SQLite

Check-in [0dd09fc034]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Avoid passing NULL to the xOpen method of an FTS3/4 tokenizer.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 0dd09fc034c127718366d3a3183e367d2f9fd82d
User & Date: dan 2011-04-19 06:43:22.111
Context
2011-04-20
13:35
Update a comment in e_createtable.test. (check-in: d8b149f5e4 user: dan tags: trunk)
2011-04-19
06:43
Avoid passing NULL to the xOpen method of an FTS3/4 tokenizer. (check-in: 0dd09fc034 user: dan tags: trunk)
2011-04-17
17:09
Change the prototype for the open() system call to agree with Posix. Though a faulty function prototype in a pointer cast is a seemingly innocuous error, the correct prototype is necessary for pthreads to work correctly on NetBSD. (check-in: 3e135748f1 user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3_write.c.
︙︙
537
538
539
540
541
542
543








544
545
546
547
548
549
550

  sqlite3_tokenizer *pTokenizer = p->pTokenizer;
  sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
  sqlite3_tokenizer_cursor *pCsr;
  int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
      const char**,int*,int*,int*,int*);

  assert( pTokenizer && pModule );









  rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr);
  if( rc!=SQLITE_OK ){
    return rc;
  }
  pCsr->pTokenizer = pTokenizer;









>
>
>
>
>
>
>
>








537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558

  sqlite3_tokenizer *pTokenizer = p->pTokenizer;
  sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
  sqlite3_tokenizer_cursor *pCsr;
  int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
      const char**,int*,int*,int*,int*);

  assert( pTokenizer && pModule );

  /* If the user has inserted a NULL value, this function may be called with
  ** zText==0. In this case, add zero token entries to the hash table and 
  ** return early. */
  if( zText==0 ){
    *pnWord = 0;
    return SQLITE_OK;
  }

  rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr);
  if( rc!=SQLITE_OK ){
    return rc;
  }
  pCsr->pTokenizer = pTokenizer;
︙︙
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646

** Argument apVal is the same as the similarly named argument passed to
** fts3InsertData(). Parameter iDocid is the docid of the new row.
*/
static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){
  int i;                          /* Iterator variable */
  for(i=2; i<p->nColumn+2; i++){
    const char *zText = (const char *)sqlite3_value_text(apVal[i]);
    if( zText ){
      int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]);
      if( rc!=SQLITE_OK ){
        return rc;
      }
    }
    aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]);
  }
  return SQLITE_OK;
}

/*








<
|
|
|
<








636
637
638
639
640
641
642

643
644
645

646
647
648
649
650
651
652

** Argument apVal is the same as the similarly named argument passed to
** fts3InsertData(). Parameter iDocid is the docid of the new row.
*/
static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){
  int i;                          /* Iterator variable */
  for(i=2; i<p->nColumn+2; i++){
    const char *zText = (const char *)sqlite3_value_text(apVal[i]);

    int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]);
    if( rc!=SQLITE_OK ){
      return rc;

    }
    aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]);
  }
  return SQLITE_OK;
}

/*
︙︙
Changes to test/fts3atoken.test.
︙︙
20
21
22
23
24
25
26


27
28
29
30
31
32
33


# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}



proc escape_string {str} {
  set out ""
  foreach char [split $str ""] {
    scan $char %c i
    if {$i<=127} {
      append out $char
    } else {








>
>








20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35


# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}

set ::testprefix fts3token

proc escape_string {str} {
  set out ""
  foreach char [split $str ""] {
    scan $char %c i
    if {$i<=127} {
      append out $char
    } else {
︙︙
162
163
164
165
166
167
168







169
170
171
172
173

174



  append output "2 then then "
  append output "3 [string tolower $longtoken] $longtoken"

  do_icu_test fts3token-4.6 MiddleOfTheOcean  $input $output
  do_icu_test fts3token-4.7 th_TH  $input $output
  do_icu_test fts3token-4.8 en_US  $input $output
}








do_test fts3token-internal {
  execsql { SELECT fts3_tokenizer_internal_test() }
} {ok}


finish_test










>
>
>
>
>
>
>





>

>
>

164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186

  append output "2 then then "
  append output "3 [string tolower $longtoken] $longtoken"

  do_icu_test fts3token-4.6 MiddleOfTheOcean  $input $output
  do_icu_test fts3token-4.7 th_TH  $input $output
  do_icu_test fts3token-4.8 en_US  $input $output
}

do_execsql_test 5.1 {
  CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US);
  insert into x1 (name) values (NULL);
  insert into x1 (name) values (NULL);
  delete from x1;
}

do_test fts3token-internal {
  execsql { SELECT fts3_tokenizer_internal_test() }
} {ok}


finish_test