/ Check-in [14dc46a7]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix handling of an OOM error in the fts3 offsets() function. Fix a couple of snippet related test cases in e_fts3.test.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 14dc46a74aafe44c0bf7dffd26268395b2c5edb2
User & Date: dan 2010-01-09 07:33:54
Context
2010-01-11
12:00
Modify snippets code to run more efficiently. And to avoid a bug relating to snippets based on full-text queries that contain duplicate terms. check-in: a2b1183d user: dan tags: trunk
2010-01-09
07:33
Fix handling of an OOM error in the fts3 offsets() function. Fix a couple of snippet related test cases in e_fts3.test. check-in: 14dc46a7 user: dan tags: trunk
2010-01-08
23:01
Update comments in fts3.c to more accurately describe the doclist format. check-in: e424a030 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3_snippet.c.

904
905
906
907
908
909
910
911





912
913









914
915
916
917
918
919
920

    /* Initialize the contents of sCtx.aTerm[] for column iCol. */
    sCtx.iCol = iCol;
    sCtx.iTerm = 0;
    rc = fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx);
    if( rc!=SQLITE_OK ) goto offsets_out;

    /* Initialize a tokenizer iterator to iterate through column iCol. */





    zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1);
    nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1);









    rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
    if( rc!=SQLITE_OK ) goto offsets_out;
    pC->pTokenizer = pTab->pTokenizer;

    rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent);
    while( rc==SQLITE_OK ){
      int i;                      /* Used to loop through terms */







|
>
>
>
>
>


>
>
>
>
>
>
>
>
>







904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934

    /* Initialize the contents of sCtx.aTerm[] for column iCol. */
    sCtx.iCol = iCol;
    sCtx.iTerm = 0;
    rc = fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx);
    if( rc!=SQLITE_OK ) goto offsets_out;

    /* Retreive the text stored in column iCol. If an SQL NULL is stored 
    ** in column iCol, jump immediately to the next iteration of the loop.
    ** If an OOM occurs while retrieving the data (this can happen if SQLite
    ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM 
    ** to the caller. 
    */
    zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1);
    nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
    if( zDoc==0 ){
      if( sqlite3_column_type(pCsr->pStmt, iCol+1)==SQLITE_NULL ){
        continue;
      }
      rc = SQLITE_NOMEM;
      goto offsets_out;
    }

    /* Initialize a tokenizer iterator to iterate through column iCol. */
    rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
    if( rc!=SQLITE_OK ) goto offsets_out;
    pC->pTokenizer = pTab->pTokenizer;

    rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent);
    while( rc==SQLITE_OK ){
      int i;                      /* Used to loop through terms */

Changes to test/e_fts3.test.

407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
...
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
...
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
} {{1 0 5 7 1 0 30 7}}
read_test  1.7.1.6 { 
  SELECT offsets(mail) FROM mail WHERE mail MATCH '"serious mail"'
} {{1 0 28 7 1 1 36 4}}

ddl_test   1.7.2.1 { CREATE VIRTUAL TABLE text USING fts3() }

write_test 3.2.2 text_content {
  INSERT INTO text VALUES('
    During 30 Nov-1 Dec, 2-3oC drops. Cool in the upper portion, minimum temperature 14-16oC and cool elsewhere, minimum temperature 17-20oC. Cold to very cold on mountaintops, minimum temperature 6-12oC. Northeasterly winds 15-30 km/hr. After that, temperature increases. Northeasterly winds 15-30 km/hr.
  ');
}

read_test  1.7.2.3 {
  SELECT snippet(text) FROM text WHERE text MATCH 'cold'
} {{<b>...</b> elsewhere, minimum temperature 17-20oC. <b>Cold</b> to very <b>cold</b> on mountaintops, minimum <b>...</b>}}

read_test  1.7.2.4 {
  SELECT snippet(text, '[', ']', '...') FROM text WHERE text MATCH '"min* tem*"'
} {{... 2-3oC drops. Cool in the upper portion, [minimum] [temperature] 14-16oC and cool elsewhere, [minimum] ...}}

##########################################################################
# Test the example in section 5 (custom tokenizers).
#
ddl_test   1.8.1.1 { CREATE VIRTUAL TABLE simple USING fts3(tokenize=simple) } 
write_test 1.8.1.2 simple_content { 
  INSERT INTO simple VALUES('Right now they''re very frustrated')
................................................................................
error_test 2.1.6 {
  SELECT snippet(a) FROM t1 WHERE a MATCH 'one'
} {illegal first argument to snippet}
error_test 2.1.7 {
  SELECT snippet() FROM t1 WHERE a MATCH 'one'
} {unable to use function snippet in the requested context}
error_test 2.1.8 {
  SELECT snippet(a, b, 'A', 'B', 'C') FROM t1 WHERE a MATCH 'one'
} {wrong number of arguments to function snippet()}
#-------------------------------------------------------------------------

#-------------------------------------------------------------------------
# Test the effect of an OOM error while installing the FTS3 module (i.e.
# opening a database handle). This case was not tested by the OOM testing
# of the document examples above.
................................................................................
set DO_MALLOC_TEST 0
ddl_test   5.1 { CREATE VIRTUAL TABLE t5 USING fts3(x) }
write_test 5.2 t5_content {
  INSERT INTO t5 VALUES('In Xanadu did Kubla Khan A stately pleasure-dome decree Where Alph, the sacred river, ran Through caverns measureless to man Down to a sunless sea.  So twice five miles of fertile ground With walls and towers were girdled round : And there were gardens bright with sinuous rills, Where blossomed many an incense-bearing tree ; And here were forests ancient as the hills, Enfolding sunny spots of greenery.');
}
read_test 5.3 { 
  SELECT snippet(t5) FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b> Down to a sunless sea.  So twice five <b>miles</b> of fertile ground With walls and towers were <b>...</b>}}
read_test 5.4 { 
  SELECT snippet(t5, '<i>') FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b> Down to a sunless sea.  So twice five <i>miles</b> of fertile ground With walls and towers were <b>...</b>}}
read_test 5.5 { 
  SELECT snippet(t5, '<i>', '</i>') FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b> Down to a sunless sea.  So twice five <i>miles</i> of fertile ground With walls and towers were <b>...</b>}}
read_test 5.6 { 
  SELECT snippet(t5, '<i>', '</i>', 'XXX') FROM t5 WHERE t5 MATCH 'miles'
} {{XXX Down to a sunless sea.  So twice five <i>miles</i> of fertile ground With walls and towers were XXX}}
#-------------------------------------------------------------------------

#-------------------------------------------------------------------------
# Test that an empty MATCH expression returns an empty result set. As
# does passing a NULL value as a MATCH expression.
#
set DO_MALLOC_TEST 0







|







|



|







 







|







 







|


|


|


|







407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
...
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
...
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
} {{1 0 5 7 1 0 30 7}}
read_test  1.7.1.6 { 
  SELECT offsets(mail) FROM mail WHERE mail MATCH '"serious mail"'
} {{1 0 28 7 1 1 36 4}}

ddl_test   1.7.2.1 { CREATE VIRTUAL TABLE text USING fts3() }

write_test 1.7.2.2 text_content {
  INSERT INTO text VALUES('
    During 30 Nov-1 Dec, 2-3oC drops. Cool in the upper portion, minimum temperature 14-16oC and cool elsewhere, minimum temperature 17-20oC. Cold to very cold on mountaintops, minimum temperature 6-12oC. Northeasterly winds 15-30 km/hr. After that, temperature increases. Northeasterly winds 15-30 km/hr.
  ');
}

read_test  1.7.2.3 {
  SELECT snippet(text) FROM text WHERE text MATCH 'cold'
} {{<b>...</b>cool elsewhere, minimum temperature 17-20oC. <b>Cold</b> to very <b>cold</b> on mountaintops, minimum temperature 6<b>...</b>}}

read_test  1.7.2.4 {
  SELECT snippet(text, '[', ']', '...') FROM text WHERE text MATCH '"min* tem*"'
} {{...the upper portion, [minimum] [temperature] 14-16oC and cool elsewhere, [minimum] [temperature] 17-20oC. Cold...}}

##########################################################################
# Test the example in section 5 (custom tokenizers).
#
ddl_test   1.8.1.1 { CREATE VIRTUAL TABLE simple USING fts3(tokenize=simple) } 
write_test 1.8.1.2 simple_content { 
  INSERT INTO simple VALUES('Right now they''re very frustrated')
................................................................................
error_test 2.1.6 {
  SELECT snippet(a) FROM t1 WHERE a MATCH 'one'
} {illegal first argument to snippet}
error_test 2.1.7 {
  SELECT snippet() FROM t1 WHERE a MATCH 'one'
} {unable to use function snippet in the requested context}
error_test 2.1.8 {
  SELECT snippet(a, b, 'A', 'B', 'C', 'D', 'E') FROM t1 WHERE a MATCH 'one'
} {wrong number of arguments to function snippet()}
#-------------------------------------------------------------------------

#-------------------------------------------------------------------------
# Test the effect of an OOM error while installing the FTS3 module (i.e.
# opening a database handle). This case was not tested by the OOM testing
# of the document examples above.
................................................................................
set DO_MALLOC_TEST 0
ddl_test   5.1 { CREATE VIRTUAL TABLE t5 USING fts3(x) }
write_test 5.2 t5_content {
  INSERT INTO t5 VALUES('In Xanadu did Kubla Khan A stately pleasure-dome decree Where Alph, the sacred river, ran Through caverns measureless to man Down to a sunless sea.  So twice five miles of fertile ground With walls and towers were girdled round : And there were gardens bright with sinuous rills, Where blossomed many an incense-bearing tree ; And here were forests ancient as the hills, Enfolding sunny spots of greenery.');
}
read_test 5.3 { 
  SELECT snippet(t5) FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b>to a sunless sea.  So twice five <b>miles</b> of fertile ground With walls and towers<b>...</b>}}
read_test 5.4 { 
  SELECT snippet(t5, '<i>') FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b>to a sunless sea.  So twice five <i>miles</b> of fertile ground With walls and towers<b>...</b>}}
read_test 5.5 { 
  SELECT snippet(t5, '<i>', '</i>') FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b>to a sunless sea.  So twice five <i>miles</i> of fertile ground With walls and towers<b>...</b>}}
read_test 5.6 { 
  SELECT snippet(t5, '<i>', '</i>', 'XXX') FROM t5 WHERE t5 MATCH 'miles'
} {{XXXto a sunless sea.  So twice five <i>miles</i> of fertile ground With walls and towersXXX}}
#-------------------------------------------------------------------------

#-------------------------------------------------------------------------
# Test that an empty MATCH expression returns an empty result set. As
# does passing a NULL value as a MATCH expression.
#
set DO_MALLOC_TEST 0