/ Check-in [04907fba]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix another bug caused by NEAR/matchinfo/order=DESC interaction.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts3-prefix-search
Files: files | file ages | folders
SHA1: 04907fbadeb743c95cc9f3529e63ef388684799f
User & Date: dan 2011-06-14 09:00:27
Context
2011-06-14
11:32
Add a couple of extra tests. Closed-Leaf check-in: aefd46df user: dan tags: fts3-prefix-search
09:00
Fix another bug caused by NEAR/matchinfo/order=DESC interaction. check-in: 04907fba user: dan tags: fts3-prefix-search
07:22
Merge recent trunk changes into fts3-prefix-search branch. check-in: 135ce30f user: dan tags: fts3-prefix-search
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

  3383   3383     int nDoclist,                   /* Length of aDoclist in bytes */
  3384   3384     char **ppIter,                  /* IN/OUT: Iterator pointer */
  3385   3385     sqlite3_int64 *piDocid,         /* IN/OUT: Docid pointer */
  3386   3386     int *pnList,                    /* IN/OUT: List length pointer */
  3387   3387     u8 *pbEof                       /* OUT: End-of-file flag */
  3388   3388   ){
  3389   3389     char *p = *ppIter;
  3390         -  int iMul = (bDescIdx ? -1 : 1);
  3391   3390   
  3392   3391     assert( nDoclist>0 );
  3393   3392     assert( *pbEof==0 );
  3394   3393     assert( p || *piDocid==0 );
  3395   3394     assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) );
  3396   3395   
  3397   3396     if( p==0 ){
  3398   3397       sqlite3_int64 iDocid = 0;
  3399   3398       char *pNext = 0;
  3400   3399       char *pDocid = aDoclist;
  3401   3400       char *pEnd = &aDoclist[nDoclist];
         3401  +    int iMul = 1;
  3402   3402   
  3403         -    pDocid += sqlite3Fts3GetVarint(pDocid, &iDocid);
  3404         -    pNext = pDocid;
  3405         -    fts3PoslistCopy(0, &pDocid);
  3406   3403       while( pDocid<pEnd ){
  3407   3404         sqlite3_int64 iDelta;
  3408   3405         pDocid += sqlite3Fts3GetVarint(pDocid, &iDelta);
  3409   3406         iDocid += (iMul * iDelta);
  3410   3407         pNext = pDocid;
  3411   3408         fts3PoslistCopy(0, &pDocid);
  3412   3409         while( pDocid<pEnd && *pDocid==0 ) pDocid++;
         3410  +      iMul = (bDescIdx ? -1 : 1);
  3413   3411       }
  3414   3412   
  3415   3413       *pnList = pEnd - pNext;
  3416   3414       *ppIter = pNext;
  3417   3415       *piDocid = iDocid;
  3418   3416     }else{
         3417  +    int iMul = (bDescIdx ? -1 : 1);
  3419   3418       sqlite3_int64 iDelta;
  3420   3419       fts3GetReverseVarint(&p, aDoclist, &iDelta);
  3421   3420       *piDocid -= (iMul * iDelta);
  3422   3421   
  3423   3422       if( p==aDoclist ){
  3424   3423         *pbEof = 1;
  3425   3424       }else{

Changes to test/fts3auto.test.

    35     35   #
    36     36   # $MATCHINFO may be any expression accepted by the FTS4 MATCH operator, 
    37     37   # except that the "<column-name>:token" syntax is not supported. Tcl list
    38     38   # commands are used to tokenize the expression. Any parenthesis must appear
    39     39   # either as separate list elements, or as the first (for opening) or last
    40     40   # (for closing) character of a list element. i.e. the expression "(a OR b)c"
    41     41   # will not be parsed correctly, but "( a OR b) c" will.
           42  +#
           43  +# Available OPTIONS are:
           44  +#
           45  +#     -deferred TOKENLIST
           46  +#
           47  +# If the "deferred" option is supplied, it is passed a list of tokens that
           48  +# are deferred by FTS and result in the relevant matchinfo() stats being an
           49  +# approximation. 
    42     50   #
    43     51   set sqlite_fts3_enable_parentheses 1
    44         -proc do_fts3query_test {tn tbl expr} {
           52  +proc do_fts3query_test {tn args} {
    45     53   
    46         -  get_near_results $tbl $expr aMatchinfo
    47         -  set match $expr
           54  +  set nArg [llength $args]
           55  +  if {$nArg < 2 || ($nArg % 2)} {
           56  +    set cmd do_fts3query_test
           57  +    error "wrong # args: should be \"$cmd ?-deferred LIST? TABLE MATCHEXPR\""
           58  +  }
           59  +  set tbl   [lindex $args [expr $nArg-2]]
           60  +  set match [lindex $args [expr $nArg-1]]
           61  +  set deferred [list]
           62  +
           63  +  foreach {k v} [lrange $args 0 [expr $nArg-3]] {
           64  +    switch -- $k {
           65  +      -deferred {
           66  +        set deferred $v
           67  +      }
           68  +      default {
           69  +        error "bad option \"$k\": must be -deferred"
           70  +      }
           71  +    }
           72  +  }
           73  +
           74  +  get_near_results $tbl $match $deferred aMatchinfo
    48     75   
    49     76     set matchinfo_asc [list]
    50     77     foreach docid [lsort -integer -incr [array names aMatchinfo]] {
    51     78       lappend matchinfo_asc $docid $aMatchinfo($docid)
    52     79     }
    53     80     set matchinfo_desc [list]
    54     81     foreach docid [lsort -integer -decr [array names aMatchinfo]] {
................................................................................
    93    120       if {[string match -nocase near/* $a]} { set a [string range $a 5 end] }
    94    121       lappend out $a
    95    122       lappend out $b
    96    123     }
    97    124     return $out
    98    125   }
    99    126   
   100         -proc get_single_near_results {tbl expr arrayvar nullvar} {
          127  +proc get_single_near_results {tbl expr deferred arrayvar nullvar} {
   101    128     upvar $arrayvar aMatchinfo
   102    129     upvar $nullvar nullentry
   103    130     catch {array unset aMatchinfo}
   104    131   
   105    132     set expr [fix_near_expr $expr]
   106    133   
   107    134     # Calculate the expected results using [fts3_near_match]. The following
................................................................................
   149    176       foreach c $counts($key) {
   150    177         if {$c>0} { incr nDoc($iPhrase,$iCol) 1 }
   151    178         incr nHit($iPhrase,$iCol) $c
   152    179         incr iPhrase
   153    180       }
   154    181     }
   155    182   
   156         -  if {[info exists ::fts3_deferred] && [llength $expr]==1} {
          183  +  if {[llength $deferred] && [llength $expr]==1} {
   157    184       set phrase [lindex $expr 0]
   158    185       set rewritten [list]
   159    186       set partial 0
   160    187       foreach tok $phrase {
   161         -      if {[lsearch $::fts3_deferred $tok]>=0} {
          188  +      if {[lsearch $deferred $tok]>=0} {
   162    189           lappend rewritten *
   163    190         } else {
   164    191           lappend rewritten $tok
   165    192           set partial 1
   166    193         }
   167    194       }
   168    195       if {$partial==0} {
................................................................................
   174    201       } elseif {$rewritten != $phrase} {
   175    202         while {[lindex $rewritten end] == "*"} {
   176    203           set rewritten [lrange $rewritten 0 end-1]
   177    204         }
   178    205         while {[lindex $rewritten 0] == "*"} {
   179    206           set rewritten [lrange $rewritten 1 end]
   180    207         }
   181         -      get_single_near_results $tbl [list $rewritten] aRewrite nullentry
          208  +      get_single_near_results $tbl [list $rewritten] {} aRewrite nullentry
   182    209         foreach docid [array names hits] {
   183    210           set aMatchinfo($docid) $aRewrite($docid)
   184    211         }
   185    212         return
   186    213       }
   187    214     }
   188    215   
................................................................................
   224    251       if {$c == "("} {incr iBracket}
   225    252       if {$c == ")"} {incr iBracket -1}
   226    253     }
   227    254   
   228    255     return [expr ($iBracket==0 && $i==$nExpr)]
   229    256   }
   230    257   
   231         -proc get_near_results {tbl expr arrayvar {nullvar ""}} {
          258  +proc get_near_results {tbl expr deferred arrayvar {nullvar ""}} {
   232    259     upvar $arrayvar aMatchinfo
   233    260     if {$nullvar != ""} { upvar $nullvar nullentry }
   234    261   
   235    262     set expr [string trim $expr]
   236    263     while { [matching_brackets $expr] } {
   237    264       set expr [string trim [string range $expr 1 end-1]]
   238    265     }
................................................................................
   256    283           if {$c == ")"} { incr iBracket -1 }
   257    284         }
   258    285       }
   259    286     }
   260    287     if {$iBracket!=0} { error "mismatched brackets in: $expr" }
   261    288   
   262    289     if {[info exists opidx]==0} {
   263         -    get_single_near_results $tbl $expr aMatchinfo nullentry
          290  +    get_single_near_results $tbl $expr $deferred aMatchinfo nullentry
   264    291     } else {
   265    292       set eLeft  [lrange $expr 0 [expr $opidx-1]]
   266    293       set eRight [lrange $expr [expr $opidx+1] end]
   267    294   
   268         -    get_near_results $tbl $eLeft  aLeft  nullleft
   269         -    get_near_results $tbl $eRight aRight nullright
          295  +    get_near_results $tbl $eLeft  $deferred aLeft  nullleft
          296  +    get_near_results $tbl $eRight $deferred aRight nullright
   270    297   
   271    298       switch -- [lindex $expr $opidx] {
   272    299         "NOT" {
   273    300           foreach hit [array names aLeft] {
   274    301             if {0==[info exists aRight($hit)]} {
   275    302               set aMatchinfo($hit) $aLeft($hit)
   276    303             }
................................................................................
   386    413   
   387    414   #--------------------------------------------------------------------------
   388    415   # Some test cases involving deferred tokens.
   389    416   #
   390    417   proc make_token_deferrable {tbl token} {
   391    418     set nRow [db one "SELECT count(*) FROM $tbl"]
   392    419     set pgsz [db one "PRAGMA page_size"]
   393         -
   394         -  execsql "INSERT INTO $tbl ($tbl) VALUES('maxpending=100000000')"
   395    420     execsql BEGIN
   396    421     for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} {
   397    422       set doc [string repeat "$token " 100]
   398    423       execsql "INSERT INTO $tbl VALUES(\$doc)"
   399    424     }
   400    425     execsql "INSERT INTO $tbl VALUES('aaaaaaa ${token}aaaaa')"
   401    426     execsql COMMIT
................................................................................
   417    442       INSERT INTO t1(docid, x) VALUES(4, 'a c e g i k');
   418    443       INSERT INTO t1(docid, x) VALUES(5, 'a d g j');
   419    444       INSERT INTO t1(docid, x) VALUES(6, 'c a b');
   420    445     }
   421    446   
   422    447     make_token_deferrable t1 c
   423    448   
   424         -  set ::fts3_deferred [list]
   425    449     foreach {tn2 expr} {
   426    450       1     {a OR c}
   427    451     } {
   428    452       do_fts3query_test 3.$tn.2.$tn2 t1 $expr
   429    453     }
   430    454   
   431         -  set ::fts3_deferred [list c]
   432    455     execsql { 
   433    456       UPDATE t1_segments 
   434    457       SET block = zeroblob(length(block)) 
   435    458       WHERE length(block)>10000 AND 0
   436    459     }
          460  +  foreach {tn2 expr def} {
          461  +    1     {a NEAR c}            {}
          462  +    2     {a AND c}             c
          463  +    3     {"a c"}               c
          464  +    4     {"c a"}               c
          465  +    5     {"a c" NEAR/1 g}      {}
          466  +    6     {"a c" NEAR/0 g}      {}
          467  +  } {
          468  +    do_fts3query_test 3.$tn.2.$tn2 -deferred $def t1 $expr
          469  +  }
          470  +}
          471  +
          472  +#--------------------------------------------------------------------------
          473  +#
          474  +foreach {tn create} {
          475  +  1    "fts4(x, y)"
          476  +  2    "fts4(x, y, order=DESC)"
          477  +} {
          478  +  catchsql { DROP TABLE t1 }
          479  +  execsql  "CREATE VIRTUAL TABLE t1 USING $create"
          480  +
          481  +  foreach {x y} {
          482  +    {one two five four five} {}
          483  +    {} {one two five four five}
          484  +    {one two} {five four five}
          485  +  } {
          486  +    execsql {INSERT INTO t1 VALUES($x, $y)}
          487  +  }
          488  +
   437    489     foreach {tn2 expr} {
   438         -    1     {a NEAR c}
   439         -    2     {a AND c}
   440         -    3     {"a c"}
   441         -    4     {"c a"}
   442         -    5     {"a c" NEAR/1 g}
   443         -    6     {"a c" NEAR/0 g}
          490  +    1     {one AND five}
          491  +    2     {one NEAR five}
          492  +    3     {one NEAR/1 five}
          493  +    4     {one NEAR/2 five}
          494  +    5     {one NEAR/3 five}
   444    495     } {
   445         -    do_fts3query_test 3.$tn.2.$tn2 t1 $expr
          496  +    do_fts3query_test 4.$tn.2.$tn2 t1 $expr
   446    497     }
   447    498   }
   448    499   
   449    500   set sqlite_fts3_enable_parentheses $sfep
   450    501   finish_test
   451    502   

Changes to test/permutations.test.

   179    179     fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test
   180    180     fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test 
   181    181     fts3near.test fts3query.test fts3shared.test fts3snippet.test 
   182    182     fts3sort.test
   183    183   
   184    184     fts3fault.test fts3malloc.test fts3matchinfo.test
   185    185   
   186         -  fts3aux1.test fts3comp1.test
          186  +  fts3aux1.test fts3comp1.test fts3auto.test
   187    187   }
   188    188   
   189    189   
   190    190   lappend ::testsuitelist xxx
   191    191   #-------------------------------------------------------------------------
   192    192   # Define the coverage related test suites:
   193    193   #