/ Check-in [e4ab6cdb]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Changes to fts3auto.test to test OR, AND and NOT operations.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts3-prefix-search
Files: files | file ages | folders
SHA1: e4ab6cdb101bbeb804820425cf569ee7dc2397fb
User & Date: dan 2011-06-13 13:48:36
Context
2011-06-13
17:00
Add tests for deferred tokens to fts3auto.test. Fix a problem with OR queries and deferred tokens. check-in: b9fb69e5 user: dan tags: fts3-prefix-search
13:48
Changes to fts3auto.test to test OR, AND and NOT operations. check-in: e4ab6cdb user: dan tags: fts3-prefix-search
09:11
Fix a bug exposed by combining matchinfo(), NEAR and "ORDER BY rowid DESC". check-in: 5f6b87f4 user: dan tags: fts3-prefix-search
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to test/fts3auto.test.

    12     12   
    13     13   # If this build does not include FTS3, skip the tests in this file.
    14     14   #
    15     15   ifcapable !fts3 { finish_test ; return }
    16     16   source $testdir/fts3_common.tcl
    17     17   source $testdir/malloc_common.tcl
    18     18   
    19         -set testprefix fts3rnd2
           19  +set testprefix fts3auto
           20  +set sfep $sqlite_fts3_enable_parentheses
           21  +set sqlite_fts3_enable_parentheses 1
    20     22   
    21         -proc test_fts3_near_match {tn doc expr res} {
    22         -  fts3_near_match $doc $expr -phrasecountvar p
    23         -  uplevel do_test [list $tn] [list [list set {} $p]] [list $res]
    24         -}
    25         -
    26         -# Simple test cases for C routine [fts3_near_match].
           23  +#--------------------------------------------------------------------------
           24  +# Start of Tcl procs used by tests.
    27     25   #
    28         -test_fts3_near_match 1.1.1 {a b c a b} a                   {2}
    29         -test_fts3_near_match 1.1.2 {a b c a b} {a 5 b 6 c}         {2 2 1}
    30         -test_fts3_near_match 1.1.3 {a b c a b} {"a b"}             {2}
    31         -test_fts3_near_match 1.1.4 {a b c a b} {"b c"}             {1}
    32         -test_fts3_near_match 1.1.5 {a b c a b} {"c c"}             {0}
    33         -
    34         -test_fts3_near_match 1.2.1 "a b c d e f g" {b 2 f}         {0 0}
    35         -test_fts3_near_match 1.2.2 "a b c d e f g" {b 3 f}         {1 1}
    36         -test_fts3_near_match 1.2.3 "a b c d e f g" {f 2 b}         {0 0}
    37         -test_fts3_near_match 1.2.4 "a b c d e f g" {f 3 b}         {1 1}
    38         -test_fts3_near_match 1.2.5 "a b c d e f g" {"a b" 2 "f g"} {0 0}
    39         -test_fts3_near_match 1.2.6 "a b c d e f g" {"a b" 3 "f g"} {1 1}
    40         -
    41         -set A "a b c d e f g h i j k l m n o p q r s t u v w x y z"
    42         -test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"}      {0 0 0}
    43         -test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"}      {1 1 1}
    44         -
    45     26   proc mit {blob} {
    46     27     set scan(littleEndian) i*
    47     28     set scan(bigEndian) I*
    48     29     binary scan $blob $scan($::tcl_platform(byteOrder)) r
    49     30     return $r
    50     31   }
    51     32   db func mit mit
................................................................................
    58     39       if {[string match -nocase near/* $a]} { set a [string range $a 5 end] }
    59     40       lappend out $a
    60     41       lappend out $b
    61     42     }
    62     43     return $out
    63     44   }
    64     45   
    65         -proc do_near_test {tn tbl expr} {
           46  +proc get_single_near_results {tbl expr arrayvar nullvar} {
           47  +  upvar $arrayvar aMatchinfo
           48  +  upvar $nullvar nullentry
           49  +  catch {array unset aMatchinfo}
    66     50   
    67     51     set expr [fix_near_expr $expr]
    68     52   
    69         -  # Create the MATCH expression from $expr
    70         -  #
    71         -  set match [lindex $expr 0]
    72         -  if {[llength $match]>1} {
    73         -    set match "\"$match\""
    74         -  } 
    75         -  foreach {nNear phrase} [lrange $expr 1 end] {
    76         -    if {[llength $phrase]>1} {
    77         -      append match " NEAR/$nNear \"$phrase\""
    78         -    } else {
    79         -      append match " NEAR/$nNear $phrase"
    80         -    }
    81         -  }
    82         -
    83     53     # Calculate the expected results using [fts3_near_match]. The following
    84     54     # loop populates the "hits" and "counts" arrays as follows:
    85     55     # 
    86     56     #   1. For each document in the table that matches the NEAR expression,
    87     57     #      hits($docid) is set to 1. The set of docids that match the expression
    88     58     #      can therefore be found using [array names hits].
    89     59     #
................................................................................
   139    109           lappend mi [lindex $counts($docid,$iCol) $iPhrase]
   140    110           lappend mi $nDoc($iPhrase,$iCol)
   141    111           lappend mi $nHit($iPhrase,$iCol)
   142    112         }
   143    113       }
   144    114       set aMatchinfo($docid) $mi
   145    115     }
          116  +
          117  +  # Set up the nullentry output.
          118  +  #
          119  +  set nullentry [list]
          120  +  for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
          121  +    for {set iCol 0} {$iCol<$nCol} {incr iCol} {
          122  +      lappend nullentry 0 $nDoc($iPhrase,$iCol) $nHit($iPhrase,$iCol)
          123  +    }
          124  +  }
          125  +}
          126  +
          127  +
          128  +proc matching_brackets {expr} {
          129  +  if {[string range $expr 0 0]!="(" || [string range $expr end end] !=")"} { 
          130  +    return 0 
          131  +  }
          132  +
          133  +  set iBracket 1
          134  +  set nExpr [string length $expr]
          135  +  for {set i 1} {$iBracket && $i < $nExpr} {incr i} {
          136  +    set c [string range $expr $i $i]
          137  +    if {$c == "("} {incr iBracket}
          138  +    if {$c == ")"} {incr iBracket -1}
          139  +  }
          140  +
          141  +  return [expr ($iBracket==0 && $i==$nExpr)]
          142  +}
          143  +
          144  +proc get_near_results {tbl expr arrayvar {nullvar ""}} {
          145  +  upvar $arrayvar aMatchinfo
          146  +  if {$nullvar != ""} { upvar $nullvar nullentry }
          147  +
          148  +  set expr [string trim $expr]
          149  +  while { [matching_brackets $expr] } {
          150  +    set expr [string trim [string range $expr 1 end-1]]
          151  +  }
          152  +
          153  +  set prec(NOT) 1
          154  +  set prec(AND) 2
          155  +  set prec(OR)  3
          156  +
          157  +  set currentprec 0
          158  +  set iBracket 0
          159  +  set expr_length [llength $expr]
          160  +  for {set i 0} {$i < $expr_length} {incr i} {
          161  +    set op [lindex $expr $i]
          162  +    if {$iBracket==0 && [info exists prec($op)] && $prec($op)>=$currentprec } {
          163  +      set opidx $i
          164  +      set currentprec $prec($op)
          165  +    } else {
          166  +      for {set j 0} {$j < [string length $op]} {incr j} {
          167  +        set c [string range $op $j $j]
          168  +        if {$c == "("} { incr iBracket +1 }
          169  +        if {$c == ")"} { incr iBracket -1 }
          170  +      }
          171  +    }
          172  +  }
          173  +  if {$iBracket!=0} { error "mismatched brackets in: $expr" }
          174  +
          175  +  if {[info exists opidx]==0} {
          176  +    get_single_near_results $tbl $expr aMatchinfo nullentry
          177  +  } else {
          178  +    set eLeft  [lrange $expr 0 [expr $opidx-1]]
          179  +    set eRight [lrange $expr [expr $opidx+1] end]
          180  +
          181  +    get_near_results $tbl $eLeft  aLeft  nullleft
          182  +    get_near_results $tbl $eRight aRight nullright
          183  +
          184  +    switch -- [lindex $expr $opidx] {
          185  +      "NOT" {
          186  +        foreach hit [array names aLeft] {
          187  +          if {0==[info exists aRight($hit)]} {
          188  +            set aMatchinfo($hit) $aLeft($hit)
          189  +          }
          190  +        }
          191  +        set nullentry $nullleft
          192  +      }
          193  +
          194  +      "AND" {
          195  +        foreach hit [array names aLeft] {
          196  +          if {[info exists aRight($hit)]} {
          197  +            set aMatchinfo($hit) [concat $aLeft($hit) $aRight($hit)]
          198  +          }
          199  +        }
          200  +        set nullentry [concat $nullleft $nullright]
          201  +      }
          202  +
          203  +      "OR" {
          204  +        foreach hit [array names aLeft] {
          205  +          if {[info exists aRight($hit)]} {
          206  +            set aMatchinfo($hit) [concat $aLeft($hit) $aRight($hit)]
          207  +            unset aRight($hit)
          208  +          } else {
          209  +            set aMatchinfo($hit) [concat $aLeft($hit) $nullright]
          210  +          }
          211  +        }
          212  +        foreach hit [array names aRight] {
          213  +          set aMatchinfo($hit) [concat $nullleft $aRight($hit)]
          214  +        }
          215  +
          216  +        set nullentry [concat $nullleft $nullright]
          217  +      }
          218  +    }
          219  +  }
          220  +}
          221  +
          222  +proc do_near_test {tn tbl expr} {
          223  +
          224  +  get_near_results $tbl $expr aMatchinfo
          225  +  set match $expr
   146    226   
   147    227     set matchinfo_asc [list]
   148    228     foreach docid [lsort -integer -incr [array names aMatchinfo]] {
   149    229       lappend matchinfo_asc $docid $aMatchinfo($docid)
   150    230     }
   151    231     set matchinfo_desc [list]
   152    232     foreach docid [lsort -integer -decr [array names aMatchinfo]] {
   153    233       lappend matchinfo_desc $docid $aMatchinfo($docid)
   154    234     }
   155    235   
   156         -  set title "(\"$match\" -> [llength [array names hits]] rows)"
          236  +  set title "(\"$match\" -> [llength [array names aMatchinfo]] rows)"
   157    237   
   158    238     do_execsql_test $tn$title.1 "
   159    239       SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC
   160         -  " [lsort -integer -incr [array names hits]] 
          240  +  " [lsort -integer -incr [array names aMatchinfo]] 
   161    241   
   162    242     do_execsql_test $tn$title.2 "
   163    243       SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid DESC
   164         -  " [lsort -integer -decr [array names hits]] 
          244  +  " [lsort -integer -decr [array names aMatchinfo]] 
   165    245   
   166    246     do_execsql_test $tn$title.3 "
   167    247       SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl 
   168    248       WHERE $tbl MATCH '$match' ORDER BY docid DESC
   169    249     " $matchinfo_desc
   170    250   
   171    251     do_execsql_test $tn$title.4 "
   172    252       SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl 
   173    253       WHERE $tbl MATCH '$match' ORDER BY docid ASC
   174    254     " $matchinfo_asc
   175    255   } 
   176    256   
   177         -do_test 2.1 {
   178         -  execsql { CREATE VIRTUAL TABLE t1 USING fts3(a, b) }
   179         -  for {set i 0} {$i<32} {incr i} {
   180         -    set doc [list]
   181         -    if {$i&0x01} {lappend doc one}
   182         -    if {$i&0x02} {lappend doc two}
   183         -    if {$i&0x04} {lappend doc three}
   184         -    if {$i&0x08} {lappend doc four}
   185         -    if {$i&0x10} {lappend doc five}
   186         -    execsql { INSERT INTO t1 VALUES($doc, null) }
   187         -  }
   188         -} {}
   189         -foreach {tn expr} {
   190         -  1     {one}
   191         -  2     {one NEAR/1 five}
   192         -  3     {t*}
   193         -  4     {t* NEAR/0 five}
   194         -  5     {o* NEAR/1 f*}
   195         -  6     {one NEAR five NEAR two NEAR four NEAR three}
          257  +
          258  +# End of test procs. Actual tests are below this line.
          259  +#--------------------------------------------------------------------------
          260  +
          261  +#--------------------------------------------------------------------------
          262  +# The following test cases - fts3auto-1.* - focus on testing the Tcl 
          263  +# command [fts3_near_match], which is used by other tests in this file.
          264  +#
          265  +proc test_fts3_near_match {tn doc expr res} {
          266  +  fts3_near_match $doc $expr -phrasecountvar p
          267  +  uplevel do_test [list $tn] [list [list set {} $p]] [list $res]
          268  +}
          269  +
          270  +test_fts3_near_match 1.1.1 {a b c a b} a                   {2}
          271  +test_fts3_near_match 1.1.2 {a b c a b} {a 5 b 6 c}         {2 2 1}
          272  +test_fts3_near_match 1.1.3 {a b c a b} {"a b"}             {2}
          273  +test_fts3_near_match 1.1.4 {a b c a b} {"b c"}             {1}
          274  +test_fts3_near_match 1.1.5 {a b c a b} {"c c"}             {0}
          275  +
          276  +test_fts3_near_match 1.2.1 "a b c d e f g" {b 2 f}         {0 0}
          277  +test_fts3_near_match 1.2.2 "a b c d e f g" {b 3 f}         {1 1}
          278  +test_fts3_near_match 1.2.3 "a b c d e f g" {f 2 b}         {0 0}
          279  +test_fts3_near_match 1.2.4 "a b c d e f g" {f 3 b}         {1 1}
          280  +test_fts3_near_match 1.2.5 "a b c d e f g" {"a b" 2 "f g"} {0 0}
          281  +test_fts3_near_match 1.2.6 "a b c d e f g" {"a b" 3 "f g"} {1 1}
          282  +
          283  +set A "a b c d e f g h i j k l m n o p q r s t u v w x y z"
          284  +test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"}      {0 0 0}
          285  +test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"}      {1 1 1}
          286  +
          287  +#--------------------------------------------------------------------------
          288  +# Test cases fts3auto-2.* run some simple tests using the 
          289  +# [do_near_test] proc.
          290  +#
          291  +foreach {tn create} {
          292  +  1    "CREATE VIRTUAL TABLE t1 USING fts4(a, b)"
          293  +  2    "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC)"
          294  +  3    "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC)"
          295  +  4    "CREATE VIRTUAL TABLE t1 USING fts4(a, b, prefix=1)"
          296  +  5    "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC, prefix=1)"
          297  +  6    "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC, prefix=1)"
   196    298   } {
   197         -  do_near_test 2.2.$tn t1 $expr
          299  +  do_test 2.$tn.1 {
          300  +    catchsql { DROP TABLE t1 }
          301  +    execsql  $create
          302  +    for {set i 0} {$i<32} {incr i} {
          303  +      set doc [list]
          304  +      if {$i&0x01} {lappend doc one}
          305  +      if {$i&0x02} {lappend doc two}
          306  +      if {$i&0x04} {lappend doc three}
          307  +      if {$i&0x08} {lappend doc four}
          308  +      if {$i&0x10} {lappend doc five}
          309  +      execsql { INSERT INTO t1 VALUES($doc, null) }
          310  +    }
          311  +  } {}
          312  +  foreach {tn2 expr} {
          313  +    1     {one}
          314  +    2     {one NEAR/1 five}
          315  +    3     {t*}
          316  +    4     {t* NEAR/0 five}
          317  +    5     {o* NEAR/1 f*}
          318  +    6     {one NEAR five NEAR two NEAR four NEAR three}
          319  +    7     {one NEAR xyz}
          320  +    8     {one OR two}
          321  +    9     {one AND two}
          322  +    10    {one NOT two}
          323  +    11    {one AND two OR three}
          324  +    12    {three OR one AND two}
          325  +    13    {(three OR one) AND two}
          326  +    14    {(three OR one) AND two NOT (five NOT four)}
          327  +    15    {"one two"}
          328  +    16    {"one two" NOT "three four"}
          329  +  } {
          330  +    do_near_test 2.$tn.2.$tn2 t1 $expr
          331  +  }
   198    332   }
   199    333   
          334  +set sqlite_fts3_enable_parentheses $sfep
   200    335   finish_test
   201    336