/ Check-in [3bb13a06]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix another bug in 3-way NEAR queries.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 3bb13a06521b54194c9f3eb44e0dc42bacf016a4
User & Date: dan 2009-12-05 14:29:23
Context
2009-12-05
18:03
Catch an obscure case of database corruption. check-in: 0f9b17a6 user: dan tags: trunk
14:29
Fix another bug in 3-way NEAR queries. check-in: 3bb13a06 user: dan tags: trunk
11:37
Fix some problems with FTS3 and 3-way NEAR queries. check-in: 23dc6fb5 user: dan tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

1120
1121
1122
1123
1124
1125
1126
1127
1128
1129

1130



1131


1132
1133
1134
1135
1136
1137
1138
....
1376
1377
1378
1379
1380
1381
1382




1383
1384
1385
1386
1387
1388
1389
  char **pp1,                     /* Left input list */
  char **pp2                      /* Right input list */
){
  char *p = *pp;
  char *p1 = *pp1;
  char *p2 = *pp2;

  while( *p1 && *p2 ){
    int iCol1 = 0;
    int iCol2 = 0;

    if( *p1==0x01 ) sqlite3Fts3GetVarint32(&p1[1], &iCol1);



    if( *p2==0x01 ) sqlite3Fts3GetVarint32(&p2[1], &iCol2);



    if( iCol1==iCol2 ){
      sqlite3_int64 i1 = 0;
      sqlite3_int64 i2 = 0;
      sqlite3_int64 iPrev = 0;
      int n = fts3PutColNumber(&p, iCol1);
      p1 += n;
................................................................................
       || mergetype==MERGE_AND    || mergetype==MERGE_NOT
       || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE
       || mergetype==MERGE_NEAR   || mergetype==MERGE_POS_NEAR
  );

  if( !aBuffer ){
    return SQLITE_NOMEM;




  }

  /* Read the first docid from each doclist */
  fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  fts3GetDeltaVarint2(&p2, pEnd2, &i2);

  switch( mergetype ){







|
|
|
>

>
>
>

>
>







 







>
>
>
>







1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
....
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
  char **pp1,                     /* Left input list */
  char **pp2                      /* Right input list */
){
  char *p = *pp;
  char *p1 = *pp1;
  char *p2 = *pp2;

  while( *p1 || *p2 ){
    int iCol1;
    int iCol2;

    if( *p1==0x01 ) sqlite3Fts3GetVarint32(&p1[1], &iCol1);
    else if( *p1==0x00 ) iCol1 = OFFSET_LIST_END;
    else iCol1 = 0;

    if( *p2==0x01 ) sqlite3Fts3GetVarint32(&p2[1], &iCol2);
    else if( *p2==0x00 ) iCol2 = OFFSET_LIST_END;
    else iCol2 = 0;

    if( iCol1==iCol2 ){
      sqlite3_int64 i1 = 0;
      sqlite3_int64 i2 = 0;
      sqlite3_int64 iPrev = 0;
      int n = fts3PutColNumber(&p, iCol1);
      p1 += n;
................................................................................
       || mergetype==MERGE_AND    || mergetype==MERGE_NOT
       || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE
       || mergetype==MERGE_NEAR   || mergetype==MERGE_POS_NEAR
  );

  if( !aBuffer ){
    return SQLITE_NOMEM;
  }
  if( n1==0 && n2==0 ){
    *pnBuffer = 0;
    return SQLITE_OK;
  }

  /* Read the first docid from each doclist */
  fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  fts3GetDeltaVarint2(&p2, pEnd2, &i2);

  switch( mergetype ){

Changes to test/fts3rnd.test.

89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

110
111
112
113
114
115
116
117
118

119
120
121
122
123
124
125
...
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
...
206
207
208
209
210
211
212
213
214
215
216
217

218


219
220
221

222
223
224
225
226
227
  }
  lsort -integer $ret
}

proc simple_near {termlist nNear} {
  set ret [list]

  # List of nNear+1 elements that contains no terms.
  #
  set dummyfields "[string repeat "X " $nNear]X"

  foreach {key value} [array get ::t1] {
    foreach v $value {
      set v [concat $dummyfields $v $dummyfields]

      set l [lsearch -exact -all $v [lindex $termlist 0]]
      foreach T [lrange $termlist 1 end] {
        set l2 [list]
        foreach i $l {
          set iStart [expr $i - $nNear - 1]
          set iEnd [expr $i + $nNear + 1]

          foreach i2 [lsearch -exact -all [lrange $v $iStart $iEnd] $T] {
            incr i2 $iStart
            if {$i2 != $i} { lappend l2 $i2 } 
          }
        }
        set l $l2
      }

      if {[llength $l]} {

        lappend ret $key
      } 
    }
  }

  lsort -unique -integer $ret
}
................................................................................
      set iInsert [expr {int(rand()*1000000)}]
    }
    db transaction {
      insert_row $iInsert
      update_row $iUpdate
      delete_row $iDelete
    }
  
    # Pick 10 terms from the vocabulary. Check that the results of querying
    # the database for the set of documents containing each of these terms
    # is the same as the result obtained by scanning the contents of the Tcl 
    # array for each term.
    #
    for {set i 0} {$i < 10} {incr i} {
      set term [random_term]
................................................................................
      set terms [list [random_term] [random_term]]
      set match [join $terms " NEAR "]
      do_test fts3rnd-1.$nodesize.$iTest.5.$i.$match {
        execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }
      } [simple_near $terms 10]
    }
 
if 0 {
    # A 3-way NEAR query with terms as the arguments.
    #
    for {set i 0} {$i < 10} {incr i} {
      set terms [list [random_term] [random_term] [random_term]]

      set match [join $terms " NEAR "]


      do_test fts3rnd-1.$nodesize.$iTest.5.$i.$match {
        execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }
      } [simple_near $terms 10]

    }
}
  }
}

finish_test







<
<
<
<


<







>





|



>







 







|







 







<




>
|
>
>


|
>

<




89
90
91
92
93
94
95




96
97

98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
...
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
...
203
204
205
206
207
208
209

210
211
212
213
214
215
216
217
218
219
220
221
222

223
224
225
226
  }
  lsort -integer $ret
}

proc simple_near {termlist nNear} {
  set ret [list]





  foreach {key value} [array get ::t1] {
    foreach v $value {


      set l [lsearch -exact -all $v [lindex $termlist 0]]
      foreach T [lrange $termlist 1 end] {
        set l2 [list]
        foreach i $l {
          set iStart [expr $i - $nNear - 1]
          set iEnd [expr $i + $nNear + 1]
          if {$iStart < 0} {set iStart 0}
          foreach i2 [lsearch -exact -all [lrange $v $iStart $iEnd] $T] {
            incr i2 $iStart
            if {$i2 != $i} { lappend l2 $i2 } 
          }
        }
        set l [lsort -uniq -integer $l2]
      }

      if {[llength $l]} {
#puts "MATCH($key): $v"
        lappend ret $key
      } 
    }
  }

  lsort -unique -integer $ret
}
................................................................................
      set iInsert [expr {int(rand()*1000000)}]
    }
    db transaction {
      insert_row $iInsert
      update_row $iUpdate
      delete_row $iDelete
    }

    # Pick 10 terms from the vocabulary. Check that the results of querying
    # the database for the set of documents containing each of these terms
    # is the same as the result obtained by scanning the contents of the Tcl 
    # array for each term.
    #
    for {set i 0} {$i < 10} {incr i} {
      set term [random_term]
................................................................................
      set terms [list [random_term] [random_term]]
      set match [join $terms " NEAR "]
      do_test fts3rnd-1.$nodesize.$iTest.5.$i.$match {
        execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }
      } [simple_near $terms 10]
    }
 

    # A 3-way NEAR query with terms as the arguments.
    #
    for {set i 0} {$i < 10} {incr i} {
      set terms [list [random_term] [random_term] [random_term]]
      set nNear 11
      set match [join $terms " NEAR/$nNear "]
      set fts3 [execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }]
      set tcl [simple_near $terms $nNear]
      do_test fts3rnd-1.$nodesize.$iTest.5.$i.$match {
        execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }
      } [simple_near $terms $nNear]
if {$::nErr} {exit -1}
    }

  }
}

finish_test