/ Check-in [70495cec]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add coverage tests (and associated fixes) for new matchinfo() code.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts3-experimental
Files: files | file ages | folders
SHA1: 70495ceccc793d608930f59e330777f287ba1ede
User & Date: dan 2010-11-25 17:49:28
Context
2010-11-26
10:58
Merge with latest trunk changes. check-in: 515cb3f4 user: dan tags: fts3-experimental
2010-11-25
17:49
Add coverage tests (and associated fixes) for new matchinfo() code. check-in: 70495cec user: dan tags: fts3-experimental
10:33
Fix bugs in fts3 function matchinfo() when used with deferred tokens. check-in: ddc2b7ec user: dan tags: fts3-experimental
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3_snippet.c.

284
285
286
287
288
289
290










291
292
293
294
295
296
297
...
935
936
937
938
939
940
941

942
943
944
945
946
947
948
949
....
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
....
1082
1083
1084
1085
1086
1087
1088

1089
1090
1091
1092
1093
1094
1095
....
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
....
1148
1149
1150
1151
1152
1153
1154
1155








1156
1157
1158
1159
1160
1161

1162
1163

1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
....
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
....
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
  if( rc==SQLITE_OK ){
    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb2, 0);
  }
  if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
  if( pnToken ) *pnToken = sCtx.nToken;
  return rc;
}











/*
** Advance the position list iterator specified by the first two 
** arguments so that it points to the first element with a value greater
** than or equal to parameter iNext.
*/
static void fts3SnippetAdvance(char **ppIter, int *piIter, int iNext){
................................................................................

    case FTS3_MATCHINFO_AVGLENGTH:
    case FTS3_MATCHINFO_LENGTH:
    case FTS3_MATCHINFO_LCS:
      nVal = pInfo->nCol;
      break;


    case FTS3_MATCHINFO_HITS:
      nVal = pInfo->nCol * pInfo->nPhrase * 3;
      break;
  }

  return nVal;
}

................................................................................
      assert( aIter[i].iCol>=iCol );
      if( aIter[i].iCol==iCol ) nLive++;
    }

    while( nLive>0 ){
      LcsIterator *pAdv = 0;
      int nThisLcs = 0;
      char *aRead;
      sqlite3_int64 iRead;

      for(i=0; i<pInfo->nPhrase; i++){
        LcsIterator *pIter = &aIter[i];
        int nToken = pIter->pExpr->pPhrase->nToken;

        if( iCol!=pIter->iCol ){  
          nThisLcs = 0;
          continue;
        }

        if( pAdv==0 || pIter->iPos<pAdv->iPos ){
................................................................................
      if( fts3LcsIteratorAdvance(pAdv) ) nLive--;
    }

    pInfo->aMatchinfo[iCol] = nLcs;
  }

  sqlite3_free(aIter);

}

static int fts3MatchinfoValues(
  Fts3Cursor *pCsr,               /* FTS3 cursor object */
  int bGlobal,                    /* True to grab the global stats */
  MatchInfo *pInfo,               /* Matchinfo context object */
  const char *zArg                /* Matchinfo format string */
................................................................................
){
  int rc = SQLITE_OK;
  int i;
  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;

  sqlite3_stmt *pSelect = 0;

  for(i=0; zArg[i]; i++){

    switch( zArg[i] ){
      case FTS3_MATCHINFO_NPHRASE: 
        if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase;
        break;

      case FTS3_MATCHINFO_NCOL: 
................................................................................
            pInfo->aMatchinfo[iCol] = (u32)nToken;
          }
        }
        sqlite3_reset(pSelectDocsize);
        break;
      }

      case FTS3_MATCHINFO_HITS: {








        Fts3Expr *pExpr = pCsr->pExpr;
        rc = fts3ExprLoadDoclists(pCsr, 0, 0);
        if( rc==SQLITE_OK ){
          if( bGlobal ){
            if( pCsr->pDeferred ){
              rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0);

            }
            (void)fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);

          }
          (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
        }
        break;
      }

      case FTS3_MATCHINFO_LCS:
        rc = fts3ExprLoadDoclists(pCsr, 0, 0);
        if( rc==SQLITE_OK ){
          fts3MatchinfoLcs(pCsr, pInfo);
        }
        break;

      default:
        assert( !"this cannot happen" );
    }

    pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]);
  }

  sqlite3_reset(pSelect);
  return rc;
................................................................................
  */
  if( pCsr->aMatchinfo==0 ){
    int nMatchinfo = 0;           /* Number of u32 elements in match-info */
    int nArg;                     /* Bytes in zArg */
    int i;                        /* Used to iterate through zArg */

    /* Load doclists for each phrase in the query. */
    rc = fts3ExprLoadDoclists(pCsr, &pCsr->nPhrase, 0);
    if( rc!=SQLITE_OK ) return rc;
    sInfo.nPhrase = pCsr->nPhrase;

    for(i=0; zArg[i]; i++){
      nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]);
    }

    /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */
................................................................................
    memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo);
    pCsr->isMatchinfoNeeded = 1;
    bGlobal = 1;
  }

  sInfo.aMatchinfo = pCsr->aMatchinfo;
  sInfo.nPhrase = pCsr->nPhrase;
  if( rc==SQLITE_OK && pCsr->isMatchinfoNeeded ){
    rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg);
    pCsr->isMatchinfoNeeded = 0;
  }

  return rc;
}








>
>
>
>
>
>
>
>
>
>







 







>
|







 







<
<



<







 







>







 







|







 







|
>
>
>
>
>
>
>
>


|
|
|
|
>
|
|
>
|
|
<


<
<
<
<
<
<
<
<
<
<







 







|
<







 







|







284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
...
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
....
1062
1063
1064
1065
1066
1067
1068


1069
1070
1071

1072
1073
1074
1075
1076
1077
1078
....
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
....
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
....
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184

1185
1186










1187
1188
1189
1190
1191
1192
1193
....
1228
1229
1230
1231
1232
1233
1234
1235

1236
1237
1238
1239
1240
1241
1242
....
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
  if( rc==SQLITE_OK ){
    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb2, 0);
  }
  if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
  if( pnToken ) *pnToken = sCtx.nToken;
  return rc;
}

static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
  (*(int *)ctx)++;
  return SQLITE_OK;
}
static int fts3ExprPhraseCount(Fts3Expr *pExpr){
  int nPhrase = 0;
  (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
  return nPhrase;
}

/*
** Advance the position list iterator specified by the first two 
** arguments so that it points to the first element with a value greater
** than or equal to parameter iNext.
*/
static void fts3SnippetAdvance(char **ppIter, int *piIter, int iNext){
................................................................................

    case FTS3_MATCHINFO_AVGLENGTH:
    case FTS3_MATCHINFO_LENGTH:
    case FTS3_MATCHINFO_LCS:
      nVal = pInfo->nCol;
      break;

    default:
      assert( cArg==FTS3_MATCHINFO_HITS );
      nVal = pInfo->nCol * pInfo->nPhrase * 3;
      break;
  }

  return nVal;
}

................................................................................
      assert( aIter[i].iCol>=iCol );
      if( aIter[i].iCol==iCol ) nLive++;
    }

    while( nLive>0 ){
      LcsIterator *pAdv = 0;
      int nThisLcs = 0;



      for(i=0; i<pInfo->nPhrase; i++){
        LcsIterator *pIter = &aIter[i];


        if( iCol!=pIter->iCol ){  
          nThisLcs = 0;
          continue;
        }

        if( pAdv==0 || pIter->iPos<pAdv->iPos ){
................................................................................
      if( fts3LcsIteratorAdvance(pAdv) ) nLive--;
    }

    pInfo->aMatchinfo[iCol] = nLcs;
  }

  sqlite3_free(aIter);
  return SQLITE_OK;
}

static int fts3MatchinfoValues(
  Fts3Cursor *pCsr,               /* FTS3 cursor object */
  int bGlobal,                    /* True to grab the global stats */
  MatchInfo *pInfo,               /* Matchinfo context object */
  const char *zArg                /* Matchinfo format string */
................................................................................
){
  int rc = SQLITE_OK;
  int i;
  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;

  sqlite3_stmt *pSelect = 0;

  for(i=0; rc==SQLITE_OK && zArg[i]; i++){

    switch( zArg[i] ){
      case FTS3_MATCHINFO_NPHRASE: 
        if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase;
        break;

      case FTS3_MATCHINFO_NCOL: 
................................................................................
            pInfo->aMatchinfo[iCol] = (u32)nToken;
          }
        }
        sqlite3_reset(pSelectDocsize);
        break;
      }

      case FTS3_MATCHINFO_LCS:
        rc = fts3ExprLoadDoclists(pCsr, 0, 0);
        if( rc==SQLITE_OK ){
          rc = fts3MatchinfoLcs(pCsr, pInfo);
        }
        break;

      default: {
        assert( zArg[i]==FTS3_MATCHINFO_HITS );
        Fts3Expr *pExpr = pCsr->pExpr;
        rc = fts3ExprLoadDoclists(pCsr, 0, 0);
        if( rc!=SQLITE_OK ) break;
        if( bGlobal ){
          if( pCsr->pDeferred ){
            rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0);
            if( rc!=SQLITE_OK ) break;
          }
          rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
          if( rc!=SQLITE_OK ) break;
        }
        (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);

        break;
      }










    }

    pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]);
  }

  sqlite3_reset(pSelect);
  return rc;
................................................................................
  */
  if( pCsr->aMatchinfo==0 ){
    int nMatchinfo = 0;           /* Number of u32 elements in match-info */
    int nArg;                     /* Bytes in zArg */
    int i;                        /* Used to iterate through zArg */

    /* Load doclists for each phrase in the query. */
    pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr);

    sInfo.nPhrase = pCsr->nPhrase;

    for(i=0; zArg[i]; i++){
      nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]);
    }

    /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */
................................................................................
    memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo);
    pCsr->isMatchinfoNeeded = 1;
    bGlobal = 1;
  }

  sInfo.aMatchinfo = pCsr->aMatchinfo;
  sInfo.nPhrase = pCsr->nPhrase;
  if( pCsr->isMatchinfoNeeded ){
    rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg);
    pCsr->isMatchinfoNeeded = 0;
  }

  return rc;
}

Changes to test/fts3fault.test.

13
14
15
16
17
18
19


20
21
22
23
24
25
26
...
150
151
152
153
154
155
156
157









































































158
set testdir [file dirname $argv0]
source $testdir/tester.tcl

set ::testprefix fts3fault

# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
ifcapable !fts3 { finish_test ; return }



# Test error handling in the sqlite3Fts3Init() function. This is the 
# function that registers the FTS3 module and various support functions
# with SQLite.
#
do_faultsim_test 1 -body { 
  sqlite3 db test.db 
................................................................................
  faultsim_delete_and_reopen
} -body {
  execsql { CREATE VIRTUAL TABLE t1 USING fts4(a, b, matchnfo=fts3) }
} -test {
  faultsim_test_result {1 {unrecognized parameter: matchnfo=fts3}} \
                       {1 {vtable constructor failed: t1}}
}










































































finish_test







>
>







 








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
...
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
set testdir [file dirname $argv0]
source $testdir/tester.tcl

set ::testprefix fts3fault

# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
ifcapable !fts3 { finish_test ; return }

if 1 {

# Test error handling in the sqlite3Fts3Init() function. This is the 
# function that registers the FTS3 module and various support functions
# with SQLite.
#
do_faultsim_test 1 -body { 
  sqlite3 db test.db 
................................................................................
  faultsim_delete_and_reopen
} -body {
  execsql { CREATE VIRTUAL TABLE t1 USING fts4(a, b, matchnfo=fts3) }
} -test {
  faultsim_test_result {1 {unrecognized parameter: matchnfo=fts3}} \
                       {1 {vtable constructor failed: t1}}
}

}

proc mit {blob} {
  set scan(littleEndian) i*
  set scan(bigEndian) I*
  binary scan $blob $scan($::tcl_platform(byteOrder)) r
  return $r
}

do_test 8.0 {
  faultsim_delete_and_reopen
  execsql { CREATE VIRTUAL TABLE t8 USING fts4 }
  execsql "INSERT INTO t8 VALUES('a b c')"
  execsql "INSERT INTO t8 VALUES('b b b')"
  execsql "INSERT INTO t8 VALUES('[string repeat {c } 50000]')"
  execsql "INSERT INTO t8 VALUES('d d d')"
  execsql "INSERT INTO t8 VALUES('e e e')"
  execsql "INSERT INTO t8(t8) VALUES('optimize')"
  faultsim_save_and_close
} {}

do_faultsim_test 8.1 -prep { 
  faultsim_restore_and_reopen
  db func mit mit
} -body {
  execsql { SELECT mit(matchinfo(t8, 'x')) FROM t8 WHERE t8 MATCH 'a b c' }
} -test {
  faultsim_test_result {0 {{1 1 1 1 4 2 1 5 5}}}
}
do_faultsim_test 8.2 -faults oom-t* -prep { 
  faultsim_restore_and_reopen
  db func mit mit
} -body {
  execsql { SELECT mit(matchinfo(t8, 's')) FROM t8 WHERE t8 MATCH 'a b c' }
} -test {
  faultsim_test_result {0 3}
}
do_faultsim_test 8.3 -prep { 
  faultsim_restore_and_reopen
  db func mit mit
} -body {
  execsql { SELECT mit(matchinfo(t8, 'a')) FROM t8 WHERE t8 MATCH 'a b c' }
} -test {
  faultsim_test_result {0 10002}
}
do_faultsim_test 8.4 -prep { 
  faultsim_restore_and_reopen
  db func mit mit
} -body {
  execsql { SELECT mit(matchinfo(t8, 'l')) FROM t8 WHERE t8 MATCH 'a b c' }
} -test {
  faultsim_test_result {0 3}
}

do_test 9.0 {
  faultsim_delete_and_reopen
  execsql {
    CREATE VIRTUAL TABLE t9 USING fts4(tokenize=porter);
    INSERT INTO t9 VALUES(
      'this record is used toooooooooooooooooooooooooooooooooooooo try to'
    );
    SELECT offsets(t9) FROM t9 WHERE t9 MATCH 'to*';
  }
  faultsim_save_and_close
} {}
do_faultsim_test 9.1 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { SELECT offsets(t9) FROM t9 WHERE t9 MATCH 'to*' }
} -test {
  faultsim_test_result {0 {{0 0 20 39 0 0 64 2}}}
}

finish_test

Changes to test/fts3matchinfo.test.

261
262
263
264
265
266
267



268
















































269
270
do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'}         { s {2} }
do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'}         { s {2 1} }
do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'}         { s {2} }
do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'}       { s {3} }
do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
do_matchinfo_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }





















































finish_test








>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'}         { s {2} }
do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'}         { s {2 1} }
do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'}         { s {2} }
do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'}       { s {3} }
do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
do_matchinfo_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }

do_execsql_test 4.5.0 {
  CREATE VIRTUAL TABLE t6 USING fts4(a, b, c);
  INSERT INTO t6 VALUES('a', 'b', 'c');
}
do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'}       { s {{1 1 1}} }


#-------------------------------------------------------------------------
# Check the following restrictions:
#
#   + Matchinfo flags 'a', 'l' and 'n' can only be used with fts4, not fts3.
#   + Matchinfo flag 'l' cannot be used with matchinfo=fts3.
#
do_execsql_test 5.1 {
  CREATE VIRTUAL TABLE t7 USING fts3(a, b);
  INSERT INTO t7 VALUES('u v w', 'x y z');

  CREATE VIRTUAL TABLE t8 USING fts4(a, b, matchinfo=fts3);
  INSERT INTO t8 VALUES('u v w', 'x y z');
}

do_catchsql_test 5.2.1 { 
  SELECT matchinfo(t7, 'a') FROM t7 WHERE t7 MATCH 'x y'
} {1 {unrecognized matchinfo request: a}}
do_catchsql_test 5.2.2 { 
  SELECT matchinfo(t7, 'l') FROM t7 WHERE t7 MATCH 'x y'
} {1 {unrecognized matchinfo request: l}}
do_catchsql_test 5.2.3 { 
  SELECT matchinfo(t7, 'n') FROM t7 WHERE t7 MATCH 'x y'
} {1 {unrecognized matchinfo request: n}}

do_catchsql_test 5.3.1 { 
  SELECT matchinfo(t8, 'l') FROM t8 WHERE t8 MATCH 'x y'
} {1 {unrecognized matchinfo request: l}}

#-------------------------------------------------------------------------
# Test that the offsets() function handles corruption in the %_content
# table correctly.
#
do_execsql_test 6.1 {
  CREATE VIRTUAL TABLE t9 USING fts4;
  INSERT INTO t9 VALUES(
    'this record is used to try to dectect corruption'
  );
  SELECT offsets(t9) FROM t9 WHERE t9 MATCH 'to';
} {{0 0 20 2 0 0 27 2}}

do_catchsql_test 6.2 {
  UPDATE t9_content SET c0content = 'this record is used to'; 
  SELECT offsets(t9) FROM t9 WHERE t9 MATCH 'to';
} {1 {database disk image is malformed}}

finish_test