/ Check-in [5f6b87f4]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a bug exposed by combining matchinfo(), NEAR and "ORDER BY rowid DESC".
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts3-prefix-search
Files: files | file ages | folders
SHA1: 5f6b87f420f21749aa7c72e020c50aca74890086
User & Date: dan 2011-06-13 09:11:01
Context
2011-06-13
13:48
Changes to fts3auto.test to test OR, AND and NOT operations. check-in: e4ab6cdb user: dan tags: fts3-prefix-search
09:11
Fix a bug exposed by combining matchinfo(), NEAR and "ORDER BY rowid DESC". check-in: 5f6b87f4 user: dan tags: fts3-prefix-search
2011-06-09
10:48
Fix problems to do with using both OR and NEAR operators in a single expression. check-in: 4e8dd19e user: dan tags: fts3-prefix-search
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Changes to Makefile.in.

375
376
377
378
379
380
381
382

383
384
385
386
387
388
389
  $(TOP)/src/test_syscall.c \
  $(TOP)/src/test_stat.c \
  $(TOP)/src/test_tclvar.c \
  $(TOP)/src/test_thread.c \
  $(TOP)/src/test_vfs.c \
  $(TOP)/src/test_wholenumber.c \
  $(TOP)/src/test_wsd.c       \
  $(TOP)/ext/fts3/fts3_term.c 


# Source code to the library files needed by the test fixture
#
TESTSRC2 = \
  $(TOP)/src/attach.c \
  $(TOP)/src/backup.c \
  $(TOP)/src/bitvec.c \







|
>







375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
  $(TOP)/src/test_syscall.c \
  $(TOP)/src/test_stat.c \
  $(TOP)/src/test_tclvar.c \
  $(TOP)/src/test_thread.c \
  $(TOP)/src/test_vfs.c \
  $(TOP)/src/test_wholenumber.c \
  $(TOP)/src/test_wsd.c       \
  $(TOP)/ext/fts3/fts3_term.c \
  $(TOP)/ext/fts3/fts3_test.c 

# Source code to the library files needed by the test fixture
#
TESTSRC2 = \
  $(TOP)/src/attach.c \
  $(TOP)/src/backup.c \
  $(TOP)/src/bitvec.c \

Changes to ext/fts3/fts3.c.

2723
2724
2725
2726
2727
2728
2729
2730
2731


2732
2733
2734
2735
2736
2737
2738
....
3418
3419
3420
3421
3422
3423
3424

3425
3426
3427
3428
3429
3430
3431
/*
** When called, *ppPoslist must point to the byte immediately following the
** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function
** moves *ppPoslist so that it instead points to the first byte of the
** same position list.
*/
static void fts3ReversePoslist(char *pStart, char **ppPoslist){
  char *p = &(*ppPoslist)[-3];
  char c = p[1];


  while( p>pStart && (*p & 0x80) | c ){ 
    c = *p--; 
  }
  if( p>pStart ){ p = &p[2]; }
  while( *p++&0x80 );
  *ppPoslist = p;
}
................................................................................
    fts3PoslistCopy(0, &pDocid);
    while( pDocid<pEnd ){
      sqlite3_int64 iDelta;
      pDocid += sqlite3Fts3GetVarint(pDocid, &iDelta);
      iDocid += (iMul * iDelta);
      pNext = pDocid;
      fts3PoslistCopy(0, &pDocid);

    }

    *pnList = pEnd - pNext;
    *ppIter = pNext;
    *piDocid = iDocid;
  }else{
    sqlite3_int64 iDelta;







|
|
>
>







 







>







2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
....
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
/*
** When called, *ppPoslist must point to the byte immediately following the
** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function
** moves *ppPoslist so that it instead points to the first byte of the
** same position list.
*/
static void fts3ReversePoslist(char *pStart, char **ppPoslist){
  char *p = &(*ppPoslist)[-2];
  char c;

  while( p>pStart && (c=*p--)==0 );
  while( p>pStart && (*p & 0x80) | c ){ 
    c = *p--; 
  }
  if( p>pStart ){ p = &p[2]; }
  while( *p++&0x80 );
  *ppPoslist = p;
}
................................................................................
    fts3PoslistCopy(0, &pDocid);
    while( pDocid<pEnd ){
      sqlite3_int64 iDelta;
      pDocid += sqlite3Fts3GetVarint(pDocid, &iDelta);
      iDocid += (iMul * iDelta);
      pNext = pDocid;
      fts3PoslistCopy(0, &pDocid);
      while( pDocid<pEnd && *pDocid==0 ) pDocid++;
    }

    *pnList = pEnd - pNext;
    *ppIter = pNext;
    *piDocid = iDocid;
  }else{
    sqlite3_int64 iDelta;

Added ext/fts3/fts3_test.c.



















































































































































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
/*
** 2011 Jun 13
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
** This file is not part of the production FTS code. It is only used for
** testing. It contains a Tcl command that can be used to test if a document
** matches an FTS NEAR expression.
*/

#include <tcl.h>
#include <string.h>
#include <assert.h>

#define NM_MAX_TOKEN 12

typedef struct NearPhrase NearPhrase;
typedef struct NearDocument NearDocument;
typedef struct NearToken NearToken;

struct NearDocument {
  int nToken;                     /* Length of token in bytes */
  NearToken *aToken;              /* Token array */
};

struct NearToken {
  int n;                          /* Length of token in bytes */
  const char *z;                  /* Pointer to token string */
};

struct NearPhrase {
  int nNear;                      /* Preceding NEAR value */
  int nToken;                     /* Number of tokens in this phrase */
  NearToken aToken[NM_MAX_TOKEN]; /* Array of tokens in this phrase */
};

static int nm_phrase_match(
  NearPhrase *p,
  NearToken *aToken
){
  int ii;

  for(ii=0; ii<p->nToken; ii++){
    NearToken *pToken = &p->aToken[ii];
    if( pToken->n>0 && pToken->z[pToken->n-1]=='*' ){
      if( aToken[ii].n<(pToken->n-1) ) return 0;
      if( memcmp(aToken[ii].z, pToken->z, pToken->n-1) ) return 0;
    }else{
      if( aToken[ii].n!=pToken->n ) return 0;
      if( memcmp(aToken[ii].z, pToken->z, pToken->n) ) return 0;
    }
  }

  return 1;
}

static int nm_near_chain(
  int iDir,                       /* Direction to iterate through aPhrase[] */
  NearDocument *pDoc,             /* Document to match against */
  int iPos,                       /* Position at which iPhrase was found */
  int nPhrase,                    /* Size of phrase array */
  NearPhrase *aPhrase,            /* Phrase array */
  int iPhrase                     /* Index of phrase found */
){
  int iStart;
  int iStop;
  int ii;
  int nNear;
  int iPhrase2;
  NearPhrase *p;
  NearPhrase *pPrev;

  assert( iDir==1 || iDir==-1 );

  if( iDir==1 ){
    if( (iPhrase+1)==nPhrase ) return 1;
    nNear = aPhrase[iPhrase+1].nNear;
  }else{
    if( iPhrase==0 ) return 1;
    nNear = aPhrase[iPhrase].nNear;
  }
  pPrev = &aPhrase[iPhrase];
  iPhrase2 = iPhrase+iDir;
  p = &aPhrase[iPhrase2];

  iStart = iPos - nNear - p->nToken;
  iStop = iPos + nNear + pPrev->nToken;

  if( iStart<0 ) iStart = 0;
  if( iStop > pDoc->nToken - p->nToken ) iStop = pDoc->nToken - p->nToken;

  for(ii=iStart; ii<=iStop; ii++){
    if( nm_phrase_match(p, &pDoc->aToken[ii]) ){
      if( nm_near_chain(iDir, pDoc, ii, nPhrase, aPhrase, iPhrase2) ) return 1;
    }
  }

  return 0;
}

static int nm_match_count(
  NearDocument *pDoc,             /* Document to match against */
  int nPhrase,                    /* Size of phrase array */
  NearPhrase *aPhrase,            /* Phrase array */
  int iPhrase                     /* Index of phrase to count matches for */
){
  int nOcc = 0;
  int ii;
  NearPhrase *p = &aPhrase[iPhrase];

  for(ii=0; ii<(pDoc->nToken + 1 - p->nToken); ii++){
    if( nm_phrase_match(p, &pDoc->aToken[ii]) ){
      /* Test forward NEAR chain (i>iPhrase) */
      if( 0==nm_near_chain(1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue;

      /* Test reverse NEAR chain (i<iPhrase) */
      if( 0==nm_near_chain(-1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue;

      /* This is a real match. Increment the counter. */
      nOcc++;
    }
  } 

  return nOcc;
}

/*
** Tclcmd: fts3_near_match DOCUMENT EXPR ?OPTIONS?
*/
static int fts3_near_match_cmd(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  int nTotal = 0;
  int rc;
  int ii;
  int nPhrase;
  NearPhrase *aPhrase = 0;
  NearDocument doc = {0, 0};
  Tcl_Obj **apDocToken;
  Tcl_Obj *pRet;
  Tcl_Obj *pPhrasecount = 0;
  
  Tcl_Obj **apExprToken;
  int nExprToken;

  /* Must have 3 or more arguments. */
  if( objc<3 || (objc%2)==0 ){
    Tcl_WrongNumArgs(interp, 1, objv, "DOCUMENT EXPR ?OPTION VALUE?...");
    rc = TCL_ERROR;
    goto near_match_out;
  }

  for(ii=3; ii<objc; ii+=2){
    enum NM_enum { NM_PHRASECOUNTS };
    struct TestnmSubcmd {
      char *zName;
      enum NM_enum eOpt;
    } aOpt[] = {
      { "-phrasecountvar", NM_PHRASECOUNTS },
      { 0, 0 }
    };
    int iOpt;
    if( Tcl_GetIndexFromObjStruct(
        interp, objv[ii], aOpt, sizeof(aOpt[0]), "option", 0, &iOpt) 
    ){
      return TCL_ERROR;
    }

    switch( aOpt[iOpt].eOpt ){
      case NM_PHRASECOUNTS:
        pPhrasecount = objv[ii+1];
        break;
    }
  }

  rc = Tcl_ListObjGetElements(interp, objv[1], &doc.nToken, &apDocToken);
  if( rc!=TCL_OK ) goto near_match_out;
  doc.aToken = (NearToken *)ckalloc(doc.nToken*sizeof(NearToken));
  for(ii=0; ii<doc.nToken; ii++){
    doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &doc.aToken[ii].n);
  }

  rc = Tcl_ListObjGetElements(interp, objv[2], &nExprToken, &apExprToken);
  if( rc!=TCL_OK ) goto near_match_out;

  nPhrase = (nExprToken + 1) / 2;
  aPhrase = (NearPhrase *)ckalloc(nPhrase * sizeof(NearPhrase));
  memset(aPhrase, 0, nPhrase * sizeof(NearPhrase));
  for(ii=0; ii<nPhrase; ii++){
    Tcl_Obj *pPhrase = apExprToken[ii*2];
    Tcl_Obj **apToken;
    int nToken;
    int jj;

    rc = Tcl_ListObjGetElements(interp, pPhrase, &nToken, &apToken);
    if( rc!=TCL_OK ) goto near_match_out;
    if( nToken>NM_MAX_TOKEN ){
      Tcl_AppendResult(interp, "Too many tokens in phrase", 0);
      rc = TCL_ERROR;
      goto near_match_out;
    }
    for(jj=0; jj<nToken; jj++){
      NearToken *pT = &aPhrase[ii].aToken[jj];
      pT->z = Tcl_GetStringFromObj(apToken[jj], &pT->n);
    }
    aPhrase[ii].nToken = nToken;
  }
  for(ii=1; ii<nPhrase; ii++){
    Tcl_Obj *pNear = apExprToken[2*ii-1];
    int nNear;
    rc = Tcl_GetIntFromObj(interp, pNear, &nNear);
    if( rc!=TCL_OK ) goto near_match_out;
    aPhrase[ii].nNear = nNear;
  }

  pRet = Tcl_NewObj();
  Tcl_IncrRefCount(pRet);
  for(ii=0; ii<nPhrase; ii++){
    int nOcc = nm_match_count(&doc, nPhrase, aPhrase, ii);
    Tcl_ListObjAppendElement(interp, pRet, Tcl_NewIntObj(nOcc));
    nTotal += nOcc;
  }
  if( pPhrasecount ){
    Tcl_ObjSetVar2(interp, pPhrasecount, 0, pRet, 0);
  }
  Tcl_DecrRefCount(pRet);
  Tcl_SetObjResult(interp, Tcl_NewBooleanObj(nTotal>0));

 near_match_out: 
  ckfree((char *)aPhrase);
  ckfree((char *)doc.aToken);
  return rc;
}

int Sqlitetestfts3_Init(Tcl_Interp *interp){
  Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0);
  return TCL_OK;
}

Changes to main.mk.

216
217
218
219
220
221
222

223
224
225
226
227
228
229
  parse.h \
  sqlite3.h


# Source code to the test files.
#
TESTSRC = \

  $(TOP)/src/test1.c \
  $(TOP)/src/test2.c \
  $(TOP)/src/test3.c \
  $(TOP)/src/test4.c \
  $(TOP)/src/test5.c \
  $(TOP)/src/test6.c \
  $(TOP)/src/test7.c \







>







216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
  parse.h \
  sqlite3.h


# Source code to the test files.
#
TESTSRC = \
  $(TOP)/ext/fts3/fts3_test.c \
  $(TOP)/src/test1.c \
  $(TOP)/src/test2.c \
  $(TOP)/src/test3.c \
  $(TOP)/src/test4.c \
  $(TOP)/src/test5.c \
  $(TOP)/src/test6.c \
  $(TOP)/src/test7.c \

Changes to src/tclsqlite.c.

3580
3581
3582
3583
3584
3585
3586




3587
3588
3589
3590
3591
3592
3593
....
3620
3621
3622
3623
3624
3625
3626




3627
3628
3629
3630
3631
3632
3633
    extern int Sqlitetestrtree_Init(Tcl_Interp*);
    extern int Sqlitequota_Init(Tcl_Interp*);
    extern int Sqlitemultiplex_Init(Tcl_Interp*);
    extern int SqliteSuperlock_Init(Tcl_Interp*);
    extern int SqlitetestSyscall_Init(Tcl_Interp*);
    extern int Sqlitetestfuzzer_Init(Tcl_Interp*);
    extern int Sqlitetestwholenumber_Init(Tcl_Interp*);





#ifdef SQLITE_ENABLE_ZIPVFS
    extern int Zipvfs_Init(Tcl_Interp*);
    Zipvfs_Init(interp);
#endif

    Sqliteconfig_Init(interp);
................................................................................
    Sqlitetestrtree_Init(interp);
    Sqlitequota_Init(interp);
    Sqlitemultiplex_Init(interp);
    SqliteSuperlock_Init(interp);
    SqlitetestSyscall_Init(interp);
    Sqlitetestfuzzer_Init(interp);
    Sqlitetestwholenumber_Init(interp);





    Tcl_CreateObjCommand(interp,"load_testfixture_extensions",init_all_cmd,0,0);

#ifdef SQLITE_SSE
    Sqlitetestsse_Init(interp);
#endif
  }







>
>
>
>







 







>
>
>
>







3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
....
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
    extern int Sqlitetestrtree_Init(Tcl_Interp*);
    extern int Sqlitequota_Init(Tcl_Interp*);
    extern int Sqlitemultiplex_Init(Tcl_Interp*);
    extern int SqliteSuperlock_Init(Tcl_Interp*);
    extern int SqlitetestSyscall_Init(Tcl_Interp*);
    extern int Sqlitetestfuzzer_Init(Tcl_Interp*);
    extern int Sqlitetestwholenumber_Init(Tcl_Interp*);

#ifdef SQLITE_ENABLE_FTS3
    extern int Sqlitetestfts3_Init(Tcl_Interp *interp);
#endif

#ifdef SQLITE_ENABLE_ZIPVFS
    extern int Zipvfs_Init(Tcl_Interp*);
    Zipvfs_Init(interp);
#endif

    Sqliteconfig_Init(interp);
................................................................................
    Sqlitetestrtree_Init(interp);
    Sqlitequota_Init(interp);
    Sqlitemultiplex_Init(interp);
    SqliteSuperlock_Init(interp);
    SqlitetestSyscall_Init(interp);
    Sqlitetestfuzzer_Init(interp);
    Sqlitetestwholenumber_Init(interp);

#ifdef SQLITE_ENABLE_FTS3
    Sqlitetestfts3_Init(interp);
#endif

    Tcl_CreateObjCommand(interp,"load_testfixture_extensions",init_all_cmd,0,0);

#ifdef SQLITE_SSE
    Sqlitetestsse_Init(interp);
#endif
  }

Added test/fts3auto.test.



















































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
# 2011 June 10
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If this build does not include FTS3, skip the tests in this file.
#
ifcapable !fts3 { finish_test ; return }
source $testdir/fts3_common.tcl
source $testdir/malloc_common.tcl

set testprefix fts3rnd2

proc test_fts3_near_match {tn doc expr res} {
  fts3_near_match $doc $expr -phrasecountvar p
  uplevel do_test [list $tn] [list [list set {} $p]] [list $res]
}

# Simple test cases for C routine [fts3_near_match].
#
test_fts3_near_match 1.1.1 {a b c a b} a                   {2}
test_fts3_near_match 1.1.2 {a b c a b} {a 5 b 6 c}         {2 2 1}
test_fts3_near_match 1.1.3 {a b c a b} {"a b"}             {2}
test_fts3_near_match 1.1.4 {a b c a b} {"b c"}             {1}
test_fts3_near_match 1.1.5 {a b c a b} {"c c"}             {0}

test_fts3_near_match 1.2.1 "a b c d e f g" {b 2 f}         {0 0}
test_fts3_near_match 1.2.2 "a b c d e f g" {b 3 f}         {1 1}
test_fts3_near_match 1.2.3 "a b c d e f g" {f 2 b}         {0 0}
test_fts3_near_match 1.2.4 "a b c d e f g" {f 3 b}         {1 1}
test_fts3_near_match 1.2.5 "a b c d e f g" {"a b" 2 "f g"} {0 0}
test_fts3_near_match 1.2.6 "a b c d e f g" {"a b" 3 "f g"} {1 1}

set A "a b c d e f g h i j k l m n o p q r s t u v w x y z"
test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"}      {0 0 0}
test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"}      {1 1 1}

proc mit {blob} {
  set scan(littleEndian) i*
  set scan(bigEndian) I*
  binary scan $blob $scan($::tcl_platform(byteOrder)) r
  return $r
}
db func mit mit

proc fix_near_expr {expr} { 
  set out [list]
  lappend out [lindex $expr 0]
  foreach {a b} [lrange $expr 1 end] {
    if {[string match -nocase near $a]}   { set a 10 }
    if {[string match -nocase near/* $a]} { set a [string range $a 5 end] }
    lappend out $a
    lappend out $b
  }
  return $out
}

proc do_near_test {tn tbl expr} {

  set expr [fix_near_expr $expr]

  # Create the MATCH expression from $expr
  #
  set match [lindex $expr 0]
  if {[llength $match]>1} {
    set match "\"$match\""
  } 
  foreach {nNear phrase} [lrange $expr 1 end] {
    if {[llength $phrase]>1} {
      append match " NEAR/$nNear \"$phrase\""
    } else {
      append match " NEAR/$nNear $phrase"
    }
  }

  # Calculate the expected results using [fts3_near_match]. The following
  # loop populates the "hits" and "counts" arrays as follows:
  # 
  #   1. For each document in the table that matches the NEAR expression,
  #      hits($docid) is set to 1. The set of docids that match the expression
  #      can therefore be found using [array names hits].
  #
  #   2. For each column of each document in the table, counts($docid,$iCol)
  #      is set to the -phrasecountvar output.
  #
  set res [list]
  catch { array unset hits }
  db eval "SELECT docid, * FROM $tbl" d {
    set iCol 0
    foreach col [lrange $d(*) 1 end] {
      set docid $d(docid)
      set hit [fts3_near_match $d($col) $expr -p counts($docid,$iCol)]
      if {$hit} { set hits($docid) 1 }
      incr iCol
    }
  }
  set nPhrase [expr ([llength $expr]+1)/2]
  set nCol $iCol

  # This block populates the nHit and nDoc arrays. For each phrase/column
  # in the query/table, array elements are set as follows:
  #
  #   nHit($iPhrase,$iCol) - Total number of hits for phrase $iPhrase in 
  #                          column $iCol.
  #
  #   nDoc($iPhrase,$iCol) - Number of documents with at least one hit for
  #                          phrase $iPhrase in column $iCol.
  #
  for {set iPhrase 0} {$iPhrase < $nPhrase} {incr iPhrase} {
    for {set iCol 0} {$iCol < $nCol} {incr iCol} {
      set nHit($iPhrase,$iCol) 0
      set nDoc($iPhrase,$iCol) 0
    }
  }
  foreach key [array names counts] {
    set iCol [lindex [split $key ,] 1]
    set iPhrase 0
    foreach c $counts($key) {
      if {$c>0} { incr nHit($iPhrase,$iCol) 1 }
      incr nDoc($iPhrase,$iCol) $c
      incr iPhrase
    }
  }

  # Set up the aMatchinfo array. For each document, set aMatchinfo($docid) to
  # contain the output of matchinfo('x') for the document.
  #
  foreach docid [array names hits] {
    set mi [list]
    for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
      for {set iCol 0} {$iCol<$nCol} {incr iCol} {
        lappend mi [lindex $counts($docid,$iCol) $iPhrase]
        lappend mi $nDoc($iPhrase,$iCol)
        lappend mi $nHit($iPhrase,$iCol)
      }
    }
    set aMatchinfo($docid) $mi
  }

  set matchinfo_asc [list]
  foreach docid [lsort -integer -incr [array names aMatchinfo]] {
    lappend matchinfo_asc $docid $aMatchinfo($docid)
  }
  set matchinfo_desc [list]
  foreach docid [lsort -integer -decr [array names aMatchinfo]] {
    lappend matchinfo_desc $docid $aMatchinfo($docid)
  }

  set title "(\"$match\" -> [llength [array names hits]] rows)"

  do_execsql_test $tn$title.1 "
    SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC
  " [lsort -integer -incr [array names hits]] 

  do_execsql_test $tn$title.2 "
    SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid DESC
  " [lsort -integer -decr [array names hits]] 

  do_execsql_test $tn$title.3 "
    SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl 
    WHERE $tbl MATCH '$match' ORDER BY docid DESC
  " $matchinfo_desc

  do_execsql_test $tn$title.4 "
    SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl 
    WHERE $tbl MATCH '$match' ORDER BY docid ASC
  " $matchinfo_asc
} 

do_test 2.1 {
  execsql { CREATE VIRTUAL TABLE t1 USING fts3(a, b) }
  for {set i 0} {$i<32} {incr i} {
    set doc [list]
    if {$i&0x01} {lappend doc one}
    if {$i&0x02} {lappend doc two}
    if {$i&0x04} {lappend doc three}
    if {$i&0x08} {lappend doc four}
    if {$i&0x10} {lappend doc five}
    execsql { INSERT INTO t1 VALUES($doc, null) }
  }
} {}
foreach {tn expr} {
  1     {one}
  2     {one NEAR/1 five}
  3     {t*}
  4     {t* NEAR/0 five}
  5     {o* NEAR/1 f*}
  6     {one NEAR five NEAR two NEAR four NEAR three}
} {
  do_near_test 2.2.$tn t1 $expr
}

finish_test

Changes to test/tester.tcl.

370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
  } {
    set testname "${::testprefix}-$testname"
  }
}
    
proc do_execsql_test {testname sql {result {}}} {
  fix_testname testname
  uplevel do_test $testname [list "execsql {$sql}"] [list [list {*}$result]]
}
proc do_catchsql_test {testname sql result} {
  fix_testname testname
  uplevel do_test $testname [list "catchsql {$sql}"] [list $result]
}
proc do_eqp_test {name sql res} {
  uplevel do_execsql_test $name [list "EXPLAIN QUERY PLAN $sql"] [list $res]
}

#-------------------------------------------------------------------------
#   Usage: do_select_tests PREFIX ?SWITCHES? TESTLIST







|



|







370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
  } {
    set testname "${::testprefix}-$testname"
  }
}
    
proc do_execsql_test {testname sql {result {}}} {
  fix_testname testname
  uplevel do_test [list $testname] [list "execsql {$sql}"] [list [list {*}$result]]
}
proc do_catchsql_test {testname sql result} {
  fix_testname testname
  uplevel do_test [list $testname] [list "catchsql {$sql}"] [list $result]
}
proc do_eqp_test {name sql res} {
  uplevel do_execsql_test $name [list "EXPLAIN QUERY PLAN $sql"] [list $res]
}

#-------------------------------------------------------------------------
#   Usage: do_select_tests PREFIX ?SWITCHES? TESTLIST