SQLite4
Check-in [b1a2a17679]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add support for NEAR to the fts expression parser.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: b1a2a1767908bd53b45029c5367e7108bcef8d72
User & Date: dan 2012-12-18 15:47:02
Context
2012-12-19
20:01
Add the "CREATE INDEX idx ON tbl USING nm(...)" syntax. check-in: 8ac71062f5 user: dan tags: trunk
2012-12-18
15:47
Add support for NEAR to the fts expression parser. check-in: b1a2a17679 user: dan tags: trunk
2012-12-17
20:18
Add a test for the "colname:phrase" syntax to fts5expr1.test. check-in: c472cae982 user: dan tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/fts5.c.

9
10
11
12
13
14
15


16
17
18
19
20
21
22
...
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
...
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
...
210
211
212
213
214
215
216
217
218
219
220
221
222
















223
224
225
226
227
228
229
230
...
268
269
270
271
272
273
274











275
276
277
278
279
280
281
**    May you share freely, never taking more than you give.
**
*************************************************************************
*/

#include "sqliteInt.h"



/*
** Token types used by expression parser.
*/
#define TOKEN_EOF       0         /* end of expression - no more tokens */
#define TOKEN_PRIMITIVE 1         /* quoted string or non-keyword */
#define TOKEN_STAR      2         /* * */
#define TOKEN_PLUS      3         /* + */
................................................................................
  Fts5ExprNode *pRight;
};

struct Fts5Expr {
  Fts5ExprNode *pRoot;
};

/*
** Return true if argument c is an ASCII whitespace character.
*/
static int fts5IsWhite(char c){
  return (c==' ' || c=='\n' || c=='\r' || c=='\t');
}

/*
** Return true if argument c is one of the special non-whitespace 
** characters that ends an unquoted expression token. 
*/
static int fts5IsSpecial(char c){
  return (c==':' || c=='(' || c==')' || c=='+' || c=='"');
}
................................................................................
){
  const char *z = pParse->zExpr;
  char c;

  memset(p, 0, sizeof(Fts5ParserToken));

  /* Skip past any whitespace */
  while( fts5IsWhite(z[pParse->iExpr]) ) pParse->iExpr++;

  c = z[pParse->iExpr];
  if( c=='\0' ){
    p->eType = TOKEN_EOF;
  }

  else if( c=='(' ){
................................................................................
  }

  else{
    const char *zPrimitive = &z[pParse->iExpr];
    int n = 0;
    while( zPrimitive[n] 
        && fts5IsSpecial(zPrimitive[n])==0
        && fts5IsWhite(zPrimitive[n])==0 
    ){
      n++;
    }
    pParse->iExpr += n;

















    if( n==3 && memcmp(zPrimitive, "NOT", 3)==0 ){
      p->eType = TOKEN_NOT;
    }
    else if( n==2 && memcmp(zPrimitive, "OR", 2)==0 ){
      p->eType = TOKEN_OR;
    }
    else if( n==3 && memcmp(zPrimitive, "AND", 3)==0 ){
      p->eType = TOKEN_AND;
................................................................................
    aNew = (Fts5Str *)sqlite4DbRealloc(pParse->db, 
        pPhrase->aStr, (pPhrase->nStr+nIncr)*sizeof(Fts5Str)
    );
    if( !aNew ) return SQLITE4_NOMEM;
    memset(&aNew[pPhrase->nStr], 0, nIncr*sizeof(Fts5Str));
    pPhrase->aStr = aNew;
  }












  pPhrase->nStr++;
  return SQLITE4_OK;
}

/*
** Callback for fts5CountTokens().







>
>







 







<
<
<
<
<
<
<







 







|







 







|





>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|







 







>
>
>
>
>
>
>
>
>
>
>







9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
...
127
128
129
130
131
132
133







134
135
136
137
138
139
140
...
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
...
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
...
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
**    May you share freely, never taking more than you give.
**
*************************************************************************
*/

#include "sqliteInt.h"

#define FTS5_DEFAULT_NEAR 10

/*
** Token types used by expression parser.
*/
#define TOKEN_EOF       0         /* end of expression - no more tokens */
#define TOKEN_PRIMITIVE 1         /* quoted string or non-keyword */
#define TOKEN_STAR      2         /* * */
#define TOKEN_PLUS      3         /* + */
................................................................................
  Fts5ExprNode *pRight;
};

struct Fts5Expr {
  Fts5ExprNode *pRoot;
};








/*
** Return true if argument c is one of the special non-whitespace 
** characters that ends an unquoted expression token. 
*/
static int fts5IsSpecial(char c){
  return (c==':' || c=='(' || c==')' || c=='+' || c=='"');
}
................................................................................
){
  const char *z = pParse->zExpr;
  char c;

  memset(p, 0, sizeof(Fts5ParserToken));

  /* Skip past any whitespace */
  while( sqlite4Isspace(z[pParse->iExpr]) ) pParse->iExpr++;

  c = z[pParse->iExpr];
  if( c=='\0' ){
    p->eType = TOKEN_EOF;
  }

  else if( c=='(' ){
................................................................................
  }

  else{
    const char *zPrimitive = &z[pParse->iExpr];
    int n = 0;
    while( zPrimitive[n] 
        && fts5IsSpecial(zPrimitive[n])==0
        && sqlite4Isspace(zPrimitive[n])==0 
    ){
      n++;
    }
    pParse->iExpr += n;

    if( n>=4 && memcmp(zPrimitive, "NEAR", 4)==0 ){
      int nNear = FTS5_DEFAULT_NEAR;
      if( n>4 ){
        int i;
        nNear = 0;
        for(i=5; i<n; i++){
          if( !sqlite4Isdigit(zPrimitive[i]) ) break;
          nNear = nNear*10 + zPrimitive[i]-'0';
        }
        if( n<6 || zPrimitive[4]!='/' || i<n ){
          return SQLITE4_ERROR;
        }
      }
      p->eType = TOKEN_NEAR;
      p->n = nNear;
      p->z = 0;
    }else if( n==3 && memcmp(zPrimitive, "NOT", 3)==0 ){
      p->eType = TOKEN_NOT;
    }
    else if( n==2 && memcmp(zPrimitive, "OR", 2)==0 ){
      p->eType = TOKEN_OR;
    }
    else if( n==3 && memcmp(zPrimitive, "AND", 3)==0 ){
      p->eType = TOKEN_AND;
................................................................................
    aNew = (Fts5Str *)sqlite4DbRealloc(pParse->db, 
        pPhrase->aStr, (pPhrase->nStr+nIncr)*sizeof(Fts5Str)
    );
    if( !aNew ) return SQLITE4_NOMEM;
    memset(&aNew[pPhrase->nStr], 0, nIncr*sizeof(Fts5Str));
    pPhrase->aStr = aNew;
  }
  if( pPhrase->nStr>0 ){
    if( ((pPhrase->nStr-1) % nIncr)==0 ){
      int *aNew;
      aNew = (Fts5Str *)sqlite4DbRealloc(pParse->db, 
        pPhrase->aiNear, (pPhrase->nStr+nIncr-1)*sizeof(int)
      );
      if( !aNew ) return SQLITE4_NOMEM;
      pPhrase->aiNear = aNew;
    }
    pPhrase->aiNear[pPhrase->nStr-1] = nNear;
  }

  pPhrase->nStr++;
  return SQLITE4_OK;
}

/*
** Callback for fts5CountTokens().

Changes to test/fts5expr1.test.

34
35
36
37
38
39
40




41
42
43
44
45
46
47
  13 { "abc" "def" }            {"abc" AND "def"}
  14 { "abc" + "def" }          {"abc"+"def"}

  15 { a NOT b AND c OR d }     {(("a" NOT "b") AND "c") OR "d"}
  16 { a OR b AND c NOT d }     {"a" OR ("b" AND ("c" NOT "d"))}
  17 { (a OR b) AND c NOT d }   {("a" OR "b") AND ("c" NOT "d")}





} {
  do_execsql_test 1.$tn { 
    SELECT fts5_parse_expr('simple', $expr, '') 
  } [list [string trim $res]]
}

do_execsql_test 2.0 { 







>
>
>
>







34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
  13 { "abc" "def" }            {"abc" AND "def"}
  14 { "abc" + "def" }          {"abc"+"def"}

  15 { a NOT b AND c OR d }     {(("a" NOT "b") AND "c") OR "d"}
  16 { a OR b AND c NOT d }     {"a" OR ("b" AND ("c" NOT "d"))}
  17 { (a OR b) AND c NOT d }   {("a" OR "b") AND ("c" NOT "d")}

  18 { a NEAR/10 b }                {"a" NEAR/10 "b"}
  19 { a NEAR b }                   {"a" NEAR/10 "b"}
  20 { a AND b OR c + d NEAR/52 e } {("a" AND "b") OR ("c"+"d" NEAR/52 "e")}

} {
  do_execsql_test 1.$tn { 
    SELECT fts5_parse_expr('simple', $expr, '') 
  } [list [string trim $res]]
}

do_execsql_test 2.0 {