SQLite

  sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
  sqlite3_step(pStmt);

  return sqlite3_finalize(pStmt);
}

static
int queryFts2Tokenizer(
  sqlite3 *db, 
  char *zName,  
  const sqlite3_tokenizer_module **pp
){
  int rc;
  sqlite3_stmt *pStmt;
  const char zSql[] = "SELECT fts2_tokenizer(?)";

/*
** Implementation of the scalar function fts2_tokenizer_internal_test().
** This function is used for testing only, it is not included in the
** build unless SQLITE_TEST is defined.
**
** The purpose of this is to test that the fts2_tokenizer() function
** can be used as designed by the C-code in the queryFts2Tokenizer and
** registerTokenizer() functions above. These two functions are repeated
** in the README.tokenizer file as an example, so it is important to
** test them.
**
** To run the tests, evaluate the fts2_tokenizer_internal_test() scalar
** function with no arguments. An assert() will fail if a problem is
** detected. i.e.:

  int rc;
  const sqlite3_tokenizer_module *p1;
  const sqlite3_tokenizer_module *p2;
  sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);

  /* Test the query function */
  sqlite3Fts2SimpleTokenizerModule(&p1);
  rc = queryFts2Tokenizer(db, "simple", &p2);
  assert( rc==SQLITE_OK );
  assert( p1==p2 );
  rc = queryFts2Tokenizer(db, "nosuchtokenizer", &p2);
  assert( rc==SQLITE_ERROR );
  assert( p2==0 );
  assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );

  /* Test the storage function */
  rc = registerTokenizer(db, "nosuchtokenizer", p1);
  assert( rc==SQLITE_OK );
  rc = queryFts2Tokenizer(db, "nosuchtokenizer", &p2);
  assert( rc==SQLITE_OK );
  assert( p2==p1 );

  sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
}

#endif

** is defined to accept an argument of type int, its behaviour when passed
** an integer that falls outside of the range of the unsigned char type
** is undefined (and sometimes, "undefined" means segfault). This wrapper
** is defined to accept an argument of type char, and always returns 0 for
** any values that fall outside of the range of the unsigned char type (i.e.
** negative values).
*/
static int fts3isspace(char c){
  return (c&0x80)==0 ? isspace(c) : 0;
}

/*
** Extract the next token from buffer z (length n) using the tokenizer
** and other information (column names etc.) in pParse. Create an Fts3Expr
** structure of type FTSQUERY_PHRASE containing a phrase consisting of this

  }
  
  *pnConsumed = nConsumed;
  *ppExpr = pRet;
  return rc;
}


/*
** Enlarge a memory allocation.  If an out-of-memory allocation occurs,
** then free the old allocation.
*/
void *fts3ReallocOrFree(void *pOrig, int nNew){
  void *pRet = sqlite3_realloc(pOrig, nNew);
  if( !pRet ){
    sqlite3_free(pOrig);
  }
  return pRet;
}

    pCursor->pTokenizer = pTokenizer;
    for(ii=0; rc==SQLITE_OK; ii++){
      const char *zToken;
      int nToken, iBegin, iEnd, iPos;
      rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
      if( rc==SQLITE_OK ){
        int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
        p = fts3ReallocOrFree(p, nByte+ii*sizeof(struct PhraseToken));
        zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken);
        if( !p || !zTemp ){
          goto no_mem;
        }
        if( ii==0 ){
          memset(p, 0, nByte);
          p->pPhrase = (Fts3Phrase *)&p[1];
          p->eType = FTSQUERY_PHRASE;

  if( rc==SQLITE_DONE ){
    int jj;
    char *zNew;
    int nNew = 0;
    int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
    nByte += (p->pPhrase->nToken-1) * sizeof(struct PhraseToken);
    p = fts3ReallocOrFree(p, nByte + nTemp);
    if( !p ){
      goto no_mem;
    }
    p->pPhrase = (Fts3Phrase *)&p[1];
    zNew = &(((char *)p)[nByte]);
    memcpy(zNew, zTemp, nTemp);
    for(jj=0; jj<p->pPhrase->nToken; jj++){

*/
static int getNextNode(
  ParseContext *pParse,                   /* fts3 query parse context */
  const char *z, int n,                   /* Input string */
  Fts3Expr **ppExpr,                      /* OUT: expression */
  int *pnConsumed                         /* OUT: Number of bytes consumed */
){
  static const struct Fts3Keyword {
    char z[4];                            /* Keyword text */
    u8 n;                                 /* Length of the keyword */
    u8 parenOnly;                         /* Only valid in paren mode */
    u8 eType;                             /* Keyword code */
  } aKeyword[] = {
    { "OR" ,  2, 0, FTSQUERY_OR   },
    { "AND",  3, 1, FTSQUERY_AND  },
    { "NOT",  3, 1, FTSQUERY_NOT  },
    { "NEAR", 4, 0, FTSQUERY_NEAR }
  };
  int ii;
  int iCol;
  int iColLen;
  int rc;
  Fts3Expr *pRet = 0;

  const char *zInput = z;
  int nInput = n;

  /* Skip over any whitespace before checking for a keyword, an open or
  ** close bracket, or a quoted string. 
  */
  while( nInput>0 && fts3isspace(*zInput) ){
    nInput--;
    zInput++;
  }

  /* See if we are dealing with a keyword. */
  for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){
    const struct Fts3Keyword *pKey = &aKeyword[ii];

    if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){


      continue;
    }

    if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){
      int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM;
      int nKey = pKey->n;
      char cNext;

            nNear = nNear * 10 + (zInput[nKey] - '0');
          }
        }
      }

      /* At this point this is probably a keyword. But for that to be true,
      ** the next byte must contain either whitespace, an open or close
      ** parenthesis, a quote character, or EOF. 
      */
      cNext = zInput[nKey];
      if( fts3isspace(cNext) 
       || cNext=='"' || cNext=='(' || cNext==')' || cNext==0
      ){
        pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr));
        memset(pRet, 0, sizeof(Fts3Expr));
        pRet->eType = pKey->eType;
        pRet->nNear = nNear;
        *ppExpr = pRet;

#ifdef SQLITE_TEST

#include <stdio.h>

/*
** Function to query the hash-table of tokenizers (see README.tokenizers).
*/
static int queryTestTokenizer(
  sqlite3 *db, 
  const char *zName,  
  const sqlite3_tokenizer_module **pp
){
  int rc;
  sqlite3_stmt *pStmt;
  const char zSql[] = "SELECT fts3_tokenizer(?)";

*/
static void fts3ExprTest(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  sqlite3_tokenizer_module const *pModule = 0;
  sqlite3_tokenizer *pTokenizer = 0;
  int rc;
  char **azCol = 0;
  const char *zExpr;
  int nExpr;
  int nCol;
  int ii;
  Fts3Expr *pExpr;
  sqlite3 *db = sqlite3_context_db_handle(context);

  if( argc<3 ){
    sqlite3_result_error(context, 
        "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1
    );
    return;
  }

  rc = queryTestTokenizer(db,
                          (const char *)sqlite3_value_text(argv[0]), &pModule);
  if( rc==SQLITE_NOMEM ){
    sqlite3_result_error_nomem(context);
    goto exprtest_out;
  }else if( !pModule ){
    sqlite3_result_error(context, "No such tokenizer module", -1);
    goto exprtest_out;
  }

    sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
    sqlite3Fts3ExprFree(pExpr);
  }else{
    sqlite3_result_error(context, "Error parsing expression", -1);
  }

exprtest_out:
  if( pModule && pTokenizer ){
    rc = pModule->xDestroy(pTokenizer);
  }
  sqlite3_free(azCol);
}

/*
** Register the query expression parser test function fts3_exprtest() 

/*
** The following describes the syntax supported by the fts3 MATCH
** operator in a similar format to that used by the lemon parser
** generator. This module does not use actually lemon, it uses a
** custom parser.
**
**   query ::= andexpr (OR andexpr)*.
**

**   andexpr ::= notexpr (AND? notexpr)*.
**
**   notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*.
**   notexpr ::= LP query RP.
**
**   nearexpr ::= phrase (NEAR distance_opt nearexpr)*.
**
**   distance_opt ::= .
**   distance_opt ::= / INTEGER.
**
**   phrase ::= TOKEN.
**   phrase ::= COLUMN:TOKEN.
**   phrase ::= "TOKEN TOKEN TOKEN...".
*/

typedef struct Fts3Expr Fts3Expr;
typedef struct Fts3Phrase Fts3Phrase;

/*
** A "phrase" is a sequence of one or more tokens that must match in
** sequence.  A single token is the base case and the most common case.
** For a sequence of tokens contained in "...", nToken will be the number
** of tokens in the string.
*/
struct Fts3Phrase {
  int nToken;          /* Number of tokens in the phrase */
  int iColumn;         /* Index of column this phrase must match */
  int isNot;           /* Phrase prefixed by unary not (-) operator */
  struct PhraseToken {
    char *z;              /* Text of the token */
    int n;                /* Number of bytes in buffer pointed to by z */
    int isPrefix;         /* True if token ends in with a "*" character */
  } aToken[1];         /* One entry for each token in the phrase */
};

/*
** A tree of these objects forms the RHS of a MATCH operator.
*/
struct Fts3Expr {
  int eType;                 /* One of the FTSQUERY_XXX values defined below */
  int nNear;                 /* Valid if eType==FTSQUERY_NEAR */
  Fts3Expr *pParent;         /* pParent->pLeft==this or pParent->pRight==this */
  Fts3Expr *pLeft;           /* Left operand */
  Fts3Expr *pRight;          /* Right operand */
  Fts3Phrase *pPhrase;       /* Valid if eType==FTSQUERY_PHRASE */
};

int sqlite3Fts3ExprParse(sqlite3_tokenizer *, char **, int, int, 
                         const char *, int, Fts3Expr **);
void sqlite3Fts3ExprFree(Fts3Expr *);

#define FTSQUERY_AND    3
#define FTSQUERY_OR     4
#define FTSQUERY_PHRASE 5

#ifdef SQLITE_TEST
void sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
#endif

# 2006 September 9
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the FTS3 module.
#
# $Id: fts3expr.test,v 1.5 2009/01/01 12:34:46 drh Exp $
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {

do_test fts3expr-1.1 {
  test_fts3expr " tag "
} {PHRASE 3 0 tag}

do_test fts3expr-1.2 {
  test_fts3expr "ab AND cd"
} {AND {PHRASE 3 0 ab} {PHRASE 3 0 cd}}
do_test fts3expr-1.2.1 {
  test_fts3expr "ab cd"
} {AND {PHRASE 3 0 ab} {PHRASE 3 0 cd}}
do_test fts3expr-1.3 {
  test_fts3expr "ab OR cd"
} {OR {PHRASE 3 0 ab} {PHRASE 3 0 cd}}
do_test fts3expr-1.4 {
  test_fts3expr "ab NOT cd"
} {NOT {PHRASE 3 0 ab} {PHRASE 3 0 cd}}
do_test fts3expr-1.5 {
  test_fts3expr "ab NEAR cd"
} {NEAR/10 {PHRASE 3 0 ab} {PHRASE 3 0 cd}}
do_test fts3expr-1.6.1 {
  test_fts3expr "ab NEAR/5 cd"
} {NEAR/5 {PHRASE 3 0 ab} {PHRASE 3 0 cd}}
do_test fts3expr-1.6.2 {
  test_fts3expr "ab NEAR/87654321 cd"
} {NEAR/87654321 {PHRASE 3 0 ab} {PHRASE 3 0 cd}}
do_test fts3expr-1.6.3 {
  test_fts3expr "ab NEAR/7654321 cd"
} {NEAR/7654321 {PHRASE 3 0 ab} {PHRASE 3 0 cd}}
do_test fts3expr-1.6.4 {
  test_fts3expr "ab NEAR/654321 cd"
} {NEAR/654321 {PHRASE 3 0 ab} {PHRASE 3 0 cd}}
do_test fts3expr-1.6.5 {
  test_fts3expr "ab NEAR/54321 cd"
} {NEAR/54321 {PHRASE 3 0 ab} {PHRASE 3 0 cd}}
do_test fts3expr-1.6.6 {
  test_fts3expr "ab NEAR/4321 cd"
} {NEAR/4321 {PHRASE 3 0 ab} {PHRASE 3 0 cd}}
do_test fts3expr-1.6.7 {
  test_fts3expr "ab NEAR/321 cd"
} {NEAR/321 {PHRASE 3 0 ab} {PHRASE 3 0 cd}}
do_test fts3expr-1.6.8 {
  test_fts3expr "ab NEAR/21 cd"
} {NEAR/21 {PHRASE 3 0 ab} {PHRASE 3 0 cd}}

do_test fts3expr-1.7 {
  test_fts3expr {"one two three"}
} {PHRASE 3 0 one two three}
do_test fts3expr-1.8.1 {
  test_fts3expr {zero "one two three" four}
} {AND {AND {PHRASE 3 0 zero} {PHRASE 3 0 one two three}} {PHRASE 3 0 four}}
do_test fts3expr-1.8.2 {
  test_fts3expr {zero AND "one two three" four}
} {AND {AND {PHRASE 3 0 zero} {PHRASE 3 0 one two three}} {PHRASE 3 0 four}}
do_test fts3expr-1.8.3 {
  test_fts3expr {zero "one two three" AND four}
} {AND {AND {PHRASE 3 0 zero} {PHRASE 3 0 one two three}} {PHRASE 3 0 four}}
do_test fts3expr-1.8.4 {
  test_fts3expr {zero AND "one two three" AND four}
} {AND {AND {PHRASE 3 0 zero} {PHRASE 3 0 one two three}} {PHRASE 3 0 four}}
do_test fts3expr-1.9.1 {
  test_fts3expr {"one* two three"}
} {PHRASE 3 0 one+ two three}
do_test fts3expr-1.9.2 {
  test_fts3expr {"one two* three"}
} {PHRASE 3 0 one two+ three}
do_test fts3expr-1.9.3 {
  test_fts3expr {"one* two* three"}
} {PHRASE 3 0 one+ two+ three}
do_test fts3expr-1.9.4 {
  test_fts3expr {"one two three*"}
} {PHRASE 3 0 one two three+}
do_test fts3expr-1.9.5 {
  test_fts3expr {"one* two three*"}
} {PHRASE 3 0 one+ two three+}
do_test fts3expr-1.9.6 {
  test_fts3expr {"one two* three*"}
} {PHRASE 3 0 one two+ three+}
do_test fts3expr-1.9.7 {
  test_fts3expr {"one* two* three*"}
} {PHRASE 3 0 one+ two+ three+}

do_test fts3expr-1.10 {
  test_fts3expr {one* two}
} {AND {PHRASE 3 0 one+} {PHRASE 3 0 two}}
do_test fts3expr-1.11 {
  test_fts3expr {one two*}
} {AND {PHRASE 3 0 one} {PHRASE 3 0 two+}}

do_test fts3expr-1.14 {
  test_fts3expr {a:one two}
} {AND {PHRASE 0 0 one} {PHRASE 3 0 two}}
do_test fts3expr-1.15 {
  test_fts3expr {one b:two}
} {AND {PHRASE 3 0 one} {PHRASE 1 0 two}}

do_test fts3expr-1.16 {
  test_fts3expr {one AND two AND three AND four AND five}
} [list AND \
        [list AND \
              [list AND \
                    [list AND {PHRASE 3 0 one} {PHRASE 3 0 two}] \
                    {PHRASE 3 0 three} \
              ] \
              {PHRASE 3 0 four} \
        ] \
        {PHRASE 3 0 five} \
  ]
do_test fts3expr-1.17 {
  test_fts3expr {(one AND two) AND ((three AND four) AND five)}
} [list AND \
        [list AND {PHRASE 3 0 one} {PHRASE 3 0 two}] \
        [list AND \
              [list AND {PHRASE 3 0 three} {PHRASE 3 0 four}] \
             {PHRASE 3 0 five} \
        ] \
  ]
do_test fts3expr-1.18 {
  test_fts3expr {(one AND two) OR ((three AND four) AND five)}
} [list OR \
        [list AND {PHRASE 3 0 one} {PHRASE 3 0 two}] \
        [list AND \
              [list AND {PHRASE 3 0 three} {PHRASE 3 0 four}] \
             {PHRASE 3 0 five} \
        ] \
  ]
do_test fts3expr-1.19 {
  test_fts3expr {(one AND two) AND ((three AND four) OR five)}
} [list AND \
        [list AND {PHRASE 3 0 one} {PHRASE 3 0 two}] \
        [list OR \
              [list AND {PHRASE 3 0 three} {PHRASE 3 0 four}] \
             {PHRASE 3 0 five} \
        ] \
  ]
do_test fts3expr-1.20 {
  test_fts3expr {(one OR two) AND ((three OR four) AND five)}
} [list AND \
        [list OR {PHRASE 3 0 one} {PHRASE 3 0 two}] \
        [list AND \
              [list OR {PHRASE 3 0 three} {PHRASE 3 0 four}] \
             {PHRASE 3 0 five} \
        ] \
  ]
do_test fts3expr-1.21 {
  test_fts3expr {(one OR two) AND ((three NOT four) AND five)}
} [list AND \
        [list OR {PHRASE 3 0 one} {PHRASE 3 0 two}] \
        [list AND \
              [list NOT {PHRASE 3 0 three} {PHRASE 3 0 four}] \
             {PHRASE 3 0 five} \
        ] \
  ]
do_test fts3expr-1.22 {
  test_fts3expr {(one OR two) NOT ((three OR four) AND five)}
} [list NOT \
        [list OR {PHRASE 3 0 one} {PHRASE 3 0 two}] \
        [list AND \
              [list OR {PHRASE 3 0 three} {PHRASE 3 0 four}] \
             {PHRASE 3 0 five} \
        ] \
  ]
do_test fts3expr-1.23 {
  test_fts3expr {(((((one OR two))))) NOT (((((three OR four))) AND five))}
} [list NOT \
        [list OR {PHRASE 3 0 one} {PHRASE 3 0 two}] \
        [list AND \
              [list OR {PHRASE 3 0 three} {PHRASE 3 0 four}] \
             {PHRASE 3 0 five} \
        ] \
  ]
do_test fts3expr-1.24 {
  test_fts3expr {one NEAR two}
} [list NEAR/10 {PHRASE 3 0 one} {PHRASE 3 0 two}]
do_test fts3expr-1.25 {
  test_fts3expr {(one NEAR two)}
} [list NEAR/10 {PHRASE 3 0 one} {PHRASE 3 0 two}]
do_test fts3expr-1.26 {
  test_fts3expr {((((((one NEAR two))))))}
} [list NEAR/10 {PHRASE 3 0 one} {PHRASE 3 0 two}]
do_test fts3expr-1.27 {
  test_fts3expr {(one NEAR two) OR ((three OR four) AND five)}
} [list OR \
        [list NEAR/10 {PHRASE 3 0 one} {PHRASE 3 0 two}] \
        [list AND \
              [list OR {PHRASE 3 0 three} {PHRASE 3 0 four}] \
             {PHRASE 3 0 five} \
        ] \
  ]
do_test fts3expr-1.28 {
  test_fts3expr {(one NEAR/321 two) OR ((three OR four) AND five)}
} [list OR \
        [list NEAR/321 {PHRASE 3 0 one} {PHRASE 3 0 two}] \
        [list AND \
              [list OR {PHRASE 3 0 three} {PHRASE 3 0 four}] \
             {PHRASE 3 0 five} \
        ] \
  ]

proc strip_phrase_data {L} {
  if {[lindex $L 0] eq "PHRASE"} {
    return [lrange $L 3 end]
  }
  return [list \
    [lindex $L 0] \

  do_test fts3expr-6.$id {
    execsql { SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid }
  } $res
}

set sqlite_fts3_enable_parentheses 0
finish_test