SQLite

**
** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both
** have their respective doclists (including position information) loaded
** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from
** each doclist that are not within nNear tokens of a corresponding entry
** in the other doclist.
*/
int sqlite3Fts3ExprNearTrim(Fts3Expr *pELeft, Fts3Expr *pERight, int nNear){
  int rc;                         /* Return code */
  Fts3Phrase *pLeft = pELeft->pPhrase;
  Fts3Phrase *pRight = pERight->pPhrase;

  assert( pELeft->eType==FTSQUERY_PHRASE && pLeft );
  assert( pERight->eType==FTSQUERY_PHRASE && pRight );
  assert( pLeft->isLoaded && pRight->isLoaded );

  if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){
    sqlite3_free(pLeft->aDoclist);
    sqlite3_free(pRight->aDoclist);
    pRight->aDoclist = 0;
    pLeft->aDoclist = 0;
    rc = SQLITE_OK;
  }else{
    char *aOut;                   /* Buffer in which to assemble new doclist */
    int nOut;                     /* Size of buffer aOut in bytes */

    rc = fts3NearMerge(MERGE_POS_NEAR, nNear, 
        pLeft->nToken, pLeft->aDoclist, pLeft->nDoclist,
        pRight->nToken, pRight->aDoclist, pRight->nDoclist,
        &aOut, &nOut
    );
    if( rc!=SQLITE_OK ) return rc;
    sqlite3_free(pRight->aDoclist);
    pRight->aDoclist = aOut;
    pRight->nDoclist = nOut;

    rc = fts3NearMerge(MERGE_POS_NEAR, nNear, 
        pRight->nToken, pRight->aDoclist, pRight->nDoclist,
        pLeft->nToken, pLeft->aDoclist, pLeft->nDoclist,
        &aOut, &nOut
    );
    sqlite3_free(pLeft->aDoclist);
    pLeft->aDoclist = aOut;
    pLeft->nDoclist = nOut;
  }
  return rc;

** Load the doclist associated with expression pExpr to pExpr->aDoclist.
** The loaded doclist contains positions as well as the document ids.
** This is used by the matchinfo(), snippet() and offsets() auxillary
** functions.
*/
int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){
  int rc;
  Fts3Phrase *pPhrase = pExpr->pPhrase;
  assert( pExpr->eType==FTSQUERY_PHRASE && pPhrase );
  assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
  rc = fts3EvalExpr(pCsr, pExpr, &pPhrase->aDoclist, &pPhrase->nDoclist, 1);
  return rc;
}

/*
** TODO: This is something to do with matchinfo(). Similar to
** sqlite3ExprLoadDoclists() but slightly different.
**
** UPDATE: Only used when there are deferred tokens.
*/
int sqlite3Fts3ExprLoadFtDoclist(
  Fts3Cursor *pCsr, 
  Fts3Expr *pExpr,
  char **paDoclist,
  int *pnDoclist
){
  int rc;

*/
char *sqlite3Fts3FindPositions(
  Fts3Cursor *pCursor,            /* Associate FTS3 cursor */
  Fts3Expr *pExpr,                /* Access this expressions doclist */
  sqlite3_int64 iDocid,           /* Docid associated with requested pos-list */
  int iCol                        /* Column of requested pos-list */
){
  Fts3Phrase *pPhrase = pExpr->pPhrase;
  assert( pPhrase->isLoaded );

  if( pPhrase->aDoclist ){
    char *pEnd = &pPhrase->aDoclist[pPhrase->nDoclist];
    char *pCsr;

    if( pPhrase->pCurrent==0 ){
      if( pCursor->desc==0 ){
        pPhrase->pCurrent = pPhrase->aDoclist;
        pPhrase->iCurrent = 0;
        fts3GetDeltaVarint(&pPhrase->pCurrent, &pPhrase->iCurrent);
      }else{
        pCsr = pPhrase->aDoclist;
        while( pCsr<pEnd ){
          fts3GetDeltaVarint(&pCsr, &pPhrase->iCurrent);
          fts3PoslistCopy(0, &pCsr);
        }
        fts3ReversePoslist(pPhrase->aDoclist, &pCsr);
        pPhrase->pCurrent = pCsr;
      }
    }
    pCsr = pPhrase->pCurrent;
    assert( pCsr );

    while( (pCursor->desc==0 && pCsr<pEnd) 
        || (pCursor->desc && pCsr>pPhrase->aDoclist) 
    ){
      if( pCursor->desc==0 && pPhrase->iCurrent<iDocid ){
        fts3PoslistCopy(0, &pCsr);
        if( pCsr<pEnd ){
          fts3GetDeltaVarint(&pCsr, &pPhrase->iCurrent);
        }
        pPhrase->pCurrent = pCsr;
      }else if( pCursor->desc && pPhrase->iCurrent>iDocid ){
        fts3GetReverseDeltaVarint(&pCsr, pPhrase->aDoclist, &pPhrase->iCurrent);
        fts3ReversePoslist(pPhrase->aDoclist, &pCsr);
        pPhrase->pCurrent = pCsr;
      }else{
        if( pPhrase->iCurrent==iDocid ){
          int iThis = 0;
          if( iCol<0 ){
            /* If iCol is negative, return a pointer to the start of the
            ** position-list (instead of a pointer to the start of a list
            ** of offsets associated with a specific column).
            */
            return pCsr;

#define FTS3_FULLTEXT_SEARCH 2    /* Full-text index search */

/*
** A "phrase" is a sequence of one or more tokens that must match in
** sequence.  A single token is the base case and the most common case.
** For a sequence of tokens contained in double-quotes (i.e. "one two three")
** nToken will be the number of tokens in the string.






*/
struct Fts3PhraseToken {
  char *z;                        /* Text of the token */
  int n;                          /* Number of bytes in buffer z */
  int isPrefix;                   /* True if token ends with a "*" character */

  /* Variables above this point are populated when the expression is
  ** parsed (by code in fts3_expr.c). Below this point the variables are
  ** used when evaluating the expression. */

  int bFulltext;                  /* True if full-text index was used */
  Fts3SegReaderCursor *pSegcsr;   /* Segment-reader for this token */
  Fts3DeferredToken *pDeferred;   /* Deferred token object for this token */
};

struct Fts3Phrase {
  /* Variables populated by fts3_expr.c when parsing a MATCH expression */
  int nToken;                /* Number of tokens in the phrase */
  int iColumn;               /* Index of column this phrase must match */

  int isLoaded;              /* True if aDoclist/nDoclist are initialized. */
  char *aDoclist;            /* Buffer containing doclist */
  int nDoclist;              /* Size of aDoclist in bytes */
  sqlite3_int64 iCurrent;
  char *pCurrent;

  Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
};

/*
** A tree of these objects forms the RHS of a MATCH operator.
**
** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist 
** points to a malloced buffer, size nDoclist bytes, containing the results 
** of this phrase query in FTS3 doclist format. As usual, the initial 
** "Length" field found in doclists stored on disk is omitted from this 
** buffer.
**
** Variable pCurrent always points to the start of a docid field within
** aDoclist. Since the doclist is usually scanned in docid order, this can
** be used to accelerate seeking to the required docid within the doclist.
*/
struct Fts3Expr {
  int eType;                 /* One of the FTSQUERY_XXX values defined below */
  int nNear;                 /* Valid if eType==FTSQUERY_NEAR */
  Fts3Expr *pParent;         /* pParent->pLeft==this or pParent->pRight==this */
  Fts3Expr *pLeft;           /* Left operand */
  Fts3Expr *pRight;          /* Right operand */
  Fts3Phrase *pPhrase;       /* Valid if eType==FTSQUERY_PHRASE */







};

/*
** Candidate values for Fts3Query.eType. Note that the order of the first
** four values is in order of precedence when parsing expressions. For 
** example, the following:
**

*/
#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10

#include "fts3Int.h"
#include <string.h>
#include <assert.h>

/*
** isNot:
**   This variable is used by function getNextNode(). When getNextNode() is
**   called, it sets ParseContext.isNot to true if the 'next node' is a 
**   FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the
**   FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to
**   zero.
*/
typedef struct ParseContext ParseContext;
struct ParseContext {
  sqlite3_tokenizer *pTokenizer;      /* Tokenizer module */
  const char **azCol;                 /* Array of column names for fts3 table */
  int nCol;                           /* Number of entries in azCol[] */
  int iDefaultCol;                    /* Default column to query */
  int isNot;                          /* True if getNextNode() sees a unary - */
  sqlite3_context *pCtx;              /* Write error message here */
  int nNest;                          /* Number of nested brackets */
};

/*
** This function is equivalent to the standard isspace() function. 
**

        memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken);

        if( iEnd<n && z[iEnd]=='*' ){
          pRet->pPhrase->aToken[0].isPrefix = 1;
          iEnd++;
        }
        if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
          pParse->isNot = 1;
        }
      }
      nConsumed = iEnd;
    }

    pModule->xClose(pCursor);
  }

  sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
  int rc;
  Fts3Expr *p = 0;
  sqlite3_tokenizer_cursor *pCursor = 0;
  char *zTemp = 0;
  int nTemp = 0;

  const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
  int nToken = 0;

  /* The final Fts3Expr data structure, including the Fts3Phrase,
  ** Fts3PhraseToken structures token buffers are all stored as a single 
  ** allocation so that the expression can be freed with a single call to
  ** sqlite3_free(). Setting this up requires a two pass approach.
  **
  ** The first pass, in the block below, uses a tokenizer cursor to iterate
  ** through the tokens in the expression. This pass uses fts3ReallocOrFree()
  ** to assemble data in two dynamic buffers:
  **
  **   Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase
  **             structure, followed by the array of Fts3PhraseToken 
  **             structures. This pass only populates the Fts3PhraseToken array.
  **
  **   Buffer zTemp: Contains copies of all tokens.
  **
  ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below,
  ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase
  ** structures.
  */
  rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
  if( rc==SQLITE_OK ){
    int ii;
    pCursor->pTokenizer = pTokenizer;
    for(ii=0; rc==SQLITE_OK; ii++){
      const char *zByte;
      int nByte, iBegin, iEnd, iPos;
      rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos);
      if( rc==SQLITE_OK ){
        Fts3PhraseToken *pToken;

        p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken));

        if( !p ) goto no_mem;

        zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte);
        if( !zTemp ) goto no_mem;



        assert( nToken==ii );
        pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii];
        memset(pToken, 0, sizeof(Fts3PhraseToken));


        memcpy(&zTemp[nTemp], zByte, nByte);
        nTemp += nByte;





        pToken->n = nByte;
        pToken->isPrefix = (iEnd<nInput && zInput[iEnd]=='*');
        nToken = ii+1;
      }
    }

    pModule->xClose(pCursor);
    pCursor = 0;
  }

  if( rc==SQLITE_DONE ){
    int jj;
    char *zBuf = 0;



    p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp);
    if( !p ) goto no_mem;
    memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p);
    p->eType = FTSQUERY_PHRASE;
    p->pPhrase = (Fts3Phrase *)&p[1];
    p->pPhrase->iColumn = pParse->iDefaultCol;
    p->pPhrase->nToken = nToken;


    zBuf = (char *)&p->pPhrase->aToken[nToken];
    memcpy(zBuf, zTemp, nTemp);

    sqlite3_free(zTemp);


    for(jj=0; jj<p->pPhrase->nToken; jj++){
      p->pPhrase->aToken[jj].z = zBuf;
      zBuf += p->pPhrase->aToken[jj].n;
    }



    rc = SQLITE_OK;
  }

  *ppExpr = p;
  return rc;
no_mem:

  int iCol;
  int iColLen;
  int rc;
  Fts3Expr *pRet = 0;

  const char *zInput = z;
  int nInput = n;

  pParse->isNot = 0;

  /* Skip over any whitespace before checking for a keyword, an open or
  ** close bracket, or a quoted string. 
  */
  while( nInput>0 && fts3isspace(*zInput) ){
    nInput--;
    zInput++;

    Fts3Expr *p = 0;
    int nByte = 0;
    rc = getNextNode(pParse, zIn, nIn, &p, &nByte);
    if( rc==SQLITE_OK ){
      int isPhrase;

      if( !sqlite3_fts3_enable_parentheses 
       && p->eType==FTSQUERY_PHRASE && pParse->isNot 
      ){
        /* Create an implicit NOT operator. */
        Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
        if( !pNot ){
          sqlite3Fts3ExprFree(p);
          rc = SQLITE_NOMEM;
          goto exprparse_out;
        }
        pNot->eType = FTSQUERY_NOT;
        pNot->pRight = p;
        if( pNotBranch ){
          pNot->pLeft = pNotBranch;
        }
        pNotBranch = pNot;
        p = pPrev;
      }else{
        int eType = p->eType;

        isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft);

        /* The isRequirePhrase variable is set to true if a phrase or
        ** an expression contained in parenthesis is required. If a
        ** binary operator (AND, OR, NOT or NEAR) is encounted when
        ** isRequirePhrase is set, this is a syntax error.
        */

}

/*
** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse().
*/
void sqlite3Fts3ExprFree(Fts3Expr *p){
  if( p ){
    assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 );
    sqlite3Fts3ExprFree(p->pLeft);
    sqlite3Fts3ExprFree(p->pRight);
    if( p->pPhrase ) sqlite3_free(p->pPhrase->aDoclist);
    sqlite3_free(p);
  }
}

/****************************************************************************
*****************************************************************************
** Everything after this point is just test code.

*/
static char *exprToString(Fts3Expr *pExpr, char *zBuf){
  switch( pExpr->eType ){
    case FTSQUERY_PHRASE: {
      Fts3Phrase *pPhrase = pExpr->pPhrase;
      int i;
      zBuf = sqlite3_mprintf(
          "%zPHRASE %d 0", zBuf, pPhrase->iColumn);
      for(i=0; zBuf && i<pPhrase->nToken; i++){
        zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, 
            pPhrase->aToken[i].n, pPhrase->aToken[i].z,
            (pPhrase->aToken[i].isPrefix?"+":"")
        );
      }
      return zBuf;

/*
** This is an fts3ExprIterate() callback used while loading the doclists
** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
** fts3ExprLoadDoclists().
*/
static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
  int rc = SQLITE_OK;
  Fts3Phrase *pPhrase = pExpr->pPhrase;
  LoadDoclistCtx *p = (LoadDoclistCtx *)ctx;

  UNUSED_PARAMETER(iPhrase);

  p->nPhrase++;
  p->nToken += pPhrase->nToken;

  if( pPhrase->isLoaded==0 ){
    rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr);
    pPhrase->isLoaded = 1;
    if( rc==SQLITE_OK ){
      rc = fts3ExprNearTrim(pExpr);
    }
  }

  return rc;
}

static int fts3ExprGlobalHitsCb(
  Fts3Expr *pExpr,                /* Phrase expression node */
  int iPhrase,                    /* Phrase number (numbered from zero) */
  void *pCtx                      /* Pointer to MatchInfo structure */
){
  MatchInfo *p = (MatchInfo *)pCtx;
  Fts3Cursor *pCsr = p->pCursor;
  Fts3Phrase *pPhrase = pExpr->pPhrase; 
  char *pIter;
  char *pEnd;
  char *pFree = 0;
  u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol];

  assert( pPhrase->isLoaded );


  if( pCsr->pDeferred ){

    int ii;
    for(ii=0; ii<pPhrase->nToken; ii++){
      if( pPhrase->aToken[ii].bFulltext ) break;
    }
    if( ii<pPhrase->nToken ){
      int nFree = 0;
      int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree);
      if( rc!=SQLITE_OK ) return rc;
      pIter = pFree;
      pEnd = &pFree[nFree];
    }else{
      int iCol;                   /* Column index */
      for(iCol=0; iCol<p->nCol; iCol++){
        aOut[iCol*3 + 1] = (u32)p->nDoc;
        aOut[iCol*3 + 2] = (u32)p->nDoc;
      }
      return SQLITE_OK;
    }
  }else{
    pIter = pPhrase->aDoclist;
    pEnd = &pPhrase->aDoclist[pPhrase->nDoclist];
  }

  /* Fill in the global hit count matrix row for this phrase. */
  while( pIter<pEnd ){
    while( *pIter++ & 0x80 );      /* Skip past docid. */
    fts3LoadColumnlistCounts(&pIter, &aOut[1], 1);
  }

){
  MatchInfo *p = (MatchInfo *)pCtx;
  int iStart = iPhrase * p->nCol * 3;
  int i;

  for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0;

  if( pExpr->pPhrase->aDoclist ){
    char *pCsr;

    pCsr = sqlite3Fts3FindPositions(p->pCursor, pExpr, p->pCursor->iPrevId, -1);
    if( pCsr ){
      fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0);
    }
  }

/*
** Helper fucntion for FreeDeferredDoclists(). This function removes all
** references to deferred doclists from within the tree of Fts3Expr 
** structures headed by 
*/
static void fts3DeferredDoclistClear(Fts3Expr *pExpr){
  if( pExpr ){
    Fts3Phrase *pPhrase = pExpr->pPhrase;
    fts3DeferredDoclistClear(pExpr->pLeft);
    fts3DeferredDoclistClear(pExpr->pRight);
    if( pPhrase ){
      assert( pExpr->eType==FTSQUERY_PHRASE );
      sqlite3_free(pPhrase->aDoclist);
      pPhrase->isLoaded = 0;
      pPhrase->aDoclist = 0;
      pPhrase->nDoclist = 0;
      pPhrase->pCurrent = 0;
      pPhrase->iCurrent = 0;
    }
  }
}

/*
** Delete all cached deferred doclists. Deferred doclists are cached
** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function.

      SELECT * FROM t1 EXCEPT SELECT * FROM t3;
      SELECT * FROM t1 ORDER BY b;
      SELECT * FROM t1 GROUP BY b;
    }
    set ::update_hook
  } [list]
}

do_test hook-4.4 {
  execsql {
    CREATE TABLE t4(a UNIQUE, b);
    INSERT INTO t4 VALUES(1, 'a');
    INSERT INTO t4 VALUES(2, 'b');
  }
  set ::update_hook [list]
  execsql {
    REPLACE INTO t4 VALUES(1, 'c');
  }
  set ::update_hook
} [list INSERT main t4 3 ]
do_execsql_test hook-4.4.1 {
  SELECT * FROM t4 ORDER BY a;
} {1 c 2 b}
do_test hook-4.4.2 {
  set ::update_hook [list]
  execsql {
    PRAGMA recursive_triggers = on;
    REPLACE INTO t4 VALUES(1, 'd');
  }
  set ::update_hook
} [list INSERT main t4 4 ]
do_execsql_test hook-4.4.3 {
  SELECT * FROM t4 ORDER BY a;
} {1 d 2 b}

db update_hook {}
#
#----------------------------------------------------------------------------

#----------------------------------------------------------------------------
# Test the rollback-hook. The rollback-hook is a bit more complicated than
# either the commit or update hooks because a rollback can happen