SQLite4  Check-in [fb07003744]

  char *zErr;                     /* Error message (or NULL) */

  const char *zExpr;              /* Pointer to expression text (nul-term) */
  int iExpr;                      /* Current offset in zExpr */
  Fts5ParserToken next;           /* Next token */

  char **azCol;                   /* Column names of indexed table */
  int nCol;                       /* Size of azCol[] in bytes */
  int iRoot;                      /* Root page number of FTS index */

  /* Space for dequoted copies of strings */
  char *aSpace;
  int iSpace;
  int nSpace;                     /* Total size of aSpace in bytes */
};

struct Fts5Token {
  /* TODO: The first three members are redundant, since they can be encoded
  ** in the aPrefix[]/nPrefix key.  */
  int bPrefix;                    /* True for a prefix search */
  int n;                          /* Size of z[] in bytes */
  char *z;                        /* Token value */

  KVCursor *pCsr;                 /* Cursor to iterate thru entries for token */
  KVByteArray *aPrefix;           /* KV prefix to iterate through */
  KVSize nPrefix;                 /* Size of aPrefix in bytes */
};

struct Fts5Str {
  Fts5Token *aToken;
  int nToken;
};

};


/*
** FTS5 specific cursor data.
*/
struct Fts5Cursor {
  sqlite4 *db;
  Fts5Info *pInfo;
  Fts5Expr *pExpr;                /* MATCH expression for this cursor */

  KVByteArray *aKey;              /* Buffer for primary key */
  int nKeyAlloc;                  /* Bytes allocated at aKey[] */
};

/*
** Return true if argument c is one of the special non-whitespace 
** characters that ends an unquoted expression token. 
*/
static int fts5IsSpecial(char c){

  void *pCtx, 
  int iWeight, 
  int iOff, 
  const char *z, int n, 
  int iSrc, int nSrc
){
  struct AppendTokensCtx *p = (struct AppendTokensCtx *)pCtx;
  Fts5Parser *pParse = p->pParse;
  Fts5Token *pToken;
  char *zSpace;
  int nUsed;





  pToken = &p->pStr->aToken[p->pStr->nToken];

  zSpace = &pParse->aSpace[pParse->iSpace];
  nUsed = putVarint32((u8 *)zSpace, pParse->iRoot);
  zSpace[nUsed++] = 0x24;
  pToken->bPrefix = 0;
  pToken->z = &zSpace[nUsed];
  pToken->n = n;
  memcpy(pToken->z, z, n);
  pToken->z[n] = '\0';

  nUsed += (n+1);
  pToken->aPrefix = (u8 *)zSpace;
  pToken->nPrefix = nUsed;
  pToken->pCsr = 0;
  pParse->iSpace += nUsed;
  p->pStr->nToken++;

  assert( pParse->iSpace<=pParse->nSpace );
  return 0;
}

static int fts5AppendTokens( 
  Fts5Parser *pParse,
  Fts5Str *pStr, const char *zPrim,

  int nPrim
){
  struct AppendTokensCtx ctx;
  ctx.pParse = pParse;
  ctx.pStr = pStr;

  return pParse->pTokenizer->xTokenize(
      (void *)&ctx, pParse->p , zPrim, nPrim, fts5AppendTokensCb
  );
}

/*
** Append a new token to the current phrase.
*/
static int fts5PhraseAppend(
  Fts5Parser *pParse,
  Fts5Phrase *pPhrase,
  const char *zPrim,
  int nPrim
){
  Fts5Tokenizer *pTok = pParse->pTokenizer;

  return rc;
}

static void fts5PhraseFree(sqlite4 *db, Fts5Phrase *p){
  if( p ){
    int i;
    for(i=0; i<p->nStr; i++){
      int iTok;
      for(iTok=0; iTok<p->aStr[i].nToken; iTok++){
        sqlite4KVCursorClose(p->aStr[i].aToken[iTok].pCsr);
      }
      sqlite4DbFree(db, p->aStr[i].aToken);
    }
    sqlite4DbFree(db, p->aiNear);
    sqlite4DbFree(db, p->aStr);
    sqlite4DbFree(db, p);
  }
}

  return SQLITE4_OK;
}

static int fts5ParseExpression(
  sqlite4 *db,                    /* Database handle */
  Fts5Tokenizer *pTokenizer,      /* Tokenizer module */
  sqlite4_tokenizer *p,           /* Tokenizer instance */
  int iRoot,                      /* Root page number of FTS index */
  char **azCol,                   /* Array of column names (nul-term'd) */
  int nCol,                       /* Size of array azCol[] */
  const char *zExpr,              /* FTS expression text */
  Fts5Expr **ppExpr,              /* OUT: Expression object */
  char **pzErr                    /* OUT: Error message */
){
  int rc = SQLITE4_OK;
  Fts5Parser sParse;

  memset(&sParse, 0, sizeof(Fts5Parser));
  sParse.zExpr = zExpr;
  sParse.azCol = azCol;
  sParse.nCol = nCol;
  sParse.pTokenizer = pTokenizer;
  sParse.p = p;
  sParse.db = db;
  sParse.iRoot = iRoot;

  pExpr = sqlite4DbMallocZero(db, sizeof(Fts5Expr) + nExpr*4);
  if( !pExpr ) return SQLITE4_NOMEM;
  sParse.aSpace = (char *)&pExpr[1];
  sParse.nSpace = nExpr*4;

  rc = fts5GrowExprHier(db, &nHierAlloc, &aHier, 1);
  if( rc==SQLITE4_OK ){
    aHier[0].ppNode = &pExpr->pRoot;
    aHier[0].nOpen = 0;
    nHier = 1;
  }

      pInfo = 0;
    }
    else if( bCol ){
      int i;
      char *p;
      int nCol = pIdx->pTable->nCol;

      pInfo->azCol = (char **)(&pInfo[1]);
      p = (char *)(&pInfo->azCol[nCol]);
      for(i=0; i<nCol; i++){
        const char *zCol = pIdx->pTable->aCol[i].zName;
        int n = sqlite4Strlen30(zCol) + 1;
        pInfo->azCol[i] = p;
        memcpy(p, zCol, n);
        p += n;

      );
    }
  }

  *piCksum = sCtx.cksum;
  return rc;
}

static int fts5OpenExprCursors(sqlite4 *db, Fts5Info *pInfo, Fts5ExprNode *p){
  int rc = SQLITE4_OK;
  if( p ){
    if( p->eType==TOKEN_PRIMITIVE ){
      KVStore *pStore = db->aDb[pInfo->iDb].pKV;
      Fts5Phrase *pPhrase = p->pPhrase;
      int iStr;

      for(iStr=0; rc==SQLITE4_OK && iStr<pPhrase->nStr; iStr++){
        Fts5Str *pStr = &pPhrase->aStr[iStr];
        int i;
        for(i=0; rc==SQLITE4_OK && i<pStr->nToken; i++){
          Fts5Token *pToken = &pStr->aToken[i];
          rc = sqlite4KVStoreOpenCursor(pStore, &pToken->pCsr);
          if( rc==SQLITE4_OK ){
            rc = sqlite4KVCursorSeek(
                pToken->pCsr, pToken->aPrefix, pToken->nPrefix, 1
            );
            if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK;
          }
        }
      }
    }
    if( rc==SQLITE4_OK ) rc = fts5OpenExprCursors(db, pInfo, p->pLeft);
    if( rc==SQLITE4_OK ) rc = fts5OpenExprCursors(db, pInfo, p->pRight);
  }

  return rc;
}

/*
** Open a cursor for each token in the expression.
*/
static int fts5OpenCursors(sqlite4 *db, Fts5Info *pInfo, Fts5Cursor *pCsr){
  return fts5OpenExprCursors(db, pInfo, pCsr->pExpr->pRoot);
}

void sqlite4Fts5Close(sqlite4 *db, Fts5Cursor *pCsr){
  if( pCsr ){
    fts5ExpressionFree(db, pCsr->pExpr);
    sqlite4DbFree(db, pCsr);
  }
}

  int rc = SQLITE4_OK;
  Fts5Cursor *pCsr;

  pCsr = sqlite4DbMallocZero(db, sizeof(Fts5Cursor));
  if( !pCsr ){
    rc = SQLITE4_NOMEM;
  }else{
    pCsr->pInfo = pInfo;
    pCsr->db = db;
    rc = fts5ParseExpression(db, pInfo->pTokenizer, pInfo->p, 
        pInfo->iRoot, pInfo->azCol, pInfo->nCol, zMatch, &pCsr->pExpr, pzErr
    );
  }

  if( rc==SQLITE4_OK ){
    /* Open a KV cursor for each term in the expression. */
    rc = fts5OpenCursors(db, pInfo, pCsr);
  }
  if( rc!=SQLITE4_OK ){
    sqlite4Fts5Close(db, pCsr);
    pCsr = 0;
  }
  *ppCsr = pCsr;
  return rc;
}

int sqlite4Fts5Next(Fts5Cursor *pCsr){
  Fts5Token *pToken;
  KVCursor *pKVCsr;
  int rc;

  assert( pCsr->pExpr->pRoot->eType==TOKEN_PRIMITIVE );
  pToken = &pCsr->pExpr->pRoot->pPhrase->aStr[0].aToken[0];

  rc = sqlite4KVCursorNext(pToken->pCsr);
  if( rc==SQLITE4_OK ){
    const KVByteArray *aKey;
    KVSize nKey;
    rc = sqlite4KVCursorKey(pToken->pCsr, &aKey, &nKey);
    if( rc==SQLITE4_OK 
     && (nKey<pToken->nPrefix || memcmp(pToken->aPrefix, aKey, pToken->nPrefix))
    ){
      rc = SQLITE4_NOTFOUND;
    }
  }
  return rc;
}

/*
** Return true if the cursor passed as the second argument currently points
** to a valid entry, or false otherwise.
*/
int sqlite4Fts5Valid(Fts5Cursor *pCsr){
  const KVByteArray *aKey;
  KVSize nKey;
  KVCursor *pKVCsr;
  int rc;

  assert( pCsr->pExpr->pRoot->eType==TOKEN_PRIMITIVE );
  pKVCsr = pCsr->pExpr->pRoot->pPhrase->aStr[0].aToken[0].pCsr;

  rc = sqlite4KVCursorKey(pKVCsr, &aKey, &nKey);
  return (rc==SQLITE4_OK);
}

int sqlite4Fts5Pk(
  Fts5Cursor *pCsr, 
  int iTbl, 
  KVByteArray **paKey, 
  KVSize *pnKey
){
  const KVByteArray *aKey;
  KVSize nKey;
  KVCursor *pKVCsr;
  int rc;
  int i;
  int i2;
  int nReq;

  assert( pCsr->pExpr->pRoot->eType==TOKEN_PRIMITIVE );

  pKVCsr = pCsr->pExpr->pRoot->pPhrase->aStr[0].aToken[0].pCsr;
  rc = sqlite4KVCursorKey(pKVCsr, &aKey, &nKey);
  if( rc!=SQLITE4_OK ) return rc;

  i = sqlite4VarintLen(pCsr->pInfo->iRoot);
  while( aKey[i] ) i++;
  i++;

  nReq = sqlite4VarintLen(iTbl) + (nKey-i);
  if( nReq>pCsr->nKeyAlloc ){
    pCsr->aKey = sqlite4DbReallocOrFree(pCsr->db, pCsr->aKey, nReq*2);
    if( !pCsr->aKey ) return SQLITE4_NOMEM;
    pCsr->nKeyAlloc = nReq*2;
  }

  i2 = putVarint32(pCsr->aKey, iTbl);
  memcpy(&pCsr->aKey[i2], &aKey[i], nKey-i);

  *paKey = pCsr->aKey;
  *pnKey = nReq;
  return SQLITE4_OK;
}

/**************************************************************************
***************************************************************************
** Below this point is test code.
*/
#ifdef SQLITE4_TEST

    rc = pTok->xCreate(pTok->pCtx, 0, 0, &p);
    if( rc!=SQLITE4_OK ){
      zErr = sqlite4MPrintf(db, "error creating tokenizer: %d", rc);
      goto fts5_parse_expr_out;
    }
  }

  rc = fts5ParseExpression(
      db, pTok, p, 0, (char **)azCol, nCol, zExpr, &pExpr, &zErr);
  if( rc!=SQLITE4_OK ){
    if( zErr==0 ){
      zErr = sqlite4MPrintf(db, "error parsing expression: %d", rc);
    }
    goto fts5_parse_expr_out;
  }

*/
static int kvlsmKey(
  KVCursor *pKVCursor,         /* The cursor whose key is desired */
  const KVByteArray **paKey,   /* Make this point to the key */
  KVSize *pN                   /* Make this point to the size of the key */
){
  KVLsmCsr *pCsr = (KVLsmCsr *)pKVCursor;

  if( 0==lsm_csr_valid(pCsr->pCsr) ) return SQLITE4_DONE;
  return lsm_csr_key(pCsr->pCsr, (const void **)paKey, (int *)pN);
}

/*
** Return the data of the node the cursor is pointing to.
*/
static int kvlsmData(

** An instance of this structure is used as the p4 argument to some fts5
** related vdbe opcodes.
*/
struct Fts5Info {
  int iDb;                        /* Database containing this index */
  int iRoot;                      /* Root page number of index */
  int nCol;                       /* Number of columns in indexed table */
  char **azCol;                   /* Column names for table */
  Fts5Tokenizer *pTokenizer;      /* Tokenizer module */
  sqlite4_tokenizer *p;           /* Tokenizer instance */
};

int sqlite4WalkExpr(Walker*, Expr*);
int sqlite4WalkExprList(Walker*, ExprList*);
int sqlite4WalkSelect(Walker*, Select*);

void sqlite4Fts5IndexFree(sqlite4 *, Index *);

int sqlite4Fts5Update(sqlite4 *, Fts5Info *, Mem *pPk, Mem *aArg, int, char **);
void sqlite4Fts5FreeInfo(sqlite4 *db, Fts5Info *);
void sqlite4Fts5CodeUpdate(Parse *, Index *pIdx, int iRegPk, int iRegData, int);
void sqlite4Fts5CodeCksum(Parse *, Index *, int, int, int);
void sqlite4Fts5CodeQuery(Parse *, Index *, int, int, int);

int sqlite4Fts5Pk(Fts5Cursor *, int, KVByteArray **, KVSize *);
int sqlite4Fts5Next(Fts5Cursor *pCsr);

#endif /* _SQLITEINT_H_ */

  KVSize nKey;                    /* Size of aKey[] in bytes */
  int nShort;                     /* Size of aKey[] without PK fields */
  KVByteArray *aPkKey;
  KVSize nPkKey;

  pPk = p->apCsr[pOp->p1];
  pIdx = p->apCsr[pOp->p3];

  if( pIdx->pFts ){
    rc = sqlite4Fts5Pk(pIdx->pFts, pPk->iRoot, &aPkKey, &nPkKey);
    if( rc==SQLITE4_OK ){
      rc = sqlite4KVCursorSeek(pPk->pKVCur, aPkKey, nPkKey, 0);
      if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_CORRUPT_BKPT;
      pPk->nullRow = 0;
    }
  }else{
    assert( pIdx->pKeyInfo->nPK>0 );
    assert( pPk->pKeyInfo->nPK==0 );

    rc = sqlite4KVCursorKey(pIdx->pKVCur, &aKey, &nKey);
    if( rc==SQLITE4_OK ){
      nShort = sqlite4VdbeShortKey(aKey, nKey, 
          pIdx->pKeyInfo->nField - pIdx->pKeyInfo->nPK
      );

      nPkKey = sqlite4VarintLen(pPk->iRoot) + nKey - nShort;
      aPkKey = sqlite4DbMallocRaw(db, nPkKey);

      if( aPkKey ){
        putVarint32(aPkKey, pPk->iRoot);
        memcpy(&aPkKey[nPkKey - (nKey-nShort)], &aKey[nShort], nKey-nShort);
        rc = sqlite4KVCursorSeek(pPk->pKVCur, aPkKey, nPkKey, 0);
        if( rc==SQLITE4_NOTFOUND ){
          rc = SQLITE4_CORRUPT_BKPT;
        }
        pPk->nullRow = 0;
        sqlite4DbFree(db, aPkKey);
      }
    }
  }

  break;
}

/* Opcode: SeekGe P1 P2 P3 P4 *

  assert( pOp->p4type==P4_FTS5INFO );
  pInfo = pOp->p4.pFtsInfo;
  pCur = allocateCursor(p, pOp->p1, 0, pInfo->iDb, 0);
  if( pCur ){
    rc = sqlite4Fts5Open(db, pInfo, zMatch, pOp->p5, &pCur->pFts, &p->zErrMsg);
  }

  if( rc==SQLITE4_OK && 0==sqlite4Fts5Valid(pCur->pFts) ){
    pc = pOp->p2-1;
  }
  break;
}

/* Opcode: FtsNext P1 P2 * * *
**
** Advance FTS cursor P1 to the next entry and jump to instruction P2. Or,
** if there is no next entry, set the cursor to point to EOF and fall through
** to the next instruction.
*/
case OP_FtsNext: {
  VdbeCursor *pCsr;

  pCsr = p->apCsr[pOp->p1];
  rc = sqlite4Fts5Next(pCsr->pFts);
  if( rc==SQLITE4_OK ) pc = pOp->p2-1;
  if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK;

  break;
}

/* Opcode: FtsPk P1 P2 * * * 
**
** P1 is an FTS cursor that points to a valid entry (not EOF). Copy the PK 
** blob for the current entry to register P2.

  INSERT INTO t1 VALUES(1, 'o n e');
  INSERT INTO t1 VALUES(2, 't w o');
  INSERT INTO t1 VALUES(3, 't h r e e');
  INSERT INTO t1 VALUES(4, 'f o u r');
}



foreach {tn stmt res} {
  1 {SELECT x FROM t1 WHERE t1 MATCH 't'} {2 3}
} {
  do_execsql_test 2.$tn $stmt $res
}

finish_test