SQLite4  Check-in [4693eb7bcc]

static void fts5TokenizerCreate(
  Parse *pParse, 
  Fts5Index *pFts, 
  Fts5Tokenizer **ppTokenizer,
  sqlite4_tokenizer **pp
){
  Fts5Tokenizer *pTok;
  char *zTok;                     /* Tokenizer name */
  const char **azArg;             /* Tokenizer arguments */
  int nArg;                       /* Number of elements in azArg */

  if( pFts->nTokenizer ){
    zTok = pFts->azTokenizer[0];
    azArg = (const char **)&pFts->azTokenizer[1];
    nArg = pFts->nTokenizer-1;
  }else{
    zTok = "simple";
    azArg = 0;
    nArg = 0;
  }
 
  *ppTokenizer = pTok = fts5FindTokenizer(pParse->db, zTok);
  if( !pTok ){
    sqlite4ErrorMsg(pParse, "no such tokenizer: \"%s\"", zTok);
  }else{

    int rc = pTok->xCreate(pTok->pCtx, azArg, nArg, pp);
    if( rc!=SQLITE4_OK ){
      assert( *pp==0 );
      sqlite4ErrorMsg(pParse, "error creating tokenizer");
    }
  }
}

}


/*
** Context structure passed to tokenizer callback when tokenizing a document.
**
** The hash table maps between tokens and TokenizeTerm structures.
**
** TokenizeTerm structures are allocated using sqlite4DbMalloc(). Immediately
** following the structure in memory is the token itself (TokenizeTerm.nToken
** bytes of data). Following this is the list of token instances in the same
** format as it is stored in the database. 
**
** All of the above is a single allocation, size TokenizeTerm.nAlloc bytes.
** If the initial allocation is too small, it is extended using
** sqlite4DbRealloc().
*/
typedef struct TokenizeCtx TokenizeCtx;
typedef struct TokenizeTerm TokenizeTerm;
struct TokenizeCtx {
  int rc;
  int iCol;
  sqlite4 *db;
  int nMax;
  Hash hash;
};
struct TokenizeTerm {
  int iWeight;                    /* Weight of previous entry */
  int iCol;                       /* Column containing previous entry */
  int iOff;                       /* Token offset of previous entry */
  int nToken;                     /* Size of token in bytes */
  int nData;                      /* Bytes of data in value */
  int nAlloc;                     /* Bytes of data allocated */
};

TokenizeTerm *fts5TokenizeAppendInt(
  TokenizeCtx *p, 
  TokenizeTerm *pTerm, 
  int iVal
){
  unsigned char *a;
  int nSpace = pTerm->nAlloc-pTerm->nData-pTerm->nToken-sizeof(TokenizeTerm);

  if( nSpace < 5 ){
    int nAlloc = (pTerm->nAlloc<256) ? 256 : pTerm->nAlloc * 2;
    pTerm = sqlite4DbReallocOrFree(p->db, pTerm, nAlloc);
    if( !pTerm ) return 0;
    pTerm->nAlloc = sqlite4DbMallocSize(p->db, pTerm);
  }

  a = &(((unsigned char *)&pTerm[1])[pTerm->nToken+pTerm->nData]);
  pTerm->nData += putVarint32(a, iVal);
  return pTerm;
}


static int fts5TokenizeCb(
  void *pCtx, 
  int iWeight, 
  int iOff,
  const char *zToken, 
  int nToken, 
  int iSrc, 
  int nSrc
){
  TokenizeCtx *p = (TokenizeCtx *)pCtx;
  TokenizeTerm *pTerm = 0;
  TokenizeTerm *pOrig = 0;

  if( nToken>p->nMax ) p->nMax = nToken;

  pTerm = (TokenizeTerm *)sqlite4HashFind(&p->hash, zToken, nToken);
  if( pTerm==0 ){
    /* Size the initial allocation so that it fits in the lookaside buffer */
    int nAlloc = sizeof(TokenizeTerm) + nToken + 32;

    pTerm = sqlite4DbMallocZero(p->db, nAlloc);
    if( pTerm ){
      void *pFree;
      pTerm->nAlloc = sqlite4DbMallocSize(p->db, pTerm);
      pTerm->nToken = nToken;
      memcpy(&pTerm[1], zToken, nToken);
      pFree = sqlite4HashInsert(&p->hash, (char *)&pTerm[1], nToken, pTerm);
      if( pFree ){
        sqlite4DbFree(p->db, pFree);
        pTerm = 0;
      }
      if( pTerm==0 ) goto tokenize_cb_out;
    }
  }

  pTerm = fts5TokenizeAppendInt(p, pTerm, (iOff-pTerm->iOff) << 1);
  if( !pTerm ) goto tokenize_cb_out;
  pTerm->iOff = iOff;

tokenize_cb_out:
  if( pTerm!=pOrig ){
    sqlite4HashInsert(&p->hash, (char *)&pTerm[1], nToken, pTerm);
  }
  if( !pTerm ){
    p->rc = SQLITE4_NOMEM;
    return 1;
  }

  return 0;
}

/*
** Update an fts index.
*/
int sqlite4Fts5Update(
  sqlite4 *db,                    /* Database handle */
  Fts5Info *pInfo,                /* Description of fts index to update */
  Mem *pKey,                      /* Primary key blob */
  Mem *aArg,                      /* Array of arguments (see above) */
  int bDel,                       /* True for a delete, false for insert */
  char **pzErr                    /* OUT: Error message */
){
  int i;
  int rc = SQLITE4_OK;
  KVStore *pStore;
  TokenizeCtx sCtx;
  u8 *aKey = 0;
  int nKey = 0;
  int nTnum = 0;
  u32 dummy = 0;

  const u8 *pPK;
  int nPK;
  HashElem *pElem;

  pStore = db->aDb[pInfo->iDb].pKV;
  sCtx.rc = SQLITE4_OK;
  sCtx.db = db;
  sCtx.nMax = 0;
  sqlite4HashInit(db->pEnv, &sCtx.hash);

  pPK = (const u8 *)sqlite4_value_blob(pKey);
  nPK = sqlite4_value_bytes(pKey);
  
  nTnum = getVarint32(pPK, dummy);
  nPK -= nTnum;
  pPK += nTnum;

  for(i=0; rc==SQLITE4_OK && i<pInfo->nCol; i++){
    sqlite4_value *pArg = (sqlite4_value *)(&aArg[i]);
    if( pArg->flags & MEM_Str ){
      const char *zText;
      int nText;

      zText = (const char *)sqlite4_value_text(pArg);
      nText = sqlite4_value_bytes(pArg);
      sCtx.iCol = i;
      rc = pInfo->pTokenizer->xTokenize(

  for(pElem=sqliteHashFirst(&sCtx.hash); pElem; pElem=sqliteHashNext(pElem)){
    TokenizeTerm *pTerm = (TokenizeTerm *)sqliteHashData(pElem);
    if( rc==SQLITE4_OK ){
      int nToken = sqliteHashKeysize(pElem);
      char *zToken = (char *)sqliteHashKey(pElem);

      nKey = putVarint32(aKey, pInfo->iRoot);
      aKey[nKey++] = 0x24;
      memcpy(&aKey[nKey], zToken, nToken);
      nKey += nToken;
      aKey[nKey++] = 0x00;
      memcpy(&aKey[nKey], pPK, nPK);
      nKey += nPK;

      if( bDel ){
        /* delete key aKey/nKey... */
        assert( 0 );
      }else{
        const KVByteArray *aData = (const KVByteArray *)&pTerm[1];
        aData += pTerm->nToken;
        rc = sqlite4KVStoreReplace(pStore, aKey, nKey, aData, pTerm->nData);
      }
    }
    sqlite4DbFree(db, pTerm);
  }
  

  sqlite4DbFree(db, aKey);
  sqlite4HashClear(&sCtx.hash);
  return rc;
}

static Fts5Info *fts5InfoCreate(Parse *pParse, Index *pIdx){
  sqlite4 *db = pParse->db;
  Fts5Info *pInfo;                /* p4 argument for FtsUpdate opcode */

  pInfo = sqlite4DbMallocZero(db, sizeof(Fts5Info));
  if( pInfo ){
    pInfo->iDb = sqlite4SchemaToIndex(db, pIdx->pSchema);
    pInfo->iRoot = pIdx->tnum;
    pInfo->nCol = pIdx->pTable->nCol;
    fts5TokenizerCreate(pParse, pIdx->pFts, &pInfo->pTokenizer, &pInfo->p);

    if( pInfo->p==0 ){
      assert( pParse->nErr );
      sqlite4DbFree(db, pInfo);
      pInfo = 0;
    }
  }

  return pInfo;
}


void sqlite4Fts5CodeUpdate(
  Parse *pParse, 
  Index *pIdx, 
  int iRegPk, 
  int iRegData
){
  Vdbe *v;
  Fts5Info *pInfo;                /* p4 argument for FtsUpdate opcode */

  if( 0==(pInfo = fts5InfoCreate(pParse, pIdx)) ) return;

  v = sqlite4GetVdbe(pParse);
  sqlite4VdbeAddOp3(v, OP_FtsUpdate, iRegPk, 0, iRegData);
  sqlite4VdbeChangeP4(v, -1, (const char *)pInfo, P4_FTS5INFO);
}

void sqlite4Fts5FreeInfo(sqlite4 *db, Fts5Info *p){
  if( db->pnBytesFreed==0 ){
    if( p->p ) p->pTokenizer->xDestroy(p->p);
    sqlite4DbFree(db, p);
  }
}

void sqlite4Fts5CodeCksum(
  Parse *pParse, 
  Index *pIdx, 
  int iCksum, 
  int iReg,
  int bIdx                        /* True for fts index, false for table */
){
  Vdbe *v;
  Fts5Info *pInfo;                /* p4 argument for FtsCksum opcode */

  if( 0==(pInfo = fts5InfoCreate(pParse, pIdx)) ) return;

  v = sqlite4GetVdbe(pParse);
  sqlite4VdbeAddOp3(v, OP_FtsCksum, iCksum, 0, iReg);
  sqlite4VdbeChangeP4(v, -1, (const char *)pInfo, P4_FTS5INFO);
  sqlite4VdbeChangeP5(v, bIdx);
}

/*
** Calculate a 64-bit checksum for a term instance. The index checksum is
** the XOR of the checksum for each term instance in the table. A term
** instance checksum is calculated based on:
**
**   * the term itself,
**   * the pk of the row the instance appears in,
**   * the weight assigned to the instance,
**   * the column number, and
**   * the term offset.
*/
static i64 fts5TermInstanceCksum(
  const u8 *aTerm, int nTerm,
  const u8 *aPk, int nPk,
  int iWeight,
  int iCol,
  int iOff
){
  int i;
  i64 cksum = 0;

  /* Add the term to the checksum */
  for(i=0; i<nTerm; i++){
    cksum += (cksum << 3) + aTerm[i];
  }

  /* Add the primary key blob to the checksum */
  for(i=0; i<nPk; i++){
    cksum += (cksum << 3) + aPk[i];
  }

  /* Add the weight, column number and offset (in that order) to the checksum */
  cksum += (cksum << 3) + iWeight;
  cksum += (cksum << 3) + iCol;
  cksum += (cksum << 3) + iOff;

  return cksum;
}


int sqlite4Fts5EntryCksum(
  sqlite4 *db,                    /* Database handle */
  Fts5Info *p,                    /* Index description */
  Mem *pKey,                      /* Database key */
  Mem *pVal,                      /* Database value */
  i64 *piCksum                    /* OUT: Checksum value */
){
  i64 cksum = 0;
  u8 const *aKey; int nKey;       /* Key blob */
  u8 const *aVal; int nVal;       /* List of token instances */
  u8 const *aToken; int nToken;   /* Token for this entry */
  u8 const *aPk; int nPk;         /* Entry primary key blob */
  int nTnum;
  u32 tnum;

  int iOff = 0;
  int iCol = 0;
  int iWeight = 0;
  int i = 0;

  aKey = (const u8 *)sqlite4_value_blob(pKey);
  nKey = sqlite4_value_bytes(pKey);
  aVal = (const u8 *)sqlite4_value_blob(pVal);
  nVal = sqlite4_value_bytes(pVal);

  /* Find the token and primary key blobs for this entry. */
  nTnum = getVarint32(aKey, tnum);
  aToken = &aKey[nTnum+1];
  nToken = sqlite4Strlen30((const char *)aToken);
  aPk = &aToken[nToken+1];
  nPk = (&aKey[nKey] - aPk);

  while( i<nVal ){
    u32 iVal;
    i += getVarint32(&aVal[i], iVal);

    if( (iVal & 0x03)==0x01 ){
      iCol = (iVal>>2);
      iOff = 0;
    }
    else if( (iVal & 0x03)==0x03 ){
      iWeight = (iVal>>2);
    }
    else{
      i64 v;
      iOff += (iVal>>1);
      v = fts5TermInstanceCksum(aPk, nPk, aToken, nToken, iWeight, iCol, iOff);
      cksum = cksum ^ v;
    }
  }

  *piCksum = cksum;
  return SQLITE4_OK;
}

typedef struct CksumCtx CksumCtx;
struct CksumCtx {
  const u8 *pPK;
  int nPK;
  int iCol;
  i64 cksum;
};

static int fts5CksumCb(
  void *pCtx, 
  int iWeight, 
  int iOff,
  const char *zToken, 
  int nToken, 
  int iSrc, 
  int nSrc
){
  CksumCtx *p = (CksumCtx *)pCtx;
  i64 cksum;

  cksum = fts5TermInstanceCksum(p->pPK, p->nPK, 
      (const u8 *)zToken, nToken, iWeight, p->iCol, iOff
  );

  p->cksum = (p->cksum ^ cksum);
  return 0;
}

int sqlite4Fts5RowCksum(
  sqlite4 *db,                    /* Database handle */
  Fts5Info *pInfo,                /* Index description */
  Mem *pKey,                      /* Primary key blob */
  Mem *aArg,                      /* Array of column values */
  i64 *piCksum                    /* OUT: Checksum value */
){
  int i;
  int rc = SQLITE4_OK;
  CksumCtx sCtx;
  int nTnum = 0;
  u32 dummy = 0;

  sCtx.cksum = 0;

  sCtx.pPK = (const u8 *)sqlite4_value_blob(pKey);
  sCtx.nPK = sqlite4_value_bytes(pKey);
  nTnum = getVarint32(sCtx.pPK, dummy);
  sCtx.nPK -= nTnum;
  sCtx.pPK += nTnum;

  for(i=0; rc==SQLITE4_OK && i<pInfo->nCol; i++){
    sqlite4_value *pArg = (sqlite4_value *)(&aArg[i]);
    if( pArg->flags & MEM_Str ){
      const char *zText;
      int nText;

      zText = (const char *)sqlite4_value_text(pArg);
      nText = sqlite4_value_bytes(pArg);
      sCtx.iCol = i;
      rc = pInfo->pTokenizer->xTokenize(
          &sCtx, pInfo->p, zText, nText, fts5CksumCb
      );
    }
  }

  *piCksum = sCtx.cksum;
  return rc;
}


/**************************************************************************
***************************************************************************
** Below this point is test code.
*/
#ifdef SQLITE4_TEST
static int fts5PrintExprNode(sqlite4 *, const char **, Fts5ExprNode *, char **);

}

static int fts5SimpleTokenize(
  void *pCtx, 
  sqlite4_tokenizer *p,
  const char *zDoc,
  int nDoc,
  int(*x)(void*, int, int, const char*, int, int, int)
){
  sqlite4_env *pEnv = (sqlite4_env *)p;
  char *aBuf;
  int nBuf;
  int iBuf;
  int i;
  int brk = 0;
  int iOff = 0;

  nBuf = 128;
  aBuf = (char *)sqlite4_malloc(pEnv, nBuf);
  if( !aBuf ) return SQLITE4_NOMEM;

  iBuf = 0;
  for(i=0; brk==0 && i<nDoc; i++){
    if( sqlite4Isalnum(zDoc[i]) ){
      aBuf[iBuf++] = fts5Tolower(zDoc[i]);
    }else if( iBuf>0 ){
      brk = x(pCtx, 0, iOff++, aBuf, iBuf, i-iBuf, iBuf);
      iBuf = 0;
    }
  }
  if( iBuf>0 ) x(pCtx, 0, iOff++, aBuf, iBuf, i-iBuf, iBuf);

  sqlite4_free(pEnv, aBuf);
  return SQLITE4_OK;
}

int sqlite4InitFts5Func(sqlite4 *db){
  int rc;

  sqlite4VdbeAddOp3(v, OP_MakeRecord, regContent, pTab->nCol, regRec);
  sqlite4TableAffinityStr(v, pTab);
  sqlite4ExprCacheAffinityChange(pParse, regContent, pTab->nCol);

  /* Write the entry to each index. */
  for(i=0, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){
    assert( pIdx->eIndexType!=SQLITE4_INDEX_PRIMARYKEY || aRegIdx[i] );
    if( pIdx->eIndexType==SQLITE4_INDEX_FTS5 ){
      int iPK;
      sqlite4FindPrimaryKey(pTab, &iPK);
      sqlite4Fts5CodeUpdate(pParse, pIdx, aRegIdx[iPK], regContent);
    }
    else if( aRegIdx[i] ){
      int regData = 0;
      int flags = 0;
      if( pIdx->eIndexType==SQLITE4_INDEX_PRIMARYKEY ){
        regData = regRec;
        flags = pik_flags;
      }
      sqlite4VdbeAddOp3(v, OP_IdxInsert, baseCur+i, regData, aRegIdx[i]);

  if( sqlite4StrICmp(zLeft, "lsm_merge")==0 ){
    int nPage = zRight ? sqlite4Atoi(zRight) : 1000;
    sqlite4_kvstore_control(db, zDb, SQLITE4_KVCTRL_LSM_MERGE, &nPage);
    returnSingleInt(pParse, "nWrite", (sqlite4_int64)nPage);
  }else

  /*
  **   PRAGMA fts_check(<index>)
  */
  if( sqlite4StrICmp(zLeft, "fts_check")==0 && zRight ){
    int iCksum1;
    int iCksum2;
    Index *pIdx;
    Table *pTab;
    Vdbe *v = sqlite4GetVdbe(pParse);
    if( v==0 || sqlite4ReadSchema(pParse) ) goto pragma_out;

    iCksum1 = ++pParse->nMem;
    iCksum2 = ++pParse->nMem;
    sqlite4VdbeAddOp2(v, OP_Integer, 0, iCksum1);
    sqlite4VdbeAddOp2(v, OP_Integer, 0, iCksum2);

    pIdx = sqlite4FindIndex(db, zRight, zDb);
    if( pIdx && pIdx->eIndexType==SQLITE4_INDEX_FTS5 ){
      int iTab = pParse->nTab++;
      int iAddr;
      int iReg;
      int i;

      pTab = pIdx->pTable;
      sqlite4OpenPrimaryKey(pParse, iTab, iDb, pTab, OP_OpenRead);
      iAddr = sqlite4VdbeAddOp2(v, OP_Rewind, iTab, 0);

      iReg = pParse->nMem+1;
      pParse->nMem += (1 + pTab->nCol);

      sqlite4VdbeAddOp2(v, OP_RowKey, iTab, iReg);
      for(i=0; i<pTab->nCol; i++){
        sqlite4VdbeAddOp3(v, OP_Column, iTab, i, iReg+1+i);
      }
      sqlite4Fts5CodeCksum(pParse, pIdx, iCksum1, iReg, 0);

      sqlite4VdbeAddOp2(v, OP_Next, iTab, iAddr+1);
      sqlite4VdbeJumpHere(v, iAddr);
      sqlite4VdbeAddOp1(v, OP_Close, iTab);

      sqlite4VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb);
      iAddr = sqlite4VdbeAddOp2(v, OP_Rewind, iTab, 0);

      iReg = pParse->nMem+1;
      pParse->nMem += 2;
      sqlite4VdbeAddOp2(v, OP_RowKey, iTab, iReg);
      sqlite4VdbeAddOp2(v, OP_RowData, iTab, iReg+1);
      sqlite4Fts5CodeCksum(pParse, pIdx, iCksum2, iReg, 1);

      sqlite4VdbeAddOp2(v, OP_Next, iTab, iAddr+1);
      sqlite4VdbeJumpHere(v, iAddr);
      sqlite4VdbeAddOp1(v, OP_Close, iTab);

      iReg = ++pParse->nMem;
      sqlite4VdbeAddOp4(v, OP_String8, 0, iReg, 0, "ok", 0);
      iAddr = sqlite4VdbeAddOp3(v, OP_Eq, iCksum1, 0, iCksum2);
      sqlite4VdbeAddOp4(v, OP_String8, 0, iReg, 0, "error - cksum mismatch", 0);
      sqlite4VdbeJumpHere(v, iAddr);
      sqlite4VdbeAddOp2(v, OP_ResultRow, iReg, 1);

      sqlite4VdbeSetNumCols(v, 1);
    }
  }



#ifndef SQLITE4_OMIT_SCHEMA_PRAGMAS
  /*
  **   PRAGMA table_info(<table>)
  **

void sqlite4ShutdownFts5(sqlite4 *db);
void sqlite4CreateUsingIndex(Parse*, CreateIndex*, ExprList*, Token*, Token*);

int sqlite4Fts5IndexSz(void);
void sqlite4Fts5IndexInit(Parse *, Index *, ExprList *);
void sqlite4Fts5IndexFree(sqlite4 *, Index *);

int sqlite4Fts5Update(sqlite4 *, Fts5Info *, Mem *pPk, Mem *aArg, int, char **);
void sqlite4Fts5FreeInfo(sqlite4 *db, Fts5Info *);
void sqlite4Fts5CodeUpdate(Parse *, Index *pIdx, int iRegPk, int iRegData);
void sqlite4Fts5CodeCksum(Parse *, Index *, int, int, int);

#endif /* _SQLITEINT_H_ */

    sqlite4DebugPrintf("SQL-trace: %s\n", zTrace);
  }
#endif /* SQLITE4_DEBUG */
  break;
}
#endif

/* Opcode: FtsUpdate P1 * P3 P4 P5
**
** This opcode is used to write to an FTS index. P4 points to an Fts5Info 
** object describing the index.
**
** If argument P5 is non-zero, then entries are removed from the FTS index.
** If it is zero, then entries are inserted. In other words, when a row
** is deleted from a table with an FTS index, this opcode is invoked with
** P5==1. When a row is inserted, it is invoked with P5==0. If an existing
** row is updated, this opcode is invoked twice - once with P5==1 and then
** again with P5==0.
**
** Register P1 contains the PK (a blob in key format) of the affected row.
** P3 is the first in an array of N registers, where N is the number of
** columns in the indexed table. Each register contains the value for the

** corresponding table column.
*/
case OP_FtsUpdate: {
  Fts5Info *pInfo;                /* Description of fts5 index to update */
  Mem *pKey;                      /* Primary key of indexed row */
  Mem *aArg;                      /* Pointer to array of N arguments */

  assert( pOp->p4type==P4_FTS5INFO );
  pInfo = pOp->p4.pFtsInfo;
  aArg = &aMem[pOp->p3];
  pKey = &aMem[pOp->p1];

  rc = sqlite4Fts5Update(db, pInfo, pKey, aArg, pOp->p5, &p->zErrMsg);
  break;
}

/*
** Opcode: FtsCksum P1 * P3 P4 P5
*/
case OP_FtsCksum: {
  Fts5Info *pInfo;                /* Description of fts5 index to update */
  Mem *pKey;                      /* Primary key of row */
  Mem *aArg;                      /* Pointer to array of N values */
  i64 cksum;

  assert( pOp->p4type==P4_FTS5INFO );
  pInfo = pOp->p4.pFtsInfo;

  pOut = &aMem[pOp->p1];
  pKey = &aMem[pOp->p3];
  aArg = &aMem[pOp->p3+1];
  cksum = 0;

  if( pOp->p5 ){
    sqlite4Fts5EntryCksum(db, pInfo, pKey, aArg, &cksum);
    pOut->u.i = pOut->u.i ^ cksum;
  }else{
    sqlite4Fts5RowCksum(db, pInfo, pKey, aArg, &cksum);
    pOut->u.i = pOut->u.i ^ cksum;
  }
  break;
}


/* Opcode: Noop * * * * *
**
** Do nothing.  This instruction is often useful as a jump

          sqlite4DbFree(db, p);
        }
        break;
      }
      case P4_VTAB : {
        if( db->pnBytesFreed==0 ) sqlite4VtabUnlock((VTable *)p4);
        break;
      }
      case P4_FTS5INFO : {
        sqlite4Fts5FreeInfo(db, (Fts5Info *)p4);
        break;
      }
    }
  }
}

/*
** Free the space allocated for aOp and any p4 values allocated for the