SQLite

#define _FTS5_H

#include "sqlite3.h"

#ifdef __cplusplus
extern "C" {
#endif

#define SQLITE_FTS5_TOKENIZE_SUBTYPE ((unsigned int)'T')

/*************************************************************************
** CUSTOM AUXILIARY FUNCTIONS
**
** Virtual table implementations may overload SQL functions by implementing
** the sqlite3_module.xFindFunction() method.
*/

/**************************************************************************
** Interface to code in fts5_config.c. fts5_config.c contains contains code
** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
*/

typedef struct Fts5Config Fts5Config;

typedef struct Fts5TokenizerInst Fts5TokenizerInst;
struct Fts5TokenizerInst {
  char *zSpec;
  Fts5Tokenizer *pTok;
  fts5_tokenizer *pTokApi;
  Fts5TokenizerInst *pNext;
};

/*
** An instance of the following structure encodes all information that can
** be gleaned from the CREATE VIRTUAL TABLE statement.
**
** And all information loaded from the %_config table.
**
** nAutomerge:

**   are ignored. This value is configured using:
**
**       INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
**
*/
struct Fts5Config {
  sqlite3 *db;                    /* Database handle */
  Fts5Global *pGlobal;            /* Database wide data */
  char *zDb;                      /* Database holding FTS index (e.g. "main") */
  char *zName;                    /* Name of FTS index */
  int nCol;                       /* Number of columns */
  char **azCol;                   /* Column names */
  u8 *abUnindexed;                /* True for unindexed columns */
  int nPrefix;                    /* Number of prefix indexes */
  int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
  int eContent;                   /* An FTS5_CONTENT value */
  int bContentlessDelete;         /* "contentless_delete=" option (dflt==0) */
  char *zContent;                 /* content table */ 
  char *zContentRowid;            /* "content_rowid=" option value */ 
  int bColumnsize;                /* "columnsize=" option value (dflt==1) */
  int bTokendata;                 /* "tokendata=" option value (dflt==0) */
  int eDetail;                    /* FTS5_DETAIL_XXX value */
  char *zContentExprlist;
  Fts5TokenizerInst *pTokList;

  int bLock;                      /* True when table is preparing statement */
  int ePattern;                   /* FTS_PATTERN_XXX constant */

  /* Values loaded from the %_config table */
  int iVersion;                   /* fts5 file format 'version' */
  int iCookie;                    /* Incremented when %_config is modified */
  int pgsz;                       /* Approximate page size used in %_data */

void sqlite3Fts5ConfigFree(Fts5Config*);

int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);

int sqlite3Fts5Tokenize(
  Fts5Config *pConfig,            /* FTS5 Configuration object */
  int flags,                      /* FTS5_TOKENIZE_* flags */
  const char *pText, int nText,   /* Text to tokenize */
  void *pCtx,                     /* Context passed to xToken() */
  int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
);

int sqlite3Fts5ConfigFindTokenizer(
  Fts5Config *pConfig, 
  const char *z, 
  Fts5TokenizerInst **ppOut
);

int sqlite3Fts5SpecTokenize(
  Fts5Config *pConfig,            /* FTS5 Configuration object */
  const char *zSpec,              /* Tokenizer specification */
  int flags,                      /* FTS5_TOKENIZE_* flags */
  const char *pText, int nText,   /* Text to tokenize */
  void *pCtx,                     /* Context passed to xToken() */
  int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
);

void sqlite3Fts5Dequote(char *z);

struct Fts5Table {
  sqlite3_vtab base;              /* Base class used by SQLite core */
  Fts5Config *pConfig;            /* Virtual table configuration */
  Fts5Index *pIndex;              /* Full-text index */
};

int sqlite3Fts5GetTokenizer(



  Fts5Config*,
  const char *zSpec
);

Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);

int sqlite3Fts5FlushToDisk(Fts5Table*);

/*

);

int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
int sqlite3Fts5StorageRebuild(Fts5Storage *p);
int sqlite3Fts5StorageOptimize(Fts5Storage *p);
int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
int sqlite3Fts5StorageReset(Fts5Storage *p);

int sqlite3Fts5UnpackTokenizeBlob(
  Fts5Config *pConfig,
  sqlite3_value *pVal,
  Fts5TokenizerInst **ppTok,
  char **pzText,
  int *pbDel
);

/*
** End of interface to code in fts5_storage.c.
**************************************************************************/


/**************************************************************************

      iVal = aEnum[i].eVal;
    }
  }

  *peVal = iVal;
  return iVal<0 ? SQLITE_ERROR : SQLITE_OK;
}

int sqlite3Fts5ConfigFindTokenizer(
  Fts5Config *pConfig, 
  const char *z, 
  Fts5TokenizerInst **ppOut
){
  Fts5TokenizerInst *pRet = 0;
  int rc = SQLITE_OK;

  assert( pConfig->pzErrmsg );

  /* Search for an existing tokenizer that matches this spec */
  for(pRet=pConfig->pTokList; pRet; pRet=pRet->pNext){
    if( strcmp(pRet->zSpec, z)==0 ) break;
  }

  if( pRet==0 ){
    /* No tokenizer found - create one. */
    rc = sqlite3Fts5GetTokenizer(pConfig, z);
    if( rc==SQLITE_OK ){
      pRet = pConfig->pTokList->pNext;
    }
  }

  if( ppOut ) *ppOut = pRet;
  return rc;
}

/*
** Parse a "special" CREATE VIRTUAL TABLE directive and update
** configuration object pConfig as appropriate.
**
** If successful, object pConfig is updated and SQLITE_OK returned. If
** an error occurs, an SQLite error code is returned and an error message

      bFirst = 0;
    }
    assert( pConfig->nPrefix<=FTS5_MAX_PREFIX_INDEXES );
    return rc;
  }

  if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){







    if( pConfig->pTokList ){
      *pzErr = sqlite3_mprintf("multiple tokenize=... directives");
      rc = SQLITE_ERROR;
    }else{



      assert( pConfig->pzErrmsg==0 );

      pConfig->pzErrmsg = pzErr;













      rc = sqlite3Fts5GetTokenizer(pConfig, zArg);

      pConfig->pzErrmsg = 0;

    }





    return rc;
  }

  if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){
    if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
      *pzErr = sqlite3_mprintf("multiple content=... directives");
      rc = SQLITE_ERROR;

    return rc;
  }

  *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
  return SQLITE_ERROR;
}











/*
** Gobble up the first bareword or quoted word from the input buffer zIn.
** Return a pointer to the character immediately following the last in
** the gobbled word if successful, or a NULL pointer otherwise (failed
** to find close-quote character).
**
** Before returning, set pzOut to point to a new buffer containing a

  int i;
  sqlite3_int64 nByte;

  *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
  if( pRet==0 ) return SQLITE_NOMEM;
  memset(pRet, 0, sizeof(Fts5Config));
  pRet->db = db;
  pRet->pGlobal = pGlobal;
  pRet->iCookie = -1;

  nByte = nArg * (sizeof(char*) + sizeof(u8));
  pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte);
  pRet->abUnindexed = pRet->azCol ? (u8*)&pRet->azCol[nArg] : 0;
  pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
  pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);

    );
    rc = SQLITE_ERROR;
  }

  /* If a tokenizer= option was successfully parsed, the tokenizer has
  ** already been allocated. Otherwise, allocate an instance of the default
  ** tokenizer (unicode61) now.  */
  if( rc==SQLITE_OK && pRet->pTokList==0 ){
    rc = sqlite3Fts5GetTokenizer(pRet, 0);
  }

  /* If no zContent option was specified, fill in the default values. */
  if( rc==SQLITE_OK && pRet->zContent==0 ){
    const char *zTail = 0;
    assert( pRet->eContent==FTS5_CONTENT_NORMAL 
         || pRet->eContent==FTS5_CONTENT_NONE 

  if( rc!=SQLITE_OK ){
    sqlite3Fts5ConfigFree(pRet);
    *ppOut = 0;
  }
  return rc;
}

static void fts5ConfigFreeTokenizers(Fts5Config *pConfig){
  Fts5TokenizerInst *p = pConfig->pTokList;
  while( p ){
    Fts5TokenizerInst *pNext = p->pNext;
    p->pTokApi->xDelete(p->pTok);
    sqlite3_free(p);
    p = pNext;
  }
  pConfig->pTokList = 0;
}

/*
** Free the configuration object passed as the only argument.
*/
void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
  if( pConfig ){
    int i;
    fts5ConfigFreeTokenizers(pConfig);


    sqlite3_free(pConfig->zDb);
    sqlite3_free(pConfig->zName);
    fts5ConfigFreeTokenizers(pConfig);
    for(i=0; i<pConfig->nCol; i++){
      sqlite3_free(pConfig->azCol[i]);
    }
    sqlite3_free(pConfig->azCol);
    sqlite3_free(pConfig->aPrefix);
    sqlite3_free(pConfig->zRank);
    sqlite3_free(pConfig->zRankArgs);

int sqlite3Fts5Tokenize(
  Fts5Config *pConfig,            /* FTS5 Configuration object */
  int flags,                      /* FTS5_TOKENIZE_* flags */
  const char *pText, int nText,   /* Text to tokenize */
  void *pCtx,                     /* Context passed to xToken() */
  int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
){
  if( pText ){
    Fts5TokenizerInst *p = pConfig->pTokList;
    return p->pTokApi->xTokenize(p->pTok, pCtx, flags, pText, nText, xToken);
  }
  return SQLITE_OK;
}

int sqlite3Fts5SpecTokenize(
  Fts5Config *pConfig,            /* FTS5 Configuration object */
  const char *zSpec,              /* Tokenizer specification */
  int flags,                      /* FTS5_TOKENIZE_* flags */
  const char *pText, int nText,   /* Text to tokenize */
  void *pCtx,                     /* Context passed to xToken() */
  int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
){
  if( pText ){
    Fts5TokenizerInst *p = pConfig->pTokList;
    if( zSpec ){
      int rc = sqlite3Fts5ConfigFindTokenizer(pConfig, zSpec, &p);
      if( rc!=SQLITE_OK ) return rc;
    }
    return p->pTokApi->xTokenize(p->pTok, pCtx, flags, pText, nText, xToken);
  }
  return SQLITE_OK;
}

/*
** Argument pIn points to the first character in what is expected to be
** a comma-separated list of SQL literals followed by a ')' character.
** If it actually is this, return a pointer to the ')'. Otherwise, return
** NULL to indicate a parse error.

  void (*xDestroy)(void*);        /* Destructor function */
  Fts5TokenizerModule *pNext;     /* Next registered tokenizer module */
};

struct Fts5FullTable {
  Fts5Table p;                    /* Public class members from fts5Int.h */
  Fts5Storage *pStorage;          /* Document store */
#if 0
  Fts5Global *pGlobal;            /* Global (connection wide) data */
#endif
  Fts5Cursor *pSortCsr;           /* Sort data from this cursor */
  int iSavepoint;                 /* Successful xSavepoint()+1 */
  
#ifdef SQLITE_DEBUG
  struct Fts5TransactionState ts;
#endif
};

  pTab = (Fts5FullTable*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5FullTable));
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr);
    assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 );
  }
  if( rc==SQLITE_OK ){
    pTab->p.pConfig = pConfig;

  }

  /* Open the index sub-system */
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5IndexOpen(pConfig, bCreate, &pTab->p.pIndex, pzErr);
  }

  pInfo->idxNum = idxFlags;
  return SQLITE_OK;
}

static int fts5NewTransaction(Fts5FullTable *pTab){
  Fts5Cursor *pCsr;
  for(pCsr=pTab->p.pConfig->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
    if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK;
  }
  return sqlite3Fts5StorageReset(pTab->pStorage);
}

/*
** Implementation of xOpen method.
*/
static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
  Fts5FullTable *pTab = (Fts5FullTable*)pVTab;
  Fts5Config *pConfig = pTab->p.pConfig;
  Fts5Cursor *pCsr = 0;           /* New cursor object */
  sqlite3_int64 nByte;            /* Bytes of space to allocate */
  int rc;                         /* Return code */

  rc = fts5NewTransaction(pTab);
  if( rc==SQLITE_OK ){
    nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
    pCsr = (Fts5Cursor*)sqlite3_malloc64(nByte);
    if( pCsr ){
      Fts5Global *pGlobal = pConfig->pGlobal;
      memset(pCsr, 0, (size_t)nByte);
      pCsr->aColumnSize = (int*)&pCsr[1];
      pCsr->pNext = pGlobal->pCsr;
      pGlobal->pCsr = pCsr;
      pCsr->iCsrId = ++pGlobal->iNextId;
    }else{
      rc = SQLITE_NOMEM;

  if( pCursor ){
    Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab);
    Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
    Fts5Cursor **pp;

    fts5FreeCursorComponents(pCsr);
    /* Remove the cursor from the Fts5Global.pCsr list */
    for(pp=&pTab->p.pConfig->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
    *pp = pCsr->pNext;

    sqlite3_free(pCsr);
  }
  return SQLITE_OK;
}

/*
** Set the FTS5CSR_REQUIRE_RESEEK flag on all FTS5_PLAN_MATCH cursors 
** open on table pTab.
*/
static void fts5TripCursors(Fts5FullTable *pTab){
  Fts5Cursor *pCsr;
  for(pCsr=pTab->p.pConfig->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
    if( pCsr->ePlan==FTS5_PLAN_MATCH
     && pCsr->base.pVtab==(sqlite3_vtab*)pTab 
    ){
      CsrFlagSet(pCsr, FTS5CSR_REQUIRE_RESEEK);
    }
  }
}

** Search for an auxiliary function named zName that can be used with table
** pTab. If one is found, return a pointer to the corresponding Fts5Auxiliary
** structure. Otherwise, if no such function exists, return NULL.
*/
static Fts5Auxiliary *fts5FindAuxiliary(Fts5FullTable *pTab, const char *zName){
  Fts5Auxiliary *pAux;

  for(pAux=pTab->p.pConfig->pGlobal->pAux; pAux; pAux=pAux->pNext){
    if( sqlite3_stricmp(zName, pAux->zFunc)==0 ) return pAux;
  }

  /* No function of the specified name was found. Return 0. */
  return 0;
}

  /* Decode the arguments passed through to this function. */
  for(i=0; i<nVal; i++){
    switch( idxStr[iIdxStr++] ){
      case 'r':
        pRank = apVal[i];
        break;
      case 'M': {
        Fts5TokenizerInst *pInst = 0;
        char *zText = 0;
        int bDel = 0;

        rc = sqlite3Fts5UnpackTokenizeBlob(
            pConfig, apVal[i], &pInst, &zText, &bDel
        );

        if( zText==0 ) zText = "";
        iCol = 0;
        do{
          iCol = iCol*10 + (idxStr[iIdxStr]-'0');
          iIdxStr++;
        }while( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' );

        if( zText[0]=='*' ){
          /* The user has issued a query of the form "MATCH '*...'". This
          ** indicates that the MATCH expression is not a full text query,
          ** but a request for an internal parameter.  */
          rc = fts5SpecialMatch(pTab, pCsr, &zText[1]);
          if( bDel ) sqlite3_free(zText);
          goto filter_out;
        }else{
          char **pzErr = &pTab->p.base.zErrMsg;
          rc = sqlite3Fts5ExprNew(pConfig, 0, iCol, zText, &pExpr, pzErr);
          if( rc==SQLITE_OK ){
            rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr);
            pExpr = 0;
          }

        }

        if( bDel ) sqlite3_free(zText);
        if( rc!=SQLITE_OK ) goto filter_out;
        break;
      }
      case 'L':
      case 'G': {
        int bGlob = (idxStr[iIdxStr-1]=='G');
        const char *zText = (const char*)sqlite3_value_text(apVal[i]);
        iCol = 0;

    rc = SQLITE_ERROR;
  }

  return rc;
}

int sqlite3Fts5GetTokenizer(



  Fts5Config *pConfig,
  const char *zSpec
){

  int rc = SQLITE_OK;
  char **pzErr = pConfig->pzErrmsg;
  const char **azArg = 0;
  char *pDel = 0;
  sqlite3_int64 nArg = 0;

  assert( pzErr || (zSpec==0 && pConfig->pTokList==0) );
  if( zSpec ){
    const char *p = (const char*)zSpec;
    char *pSpace = 0;

    nArg = strlen(zSpec) + 1;
    pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
    pSpace = pDel;
    azArg = (const char**)sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);

    if( azArg && pSpace ){
      for(nArg=0; p && *p; nArg++){
        const char *p2 = fts5ConfigSkipWhitespace(p);
        if( *p2=='\'' ){
          p = fts5ConfigSkipLiteral(p2);
        }else{
          p = fts5ConfigSkipBareword(p2);
        }
        if( p ){
          memcpy(pSpace, p2, p-p2);
          azArg[nArg] = pSpace;
          sqlite3Fts5Dequote(pSpace);
          pSpace += (p - p2) + 1;
          p = fts5ConfigSkipWhitespace(p);
        }
      }
      if( p==0 ){
        *pzErr= sqlite3_mprintf("parse error in tokenize directive");
        rc = SQLITE_ERROR;
      }
    }
  }

  if( rc==SQLITE_OK ){
    Fts5TokenizerModule *pMod;
    pMod = fts5LocateTokenizer(pConfig->pGlobal, nArg==0 ? 0 : azArg[0]);
    if( pMod==0 ){
      assert( nArg>0 );
      rc = SQLITE_ERROR;
      *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]);
    }else{
      int nSpec = zSpec ? strlen(zSpec) + 1 : 0;
      int nByte = sizeof(Fts5TokenizerInst) + nSpec;
      Fts5TokenizerInst *pNew = sqlite3Fts5MallocZero(&rc, nByte);
      if( pNew ){
        if( zSpec ){
          pNew->zSpec = (char*)&pNew[1];
          memcpy(pNew->zSpec, zSpec, nSpec);
        }
        rc = pMod->x.xCreate(
            pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pNew->pTok
        );
        pNew->pTokApi = &pMod->x;
        if( rc!=SQLITE_OK ){
          if( pzErr ){
            *pzErr = sqlite3_mprintf("error in tokenizer constructor");
          }
        }else if( pConfig->pTokList==0 ){
          pConfig->ePattern = sqlite3Fts5TokenizerPattern(
              pMod->x.xCreate, pNew->pTok
          );
        }
      }

      if( rc==SQLITE_OK ){
        if( pConfig->pTokList ){
          pNew->pNext = pConfig->pTokList->pNext;
          pConfig->pTokList->pNext = pNew;
        }else{
          pConfig->pTokList = pNew;
        }
      }else{
        sqlite3_free(pNew);
      }
    }
  }

  sqlite3_free(azArg);
  sqlite3_free(pDel);
  return rc;
}

static void fts5ModuleDestroy(void *pCtx){
  Fts5TokenizerModule *pTok, *pNextTok;
  Fts5Auxiliary *pAux, *pNextAux;
  Fts5Global *pGlobal = (Fts5Global*)pCtx;

  UNUSED_PARAM2(iUnused1, iUnused2);
  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
  if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
    pCtx->szCol++;
  }
  return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken);
}

#define IS_TOKENIZE_BLOB(pVal) ( \
    sqlite3_value_subtype(pVal)==SQLITE_FTS5_TOKENIZE_SUBTYPE \
 && sqlite3_value_type(pVal)==SQLITE_BLOB \
)

static const char *fts5UnpackTokenizeBlob(
  sqlite3_value *pVal,
  const char **pzT,
  int *pnT
){
  const u8 *pBlob = sqlite3_value_blob(pVal);
  int nBlob = sqlite3_value_bytes(pVal);
  int ii;

  assert( sqlite3_value_subtype(pVal)==SQLITE_FTS5_TOKENIZE_SUBTYPE );
  assert( sqlite3_value_type(pVal)==SQLITE_BLOB );

  for(ii=0; pBlob[ii]; ii++){
    if( ii==nBlob ){
      *pzT = 0;
      *pnT = 0;
      return 0;
    }
  }

  *pzT = (const char*)&pBlob[ii+1];
  *pnT = nBlob - ii - 1;
  return (const char*)pBlob;
}


/*
** If a row with rowid iDel is present in the %_content table, add the
** delete-markers to the FTS index necessary to delete it. Do not actually
** remove the %_content row at this time though.
*/
static int fts5StorageDeleteFromIndex(

    }
  }

  ctx.pStorage = p;
  ctx.iCol = -1;
  for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
    if( pConfig->abUnindexed[iCol-1]==0 ){
      const char *zText = 0;
      const char *zTok = 0;
      int nText = 0;
      assert( pSeek==0 || apVal==0 );
      assert( pSeek!=0 || apVal!=0 );
      if( pSeek ){
        zText = (const char*)sqlite3_column_text(pSeek, iCol);
        nText = sqlite3_column_bytes(pSeek, iCol);
      }else if( ALWAYS(apVal) ){
        sqlite3_value *pVal = apVal[iCol-1];
        if( IS_TOKENIZE_BLOB(pVal) ){
          zTok = fts5UnpackTokenizeBlob(pVal, &zText, &nText);
        }else{
          zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
          nText = sqlite3_value_bytes(apVal[iCol-1]);
        }
      }else{
        continue;
      }
      ctx.szCol = 0;
      rc = sqlite3Fts5SpecTokenize(pConfig, zTok, FTS5_TOKENIZE_DOCUMENT, 
          zText, nText, (void*)&ctx, fts5StorageInsertCallback
      );
      p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
      if( p->aTotalSize[iCol-1]<0 ){
        rc = FTS5_CORRUPT;
      }
    }

    }
    if( rc==SQLITE_OK ){
      *piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
    }
  }
  return rc;
}

int sqlite3Fts5UnpackTokenizeBlob(
  Fts5Config *pConfig,
  sqlite3_value *pVal,
  Fts5TokenizerInst **ppTok,
  char **pzText,
  int *pbDel
){
  int rc = SQLITE_OK;
  if( IS_TOKENIZE_BLOB(pVal) ){
    const char *zTok = 0;
    const char *zText = 0;
    int nText = 0;
    zTok = fts5UnpackTokenizeBlob(pVal, &zText, &nText);
    rc = sqlite3Fts5ConfigFindTokenizer(pConfig, zTok, ppTok);
    *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, zText);
    *pbDel = 1;
  }else{
    *pzText = (char*)sqlite3_value_text(pVal);
    *pbDel = 0;
    *ppTok = pConfig->pTokList;
  }
  return rc;
}

/*
** Insert a new row into the FTS content table.
*/
int sqlite3Fts5StorageContentInsert(
  Fts5Storage *p, 
  sqlite3_value **apVal, 

      rc = fts5StorageNewRowid(p, piRowid);
    }
  }else{
    sqlite3_stmt *pInsert = 0;    /* Statement to write %_content table */
    int i;                        /* Counter variable */
    rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
    for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
      sqlite3_value *pVal = apVal[i];
      if( IS_TOKENIZE_BLOB(pVal) ){
        const char *zT = 0;
        int nT = 0;
        fts5UnpackTokenizeBlob(pVal, &zT, &nT);
        rc = sqlite3_bind_text(pInsert, i, zT, nT, SQLITE_STATIC);
      }else{
        rc = sqlite3_bind_value(pInsert, i, apVal[i]);
      }
    }
    if( rc==SQLITE_OK ){
      sqlite3_step(pInsert);
      rc = sqlite3_reset(pInsert);
    }
    *piRowid = sqlite3_last_insert_rowid(pConfig->db);
  }

  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);
  }
  for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
    ctx.szCol = 0;
    if( pConfig->abUnindexed[ctx.iCol]==0 ){
      sqlite3_value *pVal = apVal[ctx.iCol+2];
      const char *zText = 0;
      const char *zTok = 0;
      int nText = 0;

      if( IS_TOKENIZE_BLOB(pVal) ){
        zTok = fts5UnpackTokenizeBlob(pVal, &zText, &nText);
      }else{
        zText = (const char*)sqlite3_value_text(apVal[ctx.iCol+2]);
        nText = sqlite3_value_bytes(apVal[ctx.iCol+2]);
      }
      rc = sqlite3Fts5SpecTokenize(pConfig, 
          zTok, FTS5_TOKENIZE_DOCUMENT, zText, nText, 


          (void*)&ctx, fts5StorageInsertCallback
      );
    }
    sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
    p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
  }
  p->nTotalRow++;

  rc = sqlite3Fts5TestRegisterMatchinfo(db);
  if( rc!=SQLITE_OK ){
    Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
    return TCL_ERROR;
  }
  return TCL_OK;
}

static void f5tFree(void *p) { sqlite3_free(p); }

static void f5tScalarFunc(
  sqlite3_context *ctx, 
  int nArg, 
  sqlite3_value **apArg
){
  const char *zText = (const char*)sqlite3_value_text(apArg[0]);
  int nText = sqlite3_value_bytes(apArg[0]);
  const char *zTok = (const char*)sqlite3_value_text(apArg[1]);
  int nTok = sqlite3_value_bytes(apArg[1]);
  unsigned char *aBuf = 0; 
  int nBuf = 0;

  assert( nArg==2 );

  if( zTok==0 ) zTok = "";
  nBuf = nTok + 1 + nText;
  aBuf = (unsigned char*)sqlite3_malloc(nBuf);
  if( aBuf==0 ){
    sqlite3_result_error_nomem(ctx);
  }else{
    memcpy(aBuf, zTok, nTok+1);
    memcpy(&aBuf[nTok+1], zText, nText);
    sqlite3_result_blob(ctx, aBuf, nBuf, f5tFree);
    sqlite3_result_subtype(ctx, SQLITE_FTS5_TOKENIZE_SUBTYPE);
  }
}

static int SQLITE_TCLAPI f5tRegisterTok(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  int rc;
  sqlite3 *db = 0;
  fts5_api *pApi = 0;

  if( objc!=2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "DB");
    return TCL_ERROR;
  }
  if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){
    return TCL_ERROR;
  }

  rc = sqlite3Fts5TestRegisterTok(db, pApi);
  if( rc==SQLITE_OK ){
    rc = sqlite3_create_function(db, "fts5tokenize", 2, SQLITE_UTF8, 0, 
        f5tScalarFunc, 0, 0
    );
  }
  if( rc!=SQLITE_OK ){
    Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
    return TCL_ERROR;
  }
  
  return TCL_OK;
}

typedef struct OriginTextCtx OriginTextCtx;
struct OriginTextCtx {
  sqlite3 *db;
  fts5_api *pApi;