}

/*
** Close the cursor.  For additional information see the documentation
** on the xClose method of the virtual table interface.
*/
static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){

  Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  Fts5Cursor **pp;
  Fts5Auxdata *pData;
  Fts5Auxdata *pNext;

  fts5CsrNewrow(pCsr);
  if( pCsr->pStmt ){
    int eStmt = fts5StmtType(pCsr->idxNum);
    sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
  }
  if( pCsr->pSorter ){
    Fts5Sorter *pSorter = pCsr->pSorter;
    sqlite3_finalize(pSorter->pStmt);
    sqlite3_free(pSorter);
  }
  
  if( pCsr->idxNum!=FTS5_PLAN_SOURCE ){
    sqlite3Fts5ExprFree(pCsr->pExpr);
  }

  for(pData=pCsr->pAuxdata; pData; pData=pNext){
    pNext = pData->pNext;
    if( pData->xDelete ) pData->xDelete(pData->pPtr);
    sqlite3_free(pData);
  }

  /* Remove the cursor from the Fts5Global.pCsr list */
  for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
  *pp = pCsr->pNext;

  sqlite3_finalize(pCsr->pRankArgStmt);
  sqlite3_free(pCsr->apRankArg);

  sqlite3_free(pCsr->zSpecial);
  if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){
    sqlite3_free(pCsr->zRank);
    sqlite3_free(pCsr->zRankArgs);
  }
  sqlite3_free(pCsr);

  return SQLITE_OK;
}

static int fts5SorterNext(Fts5Cursor *pCsr){
  Fts5Sorter *pSorter = pCsr->pSorter;
  int rc;

................................................................................
){
  int rc = SQLITE_OK;
  if( pRank ){
    const char *z = (const char*)sqlite3_value_text(pRank);
    char *zRank = 0;
    char *zRankArgs = 0;




    rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs);

    if( rc==SQLITE_OK ){
      pCsr->zRank = zRank;
      pCsr->zRankArgs = zRankArgs;
      CsrFlagSet(pCsr, FTS5CSR_FREE_ZRANK);
    }else if( rc==SQLITE_ERROR ){
      pCsr->base.pVtab->zErrMsg = sqlite3_mprintf(
          "parse error in rank function: %s", z
................................................................................
  ** Update and insert operations pass:
  **
  **   1. The "old" rowid, or NULL.
  **   2. The "new" rowid.
  **   3. Values for each of the nCol matchable columns.
  **   4. Values for the two hidden columns (<tablename> and "rank").
  */
  assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) );

  eType0 = sqlite3_value_type(apVal[0]);
  eConflict = sqlite3_vtab_on_conflict(pConfig->db);

  assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
  assert( pVtab->zErrMsg==0 );


  fts5TripCursors(pTab);
  if( rc==SQLITE_OK && eType0==SQLITE_INTEGER ){
    if( fts5IsContentless(pTab) ){
      pTab->base.zErrMsg = sqlite3_mprintf(
          "cannot %s contentless fts5 table: %s", 
          (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName
      );
      rc = SQLITE_ERROR;
    }else{
      i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
    }

  }else if( nArg>1 ){
    sqlite3_value *pCmd = apVal[2 + pConfig->nCol];
    if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){
      const char *z = (const char*)sqlite3_value_text(pCmd);
      if( pConfig->eContent!=FTS5_CONTENT_NORMAL 
       && 0==sqlite3_stricmp("delete", z) 
      ){
        return fts5SpecialDelete(pTab, apVal, pRowid);
................................................................................
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
  int rc = SQLITE_OK;

  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){
    i64 iRowid = fts5CursorRowid(pCsr);
    rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);

  }
  if( iCol<0 ){
    int i;
    *pnToken = 0;
    for(i=0; i<pTab->pConfig->nCol; i++){
      *pnToken += pCsr->aColumnSize[i];
    }
................................................................................
    }
  }else{
    rc = SQLITE_NOMEM;
  }

  return rc;
}


















/*
** Find a tokenizer. This is the implementation of the 
** fts5_api.xFindTokenizer() method.
*/
static int fts5FindTokenizer(
  fts5_api *pApi,                 /* Global context (one per db handle) */
  const char *zName,              /* Name of new function */
  void **ppUserData,
  fts5_tokenizer *pTokenizer      /* Populate this object */
){
  Fts5Global *pGlobal = (Fts5Global*)pApi;
  int rc = SQLITE_OK;
  Fts5TokenizerModule *pTok;

  if( zName==0 ){
    pTok = pGlobal->pDfltTok;
  }else{
    for(pTok=pGlobal->pTok; pTok; pTok=pTok->pNext){
      if( sqlite3_stricmp(zName, pTok->zName)==0 ) break;
    }
  }


  if( pTok ){
    *pTokenizer = pTok->x;
    *ppUserData = pTok->pUserData;
  }else{
    memset(pTokenizer, 0, sizeof(fts5_tokenizer));
    rc = SQLITE_ERROR;
  }

  return rc;
}

int sqlite3Fts5GetTokenizer(
  Fts5Global *pGlobal, 
  const char **azArg,
  int nArg,
  Fts5Tokenizer **ppTok,
  fts5_tokenizer **ppTokApi

){
  Fts5TokenizerModule *pMod = 0;
  int rc = SQLITE_OK;

  if( nArg==0 ){
    pMod = pGlobal->pDfltTok;
  }else{
    for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){
      if( sqlite3_stricmp(azArg[0], pMod->zName)==0 ) break;
    }
  }

  if( pMod==0 ){

    rc = SQLITE_ERROR;

  }else{
    rc = pMod->x.xCreate(pMod->pUserData, &azArg[1], (nArg?nArg-1:0), ppTok);
    *ppTokApi = &pMod->x;



  }

  if( rc!=SQLITE_OK ){
    *ppTokApi = 0;
    *ppTok = 0;
  }

}

/*
** Close the cursor.  For additional information see the documentation
** on the xClose method of the virtual table interface.
*/
static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
  if( pCursor ){
    Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
    Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
    Fts5Cursor **pp;
    Fts5Auxdata *pData;
    Fts5Auxdata *pNext;

    fts5CsrNewrow(pCsr);
    if( pCsr->pStmt ){
      int eStmt = fts5StmtType(pCsr->idxNum);
      sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
    }
    if( pCsr->pSorter ){
      Fts5Sorter *pSorter = pCsr->pSorter;
      sqlite3_finalize(pSorter->pStmt);
      sqlite3_free(pSorter);
    }

    if( pCsr->idxNum!=FTS5_PLAN_SOURCE ){
      sqlite3Fts5ExprFree(pCsr->pExpr);
    }

    for(pData=pCsr->pAuxdata; pData; pData=pNext){
      pNext = pData->pNext;
      if( pData->xDelete ) pData->xDelete(pData->pPtr);
      sqlite3_free(pData);
    }

    /* Remove the cursor from the Fts5Global.pCsr list */
    for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
    *pp = pCsr->pNext;

    sqlite3_finalize(pCsr->pRankArgStmt);
    sqlite3_free(pCsr->apRankArg);

    sqlite3_free(pCsr->zSpecial);
    if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){
      sqlite3_free(pCsr->zRank);
      sqlite3_free(pCsr->zRankArgs);
    }
    sqlite3_free(pCsr);
  }
  return SQLITE_OK;
}

static int fts5SorterNext(Fts5Cursor *pCsr){
  Fts5Sorter *pSorter = pCsr->pSorter;
  int rc;

................................................................................
){
  int rc = SQLITE_OK;
  if( pRank ){
    const char *z = (const char*)sqlite3_value_text(pRank);
    char *zRank = 0;
    char *zRankArgs = 0;

    if( z==0 ){
      if( sqlite3_value_type(pRank)==SQLITE_NULL ) rc = SQLITE_ERROR;
    }else{
      rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs);
    }
    if( rc==SQLITE_OK ){
      pCsr->zRank = zRank;
      pCsr->zRankArgs = zRankArgs;
      CsrFlagSet(pCsr, FTS5CSR_FREE_ZRANK);
    }else if( rc==SQLITE_ERROR ){
      pCsr->base.pVtab->zErrMsg = sqlite3_mprintf(
          "parse error in rank function: %s", z
................................................................................
  ** Update and insert operations pass:
  **
  **   1. The "old" rowid, or NULL.
  **   2. The "new" rowid.
  **   3. Values for each of the nCol matchable columns.
  **   4. Values for the two hidden columns (<tablename> and "rank").
  */


  eType0 = sqlite3_value_type(apVal[0]);
  eConflict = sqlite3_vtab_on_conflict(pConfig->db);

  assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
  assert( pVtab->zErrMsg==0 );
  assert( (nArg==1 && eType0==SQLITE_INTEGER) || nArg==(2+pConfig->nCol+2) );

  fts5TripCursors(pTab);
  if( eType0==SQLITE_INTEGER ){
    if( fts5IsContentless(pTab) ){
      pTab->base.zErrMsg = sqlite3_mprintf(
          "cannot %s contentless fts5 table: %s", 
          (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName
      );
      rc = SQLITE_ERROR;
    }else{
      i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
    }
  }else{
    assert( nArg>1 );
    sqlite3_value *pCmd = apVal[2 + pConfig->nCol];
    if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){
      const char *z = (const char*)sqlite3_value_text(pCmd);
      if( pConfig->eContent!=FTS5_CONTENT_NORMAL 
       && 0==sqlite3_stricmp("delete", z) 
      ){
        return fts5SpecialDelete(pTab, apVal, pRowid);
................................................................................
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
  int rc = SQLITE_OK;

  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){
    i64 iRowid = fts5CursorRowid(pCsr);
    rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
    CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE);
  }
  if( iCol<0 ){
    int i;
    *pnToken = 0;
    for(i=0; i<pTab->pConfig->nCol; i++){
      *pnToken += pCsr->aColumnSize[i];
    }
................................................................................
    }
  }else{
    rc = SQLITE_NOMEM;
  }

  return rc;
}

static Fts5TokenizerModule *fts5LocateTokenizer(
  Fts5Global *pGlobal, 
  const char *zName
){
  Fts5TokenizerModule *pMod = 0;

  if( zName==0 ){
    pMod = pGlobal->pDfltTok;
  }else{
    for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){
      if( sqlite3_stricmp(zName, pMod->zName)==0 ) break;
    }
  }

  return pMod;
}

/*
** Find a tokenizer. This is the implementation of the 
** fts5_api.xFindTokenizer() method.
*/
static int fts5FindTokenizer(
  fts5_api *pApi,                 /* Global context (one per db handle) */
  const char *zName,              /* Name of new function */
  void **ppUserData,
  fts5_tokenizer *pTokenizer      /* Populate this object */
){

  int rc = SQLITE_OK;
  Fts5TokenizerModule *pMod;









  pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName);
  if( pMod ){
    *pTokenizer = pMod->x;
    *ppUserData = pMod->pUserData;
  }else{
    memset(pTokenizer, 0, sizeof(fts5_tokenizer));
    rc = SQLITE_ERROR;
  }

  return rc;
}

int sqlite3Fts5GetTokenizer(
  Fts5Global *pGlobal, 
  const char **azArg,
  int nArg,
  Fts5Tokenizer **ppTok,
  fts5_tokenizer **ppTokApi,
  char **pzErr
){
  Fts5TokenizerModule *pMod;
  int rc = SQLITE_OK;


  pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]);






  if( pMod==0 ){
    assert( nArg>0 );
    rc = SQLITE_ERROR;
    *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]);
  }else{
    rc = pMod->x.xCreate(pMod->pUserData, &azArg[1], (nArg?nArg-1:0), ppTok);
    *ppTokApi = &pMod->x;
    if( rc!=SQLITE_OK && pzErr ){
      *pzErr = sqlite3_mprintf("error in tokenizer constructor");
    }
  }

  if( rc!=SQLITE_OK ){
    *ppTokApi = 0;
    *ppTok = 0;
  }

typedef struct Fts5Global Fts5Global;

int sqlite3Fts5GetTokenizer(
  Fts5Global*, 
  const char **azArg,
  int nArg,
  Fts5Tokenizer**,
  fts5_tokenizer**

);

/*
** End of interface to code in fts5.c.
**************************************************************************/

/**************************************************************************

typedef struct Fts5Global Fts5Global;

int sqlite3Fts5GetTokenizer(
  Fts5Global*, 
  const char **azArg,
  int nArg,
  Fts5Tokenizer**,
  fts5_tokenizer**,
  char **pzErr
);

/*
** End of interface to code in fts5.c.
**************************************************************************/

/**************************************************************************

    if( rc==SQLITE_OK ){
      rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
    }
    fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);

    if( rc==SQLITE_OK ){
      sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
    }else{
      sqlite3_result_error_code(pCtx, rc);
    }
    sqlite3_free(ctx.zOut);
  }



}
/*
** End of highlight() implementation.
**************************************************************************/

/*
** Implementation of snippet() function.

    if( rc==SQLITE_OK ){
      rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
    }
    fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);

    if( rc==SQLITE_OK ){
      sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);


    }
    sqlite3_free(ctx.zOut);
  }
  if( rc!=SQLITE_OK ){
    sqlite3_result_error_code(pCtx, rc);
  }
}
/*
** End of highlight() implementation.
**************************************************************************/

/*
** Implementation of snippet() function.

          }
        }
        if( p==0 ){
          *pzErr = sqlite3_mprintf("parse error in tokenize directive");
          rc = SQLITE_ERROR;
        }else{
          rc = sqlite3Fts5GetTokenizer(pGlobal, 
              (const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi

          );
          if( rc!=SQLITE_OK ){
            *pzErr = sqlite3_mprintf("error in tokenizer constructor");
          }
        }
      }
    }

    sqlite3_free(azArg);
    sqlite3_free(pDel);
    return rc;
................................................................................
** Allocate an instance of the default tokenizer ("simple") at 
** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
** code if an error occurs.
*/
static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
  assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
  return sqlite3Fts5GetTokenizer(
      pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi
  );
}

/*
** Gobble up the first bareword or quoted word from the input buffer zIn.
** Return a pointer to the character immediately following the last in
** the gobbled word if successful, or a NULL pointer otherwise (failed
................................................................................

    if( rc==SQLITE_OK ){
      if( z==0 ){
        *pzErr = sqlite3_mprintf("parse error in \"%s\"", zOrig);
        rc = SQLITE_ERROR;
      }else{
        if( bOption ){
          rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo, pzErr);
        }else{
          rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr);
          zOne = 0;
        }
      }
    }

          }
        }
        if( p==0 ){
          *pzErr = sqlite3_mprintf("parse error in tokenize directive");
          rc = SQLITE_ERROR;
        }else{
          rc = sqlite3Fts5GetTokenizer(pGlobal, 
              (const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi,
              pzErr
          );



        }
      }
    }

    sqlite3_free(azArg);
    sqlite3_free(pDel);
    return rc;
................................................................................
** Allocate an instance of the default tokenizer ("simple") at 
** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
** code if an error occurs.
*/
static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
  assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
  return sqlite3Fts5GetTokenizer(
      pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi, 0
  );
}

/*
** Gobble up the first bareword or quoted word from the input buffer zIn.
** Return a pointer to the character immediately following the last in
** the gobbled word if successful, or a NULL pointer otherwise (failed
................................................................................

    if( rc==SQLITE_OK ){
      if( z==0 ){
        *pzErr = sqlite3_mprintf("parse error in \"%s\"", zOrig);
        rc = SQLITE_ERROR;
      }else{
        if( bOption ){
          rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo?zTwo:"", pzErr);
        }else{
          rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr);
          zOne = 0;
        }
      }
    }

  Fts5Expr *pNew;
  Fts5ExprPhrase **apPhrase;
  Fts5ExprNode *pNode;
  Fts5ExprNearset *pNear;
  Fts5ExprPhrase *pCopy;

  pOrig = pExpr->apExprPhrase[iPhrase];



  pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr));
  apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*));
  pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode));
  pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, 
      sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)
  );
  pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, 
      sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm
  );

  for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
    pCopy->aTerm[i].zTerm = fts5ExprStrdup(&rc, pOrig->aTerm[i].zTerm);
    pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
  }

  if( rc==SQLITE_OK ){

  Fts5Expr *pNew;
  Fts5ExprPhrase **apPhrase;
  Fts5ExprNode *pNode;
  Fts5ExprNearset *pNear;
  Fts5ExprPhrase *pCopy;

  pOrig = pExpr->apExprPhrase[iPhrase];
  pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, 
      sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm
  );
  pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr));
  apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*));
  pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode));
  pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, 
      sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)
  );




  for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
    pCopy->aTerm[i].zTerm = fts5ExprStrdup(&rc, pOrig->aTerm[i].zTerm);
    pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
  }

  if( rc==SQLITE_OK ){

/*************************************************************************
** This is a copy of the first part of the SqliteDb structure in 
** tclsqlite.c.  We need it here so that the get_sqlite_pointer routine
** can extract the sqlite3* pointer from an existing Tcl SQLite
** connection.
*/



struct SqliteDb {
  sqlite3 *db;
};

/*
** Decode a pointer to an sqlite3 object.
*/
................................................................................
    default: 
      assert( 0 );
      break;
  }
#undef CASE

  if( rc!=SQLITE_OK ){
    Tcl_AppendResult(interp, "error in api call", 0);
    return TCL_ERROR;
  }

  return TCL_OK;
}

static void xF5tFunction(
................................................................................
  Tcl_DecrRefCount(pEval);

  pInst->pContext->pCtx = pOldCtx;
  pInst->pContext->xToken = xOldToken;
  return rc;
}

extern const char *sqlite3ErrName(int);

/*
** sqlite3_fts5_token TEXT START END POS
*/
static int f5tTokenizerReturn(
  void * clientData,
  Tcl_Interp *interp,
  int objc,

/*************************************************************************
** This is a copy of the first part of the SqliteDb structure in 
** tclsqlite.c.  We need it here so that the get_sqlite_pointer routine
** can extract the sqlite3* pointer from an existing Tcl SQLite
** connection.
*/

extern const char *sqlite3ErrName(int);

struct SqliteDb {
  sqlite3 *db;
};

/*
** Decode a pointer to an sqlite3 object.
*/
................................................................................
    default: 
      assert( 0 );
      break;
  }
#undef CASE

  if( rc!=SQLITE_OK ){
    Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
    return TCL_ERROR;
  }

  return TCL_OK;
}

static void xF5tFunction(
................................................................................
  Tcl_DecrRefCount(pEval);

  pInst->pContext->pCtx = pOldCtx;
  pInst->pContext->xToken = xOldToken;
  return rc;
}



/*
** sqlite3_fts5_token TEXT START END POS
*/
static int f5tTokenizerReturn(
  void * clientData,
  Tcl_Interp *interp,
  int objc,

  const char **azArg, int nArg,
  Fts5Tokenizer **ppOut
){
  fts5_api *pApi = (fts5_api*)pCtx;
  int rc = SQLITE_OK;
  PorterTokenizer *pRet;
  void *pUserdata = 0;






  pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer));
  if( pRet ){
    memset(pRet, 0, sizeof(PorterTokenizer));
    rc = pApi->xFindTokenizer(pApi, "unicode61", &pUserdata, &pRet->tokenizer);
  }else{
    rc = SQLITE_NOMEM;
  }
  if( rc==SQLITE_OK ){
    rc = pRet->tokenizer.xCreate(pUserdata, 0, 0, &pRet->pTokenizer);
  }

  const char **azArg, int nArg,
  Fts5Tokenizer **ppOut
){
  fts5_api *pApi = (fts5_api*)pCtx;
  int rc = SQLITE_OK;
  PorterTokenizer *pRet;
  void *pUserdata = 0;
  const char *zBase = "unicode61";

  if( nArg>0 ){
    zBase = azArg[0];
  }

  pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer));
  if( pRet ){
    memset(pRet, 0, sizeof(PorterTokenizer));
    rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer);
  }else{
    rc = SQLITE_NOMEM;
  }
  if( rc==SQLITE_OK ){
    rc = pRet->tokenizer.xCreate(pUserdata, 0, 0, &pRet->pTokenizer);
  }

} {}

do_catchsql_test 12.2 {
  SELECT t2 FROM t2 WHERE t2 MATCH '*stuff'
} {1 {unknown special query: stuff}}

do_test 12.3 {
  set res [db one { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }]
  string is integer $res
} {1}

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 13.1 {

} {}

do_catchsql_test 12.2 {
  SELECT t2 FROM t2 WHERE t2 MATCH '*stuff'
} {1 {unknown special query: stuff}}

do_test 12.3 {
  set res [db eval { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }]
  string is integer $res
} {1}

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 13.1 {

do_execsql_test 4.3.3 {
  SELECT *, rank FROM t3
  WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(3)' 
  ORDER BY rank ASC
} {
  {a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2 
}










finish_test

do_execsql_test 4.3.3 {
  SELECT *, rank FROM t3
  WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(3)' 
  ORDER BY rank ASC
} {
  {a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2 
}

do_catchsql_test 4.4.3 {
  SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'xyz(3)' 
} {1 {no such function: xyz}}
do_catchsql_test 4.4.4 {
  SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
} {1 {parse error in rank function: }}



finish_test

# 2014 Dec 20
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on the auxiliary function APIs.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5aux

proc inst {cmd i} { 
  $cmd xInst $i
}
sqlite3_fts5_create_function db inst inst

proc colsize {cmd i} { 
  $cmd xColumnSize $i
}
sqlite3_fts5_create_function db colsize colsize

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE f1 USING fts5(a, b);
  INSERT INTO f1 VALUES('one two', 'two one zero');
}

do_catchsql_test 1.1 {
  SELECT inst(f1, -1) FROM f1 WHERE f1 MATCH 'two';
} {1 SQLITE_RANGE}
do_catchsql_test 1.2 {
  SELECT inst(f1, 0) FROM f1 WHERE f1 MATCH 'two';
} {0 {{0 0 1}}}
do_catchsql_test 1.3 {
  SELECT inst(f1, 1) FROM f1 WHERE f1 MATCH 'two';
} {0 {{0 1 0}}}
do_catchsql_test 1.4 {
  SELECT inst(f1, 2) FROM f1 WHERE f1 MATCH 'two';
} {1 SQLITE_RANGE}

do_catchsql_test 2.1 {
  SELECT colsize(f1, 2) FROM f1 WHERE f1 MATCH 'two';
} {1 SQLITE_RANGE}

do_execsql_test 2.2 {
  SELECT colsize(f1, 0), colsize(f1, 1) FROM f1 WHERE f1 MATCH 'zero';
} {2 3}



finish_test

  DELETE FROM x2 WHERE "key col" = 1;
  INSERT INTO t2(t2, rowid, a, c) VALUES('delete', 1, 'a b', 'e f');
  INSERT INTO t2(t2) VALUES('integrity-check');
}

do_execsql_test 4.8 { SELECT rowid FROM t2 WHERE t2 MATCH 'b'} {}
do_execsql_test 4.9 { SELECT rowid FROM t2 WHERE t2 MATCH 'y'} {-40}




















finish_test

  DELETE FROM x2 WHERE "key col" = 1;
  INSERT INTO t2(t2, rowid, a, c) VALUES('delete', 1, 'a b', 'e f');
  INSERT INTO t2(t2) VALUES('integrity-check');
}

do_execsql_test 4.8 { SELECT rowid FROM t2 WHERE t2 MATCH 'b'} {}
do_execsql_test 4.9 { SELECT rowid FROM t2 WHERE t2 MATCH 'y'} {-40}

#-------------------------------------------------------------------------
# Test that if the 'rowid' field of a 'delete' is not an integer, no
# changes are made to the FTS index.
#
do_execsql_test 5.0 {
  CREATE VIRTUAL TABLE t5 USING fts5(a, b, content=);
  INSERT INTO t5(rowid, a, b) VALUES(-1, 'one',   'two');
  INSERT INTO t5(rowid, a, b) VALUES( 0, 'three', 'four');
  INSERT INTO t5(rowid, a, b) VALUES( 1, 'five',  'six');
}

set ::checksum [execsql {SELECT md5sum(id, block) FROM t5_data}]

do_execsql_test 5.1 {
  INSERT INTO t5(t5, rowid, a, b) VALUES('delete', NULL, 'three', 'four');
  SELECT md5sum(id, block) FROM t5_data;
} $::checksum


finish_test

  for {set i 0} {$i < 500} {incr i} {
    execsql { INSERT INTO t2 VALUES(rnddoc(50)) }
  }
  execsql { INSERT INTO t2(t2) VALUES('integrity-check') }
} {}

#--------------------------------------------------------------------

#

















finish_test

  for {set i 0} {$i < 500} {incr i} {
    execsql { INSERT INTO t2 VALUES(rnddoc(50)) }
  }
  execsql { INSERT INTO t2(t2) VALUES('integrity-check') }
} {}

#--------------------------------------------------------------------
# A mundane test - missing row in the %_content table.
#
do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE t3 USING fts5(x);
  INSERT INTO t3 VALUES('one o');
  INSERT INTO t3 VALUES('two e');
  INSERT INTO t3 VALUES('three o');
  INSERT INTO t3 VALUES('four e');
  INSERT INTO t3 VALUES('five o');
}
do_execsql_test 3.1 {
  SELECT * FROM t3 WHERE t3 MATCH 'o'
} {{one o} {three o} {five o}}

do_catchsql_test 3.1 {
  DELETE FROM t3_content WHERE rowid = 3;
  SELECT * FROM t3 WHERE t3 MATCH 'o';
} {1 {database disk image is malformed}}

finish_test

# 2015 April 21
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This test is focused on edge cases in the doclist format.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5doclist


#-------------------------------------------------------------------------
# Create a table with 1000 columns. Then add some large documents to it.
# All text is in the right most column of the table.
#
do_test 1.0 {
  set cols [list]
  for {set i 0} {$i < 900} {incr i} { lappend cols "x$i" }
  execsql "CREATE VIRTUAL TABLE ccc USING fts5([join $cols ,])"
} {}

db func rnddoc fts5_rnddoc 
do_execsql_test 1.1 {
  WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
  INSERT INTO ccc(x899) SELECT rnddoc(500) FROM ii;
}

do_execsql_test 1.2 {
  INSERT INTO ccc(ccc) VALUES('integrity-check');
}


finish_test

set testprefix fts5fault4

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}



#-------------------------------------------------------------------------
# An OOM while dropping an fts5 table.
#
db func rnddoc fts5_rnddoc 
do_test 1.0 {
  execsql { CREATE VIRTUAL TABLE xx USING fts5(x) }
................................................................................
  faultsim_restore_and_reopen
  execsql { SELECT * FROM xx }
} -body {
  execsql { DROP TABLE xx }
} -test {
  faultsim_test_result [list 0 {}]
}






















































































































































finish_test

set testprefix fts5fault4

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}

if 1 {

#-------------------------------------------------------------------------
# An OOM while dropping an fts5 table.
#
db func rnddoc fts5_rnddoc 
do_test 1.0 {
  execsql { CREATE VIRTUAL TABLE xx USING fts5(x) }
................................................................................
  faultsim_restore_and_reopen
  execsql { SELECT * FROM xx }
} -body {
  execsql { DROP TABLE xx }
} -test {
  faultsim_test_result [list 0 {}]
}

#-------------------------------------------------------------------------
# An OOM within an "ORDER BY rank" query.
#
db func rnddoc fts5_rnddoc 
do_execsql_test 2.0 {
  CREATE VIRTUAL TABLE xx USING fts5(x);
  INSERT INTO xx VALUES ('abc ' || rnddoc(10));
  INSERT INTO xx VALUES ('abc abc' || rnddoc(9));
  INSERT INTO xx VALUES ('abc abc abc' || rnddoc(8));
} {}
faultsim_save_and_close

do_faultsim_test 2 -faults oom-* -prep {
  faultsim_restore_and_reopen
  execsql { SELECT * FROM xx }
} -body {
  execsql { SELECT rowid FROM xx WHERE xx MATCH 'abc' ORDER BY rank }
} -test {
  faultsim_test_result [list 0 {3 2 1}]
}

#-------------------------------------------------------------------------
# An OOM while "reseeking" an FTS cursor.
#
do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE jj USING fts5(j);
  INSERT INTO jj(rowid, j) VALUES(101, 'm t w t f s s');
  INSERT INTO jj(rowid, j) VALUES(202, 't w t f s');
  INSERT INTO jj(rowid, j) VALUES(303, 'w t f');
  INSERT INTO jj(rowid, j) VALUES(404, 't');
}
faultsim_save_and_close

do_faultsim_test 3 -faults oom-* -prep {
  faultsim_restore_and_reopen
  execsql { SELECT * FROM jj }
} -body {
  set res [list]
  db eval { SELECT rowid FROM jj WHERE jj MATCH 't' } {
    lappend res $rowid
    if {$rowid==303} {
      execsql { DELETE FROM jj WHERE rowid=404 }
    }
  }
  set res
} -test {
  faultsim_test_result [list 0 {101 202 303}]
}

#-------------------------------------------------------------------------
# An OOM within a special "*reads" query.
#
reset_db
db func rnddoc fts5_rnddoc
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE x1 USING fts5(x);
  INSERT INTO x1(x1, rank) VALUES('pgsz', 32);

  WITH ii(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10 )
  INSERT INTO x1 SELECT rnddoc(5) FROM ii;
}

set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}]

do_faultsim_test 4 -faults oom-* -body {
  db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'}
} -test {
  faultsim_test_result {0 {0 {} 3}}
}

#-------------------------------------------------------------------------
# An OOM within a query that uses a custom rank function.
#
reset_db
do_execsql_test 5.0 {
  PRAGMA encoding='utf16';
  CREATE VIRTUAL TABLE x2 USING fts5(x);
  INSERT INTO x2(rowid, x) VALUES(10, 'a b c'); -- 3
  INSERT INTO x2(rowid, x) VALUES(20, 'a b c'); -- 6
  INSERT INTO x2(rowid, x) VALUES(30, 'a b c'); -- 2
  INSERT INTO x2(rowid, x) VALUES(40, 'a b c'); -- 5
  INSERT INTO x2(rowid, x) VALUES(50, 'a b c'); -- 1
}

proc rowidmod {cmd mod} { 
  set row [$cmd xRowid]
  expr {$row % $mod}
}
sqlite3_fts5_create_function db rowidmod rowidmod

do_faultsim_test 5.1 -faults oom-* -body {
  db eval {
    SELECT rowid || '-' || rank FROM x2 WHERE x2 MATCH 'b' AND 
    rank MATCH "rowidmod('7')" ORDER BY rank
  }
} -test {
  faultsim_test_result {0 {50-1 30-2 10-3 40-5 20-6}}
}

proc rowidprefix {cmd prefix} { 
  set row [$cmd xRowid]
  set {} "${row}-${prefix}"
}
sqlite3_fts5_create_function db rowidprefix rowidprefix

set str [string repeat abcdefghijklmnopqrstuvwxyz 10]
do_faultsim_test 5.2 -faults oom-* -body {
  db eval "
    SELECT rank, x FROM x2 WHERE x2 MATCH 'b' AND 
    rank MATCH 'rowidprefix(''$::str'')'
    LIMIT 1
  "
} -test {
  faultsim_test_result "0 {10-$::str {a b c}}"
}

}


#-------------------------------------------------------------------------
# OOM errors within auxiliary functions.
#
reset_db
do_execsql_test 6.0 {
  CREATE VIRTUAL TABLE x3 USING fts5(xxx);
  INSERT INTO x3 VALUES('a b c d c b a');
}

do_faultsim_test 6.1 -faults oom-t* -body {
  db eval { SELECT highlight(x3, 0, '*', '*') FROM x3 WHERE x3 MATCH 'c' }
} -test {
  faultsim_test_result {0 {{a b *c* d *c* b a}}}
}

proc firstinst {cmd} { 
  foreach {p c o} [$cmd xInst 0] {}
  expr $c*100 + $o
}
sqlite3_fts5_create_function db firstinst firstinst

do_faultsim_test 6.2 -faults oom-t* -body {
  db eval { SELECT firstinst(x3) FROM x3 WHERE x3 MATCH 'c' }
} -test {
  faultsim_test_result {0 2} {1 SQLITE_NOMEM}
}




finish_test

# 2014 Dec 20
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file focuses on testing the planner (xBestIndex function).
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5plan

do_execsql_test 1.0 {
  CREATE TABLE t1(x, y);
  CREATE VIRTUAL TABLE f1 USING fts5(ff);
}

do_eqp_test 1.1 {
  SELECT * FROM t1, f1 WHERE f1 MATCH t1.x
} {
  0 0 0 {SCAN TABLE t1} 
  0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
}

do_eqp_test 1.2 {
  SELECT * FROM t1, f1 WHERE f1 > t1.x
} {
  0 0 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
  0 1 0 {SCAN TABLE t1} 
}

do_eqp_test 1.3 {
  SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff
} {
  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
}

do_eqp_test 1.4 {
  SELECT * FROM f1 ORDER BY rank
} {
  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
}

do_eqp_test 1.5 {
  SELECT * FROM f1 WHERE rank MATCH ?
} {
  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
}




finish_test

# 2014 Dec 20
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file focuses on testing queries that use the "rank" column.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5rank


#-------------------------------------------------------------------------
# "ORDER BY rank" + highlight() + large poslists.
#
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE xyz USING fts5(z);
}
do_test 1.1 {
  set doc [string trim [string repeat "x y " 500]]
  execsql { INSERT INTO xyz VALUES($doc) }
} {}
do_execsql_test 1.2 {
  SELECT highlight(xyz, 0, '[', ']') FROM xyz WHERE xyz MATCH 'x' ORDER BY rank
} [list [string map {x [x]} $doc]]

do_execsql_test 1.3 {
  SELECT highlight(xyz, 0, '[', ']') FROM xyz
  WHERE xyz MATCH 'x AND y' ORDER BY rank
} [list [string map {x [x] y [y]} $doc]]

finish_test

} {1 {database disk image is malformed}}

do_execsql_test 1.7 {
  INSERT INTO f1(f1) VALUES('rebuild');
  INSERT INTO f1(f1) VALUES('integrity-check');
} {}












finish_test

} {1 {database disk image is malformed}}

do_execsql_test 1.7 {
  INSERT INTO f1(f1) VALUES('rebuild');
  INSERT INTO f1(f1) VALUES('integrity-check');
} {}


#-------------------------------------------------------------------------
# Check that 'rebuild' may not be used with a contentless table.
#
do_execsql_test 2.1 {
  CREATE VIRTUAL TABLE nc USING fts5(doc, content=);
}

do_catchsql_test 2.2 {
  INSERT INTO nc(nc) VALUES('rebuild');
} {1 {'rebuild' may not be used with a contentless fts5 table}}
finish_test

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5restart

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE f1 USING fts5(ff);
}





do_test 1.1 {
  for {set i 1} {$i < 1000} {incr i} {
    execsql { INSERT INTO f1 VALUES('a b c d e') }
    lappend lRowid $i
  }
} {}

do_execsql_test 1.2 {
  SELECT rowid FROM f1 WHERE f1 MATCH 'c';
} $lRowid

breakpoint
do_test 1.3 {
  set res [list]
  db eval { SELECT rowid FROM f1 WHERE f1 MATCH 'c' } {
    if {$rowid == 100} {
      execsql { INSERT INTO f1(f1) VALUES('optimize') }
    }
    lappend res $rowid
  }
  set res
} $lRowid


































































































finish_test

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5restart

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE f1 USING fts5(ff);
}

#-------------------------------------------------------------------------
# Run the 'optimize' command. Check that it does not disturb ongoing
# full-text queries.
#
do_test 1.1 {
  for {set i 1} {$i < 1000} {incr i} {
    execsql { INSERT INTO f1 VALUES('a b c d e') }
    lappend lRowid $i
  }
} {}

do_execsql_test 1.2 {
  SELECT rowid FROM f1 WHERE f1 MATCH 'c';
} $lRowid


do_test 1.3 {
  set res [list]
  db eval { SELECT rowid FROM f1 WHERE f1 MATCH 'c' } {
    if {$rowid == 100} {
      execsql { INSERT INTO f1(f1) VALUES('optimize') }
    }
    lappend res $rowid
  }
  set res
} $lRowid

do_test 1.4.1 {
  sqlite3 db2 test.db
  set res [list]
  db2 eval { SELECT rowid FROM f1 WHERE f1 MATCH 'c' } {
    if {$rowid == 100} {
      set cres [catchsql { INSERT INTO f1(f1) VALUES('optimize') }]
    }
    lappend res $rowid
  }
  set res
} $lRowid

do_test 1.4.2 {
  db2 close
  set cres
} {1 {database is locked}}

#-------------------------------------------------------------------------
# Open a couple of cursors. Then close them in the same order.
#
do_test 2.1 {
  set ::s1 [sqlite3_prepare db "SELECT rowid FROM f1 WHERE f1 MATCH 'b'" -1 X]
  set ::s2 [sqlite3_prepare db "SELECT rowid FROM f1 WHERE f1 MATCH 'c'" -1 X]

  sqlite3_step $::s1
} {SQLITE_ROW}
do_test 2.2 {
  sqlite3_step $::s2
} {SQLITE_ROW}

do_test 2.1 {
  sqlite3_finalize $::s1
  sqlite3_finalize $::s2
} {SQLITE_OK}

#-------------------------------------------------------------------------
# Copy data between two FTS5 tables.
#
do_execsql_test 3.1 {
  CREATE VIRTUAL TABLE f2 USING fts5(gg);
  INSERT INTO f2 SELECT ff FROM f1 WHERE f1 MATCH 'b+c+d';
}
do_execsql_test 3.2 {
  SELECT rowid FROM f2 WHERE f2 MATCH 'a+b+c+d+e'
} $lRowid

#-------------------------------------------------------------------------
# Remove the row that an FTS5 cursor is currently pointing to. And 
# various other similar things. Check that this does not disturb 
# ongoing scans.
#
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE n4 USING fts5(n);
  INSERT INTO n4(rowid, n) VALUES(100, '1 2 3 4 5');
  INSERT INTO n4(rowid, n) VALUES(200, '1 2 3 4');
  INSERT INTO n4(rowid, n) VALUES(300, '2 3 4');
  INSERT INTO n4(rowid, n) VALUES(400, '2 3');
  INSERT INTO n4(rowid, n) VALUES(500, '3');
}

do_test 4.1 {
  set res [list]
  db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' } {
    if {$rowid==300} {
      execsql { DELETE FROM n4 WHERE rowid=300 }
    }
    lappend res $rowid
  }
  set res
} {100 200 300 400 500}

do_test 4.2 {
  execsql { INSERT INTO n4(rowid, n) VALUES(300, '2 3 4') }
  set res [list]
  db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' ORDER BY rowid DESC} {
    if {$rowid==300} {
      execsql { DELETE FROM n4 WHERE rowid=300 }
    }
    lappend res $rowid
  }
  set res
} {500 400 300 200 100}

do_test 4.3 {
  execsql { INSERT INTO n4(rowid, n) VALUES(300, '2 3 4') }
  set res [list]
  db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' ORDER BY rowid DESC} {
    if {$rowid==300} {
      execsql { DELETE FROM n4  }
    }
    lappend res $rowid
  }
  set res
} {500 400 300}



finish_test

  DROP TABLE ft1;
}
do_execsql_test 1.4 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter ascii');
  DROP TABLE ft1;
}









do_execsql_test 2.0 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize=porter);
  INSERT INTO ft1 VALUES('embedded databases');
}
do_execsql_test 2.1 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'embedding' } 1
do_execsql_test 2.2 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'database' } 1
do_execsql_test 2.3 { 

  DROP TABLE ft1;
}
do_execsql_test 1.4 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter ascii');
  DROP TABLE ft1;
}

do_catchsql_test 1.5 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'nosuch');
} {1 {no such tokenizer: nosuch}}

do_catchsql_test 1.6 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter nosuch');
} {1 {error in tokenizer constructor}}

do_execsql_test 2.0 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize=porter);
  INSERT INTO ft1 VALUES('embedded databases');
}
do_execsql_test 2.1 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'embedding' } 1
do_execsql_test 2.2 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'database' } 1
do_execsql_test 2.3 {