if( rc==SQLITE_OK ){
        rc = SQLITE_CORRUPT_VTAB;
      }
    }
  }
  return rc;
}










/*
** This function is called to handle an FTS INSERT command. In other words,
** an INSERT statement of the form:
**
**     INSERT INTO fts(fts) VALUES($pCmd)
**     INSERT INTO fts(fts, rank) VALUES($pCmd, $pVal)
................................................................................
** more commands are added to this function.
*/
static int fts5SpecialInsert(
  Fts5Table *pTab,                /* Fts5 table object */
  sqlite3_value *pCmd,            /* Value inserted into special column */
  sqlite3_value *pVal             /* Value inserted into rowid column */
){

  const char *z = (const char*)sqlite3_value_text(pCmd);
  int rc = SQLITE_OK;
  int bError = 0;




















  if( 0==sqlite3_stricmp("integrity-check", z) ){
    rc = sqlite3Fts5StorageIntegrity(pTab->pStorage);
  }else{
    rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, z, pVal, &bError);
    if( rc==SQLITE_OK ){
      if( bError ){
        rc = SQLITE_ERROR;
      }else{

      if( rc==SQLITE_OK ){
        rc = SQLITE_CORRUPT_VTAB;
      }
    }
  }
  return rc;
}

static void fts5SetVtabError(Fts5Table *p, const char *zFormat, ...){
  int rc;
  va_list ap;                     /* ... printf arguments */
  va_start(ap, zFormat);
  assert( p->base.zErrMsg==0 );
  p->base.zErrMsg = sqlite3_vmprintf(zFormat, ap);
  va_end(ap);
}

/*
** This function is called to handle an FTS INSERT command. In other words,
** an INSERT statement of the form:
**
**     INSERT INTO fts(fts) VALUES($pCmd)
**     INSERT INTO fts(fts, rank) VALUES($pCmd, $pVal)
................................................................................
** more commands are added to this function.
*/
static int fts5SpecialInsert(
  Fts5Table *pTab,                /* Fts5 table object */
  sqlite3_value *pCmd,            /* Value inserted into special column */
  sqlite3_value *pVal             /* Value inserted into rowid column */
){
  Fts5Config *pConfig = pTab->pConfig;
  const char *z = (const char*)sqlite3_value_text(pCmd);
  int rc = SQLITE_OK;
  int bError = 0;

  if( 0==sqlite3_stricmp("delete-all", z) ){
    if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
      fts5SetVtabError(pTab, 
          "'delete-all' may only be used with a "
          "contentless or external content fts5 table"
      );
      rc = SQLITE_ERROR;
    }else{
      rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage);
    }
  }else if( 0==sqlite3_stricmp("rebuild", z) ){
    if( pConfig->eContent==FTS5_CONTENT_NONE ){
      fts5SetVtabError(pTab, 
          "'rebuild' may not be used with a contentless fts5 table"
      );
      rc = SQLITE_ERROR;
    }else{
      rc = sqlite3Fts5StorageRebuild(pTab->pStorage);
    }
  }else if( 0==sqlite3_stricmp("integrity-check", z) ){
    rc = sqlite3Fts5StorageIntegrity(pTab->pStorage);
  }else{
    rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, z, pVal, &bError);
    if( rc==SQLITE_OK ){
      if( bError ){
        rc = SQLITE_ERROR;
      }else{

**   output by xInstCount().
**
**   Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) 
**   if an error occurs.
**
** xRowid:
**   Returns the rowid of the current row.
**
** xPoslist:
**   Iterate through phrase instances in the current row. If the iPhrase
**   argument is 0 or greater, then only instances of phrase iPhrase are
**   visited. If it is less than 0, instances of all phrases are visited.
**
**   At EOF, -1 is returned and output variable iPos set to -1.
**
**     </pre>
**       sqlite3_int64 iPos;
**       int iPhrase;
**       int ii = 0;
**
**       while( (iPhrase = pFts->xPoslist(pFts, -1, &ii, &iPos) >= 0 ){
**         int iCol = FTS5_POS2COLUMN(iPos);
**         int iOff = FTS5_POS2OFFSET(iPos);
**         // An instance of phrase iPhrase at offset iOff of column iCol.
**       }
**     </pre>
**
**
** xTokenize:
**   Tokenize text using the tokenizer belonging to the FTS5 table.
**
**
** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback):
**   This API function is used to query the FTS table for phrase iPhrase
**   of the current query. Specifically, a query equivalent to:
**
**       ... FROM ftstable WHERE ftstable MATCH $p ORDER BY DESC
**

**   output by xInstCount().
**
**   Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) 
**   if an error occurs.
**
** xRowid:
**   Returns the rowid of the current row.




















**
** xTokenize:
**   Tokenize text using the tokenizer belonging to the FTS5 table.

**
** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback):
**   This API function is used to query the FTS table for phrase iPhrase
**   of the current query. Specifically, a query equivalent to:
**
**       ... FROM ftstable WHERE ftstable MATCH $p ORDER BY DESC
**

/*
** Return the total number of entries read from the %_data table by 
** this connection since it was created.
*/
int sqlite3Fts5IndexReads(Fts5Index *p);



/*
** End of interface to code in fts5_index.c.
**************************************************************************/

/**************************************************************************
** Interface to code in fts5_hash.c. 
*/
................................................................................

int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
int sqlite3Fts5StorageRollback(Fts5Storage *p);

int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*);

int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**);




/*
** End of interface to code in fts5_storage.c.
**************************************************************************/


/**************************************************************************

/*
** Return the total number of entries read from the %_data table by 
** this connection since it was created.
*/
int sqlite3Fts5IndexReads(Fts5Index *p);

int sqlite3Fts5IndexReinit(Fts5Index *p);

/*
** End of interface to code in fts5_index.c.
**************************************************************************/

/**************************************************************************
** Interface to code in fts5_hash.c. 
*/
................................................................................

int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
int sqlite3Fts5StorageRollback(Fts5Storage *p);

int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*);

int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**);

int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
int sqlite3Fts5StorageRebuild(Fts5Storage *p);

/*
** End of interface to code in fts5_storage.c.
**************************************************************************/


/**************************************************************************

*/
int sqlite3Fts5IndexRollback(Fts5Index *p){
  fts5CloseReader(p);
  fts5IndexDiscardData(p);
  assert( p->rc==SQLITE_OK );
  return SQLITE_OK;
}





















/*
** Open a new Fts5Index handle. If the bCreate argument is true, create
** and initialize the underlying %_data table.
**
** If successful, set *pp to point to the new object and return SQLITE_OK.
** Otherwise, set *pp to NULL and return an SQLite error code.
................................................................................
  p->nCrisisMerge = FTS5_CRISIS_MERGE;
  p->nWorkUnit = FTS5_WORK_UNIT;
  p->nMaxPendingData = 1024*1024;
  p->zDataTbl = sqlite3_mprintf("%s_data", pConfig->zName);
  if( p->zDataTbl==0 ){
    rc = SQLITE_NOMEM;
  }else if( bCreate ){
    int i;
    Fts5Structure s;
    rc = sqlite3Fts5CreateTable(
        pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr
    );
    if( rc==SQLITE_OK ){
      memset(&s, 0, sizeof(Fts5Structure));
      for(i=0; i<pConfig->nPrefix+1; i++){
        fts5StructureWrite(p, i, &s);
      }
      rc = p->rc;
    }
    if( rc==SQLITE_OK ){
      rc = sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0);
    }
  }

  assert( p->rc==SQLITE_OK || rc!=SQLITE_OK );
  if( rc ){
    sqlite3Fts5IndexClose(p, 0);
    *pp = 0;

*/
int sqlite3Fts5IndexRollback(Fts5Index *p){
  fts5CloseReader(p);
  fts5IndexDiscardData(p);
  assert( p->rc==SQLITE_OK );
  return SQLITE_OK;
}

/*
** The %_data table is completely empty when this function is called. This
** function populates it with the initial structure objects for each index,
** and the initial version of the "averages" record (a zero-byte blob).
*/
int sqlite3Fts5IndexReinit(Fts5Index *p){
  int i;
  Fts5Structure s;

  memset(&s, 0, sizeof(Fts5Structure));
  for(i=0; i<p->pConfig->nPrefix+1; i++){
    fts5StructureWrite(p, i, &s);
  }
  if( p->rc==SQLITE_OK ){
    p->rc = sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0);
  }

  return fts5IndexReturn(p);
}

/*
** Open a new Fts5Index handle. If the bCreate argument is true, create
** and initialize the underlying %_data table.
**
** If successful, set *pp to point to the new object and return SQLITE_OK.
** Otherwise, set *pp to NULL and return an SQLite error code.
................................................................................
  p->nCrisisMerge = FTS5_CRISIS_MERGE;
  p->nWorkUnit = FTS5_WORK_UNIT;
  p->nMaxPendingData = 1024*1024;
  p->zDataTbl = sqlite3_mprintf("%s_data", pConfig->zName);
  if( p->zDataTbl==0 ){
    rc = SQLITE_NOMEM;
  }else if( bCreate ){


    rc = sqlite3Fts5CreateTable(
        pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr
    );
    if( rc==SQLITE_OK ){







      rc = sqlite3Fts5IndexReinit(p);
    }
  }

  assert( p->rc==SQLITE_OK || rc!=SQLITE_OK );
  if( rc ){
    sqlite3Fts5IndexClose(p, 0);
    *pp = 0;

    }
  }

  *ppStmt = p->aStmt[eStmt];
  return rc;
}
























/*
** Drop the shadow table with the postfix zPost (e.g. "content"). Return
** SQLITE_OK if successful or an SQLite error code otherwise.
*/
int sqlite3Fts5DropTable(Fts5Config *pConfig, const char *zPost){
  int rc;
  char *zSql = sqlite3_mprintf("DROP TABLE IF EXISTS %Q.'%q_%q'",
      pConfig->zDb, pConfig->zName, zPost
  );
  if( zSql==0 ){
    rc = SQLITE_NOMEM;
  }else{
    rc = sqlite3_exec(pConfig->db, zSql, 0, 0, 0);
    sqlite3_free(zSql);
  }
  return rc;
}

/*
** Create the shadow table named zPost, with definition zDefn. Return
** SQLITE_OK if successful, or an SQLite error code otherwise.
*/
int sqlite3Fts5CreateTable(
................................................................................
  Fts5Config *pConfig,            /* FTS5 configuration */
  const char *zPost,              /* Shadow table to create (e.g. "content") */
  const char *zDefn,              /* Columns etc. for shadow table */
  int bWithout,                   /* True for without rowid */
  char **pzErr                    /* OUT: Error message */
){
  int rc;


  char *zSql = sqlite3_mprintf("CREATE TABLE %Q.'%q_%q'(%s)%s",
      pConfig->zDb, pConfig->zName, zPost, zDefn, 
      (bWithout ? " WITHOUT ROWID" :"")
  );
  if( zSql==0 ){
    rc = SQLITE_NOMEM;
  }else{
    char *zErr = 0;
    assert( *pzErr==0 );
    rc = sqlite3_exec(pConfig->db, zSql, 0, 0, &zErr);
    if( zErr ){
      *pzErr = sqlite3_mprintf(
          "fts5: error creating shadow table %q_%s: %s", 
          pConfig->zName, zPost, zErr
      );
      sqlite3_free(zErr);
    }
    sqlite3_free(zSql);
  }
  return rc;
}

/*
** Open a new Fts5Index handle. If the bCreate argument is true, create
** and initialize the underlying tables 
**
................................................................................
  i64 iDel, 
  sqlite3_value **apVal
){
  Fts5Config *pConfig = p->pConfig;
  int rc;
  sqlite3_stmt *pDel;

  assert( p->pConfig->eContent!=FTS5_CONTENT_NORMAL );
  rc = fts5StorageLoadTotals(p, 1);

  /* Delete the index records */
  if( rc==SQLITE_OK ){
    int iCol;
    Fts5InsertCtx ctx;
    ctx.pStorage = p;
................................................................................

  /* Write the averages record */
  if( rc==SQLITE_OK ){
    rc = fts5StorageSaveTotals(p);
  }

  return rc;








































































}

/*
** Allocate a new rowid. This is used for "external content" tables when
** a NULL value is inserted into the rowid column. The new rowid is allocated
** by inserting a dummy row into the %_docsize table. The dummy will be
** overwritten later.

    }
  }

  *ppStmt = p->aStmt[eStmt];
  return rc;
}


static int fts5ExecPrintf(
  sqlite3 *db,
  char **pzErr,
  const char *zFormat,
  ...
){
  int rc;
  va_list ap;                     /* ... printf arguments */
  va_start(ap, zFormat);
  char *zSql = sqlite3_vmprintf(zFormat, ap);

  if( zSql==0 ){
    rc = SQLITE_NOMEM;
  }else{
    rc = sqlite3_exec(db, zSql, 0, 0, pzErr);
    sqlite3_free(zSql);
  }

  va_end(ap);
  return rc;
}

/*
** Drop the shadow table with the postfix zPost (e.g. "content"). Return
** SQLITE_OK if successful or an SQLite error code otherwise.
*/
int sqlite3Fts5DropTable(Fts5Config *pConfig, const char *zPost){

  return fts5ExecPrintf(pConfig->db, 0, "DROP TABLE IF EXISTS %Q.'%q_%q'",
      pConfig->zDb, pConfig->zName, zPost
  );







}

/*
** Create the shadow table named zPost, with definition zDefn. Return
** SQLITE_OK if successful, or an SQLite error code otherwise.
*/
int sqlite3Fts5CreateTable(
................................................................................
  Fts5Config *pConfig,            /* FTS5 configuration */
  const char *zPost,              /* Shadow table to create (e.g. "content") */
  const char *zDefn,              /* Columns etc. for shadow table */
  int bWithout,                   /* True for without rowid */
  char **pzErr                    /* OUT: Error message */
){
  int rc;
  char *zErr = 0;

  rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s",
      pConfig->zDb, pConfig->zName, zPost, zDefn, bWithout?" WITHOUT ROWID":""

  );






  if( zErr ){
    *pzErr = sqlite3_mprintf(
        "fts5: error creating shadow table %q_%s: %s", 
        pConfig->zName, zPost, zErr
    );
    sqlite3_free(zErr);
  }


  return rc;
}

/*
** Open a new Fts5Index handle. If the bCreate argument is true, create
** and initialize the underlying tables 
**
................................................................................
  i64 iDel, 
  sqlite3_value **apVal
){
  Fts5Config *pConfig = p->pConfig;
  int rc;
  sqlite3_stmt *pDel;

  assert( pConfig->eContent!=FTS5_CONTENT_NORMAL );
  rc = fts5StorageLoadTotals(p, 1);

  /* Delete the index records */
  if( rc==SQLITE_OK ){
    int iCol;
    Fts5InsertCtx ctx;
    ctx.pStorage = p;
................................................................................

  /* Write the averages record */
  if( rc==SQLITE_OK ){
    rc = fts5StorageSaveTotals(p);
  }

  return rc;
}

/*
** Delete all entries in the FTS5 index.
*/
int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){
  Fts5Config *pConfig = p->pConfig;
  int rc;

  /* Delete the contents of the %_data and %_docsize tables. */
  rc = fts5ExecPrintf(pConfig->db, 0,
      "DELETE FROM %Q.'%q_data';"
      "DELETE FROM %Q.'%q_docsize';",
      pConfig->zDb, pConfig->zName,
      pConfig->zDb, pConfig->zName
  );

  /* Reinitialize the %_data table. This call creates the initial structure
  ** and averages records.  */
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5IndexReinit(p->pIndex);
  }
  return rc;
}

int sqlite3Fts5StorageRebuild(Fts5Storage *p){
  Fts5Buffer buf = {0,0,0};
  Fts5Config *pConfig = p->pConfig;
  sqlite3_stmt *pScan = 0;
  Fts5InsertCtx ctx;
  int rc;

  memset(&ctx, 0, sizeof(Fts5InsertCtx));
  ctx.pStorage = p;
  rc = sqlite3Fts5StorageDeleteAll(p);
  if( rc==SQLITE_OK ){
    rc = fts5StorageLoadTotals(p, 1);
  }

  if( rc==SQLITE_OK ){
    rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN_ASC, &pScan, 0);
  }

  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){
    i64 iRowid = sqlite3_column_int64(pScan, 0);

    sqlite3Fts5BufferZero(&buf);
    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iRowid);
    for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
      ctx.szCol = 0;
      rc = sqlite3Fts5Tokenize(pConfig, 
          (const char*)sqlite3_column_text(pScan, ctx.iCol+1),
          sqlite3_column_bytes(pScan, ctx.iCol+1),
          (void*)&ctx,
          fts5StorageInsertCallback
      );
      sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
      p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
    }
    p->nTotalRow++;

    if( rc==SQLITE_OK ){
      rc = fts5StorageInsertDocsize(p, iRowid, &buf);
    }
  }
  sqlite3_free(buf.p);

  /* Write the averages record */
  if( rc==SQLITE_OK ){
    rc = fts5StorageSaveTotals(p);
  }
  return rc;
}

/*
** Allocate a new rowid. This is used for "external content" tables when
** a NULL value is inserted into the rowid column. The new rowid is allocated
** by inserting a dummy row into the %_docsize table. The dummy will be
** overwritten later.

} {three {t h r e e x y z}}

do_execsql_test 2.7 {
  SELECT highlight(fts_idx, 1, '[', ']') FROM fts_idx 
  WHERE fts_idx MATCH 't AND x';
} {{[t] h r e e [x] y z}}











































finish_test

} {three {t h r e e x y z}}

do_execsql_test 2.7 {
  SELECT highlight(fts_idx, 1, '[', ']') FROM fts_idx 
  WHERE fts_idx MATCH 't AND x';
} {{[t] h r e e [x] y z}}

#-------------------------------------------------------------------------
# Quick tests of the 'delete-all' command.
#
do_execsql_test 3.1 {
  CREATE VIRTUAL TABLE t3 USING fts5(x, content='');
  INSERT INTO t3 VALUES('a b c');
  INSERT INTO t3 VALUES('d e f');
}

do_execsql_test 3.2 {
  SELECT count(*) FROM t3_docsize;
  SELECT count(*) FROM t3_data;
} {2 4}

do_execsql_test 3.3 {
  INSERT INTO t3(t3) VALUES('delete-all');
  SELECT count(*) FROM t3_docsize;
  SELECT count(*) FROM t3_data;
} {0 2}

do_execsql_test 3.4 {
  INSERT INTO t3 VALUES('a b c');
  INSERT INTO t3 VALUES('d e f');
  SELECT rowid FROM t3 WHERE t3 MATCH 'e';
} {2}

do_execsql_test 3.5 {
  SELECT rowid FROM t3 WHERE t3 MATCH 'c';
} {1}

do_execsql_test 3.6 {
  SELECT count(*) FROM t3_docsize;
  SELECT count(*) FROM t3_data;
} {2 4}

do_execsql_test 3.7 {
  CREATE VIRTUAL TABLE t4 USING fts5(x);
} {}
do_catchsql_test 3.8 {
  INSERT INTO t4(t4) VALUES('delete-all');
} {1 {'delete-all' may only be used with a contentless or external content fts5 table}}

finish_test

# 2014 Dec 20
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5rebuild

do_execsql_test 1.1 {
  CREATE VIRTUAL TABLE f1 USING fts5(a, b);
  INSERT INTO f1(a, b) VALUES('one',   'o n e');
  INSERT INTO f1(a, b) VALUES('two',   't w o');
  INSERT INTO f1(a, b) VALUES('three', 't h r e e');
}

do_execsql_test 1.2 {
  INSERT INTO f1(f1) VALUES('integrity-check');
} {}

do_execsql_test 1.3 {
  INSERT INTO f1(f1) VALUES('rebuild');
} {}

do_execsql_test 1.4 {
  INSERT INTO f1(f1) VALUES('integrity-check');
} {}

do_execsql_test 1.5 {
  DELETE FROM f1_data;
} {}

do_catchsql_test 1.6 {
  INSERT INTO f1(f1) VALUES('integrity-check');
} {1 {SQL logic error or missing database}}

do_execsql_test 1.7 {
  INSERT INTO f1(f1) VALUES('rebuild');
  INSERT INTO f1(f1) VALUES('integrity-check');
} {}

finish_test