SQLite

#define SQL_SELECT_LEVEL_COUNT        12
#define SQL_SELECT_SEGDIR_COUNT_MAX   13
#define SQL_DELETE_SEGDIR_BY_LEVEL    14
#define SQL_DELETE_SEGMENTS_RANGE     15
#define SQL_CONTENT_INSERT            16
#define SQL_GET_BLOCK                 17

/*
** This function is used to obtain an SQLite prepared statement handle
** for the statement identified by the second argument. If successful,
** *pp is set to the requested statement handle and SQLITE_OK returned.
** Otherwise, an SQLite error code is returned and *pp is set to 0.
**
** If argument apVal is not NULL, then it must point to an array with
** at least as many entries as the requested statement has bound 
** parameters. The values are bound to the statements parameters before
** returning.
*/
static int fts3SqlStmt(
  Fts3Table *p,                   /* Virtual table handle */
  int eStmt,                      /* One of the SQL_XXX constants above */
  sqlite3_stmt **pp,              /* OUT: Statement handle */
  sqlite3_value **apVal           /* Values to bind to statement */
){
  const char *azSql[] = {
/* 0  */  "DELETE FROM %Q.'%q_content' WHERE rowid = ?",
/* 1  */  "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)",
/* 2  */  "DELETE FROM %Q.'%q_content'",
/* 3  */  "DELETE FROM %Q.'%q_segments'",
/* 4  */  "DELETE FROM %Q.'%q_segdir'",

            "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC",

/* 12 */  "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
/* 13 */  "SELECT count(*), max(level) FROM %Q.'%q_segdir'",

/* 14 */  "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
/* 15 */  "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
/* 16 */  "INSERT INTO %Q.'%q_content' VALUES(%z)",
/* 17 */  "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?",

  };
  int rc = SQLITE_OK;
  sqlite3_stmt *pStmt;

  assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
  assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
  
  pStmt = p->aStmt[eStmt];
  if( !pStmt ){
    char *zSql;
    if( eStmt==SQL_CONTENT_INSERT ){
      int i;                      /* Iterator variable */  
      char *zVarlist;             /* The "?, ?, ..." string */
      zVarlist = (char *)sqlite3_malloc(2*p->nColumn+2);
      if( !zVarlist ){
        *pp = 0;
        return SQLITE_NOMEM;
      }
      zVarlist[0] = '?';
      zVarlist[p->nColumn*2+1] = '\0';
      for(i=1; i<=p->nColumn; i++){
        zVarlist[i*2-1] = ',';
        zVarlist[i*2] = '?';
      }
      zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, zVarlist);
    }else{
      zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
    }
    if( !zSql ){
      rc = SQLITE_NOMEM;
    }else{
      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
      sqlite3_free(zSql);
      assert( rc==SQLITE_OK || pStmt==0 );
      p->aStmt[eStmt] = pStmt;
    }
  }
  if( apVal ){
    int i;
    int nParam = sqlite3_bind_parameter_count(pStmt);
    for(i=0; rc==SQLITE_OK && i<nParam; i++){
      rc = sqlite3_bind_value(pStmt, i+1, apVal[i]);
    }
  }
  *pp = pStmt;
  return rc;
}

/*
** Similar to fts3SqlStmt(). Except, after binding the parameters in
** array apVal[] to the SQL statement identified by eStmt, the statement
** is executed.
**
** Returns SQLITE_OK if the statement is successfully executed, or an
** SQLite error code otherwise.
*/
static int fts3SqlExec(Fts3Table *p, int eStmt, sqlite3_value **apVal){
  sqlite3_stmt *pStmt;
  int rc = fts3SqlStmt(p, eStmt, &pStmt, apVal); 
  if( rc==SQLITE_OK ){
    sqlite3_step(pStmt);
    rc = sqlite3_reset(pStmt);
  }
  return rc;
}


/*
** Read a single block from the %_segments table. If the specified block
** does not exist, return SQLITE_CORRUPT. If some other error (malloc, IO 
** etc.) occurs, return the appropriate SQLite error code.
**
** Otherwise, if successful, set *pzBlock to point to a buffer containing

**   4: root
*/
int sqlite3Fts3AllSegdirs(Fts3Table *p, sqlite3_stmt **ppStmt){
  return fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, ppStmt, 0);
}





/*
** Append a single varint to a PendingList buffer. SQLITE_OK is returned
** if successful, or an SQLite error code otherwise.
**
** This function also serves to allocate the PendingList structure itself.
** For example, to create a new PendingList structure containing two

** varints:

**
**   PendingList *p = 0;
**   fts3PendingListAppendVarint(&p, 1);
**   fts3PendingListAppendVarint(&p, 2);
*/
static int fts3PendingListAppendVarint(
  PendingList **pp,               /* IN/OUT: Pointer to PendingList struct */
  sqlite3_int64 i                 /* Value to append to data */
){
  PendingList *p = *pp;

  /* Allocate or grow the PendingList as required. */

  /* Append the new serialized varint to the end of the list. */
  p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i);
  p->aData[p->nData] = '\0';
  *pp = p;
  return SQLITE_OK;
}

/*
** Add a docid/column/position entry to a PendingList structure. Non-zero
** is returned if the structure is sqlite3_realloced as part of adding
** the entry. Otherwise, zero.
**
** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning.
** Zero is always returned in this case. Otherwise, if no OOM error occurs,
** it is set to SQLITE_OK.
*/
static int fts3PendingListAppend(
  PendingList **pp,               /* IN/OUT: PendingList structure */
  sqlite3_int64 iDocid,           /* Docid for entry to add */
  sqlite3_int64 iCol,             /* Column for entry to add */
  sqlite3_int64 iPos,             /* Position of term for entry to add */
  int *pRc                        /* OUT: Return code */
){
  PendingList *p = *pp;
  int rc = SQLITE_OK;

  assert( !p || p->iLastDocid<=iDocid );

  if( !p || p->iLastDocid!=iDocid ){

  if( p!=*pp ){
    *pp = p;
    return 1;
  }
  return 0;
}

/*
** Tokenize the nul-terminated string zText and add all tokens to the
** pending-terms hash-table. The docid used is that currently stored in
** p->iPrevDocid, and the column is specified by argument iCol.
**
** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
*/
static int fts3PendingTermsAdd(Fts3Table *p, const char *zText, int iCol){
  int rc;
  int iStart;
  int iEnd;
  int iPos;

  char const *zToken;

  sqlite3_int64 *piDocid          /* OUT: Docid for row just inserted */
){
  int rc;                         /* Return code */
  int i;                          /* Iterator variable */
  sqlite3_stmt *pContentInsert;   /* INSERT INTO %_content VALUES(...) */

  /* Locate the statement handle used to insert data into the %_content

  ** table. The SQL for this statement is:
  **
  **   INSERT INTO %_content VALUES(?, ?, ?, ...)
  **
  ** The statement features N '?' variables, where N is the number of user
  ** defined columns in the FTS3 table, plus one for the docid field.
  */




  rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]);



















  if( rc!=SQLITE_OK ){
    return rc;
  }





  /* There is a quirk here. The users INSERT statement may have specified
  ** a value for the "rowid" field, for the "docid" field, or for both.
  ** Which is a problem, since "rowid" and "docid" are aliases for the
  ** same value. For example:
  **
  **   INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2);
  **
  ** In FTS3, if a non-NULL docid value is specified, it is the value
  ** inserted. Otherwise, the rowid value is used.
  */




  if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){
    rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]);
    if( rc!=SQLITE_OK ) return rc;
  }

  /* Execute the statement to insert the record. Set *piDocid to the 
  ** new docid value. 

      }
    }
  }

  return sqlite3_reset(pSelect);
}

/*
** Forward declaration to account for the circular dependency between
** functions fts3SegmentMerge() and fts3AllocateSegdirIdx().
*/
static int fts3SegmentMerge(Fts3Table *, int);

/* 
** This function allocates a new level iLevel index in the segdir table.
** Usually, indexes are allocated within a level sequentially starting
** with 0, so the allocated index is one greater than the value returned
** by:

  pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
  assert( pNext<&pReader->aNode[pReader->nNode] );
  pReader->aDoclist = pNext;
  pReader->pOffsetList = 0;
  return SQLITE_OK;
}

/*
** Set the SegReader to point to the first docid in the doclist associated
** with the current term.
*/
static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){
  int n;
  assert( pReader->aDoclist );
  assert( !pReader->pOffsetList );
  n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
  pReader->pOffsetList = &pReader->aDoclist[n];
}

/*
** Advance the SegReader to point to the next docid in the doclist
** associated with the current term.
** 
** If arguments ppOffsetList and pnOffsetList are not NULL, then 
** *ppOffsetList is set to point to the first column-offset list
** in the doclist entry (i.e. immediately past the docid varint).
** *pnOffsetList is set to the length of the set of column-offset
** lists, not including the nul-terminator byte. For example:



*/
static void fts3SegReaderNextDocid(
  Fts3SegReader *pReader,
  char **ppOffsetList,
  int *pnOffsetList
){
  char *p = pReader->pOffsetList;

    sqlite3_int64 iDelta;
    pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
    pReader->iDocid += iDelta;
  }
}

/*
** Free all allocations associated with the iterator passed as the 
** second argument.
*/
void sqlite3Fts3SegReaderFree(Fts3Table *p, Fts3SegReader *pReader){
  if( pReader ){
    if( pReader->pStmt ){
      /* Move the leaf-range SELECT statement to the aLeavesStmt[] array,
      ** so that it can be reused when required by another query.
      */
      assert( p->nLeavesStmt<p->nLeavesTotal );
      sqlite3_reset(pReader->pStmt);
      p->aLeavesStmt[p->nLeavesStmt++] = pReader->pStmt;
    }
    sqlite3_free(pReader->zTerm);
    sqlite3_free(pReader);
  }

  if( nExtra ){
    /* The entire segment is stored in the root node. */
    pReader->aNode = (char *)&pReader[1];
    pReader->nNode = nRoot;
    memcpy(pReader->aNode, zRoot, nRoot);
  }else{
    sqlite3_stmt *pStmt;

    /* If the text of the SQL statement to iterate through a contiguous
    ** set of entries in the %_segments table has not yet been composed,
    ** compose it now.
    */
    if( !p->zSelectLeaves ){
      p->zSelectLeaves = sqlite3_mprintf(
          "SELECT block FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ? "
          "ORDER BY blockid", p->zDb, p->zName
      );
      if( !p->zSelectLeaves ){
        rc = SQLITE_NOMEM;
        goto finished;
      }
    }

    /* If there are no free statements in the aLeavesStmt[] array, prepare
    ** a new statement now. Otherwise, reuse a prepared statement from
    ** aLeavesStmt[].
    */
    if( p->nLeavesStmt==0 ){
      if( p->nLeavesTotal==p->nLeavesAlloc ){
        int nNew = p->nLeavesAlloc + 16;
        sqlite3_stmt **aNew = (sqlite3_stmt **)sqlite3_realloc(
            p->aLeavesStmt, nNew*sizeof(sqlite3_stmt *)
        );
        if( !aNew ){

      if( rc!=SQLITE_OK ){
        goto finished;
      }
      p->nLeavesTotal++;
    }else{
      pReader->pStmt = p->aLeavesStmt[--p->nLeavesStmt];
    }

    /* Bind the start and end leaf blockids to the prepared SQL statement. */
    sqlite3_bind_int64(pReader->pStmt, 1, iStartLeaf);
    sqlite3_bind_int64(pReader->pStmt, 2, iEndLeaf);
  }
  rc = fts3SegReaderNext(pReader);

 finished:
  if( rc==SQLITE_OK ){

      Fts3SegReader *p0 = apSegment[0];
      rc = xFunc(p, pContext, zTerm, nTerm, p0->aDoclist, p0->nDoclist);
      if( rc!=SQLITE_OK ) goto finished;
    }else{
      int nDoclist = 0;           /* Size of doclist */
      sqlite3_int64 iPrev = 0;    /* Previous docid stored in doclist */

      /* The current term of the first nMerge entries in the array
      ** of Fts3SegReader objects is the same. The doclists must be merged
      ** and a single term added to the new segment.
      */
      for(i=0; i<nMerge; i++){
        fts3SegReaderFirstDocid(apSegment[i]);
      }
      fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderCmp2);