SQLite

** to *pVal.
*/
static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){
  sqlite3_int64 iVal;
  *pp += sqlite3Fts3GetVarint(*pp, &iVal);
  *pVal += iVal;
}

/*
**
*/
static void fts3GetReverseDeltaVarint(
  char **pp, 
  char *pStart, 
  sqlite3_int64 *pVal
){
  sqlite3_int64 iVal;
  char *p = *pp;

  /* Pointer p now points at the first byte past the varint we are 
  ** interested in. So, unless the doclist is corrupt, the 0x80 bit is
  ** clear on character p[-1]. */
  for(p = (*pp)-2; p>=pStart && *p&0x80; p--);
  p++;
  *pp = p;

  sqlite3Fts3GetVarint(p, &iVal);
  *pVal -= iVal;
}

/*
** As long as *pp has not reached its end (pEnd), then do the same
** as fts3GetDeltaVarint(): read a single varint and add it to *pVal.
** But if we have reached the end of the varint, just set *pp=0 and
** leave *pVal unchanged.
*/

    }
  }

  if( iCons>=0 ){
    pInfo->aConstraintUsage[iCons].argvIndex = 1;
    pInfo->aConstraintUsage[iCons].omit = 1;
  } 

  /* Regardless of the strategy selected, FTS can deliver rows in rowid (or
  ** docid) order. Both ascending and descending are possible. 
  */
  if( pInfo->nOrderBy==1 ){
    struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0];
    if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){
      if( pOrder->desc ){
        pInfo->idxStr = "DESC";
      }else{
        pInfo->idxStr = "ASC";
      }
    }
    pInfo->orderByConsumed = 1;
  }

  return SQLITE_OK;
}

/*
** Implementation of xOpen method.
*/
static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){

      if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
        pCsr->isEof = 1;
        rc = sqlite3_reset(pCsr->pStmt);
        break;
      }
      pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
    }else{
      if( pCsr->desc==0 ){
        if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
          pCsr->isEof = 1;
          break;
        }

        fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
      }else{
        fts3GetReverseDeltaVarint(&pCsr->pNextId,pCsr->aDoclist,&pCsr->iPrevId);
        if( pCsr->pNextId<=pCsr->aDoclist ){
          pCsr->isEof = 1;
          break;
        }
      }
      sqlite3_reset(pCsr->pStmt);
      pCsr->isRequireSeek = 1;
      pCsr->isMatchinfoNeeded = 1;
    }
  }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 );

  return rc;
}

  sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
  int idxNum,                     /* Strategy index */
  const char *idxStr,             /* Unused */
  int nVal,                       /* Number of elements in apVal */
  sqlite3_value **apVal           /* Arguments for the indexing scheme */
){
  const char *azSql[] = {
    "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?",   /* non-full-scan */
    "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s", /* full-scan */
  };
  int rc;                         /* Return code */
  char *zSql;                     /* SQL statement used to access %_content */
  Fts3Table *p = (Fts3Table *)pCursor->pVtab;
  Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;

  UNUSED_PARAMETER(idxStr);

  /* Compile a SELECT statement for this cursor. For a full-table-scan, the
  ** statement loops through all rows of the %_content table. For a
  ** full-text query or docid lookup, the statement retrieves a single
  ** row by docid.
  */
  zSql = (char *)azSql[idxNum==FTS3_FULLSCAN_SEARCH];
  zSql = sqlite3_mprintf(
      zSql, p->zReadExprlist, p->zDb, p->zName, (idxStr ? idxStr : "ASC")
  );
  if( !zSql ){
    rc = SQLITE_NOMEM;
  }else{
    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
    sqlite3_free(zSql);
  }
  if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){
    rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
  }
  pCsr->eSearch = (i16)idxNum;

  assert( pCsr->desc==0 );
  if( rc!=SQLITE_OK ) return rc;
  if( rc==SQLITE_OK && pCsr->nDoclist>0 && idxStr && idxStr[0]=='D' ){
    sqlite3_int64 iDocid = 0;
    char *csr = pCsr->aDoclist;
    while( csr<&pCsr->aDoclist[pCsr->nDoclist] ){
      fts3GetDeltaVarint(&csr, &iDocid);
    }
    pCsr->pNextId = csr;
    pCsr->iPrevId = iDocid;
    pCsr->desc = 1;
    pCsr->isRequireSeek = 1;
    pCsr->isMatchinfoNeeded = 1;
    pCsr->eEvalmode = FTS3_EVAL_NEXT;
    return SQLITE_OK;
  }
  return fts3NextMethod(pCursor);
}

/* 
** This is the xEof method of the virtual table. SQLite calls this 
** routine to find out if it has reached the end of a result set.
*/

/*
** After ExprLoadDoclist() (see above) has been called, this function is
** used to iterate/search through the position lists that make up the doclist
** stored in pExpr->aDoclist.
*/
char *sqlite3Fts3FindPositions(
  Fts3Cursor *pCursor,            /* Associate FTS3 cursor */
  Fts3Expr *pExpr,                /* Access this expressions doclist */
  sqlite3_int64 iDocid,           /* Docid associated with requested pos-list */
  int iCol                        /* Column of requested pos-list */
){
  assert( pExpr->isLoaded );
  if( pExpr->aDoclist ){
    char *pEnd = &pExpr->aDoclist[pExpr->nDoclist];
    char *pCsr;

    if( pExpr->pCurrent==0 || pCursor->desc ){
      pExpr->pCurrent = pExpr->aDoclist;
      pExpr->iCurrent = 0;
      pExpr->pCurrent += sqlite3Fts3GetVarint(pExpr->pCurrent,&pExpr->iCurrent);
    }
    pCsr = pExpr->pCurrent;
    assert( pCsr );

  Fts3Expr *pExpr;                /* Parsed MATCH query string */
  int nPhrase;                    /* Number of matchable phrases in query */
  Fts3DeferredToken *pDeferred;   /* Deferred search tokens, if any */
  sqlite3_int64 iPrevId;          /* Previous id read from aDoclist */
  char *pNextId;                  /* Pointer into the body of aDoclist */
  char *aDoclist;                 /* List of docids for full-text queries */
  int nDoclist;                   /* Size of buffer at aDoclist */
  int desc;                       /* True to sort in descending order */
  int eEvalmode;                  /* An FTS3_EVAL_XX constant */
  int nRowAvg;                    /* Average size of database rows, in pages */

  int isMatchinfoNeeded;          /* True when aMatchinfo[] needs filling in */
  u32 *aMatchinfo;                /* Information about most recent match */
  int nMatchinfo;                 /* Number of elements in aMatchinfo[] */
  char *zMatchinfo;               /* Matchinfo specification */

/* fts3.c */
int sqlite3Fts3PutVarint(char *, sqlite3_int64);
int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
int sqlite3Fts3GetVarint32(const char *, int *);
int sqlite3Fts3VarintLen(sqlite3_uint64);
void sqlite3Fts3Dequote(char *);

char *sqlite3Fts3FindPositions(Fts3Cursor *, Fts3Expr *, sqlite3_int64, int);
int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *);
int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *);
int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);

/* fts3_tokenizer.c */
const char *sqlite3Fts3NextToken(const char *, int *);
int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);

static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
  SnippetIter *p = (SnippetIter *)ctx;
  SnippetPhrase *pPhrase = &p->aPhrase[iPhrase];
  char *pCsr;

  pPhrase->nToken = pExpr->pPhrase->nToken;

  pCsr = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->pCsr->iPrevId, p->iCol);
  if( pCsr ){
    int iFirst = 0;
    pPhrase->pList = pCsr;
    fts3GetDeltaPosition(&pCsr, &iFirst);
    pPhrase->pHead = pCsr;
    pPhrase->pTail = pCsr;
    pPhrase->iHead = iFirst;

  int i;

  for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0;

  if( pExpr->aDoclist ){
    char *pCsr;

    pCsr = sqlite3Fts3FindPositions(p->pCursor, pExpr, p->pCursor->iPrevId, -1);
    if( pCsr ){
      fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0);
    }
  }

  return SQLITE_OK;
}

  if( !aIter ) return SQLITE_NOMEM;
  memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
  (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
  for(i=0; i<pInfo->nPhrase; i++){
    LcsIterator *pIter = &aIter[i];
    nToken -= pIter->pExpr->pPhrase->nToken;
    pIter->iPosOffset = nToken;
    pIter->pRead = sqlite3Fts3FindPositions(pCsr,pIter->pExpr,pCsr->iPrevId,-1);
    if( pIter->pRead ){
      pIter->iPos = pIter->iPosOffset;
      fts3LcsIteratorAdvance(&aIter[i]);
    }else{
      pIter->iCol = LCS_ITERATOR_FINISHED;
    }
  }

struct TermOffset {
  char *pList;                    /* Position-list */
  int iPos;                       /* Position just read from pList */
  int iOff;                       /* Offset of this term from read positions */
};

struct TermOffsetCtx {
  Fts3Cursor *pCsr;
  int iCol;                       /* Column of table to populate aTerm for */
  int iTerm;
  sqlite3_int64 iDocid;
  TermOffset *aTerm;
};

/*
** This function is an fts3ExprIterate() callback used by sqlite3Fts3Offsets().
*/
static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
  TermOffsetCtx *p = (TermOffsetCtx *)ctx;
  int nTerm;                      /* Number of tokens in phrase */
  int iTerm;                      /* For looping through nTerm phrase terms */
  char *pList;                    /* Pointer to position list for phrase */
  int iPos = 0;                   /* First position in position-list */

  UNUSED_PARAMETER(iPhrase);
  pList = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->iDocid, p->iCol);
  nTerm = pExpr->pPhrase->nToken;
  if( pList ){
    fts3GetDeltaPosition(&pList, &iPos);
    assert( iPos>=0 );
  }

  for(iTerm=0; iTerm<nTerm; iTerm++){

  /* Allocate the array of TermOffset iterators. */
  sCtx.aTerm = (TermOffset *)sqlite3_malloc(sizeof(TermOffset)*nToken);
  if( 0==sCtx.aTerm ){
    rc = SQLITE_NOMEM;
    goto offsets_out;
  }
  sCtx.iDocid = pCsr->iPrevId;
  sCtx.pCsr = pCsr;

  /* Loop through the table columns, appending offset information to 
  ** string-buffer res for each column.
  */
  for(iCol=0; iCol<pTab->nColumn; iCol++){
    sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */
    int iStart;

/*
** 2011 Jan 27
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
** This file is not part of the production FTS code. It is only used for
** testing. It contains a virtual table implementation that provides direct 
** access to the full-text index of an FTS table. 
*/

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
#ifdef SQLITE_TEST

#include "fts3Int.h"
#include <string.h>

** xBestIndex - Analyze a WHERE and ORDER BY clause.
*/
static int fts3termBestIndexMethod(
  sqlite3_vtab *pVTab, 
  sqlite3_index_info *pInfo
){
  UNUSED_PARAMETER(pVTab);

  /* This vtab naturally does "ORDER BY term, docid, col, pos".  */
  if( pInfo->nOrderBy ){
    int i;
    for(i=0; i<pInfo->nOrderBy; i++){
      if( pInfo->aOrderBy[i].iColumn!=i || pInfo->aOrderBy[i].desc ) break;
    }
    if( i==pInfo->nOrderBy ){
      pInfo->orderByConsumed = 1;
    }
  }

  return SQLITE_OK;
}

/*
** xOpen - Open a cursor.
*/
static int fts3termOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){

# 2011 May 04
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the FTS3 module.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}

set testprefix fts3sort

proc build_database {nRow} {
  db close
  forcedelete test.db
  sqlite3 db test.db

  set vocab [list    aa ab ac   ba bb bc    ca cb cc   da]
  expr srand(0)

  execsql { CREATE VIRTUAL TABLE t1 USING fts4 }
  for {set i 0} {$i < $nRow} {incr i} {
    set v [expr int(rand()*1000000)]
    set doc [list]
    for {set div 1} {$div < 1000000} {set div [expr $div*10]} {
      lappend doc [lindex $vocab [expr ($v/$div) % 10]]
    }
    execsql { INSERT INTO t1 VALUES($doc) }
  }
}

set nRow 1000
do_test 1.0 {
  build_database $nRow
  execsql { SELECT count(*) FROM t1 }
} $nRow

foreach {tn query} {
  1   "SELECT docid, * FROM t1"
  2   "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa'"
  3   "SELECT docid, * FROM t1 WHERE t1 MATCH 'a*'"
  4   "SELECT docid, quote(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'a*'"
  5   "SELECT docid, quote(matchinfo(t1,'pcnxals')) FROM t1 WHERE t1 MATCH 'b*'"
  6   "SELECT docid, * FROM t1 WHERE t1 MATCH 'a* b* c*'"
  7   "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa OR da'"
  8   "SELECT docid, * FROM t1 WHERE t1 MATCH 'nosuchtoken'"
  9   "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR da'"
} {

  unset -nocomplain A B C D
  set A_list [list]
  set B_list [list]
  set C_list [list]
  set D_list [list]

  unset -nocomplain X
  db eval "$query ORDER BY rowid ASC"  X  { 
    set A($X(docid)) [array get X] 
    lappend A_list $X(docid)
  }
  unset -nocomplain X
  db eval "$query ORDER BY rowid DESC" X  { 
    set B($X(docid)) [array get X] 
    lappend B_list $X(docid)
  }
  unset -nocomplain X
  db eval "$query ORDER BY docid ASC"  X  { 
    set C($X(docid)) [array get X] 
    lappend C_list $X(docid)
  }
  unset -nocomplain X
  db eval "$query ORDER BY docid DESC" X  { 
    set D($X(docid)) [array get X] 
    lappend D_list $X(docid)
  }

  do_test 1.$tn.1 { set A_list } [lsort -integer -increasing $A_list]
  do_test 1.$tn.2 { set B_list } [lsort -integer -decreasing $B_list]
  do_test 1.$tn.3 { set C_list } [lsort -integer -increasing $C_list]
  do_test 1.$tn.4 { set D_list } [lsort -integer -decreasing $D_list]

  unset -nocomplain DATA
  unset -nocomplain X
  db eval "$query" X  { 
    set DATA($X(docid)) [array get X] 
  }

  do_test 1.$tn.5 { lsort [array get A] } [lsort [array get DATA]]
  do_test 1.$tn.6 { lsort [array get B] } [lsort [array get DATA]]
  do_test 1.$tn.7 { lsort [array get C] } [lsort [array get DATA]]
  do_test 1.$tn.8 { lsort [array get D] } [lsort [array get DATA]]
}

finish_test