SQLite

  if( rc!=SQLITE_OK ){
    if( p ){
      fts3DisconnectMethod((sqlite3_vtab *)p);
    }else if( pTokenizer ){
      pTokenizer->pModule->xDestroy(pTokenizer);
    }
  }else{
    assert( p->pSegments==0 );
    *ppVTab = &p->base;
  }
  return rc;
}

/*
** The xConnect() and xCreate() methods for the virtual table. All the

      }else{
        pInfo->idxStr = "ASC";
      }
      pInfo->orderByConsumed = 1;
    }
  }

  assert( p->pSegments==0 );
  return SQLITE_OK;
}

/*
** Implementation of xOpen method.
*/
static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){

  Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  sqlite3_finalize(pCsr->pStmt);
  sqlite3Fts3ExprFree(pCsr->pExpr);
  sqlite3Fts3FreeDeferredTokens(pCsr);
  sqlite3_free(pCsr->aDoclist);
  sqlite3_free(pCsr->aMatchinfo);
  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  sqlite3_free(pCsr);
  return SQLITE_OK;
}

/*
** Position the pCsr->pStmt statement so that it is on the row
** of the %_content table that contains the last match.  Return

    }else{
      pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
      rc = SQLITE_OK;
    }
  }else{
    rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor);
  }
  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  return rc;
}

/*
** This is the xFilter interface for the virtual table.  See
** the virtual table xFilter method documentation for additional
** information.

** the row that the supplied cursor currently points to.
*/
static int fts3ColumnMethod(
  sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
  sqlite3_context *pContext,      /* Context for sqlite3_result_xxx() calls */
  int iCol                        /* Index of column to read value from */
){
  int rc;                         /* Return Code */
  int rc = SQLITE_OK;             /* Return Code */
  Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
  Fts3Table *p = (Fts3Table *)pCursor->pVtab;

  /* The column value supplied by SQLite must be in range. */
  assert( iCol>=0 && iCol<=p->nColumn+1 );

  if( iCol==p->nColumn+1 ){
    /* This call is a request for the "docid" column. Since "docid" is an 
    ** alias for "rowid", use the xRowid() method to obtain the value.
    */
    sqlite3_int64 iRowid;
    rc = fts3RowidMethod(pCursor, &iRowid);
    sqlite3_result_int64(pContext, iRowid);
    sqlite3_result_int64(pContext, pCsr->iPrevId);
  }else if( iCol==p->nColumn ){
    /* The extra column whose name is the same as the table.
    ** Return a blob which is a pointer to the cursor.
    */
    sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
    rc = SQLITE_OK;
  }else{
    rc = fts3CursorSeek(0, pCsr);
    if( rc==SQLITE_OK ){
      sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1));
    }
  }

  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  return rc;
}

/* 
** This function is the implementation of the xUpdate callback used by 
** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
** inserted, updated or deleted.

/*
** Implementation of xBegin() method. This is a no-op.
*/
static int fts3BeginMethod(sqlite3_vtab *pVtab){
  UNUSED_PARAMETER(pVtab);
  TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
  assert( p->pSegments==0 );
  assert( p->nPendingData==0 );
  assert( p->inTransaction!=1 );
  TESTONLY( p->inTransaction = 1 );
  TESTONLY( p->mxSavepoint = -1; );
  return SQLITE_OK;
}

/*
** Implementation of xCommit() method. This is a no-op. The contents of
** the pending-terms hash-table have already been flushed into the database
** by fts3SyncMethod().
*/
static int fts3CommitMethod(sqlite3_vtab *pVtab){
  UNUSED_PARAMETER(pVtab);
  TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
  assert( p->nPendingData==0 );
  assert( p->inTransaction!=0 );
  assert( p->pSegments==0 );
  TESTONLY( p->inTransaction = 0 );
  TESTONLY( p->mxSavepoint = -1; );
  return SQLITE_OK;
}

/*
** Implementation of xRollback(). Discard the contents of the pending-terms

}

static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  Fts3Table *p = (Fts3Table*)pVtab;
  assert( p->inTransaction );
  assert( p->mxSavepoint < iSavepoint );
  TESTONLY( p->mxSavepoint = iSavepoint );
  return sqlite3Fts3PendingTermsFlush(p);
  return fts3SyncMethod(pVtab);
}
static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
  TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
  assert( p->inTransaction );
  assert( p->mxSavepoint >= iSavepoint );
  TESTONLY( p->mxSavepoint = iSavepoint-1 );
  return SQLITE_OK;

    p->bIncr = 0;
  }

  assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr );
  return rc;
}

/*
** This function is used to iterate backwards (from the end to start) 
** through doclists.
*/
void sqlite3Fts3DoclistPrev(
  int bDescIdx,                   /* True if the doclist is desc */
  char *aDoclist,                 /* Pointer to entire doclist */
  int nDoclist,                   /* Length of aDoclist in bytes */
  char **ppIter,                  /* IN/OUT: Iterator pointer */
  sqlite3_int64 *piDocid,         /* IN/OUT: Docid pointer */
  int *pnList,                    /* IN/OUT: List length pointer */
  u8 *pbEof                       /* OUT: End-of-file flag */
){
  char *p = *ppIter;
  int iMul = (bDescIdx ? -1 : 1);

  assert( nDoclist>0 );
  assert( *pbEof==0 );
  assert( p || *piDocid==0 );
  assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) );

  if( p==0 ){
    sqlite3_int64 iDocid = 0;
    char *pNext = 0;

    assert( p->nToken==1 );
    rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, 
        &pDL->iDocid, &pDL->pList, &pDL->nList
    );
    if( rc==SQLITE_OK && !pDL->pList ){
      *pbEof = 1;
    }
  }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->aAll ){
  }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){
    sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, 
        &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof
    );
    pDL->pList = pDL->pNextDocid;
  }else{
    char *pIter;
    if( pDL->pNextDocid ){

static void fts3EvalNext(
  Fts3Cursor *pCsr, 
  Fts3Expr *pExpr, 
  int *pRc
){
  if( *pRc==SQLITE_OK ){
    assert( pExpr->bEof==0 );
    pExpr->bStart = 1;

    switch( pExpr->eType ){
      case FTSQUERY_NEAR:
      case FTSQUERY_AND: {
        Fts3Expr *pLeft = pExpr->pLeft;
        Fts3Expr *pRight = pExpr->pRight;

          pExpr->iDocid = pLeft->iDocid;
          pExpr->bEof = pLeft->bEof;
        }else{
          fts3EvalNext(pCsr, pLeft, pRc);
          fts3EvalNext(pCsr, pRight, pRc);

          while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){
            int iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
            sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
            if( iDiff==0 ) break;
            if( iDiff<0 ){
              fts3EvalNext(pCsr, pLeft, pRc);
            }else{
              fts3EvalNext(pCsr, pRight, pRc);
            }
          }
    
          pExpr->iDocid = pLeft->iDocid;
          pExpr->bEof = (pLeft->bEof || pRight->bEof);
        }
        break;
      }
  
      case FTSQUERY_OR: {
        Fts3Expr *pLeft = pExpr->pLeft;
        Fts3Expr *pRight = pExpr->pRight;
        int iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
        sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);

        assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
        assert( pRight->bStart || pLeft->iDocid==pRight->iDocid );

        if( iCmp==0 ){
        if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
          fts3EvalNext(pCsr, pLeft, pRc);
        }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
          fts3EvalNext(pCsr, pRight, pRc);
        }else if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
        }else{
          fts3EvalNext(pCsr, pLeft, pRc);
        }else{
          fts3EvalNext(pCsr, pRight, pRc);
        }
  

        pExpr->bEof = (pLeft->bEof && pRight->bEof);
        iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
        if( pRight->bEof || (pLeft->bEof==0 &&  iCmp<0) ){
          pExpr->iDocid = pLeft->iDocid;
        }else{
          pExpr->iDocid = pRight->iDocid;
        }

      }

      default: {
        Fts3Phrase *pPhrase = pExpr->pPhrase;
        fts3EvalFreeDeferredDoclist(pPhrase);
        *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
        pExpr->iDocid = pPhrase->doclist.iDocid;
#if 0
        printf("token \"%.*s\" docid=%lld\n", 
            pPhrase->aToken[0].n, pPhrase->aToken[0].z, pExpr->iDocid
        );
#endif
        
        break;
      }
    }
  }
}

static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){

    rc = sqlite3Fts3EvalStart(pCsr, pExpr, 0);
    assert( pExpr->bEof==bEof );
    assert( pExpr->bStart==bStart );
    assert( rc!=SQLITE_OK || pPhrase->bIncr==0 );
    if( pExpr->bStart && !pExpr->bEof ){
      pExpr->bStart = 0;
      while( rc==SQLITE_OK && pExpr->bEof==0 && pExpr->iDocid!=iDocid ){
      while( rc==SQLITE_OK && (pExpr->bStart==0 || pExpr->iDocid!=iDocid) ){
        fts3EvalNext(pCsr, pExpr, &rc);
        assert( !pExpr->bEof );
      }
    }
  }

  *pnList = pPhrase->doclist.nAll;
  *ppList = pPhrase->doclist.aAll;
  return rc;

  sqlite3_value **apVal,          /* Array of arguments */
  sqlite_int64 *pRowid            /* OUT: The affected (or effected) rowid */
){
  Fts3Table *p = (Fts3Table *)pVtab;
  int rc = SQLITE_OK;             /* Return Code */
  int isRemove = 0;               /* True for an UPDATE or DELETE */
  sqlite3_int64 iRemove = 0;      /* Rowid removed by UPDATE or DELETE */
  u32 *aSzIns;                    /* Sizes of inserted documents */
  u32 *aSzIns = 0;                /* Sizes of inserted documents */
  u32 *aSzDel;                    /* Sizes of deleted documents */
  int nChng = 0;                  /* Net change in number of documents */
  int bInsertDone = 0;

  assert( p->pSegments==0 );

  /* Check for a "special" INSERT operation. One of the form:
  **
  **   INSERT INTO xyz(xyz) VALUES('command');
  */
  if( nArg>1 
   && sqlite3_value_type(apVal[0])==SQLITE_NULL 
   && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL 
  ){
    return fts3SpecialInsert(p, apVal[p->nColumn+2]);
    rc = fts3SpecialInsert(p, apVal[p->nColumn+2]);
    goto update_out;
  }

  /* Allocate space to hold the change in document sizes */
  aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 );
  if( aSzIns==0 ) return SQLITE_NOMEM;
  if( aSzIns==0 ){
    rc = SQLITE_NOMEM;
    goto update_out;
  }
  aSzDel = &aSzIns[p->nColumn+1];
  memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2);

  /* If this is an INSERT operation, or an UPDATE that modifies the rowid
  ** value, then this operation requires constraint handling.
  **
  ** If the on-conflict mode is REPLACE, this means that the existing row

      }else{
        rc = fts3InsertData(p, apVal, pRowid);
        bInsertDone = 1;
      }
    }
  }
  if( rc!=SQLITE_OK ){
    sqlite3_free(aSzIns);
    goto update_out;
    return rc;
  }

  /* If this is a DELETE or UPDATE operation, remove the old record. */
  if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
    assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
    rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel);
    isRemove = 1;

    nChng++;
  }

  if( p->bHasStat ){
    fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng);
  }

 update_out:
  sqlite3_free(aSzIns);
  sqlite3Fts3SegmentsClose(p);
  return rc;
}

/* 
** Flush any data in the pending-terms hash table to disk. If successful,

  #lsort -uniq -integer $ret
  set ret
}

# This [proc] is used to test the FTS3 matchinfo() function.
# 
proc simple_token_matchinfo {zToken} {
proc simple_token_matchinfo {zToken bDesc} {

  set nDoc(0) 0
  set nDoc(1) 0
  set nDoc(2) 0
  set nHit(0) 0
  set nHit(1) 0
  set nHit(2) 0

  set dir -inc
  if {$bDesc} { set dir -dec }

  foreach key [array names ::t1] {
    set value $::t1($key)
    set a($key) [list]
    foreach i {0 1 2} col $value {
      set hit [llength [lsearch -all $col $zToken]]
      lappend a($key) $hit
      incr nHit($i) $hit
      if {$hit>0} { incr nDoc($i) }
    }
  }

  set ret [list]
  foreach docid [lsort -integer [array names a]] {
  foreach docid [lsort -integer $dir [array names a]] {
    if { [lindex [lsort -integer $a($docid)] end] } {
      set matchinfo [list 1 3]
      foreach i {0 1 2} hit $a($docid) {
        lappend matchinfo $hit $nHit($i) $nDoc($i)
      }
      lappend ret $docid $matchinfo
    }

proc mit {blob} {
  set scan(littleEndian) i*
  set scan(bigEndian) I*
  binary scan $blob $scan($::tcl_platform(byteOrder)) r
  return $r
}
db func mit mit

set sqlite_fts3_enable_parentheses 1

proc do_orderbydocid_test {tn sql res} {
  uplevel [list do_select_test $tn.asc "$sql ORDER BY docid ASC" $res]
  uplevel [list do_select_test $tn.desc "$sql ORDER BY docid DESC" \
    [lsort -int -dec $res]
  ]
}
foreach nodesize {50 500 1000 2000} {

foreach {nodesize order} {
  50    DESC
  50    ASC
  500   ASC
  1000  DESC
  2000  ASC
} {
  catch { array unset ::t1 }
  set testname "$nodesize/$order"

  # Create the FTS3 table. Populate it (and the Tcl array) with 100 rows.
  #
  db transaction {
    catchsql { DROP TABLE t1 }
    execsql "CREATE VIRTUAL TABLE t1 USING fts3(a, b, c)"
    execsql "CREATE VIRTUAL TABLE t1 USING fts4(a, b, c, order=$order)"
    execsql "INSERT INTO t1(t1) VALUES('nodesize=$nodesize')"
    for {set i 0} {$i < 100} {incr i} { insert_row $i }
  }
  
  for {set iTest 1} {$iTest <= 100} {incr iTest} {
    catchsql COMMIT

    set DO_MALLOC_TEST 0
    set nRep 10
    if {$iTest==100 && $nodesize==50} { 
      set DO_MALLOC_TEST 1 
      set nRep 2
    }

    set ::testprefix fts3rnd-1.$testname.$iTest
  
    # Delete one row, update one row and insert one row.
    #
    set rows [array names ::t1]
    set nRow [llength $rows]
    set iUpdate [lindex $rows [expr {int(rand()*$nRow)}]]
    set iDelete $iUpdate

    execsql BEGIN
      insert_row $iInsert
      update_row $iUpdate
      delete_row $iDelete
    if {0==($iTest%2)} { execsql COMMIT }

    if {0==($iTest%2)} { 
      do_test fts3rnd-1.$nodesize.$iTest.0 { fts3_integrity_check t1 } ok 
      #do_test 0 { fts3_integrity_check t1 } ok 
    }

    # Pick 10 terms from the vocabulary. Check that the results of querying
    # the database for the set of documents containing each of these terms
    # is the same as the result obtained by scanning the contents of the Tcl 
    # array for each term.
    #
    for {set i 0} {$i < 10} {incr i} {
      set term [random_term]
      do_select_test fts3rnd-1.$nodesize.$iTest.1.$i {
      do_select_test 1.$i {
        SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term
        ORDER BY docid ASC
      } [simple_token_matchinfo $term]
      } [simple_token_matchinfo $term 0]
      do_select_test 1.$i {
        SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term
        ORDER BY docid DESC
      } [simple_token_matchinfo $term 1]
    }

    # This time, use the first two characters of each term as a term prefix
    # to query for. Test that querying the Tcl array produces the same results
    # as querying the FTS3 table for the prefix.
    #
    for {set i 0} {$i < $nRep} {incr i} {
      set prefix [string range [random_term] 0 end-1]
      set match "${prefix}*"
      do_select_test fts3rnd-1.$nodesize.$iTest.2.$i {
      do_orderbydocid_test 2.$i {
        SELECT docid FROM t1 WHERE t1 MATCH $match
      } [simple_phrase $match]
    }

    # Similar to the above, except for phrase queries.
    #
    for {set i 0} {$i < $nRep} {incr i} {
      set term [list [random_term] [random_term]]
      set match "\"$term\""
      do_select_test fts3rnd-1.$nodesize.$iTest.3.$i {
      do_orderbydocid_test 3.$i {
        SELECT docid FROM t1 WHERE t1 MATCH $match
      } [simple_phrase $term]
    }

    # Three word phrases.
    #
    for {set i 0} {$i < $nRep} {incr i} {
      set term [list [random_term] [random_term] [random_term]]
      set match "\"$term\""
      do_select_test fts3rnd-1.$nodesize.$iTest.4.$i {
      do_orderbydocid_test 4.$i {
        SELECT docid FROM t1 WHERE t1 MATCH $match
      } [simple_phrase $term]
    }

    # Three word phrases made up of term-prefixes.
    #
    for {set i 0} {$i < $nRep} {incr i} {
      set    query "[string range [random_term] 0 end-1]* "
      append query "[string range [random_term] 0 end-1]* "
      append query "[string range [random_term] 0 end-1]*"

      set match "\"$query\""
      do_select_test fts3rnd-1.$nodesize.$iTest.5.$i {
      do_orderbydocid_test 5.$i {
        SELECT docid FROM t1 WHERE t1 MATCH $match
      } [simple_phrase $query]
    }

    # A NEAR query with terms as the arguments.
    # A NEAR query with terms as the arguments:
    #
    #     ... MATCH '$term1 NEAR $term2' ...
    #
    for {set i 0} {$i < $nRep} {incr i} {
      set terms [list [random_term] [random_term]]
      set match [join $terms " NEAR "]
      do_select_test fts3rnd-1.$nodesize.$iTest.6.$i {
      do_orderbydocid_test 6.$i {
        SELECT docid FROM t1 WHERE t1 MATCH $match 
      } [simple_near $terms 10]
    }

    # A 3-way NEAR query with terms as the arguments.
    #
    for {set i 0} {$i < $nRep} {incr i} {
      set terms [list [random_term] [random_term] [random_term]]
      set nNear 11
      set match [join $terms " NEAR/$nNear "]
      do_select_test fts3rnd-1.$nodesize.$iTest.7.$i {
      do_orderbydocid_test 7.$i {
        SELECT docid FROM t1 WHERE t1 MATCH $match
      } [simple_near $terms $nNear]
    }
    
    # Set operations on simple term queries.
    #
    foreach {tn op proc} {
      8  OR  setop_or
      9  NOT setop_not
      10 AND setop_and
    } {
      for {set i 0} {$i < $nRep} {incr i} {
        set term1 [random_term]
        set term2 [random_term]
        set match "$term1 $op $term2"
        do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i {
        do_orderbydocid_test $tn.$i {
          SELECT docid FROM t1 WHERE t1 MATCH $match
        } [$proc [simple_phrase $term1] [simple_phrase $term2]]
      }
    }
 
    # Set operations on NEAR queries.
    #
    foreach {tn op proc} {
      8  OR  setop_or
      9  NOT setop_not
      10 AND setop_and
    } {
      for {set i 0} {$i < $nRep} {incr i} {
        set term1 [random_term]
        set term2 [random_term]
        set term3 [random_term]
        set term4 [random_term]
        set match "$term1 NEAR $term2 $op $term3 NEAR $term4"
        do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i {
        do_orderbydocid_test $tn.$i {
          SELECT docid FROM t1 WHERE t1 MATCH $match
        } [$proc                                  \
            [simple_near [list $term1 $term2] 10] \
            [simple_near [list $term3 $term4] 10]
          ]
      }
    }

    catchsql COMMIT
  }
}

finish_test