Index: Makefile.in
==================================================================
--- Makefile.in
+++ Makefile.in
@@ -418,10 +418,11 @@
   $(TOP)/ext/misc/eval.c \
   $(TOP)/ext/misc/fileio.c \
   $(TOP)/ext/misc/fuzzer.c \
   $(TOP)/ext/fts5/fts5_tcl.c \
   $(TOP)/ext/fts5/fts5_test_mi.c \
+  $(TOP)/ext/fts5/fts5_test_tok.c \
   $(TOP)/ext/misc/ieee754.c \
   $(TOP)/ext/misc/nextchar.c \
   $(TOP)/ext/misc/percentile.c \
   $(TOP)/ext/misc/regexp.c \
   $(TOP)/ext/misc/series.c \

Index: Makefile.msc
==================================================================
--- Makefile.msc
+++ Makefile.msc
@@ -1085,10 +1085,11 @@
   $(TOP)\ext\misc\eval.c \
   $(TOP)\ext\misc\fileio.c \
   $(TOP)\ext\misc\fuzzer.c \
   $(TOP)\ext\fts5\fts5_tcl.c \
   $(TOP)\ext\fts5\fts5_test_mi.c \
+  $(TOP)\ext\fts5\fts5_test_tok.c \
   $(TOP)\ext\misc\ieee754.c \
   $(TOP)\ext\misc\nextchar.c \
   $(TOP)\ext\misc\percentile.c \
   $(TOP)\ext\misc\regexp.c \
   $(TOP)\ext\misc\series.c \
@@ -1712,12 +1713,12 @@
 	$(LTCOMPILE) $(NO_WARN) -c fts5.c
 
 fts5.dll:	fts5_ext.lo
 	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ fts5_ext.lo
 
-sqlite3rbu.lo:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
-	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c
+sqlite3rbu.lo:	$(TOP)\ext\rbu\sqlite3rbu.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\rbu\sqlite3rbu.c
 
 # Rules to build the 'testfixture' application.
 #
 # If using the amalgamation, use sqlite3.c directly to build the test
 # fixture.  Otherwise link against libsqlite3.lib.  (This distinction is
@@ -1838,12 +1839,12 @@
 speedtest1.exe:	$(TOP)\test\speedtest1.c $(SQLITE3C)
 	$(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
 		$(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
 rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C)
-	$(LTLINK) $(NO_WARN) -I. -DSQLITE_ENABLE_RBU -Fe$@ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) \
-		$(LDFLAGS) $(LTLINKOPTS)
+	$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) \
+		/link $(LDFLAGS) $(LTLINKOPTS)
 
 clean:
 	del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL
 	del /Q *.bsc *.cod *.da *.bb *.bbg gmon.out 2>NUL
 	del /Q sqlite3.h opcodes.c opcodes.h 2>NUL

Index: ext/fts5/fts5Int.h
==================================================================
--- ext/fts5/fts5Int.h
+++ ext/fts5/fts5Int.h
@@ -223,17 +223,17 @@
 ** Buffer object for the incremental building of string data.
 */
 typedef struct Fts5Buffer Fts5Buffer;
 struct Fts5Buffer {
   u8 *p;
-  int n;
-  int nSpace;
+  u32 n;
+  u32 nSpace;
 };
 
-int sqlite3Fts5BufferSize(int*, Fts5Buffer*, int);
+int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);
 void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
-void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*);
+void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);
 void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
 void sqlite3Fts5BufferFree(Fts5Buffer*);
 void sqlite3Fts5BufferZero(Fts5Buffer*);
 void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
 void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
@@ -584,11 +584,11 @@
 int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
 
 int sqlite3Fts5DropAll(Fts5Config*);
 int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
 
-int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
+int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
 int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
 int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
 
 int sqlite3Fts5StorageIntegrity(Fts5Storage *p);
 
@@ -604,12 +604,10 @@
 
 int sqlite3Fts5StorageConfigValue(
     Fts5Storage *p, const char*, sqlite3_value*, int
 );
 
-int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**);
-
 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
 int sqlite3Fts5StorageRebuild(Fts5Storage *p);
 int sqlite3Fts5StorageOptimize(Fts5Storage *p);
 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
 

Index: ext/fts5/fts5_buffer.c
==================================================================
--- ext/fts5/fts5_buffer.c
+++ ext/fts5/fts5_buffer.c
@@ -13,12 +13,12 @@
 
 
 
 #include "fts5Int.h"
 
-int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, int nByte){
-  int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
+int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){
+  u32 nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
   u8 *pNew;
   while( nNew<nByte ){
     nNew = nNew * 2;
   }
   pNew = sqlite3_realloc(pBuf->p, nNew);
@@ -59,14 +59,14 @@
 ** is called, it is a no-op.
 */
 void sqlite3Fts5BufferAppendBlob(
   int *pRc,
   Fts5Buffer *pBuf, 
-  int nData, 
+  u32 nData, 
   const u8 *pData
 ){
-  assert( *pRc || nData>=0 );
+  assert_nc( *pRc || nData>=0 );
   if( fts5BufferGrow(pRc, pBuf, nData) ) return;
   memcpy(&pBuf->p[pBuf->n], pData, nData);
   pBuf->n += nData;
 }
 
@@ -320,18 +320,21 @@
 ){
   int rc = SQLITE_OK;
   *pbPresent = 0;
   if( p ){
     int i;
-    int hash;
+    int hash = 13;
     Fts5TermsetEntry *pEntry;
 
-    /* Calculate a hash value for this term */
-    hash = 104 + iIdx;
-    for(i=0; i<nTerm; i++){
-      hash += (hash << 3) + (int)pTerm[i];
+    /* Calculate a hash value for this term. This is the same hash checksum
+    ** used by the fts5_hash.c module. This is not important for correct
+    ** operation of the module, but is necessary to ensure that some tests
+    ** designed to produce hash table collisions really do work.  */
+    for(i=nTerm-1; i>=0; i--){
+      hash = (hash << 3) ^ hash ^ pTerm[i];
     }
+    hash = (hash << 3) ^ hash ^ iIdx;
     hash = hash % ArraySize(p->apHash);
 
     for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){
       if( pEntry->iIdx==iIdx 
           && pEntry->nTerm==nTerm 

Index: ext/fts5/fts5_config.c
==================================================================
--- ext/fts5/fts5_config.c
+++ ext/fts5/fts5_config.c
@@ -276,11 +276,11 @@
       while( p[0]>='0' && p[0]<='9' && nPre<1000 ){
         nPre = nPre*10 + (p[0] - '0');
         p++;
       }
 
-      if( rc==SQLITE_OK && (nPre<=0 || nPre>=1000) ){
+      if( nPre<=0 || nPre>=1000 ){
         *pzErr = sqlite3_mprintf("prefix length out of range (max 999)");
         rc = SQLITE_ERROR;
         break;
       }
 

Index: ext/fts5/fts5_expr.c
==================================================================
--- ext/fts5/fts5_expr.c
+++ ext/fts5/fts5_expr.c
@@ -832,11 +832,11 @@
   assert( pNode->eType==FTS5_TERM );
   assert( pNear->nPhrase==1 && pPhrase->nTerm==1 );
   assert( pPhrase->aTerm[0].pSynonym==0 );
 
   rc = sqlite3Fts5IterPoslist(pIter, pColset, 
-      (const u8**)&pPhrase->poslist.p, &pPhrase->poslist.n, &pNode->iRowid
+      (const u8**)&pPhrase->poslist.p, (int*)&pPhrase->poslist.n, &pNode->iRowid
   );
   pNode->bNomatch = (pPhrase->poslist.n==0);
   return rc;
 }
 
@@ -2394,11 +2394,11 @@
       aPopulator[i].bOk = 1;
     }
   }
 
   return sqlite3Fts5Tokenize(pConfig, 
-      FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
+      FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
   );
 }
 
 static void fts5ExprClearPoslists(Fts5ExprNode *pNode){
   if( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING ){
@@ -2410,53 +2410,48 @@
     }
   }
 }
 
 static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){
-  if( pNode ){
-    pNode->iRowid = iRowid;
-    pNode->bEof = 0;
-    switch( pNode->eType ){
-      case FTS5_TERM:
-      case FTS5_STRING:
-        return (pNode->pNear->apPhrase[0]->poslist.n>0);
-
-      case FTS5_AND: {
-        int i;
-        for(i=0; i<pNode->nChild; i++){
-          if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){
-            fts5ExprClearPoslists(pNode);
-            return 0;
-          }
-        }
-        break;
-      }
-
-      case FTS5_OR: {
-        int i;
-        int bRet = 0;
-        for(i=0; i<pNode->nChild; i++){
-          if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){
-            bRet = 1;
-          }
-        }
-        if( bRet==0 ){
-          fts5ExprClearPoslists(pNode);
-        }
-        return bRet;
-      }
-
-      default: {
-        assert( pNode->eType==FTS5_NOT );
-        if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid)
-         || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid)
-        ){
-          fts5ExprClearPoslists(pNode);
-          return 0;
-        }
-        break;
-      }
+  pNode->iRowid = iRowid;
+  pNode->bEof = 0;
+  switch( pNode->eType ){
+    case FTS5_TERM:
+    case FTS5_STRING:
+      return (pNode->pNear->apPhrase[0]->poslist.n>0);
+
+    case FTS5_AND: {
+      int i;
+      for(i=0; i<pNode->nChild; i++){
+        if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){
+          fts5ExprClearPoslists(pNode);
+          return 0;
+        }
+      }
+      break;
+    }
+
+    case FTS5_OR: {
+      int i;
+      int bRet = 0;
+      for(i=0; i<pNode->nChild; i++){
+        if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){
+          bRet = 1;
+        }
+      }
+      return bRet;
+    }
+
+    default: {
+      assert( pNode->eType==FTS5_NOT );
+      if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid)
+          || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid)
+        ){
+        fts5ExprClearPoslists(pNode);
+        return 0;
+      }
+      break;
     }
   }
   return 1;
 }
 

Index: ext/fts5/fts5_index.c
==================================================================
--- ext/fts5/fts5_index.c
+++ ext/fts5/fts5_index.c
@@ -1873,17 +1873,19 @@
 ){
   Fts5Data *pLeaf = pIter->pLeaf;
   int iOff;
   int bNewTerm = 0;
   int nKeep = 0;
+  u8 *a;
+  int n;
 
   assert( pbNewTerm==0 || *pbNewTerm==0 );
   assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE );
 
   /* Search for the end of the position list within the current page. */
-  u8 *a = pLeaf->p;
-  int n = pLeaf->szLeaf;
+  a = pLeaf->p;
+  n = pLeaf->szLeaf;
 
   ASSERT_SZLEAF_OK(pLeaf);
   iOff = pIter->iLeafOffset + pIter->nPos;
 
   if( iOff<n ){
@@ -5899,10 +5901,51 @@
     }
   }
 
   return iOff;
 }
+
+/*
+** This function is part of the fts5_decode() debugging function. It is 
+** only ever used with detail=none tables.
+**
+** Buffer (pData/nData) contains a doclist in the format used by detail=none
+** tables. This function appends a human-readable version of that list to
+** buffer pBuf.
+**
+** If *pRc is other than SQLITE_OK when this function is called, it is a
+** no-op. If an OOM or other error occurs within this function, *pRc is
+** set to an SQLite error code before returning. The final state of buffer
+** pBuf is undefined in this case.
+*/
+static void fts5DecodeRowidList(
+  int *pRc,                       /* IN/OUT: Error code */
+  Fts5Buffer *pBuf,               /* Buffer to append text to */
+  const u8 *pData, int nData      /* Data to decode list-of-rowids from */
+){
+  int i = 0;
+  i64 iRowid = 0;
+
+  while( i<nData ){
+    const char *zApp = "";
+    u64 iVal;
+    i += sqlite3Fts5GetVarint(&pData[i], &iVal);
+    iRowid += iVal;
+
+    if( i<nData && pData[i]==0x00 ){
+      i++;
+      if( i<nData && pData[i]==0x00 ){
+        i++;
+        zApp = "+";
+      }else{
+        zApp = "*";
+      }
+    }
+
+    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %lld%s", iRowid, zApp);
+  }
+}
 
 /*
 ** The implementation of user-defined scalar function fts5_decode().
 */
 static void fts5DecodeFunction(
@@ -5915,10 +5958,11 @@
   const u8 *aBlob; int n;         /* Record to decode */
   u8 *a = 0;
   Fts5Buffer s;                   /* Build up text to return here */
   int rc = SQLITE_OK;             /* Return code */
   int nSpace = 0;
+  int eDetailNone = (sqlite3_user_data(pCtx)!=0);
 
   assert( nArg==2 );
   memset(&s, 0, sizeof(Fts5Buffer));
   iRowid = sqlite3_value_int64(apVal[0]);
 
@@ -5956,10 +6000,58 @@
     if( iRowid==FTS5_AVERAGES_ROWID ){
       fts5DecodeAverages(&rc, &s, a, n);
     }else{
       fts5DecodeStructure(&rc, &s, a, n);
     }
+  }else if( eDetailNone ){
+    Fts5Buffer term;              /* Current term read from page */
+    int szLeaf;
+    int iPgidxOff = szLeaf = fts5GetU16(&a[2]);
+    int iTermOff;
+    int nKeep = 0;
+    int iOff;
+
+    memset(&term, 0, sizeof(Fts5Buffer));
+
+    /* Decode any entries that occur before the first term. */
+    if( szLeaf<n ){
+      iPgidxOff += fts5GetVarint32(&a[iPgidxOff], iTermOff);
+    }else{
+      iTermOff = szLeaf;
+    }
+    fts5DecodeRowidList(&rc, &s, &a[4], iTermOff-4);
+
+    iOff = iTermOff;
+    while( iOff<szLeaf ){
+      int nAppend;
+
+      /* Read the term data for the next term*/
+      iOff += fts5GetVarint32(&a[iOff], nAppend);
+      term.n = nKeep;
+      fts5BufferAppendBlob(&rc, &term, nAppend, &a[iOff]);
+      sqlite3Fts5BufferAppendPrintf(
+          &rc, &s, " term=%.*s", term.n, (const char*)term.p
+      );
+      iOff += nAppend;
+
+      /* Figure out where the doclist for this term ends */
+      if( iPgidxOff<n ){
+        int nIncr;
+        iPgidxOff += fts5GetVarint32(&a[iPgidxOff], nIncr);
+        iTermOff += nIncr;
+      }else{
+        iTermOff = szLeaf;
+      }
+
+      fts5DecodeRowidList(&rc, &s, &a[iOff], iTermOff-iOff);
+      iOff = iTermOff;
+      if( iOff<szLeaf ){
+        iOff += fts5GetVarint32(&a[iOff], nKeep);
+      }
+    }
+
+    fts5BufferFree(&term);
   }else{
     Fts5Buffer term;              /* Current term read from page */
     int szLeaf;                   /* Offset of pgidx in a[] */
     int iPgidxOff;
     int iPgidxPrev = 0;           /* Previous value read from pgidx */
@@ -6083,12 +6175,20 @@
 */
 int sqlite3Fts5IndexInit(sqlite3 *db){
   int rc = sqlite3_create_function(
       db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0
   );
+
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(
+        db, "fts5_decode_none", 2, 
+        SQLITE_UTF8, (void*)db, fts5DecodeFunction, 0, 0
+    );
+  }
+
   if( rc==SQLITE_OK ){
     rc = sqlite3_create_function(
         db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
     );
   }
   return rc;
 }

Index: ext/fts5/fts5_main.c
==================================================================
--- ext/fts5/fts5_main.c
+++ ext/fts5/fts5_main.c
@@ -842,45 +842,44 @@
   
   return rc;
 }
 
 
-static sqlite3_stmt *fts5PrepareStatement(
-  int *pRc,
+static int fts5PrepareStatement(
+  sqlite3_stmt **ppStmt,
   Fts5Config *pConfig, 
   const char *zFmt,
   ...
 ){
   sqlite3_stmt *pRet = 0;
+  int rc;
+  char *zSql;
   va_list ap;
+
   va_start(ap, zFmt);
-
-  if( *pRc==SQLITE_OK ){
-    int rc;
-    char *zSql = sqlite3_vmprintf(zFmt, ap);
-    if( zSql==0 ){
-      rc = SQLITE_NOMEM; 
-    }else{
-      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
-      if( rc!=SQLITE_OK ){
-        *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
-      }
-      sqlite3_free(zSql);
-    }
-    *pRc = rc;
+  zSql = sqlite3_vmprintf(zFmt, ap);
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM; 
+  }else{
+    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
+    if( rc!=SQLITE_OK ){
+      *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
+    }
+    sqlite3_free(zSql);
   }
 
   va_end(ap);
-  return pRet;
+  *ppStmt = pRet;
+  return rc;
 } 
 
 static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){
   Fts5Config *pConfig = pTab->pConfig;
   Fts5Sorter *pSorter;
   int nPhrase;
   int nByte;
-  int rc = SQLITE_OK;
+  int rc;
   const char *zRank = pCsr->zRank;
   const char *zRankArgs = pCsr->zRankArgs;
   
   nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
   nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1);
@@ -894,11 +893,11 @@
   ** is not possible as SQLite reference counts the virtual table objects.
   ** And since the statement required here reads from this very virtual 
   ** table, saving it creates a circular reference.
   **
   ** If SQLite a built-in statement cache, this wouldn't be a problem. */
-  pSorter->pStmt = fts5PrepareStatement(&rc, pConfig,
+  rc = fts5PrepareStatement(&pSorter->pStmt, pConfig,
       "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s",
       pConfig->zDb, pConfig->zName, zRank, pConfig->zName,
       (zRankArgs ? ", " : ""),
       (zRankArgs ? zRankArgs : ""),
       bDesc ? "DESC" : "ASC"
@@ -1403,11 +1402,11 @@
 ){
   int rc = SQLITE_OK;
   int eType1 = sqlite3_value_type(apVal[1]);
   if( eType1==SQLITE_INTEGER ){
     sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]);
-    rc = sqlite3Fts5StorageSpecialDelete(pTab->pStorage, iDel, &apVal[2]);
+    rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2]);
   }
   return rc;
 }
 
 static void fts5StorageInsert(
@@ -1510,21 +1509,21 @@
     }
 
     /* Case 1: DELETE */
     else if( nArg==1 ){
       i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
-      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
+      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0);
     }
 
     /* Case 2: INSERT */
     else if( eType0!=SQLITE_INTEGER ){     
       /* If this is a REPLACE, first remove the current entry (if any) */
       if( eConflict==SQLITE_REPLACE 
        && sqlite3_value_type(apVal[1])==SQLITE_INTEGER 
       ){
         i64 iNew = sqlite3_value_int64(apVal[1]);  /* Rowid to delete */
-        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew);
+        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
       }
       fts5StorageInsert(&rc, pTab, apVal, pRowid);
     }
 
     /* Case 2: UPDATE */
@@ -1531,26 +1530,26 @@
     else{
       i64 iOld = sqlite3_value_int64(apVal[0]);  /* Old rowid */
       i64 iNew = sqlite3_value_int64(apVal[1]);  /* New rowid */
       if( iOld!=iNew ){
         if( eConflict==SQLITE_REPLACE ){
-          rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
+          rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
           if( rc==SQLITE_OK ){
-            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew);
+            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
           }
           fts5StorageInsert(&rc, pTab, apVal, pRowid);
         }else{
           rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid);
           if( rc==SQLITE_OK ){
-            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
+            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
           }
           if( rc==SQLITE_OK ){
             rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *pRowid);
           }
         }
       }else{
-        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
+        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
         fts5StorageInsert(&rc, pTab, apVal, pRowid);
       }
     }
   }
 
@@ -1745,11 +1744,13 @@
     /* Initialize all iterators */
     for(i=0; i<nIter && rc==SQLITE_OK; i++){
       const u8 *a;
       int n; 
       rc = fts5CsrPoslist(pCsr, i, &a, &n);
-      sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
+      if( rc==SQLITE_OK ){
+        sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
+      }
     }
 
     if( rc==SQLITE_OK ){
       while( 1 ){
         int *aInst;
@@ -2035,12 +2036,19 @@
   int rc = SQLITE_OK;
   Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
   Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
 
   if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
+    Fts5Sorter *pSorter = pCsr->pSorter;
     int n;
-    rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n);
+    if( pSorter ){
+      int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
+      n = pSorter->aIdx[iPhrase] - i1;
+      pIter->a = &pSorter->aPoslist[i1];
+    }else{
+      rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n);
+    }
     if( rc==SQLITE_OK ){
       pIter->b = &pIter->a[n];
       *piCol = 0;
       fts5ApiPhraseNextColumn(pCtx, pIter, piCol);
     }

Index: ext/fts5/fts5_storage.c
==================================================================
--- ext/fts5/fts5_storage.c
+++ ext/fts5/fts5_storage.c
@@ -376,43 +376,56 @@
 /*
 ** If a row with rowid iDel is present in the %_content table, add the
 ** delete-markers to the FTS index necessary to delete it. Do not actually
 ** remove the %_content row at this time though.
 */
-static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){
+static int fts5StorageDeleteFromIndex(
+  Fts5Storage *p, 
+  i64 iDel, 
+  sqlite3_value **apVal
+){
   Fts5Config *pConfig = p->pConfig;
-  sqlite3_stmt *pSeek;            /* SELECT to read row iDel from %_data */
+  sqlite3_stmt *pSeek = 0;        /* SELECT to read row iDel from %_data */
   int rc;                         /* Return code */
+  int rc2;                        /* sqlite3_reset() return code */
+  int iCol;
+  Fts5InsertCtx ctx;
 
-  rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
-  if( rc==SQLITE_OK ){
-    int rc2;
+  if( apVal==0 ){
+    rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
+    if( rc!=SQLITE_OK ) return rc;
     sqlite3_bind_int64(pSeek, 1, iDel);
-    if( sqlite3_step(pSeek)==SQLITE_ROW ){
-      int iCol;
-      Fts5InsertCtx ctx;
-      ctx.pStorage = p;
-      ctx.iCol = -1;
-      rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
-      for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
-        if( pConfig->abUnindexed[iCol-1] ) continue;
-        ctx.szCol = 0;
-        rc = sqlite3Fts5Tokenize(pConfig, 
-            FTS5_TOKENIZE_DOCUMENT,
-            (const char*)sqlite3_column_text(pSeek, iCol),
-            sqlite3_column_bytes(pSeek, iCol),
-            (void*)&ctx,
-            fts5StorageInsertCallback
-        );
-        p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
-      }
-      p->nTotalRow--;
-    }
-    rc2 = sqlite3_reset(pSeek);
-    if( rc==SQLITE_OK ) rc = rc2;
-  }
-
+    if( sqlite3_step(pSeek)!=SQLITE_ROW ){
+      return sqlite3_reset(pSeek);
+    }
+  }
+
+  ctx.pStorage = p;
+  ctx.iCol = -1;
+  rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
+  for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
+    if( pConfig->abUnindexed[iCol-1]==0 ){
+      const char *zText;
+      int nText;
+      if( pSeek ){
+        zText = (const char*)sqlite3_column_text(pSeek, iCol);
+        nText = sqlite3_column_bytes(pSeek, iCol);
+      }else{
+        zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
+        nText = sqlite3_value_bytes(apVal[iCol-1]);
+      }
+      ctx.szCol = 0;
+      rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, 
+          zText, nText, (void*)&ctx, fts5StorageInsertCallback
+      );
+      p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
+    }
+  }
+  p->nTotalRow--;
+
+  rc2 = sqlite3_reset(pSeek);
+  if( rc==SQLITE_OK ) rc = rc2;
   return rc;
 }
 
 
 /*
@@ -488,20 +501,21 @@
 }
 
 /*
 ** Remove a row from the FTS table.
 */
-int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){
+int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
   Fts5Config *pConfig = p->pConfig;
   int rc;
   sqlite3_stmt *pDel = 0;
 
+  assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 );
   rc = fts5StorageLoadTotals(p, 1);
 
   /* Delete the index records */
   if( rc==SQLITE_OK ){
-    rc = fts5StorageDeleteFromIndex(p, iDel);
+    rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
   }
 
   /* Delete the %_docsize record */
   if( rc==SQLITE_OK && pConfig->bColumnsize ){
     rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
@@ -515,65 +529,10 @@
   /* Delete the %_content record */
   if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
     if( rc==SQLITE_OK ){
       rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0);
     }
-    if( rc==SQLITE_OK ){
-      sqlite3_bind_int64(pDel, 1, iDel);
-      sqlite3_step(pDel);
-      rc = sqlite3_reset(pDel);
-    }
-  }
-
-  /* Write the averages record */
-  if( rc==SQLITE_OK ){
-    rc = fts5StorageSaveTotals(p);
-  }
-
-  return rc;
-}
-
-int sqlite3Fts5StorageSpecialDelete(
-  Fts5Storage *p, 
-  i64 iDel, 
-  sqlite3_value **apVal
-){
-  Fts5Config *pConfig = p->pConfig;
-  int rc;
-  sqlite3_stmt *pDel = 0;
-
-  assert( pConfig->eContent!=FTS5_CONTENT_NORMAL );
-  rc = fts5StorageLoadTotals(p, 1);
-
-  /* Delete the index records */
-  if( rc==SQLITE_OK ){
-    int iCol;
-    Fts5InsertCtx ctx;
-    ctx.pStorage = p;
-    ctx.iCol = -1;
-
-    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
-    for(iCol=0; rc==SQLITE_OK && iCol<pConfig->nCol; iCol++){
-      if( pConfig->abUnindexed[iCol] ) continue;
-      ctx.szCol = 0;
-      rc = sqlite3Fts5Tokenize(pConfig, 
-        FTS5_TOKENIZE_DOCUMENT,
-        (const char*)sqlite3_value_text(apVal[iCol]),
-        sqlite3_value_bytes(apVal[iCol]),
-        (void*)&ctx,
-        fts5StorageInsertCallback
-      );
-      p->aTotalSize[iCol] -= (i64)ctx.szCol;
-    }
-    p->nTotalRow--;
-  }
-
-  /* Delete the %_docsize record */
-  if( pConfig->bColumnsize ){
-    if( rc==SQLITE_OK ){
-      rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
-    }
     if( rc==SQLITE_OK ){
       sqlite3_bind_int64(pDel, 1, iDel);
       sqlite3_step(pDel);
       rc = sqlite3_reset(pDel);
     }

Index: ext/fts5/fts5_tcl.c
==================================================================
--- ext/fts5/fts5_tcl.c
+++ ext/fts5/fts5_tcl.c
@@ -21,11 +21,12 @@
 #include "fts5.h"
 #include <string.h>
 #include <assert.h>
 
 extern int sqlite3_fts5_may_be_corrupt;
-extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *);
+extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*);
+extern int sqlite3Fts5TestRegisterTok(sqlite3*, fts5_api*);
 
 /*************************************************************************
 ** This is a copy of the first part of the SqliteDb structure in 
 ** tclsqlite.c.  We need it here so that the get_sqlite_pointer routine
 ** can extract the sqlite3* pointer from an existing Tcl SQLite
@@ -444,14 +445,16 @@
 
       if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
       zColvar = Tcl_GetString(objv[3]);
       zOffvar = Tcl_GetString(objv[4]);
 
-      for(p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);
-          iCol>=0;
-          p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff)
-      ){
+      rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);
+      if( rc!=SQLITE_OK ){
+        Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+        return TCL_ERROR;
+      }
+      for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){
         Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
         Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0);
         rc = Tcl_EvalObjEx(interp, pScript, 0);
         if( rc==TCL_CONTINUE ) rc = TCL_OK;
         if( rc!=TCL_OK ){
@@ -471,14 +474,16 @@
       Fts5PhraseIter iter;
 
       if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
       zColvar = Tcl_GetString(objv[3]);
 
-      for(p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
-          iCol>=0;
-          p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)
-      ){
+      rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
+      if( rc!=SQLITE_OK ){
+        Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+        return TCL_ERROR;
+      }
+      for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){
         Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
         rc = Tcl_EvalObjEx(interp, pScript, 0);
         if( rc==TCL_CONTINUE ) rc = TCL_OK;
         if( rc!=TCL_OK ){
           if( rc==TCL_BREAK ) rc = TCL_OK;
@@ -1075,10 +1080,36 @@
     Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
     return TCL_ERROR;
   }
   return TCL_OK;
 }
+
+static int f5tRegisterTok(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int rc;
+  sqlite3 *db = 0;
+  fts5_api *pApi = 0;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }
+  if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){
+    return TCL_ERROR;
+  }
+
+  rc = sqlite3Fts5TestRegisterTok(db, pApi);
+  if( rc!=SQLITE_OK ){
+    Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
+    return TCL_ERROR;
+  }
+  return TCL_OK;
+}
 
 /*
 ** Entry point.
 */
 int Fts5tcl_Init(Tcl_Interp *interp){
@@ -1091,11 +1122,12 @@
     { "sqlite3_fts5_token",              f5tTokenizerReturn, 1 },
     { "sqlite3_fts5_tokenize",           f5tTokenize, 0 },
     { "sqlite3_fts5_create_function",    f5tCreateFunction, 0 },
     { "sqlite3_fts5_may_be_corrupt",     f5tMayBeCorrupt, 0 },
     { "sqlite3_fts5_token_hash",         f5tTokenHash, 0 },
-    { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 }
+    { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 },
+    { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 }
   };
   int i;
   F5tTokenizerContext *pContext;
 
   pContext = (F5tTokenizerContext*)ckalloc(sizeof(F5tTokenizerContext));

ADDED   ext/fts5/fts5_test_tok.c
Index: ext/fts5/fts5_test_tok.c
==================================================================
--- /dev/null
+++ ext/fts5/fts5_test_tok.c
@@ -0,0 +1,482 @@
+/*
+** 2013 Apr 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains code for the "fts5tokenize" virtual table module.
+** An fts5tokenize virtual table is created as follows:
+**
+**   CREATE VIRTUAL TABLE <tbl> USING fts5tokenize(
+**       <tokenizer-name>, <arg-1>, ...
+**   );
+**
+** The table created has the following schema:
+**
+**   CREATE TABLE <tbl>(input HIDDEN, token, start, end, position)
+**
+** When queried, the query must include a WHERE clause of type:
+**
+**   input = <string>
+**
+** The virtual table module tokenizes this <string>, using the FTS3 
+** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE 
+** statement and returns one row for each token in the result. With
+** fields set as follows:
+**
+**   input:   Always set to a copy of <string>
+**   token:   A token from the input.
+**   start:   Byte offset of the token within the input <string>.
+**   end:     Byte offset of the byte immediately following the end of the
+**            token within the input string.
+**   pos:     Token offset of token within input.
+**
+*/
+#if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5)
+
+#include <fts5.h>
+#include <string.h>
+#include <assert.h>
+
+typedef struct Fts5tokTable Fts5tokTable;
+typedef struct Fts5tokCursor Fts5tokCursor;
+typedef struct Fts5tokRow Fts5tokRow;
+
+/*
+** Virtual table structure.
+*/
+struct Fts5tokTable {
+  sqlite3_vtab base;              /* Base class used by SQLite core */
+  fts5_tokenizer tok;             /* Tokenizer functions */
+  Fts5Tokenizer *pTok;            /* Tokenizer instance */
+};
+
+/*
+** A container for a rows values.
+*/
+struct Fts5tokRow {
+  char *zToken;
+  int iStart;
+  int iEnd;
+  int iPos;
+};
+
+/*
+** Virtual table cursor structure.
+*/
+struct Fts5tokCursor {
+  sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
+  int iRowid;                     /* Current 'rowid' value */
+  char *zInput;                   /* Input string */
+  int nRow;                       /* Number of entries in aRow[] */
+  Fts5tokRow *aRow;               /* Array of rows to return */
+};
+
+static void fts5tokDequote(char *z){
+  char q = z[0];
+
+  if( q=='[' || q=='\'' || q=='"' || q=='`' ){
+    int iIn = 1;
+    int iOut = 0;
+    if( q=='[' ) q = ']';  
+
+    while( z[iIn] ){
+      if( z[iIn]==q ){
+        if( z[iIn+1]!=q ){
+          /* Character iIn was the close quote. */
+          iIn++;
+          break;
+        }else{
+          /* Character iIn and iIn+1 form an escaped quote character. Skip
+          ** the input cursor past both and copy a single quote character 
+          ** to the output buffer. */
+          iIn += 2;
+          z[iOut++] = q;
+        }
+      }else{
+        z[iOut++] = z[iIn++];
+      }
+    }
+
+    z[iOut] = '\0';
+  }
+}
+
+/*
+** The second argument, argv[], is an array of pointers to nul-terminated
+** strings. This function makes a copy of the array and strings into a 
+** single block of memory. It then dequotes any of the strings that appear
+** to be quoted.
+**
+** If successful, output parameter *pazDequote is set to point at the
+** array of dequoted strings and SQLITE_OK is returned. The caller is
+** responsible for eventually calling sqlite3_free() to free the array
+** in this case. Or, if an error occurs, an SQLite error code is returned.
+** The final value of *pazDequote is undefined in this case.
+*/
+static int fts5tokDequoteArray(
+  int argc,                       /* Number of elements in argv[] */
+  const char * const *argv,       /* Input array */
+  char ***pazDequote              /* Output array */
+){
+  int rc = SQLITE_OK;             /* Return code */
+  if( argc==0 ){
+    *pazDequote = 0;
+  }else{
+    int i;
+    int nByte = 0;
+    char **azDequote;
+
+    for(i=0; i<argc; i++){
+      nByte += (int)(strlen(argv[i]) + 1);
+    }
+
+    *pazDequote = azDequote = sqlite3_malloc(sizeof(char *)*argc + nByte);
+    if( azDequote==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      char *pSpace = (char *)&azDequote[argc];
+      for(i=0; i<argc; i++){
+        int n = (int)strlen(argv[i]);
+        azDequote[i] = pSpace;
+        memcpy(pSpace, argv[i], n+1);
+        fts5tokDequote(pSpace);
+        pSpace += (n+1);
+      }
+    }
+  }
+
+  return rc;
+}
+
+/*
+** Schema of the tokenizer table.
+*/
+#define FTS3_TOK_SCHEMA "CREATE TABLE x(input HIDDEN, token, start, end, position)"
+
+/*
+** This function does all the work for both the xConnect and xCreate methods.
+** These tables have no persistent representation of their own, so xConnect
+** and xCreate are identical operations.
+**
+**   argv[0]: module name
+**   argv[1]: database name 
+**   argv[2]: table name
+**   argv[3]: first argument (tokenizer name)
+*/
+static int fts5tokConnectMethod(
+  sqlite3 *db,                    /* Database connection */
+  void *pCtx,                     /* Pointer to fts5_api object */
+  int argc,                       /* Number of elements in argv array */
+  const char * const *argv,       /* xCreate/xConnect argument array */
+  sqlite3_vtab **ppVtab,          /* OUT: New sqlite3_vtab object */
+  char **pzErr                    /* OUT: sqlite3_malloc'd error message */
+){
+  fts5_api *pApi = (fts5_api*)pCtx;
+  Fts5tokTable *pTab = 0;
+  int rc;
+  char **azDequote = 0;
+  int nDequote;
+
+  rc = sqlite3_declare_vtab(db, 
+       "CREATE TABLE x(input HIDDEN, token, start, end, position)"
+  );
+
+  if( rc==SQLITE_OK ){
+    nDequote = argc-3;
+    rc = fts5tokDequoteArray(nDequote, &argv[3], &azDequote);
+  }
+
+  if( rc==SQLITE_OK ){
+    pTab = (Fts5tokTable*)sqlite3_malloc(sizeof(Fts5tokTable));
+    if( pTab==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      memset(pTab, 0, sizeof(Fts5tokTable));
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    void *pTokCtx = 0;
+    const char *zModule = 0;
+    if( nDequote>0 ){
+      zModule = azDequote[0];
+    }
+
+    rc = pApi->xFindTokenizer(pApi, zModule, &pTokCtx, &pTab->tok);
+    if( rc==SQLITE_OK ){
+      const char **azArg = (const char **)&azDequote[1];
+      int nArg = nDequote>0 ? nDequote-1 : 0;
+      rc = pTab->tok.xCreate(pTokCtx, azArg, nArg, &pTab->pTok);
+    }
+  }
+
+  if( rc!=SQLITE_OK ){
+    sqlite3_free(pTab);
+    pTab = 0;
+  }
+
+  *ppVtab = (sqlite3_vtab*)pTab;
+  sqlite3_free(azDequote);
+  return rc;
+}
+
+/*
+** This function does the work for both the xDisconnect and xDestroy methods.
+** These tables have no persistent representation of their own, so xDisconnect
+** and xDestroy are identical operations.
+*/
+static int fts5tokDisconnectMethod(sqlite3_vtab *pVtab){
+  Fts5tokTable *pTab = (Fts5tokTable *)pVtab;
+  if( pTab->pTok ){
+    pTab->tok.xDelete(pTab->pTok);
+  }
+  sqlite3_free(pTab);
+  return SQLITE_OK;
+}
+
+/*
+** xBestIndex - Analyze a WHERE and ORDER BY clause.
+*/
+static int fts5tokBestIndexMethod(
+  sqlite3_vtab *pVTab, 
+  sqlite3_index_info *pInfo
+){
+  int i;
+
+  for(i=0; i<pInfo->nConstraint; i++){
+    if( pInfo->aConstraint[i].usable 
+     && pInfo->aConstraint[i].iColumn==0 
+     && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ 
+    ){
+      pInfo->idxNum = 1;
+      pInfo->aConstraintUsage[i].argvIndex = 1;
+      pInfo->aConstraintUsage[i].omit = 1;
+      pInfo->estimatedCost = 1;
+      return SQLITE_OK;
+    }
+  }
+
+  pInfo->idxNum = 0;
+  assert( pInfo->estimatedCost>1000000.0 );
+
+  return SQLITE_OK;
+}
+
+/*
+** xOpen - Open a cursor.
+*/
+static int fts5tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
+  Fts5tokCursor *pCsr;
+
+  pCsr = (Fts5tokCursor *)sqlite3_malloc(sizeof(Fts5tokCursor));
+  if( pCsr==0 ){
+    return SQLITE_NOMEM;
+  }
+  memset(pCsr, 0, sizeof(Fts5tokCursor));
+
+  *ppCsr = (sqlite3_vtab_cursor *)pCsr;
+  return SQLITE_OK;
+}
+
+/*
+** Reset the tokenizer cursor passed as the only argument. As if it had
+** just been returned by fts5tokOpenMethod().
+*/
+static void fts5tokResetCursor(Fts5tokCursor *pCsr){
+  int i;
+  for(i=0; i<pCsr->nRow; i++){
+    sqlite3_free(pCsr->aRow[i].zToken);
+  }
+  sqlite3_free(pCsr->zInput);
+  sqlite3_free(pCsr->aRow);
+  pCsr->zInput = 0;
+  pCsr->aRow = 0;
+  pCsr->nRow = 0;
+  pCsr->iRowid = 0;
+}
+
+/*
+** xClose - Close a cursor.
+*/
+static int fts5tokCloseMethod(sqlite3_vtab_cursor *pCursor){
+  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
+  fts5tokResetCursor(pCsr);
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
+/*
+** xNext - Advance the cursor to the next row, if any.
+*/
+static int fts5tokNextMethod(sqlite3_vtab_cursor *pCursor){
+  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
+  pCsr->iRowid++;
+  return SQLITE_OK;
+}
+
+static int fts5tokCb(
+  void *pCtx,         /* Pointer to Fts5tokCursor */
+  int tflags,         /* Mask of FTS5_TOKEN_* flags */
+  const char *pToken, /* Pointer to buffer containing token */
+  int nToken,         /* Size of token in bytes */
+  int iStart,         /* Byte offset of token within input text */
+  int iEnd            /* Byte offset of end of token within input text */
+){
+  Fts5tokCursor *pCsr = (Fts5tokCursor*)pCtx;
+  Fts5tokRow *pRow;
+
+  if( (pCsr->nRow & (pCsr->nRow-1))==0 ){
+    int nNew = pCsr->nRow ? pCsr->nRow*2 : 32;
+    Fts5tokRow *aNew;
+    aNew = (Fts5tokRow*)sqlite3_realloc(pCsr->aRow, nNew*sizeof(Fts5tokRow));
+    if( aNew==0 ) return SQLITE_NOMEM;
+    memset(&aNew[pCsr->nRow], 0, sizeof(Fts5tokRow)*(nNew-pCsr->nRow));
+    pCsr->aRow = aNew;
+  }
+
+  pRow = &pCsr->aRow[pCsr->nRow];
+  pRow->iStart = iStart;
+  pRow->iEnd = iEnd;
+  if( pCsr->nRow ){
+    pRow->iPos = pRow[-1].iPos + ((tflags & FTS5_TOKEN_COLOCATED) ? 0 : 1);
+  }
+  pRow->zToken = sqlite3_malloc(nToken+1);
+  if( pRow->zToken==0 ) return SQLITE_NOMEM;
+  memcpy(pRow->zToken, pToken, nToken);
+  pRow->zToken[nToken] = 0;
+  pCsr->nRow++;
+
+  return SQLITE_OK;
+}
+
+/*
+** xFilter - Initialize a cursor to point at the start of its data.
+*/
+static int fts5tokFilterMethod(
+  sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
+  int idxNum,                     /* Strategy index */
+  const char *idxStr,             /* Unused */
+  int nVal,                       /* Number of elements in apVal */
+  sqlite3_value **apVal           /* Arguments for the indexing scheme */
+){
+  int rc = SQLITE_ERROR;
+  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
+  Fts5tokTable *pTab = (Fts5tokTable *)(pCursor->pVtab);
+
+  fts5tokResetCursor(pCsr);
+  if( idxNum==1 ){
+    const char *zByte = (const char *)sqlite3_value_text(apVal[0]);
+    int nByte = sqlite3_value_bytes(apVal[0]);
+    pCsr->zInput = sqlite3_malloc(nByte+1);
+    if( pCsr->zInput==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      memcpy(pCsr->zInput, zByte, nByte);
+      pCsr->zInput[nByte] = 0;
+      rc = pTab->tok.xTokenize(
+          pTab->pTok, (void*)pCsr, 0, zByte, nByte, fts5tokCb
+      );
+    }
+  }
+
+  if( rc!=SQLITE_OK ) return rc;
+  return fts5tokNextMethod(pCursor);
+}
+
+/*
+** xEof - Return true if the cursor is at EOF, or false otherwise.
+*/
+static int fts5tokEofMethod(sqlite3_vtab_cursor *pCursor){
+  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
+  return (pCsr->iRowid>pCsr->nRow);
+}
+
+/*
+** xColumn - Return a column value.
+*/
+static int fts5tokColumnMethod(
+  sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
+  sqlite3_context *pCtx,          /* Context for sqlite3_result_xxx() calls */
+  int iCol                        /* Index of column to read value from */
+){
+  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
+  Fts5tokRow *pRow = &pCsr->aRow[pCsr->iRowid-1];
+
+  /* CREATE TABLE x(input, token, start, end, position) */
+  switch( iCol ){
+    case 0:
+      sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT);
+      break;
+    case 1:
+      sqlite3_result_text(pCtx, pRow->zToken, -1, SQLITE_TRANSIENT);
+      break;
+    case 2:
+      sqlite3_result_int(pCtx, pRow->iStart);
+      break;
+    case 3:
+      sqlite3_result_int(pCtx, pRow->iEnd);
+      break;
+    default:
+      assert( iCol==4 );
+      sqlite3_result_int(pCtx, pRow->iPos);
+      break;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** xRowid - Return the current rowid for the cursor.
+*/
+static int fts5tokRowidMethod(
+  sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
+  sqlite_int64 *pRowid            /* OUT: Rowid value */
+){
+  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
+  *pRowid = (sqlite3_int64)pCsr->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Register the fts5tok module with database connection db. Return SQLITE_OK
+** if successful or an error code if sqlite3_create_module() fails.
+*/
+int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){
+  static const sqlite3_module fts5tok_module = {
+     0,                           /* iVersion      */
+     fts5tokConnectMethod,        /* xCreate       */
+     fts5tokConnectMethod,        /* xConnect      */
+     fts5tokBestIndexMethod,      /* xBestIndex    */
+     fts5tokDisconnectMethod,     /* xDisconnect   */
+     fts5tokDisconnectMethod,     /* xDestroy      */
+     fts5tokOpenMethod,           /* xOpen         */
+     fts5tokCloseMethod,          /* xClose        */
+     fts5tokFilterMethod,         /* xFilter       */
+     fts5tokNextMethod,           /* xNext         */
+     fts5tokEofMethod,            /* xEof          */
+     fts5tokColumnMethod,         /* xColumn       */
+     fts5tokRowidMethod,          /* xRowid        */
+     0,                           /* xUpdate       */
+     0,                           /* xBegin        */
+     0,                           /* xSync         */
+     0,                           /* xCommit       */
+     0,                           /* xRollback     */
+     0,                           /* xFindFunction */
+     0,                           /* xRename       */
+     0,                           /* xSavepoint    */
+     0,                           /* xRelease      */
+     0                            /* xRollbackTo   */
+  };
+  int rc;                         /* Return code */
+
+  rc = sqlite3_create_module(db, "fts5tokenize", &fts5tok_module, (void*)pApi);
+  return rc;
+}
+
+#endif /* defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5) */

Index: ext/fts5/fts5_vocab.c
==================================================================
--- ext/fts5/fts5_vocab.c
+++ ext/fts5/fts5_vocab.c
@@ -425,10 +425,14 @@
                 int iCol = -1;
                 while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
                   int ii = FTS5_POS2COLUMN(iPos);
                   pCsr->aCnt[ii]++;
                   if( iCol!=ii ){
+                    if( ii>=nCol ){
+                      rc = FTS5_CORRUPT;
+                      break;
+                    }
                     pCsr->aDoc[ii]++;
                     iCol = ii;
                   }
                 }
               }
@@ -442,11 +446,15 @@
               Fts5Buffer buf = {0, 0, 0};
               rc = sqlite3Fts5IterPoslistBuffer(pCsr->pIter, &buf);
               if( rc==SQLITE_OK ){
                 while( 0==sqlite3Fts5PoslistNext64(buf.p, buf.n, &iOff,&iPos) ){
                   assert_nc( iPos>=0 && iPos<nCol );
-                  if( iPos<nCol ) pCsr->aDoc[iPos]++;
+                  if( iPos>=nCol ){
+                    rc = FTS5_CORRUPT;
+                    break;
+                  }
+                  pCsr->aDoc[iPos]++;
                 }
               }
               sqlite3Fts5BufferFree(&buf);
             }
             break;
@@ -470,11 +478,11 @@
         }
       }
     }
   }
 
-  if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
+  if( rc==SQLITE_OK && pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
     while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++;
     assert( pCsr->iCol<pCsr->pConfig->nCol );
   }
   return rc;
 }

Index: ext/fts5/test/fts5_common.tcl
==================================================================
--- ext/fts5/test/fts5_common.tcl
+++ ext/fts5/test/fts5_common.tcl
@@ -12,15 +12,26 @@
 
 if {![info exists testdir]} {
   set testdir [file join [file dirname [info script]] .. .. .. test]
 }
 source $testdir/tester.tcl
+
+ifcapable !fts5 {
+  finish_test
+  return
+}
 
 catch { 
   sqlite3_fts5_may_be_corrupt 0 
   reset_db
 }
+
+# If SQLITE_ENABLE_FTS5 is not defined, skip this test.
+ifcapable !fts5 {
+  finish_test
+  return
+}
 
 proc fts5_test_poslist {cmd} {
   set res [list]
   for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
     lappend res [string map {{ } .} [$cmd xInst $i]]
@@ -507,10 +518,26 @@
   foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] {
     lappend res $rowid $poslist
   }
   set res
 }
+
+proc fts5_collist_data {expr tbl {order ASC} {aDictVar ""}} {
+  set res [list]
+
+  if {$aDictVar!=""} {
+    upvar $aDictVar aDict
+    set dict aDict
+  } else {
+    set dict ""
+  }
+
+  foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] {
+    lappend res $rowid $collist
+  }
+  set res
+}
 
 #-------------------------------------------------------------------------
 #
 
 # This command will only work inside a [foreach_detail_mode] block. It tests
@@ -559,5 +586,56 @@
     set ::expr_not_ok 1
   }
   list
 }
 
+
+#-------------------------------------------------------------------------
+# Code for a simple Tcl tokenizer that supports synonyms at query time.
+#
+proc tclnum_tokenize {mode tflags text} {
+  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
+    sqlite3_fts5_token $w $iStart $iEnd
+    if {$tflags == $mode && [info exists ::tclnum_syn($w)]} {
+      foreach s $::tclnum_syn($w)  { sqlite3_fts5_token -colo $s $iStart $iEnd }
+    }
+  }
+}
+
+proc tclnum_create {args} {
+  set mode query
+  if {[llength $args]} {
+    set mode [lindex $args 0]
+  }
+  if {$mode != "query" && $mode != "document"} { error "bad mode: $mode" }
+  return [list tclnum_tokenize $mode]
+}
+
+proc fts5_tclnum_register {db} {
+  foreach SYNDICT {
+    {zero  0}
+    {one   1 i}
+    {two   2 ii}
+    {three 3 iii}
+    {four  4 iv}
+    {five  5 v}
+    {six   6 vi}
+    {seven 7 vii}
+    {eight 8 viii}
+    {nine  9 ix}
+
+    {a1 a2 a3 a4 a5 a6 a7 a8 a9}
+    {b1 b2 b3 b4 b5 b6 b7 b8 b9}
+    {c1 c2 c3 c4 c5 c6 c7 c8 c9}
+  } {
+    foreach s $SYNDICT {
+      set o [list]
+      foreach x $SYNDICT {if {$x!=$s} {lappend o $x}}
+      set ::tclnum_syn($s) $o
+    }
+  }
+  sqlite3_fts5_create_tokenizer db tclnum tclnum_create
+}
+#
+# End of tokenizer code.
+#-------------------------------------------------------------------------
+

ADDED   ext/fts5/test/fts5bigtok.test
Index: ext/fts5/test/fts5bigtok.test
==================================================================
--- /dev/null
+++ ext/fts5/test/fts5bigtok.test
@@ -0,0 +1,67 @@
+# 2016 Jan 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5bigtok
+
+proc rndterm {} {
+  set L [list a b c d e f g h i j k l m n o p q r s t u v w x y z]
+  set l [lindex $L [expr int(rand() * [llength $L])]]
+  string repeat $l [expr int(rand() * 5) + 60]
+}
+
+proc rnddoc {n} {
+  set res [list]
+  for {set i 0} {$i < $n} {incr i} {
+    lappend res [rndterm]
+  }
+  set res
+}
+
+foreach_detail_mode $::testprefix {
+  db func rnddoc rnddoc
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
+    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+    CREATE VIRTUAL TABLE t1vocab USING fts5vocab(t1, row);
+
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 )
+    INSERT INTO t1 SELECT rnddoc(3) FROM s;
+
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 )
+    INSERT INTO t1 SELECT rnddoc(3) FROM s;
+  }
+
+  foreach v [db eval {SELECT term FROM t1vocab}] {
+    set res [db eval {SELECT rowid FROM t1($v)}]
+    do_execsql_test 1.[string range $v 0 0] {
+      SELECT rowid FROM t1($v) ORDER BY rowid DESC
+    } [lsort -integer -decr $res]
+  }
+
+  do_execsql_test 2.0 {
+    INSERT INTO t1(t1) VALUES('optimize');
+  }
+
+  foreach v [db eval {SELECT term FROM t1vocab}] {
+    set res [db eval {SELECT rowid FROM t1($v)}]
+    do_execsql_test 2.[string range $v 0 0] {
+      SELECT rowid FROM t1($v) ORDER BY rowid DESC
+    } [lsort -integer -decr $res]
+  }
+}
+
+finish_test
+
+

Index: ext/fts5/test/fts5config.test
==================================================================
--- ext/fts5/test/fts5config.test
+++ ext/fts5/test/fts5config.test
@@ -42,10 +42,12 @@
   1 {prefix=x}  
   2 {prefix='x'}
   3 {prefix='$'}
   4 {prefix='1,2,'}
   5 {prefix=',1'}
+  6 {prefix='1,2,3...'}
+  7 {prefix='1,2,3xyz'}
 } {
   set res [list 1 {malformed prefix=... directive}]
   do_catchsql_test 2.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
 }
 
@@ -157,10 +159,12 @@
 # Errors in:
 #
 #   9.1.* 'pgsz' options.
 #   9.2.* 'automerge' options.
 #   9.3.* 'crisismerge' options.
+#   9.4.* a non-existant option.
+#   9.5.* 'hashsize' options.
 #
 do_execsql_test 9.0 {
   CREATE VIRTUAL TABLE abc USING fts5(a, b);
 } {}
 do_catchsql_test 9.1.1 {
@@ -200,10 +204,20 @@
 } {}
 
 do_catchsql_test 9.4.1 {
   INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1);
 } {1 {SQL logic error or missing database}}
+
+do_catchsql_test 9.5.1 {
+  INSERT INTO abc(abc, rank) VALUES('hashsize', 'not an integer');
+} {1 {SQL logic error or missing database}}
+do_catchsql_test 9.5.2 {
+  INSERT INTO abc(abc, rank) VALUES('hashsize', -500000);
+} {1 {SQL logic error or missing database}}
+do_catchsql_test 9.5.3 {
+  INSERT INTO abc(abc, rank) VALUES('hashsize', 500000);
+} {0 {}}
 
 #-------------------------------------------------------------------------
 # Too many prefix indexes. Maximum allowed is 31.
 #
 foreach {tn spec} {
@@ -211,8 +225,23 @@
   2 {prefix="1 2 3 4", prefix="5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32"}
 } {
   set sql "CREATE VIRTUAL TABLE xyz USING fts5(x, $spec)"
   do_catchsql_test 10.$tn $sql {1 {too many prefix indexes (max 31)}}
 }
+
+#-------------------------------------------------------------------------
+# errors in the detail= option.
+#
+foreach {tn opt} {
+  1 {detail=x}  
+  2 {detail='x'}
+  3 {detail='$'}
+  4 {detail='1,2,'}
+  5 {detail=',1'}
+  6 {detail=''}
+} {
+  set res [list 1 {malformed detail=... directive}]
+  do_catchsql_test 11.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
+}
 
 finish_test
 

Index: ext/fts5/test/fts5fault4.test
==================================================================
--- ext/fts5/test/fts5fault4.test
+++ ext/fts5/test/fts5fault4.test
@@ -38,31 +38,10 @@
   execsql { DROP TABLE xx }
 } -test {
   faultsim_test_result [list 0 {}]
 }
 
-#-------------------------------------------------------------------------
-# An OOM within an "ORDER BY rank" query.
-#
-db func rnddoc fts5_rnddoc 
-do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE xx USING fts5(x);
-  INSERT INTO xx VALUES ('abc ' || rnddoc(10));
-  INSERT INTO xx VALUES ('abc abc' || rnddoc(9));
-  INSERT INTO xx VALUES ('abc abc abc' || rnddoc(8));
-} {}
-faultsim_save_and_close
-
-do_faultsim_test 2 -faults oom-* -prep {
-  faultsim_restore_and_reopen
-  execsql { SELECT * FROM xx }
-} -body {
-  execsql { SELECT rowid FROM xx WHERE xx MATCH 'abc' ORDER BY rank }
-} -test {
-  faultsim_test_result [list 0 {3 2 1}]
-}
-
 #-------------------------------------------------------------------------
 # An OOM while "reseeking" an FTS cursor.
 #
 do_execsql_test 3.0 {
   CREATE VIRTUAL TABLE jj USING fts5(j);

Index: ext/fts5/test/fts5fault5.test
==================================================================
--- ext/fts5/test/fts5fault5.test
+++ ext/fts5/test/fts5fault5.test
@@ -63,23 +63,30 @@
 } -test {
   faultsim_test_result {0 {}}
 }
 
 #-------------------------------------------------------------------------
-# OOM while scanning an fts5vocab table.
+# OOM while scanning fts5vocab tables.
 #
 reset_db
 do_test 3.0 {
   execsql {
     CREATE VIRTUAL TABLE tt USING fts5(x);
     CREATE VIRTUAL TABLE tv USING fts5vocab(tt, 'row');
+
+    CREATE VIRTUAL TABLE tt2 USING fts5(x, detail=col);
+    CREATE VIRTUAL TABLE tv2 USING fts5vocab(tt2, 'col');
+
     INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
+    INSERT INTO tt2(tt2, rank) VALUES('pgsz', 32);
     BEGIN;
   }
+
   for {set i 0} {$i < 20} {incr i} {
     set str [string repeat "$i " 50]
     execsql { INSERT INTO tt VALUES($str) }
+    execsql { INSERT INTO tt2 VALUES($str) }
   }
   execsql COMMIT
 } {}
 
 do_faultsim_test 3.1 -faults oom-t* -body {
@@ -95,10 +102,32 @@
     SELECT term FROM tv WHERE term BETWEEN '1' AND '2';
   }
 } -test {
   faultsim_test_result {0 {1 10 11 12 13 14 15 16 17 18 19 2}}
 }
+
+breakpoint
+do_execsql_test 3.3.0 {
+  SELECT * FROM tv2;
+} {
+  0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {}        
+  14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19  x 1 {}     
+  2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {}   
+  9 x 1 {}
+}
+do_faultsim_test 3.3 -faults oom-t* -body {
+  db eval {
+    SELECT * FROM tv2;
+  }
+} -test {
+  faultsim_test_result [list 0 [list                                   \
+      0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {}        \
+      14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19  x 1 {}     \
+      2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {}   \
+      9 x 1 {}
+  ]]
+}
 
 
 
 finish_test
 

Index: ext/fts5/test/fts5fault8.test
==================================================================
--- ext/fts5/test/fts5fault8.test
+++ ext/fts5/test/fts5fault8.test
@@ -22,28 +22,34 @@
   return
 }
 
 foreach_detail_mode $testprefix {
 
-if {[detail_is_none]==0} continue
-
 fts5_aux_test_functions db
 do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
   INSERT INTO t1 VALUES('a b c d', '1 2 3 4');
   INSERT INTO t1 VALUES('a b a b', NULL);
   INSERT INTO t1 VALUES(NULL, '1 2 1 2');
 }
 
-do_faultsim_test 1 -faults oom-t* -body {
+do_faultsim_test 1 -faults oom-* -body {
   execsql { 
     SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2' 
   }
 } -test {
   faultsim_test_result {0 {1 {0.0.1 1.1.1} 2 {0.0.1 0.0.3} 3 {1.1.1 1.1.3}}} \
                        {1 SQLITE_NOMEM}
 }
+
+do_faultsim_test 2 -faults oom-* -body {
+  execsql { 
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+} -test {
+  faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
+}
 
 }
 
 finish_test
 

ADDED   ext/fts5/test/fts5fault9.test
Index: ext/fts5/test/fts5fault9.test
==================================================================
--- /dev/null
+++ ext/fts5/test/fts5fault9.test
@@ -0,0 +1,156 @@
+# 2015 September 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# This file is focused on OOM errors.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5fault9
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+
+fts5_aux_test_functions db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+  WITH seq(s) AS ( SELECT 1 UNION ALL SELECT s+1 FROM seq WHERE s<50)
+  INSERT INTO t1 SELECT 'x x x y y y', 'a b c d e f' FROM seq;
+}
+
+do_faultsim_test 1 -faults oom-* -body {
+  execsql { SELECT count(*) FROM t1('x AND y') }
+} -test {
+  faultsim_test_result {0 50}
+}
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, detail=%DETAIL%);
+  INSERT INTO t2(t2, rank) VALUES('pgsz', 32);
+  INSERT INTO t2 VALUES('abc cba', 'cba abc');
+  INSERT INTO t2 VALUES('abc cba', 'cba abc');
+  INSERT INTO t2 VALUES('abc cba', 'cba abc');
+
+  INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
+  INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
+  INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
+}
+
+do_faultsim_test 2 -faults oom-* -body {
+  execsql { SELECT count(*) FROM t2('a* AND c*') }
+} -test {
+  faultsim_test_result {0 6}
+}
+
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t3 USING fts5(a, detail=%DETAIL%);
+  INSERT INTO t3 VALUES('a x x a x a a a');
+  INSERT INTO t3 VALUES('x a a x a x x x');
+}
+
+do_faultsim_test 3.1 -faults oom-* -body {
+  execsql { SELECT highlight(t3, 0, '[', ']') FROM t3('a') }
+} -test {
+  faultsim_test_result {0 {{[a] x x [a] x [a] [a] [a]} {x [a] [a] x [a] x x x}}}
+}
+
+do_faultsim_test 3.2 -faults oom-t* -body {
+  execsql { SELECT fts5_test_poslist2(t3) FROM t3('x') }
+} -test {
+  faultsim_test_result \
+      {0 {{0.0.1 0.0.2 0.0.4} {0.0.0 0.0.3 0.0.5 0.0.6 0.0.7}}} \
+      {1 SQLITE_NOMEM}
+}
+
+#-------------------------------------------------------------------------
+# Test OOM injection with the xPhraseFirstColumn() API and a tokenizer
+# uses query synonyms.
+#
+fts5_tclnum_register db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t4 USING fts5(x, y, z, detail=%DETAIL%, tokenize=tclnum);
+  INSERT INTO t4 VALUES('one two three', '1 2 3', 'i ii iii');
+  INSERT INTO t4 VALUES('1 2 3', 'i ii iii', 'one two three');
+  INSERT INTO t4 VALUES('i ii iii', 'one two three', 'i ii iii');
+
+  INSERT INTO t4 VALUES('a1 a2 a3', 'a4 a5 a6', 'a7 a8 a9');
+  INSERT INTO t4 VALUES('b1 b2 b3', 'b4 b5 b6', 'b7 b8 b9');
+  INSERT INTO t4 VALUES('c1 c2 c3', 'c4 c5 c6', 'c7 c8 c9');
+}
+
+do_faultsim_test 4.1 -faults oom-t* -body {
+  execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('2') }
+} -test {
+  faultsim_test_result \
+      {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} {1 SQLITE_NOMEM}
+}
+
+do_faultsim_test 4.2 -faults oom-t* -body {
+  execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('a5 OR b5 OR c5') }
+} -test {
+  faultsim_test_result \
+      {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} {1 SQLITE_NOMEM}
+}
+
+
+#-------------------------------------------------------------------------
+# An OOM within an "ORDER BY rank" query.
+#
+db func rnddoc fts5_rnddoc 
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=%DETAIL%);
+  INSERT INTO xx VALUES ('def', 'abc ' || rnddoc(10));
+  INSERT INTO xx VALUES ('def', 'abc abc' || rnddoc(9));
+  INSERT INTO xx VALUES ('def', 'abc abc abc' || rnddoc(8));
+} {}
+faultsim_save_and_close
+
+do_faultsim_test 5 -faults oom-* -prep {
+  faultsim_restore_and_reopen
+  execsql { SELECT * FROM xx }
+} -body {
+  execsql { SELECT rowid FROM xx('abc AND def') ORDER BY rank }
+} -test {
+  faultsim_test_result [list 0 {3 2 1}]
+}
+
+set doc [string repeat "xyz " 500]
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE yy USING fts5(y, detail=%DETAIL%);
+  INSERT INTO yy(yy, rank) VALUES('pgsz', 64);
+  INSERT INTO yy VALUES ($doc);
+  INSERT INTO yy VALUES ('1 2 3');
+  INSERT INTO yy VALUES ('xyz');
+  UPDATE yy SET y = y WHERE rowid = 1;
+  UPDATE yy SET y = y WHERE rowid = 1;
+  UPDATE yy SET y = y WHERE rowid = 1;
+  UPDATE yy SET y = y WHERE rowid = 1;
+} {}
+
+do_faultsim_test 6 -faults oom-* -body {
+  execsql { SELECT rowid FROM yy('xyz') }
+} -test {
+  faultsim_test_result [list 0 {1 3}]
+}
+
+
+} ;# foreach_detail_mode...
+
+finish_test
+

Index: ext/fts5/test/fts5hash.test
==================================================================
--- ext/fts5/test/fts5hash.test
+++ ext/fts5/test/fts5hash.test
@@ -62,15 +62,17 @@
     lappend doc [lindex $vocab $j]
   }
   return $doc
 }
 
+foreach_detail_mode $testprefix {
+
 set vocab [build_vocab1]
 db func r random_doc 
 
 do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE eee USING fts5(e, ee);
+  CREATE VIRTUAL TABLE eee USING fts5(e, ee, detail=%DETAIL%);
   BEGIN;
     WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
     INSERT INTO eee SELECT r($vocab, 5), r($vocab, 7) FROM ii;
     INSERT INTO eee(eee) VALUES('integrity-check');
   COMMIT;
@@ -89,22 +91,24 @@
   for {set i 1} {$i <= 100} {incr i} {
     execsql { INSERT INTO eee VALUES( r($vocab, 5), r($vocab, 7) ) }
   }
 } {}
   
-do_test 1.2 {
+do_test 1.3 {
   db eval { SELECT term, doc FROM vocab } {
     set nRow [db one {SELECT count(*) FROM eee WHERE eee MATCH $term}]
     if {$nRow != $doc} {
       error "term=$term fts5vocab=$doc cnt=$nRow"
     }
   }
   set {} {}
 } {}
 
-do_execsql_test 1.3 {
+do_execsql_test 1.4 {
   COMMIT;
   INSERT INTO eee(eee) VALUES('integrity-check');
 }
+
+} ;# foreach_detail_mode
 
 finish_test
 

Index: ext/fts5/test/fts5integrity.test
==================================================================
--- ext/fts5/test/fts5integrity.test
+++ ext/fts5/test/fts5integrity.test
@@ -143,12 +143,71 @@
 
 do_execsql_test 5.2 {
   INSERT INTO gg(gg) VALUES('optimize');
 }
 
-breakpoint
 do_execsql_test 5.3 {
   INSERT INTO gg(gg) VALUES('integrity-check');
 }
+
+do_test 5.4.1 {
+  set ok 0
+  for {set i 0} {$i < 10000} {incr i} {
+    set T [format %.5d $i]
+    set res  [db eval { SELECT rowid FROM gg($T) ORDER BY rowid ASC  }]
+    set res2 [db eval { SELECT rowid FROM gg($T) ORDER BY rowid DESC }]
+    if {$res == [lsort -integer $res2]} { incr ok }
+  }
+  set ok
+} {10000}
+
+do_test 5.4.2 {
+  set ok 0
+  for {set i 0} {$i < 100} {incr i} {
+    set T "[format %.3d $i]*"
+    set res  [db eval { SELECT rowid FROM gg($T) ORDER BY rowid ASC  }]
+    set res2 [db eval { SELECT rowid FROM gg($T) ORDER BY rowid DESC }]
+    if {$res == [lsort -integer $res2]} { incr ok }
+  }
+  set ok
+} {100}
+
+#-------------------------------------------------------------------------
+# Similar to 5.*.
+#
+foreach {tn pgsz} {
+  1  32
+  2  36
+  3  40
+  4  44
+  5  48
+} {
+  do_execsql_test 6.$tn.1 {
+    DROP TABLE IF EXISTS hh;
+    CREATE VIRTUAL TABLE hh USING fts5(y);
+    INSERT INTO hh(hh, rank) VALUES('pgsz', $pgsz);
+
+    WITH s(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<999)
+     INSERT INTO hh SELECT printf("%.3d%.3d%.3d %.3d%.3d%.3d",i,i,i,i+1,i+1,i+1)
+     FROM s;
+
+    WITH s(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<999)
+     INSERT INTO hh SELECT printf("%.3d%.3d%.3d %.3d%.3d%.3d",i,i,i,i+1,i+1,i+1)
+     FROM s;
+
+    INSERT INTO hh(hh) VALUES('optimize');
+  }
+
+  do_test 6.$tn.2 {
+    set ok 0
+    for {set i 0} {$i < 1000} {incr i} {
+      set T [format %.3d%.3d%.3d $i $i $i]
+      set res  [db eval { SELECT rowid FROM hh($T) ORDER BY rowid ASC  }]
+      set res2 [db eval { SELECT rowid FROM hh($T) ORDER BY rowid DESC }]
+      if {$res == [lsort -integer $res2]} { incr ok }
+    }
+    set ok
+  } {1000}
+}
 
 finish_test
 

ADDED   ext/fts5/test/fts5merge2.test
Index: ext/fts5/test/fts5merge2.test
==================================================================
--- /dev/null
+++ ext/fts5/test/fts5merge2.test
@@ -0,0 +1,59 @@
+# 2014 Dec 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test that focus on incremental merges of segments.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5merge
+
+proc dump_structure {} {
+  db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} {
+    foreach lvl [lrange $t 1 end] {
+      set seg [string repeat . [expr [llength $lvl]-2]]
+      puts "[lrange $lvl 0 1] $seg"
+    }
+  }
+}
+
+foreach_detail_mode $testprefix {
+
+if {[detail_is_none]==0} continue
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+  INSERT INTO t1(t1, rank) VALUES('crisismerge', 2);
+  INSERT INTO t1 VALUES('1 2 3 4');
+}
+
+expr srand(0)
+db func rnddoc fts5_rnddoc
+do_test 1.1 {
+  for {set i 0} {$i < 100} {incr i} {
+    execsql {
+      BEGIN;
+        DELETE FROM t1 WHERE rowid = 1;
+        INSERT INTO t1(rowid, x) VALUES(1, '1 2 3 4');
+        INSERT INTO t1 VALUES(rnddoc(10));
+      COMMIT;
+    }
+  }
+} {}
+
+do_execsql_test 1.2 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+}
+
+finish_test
+

Index: ext/fts5/test/fts5rowid.test
==================================================================
--- ext/fts5/test/fts5rowid.test
+++ ext/fts5/test/fts5rowid.test
@@ -61,20 +61,20 @@
   WITH r(a, b) AS (
     SELECT rnddoc(6), rnddoc(6) UNION ALL
     SELECT rnddoc(6), rnddoc(6) FROM r
   )
   INSERT INTO x1 SELECT * FROM r LIMIT 10000;
+  DELETE FROM x1 WHERE (rowid%2);
 }
 
 set res [db one {SELECT count(*) FROM x1_data}]
 do_execsql_test 2.3 {
   SELECT count(fts5_decode(rowid, block)) FROM x1_data;
 } $res
 do_execsql_test 2.4 {
   UPDATE x1_data SET block = X'';
-  -- SELECT count(fts5_decode(rowid, block)) FROM x1_data;
-  SELECT count(*) FROM x1_data;
+  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
 } $res
 
 do_execsql_test 2.5 {
   INSERT INTO x1(x1, rank) VALUES('pgsz', 1024);
   INSERT INTO x1(x1) VALUES('rebuild');
@@ -181,8 +181,39 @@
 
 set res [db one {SELECT count(*) FROM x4_data}]
 do_execsql_test 5.2 {
   SELECT count(fts5_decode(rowid, block)) FROM x4_data;
 } $res
+
+#-------------------------------------------------------------------------
+#
+
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE x5 USING fts5(x, detail=none);
+  INSERT INTO x5(x5, rank) VALUES('pgsz', 32);
+  INSERT INTO x5 VALUES('a b c d e f');
+  INSERT INTO x5 VALUES('a b c d e f');
+  INSERT INTO x5 VALUES('a b c d e f');
+  BEGIN;
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+    ) INSERT INTO x5 SELECT 'a b c d e f' FROM s;
+  COMMIT;
+  SELECT count(fts5_decode_none(rowid, block)) FROM x5_data;
+} {32}
+
+do_execsql_test 6.1 {
+  DELETE FROM x5 WHERE rowid <= 2;
+  SELECT count(fts5_decode_none(rowid, block)) FROM x5_data;
+} {34}
+
+do_execsql_test 6.2 {
+  UPDATE x5 SET x='a b c d e f' WHERE rowid=3;
+  SELECT count(fts5_decode_none(rowid, block)) FROM x5_data;
+} {36}
+
+#db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM x5_data} {puts $r}
+
+
 
 finish_test
 

Index: ext/fts5/test/fts5simple2.test
==================================================================
--- ext/fts5/test/fts5simple2.test
+++ ext/fts5/test/fts5simple2.test
@@ -17,12 +17,10 @@
 ifcapable !fts5 {
   finish_test
   return
 }
 
-if 1 {
-
 do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
   INSERT INTO t1 VALUES('a b c');
 }
 do_execsql_test 1.1 {
@@ -265,12 +263,10 @@
 
 do_execsql_test 14.1 {
   SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank;
 } {0.0.1}
 
-}
-
 #-------------------------------------------------------------------------
 #
 reset_db
 do_execsql_test 15.1 {
   CREATE VIRTUAL TABLE t1 USING fts5(x, detail=none);
@@ -297,8 +293,46 @@
 } {}
 
 do_test 15.4 {
   execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
 } {}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 16.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(x, detail=none);
+  BEGIN;
+    INSERT INTO t2(rowid, x) VALUES(1, 'a b c');
+    INSERT INTO t2(rowid, x) VALUES(456, 'a b c');
+    INSERT INTO t2(rowid, x) VALUES(1000, 'a b c');
+  COMMIT;
+  UPDATE t2 SET x=x;
+}
+
+do_execsql_test 16.1 {
+  INSERT INTO t2(t2) VALUES('integrity-check');
+} {}
+
+do_execsql_test 16.2 {
+  SELECT rowid FROM t2('b') ORDER BY rowid DESC
+} {1000 456 1}
+
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 16.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(x, detail=none);
+  BEGIN;
+    INSERT INTO t2(rowid, x) VALUES(1, 'a b c');
+    INSERT INTO t2(rowid, x) VALUES(456, 'a b c');
+    INSERT INTO t2(rowid, x) VALUES(1000, 'a b c');
+  COMMIT;
+  UPDATE t2 SET x=x;
+  DELETE FROM t2;
+}
+
+#db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM t2_data} {puts $r}
   
 finish_test
 

Index: ext/fts5/test/fts5synonym.test
==================================================================
--- ext/fts5/test/fts5synonym.test
+++ ext/fts5/test/fts5synonym.test
@@ -19,46 +19,20 @@
 ifcapable !fts5 {
   finish_test
   return
 }
 
-foreach S {
-  {zero 0}
-  {one 1 i}
-  {two 2 ii}
-  {three 3 iii}
-  {four 4 iv}
-  {five 5 v}
-  {six 6 vi}
-  {seven 7 vii}
-  {eight 8 viii}
-  {nine 9 ix}
-} {
-  foreach s $S {
-    set o [list]
-    foreach x $S {if {$x!=$s} {lappend o $x}}
-    set ::syn($s) $o
-  }
-}
-
-proc tcl_tokenize {tflags text} {
-  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
-    sqlite3_fts5_token $w $iStart $iEnd
-  }
-}
-
-proc tcl_create {args} {
-  return "tcl_tokenize"
-}
-
-sqlite3_fts5_create_tokenizer db tcl tcl_create
+proc tcl_create {args} { return "tcl_tokenize" }
+
+foreach_detail_mode $testprefix {
 
 #-------------------------------------------------------------------------
 # Warm body test for the code in fts5_tcl.c.
 #
+fts5_tclnum_register db
 do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl);
+  CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = "tclnum document", detail=%DETAIL%);
   INSERT INTO ft VALUES('abc def ghi');
   INSERT INTO ft VALUES('jkl mno pqr');
   SELECT rowid, x FROM ft WHERE ft MATCH 'def';
   SELECT x, rowid FROM ft WHERE ft MATCH 'pqr';
 } {1 {abc def ghi} {jkl mno pqr} 2}
@@ -65,26 +39,17 @@
 
 #-------------------------------------------------------------------------
 # Test a tokenizer that supports synonyms by adding extra entries to the
 # FTS index.
 #
-
-proc tcl_tokenize {tflags text} {
-  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
-    sqlite3_fts5_token $w $iStart $iEnd
-    if {$tflags=="document" && [info exists ::syn($w)]} {
-      foreach s $::syn($w) {
-        sqlite3_fts5_token -colo $s $iStart $iEnd
-      }
-    }
-  }
-}
 reset_db
-sqlite3_fts5_create_tokenizer db tcl tcl_create
+fts5_tclnum_register db
 
 do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl);
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize = "tclnum document", detail=%DETAIL%
+  );
   INSERT INTO ft VALUES('one two three');
   INSERT INTO ft VALUES('four five six');
   INSERT INTO ft VALUES('eight nine ten');
 } {}
 
@@ -93,10 +58,11 @@
   2 "eight OR 8 OR 5" {2 3}
   3 "10" {}
   4 "1*" {1}
   5 "1 + 2" {1}
 } {
+  if {![fts5_expr_ok $expr ft]} continue
   do_execsql_test 2.1.$tn {
     SELECT rowid FROM ft WHERE ft MATCH $expr
   } $res
 }
 
@@ -178,33 +144,25 @@
 
 #-------------------------------------------------------------------------
 # Check that expressions with synonyms can be parsed and executed.
 #
 reset_db
-sqlite3_fts5_create_tokenizer db tcl tcl_create
-proc tcl_tokenize {tflags text} {
-  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
-    sqlite3_fts5_token $w $iStart $iEnd
-    if {$tflags=="query" && [info exists ::syn($w)]} {
-      foreach s $::syn($w) {
-        sqlite3_fts5_token -colo $s $iStart $iEnd
-      }
-    }
-  }
-}
+fts5_tclnum_register db
 
 foreach {tn expr res} {
   1  {abc}                           {"abc"}
   2  {one}                           {"one"|"i"|"1"}
   3  {3}                             {"3"|"iii"|"three"}
   4  {3*}                            {"3"|"iii"|"three" *}
 } {
-  do_execsql_test 4.1.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res]
+  do_execsql_test 4.1.$tn {
+    SELECT fts5_expr($expr, 'tokenize=tclnum')
+  } [list $res]
 }
 
 do_execsql_test 4.2.1 {
-  CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tcl);
+  CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tclnum, detail=%DETAIL%);
   INSERT INTO xx VALUES('one two');
   INSERT INTO xx VALUES('three four');
 }
 
 do_execsql_test 4.2.2 {
@@ -215,11 +173,11 @@
   SELECT rowid FROM xx WHERE xx MATCH '3'
 } {2}
 
 do_test 5.0 {
   execsql { 
-    CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tcl)
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tclnum, detail=%DETAIL%)
   }
   foreach {rowid a b} {
     1 {four v 4 i three} {1 3 five five 4 one}
     2 {5 1 3 4 i} {2 2 v two 4}
     3 {5 i 5 2 four 4 1} {iii ii five two 1}
@@ -283,10 +241,11 @@
   6 {"v v"} {
     1 {four v 4 i three} {1 3 [five five] 4 one}
     5 {three i v i four 4 1} {ii [five five five] iii}
   }
 } {
+  if {![fts5_expr_ok $q t1]} continue
   do_execsql_test 5.1.$tn {
     SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']')
     FROM t1 WHERE t1 MATCH $q
   } $res
 }
@@ -314,11 +273,10 @@
   do_execsql_test 5.2.$tn {
     SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH $q
   } $res
 }
 
-
 #-------------------------------------------------------------------------
 # Test terms with more than 4 synonyms.
 #
 reset_db
 sqlite3_fts5_create_tokenizer db tcl tcl_create
@@ -332,21 +290,23 @@
     }
   }
 }
 
 do_execsql_test 6.0.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl, detail=%DETAIL%);
   INSERT INTO t1 VALUES('yy xx qq');
   INSERT INTO t1 VALUES('yy xx xx');
 }
-do_execsql_test 6.0.2 {
-  SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)';
-} {{yy xx qq}}
+if {[fts5_expr_ok "NEAR(y q)" t1]} {
+  do_execsql_test 6.0.2 {
+    SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)';
+  } {{yy xx qq}}
+}
 
 do_test 6.0.3 {
   execsql { 
-    CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl)
+    CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl, detail=%DETAIL%)
   }
   foreach {rowid a b} {
     1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq aaaa}
     2 {ww oooooo bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq}
     3 {zzzz llll gggggg cccc uu} {hhhhhh aaaa ppppp rr ee jjjj}
@@ -385,10 +345,12 @@
   4 {NEAR(q y, 20)} {
     1 {[yyyy] vvvvv [qq] oo [yyyyyy] vvvv eee} {ffff uu r qq aaaa}
     2 {ww oooooo bbbbb ssssss mm} {ffffff [yy] iiii rr s ccc [qqqqq]}
   }
 } {
+  if {![fts5_expr_ok $q t2]} continue
+
   do_execsql_test 6.1.$tn.asc {
     SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']')
     FROM t2 WHERE t2 MATCH $q
   } $res
 
@@ -433,11 +395,11 @@
     }
   }
 }
 
 do_execsql_test 7.0.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl);
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl, detail=%DETAIL%);
   INSERT INTO t1 VALUES('0 2 3', '4 5 6 7');
   INSERT INTO t1 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0');
   SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH '000 AND 00 AND 0';
 } {{3 4} {2 10}}
 
@@ -444,17 +406,19 @@
 do_execsql_test 7.0.2 {
   INSERT INTO t1(t1) VALUES('integrity-check');
 }
 
 do_execsql_test 7.1.1 {
-  CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl);
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl, detail=%DETAIL%);
   INSERT INTO t2 VALUES('0 2 3', '4 5 6 7');
   INSERT INTO t2 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0');
   SELECT fts5_test_columnsize(t2) FROM t2 WHERE t2 MATCH '000 AND 00 AND 0';
 } {{3 4} {2 10}}
 
 do_execsql_test 7.1.2 {
   INSERT INTO t2(t2) VALUES('integrity-check');
 }
+
+} ;# foreach_detail_mode
 
 finish_test
 

Index: ext/fts5/test/fts5synonym2.test
==================================================================
--- ext/fts5/test/fts5synonym2.test
+++ ext/fts5/test/fts5synonym2.test
@@ -19,56 +19,26 @@
 ifcapable !fts5 {
   finish_test
   return
 }
 
-#-------------------------------------------------------------------------
-# Code for a simple Tcl tokenizer that supports synonyms at query time.
-#
-foreach SYNDICT {
-  {zero  0}
-  {one   1 i}
-  {two   2 ii}
-  {three 3 iii}
-  {four  4 iv}
-  {five  5 v}
-  {six   6 vi}
-  {seven 7 vii}
-  {eight 8 viii}
-  {nine  9 ix}
-} {
-  foreach s $SYNDICT {
-    set o [list]
-    foreach x $SYNDICT {if {$x!=$s} {lappend o $x}}
-    set ::syn($s) $o
-  }
-}
-
-proc tcl_tokenize {tflags text} {
-  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
-    sqlite3_fts5_token $w $iStart $iEnd
-    if {$tflags == "query"} {
-      foreach s $::syn($w)  { sqlite3_fts5_token -colo $s $iStart $iEnd }
-    }
-  }
-}
-
-proc tcl_create {args} {
-  return "tcl_tokenize"
-}
-
-#
-# End of tokenizer code.
-#-------------------------------------------------------------------------
-
+foreach tok {query document} {
 foreach_detail_mode $testprefix {
 
-sqlite3_fts5_create_tokenizer db tcl tcl_create
+fts5_tclnum_register db
 fts5_aux_test_functions db
 
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE ss USING fts5(a, b, tokenize=tcl, detail=%DETAIL%);
+proc fts5_rowid {cmd} { expr [$cmd xColumnText -1] }
+sqlite3_fts5_create_function db fts5_rowid fts5_rowid
+
+do_execsql_test 1.$tok.0.1 "
+  CREATE VIRTUAL TABLE ss USING fts5(a, b, 
+       tokenize='tclnum $tok', detail=%DETAIL%);
+  INSERT INTO ss(ss, rank) VALUES('rank', 'fts5_rowid()');
+"
+
+do_execsql_test 1.$tok.0.2 {
   INSERT INTO ss VALUES('5 5 five seven 3 seven i', '2 1 5 0 two 1 i');
   INSERT INTO ss VALUES('six ix iii 7 i vii iii', 'one seven nine 4 9 1 vi');
   INSERT INTO ss VALUES('6 viii i five six zero seven', '5 v iii iv iv 3');
   INSERT INTO ss VALUES('9 ii six 8 1 6', 'six 4 iv iv 7');
   INSERT INTO ss VALUES('1 5 4 eight ii iv iii', 'nine 2 eight ix v vii');
@@ -135,20 +105,25 @@
   4.1 "NEAR(one two, 2)"
   4.2 "NEAR(one two three, 2)"
   4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)"
 } {
   if {[fts5_expr_ok $expr ss]==0} {
-    do_test 1.$tn.OMITTED { list } [list]
+    do_test 1.$tok.$tn.OMITTED { list } [list]
     continue
   }
 
-  set res [fts5_query_data $expr ss ASC ::syn]
-  breakpoint
-  do_execsql_test 1.$tn.[llength $res].asc {
+  set res [fts5_query_data $expr ss ASC ::tclnum_syn]
+  do_execsql_test 1.$tok.$tn.[llength $res].asc.1 {
+    SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr)
+  } $res
+
+  do_execsql_test 1.$tok.$tn.[llength $res].asc.2 {
     SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr)
+    ORDER BY rank ASC
   } $res
 }
 
+}
 }
 
 finish_test
 

ADDED   ext/fts5/test/fts5tok1.test
Index: ext/fts5/test/fts5tok1.test
==================================================================
--- /dev/null
+++ ext/fts5/test/fts5tok1.test
@@ -0,0 +1,115 @@
+# 2016 Jan 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5tok1
+
+
+sqlite3_fts5_register_fts5tokenize db
+
+#-------------------------------------------------------------------------
+# Simple test cases. Using the default (ascii) tokenizer.
+#
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5tokenize(ascii);
+  CREATE VIRTUAL TABLE t2 USING fts5tokenize();
+  CREATE VIRTUAL TABLE t3 USING fts5tokenize(
+      ascii, 'separators', 'xyz', tokenchars, ''''
+  );
+}
+
+foreach {tn tbl} {1 t1 2 t2 3 t3} {
+  do_execsql_test 1.$tn.1 "SELECT input, * FROM $tbl ('one two three')" {
+    {one two three} one   0  3 0 
+    {one two three} two   4  7 1 
+    {one two three} three 8 13 2
+  }
+
+  do_execsql_test 1.$tn.2 "
+    SELECT token FROM $tbl WHERE input = 'OnE tWo tHrEe'
+  " {
+    one two three
+  }
+}
+
+do_execsql_test 1.4 {
+  SELECT token FROM t3 WHERE input = '1x2x3x'
+} {1 2 3}
+
+do_execsql_test 1.5 {
+  SELECT token FROM t1 WHERE input = '1x2x3x'
+} {1x2x3x}
+
+do_execsql_test 1.6 {
+  SELECT token FROM t3 WHERE input = '1''2x3x'
+} {1'2 3}
+
+do_execsql_test 1.7 {
+  SELECT token FROM t3 WHERE input = ''
+} {}
+
+do_execsql_test 1.8 {
+  SELECT token FROM t3 WHERE input = NULL
+} {}
+
+do_execsql_test 1.9 {
+  SELECT input, * FROM t3 WHERE input = 123
+} {123 123 0 3 0}
+
+do_execsql_test 1.10 {
+  SELECT input, * FROM t1 WHERE input = 'a b c' AND token = 'b';
+} {
+  {a b c} b 2 3 1
+}
+
+do_execsql_test 1.11 {
+  SELECT input, * FROM t1 WHERE token = 'b' AND input = 'a b c';
+} {
+  {a b c} b 2 3 1
+}
+
+do_execsql_test 1.12 {
+  SELECT input, * FROM t1 WHERE input < 'b' AND input = 'a b c';
+} {
+  {a b c} a 0 1 0 
+  {a b c} b 2 3 1 
+  {a b c} c 4 5 2
+}
+
+do_execsql_test 1.13.1 {
+  CREATE TABLE c1(x);
+  INSERT INTO c1(x) VALUES('a b c');
+  INSERT INTO c1(x) VALUES('d e f');
+}
+do_execsql_test 1.13.2 {
+  SELECT c1.*, input, t1.* FROM c1, t1 WHERE input = x AND c1.rowid=t1.rowid;
+} {
+  {a b c} {a b c} a 0 1 0 
+  {d e f} {d e f} e 2 3 1 
+}
+
+
+#-------------------------------------------------------------------------
+# Error cases.
+#
+do_catchsql_test 2.0 {
+  CREATE VIRTUAL TABLE tX USING fts5tokenize(nosuchtokenizer);
+} {1 {vtable constructor failed: tX}}
+
+do_catchsql_test 2.1 {
+  CREATE VIRTUAL TABLE t4 USING fts5tokenize;
+  SELECT * FROM t4;
+} {1 {SQL logic error or missing database}}
+
+
+finish_test

ADDED   ext/fts5/test/fts5tok2.test
Index: ext/fts5/test/fts5tok2.test
==================================================================
--- /dev/null
+++ ext/fts5/test/fts5tok2.test
@@ -0,0 +1,47 @@
+# 2016 Jan 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5||!fts3 { finish_test ; return }
+set ::testprefix fts5tok2
+
+sqlite3_fts5_register_fts5tokenize db
+
+#-------------------------------------------------------------------------
+# Simple test cases. Using the default (ascii) tokenizer.
+#
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t5 USING fts5tokenize(unicode61);
+  CREATE VIRTUAL TABLE t3 USING fts3tokenize(unicode61);
+}
+
+do_test 1.1 {
+  array unset -nocomplain A
+
+  for {set i 1} {$i < 65536} {incr i} {
+    set input [format "abc%cxyz" $i]
+      set expect [execsql {
+        SELECT input, token, start, end FROM t3 WHERE input=$input
+    }]
+
+    incr A([llength $expect])
+
+    set res [execsql {
+      SELECT input, token, start, end FROM t5($input)
+    }]
+    if {$res != $expect} {error "failed at i=$i"}
+  }
+} {}
+
+do_test 1.1.nTokenChars=$A(4).nSeparators=$A(8) {} {}
+
+finish_test

ADDED   ext/fts5/test/fts5update.test
Index: ext/fts5/test/fts5update.test
==================================================================
--- /dev/null
+++ ext/fts5/test/fts5update.test
@@ -0,0 +1,121 @@
+# 2016 Jan 16
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5update
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+set docs {
+  "eight zero iv eight 7"            "ix one 8 one three ii one"        
+  "1 9 9 three viii"                 "5 zero ii 6 nine ix 3"            
+  "3 zero 5 2 seven nine"            "two eight viii eight 1"           
+  "4 six two 5 9 vii"                "viii ii four 8 i i iv"            
+  "vii 0 iv seven 7 viii"            "five 1 nine vi seven"             
+  "1 zero zero iii 1"                "one one six 6 nine seven"         
+  "one v 4 zero 4 iii ii"            "2 3 eight six ix"                 
+  "six iv 7 three 5"                 "ix zero 0 8 ii 7 3"               
+  "four six nine 2 vii 3"            "five viii 5 8 0 7"                
+}
+
+foreach_detail_mode $::testprefix {
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
+} {}
+
+do_test 1.1 {
+  foreach {a b} $docs {
+    execsql {INSERT INTO t1 VALUES($a, $b)}
+  }
+} {}
+
+proc update {iRowid iA iB} {
+  set a [lindex $::docs $iA]
+  set b [lindex $::docs $iB]
+  execsql { UPDATE t1 SET a=$a, b=$b WHERE rowid=$iRowid }
+}
+
+set nDoc [llength $::docs]
+foreach n {1 5 10 50 100} {
+  do_test 1.2.$n {
+    execsql BEGIN
+    for {set i 1} {$i <= 1000} {incr i} {
+      set iRowid [expr {int(rand() * ($nDoc/2)) + 1}]
+      set iA [expr {int(rand() * $nDoc)}]
+      set iB [expr {int(rand() * $nDoc)}]
+      update $iRowid $iA $iB
+
+      if {($i % $n)==0} {
+        execsql { COMMIT; BEGIN }
+      }
+
+      if {($i % $n)==100} {
+        execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
+      }
+    }
+    execsql COMMIT
+    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
+  } {}
+}
+
+do_execsql_test 1.3 {
+  UPDATE t1 SET a=a AND b=b;
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+do_test 1.4 {
+  execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 32) }
+  for {set i 0} {$i < 50} {incr i} {
+    execsql { UPDATE t1 SET a=a AND b=b }
+    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
+  }
+} {}
+
+#-------------------------------------------------------------------------
+# Lots of deletes/inserts of the same document with the same rowid.
+#
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE x2 USING fts5(x, detail=%DETAIL%);
+  INSERT INTO x2(x2, rank) VALUES('crisismerge', 2);
+  INSERT INTO x2 VALUES('a b c');
+  INSERT INTO x2 VALUES('a b c');
+}
+do_test 2.1 {
+  for {set i 0} {$i < 1000} {incr i} {
+    execsql { DELETE FROM x2 WHERE rowid = 2 }
+    execsql { INSERT INTO x2(rowid, x) VALUES(2, 'a b c') }
+  }
+} {}
+do_execsql_test 2.1.integrity {
+  INSERT INTO x2(x2) VALUES('integrity-check');
+}
+
+do_test 2.2 {
+  for {set i 0} {$i < 1000} {incr i} {
+    execsql { UPDATE x2 SET x=x WHERE rowid=2 }
+  }
+} {}
+do_execsql_test 2.2.integrity {
+  INSERT INTO x2(x2) VALUES('integrity-check');
+}
+
+}
+finish_test
+
+

Index: ext/fts5/test/fts5vocab.test
==================================================================
--- ext/fts5/test/fts5vocab.test
+++ ext/fts5/test/fts5vocab.test
@@ -54,10 +54,12 @@
   foreach {a b c} $L {
     lappend ret $a {} $b {}
   }
   set ret
 }
+
+if 1 {
 
 do_execsql_test 1.1.1 {
   CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1, detail=%DETAIL%);
   CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row');
   PRAGMA table_info = v1;
@@ -387,9 +389,61 @@
     SELECT count(*) FROM txc, txc_c
       WHERE txc.term = txc_c.term AND txc.col=txc_c.col;
   } {57}
 }
 
+}
+
+#-------------------------------------------------------------------------
+# Test the fts5vocab tables response to a specific types of corruption:
+# where the fts5 index contains hits for columns that do not exist.
+#
+do_execsql_test 8.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a, b, c, detail=%DETAIL%);
+  INSERT INTO x1 VALUES('a b c', 'd e f', 'g h i');
+  INSERT INTO x1 VALUES('g h i', 'a b c', 'd e f');
+  INSERT INTO x1 VALUES('d e f', 'g h i', 'a b c');
+  CREATE VIRTUAL TABLE x1_r USING fts5vocab(x1, row);
+  CREATE VIRTUAL TABLE x1_c USING fts5vocab(x1, col);
+}
+
+set resr [star_from_row {a 3 3 b 3 3 c 3 3 d 3 3 e 3 3 f 3 3 g 3 3 h 3 3 i 3 3}]
+set resc [star_from_col {
+  a a 1 1 a b 1 1 a c 1 1 b a 1 1 
+  b b 1 1 b c 1 1 c a 1 1 c b 1 1 
+  c c 1 1 d a 1 1 d b 1 1 d c 1 1
+  e a 1 1 e b 1 1 e c 1 1 f a 1 1 
+  f b 1 1 f c 1 1 g a 1 1 g b 1 1 
+  g c 1 1 h a 1 1 h b 1 1 h c 1 1 
+  i a 1 1 i b 1 1 i c 1 1
+}]
+if {[detail_is_none]} { set resc [row_to_col $resr] }
+
+do_execsql_test 8.1.1 { SELECT * FROM x1_r; } $resr
+do_execsql_test 8.1.2 { SELECT * FROM x1_c } $resc
+
+do_execsql_test 8.2 {
+  PRAGMA writable_schema = 1;
+  UPDATE sqlite_master 
+  SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(a, detail=%DETAIL%)'
+  WHERE name = 'x1';
+}
+db close
+sqlite3 db test.db
+sqlite3_fts5_may_be_corrupt 1
+
+do_execsql_test 8.2.1 { SELECT * FROM x1_r } $resr
+
+if {[detail_is_none]} {
+  do_execsql_test 8.2.2 { SELECT * FROM x1_c } $resc
+} else {
+  do_catchsql_test 8.2.2 { 
+    SELECT * FROM x1_c 
+  } {1 {database disk image is malformed}}
+}
+
+sqlite3_fts5_may_be_corrupt 0
+
 }
 
 finish_test
 

Index: main.mk
==================================================================
--- main.mk
+++ main.mk
@@ -333,11 +333,12 @@
   $(TOP)/ext/misc/spellfix.c \
   $(TOP)/ext/misc/totype.c \
   $(TOP)/ext/misc/wholenumber.c \
   $(TOP)/ext/misc/vfslog.c \
   $(TOP)/ext/fts5/fts5_tcl.c \
-  $(TOP)/ext/fts5/fts5_test_mi.c
+  $(TOP)/ext/fts5/fts5_test_mi.c \
+  $(TOP)/ext/fts5/fts5_test_tok.c 
 
 
 #TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c
 #TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c
 

Index: src/analyze.c
==================================================================
--- src/analyze.c
+++ src/analyze.c
@@ -476,12 +476,11 @@
 static const FuncDef statInitFuncdef = {
   2+IsStat34,      /* nArg */
   SQLITE_UTF8,     /* funcFlags */
   0,               /* pUserData */
   0,               /* pNext */
-  statInit,        /* xFunc */
-  0,               /* xStep */
+  statInit,        /* xSFunc */
   0,               /* xFinalize */
   "stat_init",     /* zName */
   0,               /* pHash */
   0                /* pDestructor */
 };
@@ -777,12 +776,11 @@
 static const FuncDef statPushFuncdef = {
   2+IsStat34,      /* nArg */
   SQLITE_UTF8,     /* funcFlags */
   0,               /* pUserData */
   0,               /* pNext */
-  statPush,        /* xFunc */
-  0,               /* xStep */
+  statPush,        /* xSFunc */
   0,               /* xFinalize */
   "stat_push",     /* zName */
   0,               /* pHash */
   0                /* pDestructor */
 };
@@ -924,12 +922,11 @@
 static const FuncDef statGetFuncdef = {
   1+IsStat34,      /* nArg */
   SQLITE_UTF8,     /* funcFlags */
   0,               /* pUserData */
   0,               /* pNext */
-  statGet,         /* xFunc */
-  0,               /* xStep */
+  statGet,         /* xSFunc */
   0,               /* xFinalize */
   "stat_get",      /* zName */
   0,               /* pHash */
   0                /* pDestructor */
 };
@@ -941,12 +938,12 @@
 #elif SQLITE_DEBUG
   assert( iParam==STAT_GET_STAT1 );
 #else
   UNUSED_PARAMETER( iParam );
 #endif
-  sqlite3VdbeAddOp3(v, OP_Function0, 0, regStat4, regOut);
-  sqlite3VdbeChangeP4(v, -1, (char*)&statGetFuncdef, P4_FUNCDEF);
+  sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4, regOut,
+                    (char*)&statGetFuncdef, P4_FUNCDEF);
   sqlite3VdbeChangeP5(v, 1 + IsStat34);
 }
 
 /*
 ** Generate code to do an analysis of all indices associated with
@@ -1096,12 +1093,12 @@
 #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
     sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3);
 #endif
     sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1);
     sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2);
-    sqlite3VdbeAddOp3(v, OP_Function0, 0, regStat4+1, regStat4);
-    sqlite3VdbeChangeP4(v, -1, (char*)&statInitFuncdef, P4_FUNCDEF);
+    sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4+1, regStat4,
+                     (char*)&statInitFuncdef, P4_FUNCDEF);
     sqlite3VdbeChangeP5(v, 2+IsStat34);
 
     /* Implementation of the following:
     **
     **   Rewind csr
@@ -1193,12 +1190,12 @@
       sqlite3VdbeAddOp3(v, OP_MakeRecord, regKey, pPk->nKeyCol, regRowid);
       sqlite3ReleaseTempRange(pParse, regKey, pPk->nKeyCol);
     }
 #endif
     assert( regChng==(regStat4+1) );
-    sqlite3VdbeAddOp3(v, OP_Function0, 1, regStat4, regTemp);
-    sqlite3VdbeChangeP4(v, -1, (char*)&statPushFuncdef, P4_FUNCDEF);
+    sqlite3VdbeAddOp4(v, OP_Function0, 1, regStat4, regTemp,
+                     (char*)&statPushFuncdef, P4_FUNCDEF);
     sqlite3VdbeChangeP5(v, 2+IsStat34);
     sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v);
 
     /* Add the entry to the stat1 table. */
     callStatGet(v, regStat4, STAT_GET_STAT1, regStat1);

Index: src/attach.c
==================================================================
--- src/attach.c
+++ src/attach.c
@@ -357,15 +357,15 @@
   sqlite3ExprCode(pParse, pDbname, regArgs+1);
   sqlite3ExprCode(pParse, pKey, regArgs+2);
 
   assert( v || db->mallocFailed );
   if( v ){
-    sqlite3VdbeAddOp3(v, OP_Function0, 0, regArgs+3-pFunc->nArg, regArgs+3);
+    sqlite3VdbeAddOp4(v, OP_Function0, 0, regArgs+3-pFunc->nArg, regArgs+3,
+                      (char *)pFunc, P4_FUNCDEF);
     assert( pFunc->nArg==-1 || (pFunc->nArg&0xff)==pFunc->nArg );
     sqlite3VdbeChangeP5(v, (u8)(pFunc->nArg));
-    sqlite3VdbeChangeP4(v, -1, (char *)pFunc, P4_FUNCDEF);
-
+ 
     /* Code an OP_Expire. For an ATTACH statement, set P1 to true (expire this
     ** statement only). For DETACH, set it to false (expire all existing
     ** statements).
     */
     sqlite3VdbeAddOp1(v, OP_Expire, (type==SQLITE_ATTACH));
@@ -386,12 +386,11 @@
   static const FuncDef detach_func = {
     1,                /* nArg */
     SQLITE_UTF8,      /* funcFlags */
     0,                /* pUserData */
     0,                /* pNext */
-    detachFunc,       /* xFunc */
-    0,                /* xStep */
+    detachFunc,       /* xSFunc */
     0,                /* xFinalize */
     "sqlite_detach",  /* zName */
     0,                /* pHash */
     0                 /* pDestructor */
   };
@@ -407,12 +406,11 @@
   static const FuncDef attach_func = {
     3,                /* nArg */
     SQLITE_UTF8,      /* funcFlags */
     0,                /* pUserData */
     0,                /* pNext */
-    attachFunc,       /* xFunc */
-    0,                /* xStep */
+    attachFunc,       /* xSFunc */
     0,                /* xFinalize */
     "sqlite_attach",  /* zName */
     0,                /* pHash */
     0                 /* pDestructor */
   };

Index: src/btree.c
==================================================================
--- src/btree.c
+++ src/btree.c
@@ -7211,13 +7211,12 @@
     ** This must be done in advance.  Once the balance starts, the cell
     ** offset section of the btree page will be overwritten and we will no
     ** long be able to find the cells if a pointer to each cell is not saved
     ** first.
     */
-    memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*limit);
+    memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow));
     if( pOld->nOverflow>0 ){
-      memset(&b.szCell[b.nCell+limit], 0, sizeof(b.szCell[0])*pOld->nOverflow);
       limit = pOld->aiOvfl[0];
       for(j=0; j<limit; j++){
         b.apCell[b.nCell] = aData + (maskPage & get2byteAligned(piCell));
         piCell += 2;
         b.nCell++;
@@ -8565,10 +8564,18 @@
   */
   if( NEVER(pBt->pCursor) ){
     sqlite3ConnectionBlocked(p->db, pBt->pCursor->pBtree->db);
     return SQLITE_LOCKED_SHAREDCACHE;
   }
+
+  /*
+  ** It is illegal to drop the sqlite_master table on page 1.  But again,
+  ** this error is caught long before reaching this point.
+  */
+  if( NEVER(iTable<2) ){
+    return SQLITE_CORRUPT_BKPT;
+  }
 
   rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0);
   if( rc ) return rc;
   rc = sqlite3BtreeClearTable(p, iTable, 0);
   if( rc ){
@@ -8576,80 +8583,71 @@
     return rc;
   }
 
   *piMoved = 0;
 
-  if( iTable>1 ){
-#ifdef SQLITE_OMIT_AUTOVACUUM
-    freePage(pPage, &rc);
-    releasePage(pPage);
-#else
-    if( pBt->autoVacuum ){
-      Pgno maxRootPgno;
-      sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &maxRootPgno);
-
-      if( iTable==maxRootPgno ){
-        /* If the table being dropped is the table with the largest root-page
-        ** number in the database, put the root page on the free list. 
-        */
-        freePage(pPage, &rc);
-        releasePage(pPage);
-        if( rc!=SQLITE_OK ){
-          return rc;
-        }
-      }else{
-        /* The table being dropped does not have the largest root-page
-        ** number in the database. So move the page that does into the 
-        ** gap left by the deleted root-page.
-        */
-        MemPage *pMove;
-        releasePage(pPage);
-        rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0);
-        if( rc!=SQLITE_OK ){
-          return rc;
-        }
-        rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0);
-        releasePage(pMove);
-        if( rc!=SQLITE_OK ){
-          return rc;
-        }
-        pMove = 0;
-        rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0);
-        freePage(pMove, &rc);
-        releasePage(pMove);
-        if( rc!=SQLITE_OK ){
-          return rc;
-        }
-        *piMoved = maxRootPgno;
-      }
-
-      /* Set the new 'max-root-page' value in the database header. This
-      ** is the old value less one, less one more if that happens to
-      ** be a root-page number, less one again if that is the
-      ** PENDING_BYTE_PAGE.
-      */
-      maxRootPgno--;
-      while( maxRootPgno==PENDING_BYTE_PAGE(pBt)
-             || PTRMAP_ISPAGE(pBt, maxRootPgno) ){
-        maxRootPgno--;
-      }
-      assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) );
-
-      rc = sqlite3BtreeUpdateMeta(p, 4, maxRootPgno);
-    }else{
-      freePage(pPage, &rc);
-      releasePage(pPage);
-    }
-#endif
-  }else{
-    /* If sqlite3BtreeDropTable was called on page 1.
-    ** This really never should happen except in a corrupt
-    ** database. 
-    */
-    zeroPage(pPage, PTF_INTKEY|PTF_LEAF );
-    releasePage(pPage);
-  }
+#ifdef SQLITE_OMIT_AUTOVACUUM
+  freePage(pPage, &rc);
+  releasePage(pPage);
+#else
+  if( pBt->autoVacuum ){
+    Pgno maxRootPgno;
+    sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &maxRootPgno);
+
+    if( iTable==maxRootPgno ){
+      /* If the table being dropped is the table with the largest root-page
+      ** number in the database, put the root page on the free list. 
+      */
+      freePage(pPage, &rc);
+      releasePage(pPage);
+      if( rc!=SQLITE_OK ){
+        return rc;
+      }
+    }else{
+      /* The table being dropped does not have the largest root-page
+      ** number in the database. So move the page that does into the 
+      ** gap left by the deleted root-page.
+      */
+      MemPage *pMove;
+      releasePage(pPage);
+      rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0);
+      if( rc!=SQLITE_OK ){
+        return rc;
+      }
+      rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0);
+      releasePage(pMove);
+      if( rc!=SQLITE_OK ){
+        return rc;
+      }
+      pMove = 0;
+      rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0);
+      freePage(pMove, &rc);
+      releasePage(pMove);
+      if( rc!=SQLITE_OK ){
+        return rc;
+      }
+      *piMoved = maxRootPgno;
+    }
+
+    /* Set the new 'max-root-page' value in the database header. This
+    ** is the old value less one, less one more if that happens to
+    ** be a root-page number, less one again if that is the
+    ** PENDING_BYTE_PAGE.
+    */
+    maxRootPgno--;
+    while( maxRootPgno==PENDING_BYTE_PAGE(pBt)
+           || PTRMAP_ISPAGE(pBt, maxRootPgno) ){
+      maxRootPgno--;
+    }
+    assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) );
+
+    rc = sqlite3BtreeUpdateMeta(p, 4, maxRootPgno);
+  }else{
+    freePage(pPage, &rc);
+    releasePage(pPage);
+  }
+#endif
   return rc;  
 }
 int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){
   int rc;
   sqlite3BtreeEnter(p);

Index: src/build.c
==================================================================
--- src/build.c
+++ src/build.c
@@ -226,19 +226,23 @@
     /* A minimum of one cursor is required if autoincrement is used
     *  See ticket [a696379c1f08866] */
     if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1;
     sqlite3VdbeMakeReady(v, pParse);
     pParse->rc = SQLITE_DONE;
-    pParse->colNamesSet = 0;
   }else{
     pParse->rc = SQLITE_ERROR;
   }
+
+  /* We are done with this Parse object. There is no need to de-initialize it */
+#if 0
+  pParse->colNamesSet = 0;
   pParse->nTab = 0;
   pParse->nMem = 0;
   pParse->nSet = 0;
   pParse->nVar = 0;
   DbMaskZero(pParse->cookieMask);
+#endif
 }
 
 /*
 ** Run the parser and code generator recursively in order to generate
 ** code for the SQL statement given onto the end of the pParse context
@@ -493,11 +497,10 @@
     if( j<i ){
       db->aDb[j] = db->aDb[i];
     }
     j++;
   }
-  memset(&db->aDb[j], 0, (db->nDb-j)*sizeof(db->aDb[j]));
   db->nDb = j;
   if( db->nDb<=2 && db->aDb!=db->aDbStatic ){
     memcpy(db->aDbStatic, db->aDb, 2*sizeof(db->aDb[0]));
     sqlite3DbFree(db, db->aDb);
     db->aDb = db->aDbStatic;
@@ -756,11 +759,12 @@
   Token **pUnqual     /* Write the unqualified object name here */
 ){
   int iDb;                    /* Database holding the object */
   sqlite3 *db = pParse->db;
 
-  if( ALWAYS(pName2!=0) && pName2->n>0 ){
+  assert( pName2!=0 );
+  if( pName2->n>0 ){
     if( db->init.busy ) {
       sqlite3ErrorMsg(pParse, "corrupt database");
       return -1;
     }
     *pUnqual = pName2;
@@ -845,66 +849,50 @@
   sqlite3 *db = pParse->db;
   Vdbe *v;
   int iDb;         /* Database number to create the table in */
   Token *pName;    /* Unqualified name of the table to create */
 
-  /* The table or view name to create is passed to this routine via tokens
-  ** pName1 and pName2. If the table name was fully qualified, for example:
-  **
-  ** CREATE TABLE xxx.yyy (...);
-  ** 
-  ** Then pName1 is set to "xxx" and pName2 "yyy". On the other hand if
-  ** the table name is not fully qualified, i.e.:
-  **
-  ** CREATE TABLE yyy(...);
-  **
-  ** Then pName1 is set to "yyy" and pName2 is "".
-  **
-  ** The call below sets the pName pointer to point at the token (pName1 or
-  ** pName2) that stores the unqualified table name. The variable iDb is
-  ** set to the index of the database that the table or view is to be
-  ** created in.
-  */
-  iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName);
-  if( iDb<0 ) return;
-  if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){
-    /* If creating a temp table, the name may not be qualified. Unless 
-    ** the database name is "temp" anyway.  */
-    sqlite3ErrorMsg(pParse, "temporary table name must be unqualified");
-    return;
-  }
-  if( !OMIT_TEMPDB && isTemp ) iDb = 1;
-
-  pParse->sNameToken = *pName;
-  zName = sqlite3NameFromToken(db, pName);
+  if( db->init.busy && db->init.newTnum==1 ){
+    /* Special case:  Parsing the sqlite_master or sqlite_temp_master schema */
+    iDb = db->init.iDb;
+    zName = sqlite3DbStrDup(db, SCHEMA_TABLE(iDb));
+    pName = pName1;
+  }else{
+    /* The common case */
+    iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName);
+    if( iDb<0 ) return;
+    if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){
+      /* If creating a temp table, the name may not be qualified. Unless 
+      ** the database name is "temp" anyway.  */
+      sqlite3ErrorMsg(pParse, "temporary table name must be unqualified");
+      return;
+    }
+    if( !OMIT_TEMPDB && isTemp ) iDb = 1;
+    zName = sqlite3NameFromToken(db, pName);
+  }
+  pParse->sNameToken = *pName;
   if( zName==0 ) return;
   if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){
     goto begin_table_error;
   }
   if( db->init.iDb==1 ) isTemp = 1;
 #ifndef SQLITE_OMIT_AUTHORIZATION
-  assert( (isTemp & 1)==isTemp );
+  assert( isTemp==0 || isTemp==1 );
+  assert( isView==0 || isView==1 );
   {
-    int code;
+    static const u8 aCode[] = {
+       SQLITE_CREATE_TABLE,
+       SQLITE_CREATE_TEMP_TABLE,
+       SQLITE_CREATE_VIEW,
+       SQLITE_CREATE_TEMP_VIEW
+    };
     char *zDb = db->aDb[iDb].zName;
     if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(isTemp), 0, zDb) ){
       goto begin_table_error;
     }
-    if( isView ){
-      if( !OMIT_TEMPDB && isTemp ){
-        code = SQLITE_CREATE_TEMP_VIEW;
-      }else{
-        code = SQLITE_CREATE_VIEW;
-      }
-    }else{
-      if( !OMIT_TEMPDB && isTemp ){
-        code = SQLITE_CREATE_TEMP_TABLE;
-      }else{
-        code = SQLITE_CREATE_TABLE;
-      }
-    }
-    if( !isVirtual && sqlite3AuthCheck(pParse, code, zName, 0, zDb) ){
+    if( !isVirtual && sqlite3AuthCheck(pParse, (int)aCode[isTemp+2*isView],
+                                       zName, 0, zDb) ){
       goto begin_table_error;
     }
   }
 #endif
 
@@ -1862,13 +1850,17 @@
   /* If the db->init.busy is 1 it means we are reading the SQL off the
   ** "sqlite_master" or "sqlite_temp_master" table on the disk.
   ** So do not write to the disk again.  Extract the root page number
   ** for the table from the db->init.newTnum field.  (The page number
   ** should have been put there by the sqliteOpenCb routine.)
+  **
+  ** If the root page number is 1, that means this is the sqlite_master
+  ** table itself.  So mark it read-only.
   */
   if( db->init.busy ){
     p->tnum = db->init.newTnum;
+    if( p->tnum==1 ) p->tabFlags |= TF_Readonly;
   }
 
   /* Special processing for WITHOUT ROWID Tables */
   if( tabOpts & TF_WithoutRowid ){
     if( (p->tabFlags & TF_Autoincrement) ){
@@ -2317,10 +2309,11 @@
 ** erasing iTable (this can happen with an auto-vacuum database).
 */ 
 static void destroyRootPage(Parse *pParse, int iTable, int iDb){
   Vdbe *v = sqlite3GetVdbe(pParse);
   int r1 = sqlite3GetTempReg(pParse);
+  assert( iTable>1 );
   sqlite3VdbeAddOp3(v, OP_Destroy, iTable, r1, iDb);
   sqlite3MayAbort(pParse);
 #ifndef SQLITE_OMIT_AUTOVACUUM
   /* OP_Destroy stores an in integer r1. If this integer
   ** is non-zero, then it is the root page number of a table moved to
@@ -3715,13 +3708,14 @@
   Token *pDatabase    /* Database of the table */
 ){
   struct SrcList_item *pItem;
   assert( pDatabase==0 || pTable!=0 );  /* Cannot have C without B */
   if( pList==0 ){
-    pList = sqlite3DbMallocZero(db, sizeof(SrcList) );
+    pList = sqlite3DbMallocRaw(db, sizeof(SrcList) );
     if( pList==0 ) return 0;
     pList->nAlloc = 1;
+    pList->nSrc = 0;
   }
   pList = sqlite3SrcListEnlarge(db, pList, 1, pList->nSrc);
   if( db->mallocFailed ){
     sqlite3SrcListDelete(db, pList);
     return 0;
@@ -4120,11 +4114,11 @@
   assert( (errCode&0xff)==SQLITE_CONSTRAINT );
   if( onError==OE_Abort ){
     sqlite3MayAbort(pParse);
   }
   sqlite3VdbeAddOp4(v, OP_Halt, errCode, onError, 0, p4, p4type);
-  if( p5Errmsg ) sqlite3VdbeChangeP5(v, p5Errmsg);
+  sqlite3VdbeChangeP5(v, p5Errmsg);
 }
 
 /*
 ** Code an OP_Halt due to UNIQUE or PRIMARY KEY constraint violation.
 */

Index: src/callback.c
==================================================================
--- src/callback.c
+++ src/callback.c
@@ -241,12 +241,12 @@
 ** 3: encoding matches and function takes any number of arguments
 ** 4: UTF8/16 conversion required - argument count matches exactly
 ** 5: UTF16 byte order conversion required - argument count matches exactly
 ** 6: Perfect match:  encoding and argument count match exactly.
 **
-** If nArg==(-2) then any function with a non-null xStep or xFunc is
-** a perfect match and any function with both xStep and xFunc NULL is
+** If nArg==(-2) then any function with a non-null xSFunc is
+** a perfect match and any function with xSFunc NULL is
 ** a non-match.
 */
 #define FUNC_PERFECT_MATCH 6  /* The score for a perfect match */
 static int matchQuality(
   FuncDef *p,     /* The function we are evaluating for match quality */
@@ -254,11 +254,11 @@
   u8 enc          /* Desired text encoding */
 ){
   int match;
 
   /* nArg of -2 is a special case */
-  if( nArg==(-2) ) return (p->xFunc==0 && p->xStep==0) ? 0 : FUNC_PERFECT_MATCH;
+  if( nArg==(-2) ) return (p->xSFunc==0) ? 0 : FUNC_PERFECT_MATCH;
 
   /* Wrong number of arguments means "no match" */
   if( p->nArg!=nArg && p->nArg>=0 ) return 0;
 
   /* Give a better score to a function with a specific number of arguments
@@ -332,11 +332,11 @@
 ** If the createFlag argument is true, then a new (blank) FuncDef
 ** structure is created and liked into the "db" structure if a
 ** no matching function previously existed.
 **
 ** If nArg is -2, then the first valid function found is returned.  A
-** function is valid if either xFunc or xStep is non-zero.  The nArg==(-2)
+** function is valid if xSFunc is non-zero.  The nArg==(-2)
 ** case is used to see if zName is a valid function name for some number
 ** of arguments.  If nArg is -2, then createFlag must be 0.
 **
 ** If createFlag is false, then a function with the required name and
 ** number of arguments may be returned even if the eTextRep flag does not
@@ -409,11 +409,11 @@
     memcpy(pBest->zName, zName, nName);
     pBest->zName[nName] = 0;
     sqlite3FuncDefInsert(&db->aFunc, pBest);
   }
 
-  if( pBest && (pBest->xStep || pBest->xFunc || createFlag) ){
+  if( pBest && (pBest->xSFunc || createFlag) ){
     return pBest;
   }
   return 0;
 }
 

Index: src/date.c
==================================================================
--- src/date.c
+++ src/date.c
@@ -68,52 +68,68 @@
   char tzSet;        /* Timezone was set explicitly */
 };
 
 
 /*
-** Convert zDate into one or more integers.  Additional arguments
-** come in groups of 5 as follows:
-**
-**       N       number of digits in the integer
-**       min     minimum allowed value of the integer
-**       max     maximum allowed value of the integer
-**       nextC   first character after the integer
-**       pVal    where to write the integers value.
-**
-** Conversions continue until one with nextC==0 is encountered.
+** Convert zDate into one or more integers according to the conversion
+** specifier zFormat.
+**
+** zFormat[] contains 4 characters for each integer converted, except for
+** the last integer which is specified by three characters.  The meaning
+** of a four-character format specifiers ABCD is:
+**
+**    A:   number of digits to convert.  Always "2" or "4".
+**    B:   minimum value.  Always "0" or "1".
+**    C:   maximum value, decoded as:
+**           a:  12
+**           b:  14
+**           c:  24
+**           d:  31
+**           e:  59
+**           f:  9999
+**    D:   the separator character, or \000 to indicate this is the
+**         last number to convert.
+**
+** Example:  To translate an ISO-8601 date YYYY-MM-DD, the format would
+** be "40f-21a-20c".  The "40f-" indicates the 4-digit year followed by "-".
+** The "21a-" indicates the 2-digit month followed by "-".  The "20c" indicates
+** the 2-digit day which is the last integer in the set.
+**
 ** The function returns the number of successful conversions.
 */
-static int getDigits(const char *zDate, ...){
+static int getDigits(const char *zDate, const char *zFormat, ...){
+  /* The aMx[] array translates the 3rd character of each format
+  ** spec into a max size:    a   b   c   d   e     f */
+  static const u16 aMx[] = { 12, 14, 24, 31, 59, 9999 };
   va_list ap;
-  int val;
-  int N;
-  int min;
-  int max;
-  int nextC;
-  int *pVal;
   int cnt = 0;
-  va_start(ap, zDate);
+  char nextC;
+  va_start(ap, zFormat);
   do{
-    N = va_arg(ap, int);
-    min = va_arg(ap, int);
-    max = va_arg(ap, int);
-    nextC = va_arg(ap, int);
-    pVal = va_arg(ap, int*);
+    char N = zFormat[0] - '0';
+    char min = zFormat[1] - '0';
+    int val = 0;
+    u16 max;
+
+    assert( zFormat[2]>='a' && zFormat[2]<='f' );
+    max = aMx[zFormat[2] - 'a'];
+    nextC = zFormat[3];
     val = 0;
     while( N-- ){
       if( !sqlite3Isdigit(*zDate) ){
         goto end_getDigits;
       }
       val = val*10 + *zDate - '0';
       zDate++;
     }
-    if( val<min || val>max || (nextC!=0 && nextC!=*zDate) ){
+    if( val<(int)min || val>(int)max || (nextC!=0 && nextC!=*zDate) ){
       goto end_getDigits;
     }
-    *pVal = val;
+    *va_arg(ap,int*) = val;
     zDate++;
     cnt++;
+    zFormat += 4;
   }while( nextC );
 end_getDigits:
   va_end(ap);
   return cnt;
 }
@@ -150,11 +166,11 @@
     goto zulu_time;
   }else{
     return c!=0;
   }
   zDate++;
-  if( getDigits(zDate, 2, 0, 14, ':', &nHr, 2, 0, 59, 0, &nMn)!=2 ){
+  if( getDigits(zDate, "20b:20e", &nHr, &nMn)!=2 ){
     return 1;
   }
   zDate += 5;
   p->tz = sgn*(nMn + nHr*60);
 zulu_time:
@@ -171,17 +187,17 @@
 ** Return 1 if there is a parsing error and 0 on success.
 */
 static int parseHhMmSs(const char *zDate, DateTime *p){
   int h, m, s;
   double ms = 0.0;
-  if( getDigits(zDate, 2, 0, 24, ':', &h, 2, 0, 59, 0, &m)!=2 ){
+  if( getDigits(zDate, "20c:20e", &h, &m)!=2 ){
     return 1;
   }
   zDate += 5;
   if( *zDate==':' ){
     zDate++;
-    if( getDigits(zDate, 2, 0, 59, 0, &s)!=1 ){
+    if( getDigits(zDate, "20e", &s)!=1 ){
       return 1;
     }
     zDate += 2;
     if( *zDate=='.' && sqlite3Isdigit(zDate[1]) ){
       double rScale = 1.0;
@@ -265,11 +281,11 @@
     zDate++;
     neg = 1;
   }else{
     neg = 0;
   }
-  if( getDigits(zDate,4,0,9999,'-',&Y,2,1,12,'-',&M,2,1,31,0,&D)!=3 ){
+  if( getDigits(zDate, "40f-21a-21d", &Y, &M, &D)!=3 ){
     return 1;
   }
   zDate += 10;
   while( sqlite3Isspace(*zDate) || 'T'==*(u8*)zDate ){ zDate++; }
   if( parseHhMmSs(zDate, p)==0 ){

Index: src/expr.c
==================================================================
--- src/expr.c
+++ src/expr.c
@@ -1133,14 +1133,15 @@
   ExprList *pList,        /* List to which to append. Might be NULL */
   Expr *pExpr             /* Expression to be appended. Might be NULL */
 ){
   sqlite3 *db = pParse->db;
   if( pList==0 ){
-    pList = sqlite3DbMallocZero(db, sizeof(ExprList) );
+    pList = sqlite3DbMallocRaw(db, sizeof(ExprList) );
     if( pList==0 ){
       goto no_mem;
     }
+    pList->nExpr = 0;
     pList->a = sqlite3DbMallocRaw(db, sizeof(pList->a[0]));
     if( pList->a==0 ) goto no_mem;
   }else if( (pList->nExpr & (pList->nExpr-1))==0 ){
     struct ExprList_item *a;
     assert( pList->nExpr>0 );
@@ -2894,11 +2895,11 @@
       nFarg = pFarg ? pFarg->nExpr : 0;
       assert( !ExprHasProperty(pExpr, EP_IntValue) );
       zId = pExpr->u.zToken;
       nId = sqlite3Strlen30(zId);
       pDef = sqlite3FindFunction(db, zId, nId, nFarg, enc, 0);
-      if( pDef==0 || pDef->xFunc==0 ){
+      if( pDef==0 || pDef->xFinalize!=0 ){
         sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId);
         break;
       }
 
       /* Attempt a direct implementation of the built-in COALESCE() and

Index: src/insert.c
==================================================================
--- src/insert.c
+++ src/insert.c
@@ -1593,11 +1593,11 @@
     if( useSeekResult ) pik_flags = OPFLAG_USESEEKRESULT;
     if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){
       assert( pParse->nested==0 );
       pik_flags |= OPFLAG_NCHANGE;
     }
-    if( pik_flags )  sqlite3VdbeChangeP5(v, pik_flags);
+    sqlite3VdbeChangeP5(v, pik_flags);
   }
   if( !HasRowid(pTab) ) return;
   regData = regNewData + 1;
   regRec = sqlite3GetTempReg(pParse);
   sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec);
@@ -2009,13 +2009,13 @@
     }else{
       addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid);
       assert( (pDest->tabFlags & TF_Autoincrement)==0 );
     }
     sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData);
-    sqlite3VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid);
+    sqlite3VdbeAddOp4(v, OP_Insert, iDest, regData, regRowid,
+                      pDest->zName, 0);
     sqlite3VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND);
-    sqlite3VdbeChangeP4(v, -1, pDest->zName, 0);
     sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); VdbeCoverage(v);
     sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0);
     sqlite3VdbeAddOp2(v, OP_Close, iDest, 0);
   }else{
     sqlite3TableLock(pParse, iDbDest, pDest->tnum, 1, pDest->zName);

Index: src/main.c
==================================================================
--- src/main.c
+++ src/main.c
@@ -1580,11 +1580,11 @@
   sqlite3 *db,
   const char *zFunctionName,
   int nArg,
   int enc,
   void *pUserData,
-  void (*xFunc)(sqlite3_context*,int,sqlite3_value **),
+  void (*xSFunc)(sqlite3_context*,int,sqlite3_value **),
   void (*xStep)(sqlite3_context*,int,sqlite3_value **),
   void (*xFinal)(sqlite3_context*),
   FuncDestructor *pDestructor
 ){
   FuncDef *p;
@@ -1591,13 +1591,13 @@
   int nName;
   int extraFlags;
 
   assert( sqlite3_mutex_held(db->mutex) );
   if( zFunctionName==0 ||
-      (xFunc && (xFinal || xStep)) || 
-      (!xFunc && (xFinal && !xStep)) ||
-      (!xFunc && (!xFinal && xStep)) ||
+      (xSFunc && (xFinal || xStep)) || 
+      (!xSFunc && (xFinal && !xStep)) ||
+      (!xSFunc && (!xFinal && xStep)) ||
       (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) ||
       (255<(nName = sqlite3Strlen30( zFunctionName))) ){
     return SQLITE_MISUSE_BKPT;
   }
 
@@ -1616,14 +1616,14 @@
   if( enc==SQLITE_UTF16 ){
     enc = SQLITE_UTF16NATIVE;
   }else if( enc==SQLITE_ANY ){
     int rc;
     rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8|extraFlags,
-         pUserData, xFunc, xStep, xFinal, pDestructor);
+         pUserData, xSFunc, xStep, xFinal, pDestructor);
     if( rc==SQLITE_OK ){
       rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE|extraFlags,
-          pUserData, xFunc, xStep, xFinal, pDestructor);
+          pUserData, xSFunc, xStep, xFinal, pDestructor);
     }
     if( rc!=SQLITE_OK ){
       return rc;
     }
     enc = SQLITE_UTF16BE;
@@ -1663,12 +1663,11 @@
     pDestructor->nRef++;
   }
   p->pDestructor = pDestructor;
   p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags;
   testcase( p->funcFlags & SQLITE_DETERMINISTIC );
-  p->xFunc = xFunc;
-  p->xStep = xStep;
+  p->xSFunc = xSFunc ? xSFunc : xStep;
   p->xFinalize = xFinal;
   p->pUserData = pUserData;
   p->nArg = (u16)nArg;
   return SQLITE_OK;
 }
@@ -1680,25 +1679,25 @@
   sqlite3 *db,
   const char *zFunc,
   int nArg,
   int enc,
   void *p,
-  void (*xFunc)(sqlite3_context*,int,sqlite3_value **),
+  void (*xSFunc)(sqlite3_context*,int,sqlite3_value **),
   void (*xStep)(sqlite3_context*,int,sqlite3_value **),
   void (*xFinal)(sqlite3_context*)
 ){
-  return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xFunc, xStep,
+  return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xSFunc, xStep,
                                     xFinal, 0);
 }
 
 int sqlite3_create_function_v2(
   sqlite3 *db,
   const char *zFunc,
   int nArg,
   int enc,
   void *p,
-  void (*xFunc)(sqlite3_context*,int,sqlite3_value **),
+  void (*xSFunc)(sqlite3_context*,int,sqlite3_value **),
   void (*xStep)(sqlite3_context*,int,sqlite3_value **),
   void (*xFinal)(sqlite3_context*),
   void (*xDestroy)(void *)
 ){
   int rc = SQLITE_ERROR;
@@ -1717,11 +1716,11 @@
       goto out;
     }
     pArg->xDestroy = xDestroy;
     pArg->pUserData = p;
   }
-  rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xFunc, xStep, xFinal, pArg);
+  rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xSFunc, xStep, xFinal, pArg);
   if( pArg && pArg->nRef==0 ){
     assert( rc!=SQLITE_OK );
     xDestroy(p);
     sqlite3DbFree(db, pArg);
   }
@@ -1737,11 +1736,11 @@
   sqlite3 *db,
   const void *zFunctionName,
   int nArg,
   int eTextRep,
   void *p,
-  void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
+  void (*xSFunc)(sqlite3_context*,int,sqlite3_value**),
   void (*xStep)(sqlite3_context*,int,sqlite3_value**),
   void (*xFinal)(sqlite3_context*)
 ){
   int rc;
   char *zFunc8;
@@ -1750,11 +1749,11 @@
   if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT;
 #endif
   sqlite3_mutex_enter(db->mutex);
   assert( !db->mallocFailed );
   zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE);
-  rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0);
+  rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xSFunc,xStep,xFinal,0);
   sqlite3DbFree(db, zFunc8);
   rc = sqlite3ApiExit(db, rc);
   sqlite3_mutex_leave(db->mutex);
   return rc;
 }
@@ -3064,11 +3063,10 @@
                         sqlite3GlobalConfig.nLookaside);
 
   sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT);
 
 opendb_out:
-  sqlite3_free(zOpen);
   if( db ){
     assert( db->mutex!=0 || isThreadsafe==0
            || sqlite3GlobalConfig.bFullMutex==0 );
     sqlite3_mutex_leave(db->mutex);
   }
@@ -3124,10 +3122,11 @@
       }
       sqlite3_key_v2(db, 0, zKey, i/2);
     }
   }
 #endif
+  sqlite3_free(zOpen);
   return rc & 0xff;
 }
 
 /*
 ** Open a new database handle.

Index: src/malloc.c
==================================================================
--- src/malloc.c
+++ src/malloc.c
@@ -581,12 +581,13 @@
   }
   return p;
 }
 
 /*
-** Allocate and zero memory.  If the allocation fails, make
-** the mallocFailed flag in the connection pointer.
+** Allocate memory, either lookaside (if possible) or heap.  
+** If the allocation fails, set the mallocFailed flag in
+** the connection pointer.
 **
 ** If db!=0 and db->mallocFailed is true (indicating a prior malloc
 ** failure on the same database connection) then always return 0.
 ** Hence for a particular database connection, once malloc starts
 ** failing, it fails consistently until mallocFailed is reset.
@@ -598,12 +599,12 @@
 **         if( b ) a[10] = 9;
 **
 ** In other words, if a subsequent malloc (ex: "b") worked, it is assumed
 ** that all prior mallocs (ex: "a") worked too.
 */
+static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n);
 void *sqlite3DbMallocRaw(sqlite3 *db, u64 n){
-  void *p;
   assert( db==0 || sqlite3_mutex_held(db->mutex) );
   assert( db==0 || db->pnBytesFreed==0 );
 #ifndef SQLITE_OMIT_LOOKASIDE
   if( db ){
     LookasideSlot *pBuf;
@@ -629,11 +630,14 @@
 #else
   if( db && db->mallocFailed ){
     return 0;
   }
 #endif
-  p = sqlite3Malloc(n);
+  return dbMallocRawFinish(db, n);
+}
+static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n){
+  void *p = sqlite3Malloc(n);
   if( !p && db ){
     db->mallocFailed = 1;
   }
   sqlite3MemdebugSetType(p, 
          (db && db->lookaside.bEnabled) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP);

Index: src/pragma.c
==================================================================
--- src/pragma.c
+++ src/pragma.c
@@ -428,19 +428,21 @@
       { OP_IfPos,       1, 8,        0},
       { OP_Integer,     0, 1,        0},                         /* 6 */
       { OP_Noop,        0, 0,        0},
       { OP_ResultRow,   1, 1,        0},
     };
-    int addr;
+    VdbeOp *aOp;
     sqlite3VdbeUsesBtree(v, iDb);
     if( !zRight ){
       setOneColumnName(v, "cache_size");
       pParse->nMem += 2;
-      addr = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize,iLn);
-      sqlite3VdbeChangeP1(v, addr, iDb);
-      sqlite3VdbeChangeP1(v, addr+1, iDb);
-      sqlite3VdbeChangeP1(v, addr+6, SQLITE_DEFAULT_CACHE_SIZE);
+      sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(getCacheSize));
+      aOp = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize, iLn);
+      if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break;
+      aOp[0].p1 = iDb;
+      aOp[1].p1 = iDb;
+      aOp[6].p1 = SQLITE_DEFAULT_CACHE_SIZE;
     }else{
       int size = sqlite3AbsInt32(sqlite3Atoi(zRight));
       sqlite3BeginWriteOperation(pParse, 0, iDb);
       sqlite3VdbeAddOp2(v, OP_Integer, size, 1);
       sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_DEFAULT_CACHE_SIZE, 1);
@@ -696,17 +698,20 @@
           { OP_If,             1,         0,                 0},    /* 2 */
           { OP_Halt,           SQLITE_OK, OE_Abort,          0},    /* 3 */
           { OP_Integer,        0,         1,                 0},    /* 4 */
           { OP_SetCookie,      0,         BTREE_INCR_VACUUM, 1},    /* 5 */
         };
-        int iAddr;
-        iAddr = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6, iLn);
-        sqlite3VdbeChangeP1(v, iAddr, iDb);
-        sqlite3VdbeChangeP1(v, iAddr+1, iDb);
-        sqlite3VdbeChangeP2(v, iAddr+2, iAddr+4);
-        sqlite3VdbeChangeP1(v, iAddr+4, eAuto-1);
-        sqlite3VdbeChangeP1(v, iAddr+5, iDb);
+        VdbeOp *aOp;
+        int iAddr = sqlite3VdbeCurrentAddr(v);
+        sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(setMeta6));
+        aOp = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6, iLn);
+        if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break;
+        aOp[0].p1 = iDb;
+        aOp[1].p1 = iDb;
+        aOp[2].p2 = iAddr+4;
+        aOp[4].p1 = eAuto - 1;
+        aOp[5].p1 = iDb;
         sqlite3VdbeUsesBtree(v, iDb);
       }
     }
     break;
   }
@@ -1411,22 +1416,10 @@
   ** without most of the overhead of a full integrity-check.
   */
   case PragTyp_INTEGRITY_CHECK: {
     int i, j, addr, mxErr;
 
-    /* Code that appears at the end of the integrity check.  If no error
-    ** messages have been generated, output OK.  Otherwise output the
-    ** error message
-    */
-    static const int iLn = VDBE_OFFSET_LINENO(2);
-    static const VdbeOpList endCode[] = {
-      { OP_AddImm,      1, 0,        0},    /* 0 */
-      { OP_If,          1, 0,        0},    /* 1 */
-      { OP_String8,     0, 3,        0},    /* 2 */
-      { OP_ResultRow,   3, 1,        0},
-    };
-
     int isQuick = (sqlite3Tolower(zLeft[0])=='q');
 
     /* If the PRAGMA command was of the form "PRAGMA <db>.integrity_check",
     ** then iDb is set to the index of the database identified by <db>.
     ** In this case, the integrity of database iDb only is verified by
@@ -1619,14 +1612,28 @@
           sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1);
         }
 #endif /* SQLITE_OMIT_BTREECOUNT */
       } 
     }
-    addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn);
-    sqlite3VdbeChangeP2(v, addr, -mxErr);
-    sqlite3VdbeJumpHere(v, addr+1);
-    sqlite3VdbeChangeP4(v, addr+2, "ok", P4_STATIC);
+    {
+      static const int iLn = VDBE_OFFSET_LINENO(2);
+      static const VdbeOpList endCode[] = {
+        { OP_AddImm,      1, 0,        0},    /* 0 */
+        { OP_If,          1, 0,        0},    /* 1 */
+        { OP_String8,     0, 3,        0},    /* 2 */
+        { OP_ResultRow,   3, 1,        0},
+      };
+      VdbeOp *aOp;
+
+      aOp = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn);
+      if( aOp ){
+        aOp[0].p2 = -mxErr;
+        aOp[1].p2 = sqlite3VdbeCurrentAddr(v);
+        aOp[2].p4type = P4_STATIC;
+        aOp[2].p4.z = "ok";
+      }
+    }
   }
   break;
 #endif /* SQLITE_OMIT_INTEGRITY_CHECK */
 
 #ifndef SQLITE_OMIT_UTF16
@@ -1739,26 +1746,32 @@
       static const VdbeOpList setCookie[] = {
         { OP_Transaction,    0,  1,  0},    /* 0 */
         { OP_Integer,        0,  1,  0},    /* 1 */
         { OP_SetCookie,      0,  0,  1},    /* 2 */
       };
-      int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie, 0);
-      sqlite3VdbeChangeP1(v, addr, iDb);
-      sqlite3VdbeChangeP1(v, addr+1, sqlite3Atoi(zRight));
-      sqlite3VdbeChangeP1(v, addr+2, iDb);
-      sqlite3VdbeChangeP2(v, addr+2, iCookie);
+      VdbeOp *aOp;
+      sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(setCookie));
+      aOp = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie, 0);
+      if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break;
+      aOp[0].p1 = iDb;
+      aOp[1].p1 = sqlite3Atoi(zRight);
+      aOp[2].p1 = iDb;
+      aOp[2].p2 = iCookie;
     }else{
       /* Read the specified cookie value */
       static const VdbeOpList readCookie[] = {
         { OP_Transaction,     0,  0,  0},    /* 0 */
         { OP_ReadCookie,      0,  1,  0},    /* 1 */
         { OP_ResultRow,       1,  1,  0}
       };
-      int addr = sqlite3VdbeAddOpList(v, ArraySize(readCookie), readCookie, 0);
-      sqlite3VdbeChangeP1(v, addr, iDb);
-      sqlite3VdbeChangeP1(v, addr+1, iDb);
-      sqlite3VdbeChangeP3(v, addr+1, iCookie);
+      VdbeOp *aOp;
+      sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(readCookie));
+      aOp = sqlite3VdbeAddOpList(v, ArraySize(readCookie),readCookie,0);
+      if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break;
+      aOp[0].p1 = iDb;
+      aOp[1].p1 = iDb;
+      aOp[1].p3 = iCookie;
       sqlite3VdbeSetNumCols(v, 1);
       sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT);
     }
   }
   break;

Index: src/prepare.c
==================================================================
--- src/prepare.c
+++ src/prepare.c
@@ -138,65 +138,31 @@
   int rc;
   int i;
 #ifndef SQLITE_OMIT_DEPRECATED
   int size;
 #endif
-  Table *pTab;
   Db *pDb;
   char const *azArg[4];
   int meta[5];
   InitData initData;
-  char const *zMasterSchema;
-  char const *zMasterName;
+  const char *zMasterName;
   int openedTransaction = 0;
 
-  /*
-  ** The master database table has a structure like this
-  */
-  static const char master_schema[] = 
-     "CREATE TABLE sqlite_master(\n"
-     "  type text,\n"
-     "  name text,\n"
-     "  tbl_name text,\n"
-     "  rootpage integer,\n"
-     "  sql text\n"
-     ")"
-  ;
-#ifndef SQLITE_OMIT_TEMPDB
-  static const char temp_master_schema[] = 
-     "CREATE TEMP TABLE sqlite_temp_master(\n"
-     "  type text,\n"
-     "  name text,\n"
-     "  tbl_name text,\n"
-     "  rootpage integer,\n"
-     "  sql text\n"
-     ")"
-  ;
-#else
-  #define temp_master_schema 0
-#endif
-
   assert( iDb>=0 && iDb<db->nDb );
   assert( db->aDb[iDb].pSchema );
   assert( sqlite3_mutex_held(db->mutex) );
   assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) );
 
-  /* zMasterSchema and zInitScript are set to point at the master schema
-  ** and initialisation script appropriate for the database being
-  ** initialized. zMasterName is the name of the master table.
-  */
-  if( !OMIT_TEMPDB && iDb==1 ){
-    zMasterSchema = temp_master_schema;
-  }else{
-    zMasterSchema = master_schema;
-  }
-  zMasterName = SCHEMA_TABLE(iDb);
-
-  /* Construct the schema tables.  */
-  azArg[0] = zMasterName;
+  /* Construct the in-memory representation schema tables (sqlite_master or
+  ** sqlite_temp_master) by invoking the parser directly.  The appropriate
+  ** table name will be inserted automatically by the parser so we can just
+  ** use the abbreviation "x" here.  The parser will also automatically tag
+  ** the schema table as read-only. */
+  azArg[0] = zMasterName = SCHEMA_TABLE(iDb);
   azArg[1] = "1";
-  azArg[2] = zMasterSchema;
+  azArg[2] = "CREATE TABLE x(type text,name text,tbl_name text,"
+                            "rootpage integer,sql text)";
   azArg[3] = 0;
   initData.db = db;
   initData.iDb = iDb;
   initData.rc = SQLITE_OK;
   initData.pzErrMsg = pzErrMsg;
@@ -203,14 +169,10 @@
   sqlite3InitCallback(&initData, 3, (char **)azArg, 0);
   if( initData.rc ){
     rc = initData.rc;
     goto error_out;
   }
-  pTab = sqlite3FindTable(db, zMasterName, db->aDb[iDb].zName);
-  if( ALWAYS(pTab) ){
-    pTab->tabFlags |= TF_Readonly;
-  }
 
   /* Create a cursor to hold the database open
   */
   pDb = &db->aDb[iDb];
   if( pDb->pBt==0 ){
@@ -325,11 +287,11 @@
   */
   assert( db->init.busy );
   {
     char *zSql;
     zSql = sqlite3MPrintf(db, 
-        "SELECT name, rootpage, sql FROM '%q'.%s ORDER BY rowid",
+        "SELECT name, rootpage, sql FROM \"%w\".%s ORDER BY rowid",
         db->aDb[iDb].zName, zMasterName);
 #ifndef SQLITE_OMIT_AUTHORIZATION
     {
       sqlite3_xauth xAuth;
       xAuth = db->xAuth;

Index: src/resolve.c
==================================================================
--- src/resolve.c
+++ src/resolve.c
@@ -663,11 +663,11 @@
           no_such_func = 1;
         }else{
           wrong_num_args = 1;
         }
       }else{
-        is_agg = pDef->xFunc==0;
+        is_agg = pDef->xFinalize!=0;
         if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){
           ExprSetProperty(pExpr, EP_Unlikely|EP_Skip);
           if( n==2 ){
             pExpr->iTable = exprProbability(pList->a[1].pExpr);
             if( pExpr->iTable<0 ){
@@ -1391,14 +1391,16 @@
     pParse->nHeight += pExpr->nHeight;
   }
 #endif
   savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg);
   pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg);
-  memset(&w, 0, sizeof(w));
+  w.pParse = pNC->pParse;
   w.xExprCallback = resolveExprStep;
   w.xSelectCallback = resolveSelectStep;
-  w.pParse = pNC->pParse;
+  w.xSelectCallback2 = 0;
+  w.walkerDepth = 0;
+  w.eCode = 0;
   w.u.pNC = pNC;
   sqlite3WalkExpr(&w, pExpr);
 #if SQLITE_MAX_EXPR_DEPTH>0
   pNC->pParse->nHeight -= pExpr->nHeight;
 #endif

Index: src/select.c
==================================================================
--- src/select.c
+++ src/select.c
@@ -1003,19 +1003,20 @@
 /*
 ** Allocate a KeyInfo object sufficient for an index of N key columns and
 ** X extra columns.
 */
 KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){
-  KeyInfo *p = sqlite3DbMallocZero(0, 
-                   sizeof(KeyInfo) + (N+X)*(sizeof(CollSeq*)+1));
+  int nExtra = (N+X)*(sizeof(CollSeq*)+1);
+  KeyInfo *p = sqlite3Malloc(sizeof(KeyInfo) + nExtra);
   if( p ){
     p->aSortOrder = (u8*)&p->aColl[N+X];
     p->nField = (u16)N;
     p->nXField = (u16)X;
     p->enc = ENC(db);
     p->db = db;
     p->nRef = 1;
+    memset(&p[1], 0, nExtra);
   }else{
     db->mallocFailed = 1;
   }
   return p;
 }

Index: src/sqliteInt.h
==================================================================
--- src/sqliteInt.h
+++ src/sqliteInt.h
@@ -400,10 +400,25 @@
 #else
 # define ALWAYS(X)      (X)
 # define NEVER(X)       (X)
 #endif
 
+/*
+** Some malloc failures are only possible if SQLITE_TEST_REALLOC_STRESS is
+** defined.  We need to defend against those failures when testing with
+** SQLITE_TEST_REALLOC_STRESS, but we don't want the unreachable branches
+** during a normal build.  The following macro can be used to disable tests
+** that are always false except when SQLITE_TEST_REALLOC_STRESS is set.
+*/
+#if defined(SQLITE_TEST_REALLOC_STRESS)
+# define ONLY_IF_REALLOC_STRESS(X)  (X)
+#elif !defined(NDEBUG)
+# define ONLY_IF_REALLOC_STRESS(X)  ((X)?(assert(0),1):0)
+#else
+# define ONLY_IF_REALLOC_STRESS(X)  (0)
+#endif
+
 /*
 ** Declarations used for tracing the operating system interfaces.
 */
 #if defined(SQLITE_FORCE_OS_TRACE) || defined(SQLITE_TEST) || \
     (defined(SQLITE_DEBUG) && SQLITE_OS_WIN)
@@ -1371,13 +1386,12 @@
 struct FuncDef {
   i16 nArg;            /* Number of arguments.  -1 means unlimited */
   u16 funcFlags;       /* Some combination of SQLITE_FUNC_* */
   void *pUserData;     /* User data parameter */
   FuncDef *pNext;      /* Next function with same name */
-  void (*xFunc)(sqlite3_context*,int,sqlite3_value**); /* Regular function */
-  void (*xStep)(sqlite3_context*,int,sqlite3_value**); /* Aggregate step */
-  void (*xFinalize)(sqlite3_context*);                /* Aggregate finalizer */
+  void (*xSFunc)(sqlite3_context*,int,sqlite3_value**); /* func or agg-step */
+  void (*xFinalize)(sqlite3_context*);                  /* Agg finalizer */
   char *zName;         /* SQL name of the function. */
   FuncDef *pHash;      /* Next with a different name but the same hash */
   FuncDestructor *pDestructor;   /* Reference counted destructor function */
 };
 
@@ -1456,32 +1470,32 @@
 **     FuncDef.flags variable is set to the value passed as the flags
 **     parameter.
 */
 #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \
   {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
-   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0}
+   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, 0, 0}
 #define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \
   {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
-   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0}
+   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, 0, 0}
 #define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \
   {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
-   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0}
+   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, 0, 0}
 #define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \
   {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\
-   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0}
+   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, 0, 0}
 #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \
   {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
-   pArg, 0, xFunc, 0, 0, #zName, 0, 0}
+   pArg, 0, xFunc, 0, #zName, 0, 0}
 #define LIKEFUNC(zName, nArg, arg, flags) \
   {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \
-   (void *)arg, 0, likeFunc, 0, 0, #zName, 0, 0}
+   (void *)arg, 0, likeFunc, 0, #zName, 0, 0}
 #define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \
   {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL), \
-   SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0}
+   SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName,0,0}
 #define AGGREGATE2(zName, nArg, arg, nc, xStep, xFinal, extraFlags) \
   {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|extraFlags, \
-   SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0}
+   SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName,0,0}
 
 /*
 ** All current savepoints are stored in a linked list starting at
 ** sqlite3.pSavepoint. The first element in the list is the most recently
 ** opened savepoint. Savepoints are added to the list by the vdbe
@@ -2788,11 +2802,11 @@
   ** each recursion.  The boundary between these two regions is determined
   ** using offsetof(Parse,nVar) so the nVar field must be the first field
   ** in the recursive region.
   ************************************************************************/
 
-  int nVar;                 /* Number of '?' variables seen in the SQL so far */
+  ynVar nVar;               /* Number of '?' variables seen in the SQL so far */
   int nzVar;                /* Number of available slots in azVar[] */
   u8 iPkSortOrder;          /* ASC or DESC for INTEGER PRIMARY KEY */
   u8 explain;               /* True if the EXPLAIN flag is found on the query */
 #ifndef SQLITE_OMIT_VIRTUALTABLE
   u8 declareVtab;           /* True if inside sqlite3_declare_vtab() */
@@ -3070,14 +3084,14 @@
 
 /*
 ** Context pointer passed down through the tree-walk.
 */
 struct Walker {
+  Parse *pParse;                            /* Parser context.  */
   int (*xExprCallback)(Walker*, Expr*);     /* Callback for expressions */
   int (*xSelectCallback)(Walker*,Select*);  /* Callback for SELECTs */
   void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */
-  Parse *pParse;                            /* Parser context.  */
   int walkerDepth;                          /* Number of subqueries */
   u8 eCode;                                 /* A small processing code */
   union {                                   /* Extra data for callback */
     NameContext *pNC;                          /* Naming context */
     int n;                                     /* A counter */

Index: src/trigger.c
==================================================================
--- src/trigger.c
+++ src/trigger.c
@@ -950,12 +950,12 @@
   /* Code the OP_Program opcode in the parent VDBE. P4 of the OP_Program 
   ** is a pointer to the sub-vdbe containing the trigger program.  */
   if( pPrg ){
     int bRecursive = (p->zName && 0==(pParse->db->flags&SQLITE_RecTriggers));
 
-    sqlite3VdbeAddOp3(v, OP_Program, reg, ignoreJump, ++pParse->nMem);
-    sqlite3VdbeChangeP4(v, -1, (const char *)pPrg->pProgram, P4_SUBPROGRAM);
+    sqlite3VdbeAddOp4(v, OP_Program, reg, ignoreJump, ++pParse->nMem,
+                      (const char *)pPrg->pProgram, P4_SUBPROGRAM);
     VdbeComment(
         (v, "Call: %s.%s", (p->zName?p->zName:"fkey"), onErrorText(orconf)));
 
     /* Set the P5 operand of the OP_Program instruction to non-zero if
     ** recursive invocation of this trigger program is disallowed. Recursive

Index: src/vdbe.c
==================================================================
--- src/vdbe.c
+++ src/vdbe.c
@@ -1661,12 +1661,12 @@
   }
 #endif
   MemSetTypeFlag(pCtx->pOut, MEM_Null);
   pCtx->fErrorOrAux = 0;
   db->lastRowid = lastRowid;
-  (*pCtx->pFunc->xFunc)(pCtx, pCtx->argc, pCtx->argv); /* IMP: R-24505-23230 */
-  lastRowid = db->lastRowid;  /* Remember rowid changes made by xFunc */
+  (*pCtx->pFunc->xSFunc)(pCtx, pCtx->argc, pCtx->argv);/* IMP: R-24505-23230 */
+  lastRowid = db->lastRowid;  /* Remember rowid changes made by xSFunc */
 
   /* If the function returned an error, throw an exception */
   if( pCtx->fErrorOrAux ){
     if( pCtx->isError ){
       sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut));
@@ -5098,10 +5098,11 @@
 case OP_Destroy: {     /* out2 */
   int iMoved;
   int iDb;
 
   assert( p->readOnly==0 );
+  assert( pOp->p1>1 );
   pOut = out2Prerelease(p, pOp);
   pOut->flags = MEM_Null;
   if( db->nVdbeRead > db->nVDestroy+1 ){
     rc = SQLITE_LOCKED;
     p->errorAction = OE_Abort;
@@ -5902,11 +5903,11 @@
   pMem->n++;
   sqlite3VdbeMemInit(&t, db, MEM_Null);
   pCtx->pOut = &t;
   pCtx->fErrorOrAux = 0;
   pCtx->skipFlag = 0;
-  (pCtx->pFunc->xStep)(pCtx,pCtx->argc,pCtx->argv); /* IMP: R-24505-23230 */
+  (pCtx->pFunc->xSFunc)(pCtx,pCtx->argc,pCtx->argv); /* IMP: R-24505-23230 */
   if( pCtx->fErrorOrAux ){
     if( pCtx->isError ){
       sqlite3VdbeError(p, "%s", sqlite3_value_text(&t));
       rc = pCtx->isError;
     }

Index: src/vdbe.h
==================================================================
--- src/vdbe.h
+++ src/vdbe.h
@@ -178,19 +178,24 @@
 void sqlite3VdbeMultiLoad(Vdbe*,int,const char*,...);
 int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int);
 int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int);
 int sqlite3VdbeAddOp4Dup8(Vdbe*,int,int,int,int,const u8*,int);
 int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int);
-int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno);
+#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS)
+  void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N);
+#else
+# define sqlite3VdbeVerifyNoMallocRequired(A,B)
+#endif
+VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno);
 void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*);
 void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8);
 void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1);
 void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2);
 void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3);
 void sqlite3VdbeChangeP5(Vdbe*, u8 P5);
 void sqlite3VdbeJumpHere(Vdbe*, int addr);
-void sqlite3VdbeChangeToNoop(Vdbe*, int addr);
+int sqlite3VdbeChangeToNoop(Vdbe*, int addr);
 int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op);
 void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N);
 void sqlite3VdbeSetP4KeyInfo(Parse*, Index*);
 void sqlite3VdbeUsesBtree(Vdbe*, int);
 VdbeOp *sqlite3VdbeGetOp(Vdbe*, int);

Index: src/vdbeapi.c
==================================================================
--- src/vdbeapi.c
+++ src/vdbeapi.c
@@ -793,11 +793,11 @@
 ** Allocate or return the aggregate context for a user function.  A new
 ** context is allocated on the first call.  Subsequent calls return the
 ** same context that was returned on prior calls.
 */
 void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){
-  assert( p && p->pFunc && p->pFunc->xStep );
+  assert( p && p->pFunc && p->pFunc->xFinalize );
   assert( sqlite3_mutex_held(p->pOut->db->mutex) );
   testcase( nByte<0 );
   if( (p->pMem->flags & MEM_Agg)==0 ){
     return createAggContext(p, nByte);
   }else{
@@ -884,11 +884,11 @@
 ** provide only to avoid breaking legacy code.  New aggregate function
 ** implementations should keep their own counts within their aggregate
 ** context.
 */
 int sqlite3_aggregate_count(sqlite3_context *p){
-  assert( p && p->pMem && p->pFunc && p->pFunc->xStep );
+  assert( p && p->pMem && p->pFunc && p->pFunc->xFinalize );
   return p->pMem->n;
 }
 #endif
 
 /*

Index: src/vdbeaux.c
==================================================================
--- src/vdbeaux.c
+++ src/vdbeaux.c
@@ -248,12 +248,11 @@
   char c;
   va_start(ap, zTypes);
   for(i=0; (c = zTypes[i])!=0; i++){
     if( c=='s' ){
       const char *z = va_arg(ap, const char*);
-      int addr = sqlite3VdbeAddOp2(p, z==0 ? OP_Null : OP_String8, 0, iDest++);
-      if( z ) sqlite3VdbeChangeP4(p, addr, z, 0);
+      sqlite3VdbeAddOp4(p, z==0 ? OP_Null : OP_String8, 0, iDest++, 0, z, 0);
     }else{
       assert( c=='i' );
       sqlite3VdbeAddOp2(p, OP_Integer, va_arg(ap, int), iDest++);
     }
   }
@@ -604,10 +603,24 @@
 int sqlite3VdbeCurrentAddr(Vdbe *p){
   assert( p->magic==VDBE_MAGIC_INIT );
   return p->nOp;
 }
 
+/*
+** Verify that at least N opcode slots are available in p without
+** having to malloc for more space (except when compiled using
+** SQLITE_TEST_REALLOC_STRESS).  This interface is used during testing
+** to verify that certain calls to sqlite3VdbeAddOpList() can never
+** fail due to a OOM fault and hence that the return value from
+** sqlite3VdbeAddOpList() will always be non-NULL.
+*/
+#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS)
+void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N){
+  assert( p->nOp + N <= p->pParse->nOpAlloc );
+}
+#endif
+
 /*
 ** This function returns a pointer to the array of opcodes associated with
 ** the Vdbe passed as the first argument. It is the callers responsibility
 ** to arrange for the returned array to be eventually freed using the 
 ** vdbeFreeOpArray() function.
@@ -629,23 +642,27 @@
   p->aOp = 0;
   return aOp;
 }
 
 /*
-** Add a whole list of operations to the operation stack.  Return the
-** address of the first operation added.
+** Add a whole list of operations to the operation stack.  Return a
+** pointer to the first operation inserted.
 */
-int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){
-  int addr, i;
-  VdbeOp *pOut;
+VdbeOp *sqlite3VdbeAddOpList(
+  Vdbe *p,                     /* Add opcodes to the prepared statement */
+  int nOp,                     /* Number of opcodes to add */
+  VdbeOpList const *aOp,       /* The opcodes to be added */
+  int iLineno                  /* Source-file line number of first opcode */
+){
+  int i;
+  VdbeOp *pOut, *pFirst;
   assert( nOp>0 );
   assert( p->magic==VDBE_MAGIC_INIT );
   if( p->nOp + nOp > p->pParse->nOpAlloc && growOpArray(p, nOp) ){
     return 0;
   }
-  addr = p->nOp;
-  pOut = &p->aOp[addr];
+  pFirst = pOut = &p->aOp[p->nOp];
   for(i=0; i<nOp; i++, aOp++, pOut++){
     pOut->opcode = aOp->opcode;
     pOut->p1 = aOp->p1;
     pOut->p2 = aOp->p2;
     assert( aOp->p2>=0 );
@@ -661,16 +678,16 @@
 #else
     (void)iLineno;
 #endif
 #ifdef SQLITE_DEBUG
     if( p->db->flags & SQLITE_VdbeAddopTrace ){
-      sqlite3VdbePrintOp(0, i+addr, &p->aOp[i+addr]);
+      sqlite3VdbePrintOp(0, i+p->nOp, &p->aOp[i+p->nOp]);
     }
 #endif
   }
   p->nOp += nOp;
-  return addr;
+  return pFirst;
 }
 
 #if defined(SQLITE_ENABLE_STMT_SCANSTATUS)
 /*
 ** Add an entry to the array of counters managed by sqlite3_stmt_scanstatus().
@@ -714,11 +731,11 @@
 }
 void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){
   sqlite3VdbeGetOp(p,addr)->p3 = val;
 }
 void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){
-  sqlite3VdbeGetOp(p,-1)->p5 = p5;
+  if( !p->db->mallocFailed ) p->aOp[p->nOp-1].p5 = p5;
 }
 
 /*
 ** Change the P2 operand of instruction addr so that it points to
 ** the address of the next instruction to be coded.
@@ -824,28 +841,29 @@
 }
 
 /*
 ** Change the opcode at addr into OP_Noop
 */
-void sqlite3VdbeChangeToNoop(Vdbe *p, int addr){
-  if( addr<p->nOp ){
-    VdbeOp *pOp = &p->aOp[addr];
-    sqlite3 *db = p->db;
-    freeP4(db, pOp->p4type, pOp->p4.p);
-    memset(pOp, 0, sizeof(pOp[0]));
-    pOp->opcode = OP_Noop;
-  }
+int sqlite3VdbeChangeToNoop(Vdbe *p, int addr){
+  VdbeOp *pOp;
+  if( p->db->mallocFailed ) return 0;
+  assert( addr>=0 && addr<p->nOp );
+  pOp = &p->aOp[addr];
+  freeP4(p->db, pOp->p4type, pOp->p4.p);
+  pOp->p4type = P4_NOTUSED;
+  pOp->p4.z = 0;
+  pOp->opcode = OP_Noop;
+  return 1;
 }
 
 /*
 ** If the last opcode is "op" and it is not a jump destination,
 ** then remove it.  Return true if and only if an opcode was removed.
 */
 int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){
   if( (p->nOp-1)>(p->pParse->iFixedOp) && p->aOp[p->nOp-1].opcode==op ){
-    sqlite3VdbeChangeToNoop(p, p->nOp-1);
-    return 1;
+    return sqlite3VdbeChangeToNoop(p, p->nOp-1);
   }else{
     return 0;
   }
 }
 
@@ -1880,11 +1898,10 @@
   do {
     nByte = 0;
     p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), zCsr, &nFree, &nByte);
     p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), zCsr, &nFree, &nByte);
     p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), zCsr, &nFree, &nByte);
-    p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), zCsr, &nFree, &nByte);
     p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*),
                           zCsr, &nFree, &nByte);
     p->aOnceFlag = allocSpace(p->aOnceFlag, nOnce, zCsr, &nFree, &nByte);
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
     p->anExec = allocSpace(p->anExec, p->nOp*sizeof(i64), zCsr, &nFree, &nByte);
@@ -1903,15 +1920,14 @@
     for(n=0; n<nVar; n++){
       p->aVar[n].flags = MEM_Null;
       p->aVar[n].db = db;
     }
   }
-  if( p->azVar && pParse->nzVar>0 ){
-    p->nzVar = pParse->nzVar;
-    memcpy(p->azVar, pParse->azVar, p->nzVar*sizeof(p->azVar[0]));
-    memset(pParse->azVar, 0, pParse->nzVar*sizeof(pParse->azVar[0]));
-  }
+  p->nzVar = pParse->nzVar;
+  p->azVar = pParse->azVar;
+  pParse->nzVar =  0;
+  pParse->azVar = 0;
   if( p->aMem ){
     p->aMem--;                      /* aMem[] goes from 1..nMem */
     p->nMem = nMem;                 /*       not from 0..nMem-1 */
     for(n=1; n<=nMem; n++){
       p->aMem[n].flags = MEM_Undefined;
@@ -2894,10 +2910,11 @@
     pNext = pSub->pNext;
     vdbeFreeOpArray(db, pSub->aOp, pSub->nOp);
     sqlite3DbFree(db, pSub);
   }
   for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]);
+  sqlite3DbFree(db, p->azVar);
   vdbeFreeOpArray(db, p->aOp, p->nOp);
   sqlite3DbFree(db, p->aColName);
   sqlite3DbFree(db, p->zSql);
   sqlite3DbFree(db, p->pFree);
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
@@ -3638,13 +3655,13 @@
     v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc);
     n1 = v1==0 ? 0 : c1.n;
     v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc);
     n2 = v2==0 ? 0 : c2.n;
     rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2);
+    if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM;
     sqlite3VdbeMemRelease(&c1);
     sqlite3VdbeMemRelease(&c2);
-    if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM;
     return rc;
   }
 }
 
 /*
@@ -4428,12 +4445,14 @@
 ** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored
 ** in memory obtained from sqlite3_malloc) into a Vdbe.zErrMsg (text stored
 ** in memory obtained from sqlite3DbMalloc).
 */
 void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){
-  sqlite3 *db = p->db;
-  sqlite3DbFree(db, p->zErrMsg);
-  p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg);
-  sqlite3_free(pVtab->zErrMsg);
-  pVtab->zErrMsg = 0;
+  if( pVtab->zErrMsg ){
+    sqlite3 *db = p->db;
+    sqlite3DbFree(db, p->zErrMsg);
+    p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg);
+    sqlite3_free(pVtab->zErrMsg);
+    pVtab->zErrMsg = 0;
+  }
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

Index: src/vdbeblob.c
==================================================================
--- src/vdbeblob.c
+++ src/vdbeblob.c
@@ -113,42 +113,10 @@
   int flags,              /* True -> read/write access, false -> read-only */
   sqlite3_blob **ppBlob   /* Handle for accessing the blob returned here */
 ){
   int nAttempt = 0;
   int iCol;               /* Index of zColumn in row-record */
-
-  /* This VDBE program seeks a btree cursor to the identified 
-  ** db/table/row entry. The reason for using a vdbe program instead
-  ** of writing code to use the b-tree layer directly is that the
-  ** vdbe program will take advantage of the various transaction,
-  ** locking and error handling infrastructure built into the vdbe.
-  **
-  ** After seeking the cursor, the vdbe executes an OP_ResultRow.
-  ** Code external to the Vdbe then "borrows" the b-tree cursor and
-  ** uses it to implement the blob_read(), blob_write() and 
-  ** blob_bytes() functions.
-  **
-  ** The sqlite3_blob_close() function finalizes the vdbe program,
-  ** which closes the b-tree cursor and (possibly) commits the 
-  ** transaction.
-  */
-  static const int iLn = VDBE_OFFSET_LINENO(4);
-  static const VdbeOpList openBlob[] = {
-    /* {OP_Transaction, 0, 0, 0},  // 0: Inserted separately */
-    {OP_TableLock, 0, 0, 0},       /* 1: Acquire a read or write lock */
-    /* One of the following two instructions is replaced by an OP_Noop. */
-    {OP_OpenRead, 0, 0, 0},        /* 2: Open cursor 0 for reading */
-    {OP_OpenWrite, 0, 0, 0},       /* 3: Open cursor 0 for read/write */
-    {OP_Variable, 1, 1, 1},        /* 4: Push the rowid to the stack */
-    {OP_NotExists, 0, 10, 1},      /* 5: Seek the cursor */
-    {OP_Column, 0, 0, 1},          /* 6  */
-    {OP_ResultRow, 1, 0, 0},       /* 7  */
-    {OP_Goto, 0, 4, 0},            /* 8  */
-    {OP_Close, 0, 0, 0},           /* 9  */
-    {OP_Halt, 0, 0, 0},            /* 10 */
-  };
-
   int rc = SQLITE_OK;
   char *zErr = 0;
   Table *pTab;
   Parse *pParse = 0;
   Incrblob *pBlob = 0;
@@ -263,49 +231,84 @@
     }
 
     pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(pParse);
     assert( pBlob->pStmt || db->mallocFailed );
     if( pBlob->pStmt ){
+      
+      /* This VDBE program seeks a btree cursor to the identified 
+      ** db/table/row entry. The reason for using a vdbe program instead
+      ** of writing code to use the b-tree layer directly is that the
+      ** vdbe program will take advantage of the various transaction,
+      ** locking and error handling infrastructure built into the vdbe.
+      **
+      ** After seeking the cursor, the vdbe executes an OP_ResultRow.
+      ** Code external to the Vdbe then "borrows" the b-tree cursor and
+      ** uses it to implement the blob_read(), blob_write() and 
+      ** blob_bytes() functions.
+      **
+      ** The sqlite3_blob_close() function finalizes the vdbe program,
+      ** which closes the b-tree cursor and (possibly) commits the 
+      ** transaction.
+      */
+      static const int iLn = VDBE_OFFSET_LINENO(4);
+      static const VdbeOpList openBlob[] = {
+                                    /* addr/ofst */
+        /* {OP_Transaction, 0, 0, 0},  // 0/   inserted separately */
+        {OP_TableLock, 0, 0, 0},       /* 1/0: Acquire a read or write lock */
+        {OP_OpenRead, 0, 0, 0},        /* 2/1: Open a cursor */
+        {OP_Variable, 1, 1, 0},        /* 3/2: Move ?1 into reg[1] */
+        {OP_NotExists, 0, 8, 1},       /* 4/3: Seek the cursor */
+        {OP_Column, 0, 0, 1},          /* 5/4  */
+        {OP_ResultRow, 1, 0, 0},       /* 6/5  */
+        {OP_Goto, 0, 3, 0},            /* 7/6  */
+        {OP_Close, 0, 0, 0},           /* 8/7  */
+        {OP_Halt, 0, 0, 0},            /* 9/8  */
+      };
       Vdbe *v = (Vdbe *)pBlob->pStmt;
       int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
-
+      VdbeOp *aOp;
 
       sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, flags, 
                            pTab->pSchema->schema_cookie,
                            pTab->pSchema->iGeneration);
       sqlite3VdbeChangeP5(v, 1);     
-      sqlite3VdbeAddOpList(v, ArraySize(openBlob), openBlob, iLn);
+      aOp = sqlite3VdbeAddOpList(v, ArraySize(openBlob), openBlob, iLn);
 
       /* Make sure a mutex is held on the table to be accessed */
       sqlite3VdbeUsesBtree(v, iDb); 
 
-      /* Configure the OP_TableLock instruction */
+      if( db->mallocFailed==0 ){
+        assert( aOp!=0 );
+        /* Configure the OP_TableLock instruction */
 #ifdef SQLITE_OMIT_SHARED_CACHE
-      sqlite3VdbeChangeToNoop(v, 1);
+        aOp[0].opcode = OP_Noop;
 #else
-      sqlite3VdbeChangeP1(v, 1, iDb);
-      sqlite3VdbeChangeP2(v, 1, pTab->tnum);
-      sqlite3VdbeChangeP3(v, 1, flags);
-      sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT);
+        aOp[0].p1 = iDb;
+        aOp[0].p2 = pTab->tnum;
+        aOp[0].p3 = flags;
+        sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT);
+      }
+      if( db->mallocFailed==0 ){
 #endif
 
-      /* Remove either the OP_OpenWrite or OpenRead. Set the P2 
-      ** parameter of the other to pTab->tnum.  */
-      sqlite3VdbeChangeToNoop(v, 3 - flags);
-      sqlite3VdbeChangeP2(v, 2 + flags, pTab->tnum);
-      sqlite3VdbeChangeP3(v, 2 + flags, iDb);
-
-      /* Configure the number of columns. Configure the cursor to
-      ** think that the table has one more column than it really
-      ** does. An OP_Column to retrieve this imaginary column will
-      ** always return an SQL NULL. This is useful because it means
-      ** we can invoke OP_Column to fill in the vdbe cursors type 
-      ** and offset cache without causing any IO.
-      */
-      sqlite3VdbeChangeP4(v, 2+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32);
-      sqlite3VdbeChangeP2(v, 6, pTab->nCol);
-      if( !db->mallocFailed ){
+        /* Remove either the OP_OpenWrite or OpenRead. Set the P2 
+        ** parameter of the other to pTab->tnum.  */
+        if( flags ) aOp[1].opcode = OP_OpenWrite;
+        aOp[1].p2 = pTab->tnum;
+        aOp[1].p3 = iDb;   
+
+        /* Configure the number of columns. Configure the cursor to
+        ** think that the table has one more column than it really
+        ** does. An OP_Column to retrieve this imaginary column will
+        ** always return an SQL NULL. This is useful because it means
+        ** we can invoke OP_Column to fill in the vdbe cursors type 
+        ** and offset cache without causing any IO.
+        */
+        aOp[1].p4type = P4_INT32;
+        aOp[1].p4.i = pTab->nCol+1;
+        aOp[4].p2 = pTab->nCol;
+
         pParse->nVar = 1;
         pParse->nMem = 1;
         pParse->nTab = 1;
         sqlite3VdbeMakeReady(v, pParse);
       }

Index: src/vdbemem.c
==================================================================
--- src/vdbemem.c
+++ src/vdbemem.c
@@ -1222,11 +1222,11 @@
 
   assert( pCtx->pParse->rc==SQLITE_OK );
   memset(&ctx, 0, sizeof(ctx));
   ctx.pOut = pVal;
   ctx.pFunc = pFunc;
-  pFunc->xFunc(&ctx, nVal, apVal);
+  pFunc->xSFunc(&ctx, nVal, apVal);
   if( ctx.isError ){
     rc = ctx.isError;
     sqlite3ErrorMsg(pCtx->pParse, "%s", sqlite3_value_text(pVal));
   }else{
     sqlite3ValueApplyAffinity(pVal, aff, SQLITE_UTF8);

Index: src/vtab.c
==================================================================
--- src/vtab.c
+++ src/vtab.c
@@ -1014,11 +1014,11 @@
   Expr *pExpr     /* First argument to the function */
 ){
   Table *pTab;
   sqlite3_vtab *pVtab;
   sqlite3_module *pMod;
-  void (*xFunc)(sqlite3_context*,int,sqlite3_value**) = 0;
+  void (*xSFunc)(sqlite3_context*,int,sqlite3_value**) = 0;
   void *pArg = 0;
   FuncDef *pNew;
   int rc = 0;
   char *zLowerName;
   unsigned char *z;
@@ -1042,11 +1042,11 @@
   zLowerName = sqlite3DbStrDup(db, pDef->zName);
   if( zLowerName ){
     for(z=(unsigned char*)zLowerName; *z; z++){
       *z = sqlite3UpperToLower[*z];
     }
-    rc = pMod->xFindFunction(pVtab, nArg, zLowerName, &xFunc, &pArg);
+    rc = pMod->xFindFunction(pVtab, nArg, zLowerName, &xSFunc, &pArg);
     sqlite3DbFree(db, zLowerName);
   }
   if( rc==0 ){
     return pDef;
   }
@@ -1059,11 +1059,11 @@
     return pDef;
   }
   *pNew = *pDef;
   pNew->zName = (char *)&pNew[1];
   memcpy(pNew->zName, pDef->zName, sqlite3Strlen30(pDef->zName)+1);
-  pNew->xFunc = xFunc;
+  pNew->xSFunc = xSFunc;
   pNew->pUserData = pArg;
   pNew->funcFlags |= SQLITE_FUNC_EPHEM;
   return pNew;
 }
 

Index: src/wherecode.c
==================================================================
--- src/wherecode.c
+++ src/wherecode.c
@@ -325,12 +325,11 @@
     n--;
   }
 
   /* Code the OP_Affinity opcode if there is anything left to do. */
   if( n>0 ){
-    sqlite3VdbeAddOp2(v, OP_Affinity, base, n);
-    sqlite3VdbeChangeP4(v, -1, zAff, n);
+    sqlite3VdbeAddOp4(v, OP_Affinity, base, n, 0, zAff, n);
     sqlite3ExprCacheAffinityChange(pParse, base, n);
   }
 }
 
 

Index: src/whereexpr.c
==================================================================
--- src/whereexpr.c
+++ src/whereexpr.c
@@ -200,10 +200,11 @@
   int cnt;                   /* Number of non-wildcard prefix characters */
   char wc[3];                /* Wildcard characters */
   sqlite3 *db = pParse->db;  /* Database connection */
   sqlite3_value *pVal = 0;
   int op;                    /* Opcode of pRight */
+  int rc;                    /* Result code to return */
 
   if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){
     return 0;
   }
 #ifdef SQLITE_EBCDIC
@@ -265,12 +266,13 @@
     }else{
       z = 0;
     }
   }
 
+  rc = (z!=0);
   sqlite3ValueFree(pVal);
-  return (z!=0);
+  return rc;
 }
 #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */
 
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE

Index: test/threadtest3.c
==================================================================
--- test/threadtest3.c
+++ test/threadtest3.c
@@ -878,11 +878,11 @@
 
 static double currentTime(void){
   double t;
   static sqlite3_vfs *pTimelimitVfs = 0;
   if( pTimelimitVfs==0 ) pTimelimitVfs = sqlite3_vfs_find(0);
-  if( pTimelimitVfs->iVersion>=1 && pTimelimitVfs->xCurrentTimeInt64!=0 ){
+  if( pTimelimitVfs->iVersion>=2 && pTimelimitVfs->xCurrentTimeInt64!=0 ){
     sqlite3_int64 tm;
     pTimelimitVfs->xCurrentTimeInt64(pTimelimitVfs, &tm);
     t = tm/86400000.0;
   }else{
     pTimelimitVfs->xCurrentTime(pTimelimitVfs, &t);