/ Check-in [3ed49691]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge all recent enhancements from trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | apple-osx
Files: files | file ages | folders
SHA1: 3ed49691a12455e8583d8b8394ab005124442e89
User & Date: drh 2016-01-20 11:40:31
Context
2016-02-02
02:30
Merge recent enhancements from trunk. check-in: e6a4a163 user: drh tags: apple-osx
2016-01-20
11:40
Merge all recent enhancements from trunk. check-in: 3ed49691 user: drh tags: apple-osx
08:47
Improve performance of sqlite3VtabImportErrmsg(). check-in: 18d61c8e user: dan tags: trunk
2016-01-14
15:03
Merge the latest enhancements and fixes from trunk. check-in: d85774e0 user: drh tags: apple-osx
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   416    416     $(TOP)/ext/misc/amatch.c \
   417    417     $(TOP)/ext/misc/closure.c \
   418    418     $(TOP)/ext/misc/eval.c \
   419    419     $(TOP)/ext/misc/fileio.c \
   420    420     $(TOP)/ext/misc/fuzzer.c \
   421    421     $(TOP)/ext/fts5/fts5_tcl.c \
   422    422     $(TOP)/ext/fts5/fts5_test_mi.c \
          423  +  $(TOP)/ext/fts5/fts5_test_tok.c \
   423    424     $(TOP)/ext/misc/ieee754.c \
   424    425     $(TOP)/ext/misc/nextchar.c \
   425    426     $(TOP)/ext/misc/percentile.c \
   426    427     $(TOP)/ext/misc/regexp.c \
   427    428     $(TOP)/ext/misc/series.c \
   428    429     $(TOP)/ext/misc/spellfix.c \
   429    430     $(TOP)/ext/misc/totype.c \

Changes to Makefile.msc.

  1083   1083     $(TOP)\ext\misc\amatch.c \
  1084   1084     $(TOP)\ext\misc\closure.c \
  1085   1085     $(TOP)\ext\misc\eval.c \
  1086   1086     $(TOP)\ext\misc\fileio.c \
  1087   1087     $(TOP)\ext\misc\fuzzer.c \
  1088   1088     $(TOP)\ext\fts5\fts5_tcl.c \
  1089   1089     $(TOP)\ext\fts5\fts5_test_mi.c \
         1090  +  $(TOP)\ext\fts5\fts5_test_tok.c \
  1090   1091     $(TOP)\ext\misc\ieee754.c \
  1091   1092     $(TOP)\ext\misc\nextchar.c \
  1092   1093     $(TOP)\ext\misc\percentile.c \
  1093   1094     $(TOP)\ext\misc\regexp.c \
  1094   1095     $(TOP)\ext\misc\series.c \
  1095   1096     $(TOP)\ext\misc\spellfix.c \
  1096   1097     $(TOP)\ext\misc\totype.c \
................................................................................
  1710   1711   
  1711   1712   fts5_ext.lo:	fts5.c $(HDR) $(EXTHDR)
  1712   1713   	$(LTCOMPILE) $(NO_WARN) -c fts5.c
  1713   1714   
  1714   1715   fts5.dll:	fts5_ext.lo
  1715   1716   	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ fts5_ext.lo
  1716   1717   
  1717         -sqlite3rbu.lo:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
  1718         -	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c
         1718  +sqlite3rbu.lo:	$(TOP)\ext\rbu\sqlite3rbu.c $(HDR) $(EXTHDR)
         1719  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\rbu\sqlite3rbu.c
  1719   1720   
  1720   1721   # Rules to build the 'testfixture' application.
  1721   1722   #
  1722   1723   # If using the amalgamation, use sqlite3.c directly to build the test
  1723   1724   # fixture.  Otherwise link against libsqlite3.lib.  (This distinction is
  1724   1725   # necessary because the test fixture requires non-API symbols which are
  1725   1726   # hidden when the library is built via the amalgamation).
................................................................................
  1836   1837   		$(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1837   1838   
  1838   1839   speedtest1.exe:	$(TOP)\test\speedtest1.c $(SQLITE3C)
  1839   1840   	$(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1840   1841   		$(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1841   1842   
  1842   1843   rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C)
  1843         -	$(LTLINK) $(NO_WARN) -I. -DSQLITE_ENABLE_RBU -Fe$@ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) \
  1844         -		$(LDFLAGS) $(LTLINKOPTS)
         1844  +	$(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) \
         1845  +		/link $(LDFLAGS) $(LTLINKOPTS)
  1845   1846   
  1846   1847   clean:
  1847   1848   	del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL
  1848   1849   	del /Q *.bsc *.cod *.da *.bb *.bbg gmon.out 2>NUL
  1849   1850   	del /Q sqlite3.h opcodes.c opcodes.h 2>NUL
  1850   1851   	del /Q lemon.* lempar.c parse.* 2>NUL
  1851   1852   	del /Q mkkeywordhash.* keywordhash.h 2>NUL

Changes to ext/fts5/fts5Int.h.

   221    221   
   222    222   /*
   223    223   ** Buffer object for the incremental building of string data.
   224    224   */
   225    225   typedef struct Fts5Buffer Fts5Buffer;
   226    226   struct Fts5Buffer {
   227    227     u8 *p;
   228         -  int n;
   229         -  int nSpace;
          228  +  u32 n;
          229  +  u32 nSpace;
   230    230   };
   231    231   
   232         -int sqlite3Fts5BufferSize(int*, Fts5Buffer*, int);
          232  +int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);
   233    233   void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
   234         -void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*);
          234  +void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);
   235    235   void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
   236    236   void sqlite3Fts5BufferFree(Fts5Buffer*);
   237    237   void sqlite3Fts5BufferZero(Fts5Buffer*);
   238    238   void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
   239    239   void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
   240    240   
   241    241   char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
................................................................................
   582    582   int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
   583    583   int sqlite3Fts5StorageClose(Fts5Storage *p);
   584    584   int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
   585    585   
   586    586   int sqlite3Fts5DropAll(Fts5Config*);
   587    587   int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
   588    588   
   589         -int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
          589  +int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
   590    590   int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
   591    591   int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
   592    592   
   593    593   int sqlite3Fts5StorageIntegrity(Fts5Storage *p);
   594    594   
   595    595   int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
   596    596   void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
................................................................................
   602    602   int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
   603    603   int sqlite3Fts5StorageRollback(Fts5Storage *p);
   604    604   
   605    605   int sqlite3Fts5StorageConfigValue(
   606    606       Fts5Storage *p, const char*, sqlite3_value*, int
   607    607   );
   608    608   
   609         -int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**);
   610         -
   611    609   int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
   612    610   int sqlite3Fts5StorageRebuild(Fts5Storage *p);
   613    611   int sqlite3Fts5StorageOptimize(Fts5Storage *p);
   614    612   int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
   615    613   
   616    614   /*
   617    615   ** End of interface to code in fts5_storage.c.

Changes to ext/fts5/fts5_buffer.c.

    11     11   ******************************************************************************
    12     12   */
    13     13   
    14     14   
    15     15   
    16     16   #include "fts5Int.h"
    17     17   
    18         -int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, int nByte){
    19         -  int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
           18  +int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){
           19  +  u32 nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
    20     20     u8 *pNew;
    21     21     while( nNew<nByte ){
    22     22       nNew = nNew * 2;
    23     23     }
    24     24     pNew = sqlite3_realloc(pBuf->p, nNew);
    25     25     if( pNew==0 ){
    26     26       *pRc = SQLITE_NOMEM;
................................................................................
    57     57   ** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set 
    58     58   ** the error code in p. If an error has already occurred when this function
    59     59   ** is called, it is a no-op.
    60     60   */
    61     61   void sqlite3Fts5BufferAppendBlob(
    62     62     int *pRc,
    63     63     Fts5Buffer *pBuf, 
    64         -  int nData, 
           64  +  u32 nData, 
    65     65     const u8 *pData
    66     66   ){
    67         -  assert( *pRc || nData>=0 );
           67  +  assert_nc( *pRc || nData>=0 );
    68     68     if( fts5BufferGrow(pRc, pBuf, nData) ) return;
    69     69     memcpy(&pBuf->p[pBuf->n], pData, nData);
    70     70     pBuf->n += nData;
    71     71   }
    72     72   
    73     73   /*
    74     74   ** Append the nul-terminated string zStr to the buffer pBuf. This function
................................................................................
   318    318     const char *pTerm, int nTerm, 
   319    319     int *pbPresent
   320    320   ){
   321    321     int rc = SQLITE_OK;
   322    322     *pbPresent = 0;
   323    323     if( p ){
   324    324       int i;
   325         -    int hash;
          325  +    int hash = 13;
   326    326       Fts5TermsetEntry *pEntry;
   327    327   
   328         -    /* Calculate a hash value for this term */
   329         -    hash = 104 + iIdx;
   330         -    for(i=0; i<nTerm; i++){
   331         -      hash += (hash << 3) + (int)pTerm[i];
          328  +    /* Calculate a hash value for this term. This is the same hash checksum
          329  +    ** used by the fts5_hash.c module. This is not important for correct
          330  +    ** operation of the module, but is necessary to ensure that some tests
          331  +    ** designed to produce hash table collisions really do work.  */
          332  +    for(i=nTerm-1; i>=0; i--){
          333  +      hash = (hash << 3) ^ hash ^ pTerm[i];
   332    334       }
          335  +    hash = (hash << 3) ^ hash ^ iIdx;
   333    336       hash = hash % ArraySize(p->apHash);
   334    337   
   335    338       for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){
   336    339         if( pEntry->iIdx==iIdx 
   337    340             && pEntry->nTerm==nTerm 
   338    341             && memcmp(pEntry->pTerm, pTerm, nTerm)==0 
   339    342           ){

Changes to ext/fts5/fts5_config.c.

   274    274         }
   275    275   
   276    276         while( p[0]>='0' && p[0]<='9' && nPre<1000 ){
   277    277           nPre = nPre*10 + (p[0] - '0');
   278    278           p++;
   279    279         }
   280    280   
   281         -      if( rc==SQLITE_OK && (nPre<=0 || nPre>=1000) ){
          281  +      if( nPre<=0 || nPre>=1000 ){
   282    282           *pzErr = sqlite3_mprintf("prefix length out of range (max 999)");
   283    283           rc = SQLITE_ERROR;
   284    284           break;
   285    285         }
   286    286   
   287    287         pConfig->aPrefix[pConfig->nPrefix] = nPre;
   288    288         pConfig->nPrefix++;

Changes to ext/fts5/fts5_expr.c.

   830    830     int rc;
   831    831   
   832    832     assert( pNode->eType==FTS5_TERM );
   833    833     assert( pNear->nPhrase==1 && pPhrase->nTerm==1 );
   834    834     assert( pPhrase->aTerm[0].pSynonym==0 );
   835    835   
   836    836     rc = sqlite3Fts5IterPoslist(pIter, pColset, 
   837         -      (const u8**)&pPhrase->poslist.p, &pPhrase->poslist.n, &pNode->iRowid
          837  +      (const u8**)&pPhrase->poslist.p, (int*)&pPhrase->poslist.n, &pNode->iRowid
   838    838     );
   839    839     pNode->bNomatch = (pPhrase->poslist.n==0);
   840    840     return rc;
   841    841   }
   842    842   
   843    843   /*
   844    844   ** All individual term iterators in pNear are guaranteed to be valid when
................................................................................
  2392   2392         aPopulator[i].bOk = 0;
  2393   2393       }else{
  2394   2394         aPopulator[i].bOk = 1;
  2395   2395       }
  2396   2396     }
  2397   2397   
  2398   2398     return sqlite3Fts5Tokenize(pConfig, 
  2399         -      FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
         2399  +      FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
  2400   2400     );
  2401   2401   }
  2402   2402   
  2403   2403   static void fts5ExprClearPoslists(Fts5ExprNode *pNode){
  2404   2404     if( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING ){
  2405   2405       pNode->pNear->apPhrase[0]->poslist.n = 0;
  2406   2406     }else{
................................................................................
  2408   2408       for(i=0; i<pNode->nChild; i++){
  2409   2409         fts5ExprClearPoslists(pNode->apChild[i]);
  2410   2410       }
  2411   2411     }
  2412   2412   }
  2413   2413   
  2414   2414   static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){
  2415         -  if( pNode ){
  2416         -    pNode->iRowid = iRowid;
  2417         -    pNode->bEof = 0;
  2418         -    switch( pNode->eType ){
  2419         -      case FTS5_TERM:
  2420         -      case FTS5_STRING:
  2421         -        return (pNode->pNear->apPhrase[0]->poslist.n>0);
         2415  +  pNode->iRowid = iRowid;
         2416  +  pNode->bEof = 0;
         2417  +  switch( pNode->eType ){
         2418  +    case FTS5_TERM:
         2419  +    case FTS5_STRING:
         2420  +      return (pNode->pNear->apPhrase[0]->poslist.n>0);
  2422   2421   
  2423         -      case FTS5_AND: {
  2424         -        int i;
  2425         -        for(i=0; i<pNode->nChild; i++){
  2426         -          if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){
  2427         -            fts5ExprClearPoslists(pNode);
  2428         -            return 0;
  2429         -          }
  2430         -        }
  2431         -        break;
  2432         -      }
  2433         -
  2434         -      case FTS5_OR: {
  2435         -        int i;
  2436         -        int bRet = 0;
  2437         -        for(i=0; i<pNode->nChild; i++){
  2438         -          if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){
  2439         -            bRet = 1;
  2440         -          }
  2441         -        }
  2442         -        if( bRet==0 ){
  2443         -          fts5ExprClearPoslists(pNode);
  2444         -        }
  2445         -        return bRet;
  2446         -      }
  2447         -
  2448         -      default: {
  2449         -        assert( pNode->eType==FTS5_NOT );
  2450         -        if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid)
  2451         -         || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid)
  2452         -        ){
         2422  +    case FTS5_AND: {
         2423  +      int i;
         2424  +      for(i=0; i<pNode->nChild; i++){
         2425  +        if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){
  2453   2426             fts5ExprClearPoslists(pNode);
  2454   2427             return 0;
  2455   2428           }
  2456         -        break;
         2429  +      }
         2430  +      break;
         2431  +    }
         2432  +
         2433  +    case FTS5_OR: {
         2434  +      int i;
         2435  +      int bRet = 0;
         2436  +      for(i=0; i<pNode->nChild; i++){
         2437  +        if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){
         2438  +          bRet = 1;
         2439  +        }
         2440  +      }
         2441  +      return bRet;
         2442  +    }
         2443  +
         2444  +    default: {
         2445  +      assert( pNode->eType==FTS5_NOT );
         2446  +      if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid)
         2447  +          || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid)
         2448  +        ){
         2449  +        fts5ExprClearPoslists(pNode);
         2450  +        return 0;
  2457   2451         }
         2452  +      break;
  2458   2453       }
  2459   2454     }
  2460   2455     return 1;
  2461   2456   }
  2462   2457   
  2463   2458   void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){
  2464   2459     fts5ExprCheckPoslists(pExpr->pRoot, iRowid);

Changes to ext/fts5/fts5_index.c.

  1871   1871     Fts5SegIter *pIter,             /* Iterator to advance */
  1872   1872     int *pbNewTerm                  /* OUT: Set for new term */
  1873   1873   ){
  1874   1874     Fts5Data *pLeaf = pIter->pLeaf;
  1875   1875     int iOff;
  1876   1876     int bNewTerm = 0;
  1877   1877     int nKeep = 0;
         1878  +  u8 *a;
         1879  +  int n;
  1878   1880   
  1879   1881     assert( pbNewTerm==0 || *pbNewTerm==0 );
  1880   1882     assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE );
  1881   1883   
  1882   1884     /* Search for the end of the position list within the current page. */
  1883         -  u8 *a = pLeaf->p;
  1884         -  int n = pLeaf->szLeaf;
         1885  +  a = pLeaf->p;
         1886  +  n = pLeaf->szLeaf;
  1885   1887   
  1886   1888     ASSERT_SZLEAF_OK(pLeaf);
  1887   1889     iOff = pIter->iLeafOffset + pIter->nPos;
  1888   1890   
  1889   1891     if( iOff<n ){
  1890   1892       /* The next entry is on the current page. */
  1891   1893       assert_nc( iOff<=pIter->iEndofDoclist );
................................................................................
  5897   5899         iDocid += iDelta;
  5898   5900         sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid);
  5899   5901       }
  5900   5902     }
  5901   5903   
  5902   5904     return iOff;
  5903   5905   }
         5906  +
         5907  +/*
         5908  +** This function is part of the fts5_decode() debugging function. It is 
         5909  +** only ever used with detail=none tables.
         5910  +**
         5911  +** Buffer (pData/nData) contains a doclist in the format used by detail=none
         5912  +** tables. This function appends a human-readable version of that list to
         5913  +** buffer pBuf.
         5914  +**
         5915  +** If *pRc is other than SQLITE_OK when this function is called, it is a
         5916  +** no-op. If an OOM or other error occurs within this function, *pRc is
         5917  +** set to an SQLite error code before returning. The final state of buffer
         5918  +** pBuf is undefined in this case.
         5919  +*/
         5920  +static void fts5DecodeRowidList(
         5921  +  int *pRc,                       /* IN/OUT: Error code */
         5922  +  Fts5Buffer *pBuf,               /* Buffer to append text to */
         5923  +  const u8 *pData, int nData      /* Data to decode list-of-rowids from */
         5924  +){
         5925  +  int i = 0;
         5926  +  i64 iRowid = 0;
         5927  +
         5928  +  while( i<nData ){
         5929  +    const char *zApp = "";
         5930  +    u64 iVal;
         5931  +    i += sqlite3Fts5GetVarint(&pData[i], &iVal);
         5932  +    iRowid += iVal;
         5933  +
         5934  +    if( i<nData && pData[i]==0x00 ){
         5935  +      i++;
         5936  +      if( i<nData && pData[i]==0x00 ){
         5937  +        i++;
         5938  +        zApp = "+";
         5939  +      }else{
         5940  +        zApp = "*";
         5941  +      }
         5942  +    }
         5943  +
         5944  +    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %lld%s", iRowid, zApp);
         5945  +  }
         5946  +}
  5904   5947   
  5905   5948   /*
  5906   5949   ** The implementation of user-defined scalar function fts5_decode().
  5907   5950   */
  5908   5951   static void fts5DecodeFunction(
  5909   5952     sqlite3_context *pCtx,          /* Function call context */
  5910   5953     int nArg,                       /* Number of args (always 2) */
................................................................................
  5913   5956     i64 iRowid;                     /* Rowid for record being decoded */
  5914   5957     int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */
  5915   5958     const u8 *aBlob; int n;         /* Record to decode */
  5916   5959     u8 *a = 0;
  5917   5960     Fts5Buffer s;                   /* Build up text to return here */
  5918   5961     int rc = SQLITE_OK;             /* Return code */
  5919   5962     int nSpace = 0;
         5963  +  int eDetailNone = (sqlite3_user_data(pCtx)!=0);
  5920   5964   
  5921   5965     assert( nArg==2 );
  5922   5966     memset(&s, 0, sizeof(Fts5Buffer));
  5923   5967     iRowid = sqlite3_value_int64(apVal[0]);
  5924   5968   
  5925   5969     /* Make a copy of the second argument (a blob) in aBlob[]. The aBlob[]
  5926   5970     ** copy is followed by FTS5_DATA_ZERO_PADDING 0x00 bytes, which prevents
................................................................................
  5954   5998       }
  5955   5999     }else if( iSegid==0 ){
  5956   6000       if( iRowid==FTS5_AVERAGES_ROWID ){
  5957   6001         fts5DecodeAverages(&rc, &s, a, n);
  5958   6002       }else{
  5959   6003         fts5DecodeStructure(&rc, &s, a, n);
  5960   6004       }
         6005  +  }else if( eDetailNone ){
         6006  +    Fts5Buffer term;              /* Current term read from page */
         6007  +    int szLeaf;
         6008  +    int iPgidxOff = szLeaf = fts5GetU16(&a[2]);
         6009  +    int iTermOff;
         6010  +    int nKeep = 0;
         6011  +    int iOff;
         6012  +
         6013  +    memset(&term, 0, sizeof(Fts5Buffer));
         6014  +
         6015  +    /* Decode any entries that occur before the first term. */
         6016  +    if( szLeaf<n ){
         6017  +      iPgidxOff += fts5GetVarint32(&a[iPgidxOff], iTermOff);
         6018  +    }else{
         6019  +      iTermOff = szLeaf;
         6020  +    }
         6021  +    fts5DecodeRowidList(&rc, &s, &a[4], iTermOff-4);
         6022  +
         6023  +    iOff = iTermOff;
         6024  +    while( iOff<szLeaf ){
         6025  +      int nAppend;
         6026  +
         6027  +      /* Read the term data for the next term*/
         6028  +      iOff += fts5GetVarint32(&a[iOff], nAppend);
         6029  +      term.n = nKeep;
         6030  +      fts5BufferAppendBlob(&rc, &term, nAppend, &a[iOff]);
         6031  +      sqlite3Fts5BufferAppendPrintf(
         6032  +          &rc, &s, " term=%.*s", term.n, (const char*)term.p
         6033  +      );
         6034  +      iOff += nAppend;
         6035  +
         6036  +      /* Figure out where the doclist for this term ends */
         6037  +      if( iPgidxOff<n ){
         6038  +        int nIncr;
         6039  +        iPgidxOff += fts5GetVarint32(&a[iPgidxOff], nIncr);
         6040  +        iTermOff += nIncr;
         6041  +      }else{
         6042  +        iTermOff = szLeaf;
         6043  +      }
         6044  +
         6045  +      fts5DecodeRowidList(&rc, &s, &a[iOff], iTermOff-iOff);
         6046  +      iOff = iTermOff;
         6047  +      if( iOff<szLeaf ){
         6048  +        iOff += fts5GetVarint32(&a[iOff], nKeep);
         6049  +      }
         6050  +    }
         6051  +
         6052  +    fts5BufferFree(&term);
  5961   6053     }else{
  5962   6054       Fts5Buffer term;              /* Current term read from page */
  5963   6055       int szLeaf;                   /* Offset of pgidx in a[] */
  5964   6056       int iPgidxOff;
  5965   6057       int iPgidxPrev = 0;           /* Previous value read from pgidx */
  5966   6058       int iTermOff = 0;
  5967   6059       int iRowidOff = 0;
................................................................................
  6081   6173   ** If successful, SQLITE_OK is returned. If an error occurs, some other
  6082   6174   ** SQLite error code is returned instead.
  6083   6175   */
  6084   6176   int sqlite3Fts5IndexInit(sqlite3 *db){
  6085   6177     int rc = sqlite3_create_function(
  6086   6178         db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0
  6087   6179     );
         6180  +
         6181  +  if( rc==SQLITE_OK ){
         6182  +    rc = sqlite3_create_function(
         6183  +        db, "fts5_decode_none", 2, 
         6184  +        SQLITE_UTF8, (void*)db, fts5DecodeFunction, 0, 0
         6185  +    );
         6186  +  }
         6187  +
  6088   6188     if( rc==SQLITE_OK ){
  6089   6189       rc = sqlite3_create_function(
  6090   6190           db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
  6091   6191       );
  6092   6192     }
  6093   6193     return rc;
  6094   6194   }

Changes to ext/fts5/fts5_main.c.

   840    840       }
   841    841     }
   842    842     
   843    843     return rc;
   844    844   }
   845    845   
   846    846   
   847         -static sqlite3_stmt *fts5PrepareStatement(
   848         -  int *pRc,
          847  +static int fts5PrepareStatement(
          848  +  sqlite3_stmt **ppStmt,
   849    849     Fts5Config *pConfig, 
   850    850     const char *zFmt,
   851    851     ...
   852    852   ){
   853    853     sqlite3_stmt *pRet = 0;
          854  +  int rc;
          855  +  char *zSql;
   854    856     va_list ap;
          857  +
   855    858     va_start(ap, zFmt);
   856         -
   857         -  if( *pRc==SQLITE_OK ){
   858         -    int rc;
   859         -    char *zSql = sqlite3_vmprintf(zFmt, ap);
   860         -    if( zSql==0 ){
   861         -      rc = SQLITE_NOMEM; 
   862         -    }else{
   863         -      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
   864         -      if( rc!=SQLITE_OK ){
   865         -        *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
   866         -      }
   867         -      sqlite3_free(zSql);
          859  +  zSql = sqlite3_vmprintf(zFmt, ap);
          860  +  if( zSql==0 ){
          861  +    rc = SQLITE_NOMEM; 
          862  +  }else{
          863  +    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
          864  +    if( rc!=SQLITE_OK ){
          865  +      *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
   868    866       }
   869         -    *pRc = rc;
          867  +    sqlite3_free(zSql);
   870    868     }
   871    869   
   872    870     va_end(ap);
   873         -  return pRet;
          871  +  *ppStmt = pRet;
          872  +  return rc;
   874    873   } 
   875    874   
   876    875   static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){
   877    876     Fts5Config *pConfig = pTab->pConfig;
   878    877     Fts5Sorter *pSorter;
   879    878     int nPhrase;
   880    879     int nByte;
   881         -  int rc = SQLITE_OK;
          880  +  int rc;
   882    881     const char *zRank = pCsr->zRank;
   883    882     const char *zRankArgs = pCsr->zRankArgs;
   884    883     
   885    884     nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
   886    885     nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1);
   887    886     pSorter = (Fts5Sorter*)sqlite3_malloc(nByte);
   888    887     if( pSorter==0 ) return SQLITE_NOMEM;
................................................................................
   892    891     /* TODO: It would be better to have some system for reusing statement
   893    892     ** handles here, rather than preparing a new one for each query. But that
   894    893     ** is not possible as SQLite reference counts the virtual table objects.
   895    894     ** And since the statement required here reads from this very virtual 
   896    895     ** table, saving it creates a circular reference.
   897    896     **
   898    897     ** If SQLite a built-in statement cache, this wouldn't be a problem. */
   899         -  pSorter->pStmt = fts5PrepareStatement(&rc, pConfig,
          898  +  rc = fts5PrepareStatement(&pSorter->pStmt, pConfig,
   900    899         "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s",
   901    900         pConfig->zDb, pConfig->zName, zRank, pConfig->zName,
   902    901         (zRankArgs ? ", " : ""),
   903    902         (zRankArgs ? zRankArgs : ""),
   904    903         bDesc ? "DESC" : "ASC"
   905    904     );
   906    905   
................................................................................
  1401   1400     sqlite3_value **apVal, 
  1402   1401     sqlite3_int64 *piRowid
  1403   1402   ){
  1404   1403     int rc = SQLITE_OK;
  1405   1404     int eType1 = sqlite3_value_type(apVal[1]);
  1406   1405     if( eType1==SQLITE_INTEGER ){
  1407   1406       sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]);
  1408         -    rc = sqlite3Fts5StorageSpecialDelete(pTab->pStorage, iDel, &apVal[2]);
         1407  +    rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2]);
  1409   1408     }
  1410   1409     return rc;
  1411   1410   }
  1412   1411   
  1413   1412   static void fts5StorageInsert(
  1414   1413     int *pRc, 
  1415   1414     Fts5Table *pTab, 
................................................................................
  1508   1507         );
  1509   1508         rc = SQLITE_ERROR;
  1510   1509       }
  1511   1510   
  1512   1511       /* Case 1: DELETE */
  1513   1512       else if( nArg==1 ){
  1514   1513         i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
  1515         -      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
         1514  +      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0);
  1516   1515       }
  1517   1516   
  1518   1517       /* Case 2: INSERT */
  1519   1518       else if( eType0!=SQLITE_INTEGER ){     
  1520   1519         /* If this is a REPLACE, first remove the current entry (if any) */
  1521   1520         if( eConflict==SQLITE_REPLACE 
  1522   1521          && sqlite3_value_type(apVal[1])==SQLITE_INTEGER 
  1523   1522         ){
  1524   1523           i64 iNew = sqlite3_value_int64(apVal[1]);  /* Rowid to delete */
  1525         -        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew);
         1524  +        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
  1526   1525         }
  1527   1526         fts5StorageInsert(&rc, pTab, apVal, pRowid);
  1528   1527       }
  1529   1528   
  1530   1529       /* Case 2: UPDATE */
  1531   1530       else{
  1532   1531         i64 iOld = sqlite3_value_int64(apVal[0]);  /* Old rowid */
  1533   1532         i64 iNew = sqlite3_value_int64(apVal[1]);  /* New rowid */
  1534   1533         if( iOld!=iNew ){
  1535   1534           if( eConflict==SQLITE_REPLACE ){
  1536         -          rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
         1535  +          rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
  1537   1536             if( rc==SQLITE_OK ){
  1538         -            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew);
         1537  +            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
  1539   1538             }
  1540   1539             fts5StorageInsert(&rc, pTab, apVal, pRowid);
  1541   1540           }else{
  1542   1541             rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid);
  1543   1542             if( rc==SQLITE_OK ){
  1544         -            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
         1543  +            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
  1545   1544             }
  1546   1545             if( rc==SQLITE_OK ){
  1547   1546               rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *pRowid);
  1548   1547             }
  1549   1548           }
  1550   1549         }else{
  1551         -        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
         1550  +        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
  1552   1551           fts5StorageInsert(&rc, pTab, apVal, pRowid);
  1553   1552         }
  1554   1553       }
  1555   1554     }
  1556   1555   
  1557   1556     pTab->pConfig->pzErrmsg = 0;
  1558   1557     return rc;
................................................................................
  1743   1742       int i;
  1744   1743   
  1745   1744       /* Initialize all iterators */
  1746   1745       for(i=0; i<nIter && rc==SQLITE_OK; i++){
  1747   1746         const u8 *a;
  1748   1747         int n; 
  1749   1748         rc = fts5CsrPoslist(pCsr, i, &a, &n);
  1750         -      sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
         1749  +      if( rc==SQLITE_OK ){
         1750  +        sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
         1751  +      }
  1751   1752       }
  1752   1753   
  1753   1754       if( rc==SQLITE_OK ){
  1754   1755         while( 1 ){
  1755   1756           int *aInst;
  1756   1757           int iBest = -1;
  1757   1758           for(i=0; i<nIter; i++){
................................................................................
  2033   2034     int *piCol
  2034   2035   ){
  2035   2036     int rc = SQLITE_OK;
  2036   2037     Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  2037   2038     Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
  2038   2039   
  2039   2040     if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
         2041  +    Fts5Sorter *pSorter = pCsr->pSorter;
  2040   2042       int n;
  2041         -    rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n);
         2043  +    if( pSorter ){
         2044  +      int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
         2045  +      n = pSorter->aIdx[iPhrase] - i1;
         2046  +      pIter->a = &pSorter->aPoslist[i1];
         2047  +    }else{
         2048  +      rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n);
         2049  +    }
  2042   2050       if( rc==SQLITE_OK ){
  2043   2051         pIter->b = &pIter->a[n];
  2044   2052         *piCol = 0;
  2045   2053         fts5ApiPhraseNextColumn(pCtx, pIter, piCol);
  2046   2054       }
  2047   2055     }else{
  2048   2056       int n;

Changes to ext/fts5/fts5_storage.c.

   374    374   }
   375    375   
   376    376   /*
   377    377   ** If a row with rowid iDel is present in the %_content table, add the
   378    378   ** delete-markers to the FTS index necessary to delete it. Do not actually
   379    379   ** remove the %_content row at this time though.
   380    380   */
   381         -static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){
          381  +static int fts5StorageDeleteFromIndex(
          382  +  Fts5Storage *p, 
          383  +  i64 iDel, 
          384  +  sqlite3_value **apVal
          385  +){
   382    386     Fts5Config *pConfig = p->pConfig;
   383         -  sqlite3_stmt *pSeek;            /* SELECT to read row iDel from %_data */
          387  +  sqlite3_stmt *pSeek = 0;        /* SELECT to read row iDel from %_data */
   384    388     int rc;                         /* Return code */
          389  +  int rc2;                        /* sqlite3_reset() return code */
          390  +  int iCol;
          391  +  Fts5InsertCtx ctx;
   385    392   
   386         -  rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
   387         -  if( rc==SQLITE_OK ){
   388         -    int rc2;
          393  +  if( apVal==0 ){
          394  +    rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
          395  +    if( rc!=SQLITE_OK ) return rc;
   389    396       sqlite3_bind_int64(pSeek, 1, iDel);
   390         -    if( sqlite3_step(pSeek)==SQLITE_ROW ){
   391         -      int iCol;
   392         -      Fts5InsertCtx ctx;
   393         -      ctx.pStorage = p;
   394         -      ctx.iCol = -1;
   395         -      rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
   396         -      for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
   397         -        if( pConfig->abUnindexed[iCol-1] ) continue;
   398         -        ctx.szCol = 0;
   399         -        rc = sqlite3Fts5Tokenize(pConfig, 
   400         -            FTS5_TOKENIZE_DOCUMENT,
   401         -            (const char*)sqlite3_column_text(pSeek, iCol),
   402         -            sqlite3_column_bytes(pSeek, iCol),
   403         -            (void*)&ctx,
   404         -            fts5StorageInsertCallback
   405         -        );
   406         -        p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
          397  +    if( sqlite3_step(pSeek)!=SQLITE_ROW ){
          398  +      return sqlite3_reset(pSeek);
          399  +    }
          400  +  }
          401  +
          402  +  ctx.pStorage = p;
          403  +  ctx.iCol = -1;
          404  +  rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
          405  +  for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
          406  +    if( pConfig->abUnindexed[iCol-1]==0 ){
          407  +      const char *zText;
          408  +      int nText;
          409  +      if( pSeek ){
          410  +        zText = (const char*)sqlite3_column_text(pSeek, iCol);
          411  +        nText = sqlite3_column_bytes(pSeek, iCol);
          412  +      }else{
          413  +        zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
          414  +        nText = sqlite3_value_bytes(apVal[iCol-1]);
   407    415         }
   408         -      p->nTotalRow--;
          416  +      ctx.szCol = 0;
          417  +      rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, 
          418  +          zText, nText, (void*)&ctx, fts5StorageInsertCallback
          419  +      );
          420  +      p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
   409    421       }
   410         -    rc2 = sqlite3_reset(pSeek);
   411         -    if( rc==SQLITE_OK ) rc = rc2;
   412    422     }
          423  +  p->nTotalRow--;
   413    424   
          425  +  rc2 = sqlite3_reset(pSeek);
          426  +  if( rc==SQLITE_OK ) rc = rc2;
   414    427     return rc;
   415    428   }
   416    429   
   417    430   
   418    431   /*
   419    432   ** Insert a record into the %_docsize table. Specifically, do:
   420    433   **
................................................................................
   486    499   
   487    500     return rc;
   488    501   }
   489    502   
   490    503   /*
   491    504   ** Remove a row from the FTS table.
   492    505   */
   493         -int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){
          506  +int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
   494    507     Fts5Config *pConfig = p->pConfig;
   495    508     int rc;
   496    509     sqlite3_stmt *pDel = 0;
   497    510   
          511  +  assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 );
   498    512     rc = fts5StorageLoadTotals(p, 1);
   499    513   
   500    514     /* Delete the index records */
   501    515     if( rc==SQLITE_OK ){
   502         -    rc = fts5StorageDeleteFromIndex(p, iDel);
          516  +    rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
   503    517     }
   504    518   
   505    519     /* Delete the %_docsize record */
   506    520     if( rc==SQLITE_OK && pConfig->bColumnsize ){
   507    521       rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
   508    522       if( rc==SQLITE_OK ){
   509    523         sqlite3_bind_int64(pDel, 1, iDel);
................................................................................
   513    527     }
   514    528   
   515    529     /* Delete the %_content record */
   516    530     if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
   517    531       if( rc==SQLITE_OK ){
   518    532         rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0);
   519    533       }
   520         -    if( rc==SQLITE_OK ){
   521         -      sqlite3_bind_int64(pDel, 1, iDel);
   522         -      sqlite3_step(pDel);
   523         -      rc = sqlite3_reset(pDel);
   524         -    }
   525         -  }
   526         -
   527         -  /* Write the averages record */
   528         -  if( rc==SQLITE_OK ){
   529         -    rc = fts5StorageSaveTotals(p);
   530         -  }
   531         -
   532         -  return rc;
   533         -}
   534         -
   535         -int sqlite3Fts5StorageSpecialDelete(
   536         -  Fts5Storage *p, 
   537         -  i64 iDel, 
   538         -  sqlite3_value **apVal
   539         -){
   540         -  Fts5Config *pConfig = p->pConfig;
   541         -  int rc;
   542         -  sqlite3_stmt *pDel = 0;
   543         -
   544         -  assert( pConfig->eContent!=FTS5_CONTENT_NORMAL );
   545         -  rc = fts5StorageLoadTotals(p, 1);
   546         -
   547         -  /* Delete the index records */
   548         -  if( rc==SQLITE_OK ){
   549         -    int iCol;
   550         -    Fts5InsertCtx ctx;
   551         -    ctx.pStorage = p;
   552         -    ctx.iCol = -1;
   553         -
   554         -    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
   555         -    for(iCol=0; rc==SQLITE_OK && iCol<pConfig->nCol; iCol++){
   556         -      if( pConfig->abUnindexed[iCol] ) continue;
   557         -      ctx.szCol = 0;
   558         -      rc = sqlite3Fts5Tokenize(pConfig, 
   559         -        FTS5_TOKENIZE_DOCUMENT,
   560         -        (const char*)sqlite3_value_text(apVal[iCol]),
   561         -        sqlite3_value_bytes(apVal[iCol]),
   562         -        (void*)&ctx,
   563         -        fts5StorageInsertCallback
   564         -      );
   565         -      p->aTotalSize[iCol] -= (i64)ctx.szCol;
   566         -    }
   567         -    p->nTotalRow--;
   568         -  }
   569         -
   570         -  /* Delete the %_docsize record */
   571         -  if( pConfig->bColumnsize ){
   572         -    if( rc==SQLITE_OK ){
   573         -      rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
   574         -    }
   575    534       if( rc==SQLITE_OK ){
   576    535         sqlite3_bind_int64(pDel, 1, iDel);
   577    536         sqlite3_step(pDel);
   578    537         rc = sqlite3_reset(pDel);
   579    538       }
   580    539     }
   581    540   

Changes to ext/fts5/fts5_tcl.c.

    19     19   #ifdef SQLITE_ENABLE_FTS5
    20     20   
    21     21   #include "fts5.h"
    22     22   #include <string.h>
    23     23   #include <assert.h>
    24     24   
    25     25   extern int sqlite3_fts5_may_be_corrupt;
    26         -extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *);
           26  +extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*);
           27  +extern int sqlite3Fts5TestRegisterTok(sqlite3*, fts5_api*);
    27     28   
    28     29   /*************************************************************************
    29     30   ** This is a copy of the first part of the SqliteDb structure in 
    30     31   ** tclsqlite.c.  We need it here so that the get_sqlite_pointer routine
    31     32   ** can extract the sqlite3* pointer from an existing Tcl SQLite
    32     33   ** connection.
    33     34   */
................................................................................
   442    443         Tcl_Obj *pScript = objv[5];
   443    444         Fts5PhraseIter iter;
   444    445   
   445    446         if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
   446    447         zColvar = Tcl_GetString(objv[3]);
   447    448         zOffvar = Tcl_GetString(objv[4]);
   448    449   
   449         -      for(p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);
   450         -          iCol>=0;
   451         -          p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff)
   452         -      ){
          450  +      rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);
          451  +      if( rc!=SQLITE_OK ){
          452  +        Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
          453  +        return TCL_ERROR;
          454  +      }
          455  +      for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){
   453    456           Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
   454    457           Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0);
   455    458           rc = Tcl_EvalObjEx(interp, pScript, 0);
   456    459           if( rc==TCL_CONTINUE ) rc = TCL_OK;
   457    460           if( rc!=TCL_OK ){
   458    461             if( rc==TCL_BREAK ) rc = TCL_OK;
   459    462             break;
................................................................................
   469    472         const char *zColvar;
   470    473         Tcl_Obj *pScript = objv[4];
   471    474         Fts5PhraseIter iter;
   472    475   
   473    476         if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
   474    477         zColvar = Tcl_GetString(objv[3]);
   475    478   
   476         -      for(p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
   477         -          iCol>=0;
   478         -          p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)
   479         -      ){
          479  +      rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
          480  +      if( rc!=SQLITE_OK ){
          481  +        Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
          482  +        return TCL_ERROR;
          483  +      }
          484  +      for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){
   480    485           Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
   481    486           rc = Tcl_EvalObjEx(interp, pScript, 0);
   482    487           if( rc==TCL_CONTINUE ) rc = TCL_OK;
   483    488           if( rc!=TCL_OK ){
   484    489             if( rc==TCL_BREAK ) rc = TCL_OK;
   485    490             break;
   486    491           }
................................................................................
  1073   1078     rc = sqlite3Fts5TestRegisterMatchinfo(db);
  1074   1079     if( rc!=SQLITE_OK ){
  1075   1080       Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
  1076   1081       return TCL_ERROR;
  1077   1082     }
  1078   1083     return TCL_OK;
  1079   1084   }
         1085  +
         1086  +static int f5tRegisterTok(
         1087  +  void * clientData,
         1088  +  Tcl_Interp *interp,
         1089  +  int objc,
         1090  +  Tcl_Obj *CONST objv[]
         1091  +){
         1092  +  int rc;
         1093  +  sqlite3 *db = 0;
         1094  +  fts5_api *pApi = 0;
         1095  +
         1096  +  if( objc!=2 ){
         1097  +    Tcl_WrongNumArgs(interp, 1, objv, "DB");
         1098  +    return TCL_ERROR;
         1099  +  }
         1100  +  if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){
         1101  +    return TCL_ERROR;
         1102  +  }
         1103  +
         1104  +  rc = sqlite3Fts5TestRegisterTok(db, pApi);
         1105  +  if( rc!=SQLITE_OK ){
         1106  +    Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
         1107  +    return TCL_ERROR;
         1108  +  }
         1109  +  return TCL_OK;
         1110  +}
  1080   1111   
  1081   1112   /*
  1082   1113   ** Entry point.
  1083   1114   */
  1084   1115   int Fts5tcl_Init(Tcl_Interp *interp){
  1085   1116     static struct Cmd {
  1086   1117       char *zName;
................................................................................
  1089   1120     } aCmd[] = {
  1090   1121       { "sqlite3_fts5_create_tokenizer",   f5tCreateTokenizer, 1 },
  1091   1122       { "sqlite3_fts5_token",              f5tTokenizerReturn, 1 },
  1092   1123       { "sqlite3_fts5_tokenize",           f5tTokenize, 0 },
  1093   1124       { "sqlite3_fts5_create_function",    f5tCreateFunction, 0 },
  1094   1125       { "sqlite3_fts5_may_be_corrupt",     f5tMayBeCorrupt, 0 },
  1095   1126       { "sqlite3_fts5_token_hash",         f5tTokenHash, 0 },
  1096         -    { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 }
         1127  +    { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 },
         1128  +    { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 }
  1097   1129     };
  1098   1130     int i;
  1099   1131     F5tTokenizerContext *pContext;
  1100   1132   
  1101   1133     pContext = (F5tTokenizerContext*)ckalloc(sizeof(F5tTokenizerContext));
  1102   1134     memset(pContext, 0, sizeof(*pContext));
  1103   1135   

Added ext/fts5/fts5_test_tok.c.

            1  +/*
            2  +** 2013 Apr 22
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +** This file contains code for the "fts5tokenize" virtual table module.
           14  +** An fts5tokenize virtual table is created as follows:
           15  +**
           16  +**   CREATE VIRTUAL TABLE <tbl> USING fts5tokenize(
           17  +**       <tokenizer-name>, <arg-1>, ...
           18  +**   );
           19  +**
           20  +** The table created has the following schema:
           21  +**
           22  +**   CREATE TABLE <tbl>(input HIDDEN, token, start, end, position)
           23  +**
           24  +** When queried, the query must include a WHERE clause of type:
           25  +**
           26  +**   input = <string>
           27  +**
           28  +** The virtual table module tokenizes this <string>, using the FTS3 
           29  +** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE 
           30  +** statement and returns one row for each token in the result. With
           31  +** fields set as follows:
           32  +**
           33  +**   input:   Always set to a copy of <string>
           34  +**   token:   A token from the input.
           35  +**   start:   Byte offset of the token within the input <string>.
           36  +**   end:     Byte offset of the byte immediately following the end of the
           37  +**            token within the input string.
           38  +**   pos:     Token offset of token within input.
           39  +**
           40  +*/
           41  +#if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5)
           42  +
           43  +#include <fts5.h>
           44  +#include <string.h>
           45  +#include <assert.h>
           46  +
           47  +typedef struct Fts5tokTable Fts5tokTable;
           48  +typedef struct Fts5tokCursor Fts5tokCursor;
           49  +typedef struct Fts5tokRow Fts5tokRow;
           50  +
           51  +/*
           52  +** Virtual table structure.
           53  +*/
           54  +struct Fts5tokTable {
           55  +  sqlite3_vtab base;              /* Base class used by SQLite core */
           56  +  fts5_tokenizer tok;             /* Tokenizer functions */
           57  +  Fts5Tokenizer *pTok;            /* Tokenizer instance */
           58  +};
           59  +
           60  +/*
           61  +** A container for a rows values.
           62  +*/
           63  +struct Fts5tokRow {
           64  +  char *zToken;
           65  +  int iStart;
           66  +  int iEnd;
           67  +  int iPos;
           68  +};
           69  +
           70  +/*
           71  +** Virtual table cursor structure.
           72  +*/
           73  +struct Fts5tokCursor {
           74  +  sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
           75  +  int iRowid;                     /* Current 'rowid' value */
           76  +  char *zInput;                   /* Input string */
           77  +  int nRow;                       /* Number of entries in aRow[] */
           78  +  Fts5tokRow *aRow;               /* Array of rows to return */
           79  +};
           80  +
           81  +static void fts5tokDequote(char *z){
           82  +  char q = z[0];
           83  +
           84  +  if( q=='[' || q=='\'' || q=='"' || q=='`' ){
           85  +    int iIn = 1;
           86  +    int iOut = 0;
           87  +    if( q=='[' ) q = ']';  
           88  +
           89  +    while( z[iIn] ){
           90  +      if( z[iIn]==q ){
           91  +        if( z[iIn+1]!=q ){
           92  +          /* Character iIn was the close quote. */
           93  +          iIn++;
           94  +          break;
           95  +        }else{
           96  +          /* Character iIn and iIn+1 form an escaped quote character. Skip
           97  +          ** the input cursor past both and copy a single quote character 
           98  +          ** to the output buffer. */
           99  +          iIn += 2;
          100  +          z[iOut++] = q;
          101  +        }
          102  +      }else{
          103  +        z[iOut++] = z[iIn++];
          104  +      }
          105  +    }
          106  +
          107  +    z[iOut] = '\0';
          108  +  }
          109  +}
          110  +
          111  +/*
          112  +** The second argument, argv[], is an array of pointers to nul-terminated
          113  +** strings. This function makes a copy of the array and strings into a 
          114  +** single block of memory. It then dequotes any of the strings that appear
          115  +** to be quoted.
          116  +**
          117  +** If successful, output parameter *pazDequote is set to point at the
          118  +** array of dequoted strings and SQLITE_OK is returned. The caller is
          119  +** responsible for eventually calling sqlite3_free() to free the array
          120  +** in this case. Or, if an error occurs, an SQLite error code is returned.
          121  +** The final value of *pazDequote is undefined in this case.
          122  +*/
          123  +static int fts5tokDequoteArray(
          124  +  int argc,                       /* Number of elements in argv[] */
          125  +  const char * const *argv,       /* Input array */
          126  +  char ***pazDequote              /* Output array */
          127  +){
          128  +  int rc = SQLITE_OK;             /* Return code */
          129  +  if( argc==0 ){
          130  +    *pazDequote = 0;
          131  +  }else{
          132  +    int i;
          133  +    int nByte = 0;
          134  +    char **azDequote;
          135  +
          136  +    for(i=0; i<argc; i++){
          137  +      nByte += (int)(strlen(argv[i]) + 1);
          138  +    }
          139  +
          140  +    *pazDequote = azDequote = sqlite3_malloc(sizeof(char *)*argc + nByte);
          141  +    if( azDequote==0 ){
          142  +      rc = SQLITE_NOMEM;
          143  +    }else{
          144  +      char *pSpace = (char *)&azDequote[argc];
          145  +      for(i=0; i<argc; i++){
          146  +        int n = (int)strlen(argv[i]);
          147  +        azDequote[i] = pSpace;
          148  +        memcpy(pSpace, argv[i], n+1);
          149  +        fts5tokDequote(pSpace);
          150  +        pSpace += (n+1);
          151  +      }
          152  +    }
          153  +  }
          154  +
          155  +  return rc;
          156  +}
          157  +
          158  +/*
          159  +** Schema of the tokenizer table.
          160  +*/
          161  +#define FTS3_TOK_SCHEMA "CREATE TABLE x(input HIDDEN, token, start, end, position)"
          162  +
          163  +/*
          164  +** This function does all the work for both the xConnect and xCreate methods.
          165  +** These tables have no persistent representation of their own, so xConnect
          166  +** and xCreate are identical operations.
          167  +**
          168  +**   argv[0]: module name
          169  +**   argv[1]: database name 
          170  +**   argv[2]: table name
          171  +**   argv[3]: first argument (tokenizer name)
          172  +*/
          173  +static int fts5tokConnectMethod(
          174  +  sqlite3 *db,                    /* Database connection */
          175  +  void *pCtx,                     /* Pointer to fts5_api object */
          176  +  int argc,                       /* Number of elements in argv array */
          177  +  const char * const *argv,       /* xCreate/xConnect argument array */
          178  +  sqlite3_vtab **ppVtab,          /* OUT: New sqlite3_vtab object */
          179  +  char **pzErr                    /* OUT: sqlite3_malloc'd error message */
          180  +){
          181  +  fts5_api *pApi = (fts5_api*)pCtx;
          182  +  Fts5tokTable *pTab = 0;
          183  +  int rc;
          184  +  char **azDequote = 0;
          185  +  int nDequote;
          186  +
          187  +  rc = sqlite3_declare_vtab(db, 
          188  +       "CREATE TABLE x(input HIDDEN, token, start, end, position)"
          189  +  );
          190  +
          191  +  if( rc==SQLITE_OK ){
          192  +    nDequote = argc-3;
          193  +    rc = fts5tokDequoteArray(nDequote, &argv[3], &azDequote);
          194  +  }
          195  +
          196  +  if( rc==SQLITE_OK ){
          197  +    pTab = (Fts5tokTable*)sqlite3_malloc(sizeof(Fts5tokTable));
          198  +    if( pTab==0 ){
          199  +      rc = SQLITE_NOMEM;
          200  +    }else{
          201  +      memset(pTab, 0, sizeof(Fts5tokTable));
          202  +    }
          203  +  }
          204  +
          205  +  if( rc==SQLITE_OK ){
          206  +    void *pTokCtx = 0;
          207  +    const char *zModule = 0;
          208  +    if( nDequote>0 ){
          209  +      zModule = azDequote[0];
          210  +    }
          211  +
          212  +    rc = pApi->xFindTokenizer(pApi, zModule, &pTokCtx, &pTab->tok);
          213  +    if( rc==SQLITE_OK ){
          214  +      const char **azArg = (const char **)&azDequote[1];
          215  +      int nArg = nDequote>0 ? nDequote-1 : 0;
          216  +      rc = pTab->tok.xCreate(pTokCtx, azArg, nArg, &pTab->pTok);
          217  +    }
          218  +  }
          219  +
          220  +  if( rc!=SQLITE_OK ){
          221  +    sqlite3_free(pTab);
          222  +    pTab = 0;
          223  +  }
          224  +
          225  +  *ppVtab = (sqlite3_vtab*)pTab;
          226  +  sqlite3_free(azDequote);
          227  +  return rc;
          228  +}
          229  +
          230  +/*
          231  +** This function does the work for both the xDisconnect and xDestroy methods.
          232  +** These tables have no persistent representation of their own, so xDisconnect
          233  +** and xDestroy are identical operations.
          234  +*/
          235  +static int fts5tokDisconnectMethod(sqlite3_vtab *pVtab){
          236  +  Fts5tokTable *pTab = (Fts5tokTable *)pVtab;
          237  +  if( pTab->pTok ){
          238  +    pTab->tok.xDelete(pTab->pTok);
          239  +  }
          240  +  sqlite3_free(pTab);
          241  +  return SQLITE_OK;
          242  +}
          243  +
          244  +/*
          245  +** xBestIndex - Analyze a WHERE and ORDER BY clause.
          246  +*/
          247  +static int fts5tokBestIndexMethod(
          248  +  sqlite3_vtab *pVTab, 
          249  +  sqlite3_index_info *pInfo
          250  +){
          251  +  int i;
          252  +
          253  +  for(i=0; i<pInfo->nConstraint; i++){
          254  +    if( pInfo->aConstraint[i].usable 
          255  +     && pInfo->aConstraint[i].iColumn==0 
          256  +     && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ 
          257  +    ){
          258  +      pInfo->idxNum = 1;
          259  +      pInfo->aConstraintUsage[i].argvIndex = 1;
          260  +      pInfo->aConstraintUsage[i].omit = 1;
          261  +      pInfo->estimatedCost = 1;
          262  +      return SQLITE_OK;
          263  +    }
          264  +  }
          265  +
          266  +  pInfo->idxNum = 0;
          267  +  assert( pInfo->estimatedCost>1000000.0 );
          268  +
          269  +  return SQLITE_OK;
          270  +}
          271  +
          272  +/*
          273  +** xOpen - Open a cursor.
          274  +*/
          275  +static int fts5tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
          276  +  Fts5tokCursor *pCsr;
          277  +
          278  +  pCsr = (Fts5tokCursor *)sqlite3_malloc(sizeof(Fts5tokCursor));
          279  +  if( pCsr==0 ){
          280  +    return SQLITE_NOMEM;
          281  +  }
          282  +  memset(pCsr, 0, sizeof(Fts5tokCursor));
          283  +
          284  +  *ppCsr = (sqlite3_vtab_cursor *)pCsr;
          285  +  return SQLITE_OK;
          286  +}
          287  +
          288  +/*
          289  +** Reset the tokenizer cursor passed as the only argument. As if it had
          290  +** just been returned by fts5tokOpenMethod().
          291  +*/
          292  +static void fts5tokResetCursor(Fts5tokCursor *pCsr){
          293  +  int i;
          294  +  for(i=0; i<pCsr->nRow; i++){
          295  +    sqlite3_free(pCsr->aRow[i].zToken);
          296  +  }
          297  +  sqlite3_free(pCsr->zInput);
          298  +  sqlite3_free(pCsr->aRow);
          299  +  pCsr->zInput = 0;
          300  +  pCsr->aRow = 0;
          301  +  pCsr->nRow = 0;
          302  +  pCsr->iRowid = 0;
          303  +}
          304  +
          305  +/*
          306  +** xClose - Close a cursor.
          307  +*/
          308  +static int fts5tokCloseMethod(sqlite3_vtab_cursor *pCursor){
          309  +  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
          310  +  fts5tokResetCursor(pCsr);
          311  +  sqlite3_free(pCsr);
          312  +  return SQLITE_OK;
          313  +}
          314  +
          315  +/*
          316  +** xNext - Advance the cursor to the next row, if any.
          317  +*/
          318  +static int fts5tokNextMethod(sqlite3_vtab_cursor *pCursor){
          319  +  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
          320  +  pCsr->iRowid++;
          321  +  return SQLITE_OK;
          322  +}
          323  +
          324  +static int fts5tokCb(
          325  +  void *pCtx,         /* Pointer to Fts5tokCursor */
          326  +  int tflags,         /* Mask of FTS5_TOKEN_* flags */
          327  +  const char *pToken, /* Pointer to buffer containing token */
          328  +  int nToken,         /* Size of token in bytes */
          329  +  int iStart,         /* Byte offset of token within input text */
          330  +  int iEnd            /* Byte offset of end of token within input text */
          331  +){
          332  +  Fts5tokCursor *pCsr = (Fts5tokCursor*)pCtx;
          333  +  Fts5tokRow *pRow;
          334  +
          335  +  if( (pCsr->nRow & (pCsr->nRow-1))==0 ){
          336  +    int nNew = pCsr->nRow ? pCsr->nRow*2 : 32;
          337  +    Fts5tokRow *aNew;
          338  +    aNew = (Fts5tokRow*)sqlite3_realloc(pCsr->aRow, nNew*sizeof(Fts5tokRow));
          339  +    if( aNew==0 ) return SQLITE_NOMEM;
          340  +    memset(&aNew[pCsr->nRow], 0, sizeof(Fts5tokRow)*(nNew-pCsr->nRow));
          341  +    pCsr->aRow = aNew;
          342  +  }
          343  +
          344  +  pRow = &pCsr->aRow[pCsr->nRow];
          345  +  pRow->iStart = iStart;
          346  +  pRow->iEnd = iEnd;
          347  +  if( pCsr->nRow ){
          348  +    pRow->iPos = pRow[-1].iPos + ((tflags & FTS5_TOKEN_COLOCATED) ? 0 : 1);
          349  +  }
          350  +  pRow->zToken = sqlite3_malloc(nToken+1);
          351  +  if( pRow->zToken==0 ) return SQLITE_NOMEM;
          352  +  memcpy(pRow->zToken, pToken, nToken);
          353  +  pRow->zToken[nToken] = 0;
          354  +  pCsr->nRow++;
          355  +
          356  +  return SQLITE_OK;
          357  +}
          358  +
          359  +/*
          360  +** xFilter - Initialize a cursor to point at the start of its data.
          361  +*/
          362  +static int fts5tokFilterMethod(
          363  +  sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
          364  +  int idxNum,                     /* Strategy index */
          365  +  const char *idxStr,             /* Unused */
          366  +  int nVal,                       /* Number of elements in apVal */
          367  +  sqlite3_value **apVal           /* Arguments for the indexing scheme */
          368  +){
          369  +  int rc = SQLITE_ERROR;
          370  +  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
          371  +  Fts5tokTable *pTab = (Fts5tokTable *)(pCursor->pVtab);
          372  +
          373  +  fts5tokResetCursor(pCsr);
          374  +  if( idxNum==1 ){
          375  +    const char *zByte = (const char *)sqlite3_value_text(apVal[0]);
          376  +    int nByte = sqlite3_value_bytes(apVal[0]);
          377  +    pCsr->zInput = sqlite3_malloc(nByte+1);
          378  +    if( pCsr->zInput==0 ){
          379  +      rc = SQLITE_NOMEM;
          380  +    }else{
          381  +      memcpy(pCsr->zInput, zByte, nByte);
          382  +      pCsr->zInput[nByte] = 0;
          383  +      rc = pTab->tok.xTokenize(
          384  +          pTab->pTok, (void*)pCsr, 0, zByte, nByte, fts5tokCb
          385  +      );
          386  +    }
          387  +  }
          388  +
          389  +  if( rc!=SQLITE_OK ) return rc;
          390  +  return fts5tokNextMethod(pCursor);
          391  +}
          392  +
          393  +/*
          394  +** xEof - Return true if the cursor is at EOF, or false otherwise.
          395  +*/
          396  +static int fts5tokEofMethod(sqlite3_vtab_cursor *pCursor){
          397  +  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
          398  +  return (pCsr->iRowid>pCsr->nRow);
          399  +}
          400  +
          401  +/*
          402  +** xColumn - Return a column value.
          403  +*/
          404  +static int fts5tokColumnMethod(
          405  +  sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
          406  +  sqlite3_context *pCtx,          /* Context for sqlite3_result_xxx() calls */
          407  +  int iCol                        /* Index of column to read value from */
          408  +){
          409  +  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
          410  +  Fts5tokRow *pRow = &pCsr->aRow[pCsr->iRowid-1];
          411  +
          412  +  /* CREATE TABLE x(input, token, start, end, position) */
          413  +  switch( iCol ){
          414  +    case 0:
          415  +      sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT);
          416  +      break;
          417  +    case 1:
          418  +      sqlite3_result_text(pCtx, pRow->zToken, -1, SQLITE_TRANSIENT);
          419  +      break;
          420  +    case 2:
          421  +      sqlite3_result_int(pCtx, pRow->iStart);
          422  +      break;
          423  +    case 3:
          424  +      sqlite3_result_int(pCtx, pRow->iEnd);
          425  +      break;
          426  +    default:
          427  +      assert( iCol==4 );
          428  +      sqlite3_result_int(pCtx, pRow->iPos);
          429  +      break;
          430  +  }
          431  +  return SQLITE_OK;
          432  +}
          433  +
          434  +/*
          435  +** xRowid - Return the current rowid for the cursor.
          436  +*/
          437  +static int fts5tokRowidMethod(
          438  +  sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
          439  +  sqlite_int64 *pRowid            /* OUT: Rowid value */
          440  +){
          441  +  Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
          442  +  *pRowid = (sqlite3_int64)pCsr->iRowid;
          443  +  return SQLITE_OK;
          444  +}
          445  +
          446  +/*
          447  +** Register the fts5tok module with database connection db. Return SQLITE_OK
          448  +** if successful or an error code if sqlite3_create_module() fails.
          449  +*/
          450  +int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){
          451  +  static const sqlite3_module fts5tok_module = {
          452  +     0,                           /* iVersion      */
          453  +     fts5tokConnectMethod,        /* xCreate       */
          454  +     fts5tokConnectMethod,        /* xConnect      */
          455  +     fts5tokBestIndexMethod,      /* xBestIndex    */
          456  +     fts5tokDisconnectMethod,     /* xDisconnect   */
          457  +     fts5tokDisconnectMethod,     /* xDestroy      */
          458  +     fts5tokOpenMethod,           /* xOpen         */
          459  +     fts5tokCloseMethod,          /* xClose        */
          460  +     fts5tokFilterMethod,         /* xFilter       */
          461  +     fts5tokNextMethod,           /* xNext         */
          462  +     fts5tokEofMethod,            /* xEof          */
          463  +     fts5tokColumnMethod,         /* xColumn       */
          464  +     fts5tokRowidMethod,          /* xRowid        */
          465  +     0,                           /* xUpdate       */
          466  +     0,                           /* xBegin        */
          467  +     0,                           /* xSync         */
          468  +     0,                           /* xCommit       */
          469  +     0,                           /* xRollback     */
          470  +     0,                           /* xFindFunction */
          471  +     0,                           /* xRename       */
          472  +     0,                           /* xSavepoint    */
          473  +     0,                           /* xRelease      */
          474  +     0                            /* xRollbackTo   */
          475  +  };
          476  +  int rc;                         /* Return code */
          477  +
          478  +  rc = sqlite3_create_module(db, "fts5tokenize", &fts5tok_module, (void*)pApi);
          479  +  return rc;
          480  +}
          481  +
          482  +#endif /* defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5) */

Changes to ext/fts5/fts5_vocab.c.

   423    423                   pCsr->aDoc[0]++;
   424    424                 }else{
   425    425                   int iCol = -1;
   426    426                   while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
   427    427                     int ii = FTS5_POS2COLUMN(iPos);
   428    428                     pCsr->aCnt[ii]++;
   429    429                     if( iCol!=ii ){
          430  +                    if( ii>=nCol ){
          431  +                      rc = FTS5_CORRUPT;
          432  +                      break;
          433  +                    }
   430    434                       pCsr->aDoc[ii]++;
   431    435                       iCol = ii;
   432    436                     }
   433    437                   }
   434    438                 }
   435    439               }
   436    440               break;
................................................................................
   440    444                 pCsr->aDoc[0]++;
   441    445               }else{
   442    446                 Fts5Buffer buf = {0, 0, 0};
   443    447                 rc = sqlite3Fts5IterPoslistBuffer(pCsr->pIter, &buf);
   444    448                 if( rc==SQLITE_OK ){
   445    449                   while( 0==sqlite3Fts5PoslistNext64(buf.p, buf.n, &iOff,&iPos) ){
   446    450                     assert_nc( iPos>=0 && iPos<nCol );
   447         -                  if( iPos<nCol ) pCsr->aDoc[iPos]++;
          451  +                  if( iPos>=nCol ){
          452  +                    rc = FTS5_CORRUPT;
          453  +                    break;
          454  +                  }
          455  +                  pCsr->aDoc[iPos]++;
   448    456                   }
   449    457                 }
   450    458                 sqlite3Fts5BufferFree(&buf);
   451    459               }
   452    460               break;
   453    461   
   454    462             default: 
................................................................................
   468    476             }
   469    477             if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
   470    478           }
   471    479         }
   472    480       }
   473    481     }
   474    482   
   475         -  if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
          483  +  if( rc==SQLITE_OK && pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
   476    484       while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++;
   477    485       assert( pCsr->iCol<pCsr->pConfig->nCol );
   478    486     }
   479    487     return rc;
   480    488   }
   481    489   
   482    490   /*

Changes to ext/fts5/test/fts5_common.tcl.

    10     10   #***********************************************************************
    11     11   #
    12     12   
    13     13   if {![info exists testdir]} {
    14     14     set testdir [file join [file dirname [info script]] .. .. .. test]
    15     15   }
    16     16   source $testdir/tester.tcl
           17  +
           18  +ifcapable !fts5 {
           19  +  finish_test
           20  +  return
           21  +}
    17     22   
    18     23   catch { 
    19     24     sqlite3_fts5_may_be_corrupt 0 
    20     25     reset_db
    21     26   }
           27  +
           28  +# If SQLITE_ENABLE_FTS5 is not defined, skip this test.
           29  +ifcapable !fts5 {
           30  +  finish_test
           31  +  return
           32  +}
    22     33   
    23     34   proc fts5_test_poslist {cmd} {
    24     35     set res [list]
    25     36     for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
    26     37       lappend res [string map {{ } .} [$cmd xInst $i]]
    27     38     }
    28     39     set res
................................................................................
   505    516     }
   506    517   
   507    518     foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] {
   508    519       lappend res $rowid $poslist
   509    520     }
   510    521     set res
   511    522   }
          523  +
          524  +proc fts5_collist_data {expr tbl {order ASC} {aDictVar ""}} {
          525  +  set res [list]
          526  +
          527  +  if {$aDictVar!=""} {
          528  +    upvar $aDictVar aDict
          529  +    set dict aDict
          530  +  } else {
          531  +    set dict ""
          532  +  }
          533  +
          534  +  foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] {
          535  +    lappend res $rowid $collist
          536  +  }
          537  +  set res
          538  +}
   512    539   
   513    540   #-------------------------------------------------------------------------
   514    541   #
   515    542   
   516    543   # This command will only work inside a [foreach_detail_mode] block. It tests
   517    544   # whether or not expression $expr run on FTS5 table $tbl is supported by
   518    545   # the current mode. If so, 1 is returned. If not, 0.
................................................................................
   557    584     nearset_rf $aCol {*}$args
   558    585     if {[lsearch $args -col]>=0} { 
   559    586       set ::expr_not_ok 1
   560    587     }
   561    588     list
   562    589   }
   563    590   
          591  +
          592  +#-------------------------------------------------------------------------
          593  +# Code for a simple Tcl tokenizer that supports synonyms at query time.
          594  +#
          595  +proc tclnum_tokenize {mode tflags text} {
          596  +  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
          597  +    sqlite3_fts5_token $w $iStart $iEnd
          598  +    if {$tflags == $mode && [info exists ::tclnum_syn($w)]} {
          599  +      foreach s $::tclnum_syn($w)  { sqlite3_fts5_token -colo $s $iStart $iEnd }
          600  +    }
          601  +  }
          602  +}
          603  +
          604  +proc tclnum_create {args} {
          605  +  set mode query
          606  +  if {[llength $args]} {
          607  +    set mode [lindex $args 0]
          608  +  }
          609  +  if {$mode != "query" && $mode != "document"} { error "bad mode: $mode" }
          610  +  return [list tclnum_tokenize $mode]
          611  +}
          612  +
          613  +proc fts5_tclnum_register {db} {
          614  +  foreach SYNDICT {
          615  +    {zero  0}
          616  +    {one   1 i}
          617  +    {two   2 ii}
          618  +    {three 3 iii}
          619  +    {four  4 iv}
          620  +    {five  5 v}
          621  +    {six   6 vi}
          622  +    {seven 7 vii}
          623  +    {eight 8 viii}
          624  +    {nine  9 ix}
          625  +
          626  +    {a1 a2 a3 a4 a5 a6 a7 a8 a9}
          627  +    {b1 b2 b3 b4 b5 b6 b7 b8 b9}
          628  +    {c1 c2 c3 c4 c5 c6 c7 c8 c9}
          629  +  } {
          630  +    foreach s $SYNDICT {
          631  +      set o [list]
          632  +      foreach x $SYNDICT {if {$x!=$s} {lappend o $x}}
          633  +      set ::tclnum_syn($s) $o
          634  +    }
          635  +  }
          636  +  sqlite3_fts5_create_tokenizer db tclnum tclnum_create
          637  +}
          638  +#
          639  +# End of tokenizer code.
          640  +#-------------------------------------------------------------------------
          641  +

Added ext/fts5/test/fts5bigtok.test.

            1  +# 2016 Jan 19
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5bigtok
           17  +
           18  +proc rndterm {} {
           19  +  set L [list a b c d e f g h i j k l m n o p q r s t u v w x y z]
           20  +  set l [lindex $L [expr int(rand() * [llength $L])]]
           21  +  string repeat $l [expr int(rand() * 5) + 60]
           22  +}
           23  +
           24  +proc rnddoc {n} {
           25  +  set res [list]
           26  +  for {set i 0} {$i < $n} {incr i} {
           27  +    lappend res [rndterm]
           28  +  }
           29  +  set res
           30  +}
           31  +
           32  +foreach_detail_mode $::testprefix {
           33  +  db func rnddoc rnddoc
           34  +  do_execsql_test 1.0 {
           35  +    CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
           36  +    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
           37  +    CREATE VIRTUAL TABLE t1vocab USING fts5vocab(t1, row);
           38  +
           39  +    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 )
           40  +    INSERT INTO t1 SELECT rnddoc(3) FROM s;
           41  +
           42  +    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 )
           43  +    INSERT INTO t1 SELECT rnddoc(3) FROM s;
           44  +  }
           45  +
           46  +  foreach v [db eval {SELECT term FROM t1vocab}] {
           47  +    set res [db eval {SELECT rowid FROM t1($v)}]
           48  +    do_execsql_test 1.[string range $v 0 0] {
           49  +      SELECT rowid FROM t1($v) ORDER BY rowid DESC
           50  +    } [lsort -integer -decr $res]
           51  +  }
           52  +
           53  +  do_execsql_test 2.0 {
           54  +    INSERT INTO t1(t1) VALUES('optimize');
           55  +  }
           56  +
           57  +  foreach v [db eval {SELECT term FROM t1vocab}] {
           58  +    set res [db eval {SELECT rowid FROM t1($v)}]
           59  +    do_execsql_test 2.[string range $v 0 0] {
           60  +      SELECT rowid FROM t1($v) ORDER BY rowid DESC
           61  +    } [lsort -integer -decr $res]
           62  +  }
           63  +}
           64  +
           65  +finish_test
           66  +
           67  +

Changes to ext/fts5/test/fts5config.test.

    40     40   #
    41     41   foreach {tn opt} {
    42     42     1 {prefix=x}  
    43     43     2 {prefix='x'}
    44     44     3 {prefix='$'}
    45     45     4 {prefix='1,2,'}
    46     46     5 {prefix=',1'}
           47  +  6 {prefix='1,2,3...'}
           48  +  7 {prefix='1,2,3xyz'}
    47     49   } {
    48     50     set res [list 1 {malformed prefix=... directive}]
    49     51     do_catchsql_test 2.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
    50     52   }
    51     53   
    52     54   #-------------------------------------------------------------------------
    53     55   # Syntax errors in the 'rank' option.
................................................................................
   155    157   
   156    158   #-------------------------------------------------------------------------
   157    159   # Errors in:
   158    160   #
   159    161   #   9.1.* 'pgsz' options.
   160    162   #   9.2.* 'automerge' options.
   161    163   #   9.3.* 'crisismerge' options.
          164  +#   9.4.* a non-existant option.
          165  +#   9.5.* 'hashsize' options.
   162    166   #
   163    167   do_execsql_test 9.0 {
   164    168     CREATE VIRTUAL TABLE abc USING fts5(a, b);
   165    169   } {}
   166    170   do_catchsql_test 9.1.1 {
   167    171     INSERT INTO abc(abc, rank) VALUES('pgsz', -5);
   168    172   } {1 {SQL logic error or missing database}}
................................................................................
   198    202   do_execsql_test 9.3.4 {
   199    203     INSERT INTO abc(abc, rank) VALUES('crisismerge', 50000000);
   200    204   } {}
   201    205   
   202    206   do_catchsql_test 9.4.1 {
   203    207     INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1);
   204    208   } {1 {SQL logic error or missing database}}
          209  +
          210  +do_catchsql_test 9.5.1 {
          211  +  INSERT INTO abc(abc, rank) VALUES('hashsize', 'not an integer');
          212  +} {1 {SQL logic error or missing database}}
          213  +do_catchsql_test 9.5.2 {
          214  +  INSERT INTO abc(abc, rank) VALUES('hashsize', -500000);
          215  +} {1 {SQL logic error or missing database}}
          216  +do_catchsql_test 9.5.3 {
          217  +  INSERT INTO abc(abc, rank) VALUES('hashsize', 500000);
          218  +} {0 {}}
   205    219   
   206    220   #-------------------------------------------------------------------------
   207    221   # Too many prefix indexes. Maximum allowed is 31.
   208    222   #
   209    223   foreach {tn spec} {
   210    224     1 {prefix="1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32"}
   211    225     2 {prefix="1 2 3 4", prefix="5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32"}
   212    226   } {
   213    227     set sql "CREATE VIRTUAL TABLE xyz USING fts5(x, $spec)"
   214    228     do_catchsql_test 10.$tn $sql {1 {too many prefix indexes (max 31)}}
   215    229   }
          230  +
          231  +#-------------------------------------------------------------------------
          232  +# errors in the detail= option.
          233  +#
          234  +foreach {tn opt} {
          235  +  1 {detail=x}  
          236  +  2 {detail='x'}
          237  +  3 {detail='$'}
          238  +  4 {detail='1,2,'}
          239  +  5 {detail=',1'}
          240  +  6 {detail=''}
          241  +} {
          242  +  set res [list 1 {malformed detail=... directive}]
          243  +  do_catchsql_test 11.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
          244  +}
   216    245   
   217    246   finish_test
   218    247   

Changes to ext/fts5/test/fts5fault4.test.

    36     36     execsql { SELECT * FROM xx }
    37     37   } -body {
    38     38     execsql { DROP TABLE xx }
    39     39   } -test {
    40     40     faultsim_test_result [list 0 {}]
    41     41   }
    42     42   
    43         -#-------------------------------------------------------------------------
    44         -# An OOM within an "ORDER BY rank" query.
    45         -#
    46         -db func rnddoc fts5_rnddoc 
    47         -do_execsql_test 2.0 {
    48         -  CREATE VIRTUAL TABLE xx USING fts5(x);
    49         -  INSERT INTO xx VALUES ('abc ' || rnddoc(10));
    50         -  INSERT INTO xx VALUES ('abc abc' || rnddoc(9));
    51         -  INSERT INTO xx VALUES ('abc abc abc' || rnddoc(8));
    52         -} {}
    53         -faultsim_save_and_close
    54         -
    55         -do_faultsim_test 2 -faults oom-* -prep {
    56         -  faultsim_restore_and_reopen
    57         -  execsql { SELECT * FROM xx }
    58         -} -body {
    59         -  execsql { SELECT rowid FROM xx WHERE xx MATCH 'abc' ORDER BY rank }
    60         -} -test {
    61         -  faultsim_test_result [list 0 {3 2 1}]
    62         -}
    63         -
    64     43   #-------------------------------------------------------------------------
    65     44   # An OOM while "reseeking" an FTS cursor.
    66     45   #
    67     46   do_execsql_test 3.0 {
    68     47     CREATE VIRTUAL TABLE jj USING fts5(j);
    69     48     INSERT INTO jj(rowid, j) VALUES(101, 'm t w t f s s');
    70     49     INSERT INTO jj(rowid, j) VALUES(202, 't w t f s');

Changes to ext/fts5/test/fts5fault5.test.

    61     61   do_faultsim_test 2.2 -faults oom-t* -body {
    62     62     db eval { INSERT INTO tt(tt) VALUES('integrity-check') }
    63     63   } -test {
    64     64     faultsim_test_result {0 {}}
    65     65   }
    66     66   
    67     67   #-------------------------------------------------------------------------
    68         -# OOM while scanning an fts5vocab table.
           68  +# OOM while scanning fts5vocab tables.
    69     69   #
    70     70   reset_db
    71     71   do_test 3.0 {
    72     72     execsql {
    73     73       CREATE VIRTUAL TABLE tt USING fts5(x);
    74     74       CREATE VIRTUAL TABLE tv USING fts5vocab(tt, 'row');
           75  +
           76  +    CREATE VIRTUAL TABLE tt2 USING fts5(x, detail=col);
           77  +    CREATE VIRTUAL TABLE tv2 USING fts5vocab(tt2, 'col');
           78  +
    75     79       INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
           80  +    INSERT INTO tt2(tt2, rank) VALUES('pgsz', 32);
    76     81       BEGIN;
    77     82     }
           83  +
    78     84     for {set i 0} {$i < 20} {incr i} {
    79     85       set str [string repeat "$i " 50]
    80     86       execsql { INSERT INTO tt VALUES($str) }
           87  +    execsql { INSERT INTO tt2 VALUES($str) }
    81     88     }
    82     89     execsql COMMIT
    83     90   } {}
    84     91   
    85     92   do_faultsim_test 3.1 -faults oom-t* -body {
    86     93     db eval {
    87     94       SELECT term FROM tv;
................................................................................
    93    100   do_faultsim_test 3.2 -faults oom-t* -body {
    94    101     db eval {
    95    102       SELECT term FROM tv WHERE term BETWEEN '1' AND '2';
    96    103     }
    97    104   } -test {
    98    105     faultsim_test_result {0 {1 10 11 12 13 14 15 16 17 18 19 2}}
    99    106   }
          107  +
          108  +breakpoint
          109  +do_execsql_test 3.3.0 {
          110  +  SELECT * FROM tv2;
          111  +} {
          112  +  0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {}        
          113  +  14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19  x 1 {}     
          114  +  2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {}   
          115  +  9 x 1 {}
          116  +}
          117  +do_faultsim_test 3.3 -faults oom-t* -body {
          118  +  db eval {
          119  +    SELECT * FROM tv2;
          120  +  }
          121  +} -test {
          122  +  faultsim_test_result [list 0 [list                                   \
          123  +      0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {}        \
          124  +      14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19  x 1 {}     \
          125  +      2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {}   \
          126  +      9 x 1 {}
          127  +  ]]
          128  +}
   100    129   
   101    130   
   102    131   
   103    132   finish_test
   104    133   

Changes to ext/fts5/test/fts5fault8.test.

    20     20   ifcapable !fts5 {
    21     21     finish_test
    22     22     return
    23     23   }
    24     24   
    25     25   foreach_detail_mode $testprefix {
    26     26   
    27         -if {[detail_is_none]==0} continue
    28         -
    29     27   fts5_aux_test_functions db
    30     28   do_execsql_test 1.0 {
    31     29     CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
    32     30     INSERT INTO t1 VALUES('a b c d', '1 2 3 4');
    33     31     INSERT INTO t1 VALUES('a b a b', NULL);
    34     32     INSERT INTO t1 VALUES(NULL, '1 2 1 2');
    35     33   }
    36     34   
    37         -do_faultsim_test 1 -faults oom-t* -body {
           35  +do_faultsim_test 1 -faults oom-* -body {
    38     36     execsql { 
    39     37       SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2' 
    40     38     }
    41     39   } -test {
    42     40     faultsim_test_result {0 {1 {0.0.1 1.1.1} 2 {0.0.1 0.0.3} 3 {1.1.1 1.1.3}}} \
    43     41                          {1 SQLITE_NOMEM}
    44     42   }
           43  +
           44  +do_faultsim_test 2 -faults oom-* -body {
           45  +  execsql { 
           46  +    INSERT INTO t1(t1) VALUES('integrity-check');
           47  +  }
           48  +} -test {
           49  +  faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
           50  +}
    45     51   
    46     52   }
    47     53   
    48     54   finish_test
    49     55   

Added ext/fts5/test/fts5fault9.test.

            1  +# 2015 September 3
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +#
           12  +# This file is focused on OOM errors.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +source $testdir/malloc_common.tcl
           17  +set testprefix fts5fault9
           18  +
           19  +# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           20  +ifcapable !fts5 {
           21  +  finish_test
           22  +  return
           23  +}
           24  +
           25  +foreach_detail_mode $testprefix {
           26  +
           27  +fts5_aux_test_functions db
           28  +
           29  +do_execsql_test 1.0 {
           30  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
           31  +  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
           32  +  WITH seq(s) AS ( SELECT 1 UNION ALL SELECT s+1 FROM seq WHERE s<50)
           33  +  INSERT INTO t1 SELECT 'x x x y y y', 'a b c d e f' FROM seq;
           34  +}
           35  +
           36  +do_faultsim_test 1 -faults oom-* -body {
           37  +  execsql { SELECT count(*) FROM t1('x AND y') }
           38  +} -test {
           39  +  faultsim_test_result {0 50}
           40  +}
           41  +
           42  +do_execsql_test 2.0 {
           43  +  CREATE VIRTUAL TABLE t2 USING fts5(a, b, detail=%DETAIL%);
           44  +  INSERT INTO t2(t2, rank) VALUES('pgsz', 32);
           45  +  INSERT INTO t2 VALUES('abc cba', 'cba abc');
           46  +  INSERT INTO t2 VALUES('abc cba', 'cba abc');
           47  +  INSERT INTO t2 VALUES('abc cba', 'cba abc');
           48  +
           49  +  INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
           50  +  INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
           51  +  INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
           52  +}
           53  +
           54  +do_faultsim_test 2 -faults oom-* -body {
           55  +  execsql { SELECT count(*) FROM t2('a* AND c*') }
           56  +} -test {
           57  +  faultsim_test_result {0 6}
           58  +}
           59  +
           60  +
           61  +do_execsql_test 3.0 {
           62  +  CREATE VIRTUAL TABLE t3 USING fts5(a, detail=%DETAIL%);
           63  +  INSERT INTO t3 VALUES('a x x a x a a a');
           64  +  INSERT INTO t3 VALUES('x a a x a x x x');
           65  +}
           66  +
           67  +do_faultsim_test 3.1 -faults oom-* -body {
           68  +  execsql { SELECT highlight(t3, 0, '[', ']') FROM t3('a') }
           69  +} -test {
           70  +  faultsim_test_result {0 {{[a] x x [a] x [a] [a] [a]} {x [a] [a] x [a] x x x}}}
           71  +}
           72  +
           73  +do_faultsim_test 3.2 -faults oom-t* -body {
           74  +  execsql { SELECT fts5_test_poslist2(t3) FROM t3('x') }
           75  +} -test {
           76  +  faultsim_test_result \
           77  +      {0 {{0.0.1 0.0.2 0.0.4} {0.0.0 0.0.3 0.0.5 0.0.6 0.0.7}}} \
           78  +      {1 SQLITE_NOMEM}
           79  +}
           80  +
           81  +#-------------------------------------------------------------------------
           82  +# Test OOM injection with the xPhraseFirstColumn() API and a tokenizer
           83  +# uses query synonyms.
           84  +#
           85  +fts5_tclnum_register db
           86  +do_execsql_test 4.0 {
           87  +  CREATE VIRTUAL TABLE t4 USING fts5(x, y, z, detail=%DETAIL%, tokenize=tclnum);
           88  +  INSERT INTO t4 VALUES('one two three', '1 2 3', 'i ii iii');
           89  +  INSERT INTO t4 VALUES('1 2 3', 'i ii iii', 'one two three');
           90  +  INSERT INTO t4 VALUES('i ii iii', 'one two three', 'i ii iii');
           91  +
           92  +  INSERT INTO t4 VALUES('a1 a2 a3', 'a4 a5 a6', 'a7 a8 a9');
           93  +  INSERT INTO t4 VALUES('b1 b2 b3', 'b4 b5 b6', 'b7 b8 b9');
           94  +  INSERT INTO t4 VALUES('c1 c2 c3', 'c4 c5 c6', 'c7 c8 c9');
           95  +}
           96  +
           97  +do_faultsim_test 4.1 -faults oom-t* -body {
           98  +  execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('2') }
           99  +} -test {
          100  +  faultsim_test_result \
          101  +      {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} {1 SQLITE_NOMEM}
          102  +}
          103  +
          104  +do_faultsim_test 4.2 -faults oom-t* -body {
          105  +  execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('a5 OR b5 OR c5') }
          106  +} -test {
          107  +  faultsim_test_result \
          108  +      {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} {1 SQLITE_NOMEM}
          109  +}
          110  +
          111  +
          112  +#-------------------------------------------------------------------------
          113  +# An OOM within an "ORDER BY rank" query.
          114  +#
          115  +db func rnddoc fts5_rnddoc 
          116  +do_execsql_test 5.0 {
          117  +  CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=%DETAIL%);
          118  +  INSERT INTO xx VALUES ('def', 'abc ' || rnddoc(10));
          119  +  INSERT INTO xx VALUES ('def', 'abc abc' || rnddoc(9));
          120  +  INSERT INTO xx VALUES ('def', 'abc abc abc' || rnddoc(8));
          121  +} {}
          122  +faultsim_save_and_close
          123  +
          124  +do_faultsim_test 5 -faults oom-* -prep {
          125  +  faultsim_restore_and_reopen
          126  +  execsql { SELECT * FROM xx }
          127  +} -body {
          128  +  execsql { SELECT rowid FROM xx('abc AND def') ORDER BY rank }
          129  +} -test {
          130  +  faultsim_test_result [list 0 {3 2 1}]
          131  +}
          132  +
          133  +set doc [string repeat "xyz " 500]
          134  +do_execsql_test 6.0 {
          135  +  CREATE VIRTUAL TABLE yy USING fts5(y, detail=%DETAIL%);
          136  +  INSERT INTO yy(yy, rank) VALUES('pgsz', 64);
          137  +  INSERT INTO yy VALUES ($doc);
          138  +  INSERT INTO yy VALUES ('1 2 3');
          139  +  INSERT INTO yy VALUES ('xyz');
          140  +  UPDATE yy SET y = y WHERE rowid = 1;
          141  +  UPDATE yy SET y = y WHERE rowid = 1;
          142  +  UPDATE yy SET y = y WHERE rowid = 1;
          143  +  UPDATE yy SET y = y WHERE rowid = 1;
          144  +} {}
          145  +
          146  +do_faultsim_test 6 -faults oom-* -body {
          147  +  execsql { SELECT rowid FROM yy('xyz') }
          148  +} -test {
          149  +  faultsim_test_result [list 0 {1 3}]
          150  +}
          151  +
          152  +
          153  +} ;# foreach_detail_mode...
          154  +
          155  +finish_test
          156  +

Changes to ext/fts5/test/fts5hash.test.

    60     60     for {set i 0} {$i<$nWord} {incr i} {
    61     61       set j [expr {int(rand() * $nVocab)}]
    62     62       lappend doc [lindex $vocab $j]
    63     63     }
    64     64     return $doc
    65     65   }
    66     66   
           67  +foreach_detail_mode $testprefix {
           68  +
    67     69   set vocab [build_vocab1]
    68     70   db func r random_doc 
    69     71   
    70     72   do_execsql_test 1.0 {
    71         -  CREATE VIRTUAL TABLE eee USING fts5(e, ee);
           73  +  CREATE VIRTUAL TABLE eee USING fts5(e, ee, detail=%DETAIL%);
    72     74     BEGIN;
    73     75       WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
    74     76       INSERT INTO eee SELECT r($vocab, 5), r($vocab, 7) FROM ii;
    75     77       INSERT INTO eee(eee) VALUES('integrity-check');
    76     78     COMMIT;
    77     79     INSERT INTO eee(eee) VALUES('integrity-check');
    78     80   }
................................................................................
    87     89   }
    88     90   do_test 1.2 {
    89     91     for {set i 1} {$i <= 100} {incr i} {
    90     92       execsql { INSERT INTO eee VALUES( r($vocab, 5), r($vocab, 7) ) }
    91     93     }
    92     94   } {}
    93     95     
    94         -do_test 1.2 {
           96  +do_test 1.3 {
    95     97     db eval { SELECT term, doc FROM vocab } {
    96     98       set nRow [db one {SELECT count(*) FROM eee WHERE eee MATCH $term}]
    97     99       if {$nRow != $doc} {
    98    100         error "term=$term fts5vocab=$doc cnt=$nRow"
    99    101       }
   100    102     }
   101    103     set {} {}
   102    104   } {}
   103    105   
   104         -do_execsql_test 1.3 {
          106  +do_execsql_test 1.4 {
   105    107     COMMIT;
   106    108     INSERT INTO eee(eee) VALUES('integrity-check');
   107    109   }
          110  +
          111  +} ;# foreach_detail_mode
   108    112   
   109    113   finish_test
   110    114   

Changes to ext/fts5/test/fts5integrity.test.

   141    141     INSERT INTO gg(gg) VALUES('integrity-check');
   142    142   }
   143    143   
   144    144   do_execsql_test 5.2 {
   145    145     INSERT INTO gg(gg) VALUES('optimize');
   146    146   }
   147    147   
   148         -breakpoint
   149    148   do_execsql_test 5.3 {
   150    149     INSERT INTO gg(gg) VALUES('integrity-check');
   151    150   }
          151  +
          152  +do_test 5.4.1 {
          153  +  set ok 0
          154  +  for {set i 0} {$i < 10000} {incr i} {
          155  +    set T [format %.5d $i]
          156  +    set res  [db eval { SELECT rowid FROM gg($T) ORDER BY rowid ASC  }]
          157  +    set res2 [db eval { SELECT rowid FROM gg($T) ORDER BY rowid DESC }]
          158  +    if {$res == [lsort -integer $res2]} { incr ok }
          159  +  }
          160  +  set ok
          161  +} {10000}
          162  +
          163  +do_test 5.4.2 {
          164  +  set ok 0
          165  +  for {set i 0} {$i < 100} {incr i} {
          166  +    set T "[format %.3d $i]*"
          167  +    set res  [db eval { SELECT rowid FROM gg($T) ORDER BY rowid ASC  }]
          168  +    set res2 [db eval { SELECT rowid FROM gg($T) ORDER BY rowid DESC }]
          169  +    if {$res == [lsort -integer $res2]} { incr ok }
          170  +  }
          171  +  set ok
          172  +} {100}
          173  +
          174  +#-------------------------------------------------------------------------
          175  +# Similar to 5.*.
          176  +#
          177  +foreach {tn pgsz} {
          178  +  1  32
          179  +  2  36
          180  +  3  40
          181  +  4  44
          182  +  5  48
          183  +} {
          184  +  do_execsql_test 6.$tn.1 {
          185  +    DROP TABLE IF EXISTS hh;
          186  +    CREATE VIRTUAL TABLE hh USING fts5(y);
          187  +    INSERT INTO hh(hh, rank) VALUES('pgsz', $pgsz);
          188  +
          189  +    WITH s(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<999)
          190  +     INSERT INTO hh SELECT printf("%.3d%.3d%.3d %.3d%.3d%.3d",i,i,i,i+1,i+1,i+1)
          191  +     FROM s;
          192  +
          193  +    WITH s(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<999)
          194  +     INSERT INTO hh SELECT printf("%.3d%.3d%.3d %.3d%.3d%.3d",i,i,i,i+1,i+1,i+1)
          195  +     FROM s;
          196  +
          197  +    INSERT INTO hh(hh) VALUES('optimize');
          198  +  }
          199  +
          200  +  do_test 6.$tn.2 {
          201  +    set ok 0
          202  +    for {set i 0} {$i < 1000} {incr i} {
          203  +      set T [format %.3d%.3d%.3d $i $i $i]
          204  +      set res  [db eval { SELECT rowid FROM hh($T) ORDER BY rowid ASC  }]
          205  +      set res2 [db eval { SELECT rowid FROM hh($T) ORDER BY rowid DESC }]
          206  +      if {$res == [lsort -integer $res2]} { incr ok }
          207  +    }
          208  +    set ok
          209  +  } {1000}
          210  +}
   152    211   
   153    212   finish_test
   154    213   

Added ext/fts5/test/fts5merge2.test.

            1  +# 2014 Dec 20
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# Test that focus on incremental merges of segments.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5merge
           17  +
           18  +proc dump_structure {} {
           19  +  db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} {
           20  +    foreach lvl [lrange $t 1 end] {
           21  +      set seg [string repeat . [expr [llength $lvl]-2]]
           22  +      puts "[lrange $lvl 0 1] $seg"
           23  +    }
           24  +  }
           25  +}
           26  +
           27  +foreach_detail_mode $testprefix {
           28  +
           29  +if {[detail_is_none]==0} continue
           30  +
           31  +do_execsql_test 1.0 {
           32  +  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
           33  +  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
           34  +  INSERT INTO t1(t1, rank) VALUES('crisismerge', 2);
           35  +  INSERT INTO t1 VALUES('1 2 3 4');
           36  +}
           37  +
           38  +expr srand(0)
           39  +db func rnddoc fts5_rnddoc
           40  +do_test 1.1 {
           41  +  for {set i 0} {$i < 100} {incr i} {
           42  +    execsql {
           43  +      BEGIN;
           44  +        DELETE FROM t1 WHERE rowid = 1;
           45  +        INSERT INTO t1(rowid, x) VALUES(1, '1 2 3 4');
           46  +        INSERT INTO t1 VALUES(rnddoc(10));
           47  +      COMMIT;
           48  +    }
           49  +  }
           50  +} {}
           51  +
           52  +do_execsql_test 1.2 {
           53  +  INSERT INTO t1(t1) VALUES('integrity-check');
           54  +}
           55  +
           56  +}
           57  +
           58  +finish_test
           59  +

Changes to ext/fts5/test/fts5rowid.test.

    59     59   
    60     60   do_execsql_test 2.2 {
    61     61     WITH r(a, b) AS (
    62     62       SELECT rnddoc(6), rnddoc(6) UNION ALL
    63     63       SELECT rnddoc(6), rnddoc(6) FROM r
    64     64     )
    65     65     INSERT INTO x1 SELECT * FROM r LIMIT 10000;
           66  +  DELETE FROM x1 WHERE (rowid%2);
    66     67   }
    67     68   
    68     69   set res [db one {SELECT count(*) FROM x1_data}]
    69     70   do_execsql_test 2.3 {
    70     71     SELECT count(fts5_decode(rowid, block)) FROM x1_data;
    71     72   } $res
    72     73   do_execsql_test 2.4 {
    73     74     UPDATE x1_data SET block = X'';
    74         -  -- SELECT count(fts5_decode(rowid, block)) FROM x1_data;
    75         -  SELECT count(*) FROM x1_data;
           75  +  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
    76     76   } $res
    77     77   
    78     78   do_execsql_test 2.5 {
    79     79     INSERT INTO x1(x1, rank) VALUES('pgsz', 1024);
    80     80     INSERT INTO x1(x1) VALUES('rebuild');
    81     81   }
    82     82   
................................................................................
   179    179     SELECT rowid FROM x4 WHERE x4 MATCH 'a'
   180    180   } {1 2 3 4}
   181    181   
   182    182   set res [db one {SELECT count(*) FROM x4_data}]
   183    183   do_execsql_test 5.2 {
   184    184     SELECT count(fts5_decode(rowid, block)) FROM x4_data;
   185    185   } $res
          186  +
          187  +#-------------------------------------------------------------------------
          188  +#
          189  +
          190  +do_execsql_test 6.0 {
          191  +  CREATE VIRTUAL TABLE x5 USING fts5(x, detail=none);
          192  +  INSERT INTO x5(x5, rank) VALUES('pgsz', 32);
          193  +  INSERT INTO x5 VALUES('a b c d e f');
          194  +  INSERT INTO x5 VALUES('a b c d e f');
          195  +  INSERT INTO x5 VALUES('a b c d e f');
          196  +  BEGIN;
          197  +    WITH s(i) AS (
          198  +      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
          199  +    ) INSERT INTO x5 SELECT 'a b c d e f' FROM s;
          200  +  COMMIT;
          201  +  SELECT count(fts5_decode_none(rowid, block)) FROM x5_data;
          202  +} {32}
          203  +
          204  +do_execsql_test 6.1 {
          205  +  DELETE FROM x5 WHERE rowid <= 2;
          206  +  SELECT count(fts5_decode_none(rowid, block)) FROM x5_data;
          207  +} {34}
          208  +
          209  +do_execsql_test 6.2 {
          210  +  UPDATE x5 SET x='a b c d e f' WHERE rowid=3;
          211  +  SELECT count(fts5_decode_none(rowid, block)) FROM x5_data;
          212  +} {36}
          213  +
          214  +#db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM x5_data} {puts $r}
          215  +
          216  +
   186    217   
   187    218   finish_test
   188    219   

Changes to ext/fts5/test/fts5simple2.test.

    15     15   
    16     16   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    17     17   ifcapable !fts5 {
    18     18     finish_test
    19     19     return
    20     20   }
    21     21   
    22         -if 1 {
    23         -
    24     22   do_execsql_test 1.0 {
    25     23     CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
    26     24     INSERT INTO t1 VALUES('a b c');
    27     25   }
    28     26   do_execsql_test 1.1 {
    29     27     SELECT rowid FROM t1('c a b')
    30     28   } {1}
................................................................................
   263    261     INSERT INTO t1 VALUES('a b c d');
   264    262   } {}
   265    263   
   266    264   do_execsql_test 14.1 {
   267    265     SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank;
   268    266   } {0.0.1}
   269    267   
   270         -}
   271         -
   272    268   #-------------------------------------------------------------------------
   273    269   #
   274    270   reset_db
   275    271   do_execsql_test 15.1 {
   276    272     CREATE VIRTUAL TABLE t1 USING fts5(x, detail=none);
   277    273     BEGIN;
   278    274       INSERT INTO t1(rowid, x) VALUES(1, 'sqlite');
................................................................................
   295    291   do_execsql_test 15.3.2 {
   296    292     SELECT rowid FROM t1('sqlite') ORDER BY rowid DESC;
   297    293   } {}
   298    294   
   299    295   do_test 15.4 {
   300    296     execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
   301    297   } {}
          298  +
          299  +#-------------------------------------------------------------------------
          300  +#
          301  +reset_db
          302  +do_execsql_test 16.0 {
          303  +  CREATE VIRTUAL TABLE t2 USING fts5(x, detail=none);
          304  +  BEGIN;
          305  +    INSERT INTO t2(rowid, x) VALUES(1, 'a b c');
          306  +    INSERT INTO t2(rowid, x) VALUES(456, 'a b c');
          307  +    INSERT INTO t2(rowid, x) VALUES(1000, 'a b c');
          308  +  COMMIT;
          309  +  UPDATE t2 SET x=x;
          310  +}
          311  +
          312  +do_execsql_test 16.1 {
          313  +  INSERT INTO t2(t2) VALUES('integrity-check');
          314  +} {}
          315  +
          316  +do_execsql_test 16.2 {
          317  +  SELECT rowid FROM t2('b') ORDER BY rowid DESC
          318  +} {1000 456 1}
          319  +
          320  +
          321  +#-------------------------------------------------------------------------
          322  +#
          323  +reset_db
          324  +do_execsql_test 16.0 {
          325  +  CREATE VIRTUAL TABLE t2 USING fts5(x, detail=none);
          326  +  BEGIN;
          327  +    INSERT INTO t2(rowid, x) VALUES(1, 'a b c');
          328  +    INSERT INTO t2(rowid, x) VALUES(456, 'a b c');
          329  +    INSERT INTO t2(rowid, x) VALUES(1000, 'a b c');
          330  +  COMMIT;
          331  +  UPDATE t2 SET x=x;
          332  +  DELETE FROM t2;
          333  +}
          334  +
          335  +#db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM t2_data} {puts $r}
   302    336     
   303    337   finish_test
   304    338   

Changes to ext/fts5/test/fts5synonym.test.

    17     17   
    18     18   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    19     19   ifcapable !fts5 {
    20     20     finish_test
    21     21     return
    22     22   }
    23     23   
    24         -foreach S {
    25         -  {zero 0}
    26         -  {one 1 i}
    27         -  {two 2 ii}
    28         -  {three 3 iii}
    29         -  {four 4 iv}
    30         -  {five 5 v}
    31         -  {six 6 vi}
    32         -  {seven 7 vii}
    33         -  {eight 8 viii}
    34         -  {nine 9 ix}
    35         -} {
    36         -  foreach s $S {
    37         -    set o [list]
    38         -    foreach x $S {if {$x!=$s} {lappend o $x}}
    39         -    set ::syn($s) $o
    40         -  }
    41         -}
           24  +proc tcl_create {args} { return "tcl_tokenize" }
    42     25   
    43         -proc tcl_tokenize {tflags text} {
    44         -  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
    45         -    sqlite3_fts5_token $w $iStart $iEnd
    46         -  }
    47         -}
    48         -
    49         -proc tcl_create {args} {
    50         -  return "tcl_tokenize"
    51         -}
    52         -
    53         -sqlite3_fts5_create_tokenizer db tcl tcl_create
           26  +foreach_detail_mode $testprefix {
    54     27   
    55     28   #-------------------------------------------------------------------------
    56     29   # Warm body test for the code in fts5_tcl.c.
    57     30   #
           31  +fts5_tclnum_register db
    58     32   do_execsql_test 1.0 {
    59         -  CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl);
           33  +  CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = "tclnum document", detail=%DETAIL%);
    60     34     INSERT INTO ft VALUES('abc def ghi');
    61     35     INSERT INTO ft VALUES('jkl mno pqr');
    62     36     SELECT rowid, x FROM ft WHERE ft MATCH 'def';
    63     37     SELECT x, rowid FROM ft WHERE ft MATCH 'pqr';
    64     38   } {1 {abc def ghi} {jkl mno pqr} 2}
    65     39   
    66     40   #-------------------------------------------------------------------------
    67     41   # Test a tokenizer that supports synonyms by adding extra entries to the
    68     42   # FTS index.
    69     43   #
    70         -
    71         -proc tcl_tokenize {tflags text} {
    72         -  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
    73         -    sqlite3_fts5_token $w $iStart $iEnd
    74         -    if {$tflags=="document" && [info exists ::syn($w)]} {
    75         -      foreach s $::syn($w) {
    76         -        sqlite3_fts5_token -colo $s $iStart $iEnd
    77         -      }
    78         -    }
    79         -  }
    80         -}
    81     44   reset_db
    82         -sqlite3_fts5_create_tokenizer db tcl tcl_create
           45  +fts5_tclnum_register db
    83     46   
    84     47   do_execsql_test 2.0 {
    85         -  CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl);
           48  +  CREATE VIRTUAL TABLE ft USING fts5(
           49  +      x, tokenize = "tclnum document", detail=%DETAIL%
           50  +  );
    86     51     INSERT INTO ft VALUES('one two three');
    87     52     INSERT INTO ft VALUES('four five six');
    88     53     INSERT INTO ft VALUES('eight nine ten');
    89     54   } {}
    90     55   
    91     56   foreach {tn expr res} {
    92     57     1 "3" 1
    93     58     2 "eight OR 8 OR 5" {2 3}
    94     59     3 "10" {}
    95     60     4 "1*" {1}
    96     61     5 "1 + 2" {1}
    97     62   } {
           63  +  if {![fts5_expr_ok $expr ft]} continue
    98     64     do_execsql_test 2.1.$tn {
    99     65       SELECT rowid FROM ft WHERE ft MATCH $expr
   100     66     } $res
   101     67   }
   102     68   
   103     69   #-------------------------------------------------------------------------
   104     70   # Test some broken tokenizers:
................................................................................
   176    142     SELECT rowid FROM ft WHERE ft MATCH 'one + two + two + three';
   177    143   } {}
   178    144   
   179    145   #-------------------------------------------------------------------------
   180    146   # Check that expressions with synonyms can be parsed and executed.
   181    147   #
   182    148   reset_db
   183         -sqlite3_fts5_create_tokenizer db tcl tcl_create
   184         -proc tcl_tokenize {tflags text} {
   185         -  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
   186         -    sqlite3_fts5_token $w $iStart $iEnd
   187         -    if {$tflags=="query" && [info exists ::syn($w)]} {
   188         -      foreach s $::syn($w) {
   189         -        sqlite3_fts5_token -colo $s $iStart $iEnd
   190         -      }
   191         -    }
   192         -  }
   193         -}
          149  +fts5_tclnum_register db
   194    150   
   195    151   foreach {tn expr res} {
   196    152     1  {abc}                           {"abc"}
   197    153     2  {one}                           {"one"|"i"|"1"}
   198    154     3  {3}                             {"3"|"iii"|"three"}
   199    155     4  {3*}                            {"3"|"iii"|"three" *}
   200    156   } {
   201         -  do_execsql_test 4.1.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res]
          157  +  do_execsql_test 4.1.$tn {
          158  +    SELECT fts5_expr($expr, 'tokenize=tclnum')
          159  +  } [list $res]
   202    160   }
   203    161   
   204    162   do_execsql_test 4.2.1 {
   205         -  CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tcl);
          163  +  CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tclnum, detail=%DETAIL%);
   206    164     INSERT INTO xx VALUES('one two');
   207    165     INSERT INTO xx VALUES('three four');
   208    166   }
   209    167   
   210    168   do_execsql_test 4.2.2 {
   211    169     SELECT rowid FROM xx WHERE xx MATCH '2'
   212    170   } {1}
................................................................................
   213    171   
   214    172   do_execsql_test 4.2.3 {
   215    173     SELECT rowid FROM xx WHERE xx MATCH '3'
   216    174   } {2}
   217    175   
   218    176   do_test 5.0 {
   219    177     execsql { 
   220         -    CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tcl)
          178  +    CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tclnum, detail=%DETAIL%)
   221    179     }
   222    180     foreach {rowid a b} {
   223    181       1 {four v 4 i three} {1 3 five five 4 one}
   224    182       2 {5 1 3 4 i} {2 2 v two 4}
   225    183       3 {5 i 5 2 four 4 1} {iii ii five two 1}
   226    184       4 {ii four 4 one 5 three five} {one 5 1 iii 4 3}
   227    185       5 {three i v i four 4 1} {ii five five five iii}
................................................................................
   281    239     }
   282    240   
   283    241     6 {"v v"} {
   284    242       1 {four v 4 i three} {1 3 [five five] 4 one}
   285    243       5 {three i v i four 4 1} {ii [five five five] iii}
   286    244     }
   287    245   } {
          246  +  if {![fts5_expr_ok $q t1]} continue
   288    247     do_execsql_test 5.1.$tn {
   289    248       SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']')
   290    249       FROM t1 WHERE t1 MATCH $q
   291    250     } $res
   292    251   }
   293    252   
   294    253   # Test that the xQueryPhrase() API works with synonyms.
................................................................................
   312    271     }
   313    272   } {
   314    273     do_execsql_test 5.2.$tn {
   315    274       SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH $q
   316    275     } $res
   317    276   }
   318    277   
   319         -
   320    278   #-------------------------------------------------------------------------
   321    279   # Test terms with more than 4 synonyms.
   322    280   #
   323    281   reset_db
   324    282   sqlite3_fts5_create_tokenizer db tcl tcl_create
   325    283   proc tcl_tokenize {tflags text} {
   326    284     foreach {w iStart iEnd} [fts5_tokenize_split $text] {
................................................................................
   330    288           sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd
   331    289         }
   332    290       }
   333    291     }
   334    292   }
   335    293   
   336    294   do_execsql_test 6.0.1 {
   337         -  CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl);
          295  +  CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl, detail=%DETAIL%);
   338    296     INSERT INTO t1 VALUES('yy xx qq');
   339    297     INSERT INTO t1 VALUES('yy xx xx');
   340    298   }
   341         -do_execsql_test 6.0.2 {
   342         -  SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)';
   343         -} {{yy xx qq}}
          299  +if {[fts5_expr_ok "NEAR(y q)" t1]} {
          300  +  do_execsql_test 6.0.2 {
          301  +    SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)';
          302  +  } {{yy xx qq}}
          303  +}
   344    304   
   345    305   do_test 6.0.3 {
   346    306     execsql { 
   347         -    CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl)
          307  +    CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl, detail=%DETAIL%)
   348    308     }
   349    309     foreach {rowid a b} {
   350    310       1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq aaaa}
   351    311       2 {ww oooooo bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq}
   352    312       3 {zzzz llll gggggg cccc uu} {hhhhhh aaaa ppppp rr ee jjjj}
   353    313       4 {r f i rrrrrr ww hhh} {aa yyy t x aaaaa ii}
   354    314       5 {fffff mm vvvv ooo ffffff kkkk tttt} {cccccc bb e zzz d n}
................................................................................
   383    343     }
   384    344   
   385    345     4 {NEAR(q y, 20)} {
   386    346       1 {[yyyy] vvvvv [qq] oo [yyyyyy] vvvv eee} {ffff uu r qq aaaa}
   387    347       2 {ww oooooo bbbbb ssssss mm} {ffffff [yy] iiii rr s ccc [qqqqq]}
   388    348     }
   389    349   } {
          350  +  if {![fts5_expr_ok $q t2]} continue
          351  +
   390    352     do_execsql_test 6.1.$tn.asc {
   391    353       SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']')
   392    354       FROM t2 WHERE t2 MATCH $q
   393    355     } $res
   394    356   
   395    357     set res2 [list]
   396    358     foreach {rowid a b} $res {
................................................................................
   431    393           sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd
   432    394         }
   433    395       }
   434    396     }
   435    397   }
   436    398   
   437    399   do_execsql_test 7.0.1 {
   438         -  CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl);
          400  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl, detail=%DETAIL%);
   439    401     INSERT INTO t1 VALUES('0 2 3', '4 5 6 7');
   440    402     INSERT INTO t1 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0');
   441    403     SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH '000 AND 00 AND 0';
   442    404   } {{3 4} {2 10}}
   443    405   
   444    406   do_execsql_test 7.0.2 {
   445    407     INSERT INTO t1(t1) VALUES('integrity-check');
   446    408   }
   447    409   
   448    410   do_execsql_test 7.1.1 {
   449         -  CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl);
          411  +  CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl, detail=%DETAIL%);
   450    412     INSERT INTO t2 VALUES('0 2 3', '4 5 6 7');
   451    413     INSERT INTO t2 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0');
   452    414     SELECT fts5_test_columnsize(t2) FROM t2 WHERE t2 MATCH '000 AND 00 AND 0';
   453    415   } {{3 4} {2 10}}
   454    416   
   455    417   do_execsql_test 7.1.2 {
   456    418     INSERT INTO t2(t2) VALUES('integrity-check');
   457    419   }
          420  +
          421  +} ;# foreach_detail_mode
   458    422   
   459    423   finish_test
   460    424   

Changes to ext/fts5/test/fts5synonym2.test.

    17     17   
    18     18   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    19     19   ifcapable !fts5 {
    20     20     finish_test
    21     21     return
    22     22   }
    23     23   
    24         -#-------------------------------------------------------------------------
    25         -# Code for a simple Tcl tokenizer that supports synonyms at query time.
    26         -#
    27         -foreach SYNDICT {
    28         -  {zero  0}
    29         -  {one   1 i}
    30         -  {two   2 ii}
    31         -  {three 3 iii}
    32         -  {four  4 iv}
    33         -  {five  5 v}
    34         -  {six   6 vi}
    35         -  {seven 7 vii}
    36         -  {eight 8 viii}
    37         -  {nine  9 ix}
    38         -} {
    39         -  foreach s $SYNDICT {
    40         -    set o [list]
    41         -    foreach x $SYNDICT {if {$x!=$s} {lappend o $x}}
    42         -    set ::syn($s) $o
    43         -  }
    44         -}
    45         -
    46         -proc tcl_tokenize {tflags text} {
    47         -  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
    48         -    sqlite3_fts5_token $w $iStart $iEnd
    49         -    if {$tflags == "query"} {
    50         -      foreach s $::syn($w)  { sqlite3_fts5_token -colo $s $iStart $iEnd }
    51         -    }
    52         -  }
    53         -}
    54         -
    55         -proc tcl_create {args} {
    56         -  return "tcl_tokenize"
    57         -}
    58         -
    59         -#
    60         -# End of tokenizer code.
    61         -#-------------------------------------------------------------------------
    62         -
           24  +foreach tok {query document} {
    63     25   foreach_detail_mode $testprefix {
    64     26   
    65         -sqlite3_fts5_create_tokenizer db tcl tcl_create
           27  +fts5_tclnum_register db
    66     28   fts5_aux_test_functions db
    67     29   
    68         -do_execsql_test 1.0 {
    69         -  CREATE VIRTUAL TABLE ss USING fts5(a, b, tokenize=tcl, detail=%DETAIL%);
           30  +proc fts5_rowid {cmd} { expr [$cmd xColumnText -1] }
           31  +sqlite3_fts5_create_function db fts5_rowid fts5_rowid
           32  +
           33  +do_execsql_test 1.$tok.0.1 "
           34  +  CREATE VIRTUAL TABLE ss USING fts5(a, b, 
           35  +       tokenize='tclnum $tok', detail=%DETAIL%);
           36  +  INSERT INTO ss(ss, rank) VALUES('rank', 'fts5_rowid()');
           37  +"
           38  +
           39  +do_execsql_test 1.$tok.0.2 {
    70     40     INSERT INTO ss VALUES('5 5 five seven 3 seven i', '2 1 5 0 two 1 i');
    71     41     INSERT INTO ss VALUES('six ix iii 7 i vii iii', 'one seven nine 4 9 1 vi');
    72     42     INSERT INTO ss VALUES('6 viii i five six zero seven', '5 v iii iv iv 3');
    73     43     INSERT INTO ss VALUES('9 ii six 8 1 6', 'six 4 iv iv 7');
    74     44     INSERT INTO ss VALUES('1 5 4 eight ii iv iii', 'nine 2 eight ix v vii');
    75     45     INSERT INTO ss VALUES('one 7 seven six 2 two', '1 2 four 7 4 3 4');
    76     46     INSERT INTO ss VALUES('eight iv 4 nine vii six 1', '5 6 v one zero 4');
................................................................................
   133    103     3.3 "a:one OR b:1 OR {a b} : i"
   134    104   
   135    105     4.1 "NEAR(one two, 2)"
   136    106     4.2 "NEAR(one two three, 2)"
   137    107     4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)"
   138    108   } {
   139    109     if {[fts5_expr_ok $expr ss]==0} {
   140         -    do_test 1.$tn.OMITTED { list } [list]
          110  +    do_test 1.$tok.$tn.OMITTED { list } [list]
   141    111       continue
   142    112     }
   143    113   
   144         -  set res [fts5_query_data $expr ss ASC ::syn]
   145         -  breakpoint
   146         -  do_execsql_test 1.$tn.[llength $res].asc {
          114  +  set res [fts5_query_data $expr ss ASC ::tclnum_syn]
          115  +  do_execsql_test 1.$tok.$tn.[llength $res].asc.1 {
          116  +    SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr)
          117  +  } $res
          118  +
          119  +  do_execsql_test 1.$tok.$tn.[llength $res].asc.2 {
   147    120       SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr)
          121  +    ORDER BY rank ASC
   148    122     } $res
   149    123   }
   150    124   
          125  +}
   151    126   }
   152    127   
   153    128   finish_test
   154    129   

Added ext/fts5/test/fts5tok1.test.

            1  +# 2016 Jan 15
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +#
           12  +
           13  +source [file join [file dirname [info script]] fts5_common.tcl]
           14  +ifcapable !fts5 { finish_test ; return }
           15  +set ::testprefix fts5tok1
           16  +
           17  +
           18  +sqlite3_fts5_register_fts5tokenize db
           19  +
           20  +#-------------------------------------------------------------------------
           21  +# Simple test cases. Using the default (ascii) tokenizer.
           22  +#
           23  +do_execsql_test 1.0 {
           24  +  CREATE VIRTUAL TABLE t1 USING fts5tokenize(ascii);
           25  +  CREATE VIRTUAL TABLE t2 USING fts5tokenize();
           26  +  CREATE VIRTUAL TABLE t3 USING fts5tokenize(
           27  +      ascii, 'separators', 'xyz', tokenchars, ''''
           28  +  );
           29  +}
           30  +
           31  +foreach {tn tbl} {1 t1 2 t2 3 t3} {
           32  +  do_execsql_test 1.$tn.1 "SELECT input, * FROM $tbl ('one two three')" {
           33  +    {one two three} one   0  3 0 
           34  +    {one two three} two   4  7 1 
           35  +    {one two three} three 8 13 2
           36  +  }
           37  +
           38  +  do_execsql_test 1.$tn.2 "
           39  +    SELECT token FROM $tbl WHERE input = 'OnE tWo tHrEe'
           40  +  " {
           41  +    one two three
           42  +  }
           43  +}
           44  +
           45  +do_execsql_test 1.4 {
           46  +  SELECT token FROM t3 WHERE input = '1x2x3x'
           47  +} {1 2 3}
           48  +
           49  +do_execsql_test 1.5 {
           50  +  SELECT token FROM t1 WHERE input = '1x2x3x'
           51  +} {1x2x3x}
           52  +
           53  +do_execsql_test 1.6 {
           54  +  SELECT token FROM t3 WHERE input = '1''2x3x'
           55  +} {1'2 3}
           56  +
           57  +do_execsql_test 1.7 {
           58  +  SELECT token FROM t3 WHERE input = ''
           59  +} {}
           60  +
           61  +do_execsql_test 1.8 {
           62  +  SELECT token FROM t3 WHERE input = NULL
           63  +} {}
           64  +
           65  +do_execsql_test 1.9 {
           66  +  SELECT input, * FROM t3 WHERE input = 123
           67  +} {123 123 0 3 0}
           68  +
           69  +do_execsql_test 1.10 {
           70  +  SELECT input, * FROM t1 WHERE input = 'a b c' AND token = 'b';
           71  +} {
           72  +  {a b c} b 2 3 1
           73  +}
           74  +
           75  +do_execsql_test 1.11 {
           76  +  SELECT input, * FROM t1 WHERE token = 'b' AND input = 'a b c';
           77  +} {
           78  +  {a b c} b 2 3 1
           79  +}
           80  +
           81  +do_execsql_test 1.12 {
           82  +  SELECT input, * FROM t1 WHERE input < 'b' AND input = 'a b c';
           83  +} {
           84  +  {a b c} a 0 1 0 
           85  +  {a b c} b 2 3 1 
           86  +  {a b c} c 4 5 2
           87  +}
           88  +
           89  +do_execsql_test 1.13.1 {
           90  +  CREATE TABLE c1(x);
           91  +  INSERT INTO c1(x) VALUES('a b c');
           92  +  INSERT INTO c1(x) VALUES('d e f');
           93  +}
           94  +do_execsql_test 1.13.2 {
           95  +  SELECT c1.*, input, t1.* FROM c1, t1 WHERE input = x AND c1.rowid=t1.rowid;
           96  +} {
           97  +  {a b c} {a b c} a 0 1 0 
           98  +  {d e f} {d e f} e 2 3 1 
           99  +}
          100  +
          101  +
          102  +#-------------------------------------------------------------------------
          103  +# Error cases.
          104  +#
          105  +do_catchsql_test 2.0 {
          106  +  CREATE VIRTUAL TABLE tX USING fts5tokenize(nosuchtokenizer);
          107  +} {1 {vtable constructor failed: tX}}
          108  +
          109  +do_catchsql_test 2.1 {
          110  +  CREATE VIRTUAL TABLE t4 USING fts5tokenize;
          111  +  SELECT * FROM t4;
          112  +} {1 {SQL logic error or missing database}}
          113  +
          114  +
          115  +finish_test

Added ext/fts5/test/fts5tok2.test.

            1  +# 2016 Jan 15
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +#
           12  +
           13  +source [file join [file dirname [info script]] fts5_common.tcl]
           14  +ifcapable !fts5||!fts3 { finish_test ; return }
           15  +set ::testprefix fts5tok2
           16  +
           17  +sqlite3_fts5_register_fts5tokenize db
           18  +
           19  +#-------------------------------------------------------------------------
           20  +# Simple test cases. Using the default (ascii) tokenizer.
           21  +#
           22  +do_execsql_test 1.0 {
           23  +  CREATE VIRTUAL TABLE t5 USING fts5tokenize(unicode61);
           24  +  CREATE VIRTUAL TABLE t3 USING fts3tokenize(unicode61);
           25  +}
           26  +
           27  +do_test 1.1 {
           28  +  array unset -nocomplain A
           29  +
           30  +  for {set i 1} {$i < 65536} {incr i} {
           31  +    set input [format "abc%cxyz" $i]
           32  +      set expect [execsql {
           33  +        SELECT input, token, start, end FROM t3 WHERE input=$input
           34  +    }]
           35  +
           36  +    incr A([llength $expect])
           37  +
           38  +    set res [execsql {
           39  +      SELECT input, token, start, end FROM t5($input)
           40  +    }]
           41  +    if {$res != $expect} {error "failed at i=$i"}
           42  +  }
           43  +} {}
           44  +
           45  +do_test 1.1.nTokenChars=$A(4).nSeparators=$A(8) {} {}
           46  +
           47  +finish_test

Added ext/fts5/test/fts5update.test.

            1  +# 2016 Jan 16
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5update
           17  +
           18  +# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
           19  +ifcapable !fts5 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +set docs {
           25  +  "eight zero iv eight 7"            "ix one 8 one three ii one"        
           26  +  "1 9 9 three viii"                 "5 zero ii 6 nine ix 3"            
           27  +  "3 zero 5 2 seven nine"            "two eight viii eight 1"           
           28  +  "4 six two 5 9 vii"                "viii ii four 8 i i iv"            
           29  +  "vii 0 iv seven 7 viii"            "five 1 nine vi seven"             
           30  +  "1 zero zero iii 1"                "one one six 6 nine seven"         
           31  +  "one v 4 zero 4 iii ii"            "2 3 eight six ix"                 
           32  +  "six iv 7 three 5"                 "ix zero 0 8 ii 7 3"               
           33  +  "four six nine 2 vii 3"            "five viii 5 8 0 7"                
           34  +}
           35  +
           36  +foreach_detail_mode $::testprefix {
           37  +
           38  +do_execsql_test 1.0 {
           39  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
           40  +} {}
           41  +
           42  +do_test 1.1 {
           43  +  foreach {a b} $docs {
           44  +    execsql {INSERT INTO t1 VALUES($a, $b)}
           45  +  }
           46  +} {}
           47  +
           48  +proc update {iRowid iA iB} {
           49  +  set a [lindex $::docs $iA]
           50  +  set b [lindex $::docs $iB]
           51  +  execsql { UPDATE t1 SET a=$a, b=$b WHERE rowid=$iRowid }
           52  +}
           53  +
           54  +set nDoc [llength $::docs]
           55  +foreach n {1 5 10 50 100} {
           56  +  do_test 1.2.$n {
           57  +    execsql BEGIN
           58  +    for {set i 1} {$i <= 1000} {incr i} {
           59  +      set iRowid [expr {int(rand() * ($nDoc/2)) + 1}]
           60  +      set iA [expr {int(rand() * $nDoc)}]
           61  +      set iB [expr {int(rand() * $nDoc)}]
           62  +      update $iRowid $iA $iB
           63  +
           64  +      if {($i % $n)==0} {
           65  +        execsql { COMMIT; BEGIN }
           66  +      }
           67  +
           68  +      if {($i % $n)==100} {
           69  +        execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
           70  +      }
           71  +    }
           72  +    execsql COMMIT
           73  +    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
           74  +  } {}
           75  +}
           76  +
           77  +do_execsql_test 1.3 {
           78  +  UPDATE t1 SET a=a AND b=b;
           79  +  INSERT INTO t1(t1) VALUES('integrity-check');
           80  +}
           81  +
           82  +do_test 1.4 {
           83  +  execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 32) }
           84  +  for {set i 0} {$i < 50} {incr i} {
           85  +    execsql { UPDATE t1 SET a=a AND b=b }
           86  +    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
           87  +  }
           88  +} {}
           89  +
           90  +#-------------------------------------------------------------------------
           91  +# Lots of deletes/inserts of the same document with the same rowid.
           92  +#
           93  +do_execsql_test 2.0 {
           94  +  CREATE VIRTUAL TABLE x2 USING fts5(x, detail=%DETAIL%);
           95  +  INSERT INTO x2(x2, rank) VALUES('crisismerge', 2);
           96  +  INSERT INTO x2 VALUES('a b c');
           97  +  INSERT INTO x2 VALUES('a b c');
           98  +}
           99  +do_test 2.1 {
          100  +  for {set i 0} {$i < 1000} {incr i} {
          101  +    execsql { DELETE FROM x2 WHERE rowid = 2 }
          102  +    execsql { INSERT INTO x2(rowid, x) VALUES(2, 'a b c') }
          103  +  }
          104  +} {}
          105  +do_execsql_test 2.1.integrity {
          106  +  INSERT INTO x2(x2) VALUES('integrity-check');
          107  +}
          108  +
          109  +do_test 2.2 {
          110  +  for {set i 0} {$i < 1000} {incr i} {
          111  +    execsql { UPDATE x2 SET x=x WHERE rowid=2 }
          112  +  }
          113  +} {}
          114  +do_execsql_test 2.2.integrity {
          115  +  INSERT INTO x2(x2) VALUES('integrity-check');
          116  +}
          117  +
          118  +}
          119  +finish_test
          120  +
          121  +

Changes to ext/fts5/test/fts5vocab.test.

    52     52     if {[detail_is_none]==0} { error "this is for detail=none mode" }
    53     53     set ret [list]
    54     54     foreach {a b c} $L {
    55     55       lappend ret $a {} $b {}
    56     56     }
    57     57     set ret
    58     58   }
           59  +
           60  +if 1 {
    59     61   
    60     62   do_execsql_test 1.1.1 {
    61     63     CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1, detail=%DETAIL%);
    62     64     CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row');
    63     65     PRAGMA table_info = v1;
    64     66   } {
    65     67     0 term {} 0 {} 0
................................................................................
   385    387   if {![detail_is_none]} {
   386    388     do_execsql_test 7.3.2 {
   387    389       SELECT count(*) FROM txc, txc_c
   388    390         WHERE txc.term = txc_c.term AND txc.col=txc_c.col;
   389    391     } {57}
   390    392   }
   391    393   
          394  +}
          395  +
          396  +#-------------------------------------------------------------------------
          397  +# Test the fts5vocab tables response to a specific types of corruption:
          398  +# where the fts5 index contains hits for columns that do not exist.
          399  +#
          400  +do_execsql_test 8.0 {
          401  +  CREATE VIRTUAL TABLE x1 USING fts5(a, b, c, detail=%DETAIL%);
          402  +  INSERT INTO x1 VALUES('a b c', 'd e f', 'g h i');
          403  +  INSERT INTO x1 VALUES('g h i', 'a b c', 'd e f');
          404  +  INSERT INTO x1 VALUES('d e f', 'g h i', 'a b c');
          405  +  CREATE VIRTUAL TABLE x1_r USING fts5vocab(x1, row);
          406  +  CREATE VIRTUAL TABLE x1_c USING fts5vocab(x1, col);
          407  +}
          408  +
          409  +set resr [star_from_row {a 3 3 b 3 3 c 3 3 d 3 3 e 3 3 f 3 3 g 3 3 h 3 3 i 3 3}]
          410  +set resc [star_from_col {
          411  +  a a 1 1 a b 1 1 a c 1 1 b a 1 1 
          412  +  b b 1 1 b c 1 1 c a 1 1 c b 1 1 
          413  +  c c 1 1 d a 1 1 d b 1 1 d c 1 1
          414  +  e a 1 1 e b 1 1 e c 1 1 f a 1 1 
          415  +  f b 1 1 f c 1 1 g a 1 1 g b 1 1 
          416  +  g c 1 1 h a 1 1 h b 1 1 h c 1 1 
          417  +  i a 1 1 i b 1 1 i c 1 1
          418  +}]
          419  +if {[detail_is_none]} { set resc [row_to_col $resr] }
          420  +
          421  +do_execsql_test 8.1.1 { SELECT * FROM x1_r; } $resr
          422  +do_execsql_test 8.1.2 { SELECT * FROM x1_c } $resc
          423  +
          424  +do_execsql_test 8.2 {
          425  +  PRAGMA writable_schema = 1;
          426  +  UPDATE sqlite_master 
          427  +  SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(a, detail=%DETAIL%)'
          428  +  WHERE name = 'x1';
          429  +}
          430  +db close
          431  +sqlite3 db test.db
          432  +sqlite3_fts5_may_be_corrupt 1
          433  +
          434  +do_execsql_test 8.2.1 { SELECT * FROM x1_r } $resr
          435  +
          436  +if {[detail_is_none]} {
          437  +  do_execsql_test 8.2.2 { SELECT * FROM x1_c } $resc
          438  +} else {
          439  +  do_catchsql_test 8.2.2 { 
          440  +    SELECT * FROM x1_c 
          441  +  } {1 {database disk image is malformed}}
          442  +}
          443  +
          444  +sqlite3_fts5_may_be_corrupt 0
          445  +
   392    446   }
   393    447   
   394    448   finish_test
   395    449   

Changes to main.mk.

   331    331     $(TOP)/ext/misc/regexp.c \
   332    332     $(TOP)/ext/misc/series.c \
   333    333     $(TOP)/ext/misc/spellfix.c \
   334    334     $(TOP)/ext/misc/totype.c \
   335    335     $(TOP)/ext/misc/wholenumber.c \
   336    336     $(TOP)/ext/misc/vfslog.c \
   337    337     $(TOP)/ext/fts5/fts5_tcl.c \
   338         -  $(TOP)/ext/fts5/fts5_test_mi.c
          338  +  $(TOP)/ext/fts5/fts5_test_mi.c \
          339  +  $(TOP)/ext/fts5/fts5_test_tok.c 
   339    340   
   340    341   
   341    342   #TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c
   342    343   #TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c
   343    344   
   344    345   TESTSRC2 = \
   345    346     $(TOP)/src/attach.c \

Changes to src/analyze.c.

   474    474     sqlite3_result_blob(context, p, sizeof(*p), stat4Destructor);
   475    475   }
   476    476   static const FuncDef statInitFuncdef = {
   477    477     2+IsStat34,      /* nArg */
   478    478     SQLITE_UTF8,     /* funcFlags */
   479    479     0,               /* pUserData */
   480    480     0,               /* pNext */
   481         -  statInit,        /* xFunc */
   482         -  0,               /* xStep */
          481  +  statInit,        /* xSFunc */
   483    482     0,               /* xFinalize */
   484    483     "stat_init",     /* zName */
   485    484     0,               /* pHash */
   486    485     0                /* pDestructor */
   487    486   };
   488    487   
   489    488   #ifdef SQLITE_ENABLE_STAT4
................................................................................
   775    774   #endif
   776    775   }
   777    776   static const FuncDef statPushFuncdef = {
   778    777     2+IsStat34,      /* nArg */
   779    778     SQLITE_UTF8,     /* funcFlags */
   780    779     0,               /* pUserData */
   781    780     0,               /* pNext */
   782         -  statPush,        /* xFunc */
   783         -  0,               /* xStep */
          781  +  statPush,        /* xSFunc */
   784    782     0,               /* xFinalize */
   785    783     "stat_push",     /* zName */
   786    784     0,               /* pHash */
   787    785     0                /* pDestructor */
   788    786   };
   789    787   
   790    788   #define STAT_GET_STAT1 0          /* "stat" column of stat1 table */
................................................................................
   922    920   #endif
   923    921   }
   924    922   static const FuncDef statGetFuncdef = {
   925    923     1+IsStat34,      /* nArg */
   926    924     SQLITE_UTF8,     /* funcFlags */
   927    925     0,               /* pUserData */
   928    926     0,               /* pNext */
   929         -  statGet,         /* xFunc */
   930         -  0,               /* xStep */
          927  +  statGet,         /* xSFunc */
   931    928     0,               /* xFinalize */
   932    929     "stat_get",      /* zName */
   933    930     0,               /* pHash */
   934    931     0                /* pDestructor */
   935    932   };
   936    933   
   937    934   static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){
................................................................................
   939    936   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
   940    937     sqlite3VdbeAddOp2(v, OP_Integer, iParam, regStat4+1);
   941    938   #elif SQLITE_DEBUG
   942    939     assert( iParam==STAT_GET_STAT1 );
   943    940   #else
   944    941     UNUSED_PARAMETER( iParam );
   945    942   #endif
   946         -  sqlite3VdbeAddOp3(v, OP_Function0, 0, regStat4, regOut);
   947         -  sqlite3VdbeChangeP4(v, -1, (char*)&statGetFuncdef, P4_FUNCDEF);
          943  +  sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4, regOut,
          944  +                    (char*)&statGetFuncdef, P4_FUNCDEF);
   948    945     sqlite3VdbeChangeP5(v, 1 + IsStat34);
   949    946   }
   950    947   
   951    948   /*
   952    949   ** Generate code to do an analysis of all indices associated with
   953    950   ** a single table.
   954    951   */
................................................................................
  1094   1091       ** The third argument is only used for STAT3 and STAT4
  1095   1092       */
  1096   1093   #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  1097   1094       sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3);
  1098   1095   #endif
  1099   1096       sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1);
  1100   1097       sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2);
  1101         -    sqlite3VdbeAddOp3(v, OP_Function0, 0, regStat4+1, regStat4);
  1102         -    sqlite3VdbeChangeP4(v, -1, (char*)&statInitFuncdef, P4_FUNCDEF);
         1098  +    sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4+1, regStat4,
         1099  +                     (char*)&statInitFuncdef, P4_FUNCDEF);
  1103   1100       sqlite3VdbeChangeP5(v, 2+IsStat34);
  1104   1101   
  1105   1102       /* Implementation of the following:
  1106   1103       **
  1107   1104       **   Rewind csr
  1108   1105       **   if eof(csr) goto end_of_scan;
  1109   1106       **   regChng = 0
................................................................................
  1191   1188           VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName));
  1192   1189         }
  1193   1190         sqlite3VdbeAddOp3(v, OP_MakeRecord, regKey, pPk->nKeyCol, regRowid);
  1194   1191         sqlite3ReleaseTempRange(pParse, regKey, pPk->nKeyCol);
  1195   1192       }
  1196   1193   #endif
  1197   1194       assert( regChng==(regStat4+1) );
  1198         -    sqlite3VdbeAddOp3(v, OP_Function0, 1, regStat4, regTemp);
  1199         -    sqlite3VdbeChangeP4(v, -1, (char*)&statPushFuncdef, P4_FUNCDEF);
         1195  +    sqlite3VdbeAddOp4(v, OP_Function0, 1, regStat4, regTemp,
         1196  +                     (char*)&statPushFuncdef, P4_FUNCDEF);
  1200   1197       sqlite3VdbeChangeP5(v, 2+IsStat34);
  1201   1198       sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v);
  1202   1199   
  1203   1200       /* Add the entry to the stat1 table. */
  1204   1201       callStatGet(v, regStat4, STAT_GET_STAT1, regStat1);
  1205   1202       assert( "BBB"[0]==SQLITE_AFF_TEXT );
  1206   1203       sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0);

Changes to src/attach.c.

   355    355     regArgs = sqlite3GetTempRange(pParse, 4);
   356    356     sqlite3ExprCode(pParse, pFilename, regArgs);
   357    357     sqlite3ExprCode(pParse, pDbname, regArgs+1);
   358    358     sqlite3ExprCode(pParse, pKey, regArgs+2);
   359    359   
   360    360     assert( v || db->mallocFailed );
   361    361     if( v ){
   362         -    sqlite3VdbeAddOp3(v, OP_Function0, 0, regArgs+3-pFunc->nArg, regArgs+3);
          362  +    sqlite3VdbeAddOp4(v, OP_Function0, 0, regArgs+3-pFunc->nArg, regArgs+3,
          363  +                      (char *)pFunc, P4_FUNCDEF);
   363    364       assert( pFunc->nArg==-1 || (pFunc->nArg&0xff)==pFunc->nArg );
   364    365       sqlite3VdbeChangeP5(v, (u8)(pFunc->nArg));
   365         -    sqlite3VdbeChangeP4(v, -1, (char *)pFunc, P4_FUNCDEF);
   366         -
          366  + 
   367    367       /* Code an OP_Expire. For an ATTACH statement, set P1 to true (expire this
   368    368       ** statement only). For DETACH, set it to false (expire all existing
   369    369       ** statements).
   370    370       */
   371    371       sqlite3VdbeAddOp1(v, OP_Expire, (type==SQLITE_ATTACH));
   372    372     }
   373    373     
................................................................................
   384    384   */
   385    385   void sqlite3Detach(Parse *pParse, Expr *pDbname){
   386    386     static const FuncDef detach_func = {
   387    387       1,                /* nArg */
   388    388       SQLITE_UTF8,      /* funcFlags */
   389    389       0,                /* pUserData */
   390    390       0,                /* pNext */
   391         -    detachFunc,       /* xFunc */
   392         -    0,                /* xStep */
          391  +    detachFunc,       /* xSFunc */
   393    392       0,                /* xFinalize */
   394    393       "sqlite_detach",  /* zName */
   395    394       0,                /* pHash */
   396    395       0                 /* pDestructor */
   397    396     };
   398    397     codeAttach(pParse, SQLITE_DETACH, &detach_func, pDbname, 0, 0, pDbname);
   399    398   }
................................................................................
   405    404   */
   406    405   void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *pKey){
   407    406     static const FuncDef attach_func = {
   408    407       3,                /* nArg */
   409    408       SQLITE_UTF8,      /* funcFlags */
   410    409       0,                /* pUserData */
   411    410       0,                /* pNext */
   412         -    attachFunc,       /* xFunc */
   413         -    0,                /* xStep */
          411  +    attachFunc,       /* xSFunc */
   414    412       0,                /* xFinalize */
   415    413       "sqlite_attach",  /* zName */
   416    414       0,                /* pHash */
   417    415       0                 /* pDestructor */
   418    416     };
   419    417     codeAttach(pParse, SQLITE_ATTACH, &attach_func, p, p, pDbname, pKey);
   420    418   }

Changes to src/btree.c.

  7209   7209       ** invariant.
  7210   7210       **
  7211   7211       ** This must be done in advance.  Once the balance starts, the cell
  7212   7212       ** offset section of the btree page will be overwritten and we will no
  7213   7213       ** long be able to find the cells if a pointer to each cell is not saved
  7214   7214       ** first.
  7215   7215       */
  7216         -    memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*limit);
         7216  +    memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow));
  7217   7217       if( pOld->nOverflow>0 ){
  7218         -      memset(&b.szCell[b.nCell+limit], 0, sizeof(b.szCell[0])*pOld->nOverflow);
  7219   7218         limit = pOld->aiOvfl[0];
  7220   7219         for(j=0; j<limit; j++){
  7221   7220           b.apCell[b.nCell] = aData + (maskPage & get2byteAligned(piCell));
  7222   7221           piCell += 2;
  7223   7222           b.nCell++;
  7224   7223         }
  7225   7224         for(k=0; k<pOld->nOverflow; k++){
................................................................................
  8563   8562     **
  8564   8563     ** This error is caught long before control reaches this point.
  8565   8564     */
  8566   8565     if( NEVER(pBt->pCursor) ){
  8567   8566       sqlite3ConnectionBlocked(p->db, pBt->pCursor->pBtree->db);
  8568   8567       return SQLITE_LOCKED_SHAREDCACHE;
  8569   8568     }
         8569  +
         8570  +  /*
         8571  +  ** It is illegal to drop the sqlite_master table on page 1.  But again,
         8572  +  ** this error is caught long before reaching this point.
         8573  +  */
         8574  +  if( NEVER(iTable<2) ){
         8575  +    return SQLITE_CORRUPT_BKPT;
         8576  +  }
  8570   8577   
  8571   8578     rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0);
  8572   8579     if( rc ) return rc;
  8573   8580     rc = sqlite3BtreeClearTable(p, iTable, 0);
  8574   8581     if( rc ){
  8575   8582       releasePage(pPage);
  8576   8583       return rc;
  8577   8584     }
  8578   8585   
  8579   8586     *piMoved = 0;
  8580   8587   
  8581         -  if( iTable>1 ){
  8582   8588   #ifdef SQLITE_OMIT_AUTOVACUUM
  8583         -    freePage(pPage, &rc);
  8584         -    releasePage(pPage);
         8589  +  freePage(pPage, &rc);
         8590  +  releasePage(pPage);
  8585   8591   #else
  8586         -    if( pBt->autoVacuum ){
  8587         -      Pgno maxRootPgno;
  8588         -      sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &maxRootPgno);
  8589         -
  8590         -      if( iTable==maxRootPgno ){
  8591         -        /* If the table being dropped is the table with the largest root-page
  8592         -        ** number in the database, put the root page on the free list. 
  8593         -        */
  8594         -        freePage(pPage, &rc);
  8595         -        releasePage(pPage);
  8596         -        if( rc!=SQLITE_OK ){
  8597         -          return rc;
  8598         -        }
  8599         -      }else{
  8600         -        /* The table being dropped does not have the largest root-page
  8601         -        ** number in the database. So move the page that does into the 
  8602         -        ** gap left by the deleted root-page.
  8603         -        */
  8604         -        MemPage *pMove;
  8605         -        releasePage(pPage);
  8606         -        rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0);
  8607         -        if( rc!=SQLITE_OK ){
  8608         -          return rc;
  8609         -        }
  8610         -        rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0);
  8611         -        releasePage(pMove);
  8612         -        if( rc!=SQLITE_OK ){
  8613         -          return rc;
  8614         -        }
  8615         -        pMove = 0;
  8616         -        rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0);
  8617         -        freePage(pMove, &rc);
  8618         -        releasePage(pMove);
  8619         -        if( rc!=SQLITE_OK ){
  8620         -          return rc;
  8621         -        }
  8622         -        *piMoved = maxRootPgno;
  8623         -      }
  8624         -
  8625         -      /* Set the new 'max-root-page' value in the database header. This
  8626         -      ** is the old value less one, less one more if that happens to
  8627         -      ** be a root-page number, less one again if that is the
  8628         -      ** PENDING_BYTE_PAGE.
  8629         -      */
  8630         -      maxRootPgno--;
  8631         -      while( maxRootPgno==PENDING_BYTE_PAGE(pBt)
  8632         -             || PTRMAP_ISPAGE(pBt, maxRootPgno) ){
  8633         -        maxRootPgno--;
  8634         -      }
  8635         -      assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) );
  8636         -
  8637         -      rc = sqlite3BtreeUpdateMeta(p, 4, maxRootPgno);
  8638         -    }else{
         8592  +  if( pBt->autoVacuum ){
         8593  +    Pgno maxRootPgno;
         8594  +    sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &maxRootPgno);
         8595  +
         8596  +    if( iTable==maxRootPgno ){
         8597  +      /* If the table being dropped is the table with the largest root-page
         8598  +      ** number in the database, put the root page on the free list. 
         8599  +      */
  8639   8600         freePage(pPage, &rc);
  8640   8601         releasePage(pPage);
         8602  +      if( rc!=SQLITE_OK ){
         8603  +        return rc;
         8604  +      }
         8605  +    }else{
         8606  +      /* The table being dropped does not have the largest root-page
         8607  +      ** number in the database. So move the page that does into the 
         8608  +      ** gap left by the deleted root-page.
         8609  +      */
         8610  +      MemPage *pMove;
         8611  +      releasePage(pPage);
         8612  +      rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0);
         8613  +      if( rc!=SQLITE_OK ){
         8614  +        return rc;
         8615  +      }
         8616  +      rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0);
         8617  +      releasePage(pMove);
         8618  +      if( rc!=SQLITE_OK ){
         8619  +        return rc;
         8620  +      }
         8621  +      pMove = 0;
         8622  +      rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0);
         8623  +      freePage(pMove, &rc);
         8624  +      releasePage(pMove);
         8625  +      if( rc!=SQLITE_OK ){
         8626  +        return rc;
         8627  +      }
         8628  +      *piMoved = maxRootPgno;
  8641   8629       }
  8642         -#endif
  8643         -  }else{
  8644         -    /* If sqlite3BtreeDropTable was called on page 1.
  8645         -    ** This really never should happen except in a corrupt
  8646         -    ** database. 
         8630  +
         8631  +    /* Set the new 'max-root-page' value in the database header. This
         8632  +    ** is the old value less one, less one more if that happens to
         8633  +    ** be a root-page number, less one again if that is the
         8634  +    ** PENDING_BYTE_PAGE.
  8647   8635       */
  8648         -    zeroPage(pPage, PTF_INTKEY|PTF_LEAF );
         8636  +    maxRootPgno--;
         8637  +    while( maxRootPgno==PENDING_BYTE_PAGE(pBt)
         8638  +           || PTRMAP_ISPAGE(pBt, maxRootPgno) ){
         8639  +      maxRootPgno--;
         8640  +    }
         8641  +    assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) );
         8642  +
         8643  +    rc = sqlite3BtreeUpdateMeta(p, 4, maxRootPgno);
         8644  +  }else{
         8645  +    freePage(pPage, &rc);
  8649   8646       releasePage(pPage);
  8650   8647     }
         8648  +#endif
  8651   8649     return rc;  
  8652   8650   }
  8653   8651   int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){
  8654   8652     int rc;
  8655   8653     sqlite3BtreeEnter(p);
  8656   8654     rc = btreeDropTable(p, iTable, piMoved);
  8657   8655     sqlite3BtreeLeave(p);

Changes to src/build.c.

   224    224     if( v && pParse->nErr==0 && !db->mallocFailed ){
   225    225       assert( pParse->iCacheLevel==0 );  /* Disables and re-enables match */
   226    226       /* A minimum of one cursor is required if autoincrement is used
   227    227       *  See ticket [a696379c1f08866] */
   228    228       if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1;
   229    229       sqlite3VdbeMakeReady(v, pParse);
   230    230       pParse->rc = SQLITE_DONE;
   231         -    pParse->colNamesSet = 0;
   232    231     }else{
   233    232       pParse->rc = SQLITE_ERROR;
   234    233     }
          234  +
          235  +  /* We are done with this Parse object. There is no need to de-initialize it */
          236  +#if 0
          237  +  pParse->colNamesSet = 0;
   235    238     pParse->nTab = 0;
   236    239     pParse->nMem = 0;
   237    240     pParse->nSet = 0;
   238    241     pParse->nVar = 0;
   239    242     DbMaskZero(pParse->cookieMask);
          243  +#endif
   240    244   }
   241    245   
   242    246   /*
   243    247   ** Run the parser and code generator recursively in order to generate
   244    248   ** code for the SQL statement given onto the end of the pParse context
   245    249   ** currently under construction.  When the parser is run recursively
   246    250   ** this way, the final OP_Halt is not appended and other initialization
................................................................................
   491    495         continue;
   492    496       }
   493    497       if( j<i ){
   494    498         db->aDb[j] = db->aDb[i];
   495    499       }
   496    500       j++;
   497    501     }
   498         -  memset(&db->aDb[j], 0, (db->nDb-j)*sizeof(db->aDb[j]));
   499    502     db->nDb = j;
   500    503     if( db->nDb<=2 && db->aDb!=db->aDbStatic ){
   501    504       memcpy(db->aDbStatic, db->aDb, 2*sizeof(db->aDb[0]));
   502    505       sqlite3DbFree(db, db->aDb);
   503    506       db->aDb = db->aDbStatic;
   504    507     }
   505    508   }
................................................................................
   754    757     Token *pName1,      /* The "xxx" in the name "xxx.yyy" or "xxx" */
   755    758     Token *pName2,      /* The "yyy" in the name "xxx.yyy" */
   756    759     Token **pUnqual     /* Write the unqualified object name here */
   757    760   ){
   758    761     int iDb;                    /* Database holding the object */
   759    762     sqlite3 *db = pParse->db;
   760    763   
   761         -  if( ALWAYS(pName2!=0) && pName2->n>0 ){
          764  +  assert( pName2!=0 );
          765  +  if( pName2->n>0 ){
   762    766       if( db->init.busy ) {
   763    767         sqlite3ErrorMsg(pParse, "corrupt database");
   764    768         return -1;
   765    769       }
   766    770       *pUnqual = pName2;
   767    771       iDb = sqlite3FindDb(db, pName1);
   768    772       if( iDb<0 ){
................................................................................
   843    847     Table *pTable;
   844    848     char *zName = 0; /* The name of the new table */
   845    849     sqlite3 *db = pParse->db;
   846    850     Vdbe *v;
   847    851     int iDb;         /* Database number to create the table in */
   848    852     Token *pName;    /* Unqualified name of the table to create */
   849    853   
   850         -  /* The table or view name to create is passed to this routine via tokens
   851         -  ** pName1 and pName2. If the table name was fully qualified, for example:
   852         -  **
   853         -  ** CREATE TABLE xxx.yyy (...);
   854         -  ** 
   855         -  ** Then pName1 is set to "xxx" and pName2 "yyy". On the other hand if
   856         -  ** the table name is not fully qualified, i.e.:
   857         -  **
   858         -  ** CREATE TABLE yyy(...);
   859         -  **
   860         -  ** Then pName1 is set to "yyy" and pName2 is "".
   861         -  **
   862         -  ** The call below sets the pName pointer to point at the token (pName1 or
   863         -  ** pName2) that stores the unqualified table name. The variable iDb is
   864         -  ** set to the index of the database that the table or view is to be
   865         -  ** created in.
   866         -  */
   867         -  iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName);
   868         -  if( iDb<0 ) return;
   869         -  if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){
   870         -    /* If creating a temp table, the name may not be qualified. Unless 
   871         -    ** the database name is "temp" anyway.  */
   872         -    sqlite3ErrorMsg(pParse, "temporary table name must be unqualified");
   873         -    return;
   874         -  }
   875         -  if( !OMIT_TEMPDB && isTemp ) iDb = 1;
   876         -
   877         -  pParse->sNameToken = *pName;
   878         -  zName = sqlite3NameFromToken(db, pName);
          854  +  if( db->init.busy && db->init.newTnum==1 ){
          855  +    /* Special case:  Parsing the sqlite_master or sqlite_temp_master schema */
          856  +    iDb = db->init.iDb;
          857  +    zName = sqlite3DbStrDup(db, SCHEMA_TABLE(iDb));
          858  +    pName = pName1;
          859  +  }else{
          860  +    /* The common case */
          861  +    iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName);
          862  +    if( iDb<0 ) return;
          863  +    if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){
          864  +      /* If creating a temp table, the name may not be qualified. Unless 
          865  +      ** the database name is "temp" anyway.  */
          866  +      sqlite3ErrorMsg(pParse, "temporary table name must be unqualified");
          867  +      return;
          868  +    }
          869  +    if( !OMIT_TEMPDB && isTemp ) iDb = 1;
          870  +    zName = sqlite3NameFromToken(db, pName);
          871  +  }
          872  +  pParse->sNameToken = *pName;
   879    873     if( zName==0 ) return;
   880    874     if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){
   881    875       goto begin_table_error;
   882    876     }
   883    877     if( db->init.iDb==1 ) isTemp = 1;
   884    878   #ifndef SQLITE_OMIT_AUTHORIZATION
   885         -  assert( (isTemp & 1)==isTemp );
          879  +  assert( isTemp==0 || isTemp==1 );
          880  +  assert( isView==0 || isView==1 );
   886    881     {
   887         -    int code;
          882  +    static const u8 aCode[] = {
          883  +       SQLITE_CREATE_TABLE,
          884  +       SQLITE_CREATE_TEMP_TABLE,
          885  +       SQLITE_CREATE_VIEW,
          886  +       SQLITE_CREATE_TEMP_VIEW
          887  +    };
   888    888       char *zDb = db->aDb[iDb].zName;
   889    889       if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(isTemp), 0, zDb) ){
   890    890         goto begin_table_error;
   891    891       }
   892         -    if( isView ){
   893         -      if( !OMIT_TEMPDB && isTemp ){
   894         -        code = SQLITE_CREATE_TEMP_VIEW;
   895         -      }else{
   896         -        code = SQLITE_CREATE_VIEW;
   897         -      }
   898         -    }else{
   899         -      if( !OMIT_TEMPDB && isTemp ){
   900         -        code = SQLITE_CREATE_TEMP_TABLE;
   901         -      }else{
   902         -        code = SQLITE_CREATE_TABLE;
   903         -      }
   904         -    }
   905         -    if( !isVirtual && sqlite3AuthCheck(pParse, code, zName, 0, zDb) ){
          892  +    if( !isVirtual && sqlite3AuthCheck(pParse, (int)aCode[isTemp+2*isView],
          893  +                                       zName, 0, zDb) ){
   906    894         goto begin_table_error;
   907    895       }
   908    896     }
   909    897   #endif
   910    898   
   911    899     /* Make sure the new table name does not collide with an existing
   912    900     ** index or table name in the same database.  Issue an error message if
................................................................................
  1860   1848     assert( !db->init.busy || !pSelect );
  1861   1849   
  1862   1850     /* If the db->init.busy is 1 it means we are reading the SQL off the
  1863   1851     ** "sqlite_master" or "sqlite_temp_master" table on the disk.
  1864   1852     ** So do not write to the disk again.  Extract the root page number
  1865   1853     ** for the table from the db->init.newTnum field.  (The page number
  1866   1854     ** should have been put there by the sqliteOpenCb routine.)
         1855  +  **
         1856  +  ** If the root page number is 1, that means this is the sqlite_master
         1857  +  ** table itself.  So mark it read-only.
  1867   1858     */
  1868   1859     if( db->init.busy ){
  1869   1860       p->tnum = db->init.newTnum;
         1861  +    if( p->tnum==1 ) p->tabFlags |= TF_Readonly;
  1870   1862     }
  1871   1863   
  1872   1864     /* Special processing for WITHOUT ROWID Tables */
  1873   1865     if( tabOpts & TF_WithoutRowid ){
  1874   1866       if( (p->tabFlags & TF_Autoincrement) ){
  1875   1867         sqlite3ErrorMsg(pParse,
  1876   1868             "AUTOINCREMENT not allowed on WITHOUT ROWID tables");
................................................................................
  2315   2307   ** Also write code to modify the sqlite_master table and internal schema
  2316   2308   ** if a root-page of another table is moved by the btree-layer whilst
  2317   2309   ** erasing iTable (this can happen with an auto-vacuum database).
  2318   2310   */ 
  2319   2311   static void destroyRootPage(Parse *pParse, int iTable, int iDb){
  2320   2312     Vdbe *v = sqlite3GetVdbe(pParse);
  2321   2313     int r1 = sqlite3GetTempReg(pParse);
         2314  +  assert( iTable>1 );
  2322   2315     sqlite3VdbeAddOp3(v, OP_Destroy, iTable, r1, iDb);
  2323   2316     sqlite3MayAbort(pParse);
  2324   2317   #ifndef SQLITE_OMIT_AUTOVACUUM
  2325   2318     /* OP_Destroy stores an in integer r1. If this integer
  2326   2319     ** is non-zero, then it is the root page number of a table moved to
  2327   2320     ** location iTable. The following code modifies the sqlite_master table to
  2328   2321     ** reflect this.
................................................................................
  3713   3706     SrcList *pList,     /* Append to this SrcList. NULL creates a new SrcList */
  3714   3707     Token *pTable,      /* Table to append */
  3715   3708     Token *pDatabase    /* Database of the table */
  3716   3709   ){
  3717   3710     struct SrcList_item *pItem;
  3718   3711     assert( pDatabase==0 || pTable!=0 );  /* Cannot have C without B */
  3719   3712     if( pList==0 ){
  3720         -    pList = sqlite3DbMallocZero(db, sizeof(SrcList) );
         3713  +    pList = sqlite3DbMallocRaw(db, sizeof(SrcList) );
  3721   3714       if( pList==0 ) return 0;
  3722   3715       pList->nAlloc = 1;
         3716  +    pList->nSrc = 0;
  3723   3717     }
  3724   3718     pList = sqlite3SrcListEnlarge(db, pList, 1, pList->nSrc);
  3725   3719     if( db->mallocFailed ){
  3726   3720       sqlite3SrcListDelete(db, pList);
  3727   3721       return 0;
  3728   3722     }
  3729   3723     pItem = &pList->a[pList->nSrc-1];
................................................................................
  4118   4112   ){
  4119   4113     Vdbe *v = sqlite3GetVdbe(pParse);
  4120   4114     assert( (errCode&0xff)==SQLITE_CONSTRAINT );
  4121   4115     if( onError==OE_Abort ){
  4122   4116       sqlite3MayAbort(pParse);
  4123   4117     }
  4124   4118     sqlite3VdbeAddOp4(v, OP_Halt, errCode, onError, 0, p4, p4type);
  4125         -  if( p5Errmsg ) sqlite3VdbeChangeP5(v, p5Errmsg);
         4119  +  sqlite3VdbeChangeP5(v, p5Errmsg);
  4126   4120   }
  4127   4121   
  4128   4122   /*
  4129   4123   ** Code an OP_Halt due to UNIQUE or PRIMARY KEY constraint violation.
  4130   4124   */
  4131   4125   void sqlite3UniqueConstraint(
  4132   4126     Parse *pParse,    /* Parsing context */

Changes to src/callback.c.

   239    239   ** 1: UTF8/16 conversion required and function takes any number of arguments.
   240    240   ** 2: UTF16 byte order change required and function takes any number of args.
   241    241   ** 3: encoding matches and function takes any number of arguments
   242    242   ** 4: UTF8/16 conversion required - argument count matches exactly
   243    243   ** 5: UTF16 byte order conversion required - argument count matches exactly
   244    244   ** 6: Perfect match:  encoding and argument count match exactly.
   245    245   **
   246         -** If nArg==(-2) then any function with a non-null xStep or xFunc is
   247         -** a perfect match and any function with both xStep and xFunc NULL is
          246  +** If nArg==(-2) then any function with a non-null xSFunc is
          247  +** a perfect match and any function with xSFunc NULL is
   248    248   ** a non-match.
   249    249   */
   250    250   #define FUNC_PERFECT_MATCH 6  /* The score for a perfect match */
   251    251   static int matchQuality(
   252    252     FuncDef *p,     /* The function we are evaluating for match quality */
   253    253     int nArg,       /* Desired number of arguments.  (-1)==any */
   254    254     u8 enc          /* Desired text encoding */
   255    255   ){
   256    256     int match;
   257    257   
   258    258     /* nArg of -2 is a special case */
   259         -  if( nArg==(-2) ) return (p->xFunc==0 && p->xStep==0) ? 0 : FUNC_PERFECT_MATCH;
          259  +  if( nArg==(-2) ) return (p->xSFunc==0) ? 0 : FUNC_PERFECT_MATCH;
   260    260   
   261    261     /* Wrong number of arguments means "no match" */
   262    262     if( p->nArg!=nArg && p->nArg>=0 ) return 0;
   263    263   
   264    264     /* Give a better score to a function with a specific number of arguments
   265    265     ** than to function that accepts any number of arguments. */
   266    266     if( p->nArg==nArg ){
................................................................................
   330    330   ** NULL if the function does not exist.
   331    331   **
   332    332   ** If the createFlag argument is true, then a new (blank) FuncDef
   333    333   ** structure is created and liked into the "db" structure if a
   334    334   ** no matching function previously existed.
   335    335   **
   336    336   ** If nArg is -2, then the first valid function found is returned.  A
   337         -** function is valid if either xFunc or xStep is non-zero.  The nArg==(-2)
          337  +** function is valid if xSFunc is non-zero.  The nArg==(-2)
   338    338   ** case is used to see if zName is a valid function name for some number
   339    339   ** of arguments.  If nArg is -2, then createFlag must be 0.
   340    340   **
   341    341   ** If createFlag is false, then a function with the required name and
   342    342   ** number of arguments may be returned even if the eTextRep flag does not
   343    343   ** match that requested.
   344    344   */
................................................................................
   407    407       pBest->nArg = (u16)nArg;
   408    408       pBest->funcFlags = enc;
   409    409       memcpy(pBest->zName, zName, nName);
   410    410       pBest->zName[nName] = 0;
   411    411       sqlite3FuncDefInsert(&db->aFunc, pBest);
   412    412     }
   413    413   
   414         -  if( pBest && (pBest->xStep || pBest->xFunc || createFlag) ){
          414  +  if( pBest && (pBest->xSFunc || createFlag) ){
   415    415       return pBest;
   416    416     }
   417    417     return 0;
   418    418   }
   419    419   
   420    420   /*
   421    421   ** Free all resources held by the schema structure. The void* argument points

Changes to src/date.c.

    66     66     char validJD;      /* True (1) if iJD is valid */
    67     67     char validTZ;      /* True (1) if tz is valid */
    68     68     char tzSet;        /* Timezone was set explicitly */
    69     69   };
    70     70   
    71     71   
    72     72   /*
    73         -** Convert zDate into one or more integers.  Additional arguments
    74         -** come in groups of 5 as follows:
           73  +** Convert zDate into one or more integers according to the conversion
           74  +** specifier zFormat.
    75     75   **
    76         -**       N       number of digits in the integer
    77         -**       min     minimum allowed value of the integer
    78         -**       max     maximum allowed value of the integer
    79         -**       nextC   first character after the integer
    80         -**       pVal    where to write the integers value.
           76  +** zFormat[] contains 4 characters for each integer converted, except for
           77  +** the last integer which is specified by three characters.  The meaning
           78  +** of a four-character format specifiers ABCD is:
    81     79   **
    82         -** Conversions continue until one with nextC==0 is encountered.
           80  +**    A:   number of digits to convert.  Always "2" or "4".
           81  +**    B:   minimum value.  Always "0" or "1".
           82  +**    C:   maximum value, decoded as:
           83  +**           a:  12
           84  +**           b:  14
           85  +**           c:  24
           86  +**           d:  31
           87  +**           e:  59
           88  +**           f:  9999
           89  +**    D:   the separator character, or \000 to indicate this is the
           90  +**         last number to convert.
           91  +**
           92  +** Example:  To translate an ISO-8601 date YYYY-MM-DD, the format would
           93  +** be "40f-21a-20c".  The "40f-" indicates the 4-digit year followed by "-".
           94  +** The "21a-" indicates the 2-digit month followed by "-".  The "20c" indicates
           95  +** the 2-digit day which is the last integer in the set.
           96  +**
    83     97   ** The function returns the number of successful conversions.
    84     98   */
    85         -static int getDigits(const char *zDate, ...){
           99  +static int getDigits(const char *zDate, const char *zFormat, ...){
          100  +  /* The aMx[] array translates the 3rd character of each format
          101  +  ** spec into a max size:    a   b   c   d   e     f */
          102  +  static const u16 aMx[] = { 12, 14, 24, 31, 59, 9999 };
    86    103     va_list ap;
    87         -  int val;
    88         -  int N;
    89         -  int min;
    90         -  int max;
    91         -  int nextC;
    92         -  int *pVal;
    93    104     int cnt = 0;
    94         -  va_start(ap, zDate);
          105  +  char nextC;
          106  +  va_start(ap, zFormat);
    95    107     do{
    96         -    N = va_arg(ap, int);
    97         -    min = va_arg(ap, int);
    98         -    max = va_arg(ap, int);
    99         -    nextC = va_arg(ap, int);
   100         -    pVal = va_arg(ap, int*);
          108  +    char N = zFormat[0] - '0';
          109  +    char min = zFormat[1] - '0';
          110  +    int val = 0;
          111  +    u16 max;
          112  +
          113  +    assert( zFormat[2]>='a' && zFormat[2]<='f' );
          114  +    max = aMx[zFormat[2] - 'a'];
          115  +    nextC = zFormat[3];
   101    116       val = 0;
   102    117       while( N-- ){
   103    118         if( !sqlite3Isdigit(*zDate) ){
   104    119           goto end_getDigits;
   105    120         }
   106    121         val = val*10 + *zDate - '0';
   107    122         zDate++;
   108    123       }
   109         -    if( val<min || val>max || (nextC!=0 && nextC!=*zDate) ){
          124  +    if( val<(int)min || val>(int)max || (nextC!=0 && nextC!=*zDate) ){
   110    125         goto end_getDigits;
   111    126       }
   112         -    *pVal = val;
          127  +    *va_arg(ap,int*) = val;
   113    128       zDate++;
   114    129       cnt++;
          130  +    zFormat += 4;
   115    131     }while( nextC );
   116    132   end_getDigits:
   117    133     va_end(ap);
   118    134     return cnt;
   119    135   }
   120    136   
   121    137   /*
................................................................................
   148    164     }else if( c=='Z' || c=='z' ){
   149    165       zDate++;
   150    166       goto zulu_time;
   151    167     }else{
   152    168       return c!=0;
   153    169     }
   154    170     zDate++;
   155         -  if( getDigits(zDate, 2, 0, 14, ':', &nHr, 2, 0, 59, 0, &nMn)!=2 ){
          171  +  if( getDigits(zDate, "20b:20e", &nHr, &nMn)!=2 ){
   156    172       return 1;
   157    173     }
   158    174     zDate += 5;
   159    175     p->tz = sgn*(nMn + nHr*60);
   160    176   zulu_time:
   161    177     while( sqlite3Isspace(*zDate) ){ zDate++; }
   162    178     p->tzSet = 1;
................................................................................
   169    185   ** fractional seconds FFFF can be one or more digits.
   170    186   **
   171    187   ** Return 1 if there is a parsing error and 0 on success.
   172    188   */
   173    189   static int parseHhMmSs(const char *zDate, DateTime *p){
   174    190     int h, m, s;
   175    191     double ms = 0.0;
   176         -  if( getDigits(zDate, 2, 0, 24, ':', &h, 2, 0, 59, 0, &m)!=2 ){
          192  +  if( getDigits(zDate, "20c:20e", &h, &m)!=2 ){
   177    193       return 1;
   178    194     }
   179    195     zDate += 5;
   180    196     if( *zDate==':' ){
   181    197       zDate++;
   182         -    if( getDigits(zDate, 2, 0, 59, 0, &s)!=1 ){
          198  +    if( getDigits(zDate, "20e", &s)!=1 ){
   183    199         return 1;
   184    200       }
   185    201       zDate += 2;
   186    202       if( *zDate=='.' && sqlite3Isdigit(zDate[1]) ){
   187    203         double rScale = 1.0;
   188    204         zDate++;
   189    205         while( sqlite3Isdigit(*zDate) ){
................................................................................
   263    279   
   264    280     if( zDate[0]=='-' ){
   265    281       zDate++;
   266    282       neg = 1;
   267    283     }else{
   268    284       neg = 0;
   269    285     }
   270         -  if( getDigits(zDate,4,0,9999,'-',&Y,2,1,12,'-',&M,2,1,31,0,&D)!=3 ){
          286  +  if( getDigits(zDate, "40f-21a-21d", &Y, &M, &D)!=3 ){
   271    287       return 1;
   272    288     }
   273    289     zDate += 10;
   274    290     while( sqlite3Isspace(*zDate) || 'T'==*(u8*)zDate ){ zDate++; }
   275    291     if( parseHhMmSs(zDate, p)==0 ){
   276    292       /* We got the time */
   277    293     }else if( *zDate==0 ){

Changes to src/expr.c.

  1131   1131   ExprList *sqlite3ExprListAppend(
  1132   1132     Parse *pParse,          /* Parsing context */
  1133   1133     ExprList *pList,        /* List to which to append. Might be NULL */
  1134   1134     Expr *pExpr             /* Expression to be appended. Might be NULL */
  1135   1135   ){
  1136   1136     sqlite3 *db = pParse->db;
  1137   1137     if( pList==0 ){
  1138         -    pList = sqlite3DbMallocZero(db, sizeof(ExprList) );
         1138  +    pList = sqlite3DbMallocRaw(db, sizeof(ExprList) );
  1139   1139       if( pList==0 ){
  1140   1140         goto no_mem;
  1141   1141       }
         1142  +    pList->nExpr = 0;
  1142   1143       pList->a = sqlite3DbMallocRaw(db, sizeof(pList->a[0]));
  1143   1144       if( pList->a==0 ) goto no_mem;
  1144   1145     }else if( (pList->nExpr & (pList->nExpr-1))==0 ){
  1145   1146       struct ExprList_item *a;
  1146   1147       assert( pList->nExpr>0 );
  1147   1148       a = sqlite3DbRealloc(db, pList->a, pList->nExpr*2*sizeof(pList->a[0]));
  1148   1149       if( a==0 ){
................................................................................
  2892   2893           pFarg = pExpr->x.pList;
  2893   2894         }
  2894   2895         nFarg = pFarg ? pFarg->nExpr : 0;
  2895   2896         assert( !ExprHasProperty(pExpr, EP_IntValue) );
  2896   2897         zId = pExpr->u.zToken;
  2897   2898         nId = sqlite3Strlen30(zId);
  2898   2899         pDef = sqlite3FindFunction(db, zId, nId, nFarg, enc, 0);
  2899         -      if( pDef==0 || pDef->xFunc==0 ){
         2900  +      if( pDef==0 || pDef->xFinalize!=0 ){
  2900   2901           sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId);
  2901   2902           break;
  2902   2903         }
  2903   2904   
  2904   2905         /* Attempt a direct implementation of the built-in COALESCE() and
  2905   2906         ** IFNULL() functions.  This avoids unnecessary evaluation of
  2906   2907         ** arguments past the first non-NULL argument.

Changes to src/insert.c.

  1591   1591       sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdxCur+i, aRegIdx[i]);
  1592   1592       pik_flags = 0;
  1593   1593       if( useSeekResult ) pik_flags = OPFLAG_USESEEKRESULT;
  1594   1594       if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){
  1595   1595         assert( pParse->nested==0 );
  1596   1596         pik_flags |= OPFLAG_NCHANGE;
  1597   1597       }
  1598         -    if( pik_flags )  sqlite3VdbeChangeP5(v, pik_flags);
         1598  +    sqlite3VdbeChangeP5(v, pik_flags);
  1599   1599     }
  1600   1600     if( !HasRowid(pTab) ) return;
  1601   1601     regData = regNewData + 1;
  1602   1602     regRec = sqlite3GetTempReg(pParse);
  1603   1603     sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec);
  1604   1604     if( !bAffinityDone ) sqlite3TableAffinity(v, pTab, 0);
  1605   1605     sqlite3ExprCacheAffinityChange(pParse, regData, pTab->nCol);
................................................................................
  2007   2007       }else if( pDest->pIndex==0 ){
  2008   2008         addr1 = sqlite3VdbeAddOp2(v, OP_NewRowid, iDest, regRowid);
  2009   2009       }else{
  2010   2010         addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid);
  2011   2011         assert( (pDest->tabFlags & TF_Autoincrement)==0 );
  2012   2012       }
  2013   2013       sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData);
  2014         -    sqlite3VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid);
         2014  +    sqlite3VdbeAddOp4(v, OP_Insert, iDest, regData, regRowid,
         2015  +                      pDest->zName, 0);
  2015   2016       sqlite3VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND);
  2016         -    sqlite3VdbeChangeP4(v, -1, pDest->zName, 0);
  2017   2017       sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); VdbeCoverage(v);
  2018   2018       sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0);
  2019   2019       sqlite3VdbeAddOp2(v, OP_Close, iDest, 0);
  2020   2020     }else{
  2021   2021       sqlite3TableLock(pParse, iDbDest, pDest->tnum, 1, pDest->zName);
  2022   2022       sqlite3TableLock(pParse, iDbSrc, pSrc->tnum, 0, pSrc->zName);
  2023   2023     }

Changes to src/main.c.

  1578   1578   */
  1579   1579   int sqlite3CreateFunc(
  1580   1580     sqlite3 *db,
  1581   1581     const char *zFunctionName,
  1582   1582     int nArg,
  1583   1583     int enc,
  1584   1584     void *pUserData,
  1585         -  void (*xFunc)(sqlite3_context*,int,sqlite3_value **),
         1585  +  void (*xSFunc)(sqlite3_context*,int,sqlite3_value **),
  1586   1586     void (*xStep)(sqlite3_context*,int,sqlite3_value **),
  1587   1587     void (*xFinal)(sqlite3_context*),
  1588   1588     FuncDestructor *pDestructor
  1589   1589   ){
  1590   1590     FuncDef *p;
  1591   1591     int nName;
  1592   1592     int extraFlags;
  1593   1593   
  1594   1594     assert( sqlite3_mutex_held(db->mutex) );
  1595   1595     if( zFunctionName==0 ||
  1596         -      (xFunc && (xFinal || xStep)) || 
  1597         -      (!xFunc && (xFinal && !xStep)) ||
  1598         -      (!xFunc && (!xFinal && xStep)) ||
         1596  +      (xSFunc && (xFinal || xStep)) || 
         1597  +      (!xSFunc && (xFinal && !xStep)) ||
         1598  +      (!xSFunc && (!xFinal && xStep)) ||
  1599   1599         (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) ||
  1600   1600         (255<(nName = sqlite3Strlen30( zFunctionName))) ){
  1601   1601       return SQLITE_MISUSE_BKPT;
  1602   1602     }
  1603   1603   
  1604   1604     assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC );
  1605   1605     extraFlags = enc &  SQLITE_DETERMINISTIC;
................................................................................
  1614   1614     ** to the hash table.
  1615   1615     */
  1616   1616     if( enc==SQLITE_UTF16 ){
  1617   1617       enc = SQLITE_UTF16NATIVE;
  1618   1618     }else if( enc==SQLITE_ANY ){
  1619   1619       int rc;
  1620   1620       rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8|extraFlags,
  1621         -         pUserData, xFunc, xStep, xFinal, pDestructor);
         1621  +         pUserData, xSFunc, xStep, xFinal, pDestructor);
  1622   1622       if( rc==SQLITE_OK ){
  1623   1623         rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE|extraFlags,
  1624         -          pUserData, xFunc, xStep, xFinal, pDestructor);
         1624  +          pUserData, xSFunc, xStep, xFinal, pDestructor);
  1625   1625       }
  1626   1626       if( rc!=SQLITE_OK ){
  1627   1627         return rc;
  1628   1628       }
  1629   1629       enc = SQLITE_UTF16BE;
  1630   1630     }
  1631   1631   #else
................................................................................
  1661   1661   
  1662   1662     if( pDestructor ){
  1663   1663       pDestructor->nRef++;
  1664   1664     }
  1665   1665     p->pDestructor = pDestructor;
  1666   1666     p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags;
  1667   1667     testcase( p->funcFlags & SQLITE_DETERMINISTIC );
  1668         -  p->xFunc = xFunc;
  1669         -  p->xStep = xStep;
         1668  +  p->xSFunc = xSFunc ? xSFunc : xStep;
  1670   1669     p->xFinalize = xFinal;
  1671   1670     p->pUserData = pUserData;
  1672   1671     p->nArg = (u16)nArg;
  1673   1672     return SQLITE_OK;
  1674   1673   }
  1675   1674   
  1676   1675   /*
................................................................................
  1678   1677   */
  1679   1678   int sqlite3_create_function(
  1680   1679     sqlite3 *db,
  1681   1680     const char *zFunc,
  1682   1681     int nArg,
  1683   1682     int enc,
  1684   1683     void *p,
  1685         -  void (*xFunc)(sqlite3_context*,int,sqlite3_value **),
         1684  +  void (*xSFunc)(sqlite3_context*,int,sqlite3_value **),
  1686   1685     void (*xStep)(sqlite3_context*,int,sqlite3_value **),
  1687   1686     void (*xFinal)(sqlite3_context*)
  1688   1687   ){
  1689         -  return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xFunc, xStep,
         1688  +  return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xSFunc, xStep,
  1690   1689                                       xFinal, 0);
  1691   1690   }
  1692   1691   
  1693   1692   int sqlite3_create_function_v2(
  1694   1693     sqlite3 *db,
  1695   1694     const char *zFunc,
  1696   1695     int nArg,
  1697   1696     int enc,
  1698   1697     void *p,
  1699         -  void (*xFunc)(sqlite3_context*,int,sqlite3_value **),
         1698  +  void (*xSFunc)(sqlite3_context*,int,sqlite3_value **),
  1700   1699     void (*xStep)(sqlite3_context*,int,sqlite3_value **),
  1701   1700     void (*xFinal)(sqlite3_context*),
  1702   1701     void (*xDestroy)(void *)
  1703   1702   ){
  1704   1703     int rc = SQLITE_ERROR;
  1705   1704     FuncDestructor *pArg = 0;
  1706   1705   
................................................................................
  1715   1714       if( !pArg ){
  1716   1715         xDestroy(p);
  1717   1716         goto out;
  1718   1717       }
  1719   1718       pArg->xDestroy = xDestroy;
  1720   1719       pArg->pUserData = p;
  1721   1720     }
  1722         -  rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xFunc, xStep, xFinal, pArg);
         1721  +  rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xSFunc, xStep, xFinal, pArg);
  1723   1722     if( pArg && pArg->nRef==0 ){
  1724   1723       assert( rc!=SQLITE_OK );
  1725   1724       xDestroy(p);
  1726   1725       sqlite3DbFree(db, pArg);
  1727   1726     }
  1728   1727   
  1729   1728    out:
................................................................................
  1735   1734   #ifndef SQLITE_OMIT_UTF16
  1736   1735   int sqlite3_create_function16(
  1737   1736     sqlite3 *db,
  1738   1737     const void *zFunctionName,
  1739   1738     int nArg,
  1740   1739     int eTextRep,
  1741   1740     void *p,
  1742         -  void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
         1741  +  void (*xSFunc)(sqlite3_context*,int,sqlite3_value**),
  1743   1742     void (*xStep)(sqlite3_context*,int,sqlite3_value**),
  1744   1743     void (*xFinal)(sqlite3_context*)
  1745   1744   ){
  1746   1745     int rc;
  1747   1746     char *zFunc8;
  1748   1747   
  1749   1748   #ifdef SQLITE_ENABLE_API_ARMOR
  1750   1749     if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT;
  1751   1750   #endif
  1752   1751     sqlite3_mutex_enter(db->mutex);
  1753   1752     assert( !db->mallocFailed );
  1754   1753     zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE);
  1755         -  rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0);
         1754  +  rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xSFunc,xStep,xFinal,0);
  1756   1755     sqlite3DbFree(db, zFunc8);
  1757   1756     rc = sqlite3ApiExit(db, rc);
  1758   1757     sqlite3_mutex_leave(db->mutex);
  1759   1758     return rc;
  1760   1759   }
  1761   1760   #endif
  1762   1761   
................................................................................
  3062   3061     /* Enable the lookaside-malloc subsystem */
  3063   3062     setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside,
  3064   3063                           sqlite3GlobalConfig.nLookaside);
  3065   3064   
  3066   3065     sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT);
  3067   3066   
  3068   3067   opendb_out:
  3069         -  sqlite3_free(zOpen);
  3070   3068     if( db ){
  3071   3069       assert( db->mutex!=0 || isThreadsafe==0
  3072   3070              || sqlite3GlobalConfig.bFullMutex==0 );
  3073   3071       sqlite3_mutex_leave(db->mutex);
  3074   3072     }
  3075   3073     rc = sqlite3_errcode(db);
  3076   3074     assert( db!=0 || rc==SQLITE_NOMEM );
................................................................................
  3122   3120           iByte = (iByte<<4) + sqlite3HexToInt(zHexKey[i]);
  3123   3121           if( (i&1)!=0 ) zKey[i/2] = iByte;
  3124   3122         }
  3125   3123         sqlite3_key_v2(db, 0, zKey, i/2);
  3126   3124       }
  3127   3125     }
  3128   3126   #endif
         3127  +  sqlite3_free(zOpen);
  3129   3128     return rc & 0xff;
  3130   3129   }
  3131   3130   
  3132   3131   /*
  3133   3132   ** Open a new database handle.
  3134   3133   */
  3135   3134   int sqlite3_open(

Changes to src/malloc.c.

   579    579     if( p ){
   580    580       memset(p, 0, (size_t)n);
   581    581     }
   582    582     return p;
   583    583   }
   584    584   
   585    585   /*
   586         -** Allocate and zero memory.  If the allocation fails, make
   587         -** the mallocFailed flag in the connection pointer.
          586  +** Allocate memory, either lookaside (if possible) or heap.  
          587  +** If the allocation fails, set the mallocFailed flag in
          588  +** the connection pointer.
   588    589   **
   589    590   ** If db!=0 and db->mallocFailed is true (indicating a prior malloc
   590    591   ** failure on the same database connection) then always return 0.
   591    592   ** Hence for a particular database connection, once malloc starts
   592    593   ** failing, it fails consistently until mallocFailed is reset.
   593    594   ** This is an important assumption.  There are many places in the
   594    595   ** code that do things like this:
................................................................................
   596    597   **         int *a = (int*)sqlite3DbMallocRaw(db, 100);
   597    598   **         int *b = (int*)sqlite3DbMallocRaw(db, 200);
   598    599   **         if( b ) a[10] = 9;
   599    600   **
   600    601   ** In other words, if a subsequent malloc (ex: "b") worked, it is assumed
   601    602   ** that all prior mallocs (ex: "a") worked too.
   602    603   */
          604  +static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n);
   603    605   void *sqlite3DbMallocRaw(sqlite3 *db, u64 n){
   604         -  void *p;
   605    606     assert( db==0 || sqlite3_mutex_held(db->mutex) );
   606    607     assert( db==0 || db->pnBytesFreed==0 );
   607    608   #ifndef SQLITE_OMIT_LOOKASIDE
   608    609     if( db ){
   609    610       LookasideSlot *pBuf;
   610    611       if( db->mallocFailed ){
   611    612         return 0;
................................................................................
   627    628       }
   628    629     }
   629    630   #else
   630    631     if( db && db->mallocFailed ){
   631    632       return 0;
   632    633     }
   633    634   #endif
   634         -  p = sqlite3Malloc(n);
          635  +  return dbMallocRawFinish(db, n);
          636  +}
          637  +static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n){
          638  +  void *p = sqlite3Malloc(n);
   635    639     if( !p && db ){
   636    640       db->mallocFailed = 1;
   637    641     }
   638    642     sqlite3MemdebugSetType(p, 
   639    643            (db && db->lookaside.bEnabled) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP);
   640    644     return p;
   641    645   }

Changes to src/pragma.c.

   426    426         { OP_Integer,     0, 2,        0},
   427    427         { OP_Subtract,    1, 2,        1},
   428    428         { OP_IfPos,       1, 8,        0},
   429    429         { OP_Integer,     0, 1,        0},                         /* 6 */
   430    430         { OP_Noop,        0, 0,        0},
   431    431         { OP_ResultRow,   1, 1,        0},
   432    432       };
   433         -    int addr;
          433  +    VdbeOp *aOp;
   434    434       sqlite3VdbeUsesBtree(v, iDb);
   435    435       if( !zRight ){
   436    436         setOneColumnName(v, "cache_size");
   437    437         pParse->nMem += 2;
   438         -      addr = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize,iLn);
   439         -      sqlite3VdbeChangeP1(v, addr, iDb);
   440         -      sqlite3VdbeChangeP1(v, addr+1, iDb);
   441         -      sqlite3VdbeChangeP1(v, addr+6, SQLITE_DEFAULT_CACHE_SIZE);
          438  +      sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(getCacheSize));
          439  +      aOp = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize, iLn);
          440  +      if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break;
          441  +      aOp[0].p1 = iDb;
          442  +      aOp[1].p1 = iDb;
          443  +      aOp[6].p1 = SQLITE_DEFAULT_CACHE_SIZE;
   442    444       }else{
   443    445         int size = sqlite3AbsInt32(sqlite3Atoi(zRight));
   444    446         sqlite3BeginWriteOperation(pParse, 0, iDb);
   445    447         sqlite3VdbeAddOp2(v, OP_Integer, size, 1);
   446    448         sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_DEFAULT_CACHE_SIZE, 1);
   447    449         assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
   448    450         pDb->pSchema->cache_size = size;
................................................................................
   694    696             { OP_Transaction,    0,         1,                 0},    /* 0 */
   695    697             { OP_ReadCookie,     0,         1,         BTREE_LARGEST_ROOT_PAGE},
   696    698             { OP_If,             1,         0,                 0},    /* 2 */
   697    699             { OP_Halt,           SQLITE_OK, OE_Abort,          0},    /* 3 */
   698    700             { OP_Integer,        0,         1,                 0},    /* 4 */
   699    701             { OP_SetCookie,      0,         BTREE_INCR_VACUUM, 1},    /* 5 */
   700    702           };
   701         -        int iAddr;
   702         -        iAddr = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6, iLn);
   703         -        sqlite3VdbeChangeP1(v, iAddr, iDb);
   704         -        sqlite3VdbeChangeP1(v, iAddr+1, iDb);
   705         -        sqlite3VdbeChangeP2(v, iAddr+2, iAddr+4);
   706         -        sqlite3VdbeChangeP1(v, iAddr+4, eAuto-1);
   707         -        sqlite3VdbeChangeP1(v, iAddr+5, iDb);
          703  +        VdbeOp *aOp;
          704  +        int iAddr = sqlite3VdbeCurrentAddr(v);
          705  +        sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(setMeta6));
          706  +        aOp = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6, iLn);
          707  +        if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break;
          708  +        aOp[0].p1 = iDb;
          709  +        aOp[1].p1 = iDb;
          710  +        aOp[2].p2 = iAddr+4;
          711  +        aOp[4].p1 = eAuto - 1;
          712  +        aOp[5].p1 = iDb;
   708    713           sqlite3VdbeUsesBtree(v, iDb);
   709    714         }
   710    715       }
   711    716       break;
   712    717     }
   713    718   #endif
   714    719   
................................................................................
  1409   1414     /* Pragma "quick_check" is reduced version of 
  1410   1415     ** integrity_check designed to detect most database corruption
  1411   1416     ** without most of the overhead of a full integrity-check.
  1412   1417     */
  1413   1418     case PragTyp_INTEGRITY_CHECK: {
  1414   1419       int i, j, addr, mxErr;
  1415   1420   
  1416         -    /* Code that appears at the end of the integrity check.  If no error
  1417         -    ** messages have been generated, output OK.  Otherwise output the
  1418         -    ** error message
  1419         -    */
  1420         -    static const int iLn = VDBE_OFFSET_LINENO(2);
  1421         -    static const VdbeOpList endCode[] = {
  1422         -      { OP_AddImm,      1, 0,        0},    /* 0 */
  1423         -      { OP_If,          1, 0,        0},    /* 1 */
  1424         -      { OP_String8,     0, 3,        0},    /* 2 */
  1425         -      { OP_ResultRow,   3, 1,        0},
  1426         -    };
  1427         -
  1428   1421       int isQuick = (sqlite3Tolower(zLeft[0])=='q');
  1429   1422   
  1430   1423       /* If the PRAGMA command was of the form "PRAGMA <db>.integrity_check",
  1431   1424       ** then iDb is set to the index of the database identified by <db>.
  1432   1425       ** In this case, the integrity of database iDb only is verified by
  1433   1426       ** the VDBE created below.
  1434   1427       **
................................................................................
  1617   1610             sqlite3VdbeLoadString(v, 3, pIdx->zName);
  1618   1611             sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7);
  1619   1612             sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1);
  1620   1613           }
  1621   1614   #endif /* SQLITE_OMIT_BTREECOUNT */
  1622   1615         } 
  1623   1616       }
  1624         -    addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn);
  1625         -    sqlite3VdbeChangeP2(v, addr, -mxErr);
  1626         -    sqlite3VdbeJumpHere(v, addr+1);
  1627         -    sqlite3VdbeChangeP4(v, addr+2, "ok", P4_STATIC);
         1617  +    {
         1618  +      static const int iLn = VDBE_OFFSET_LINENO(2);
         1619  +      static const VdbeOpList endCode[] = {
         1620  +        { OP_AddImm,      1, 0,        0},    /* 0 */
         1621  +        { OP_If,          1, 0,        0},    /* 1 */
         1622  +        { OP_String8,     0, 3,        0},    /* 2 */
         1623  +        { OP_ResultRow,   3, 1,        0},
         1624  +      };
         1625  +      VdbeOp *aOp;
         1626  +
         1627  +      aOp = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn);
         1628  +      if( aOp ){
         1629  +        aOp[0].p2 = -mxErr;
         1630  +        aOp[1].p2 = sqlite3VdbeCurrentAddr(v);
         1631  +        aOp[2].p4type = P4_STATIC;
         1632  +        aOp[2].p4.z = "ok";
         1633  +      }
         1634  +    }
  1628   1635     }
  1629   1636     break;
  1630   1637   #endif /* SQLITE_OMIT_INTEGRITY_CHECK */
  1631   1638   
  1632   1639   #ifndef SQLITE_OMIT_UTF16
  1633   1640     /*
  1634   1641     **   PRAGMA encoding
................................................................................
  1737   1744       if( zRight && (pPragma->mPragFlag & PragFlag_ReadOnly)==0 ){
  1738   1745         /* Write the specified cookie value */
  1739   1746         static const VdbeOpList setCookie[] = {
  1740   1747           { OP_Transaction,    0,  1,  0},    /* 0 */
  1741   1748           { OP_Integer,        0,  1,  0},    /* 1 */
  1742   1749           { OP_SetCookie,      0,  0,  1},    /* 2 */
  1743   1750         };
  1744         -      int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie, 0);
  1745         -      sqlite3VdbeChangeP1(v, addr, iDb);
  1746         -      sqlite3VdbeChangeP1(v, addr+1, sqlite3Atoi(zRight));
  1747         -      sqlite3VdbeChangeP1(v, addr+2, iDb);
  1748         -      sqlite3VdbeChangeP2(v, addr+2, iCookie);
         1751  +      VdbeOp *aOp;
         1752  +      sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(setCookie));
         1753  +      aOp = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie, 0);
         1754  +      if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break;
         1755  +      aOp[0].p1 = iDb;
         1756  +      aOp[1].p1 = sqlite3Atoi(zRight);
         1757  +      aOp[2].p1 = iDb;
         1758  +      aOp[2].p2 = iCookie;
  1749   1759       }else{
  1750   1760         /* Read the specified cookie value */
  1751   1761         static const VdbeOpList readCookie[] = {
  1752   1762           { OP_Transaction,     0,  0,  0},    /* 0 */
  1753   1763           { OP_ReadCookie,      0,  1,  0},    /* 1 */
  1754   1764           { OP_ResultRow,       1,  1,  0}
  1755   1765         };
  1756         -      int addr = sqlite3VdbeAddOpList(v, ArraySize(readCookie), readCookie, 0);
  1757         -      sqlite3VdbeChangeP1(v, addr, iDb);
  1758         -      sqlite3VdbeChangeP1(v, addr+1, iDb);
  1759         -      sqlite3VdbeChangeP3(v, addr+1, iCookie);
         1766  +      VdbeOp *aOp;
         1767  +      sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(readCookie));
         1768  +      aOp = sqlite3VdbeAddOpList(v, ArraySize(readCookie),readCookie,0);
         1769  +      if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break;
         1770  +      aOp[0].p1 = iDb;
         1771  +      aOp[1].p1 = iDb;
         1772  +      aOp[1].p3 = iCookie;
  1760   1773         sqlite3VdbeSetNumCols(v, 1);
  1761   1774         sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT);
  1762   1775       }
  1763   1776     }
  1764   1777     break;
  1765   1778   #endif /* SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS */
  1766   1779   

Changes to src/prepare.c.

   136    136   */
   137    137   static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){
   138    138     int rc;
   139    139     int i;
   140    140   #ifndef SQLITE_OMIT_DEPRECATED
   141    141     int size;
   142    142   #endif
   143         -  Table *pTab;
   144    143     Db *pDb;
   145    144     char const *azArg[4];
   146    145     int meta[5];
   147    146     InitData initData;
   148         -  char const *zMasterSchema;
   149         -  char const *zMasterName;
          147  +  const char *zMasterName;
   150    148     int openedTransaction = 0;
   151    149   
   152         -  /*
   153         -  ** The master database table has a structure like this
   154         -  */
   155         -  static const char master_schema[] = 
   156         -     "CREATE TABLE sqlite_master(\n"
   157         -     "  type text,\n"
   158         -     "  name text,\n"
   159         -     "  tbl_name text,\n"
   160         -     "  rootpage integer,\n"
   161         -     "  sql text\n"
   162         -     ")"
   163         -  ;
   164         -#ifndef SQLITE_OMIT_TEMPDB
   165         -  static const char temp_master_schema[] = 
   166         -     "CREATE TEMP TABLE sqlite_temp_master(\n"
   167         -     "  type text,\n"
   168         -     "  name text,\n"
   169         -     "  tbl_name text,\n"
   170         -     "  rootpage integer,\n"
   171         -     "  sql text\n"
   172         -     ")"
   173         -  ;
   174         -#else
   175         -  #define temp_master_schema 0
   176         -#endif
   177         -
   178    150     assert( iDb>=0 && iDb<db->nDb );
   179    151     assert( db->aDb[iDb].pSchema );
   180    152     assert( sqlite3_mutex_held(db->mutex) );
   181    153     assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) );
   182    154   
   183         -  /* zMasterSchema and zInitScript are set to point at the master schema
   184         -  ** and initialisation script appropriate for the database being
   185         -  ** initialized. zMasterName is the name of the master table.
   186         -  */
   187         -  if( !OMIT_TEMPDB && iDb==1 ){
   188         -    zMasterSchema = temp_master_schema;
   189         -  }else{
   190         -    zMasterSchema = master_schema;
   191         -  }
   192         -  zMasterName = SCHEMA_TABLE(iDb);
   193         -
   194         -  /* Construct the schema tables.  */
   195         -  azArg[0] = zMasterName;
          155  +  /* Construct the in-memory representation schema tables (sqlite_master or
          156  +  ** sqlite_temp_master) by invoking the parser directly.  The appropriate
          157  +  ** table name will be inserted automatically by the parser so we can just
          158  +  ** use the abbreviation "x" here.  The parser will also automatically tag
          159  +  ** the schema table as read-only. */
          160  +  azArg[0] = zMasterName = SCHEMA_TABLE(iDb);
   196    161     azArg[1] = "1";
   197         -  azArg[2] = zMasterSchema;
          162  +  azArg[2] = "CREATE TABLE x(type text,name text,tbl_name text,"
          163  +                            "rootpage integer,sql text)";
   198    164     azArg[3] = 0;
   199    165     initData.db = db;
   200    166     initData.iDb = iDb;
   201    167     initData.rc = SQLITE_OK;
   202    168     initData.pzErrMsg = pzErrMsg;
   203    169     sqlite3InitCallback(&initData, 3, (char **)azArg, 0);
   204    170     if( initData.rc ){
   205    171       rc = initData.rc;
   206    172       goto error_out;
   207    173     }
   208         -  pTab = sqlite3FindTable(db, zMasterName, db->aDb[iDb].zName);
   209         -  if( ALWAYS(pTab) ){
   210         -    pTab->tabFlags |= TF_Readonly;
   211         -  }
   212    174   
   213    175     /* Create a cursor to hold the database open
   214    176     */
   215    177     pDb = &db->aDb[iDb];
   216    178     if( pDb->pBt==0 ){
   217    179       if( !OMIT_TEMPDB && ALWAYS(iDb==1) ){
   218    180         DbSetProperty(db, 1, DB_SchemaLoaded);
................................................................................
   323    285   
   324    286     /* Read the schema information out of the schema tables
   325    287     */
   326    288     assert( db->init.busy );
   327    289     {
   328    290       char *zSql;
   329    291       zSql = sqlite3MPrintf(db, 
   330         -        "SELECT name, rootpage, sql FROM '%q'.%s ORDER BY rowid",
          292  +        "SELECT name, rootpage, sql FROM \"%w\".%s ORDER BY rowid",
   331    293           db->aDb[iDb].zName, zMasterName);
   332    294   #ifndef SQLITE_OMIT_AUTHORIZATION
   333    295       {
   334    296         sqlite3_xauth xAuth;
   335    297         xAuth = db->xAuth;
   336    298         db->xAuth = 0;
   337    299   #endif

Changes to src/resolve.c.

   661    661           pDef = sqlite3FindFunction(pParse->db, zId, nId, -2, enc, 0);
   662    662           if( pDef==0 ){
   663    663             no_such_func = 1;
   664    664           }else{
   665    665             wrong_num_args = 1;
   666    666           }
   667    667         }else{
   668         -        is_agg = pDef->xFunc==0;
          668  +        is_agg = pDef->xFinalize!=0;
   669    669           if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){
   670    670             ExprSetProperty(pExpr, EP_Unlikely|EP_Skip);
   671    671             if( n==2 ){
   672    672               pExpr->iTable = exprProbability(pList->a[1].pExpr);
   673    673               if( pExpr->iTable<0 ){
   674    674                 sqlite3ErrorMsg(pParse,
   675    675                   "second argument to likelihood() must be a "
................................................................................
  1389   1389         return 1;
  1390   1390       }
  1391   1391       pParse->nHeight += pExpr->nHeight;
  1392   1392     }
  1393   1393   #endif
  1394   1394     savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg);
  1395   1395     pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg);
  1396         -  memset(&w, 0, sizeof(w));
         1396  +  w.pParse = pNC->pParse;
  1397   1397     w.xExprCallback = resolveExprStep;
  1398   1398     w.xSelectCallback = resolveSelectStep;
  1399         -  w.pParse = pNC->pParse;
         1399  +  w.xSelectCallback2 = 0;
         1400  +  w.walkerDepth = 0;
         1401  +  w.eCode = 0;
  1400   1402     w.u.pNC = pNC;
  1401   1403     sqlite3WalkExpr(&w, pExpr);
  1402   1404   #if SQLITE_MAX_EXPR_DEPTH>0
  1403   1405     pNC->pParse->nHeight -= pExpr->nHeight;
  1404   1406   #endif
  1405   1407     if( pNC->nErr>0 || w.pParse->nErr>0 ){
  1406   1408       ExprSetProperty(pExpr, EP_Error);

Changes to src/select.c.

  1001   1001   }
  1002   1002   
  1003   1003   /*
  1004   1004   ** Allocate a KeyInfo object sufficient for an index of N key columns and
  1005   1005   ** X extra columns.
  1006   1006   */
  1007   1007   KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){
  1008         -  KeyInfo *p = sqlite3DbMallocZero(0, 
  1009         -                   sizeof(KeyInfo) + (N+X)*(sizeof(CollSeq*)+1));
         1008  +  int nExtra = (N+X)*(sizeof(CollSeq*)+1);
         1009  +  KeyInfo *p = sqlite3Malloc(sizeof(KeyInfo) + nExtra);
  1010   1010     if( p ){
  1011   1011       p->aSortOrder = (u8*)&p->aColl[N+X];
  1012   1012       p->nField = (u16)N;
  1013   1013       p->nXField = (u16)X;
  1014   1014       p->enc = ENC(db);
  1015   1015       p->db = db;
  1016   1016       p->nRef = 1;
         1017  +    memset(&p[1], 0, nExtra);
  1017   1018     }else{
  1018   1019       db->mallocFailed = 1;
  1019   1020     }
  1020   1021     return p;
  1021   1022   }
  1022   1023   
  1023   1024   /*

Changes to src/sqliteInt.h.

   398    398   # define ALWAYS(X)      ((X)?1:(assert(0),0))
   399    399   # define NEVER(X)       ((X)?(assert(0),1):0)
   400    400   #else
   401    401   # define ALWAYS(X)      (X)
   402    402   # define NEVER(X)       (X)
   403    403   #endif
   404    404   
          405  +/*
          406  +** Some malloc failures are only possible if SQLITE_TEST_REALLOC_STRESS is
          407  +** defined.  We need to defend against those failures when testing with
          408  +** SQLITE_TEST_REALLOC_STRESS, but we don't want the unreachable branches
          409  +** during a normal build.  The following macro can be used to disable tests
          410  +** that are always false except when SQLITE_TEST_REALLOC_STRESS is set.
          411  +*/
          412  +#if defined(SQLITE_TEST_REALLOC_STRESS)
          413  +# define ONLY_IF_REALLOC_STRESS(X)  (X)
          414  +#elif !defined(NDEBUG)
          415  +# define ONLY_IF_REALLOC_STRESS(X)  ((X)?(assert(0),1):0)
          416  +#else
          417  +# define ONLY_IF_REALLOC_STRESS(X)  (0)
          418  +#endif
          419  +
   405    420   /*
   406    421   ** Declarations used for tracing the operating system interfaces.
   407    422   */
   408    423   #if defined(SQLITE_FORCE_OS_TRACE) || defined(SQLITE_TEST) || \
   409    424       (defined(SQLITE_DEBUG) && SQLITE_OS_WIN)
   410    425     extern int sqlite3OSTrace;
   411    426   # define OSTRACE(X)          if( sqlite3OSTrace ) sqlite3DebugPrintf X
................................................................................
  1369   1384   ** points to a linked list of these structures.
  1370   1385   */
  1371   1386   struct FuncDef {
  1372   1387     i16 nArg;            /* Number of arguments.  -1 means unlimited */
  1373   1388     u16 funcFlags;       /* Some combination of SQLITE_FUNC_* */
  1374   1389     void *pUserData;     /* User data parameter */
  1375   1390     FuncDef *pNext;      /* Next function with same name */
  1376         -  void (*xFunc)(sqlite3_context*,int,sqlite3_value**); /* Regular function */
  1377         -  void (*xStep)(sqlite3_context*,int,sqlite3_value**); /* Aggregate step */
  1378         -  void (*xFinalize)(sqlite3_context*);                /* Aggregate finalizer */
         1391  +  void (*xSFunc)(sqlite3_context*,int,sqlite3_value**); /* func or agg-step */
         1392  +  void (*xFinalize)(sqlite3_context*);                  /* Agg finalizer */
  1379   1393     char *zName;         /* SQL name of the function. */
  1380   1394     FuncDef *pHash;      /* Next with a different name but the same hash */
  1381   1395     FuncDestructor *pDestructor;   /* Reference counted destructor function */
  1382   1396   };
  1383   1397   
  1384   1398   /*
  1385   1399   ** This structure encapsulates a user-function destructor callback (as
................................................................................
  1454   1468   **     function likeFunc. Argument pArg is cast to a (void *) and made
  1455   1469   **     available as the function user-data (sqlite3_user_data()). The
  1456   1470   **     FuncDef.flags variable is set to the value passed as the flags
  1457   1471   **     parameter.
  1458   1472   */
  1459   1473   #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \
  1460   1474     {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
  1461         -   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0}
         1475  +   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, 0, 0}
  1462   1476   #define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \
  1463   1477     {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
  1464         -   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0}
         1478  +   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, 0, 0}
  1465   1479   #define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \
  1466   1480     {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
  1467         -   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0}
         1481  +   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, 0, 0}
  1468   1482   #define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \
  1469   1483     {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\
  1470         -   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0}
         1484  +   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, 0, 0}
  1471   1485   #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \
  1472   1486     {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
  1473         -   pArg, 0, xFunc, 0, 0, #zName, 0, 0}
         1487  +   pArg, 0, xFunc, 0, #zName, 0, 0}
  1474   1488   #define LIKEFUNC(zName, nArg, arg, flags) \
  1475   1489     {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \
  1476         -   (void *)arg, 0, likeFunc, 0, 0, #zName, 0, 0}
         1490  +   (void *)arg, 0, likeFunc, 0, #zName, 0, 0}
  1477   1491   #define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \
  1478   1492     {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL), \
  1479         -   SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0}
         1493  +   SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName,0,0}
  1480   1494   #define AGGREGATE2(zName, nArg, arg, nc, xStep, xFinal, extraFlags) \
  1481   1495     {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|extraFlags, \
  1482         -   SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0}
         1496  +   SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName,0,0}
  1483   1497   
  1484   1498   /*
  1485   1499   ** All current savepoints are stored in a linked list starting at
  1486   1500   ** sqlite3.pSavepoint. The first element in the list is the most recently
  1487   1501   ** opened savepoint. Savepoints are added to the list by the vdbe
  1488   1502   ** OP_Savepoint instruction.
  1489   1503   */
................................................................................
  2786   2800     /************************************************************************
  2787   2801     ** Above is constant between recursions.  Below is reset before and after
  2788   2802     ** each recursion.  The boundary between these two regions is determined
  2789   2803     ** using offsetof(Parse,nVar) so the nVar field must be the first field
  2790   2804     ** in the recursive region.
  2791   2805     ************************************************************************/
  2792   2806   
  2793         -  int nVar;                 /* Number of '?' variables seen in the SQL so far */
         2807  +  ynVar nVar;               /* Number of '?' variables seen in the SQL so far */
  2794   2808     int nzVar;                /* Number of available slots in azVar[] */
  2795   2809     u8 iPkSortOrder;          /* ASC or DESC for INTEGER PRIMARY KEY */
  2796   2810     u8 explain;               /* True if the EXPLAIN flag is found on the query */
  2797   2811   #ifndef SQLITE_OMIT_VIRTUALTABLE
  2798   2812     u8 declareVtab;           /* True if inside sqlite3_declare_vtab() */
  2799   2813     int nVtabLock;            /* Number of virtual tables to lock */
  2800   2814   #endif
................................................................................
  3068   3082   */
  3069   3083   #define CORRUPT_DB  (sqlite3Config.neverCorrupt==0)
  3070   3084   
  3071   3085   /*
  3072   3086   ** Context pointer passed down through the tree-walk.
  3073   3087   */
  3074   3088   struct Walker {
         3089  +  Parse *pParse;                            /* Parser context.  */
  3075   3090     int (*xExprCallback)(Walker*, Expr*);     /* Callback for expressions */
  3076   3091     int (*xSelectCallback)(Walker*,Select*);  /* Callback for SELECTs */
  3077   3092     void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */
  3078         -  Parse *pParse;                            /* Parser context.  */
  3079   3093     int walkerDepth;                          /* Number of subqueries */
  3080   3094     u8 eCode;                                 /* A small processing code */
  3081   3095     union {                                   /* Extra data for callback */
  3082   3096       NameContext *pNC;                          /* Naming context */
  3083   3097       int n;                                     /* A counter */
  3084   3098       int iCur;                                  /* A cursor number */
  3085   3099       SrcList *pSrcList;                         /* FROM clause */

Changes to src/trigger.c.

   948    948     assert( pPrg || pParse->nErr || pParse->db->mallocFailed );
   949    949   
   950    950     /* Code the OP_Program opcode in the parent VDBE. P4 of the OP_Program 
   951    951     ** is a pointer to the sub-vdbe containing the trigger program.  */
   952    952     if( pPrg ){
   953    953       int bRecursive = (p->zName && 0==(pParse->db->flags&SQLITE_RecTriggers));
   954    954   
   955         -    sqlite3VdbeAddOp3(v, OP_Program, reg, ignoreJump, ++pParse->nMem);
   956         -    sqlite3VdbeChangeP4(v, -1, (const char *)pPrg->pProgram, P4_SUBPROGRAM);
          955  +    sqlite3VdbeAddOp4(v, OP_Program, reg, ignoreJump, ++pParse->nMem,
          956  +                      (const char *)pPrg->pProgram, P4_SUBPROGRAM);
   957    957       VdbeComment(
   958    958           (v, "Call: %s.%s", (p->zName?p->zName:"fkey"), onErrorText(orconf)));
   959    959   
   960    960       /* Set the P5 operand of the OP_Program instruction to non-zero if
   961    961       ** recursive invocation of this trigger program is disallowed. Recursive
   962    962       ** invocation is disallowed if (a) the sub-program is really a trigger,
   963    963       ** not a foreign key action, and (b) the flag to enable recursive triggers

Changes to src/vdbe.c.

  1659   1659       assert( memIsValid(pCtx->argv[i]) );
  1660   1660       REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]);
  1661   1661     }
  1662   1662   #endif
  1663   1663     MemSetTypeFlag(pCtx->pOut, MEM_Null);
  1664   1664     pCtx->fErrorOrAux = 0;
  1665   1665     db->lastRowid = lastRowid;
  1666         -  (*pCtx->pFunc->xFunc)(pCtx, pCtx->argc, pCtx->argv); /* IMP: R-24505-23230 */
  1667         -  lastRowid = db->lastRowid;  /* Remember rowid changes made by xFunc */
         1666  +  (*pCtx->pFunc->xSFunc)(pCtx, pCtx->argc, pCtx->argv);/* IMP: R-24505-23230 */
         1667  +  lastRowid = db->lastRowid;  /* Remember rowid changes made by xSFunc */
  1668   1668   
  1669   1669     /* If the function returned an error, throw an exception */
  1670   1670     if( pCtx->fErrorOrAux ){
  1671   1671       if( pCtx->isError ){
  1672   1672         sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut));
  1673   1673         rc = pCtx->isError;
  1674   1674       }
................................................................................
  5096   5096   ** See also: Clear
  5097   5097   */
  5098   5098   case OP_Destroy: {     /* out2 */
  5099   5099     int iMoved;
  5100   5100     int iDb;
  5101   5101   
  5102   5102     assert( p->readOnly==0 );
         5103  +  assert( pOp->p1>1 );
  5103   5104     pOut = out2Prerelease(p, pOp);
  5104   5105     pOut->flags = MEM_Null;
  5105   5106     if( db->nVdbeRead > db->nVDestroy+1 ){
  5106   5107       rc = SQLITE_LOCKED;
  5107   5108       p->errorAction = OE_Abort;
  5108   5109     }else{
  5109   5110       iDb = pOp->p3;
................................................................................
  5900   5901   #endif
  5901   5902   
  5902   5903     pMem->n++;
  5903   5904     sqlite3VdbeMemInit(&t, db, MEM_Null);
  5904   5905     pCtx->pOut = &t;
  5905   5906     pCtx->fErrorOrAux = 0;
  5906   5907     pCtx->skipFlag = 0;
  5907         -  (pCtx->pFunc->xStep)(pCtx,pCtx->argc,pCtx->argv); /* IMP: R-24505-23230 */
         5908  +  (pCtx->pFunc->xSFunc)(pCtx,pCtx->argc,pCtx->argv); /* IMP: R-24505-23230 */
  5908   5909     if( pCtx->fErrorOrAux ){
  5909   5910       if( pCtx->isError ){
  5910   5911         sqlite3VdbeError(p, "%s", sqlite3_value_text(&t));
  5911   5912         rc = pCtx->isError;
  5912   5913       }
  5913   5914       sqlite3VdbeMemRelease(&t);
  5914   5915     }else{

Changes to src/vdbe.h.

   176    176   int sqlite3VdbeGoto(Vdbe*,int);
   177    177   int sqlite3VdbeLoadString(Vdbe*,int,const char*);
   178    178   void sqlite3VdbeMultiLoad(Vdbe*,int,const char*,...);
   179    179   int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int);
   180    180   int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int);
   181    181   int sqlite3VdbeAddOp4Dup8(Vdbe*,int,int,int,int,const u8*,int);
   182    182   int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int);
   183         -int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno);
          183  +#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS)
          184  +  void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N);
          185  +#else
          186  +# define sqlite3VdbeVerifyNoMallocRequired(A,B)
          187  +#endif
          188  +VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno);
   184    189   void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*);
   185    190   void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8);
   186    191   void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1);
   187    192   void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2);
   188    193   void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3);
   189    194   void sqlite3VdbeChangeP5(Vdbe*, u8 P5);
   190    195   void sqlite3VdbeJumpHere(Vdbe*, int addr);
   191         -void sqlite3VdbeChangeToNoop(Vdbe*, int addr);
          196  +int sqlite3VdbeChangeToNoop(Vdbe*, int addr);
   192    197   int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op);
   193    198   void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N);
   194    199   void sqlite3VdbeSetP4KeyInfo(Parse*, Index*);
   195    200   void sqlite3VdbeUsesBtree(Vdbe*, int);
   196    201   VdbeOp *sqlite3VdbeGetOp(Vdbe*, int);
   197    202   int sqlite3VdbeMakeLabel(Vdbe*);
   198    203   void sqlite3VdbeRunOnlyOnce(Vdbe*);

Changes to src/vdbeapi.c.

   791    791   
   792    792   /*
   793    793   ** Allocate or return the aggregate context for a user function.  A new
   794    794   ** context is allocated on the first call.  Subsequent calls return the
   795    795   ** same context that was returned on prior calls.
   796    796   */
   797    797   void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){
   798         -  assert( p && p->pFunc && p->pFunc->xStep );
          798  +  assert( p && p->pFunc && p->pFunc->xFinalize );
   799    799     assert( sqlite3_mutex_held(p->pOut->db->mutex) );
   800    800     testcase( nByte<0 );
   801    801     if( (p->pMem->flags & MEM_Agg)==0 ){
   802    802       return createAggContext(p, nByte);
   803    803     }else{
   804    804       return (void*)p->pMem->z;
   805    805     }
................................................................................
   882    882   **
   883    883   ** This function is deprecated.  Do not use it for new code.  It is
   884    884   ** provide only to avoid breaking legacy code.  New aggregate function
   885    885   ** implementations should keep their own counts within their aggregate
   886    886   ** context.
   887    887   */
   888    888   int sqlite3_aggregate_count(sqlite3_context *p){
   889         -  assert( p && p->pMem && p->pFunc && p->pFunc->xStep );
          889  +  assert( p && p->pMem && p->pFunc && p->pFunc->xFinalize );
   890    890     return p->pMem->n;
   891    891   }
   892    892   #endif
   893    893   
   894    894   /*
   895    895   ** Return the number of columns in the result set for the statement pStmt.
   896    896   */

Changes to src/vdbeaux.c.

   246    246     va_list ap;
   247    247     int i;
   248    248     char c;
   249    249     va_start(ap, zTypes);
   250    250     for(i=0; (c = zTypes[i])!=0; i++){
   251    251       if( c=='s' ){
   252    252         const char *z = va_arg(ap, const char*);
   253         -      int addr = sqlite3VdbeAddOp2(p, z==0 ? OP_Null : OP_String8, 0, iDest++);
   254         -      if( z ) sqlite3VdbeChangeP4(p, addr, z, 0);
          253  +      sqlite3VdbeAddOp4(p, z==0 ? OP_Null : OP_String8, 0, iDest++, 0, z, 0);
   255    254       }else{
   256    255         assert( c=='i' );
   257    256         sqlite3VdbeAddOp2(p, OP_Integer, va_arg(ap, int), iDest++);
   258    257       }
   259    258     }
   260    259     va_end(ap);
   261    260   }
................................................................................
   602    601   ** Return the address of the next instruction to be inserted.
   603    602   */
   604    603   int sqlite3VdbeCurrentAddr(Vdbe *p){
   605    604     assert( p->magic==VDBE_MAGIC_INIT );
   606    605     return p->nOp;
   607    606   }
   608    607   
          608  +/*
          609  +** Verify that at least N opcode slots are available in p without
          610  +** having to malloc for more space (except when compiled using
          611  +** SQLITE_TEST_REALLOC_STRESS).  This interface is used during testing
          612  +** to verify that certain calls to sqlite3VdbeAddOpList() can never
          613  +** fail due to a OOM fault and hence that the return value from
          614  +** sqlite3VdbeAddOpList() will always be non-NULL.
          615  +*/
          616  +#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS)
          617  +void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N){
          618  +  assert( p->nOp + N <= p->pParse->nOpAlloc );
          619  +}
          620  +#endif
          621  +
   609    622   /*
   610    623   ** This function returns a pointer to the array of opcodes associated with
   611    624   ** the Vdbe passed as the first argument. It is the callers responsibility
   612    625   ** to arrange for the returned array to be eventually freed using the 
   613    626   ** vdbeFreeOpArray() function.
   614    627   **
   615    628   ** Before returning, *pnOp is set to the number of entries in the returned
................................................................................
   627    640     resolveP2Values(p, pnMaxArg);
   628    641     *pnOp = p->nOp;
   629    642     p->aOp = 0;
   630    643     return aOp;
   631    644   }
   632    645   
   633    646   /*
   634         -** Add a whole list of operations to the operation stack.  Return the
   635         -** address of the first operation added.
          647  +** Add a whole list of operations to the operation stack.  Return a
          648  +** pointer to the first operation inserted.
   636    649   */
   637         -int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){
   638         -  int addr, i;
   639         -  VdbeOp *pOut;
          650  +VdbeOp *sqlite3VdbeAddOpList(
          651  +  Vdbe *p,                     /* Add opcodes to the prepared statement */
          652  +  int nOp,                     /* Number of opcodes to add */
          653  +  VdbeOpList const *aOp,       /* The opcodes to be added */
          654  +  int iLineno                  /* Source-file line number of first opcode */
          655  +){
          656  +  int i;
          657  +  VdbeOp *pOut, *pFirst;
   640    658     assert( nOp>0 );
   641    659     assert( p->magic==VDBE_MAGIC_INIT );
   642    660     if( p->nOp + nOp > p->pParse->nOpAlloc && growOpArray(p, nOp) ){
   643    661       return 0;
   644    662     }
   645         -  addr = p->nOp;
   646         -  pOut = &p->aOp[addr];
          663  +  pFirst = pOut = &p->aOp[p->nOp];
   647    664     for(i=0; i<nOp; i++, aOp++, pOut++){
   648    665       pOut->opcode = aOp->opcode;
   649    666       pOut->p1 = aOp->p1;
   650    667       pOut->p2 = aOp->p2;
   651    668       assert( aOp->p2>=0 );
   652    669       pOut->p3 = aOp->p3;
   653    670       pOut->p4type = P4_NOTUSED;
................................................................................
   659    676   #ifdef SQLITE_VDBE_COVERAGE
   660    677       pOut->iSrcLine = iLineno+i;
   661    678   #else
   662    679       (void)iLineno;
   663    680   #endif
   664    681   #ifdef SQLITE_DEBUG
   665    682       if( p->db->flags & SQLITE_VdbeAddopTrace ){
   666         -      sqlite3VdbePrintOp(0, i+addr, &p->aOp[i+addr]);
          683  +      sqlite3VdbePrintOp(0, i+p->nOp, &p->aOp[i+p->nOp]);
   667    684       }
   668    685   #endif
   669    686     }
   670    687     p->nOp += nOp;
   671         -  return addr;
          688  +  return pFirst;
   672    689   }
   673    690   
   674    691   #if defined(SQLITE_ENABLE_STMT_SCANSTATUS)
   675    692   /*
   676    693   ** Add an entry to the array of counters managed by sqlite3_stmt_scanstatus().
   677    694   */
   678    695   void sqlite3VdbeScanStatus(
................................................................................
   712    729   void sqlite3VdbeChangeP2(Vdbe *p, u32 addr, int val){
   713    730     sqlite3VdbeGetOp(p,addr)->p2 = val;
   714    731   }
   715    732   void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){
   716    733     sqlite3VdbeGetOp(p,addr)->p3 = val;
   717    734   }
   718    735   void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){
   719         -  sqlite3VdbeGetOp(p,-1)->p5 = p5;
          736  +  if( !p->db->mallocFailed ) p->aOp[p->nOp-1].p5 = p5;
   720    737   }
   721    738   
   722    739   /*
   723    740   ** Change the P2 operand of instruction addr so that it points to
   724    741   ** the address of the next instruction to be coded.
   725    742   */
   726    743   void sqlite3VdbeJumpHere(Vdbe *p, int addr){
................................................................................
   822    839     p->pNext = pVdbe->pProgram;
   823    840     pVdbe->pProgram = p;
   824    841   }
   825    842   
   826    843   /*
   827    844   ** Change the opcode at addr into OP_Noop
   828    845   */
   829         -void sqlite3VdbeChangeToNoop(Vdbe *p, int addr){
   830         -  if( addr<p->nOp ){
   831         -    VdbeOp *pOp = &p->aOp[addr];
   832         -    sqlite3 *db = p->db;
   833         -    freeP4(db, pOp->p4type, pOp->p4.p);
   834         -    memset(pOp, 0, sizeof(pOp[0]));
   835         -    pOp->opcode = OP_Noop;
   836         -  }
          846  +int sqlite3VdbeChangeToNoop(Vdbe *p, int addr){
          847  +  VdbeOp *pOp;
          848  +  if( p->db->mallocFailed ) return 0;
          849  +  assert( addr>=0 && addr<p->nOp );
          850  +  pOp = &p->aOp[addr];
          851  +  freeP4(p->db, pOp->p4type, pOp->p4.p);
          852  +  pOp->p4type = P4_NOTUSED;
          853  +  pOp->p4.z = 0;
          854  +  pOp->opcode = OP_Noop;
          855  +  return 1;
   837    856   }
   838    857   
   839    858   /*
   840    859   ** If the last opcode is "op" and it is not a jump destination,
   841    860   ** then remove it.  Return true if and only if an opcode was removed.
   842    861   */
   843    862   int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){
   844    863     if( (p->nOp-1)>(p->pParse->iFixedOp) && p->aOp[p->nOp-1].opcode==op ){
   845         -    sqlite3VdbeChangeToNoop(p, p->nOp-1);
   846         -    return 1;
          864  +    return sqlite3VdbeChangeToNoop(p, p->nOp-1);
   847    865     }else{
   848    866       return 0;
   849    867     }
   850    868   }
   851    869   
   852    870   /*
   853    871   ** Change the value of the P4 operand for a specific instruction.
................................................................................
  1878   1896     ** reduce the amount of memory held by a prepared statement.
  1879   1897     */
  1880   1898     do {
  1881   1899       nByte = 0;
  1882   1900       p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), zCsr, &nFree, &nByte);
  1883   1901       p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), zCsr, &nFree, &nByte);
  1884   1902       p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), zCsr, &nFree, &nByte);
  1885         -    p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), zCsr, &nFree, &nByte);
  1886   1903       p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*),
  1887   1904                             zCsr, &nFree, &nByte);
  1888   1905       p->aOnceFlag = allocSpace(p->aOnceFlag, nOnce, zCsr, &nFree, &nByte);
  1889   1906   #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
  1890   1907       p->anExec = allocSpace(p->anExec, p->nOp*sizeof(i64), zCsr, &nFree, &nByte);
  1891   1908   #endif
  1892   1909       if( nByte ){
................................................................................
  1901   1918     if( p->aVar ){
  1902   1919       p->nVar = (ynVar)nVar;
  1903   1920       for(n=0; n<nVar; n++){
  1904   1921         p->aVar[n].flags = MEM_Null;
  1905   1922         p->aVar[n].db = db;
  1906   1923       }
  1907   1924     }
  1908         -  if( p->azVar && pParse->nzVar>0 ){
  1909         -    p->nzVar = pParse->nzVar;
  1910         -    memcpy(p->azVar, pParse->azVar, p->nzVar*sizeof(p->azVar[0]));
  1911         -    memset(pParse->azVar, 0, pParse->nzVar*sizeof(pParse->azVar[0]));
  1912         -  }
         1925  +  p->nzVar = pParse->nzVar;
         1926  +  p->azVar = pParse->azVar;
         1927  +  pParse->nzVar =  0;
         1928  +  pParse->azVar = 0;
  1913   1929     if( p->aMem ){
  1914   1930       p->aMem--;                      /* aMem[] goes from 1..nMem */
  1915   1931       p->nMem = nMem;                 /*       not from 0..nMem-1 */
  1916   1932       for(n=1; n<=nMem; n++){
  1917   1933         p->aMem[n].flags = MEM_Undefined;
  1918   1934         p->aMem[n].db = db;
  1919   1935       }
................................................................................
  2892   2908     releaseMemArray(p->aColName, p->nResColumn*COLNAME_N);
  2893   2909     for(pSub=p->pProgram; pSub; pSub=pNext){
  2894   2910       pNext = pSub->pNext;
  2895   2911       vdbeFreeOpArray(db, pSub->aOp, pSub->nOp);
  2896   2912       sqlite3DbFree(db, pSub);
  2897   2913     }
  2898   2914     for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]);
         2915  +  sqlite3DbFree(db, p->azVar);
  2899   2916     vdbeFreeOpArray(db, p->aOp, p->nOp);
  2900   2917     sqlite3DbFree(db, p->aColName);
  2901   2918     sqlite3DbFree(db, p->zSql);
  2902   2919     sqlite3DbFree(db, p->pFree);
  2903   2920   #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
  2904   2921     for(i=0; i<p->nScan; i++){
  2905   2922       sqlite3DbFree(db, p->aScan[i].zName);
................................................................................
  3636   3653       sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem);
  3637   3654       sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem);
  3638   3655       v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc);
  3639   3656       n1 = v1==0 ? 0 : c1.n;
  3640   3657       v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc);
  3641   3658       n2 = v2==0 ? 0 : c2.n;
  3642   3659       rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2);
         3660  +    if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM;
  3643   3661       sqlite3VdbeMemRelease(&c1);
  3644   3662       sqlite3VdbeMemRelease(&c2);
  3645         -    if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM;
  3646   3663       return rc;
  3647   3664     }
  3648   3665   }
  3649   3666   
  3650   3667   /*
  3651   3668   ** Compare two blobs.  Return negative, zero, or positive if the first
  3652   3669   ** is less than, equal to, or greater than the second, respectively.
................................................................................
  4426   4443   #ifndef SQLITE_OMIT_VIRTUALTABLE
  4427   4444   /*
  4428   4445   ** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored
  4429   4446   ** in memory obtained from sqlite3_malloc) into a Vdbe.zErrMsg (text stored
  4430   4447   ** in memory obtained from sqlite3DbMalloc).
  4431   4448   */
  4432   4449   void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){
  4433         -  sqlite3 *db = p->db;
  4434         -  sqlite3DbFree(db, p->zErrMsg);
  4435         -  p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg);
  4436         -  sqlite3_free(pVtab->zErrMsg);
  4437         -  pVtab->zErrMsg = 0;
         4450  +  if( pVtab->zErrMsg ){
         4451  +    sqlite3 *db = p->db;
         4452  +    sqlite3DbFree(db, p->zErrMsg);
         4453  +    p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg);
         4454  +    sqlite3_free(pVtab->zErrMsg);
         4455  +    pVtab->zErrMsg = 0;
         4456  +  }
  4438   4457   }
  4439   4458   #endif /* SQLITE_OMIT_VIRTUALTABLE */

Changes to src/vdbeblob.c.

   111    111     const char *zColumn,    /* The column containing the blob */
   112    112     sqlite_int64 iRow,      /* The row containing the glob */
   113    113     int flags,              /* True -> read/write access, false -> read-only */
   114    114     sqlite3_blob **ppBlob   /* Handle for accessing the blob returned here */
   115    115   ){
   116    116     int nAttempt = 0;
   117    117     int iCol;               /* Index of zColumn in row-record */
   118         -
   119         -  /* This VDBE program seeks a btree cursor to the identified 
   120         -  ** db/table/row entry. The reason for using a vdbe program instead
   121         -  ** of writing code to use the b-tree layer directly is that the
   122         -  ** vdbe program will take advantage of the various transaction,
   123         -  ** locking and error handling infrastructure built into the vdbe.
   124         -  **
   125         -  ** After seeking the cursor, the vdbe executes an OP_ResultRow.
   126         -  ** Code external to the Vdbe then "borrows" the b-tree cursor and
   127         -  ** uses it to implement the blob_read(), blob_write() and 
   128         -  ** blob_bytes() functions.
   129         -  **
   130         -  ** The sqlite3_blob_close() function finalizes the vdbe program,
   131         -  ** which closes the b-tree cursor and (possibly) commits the 
   132         -  ** transaction.
   133         -  */
   134         -  static const int iLn = VDBE_OFFSET_LINENO(4);
   135         -  static const VdbeOpList openBlob[] = {
   136         -    /* {OP_Transaction, 0, 0, 0},  // 0: Inserted separately */
   137         -    {OP_TableLock, 0, 0, 0},       /* 1: Acquire a read or write lock */
   138         -    /* One of the following two instructions is replaced by an OP_Noop. */
   139         -    {OP_OpenRead, 0, 0, 0},        /* 2: Open cursor 0 for reading */
   140         -    {OP_OpenWrite, 0, 0, 0},       /* 3: Open cursor 0 for read/write */
   141         -    {OP_Variable, 1, 1, 1},        /* 4: Push the rowid to the stack */
   142         -    {OP_NotExists, 0, 10, 1},      /* 5: Seek the cursor */
   143         -    {OP_Column, 0, 0, 1},          /* 6  */
   144         -    {OP_ResultRow, 1, 0, 0},       /* 7  */
   145         -    {OP_Goto, 0, 4, 0},            /* 8  */
   146         -    {OP_Close, 0, 0, 0},           /* 9  */
   147         -    {OP_Halt, 0, 0, 0},            /* 10 */
   148         -  };
   149         -
   150    118     int rc = SQLITE_OK;
   151    119     char *zErr = 0;
   152    120     Table *pTab;
   153    121     Parse *pParse = 0;
   154    122     Incrblob *pBlob = 0;
   155    123   
   156    124   #ifdef SQLITE_ENABLE_API_ARMOR
................................................................................
   261    229           goto blob_open_out;
   262    230         }
   263    231       }
   264    232   
   265    233       pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(pParse);
   266    234       assert( pBlob->pStmt || db->mallocFailed );
   267    235       if( pBlob->pStmt ){
          236  +      
          237  +      /* This VDBE program seeks a btree cursor to the identified 
          238  +      ** db/table/row entry. The reason for using a vdbe program instead
          239  +      ** of writing code to use the b-tree layer directly is that the
          240  +      ** vdbe program will take advantage of the various transaction,
          241  +      ** locking and error handling infrastructure built into the vdbe.
          242  +      **
          243  +      ** After seeking the cursor, the vdbe executes an OP_ResultRow.
          244  +      ** Code external to the Vdbe then "borrows" the b-tree cursor and
          245  +      ** uses it to implement the blob_read(), blob_write() and 
          246  +      ** blob_bytes() functions.
          247  +      **
          248  +      ** The sqlite3_blob_close() function finalizes the vdbe program,
          249  +      ** which closes the b-tree cursor and (possibly) commits the 
          250  +      ** transaction.
          251  +      */
          252  +      static const int iLn = VDBE_OFFSET_LINENO(4);
          253  +      static const VdbeOpList openBlob[] = {
          254  +                                    /* addr/ofst */
          255  +        /* {OP_Transaction, 0, 0, 0},  // 0/   inserted separately */
          256  +        {OP_TableLock, 0, 0, 0},       /* 1/0: Acquire a read or write lock */
          257  +        {OP_OpenRead, 0, 0, 0},        /* 2/1: Open a cursor */
          258  +        {OP_Variable, 1, 1, 0},        /* 3/2: Move ?1 into reg[1] */
          259  +        {OP_NotExists, 0, 8, 1},       /* 4/3: Seek the cursor */
          260  +        {OP_Column, 0, 0, 1},          /* 5/4  */
          261  +        {OP_ResultRow, 1, 0, 0},       /* 6/5  */
          262  +        {OP_Goto, 0, 3, 0},            /* 7/6  */
          263  +        {OP_Close, 0, 0, 0},           /* 8/7  */
          264  +        {OP_Halt, 0, 0, 0},            /* 9/8  */
          265  +      };
   268    266         Vdbe *v = (Vdbe *)pBlob->pStmt;
   269    267         int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
   270         -
          268  +      VdbeOp *aOp;
   271    269   
   272    270         sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, flags, 
   273    271                              pTab->pSchema->schema_cookie,
   274    272                              pTab->pSchema->iGeneration);
   275    273         sqlite3VdbeChangeP5(v, 1);     
   276         -      sqlite3VdbeAddOpList(v, ArraySize(openBlob), openBlob, iLn);
          274  +      aOp = sqlite3VdbeAddOpList(v, ArraySize(openBlob), openBlob, iLn);
   277    275   
   278    276         /* Make sure a mutex is held on the table to be accessed */
   279    277         sqlite3VdbeUsesBtree(v, iDb); 
   280    278   
   281         -      /* Configure the OP_TableLock instruction */
          279  +      if( db->mallocFailed==0 ){
          280  +        assert( aOp!=0 );
          281  +        /* Configure the OP_TableLock instruction */
   282    282   #ifdef SQLITE_OMIT_SHARED_CACHE
   283         -      sqlite3VdbeChangeToNoop(v, 1);
          283  +        aOp[0].opcode = OP_Noop;
   284    284   #else
   285         -      sqlite3VdbeChangeP1(v, 1, iDb);
   286         -      sqlite3VdbeChangeP2(v, 1, pTab->tnum);
   287         -      sqlite3VdbeChangeP3(v, 1, flags);
   288         -      sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT);
          285  +        aOp[0].p1 = iDb;
          286  +        aOp[0].p2 = pTab->tnum;
          287  +        aOp[0].p3 = flags;
          288  +        sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT);
          289  +      }
          290  +      if( db->mallocFailed==0 ){
   289    291   #endif
   290    292   
   291         -      /* Remove either the OP_OpenWrite or OpenRead. Set the P2 
   292         -      ** parameter of the other to pTab->tnum.  */
   293         -      sqlite3VdbeChangeToNoop(v, 3 - flags);
   294         -      sqlite3VdbeChangeP2(v, 2 + flags, pTab->tnum);
   295         -      sqlite3VdbeChangeP3(v, 2 + flags, iDb);
          293  +        /* Remove either the OP_OpenWrite or OpenRead. Set the P2 
          294  +        ** parameter of the other to pTab->tnum.  */
          295  +        if( flags ) aOp[1].opcode = OP_OpenWrite;
          296  +        aOp[1].p2 = pTab->tnum;
          297  +        aOp[1].p3 = iDb;   
   296    298   
   297         -      /* Configure the number of columns. Configure the cursor to
   298         -      ** think that the table has one more column than it really
   299         -      ** does. An OP_Column to retrieve this imaginary column will
   300         -      ** always return an SQL NULL. This is useful because it means
   301         -      ** we can invoke OP_Column to fill in the vdbe cursors type 
   302         -      ** and offset cache without causing any IO.
   303         -      */
   304         -      sqlite3VdbeChangeP4(v, 2+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32);
   305         -      sqlite3VdbeChangeP2(v, 6, pTab->nCol);
   306         -      if( !db->mallocFailed ){
          299  +        /* Configure the number of columns. Configure the cursor to
          300  +        ** think that the table has one more column than it really
          301  +        ** does. An OP_Column to retrieve this imaginary column will
          302  +        ** always return an SQL NULL. This is useful because it means
          303  +        ** we can invoke OP_Column to fill in the vdbe cursors type 
          304  +        ** and offset cache without causing any IO.
          305  +        */
          306  +        aOp[1].p4type = P4_INT32;
          307  +        aOp[1].p4.i = pTab->nCol+1;
          308  +        aOp[4].p2 = pTab->nCol;
          309  +
   307    310           pParse->nVar = 1;
   308    311           pParse->nMem = 1;
   309    312           pParse->nTab = 1;
   310    313           sqlite3VdbeMakeReady(v, pParse);
   311    314         }
   312    315       }
   313    316      

Changes to src/vdbemem.c.

  1220   1220       goto value_from_function_out;
  1221   1221     }
  1222   1222   
  1223   1223     assert( pCtx->pParse->rc==SQLITE_OK );
  1224   1224     memset(&ctx, 0, sizeof(ctx));
  1225   1225     ctx.pOut = pVal;
  1226   1226     ctx.pFunc = pFunc;
  1227         -  pFunc->xFunc(&ctx, nVal, apVal);
         1227  +  pFunc->xSFunc(&ctx, nVal, apVal);
  1228   1228     if( ctx.isError ){
  1229   1229       rc = ctx.isError;
  1230   1230       sqlite3ErrorMsg(pCtx->pParse, "%s", sqlite3_value_text(pVal));
  1231   1231     }else{
  1232   1232       sqlite3ValueApplyAffinity(pVal, aff, SQLITE_UTF8);
  1233   1233       assert( rc==SQLITE_OK );
  1234   1234       rc = sqlite3VdbeChangeEncoding(pVal, enc);

Changes to src/vtab.c.

  1012   1012     FuncDef *pDef,  /* Function to possibly overload */
  1013   1013     int nArg,       /* Number of arguments to the function */
  1014   1014     Expr *pExpr     /* First argument to the function */
  1015   1015   ){
  1016   1016     Table *pTab;
  1017   1017     sqlite3_vtab *pVtab;
  1018   1018     sqlite3_module *pMod;
  1019         -  void (*xFunc)(sqlite3_context*,int,sqlite3_value**) = 0;
         1019  +  void (*xSFunc)(sqlite3_context*,int,sqlite3_value**) = 0;
  1020   1020     void *pArg = 0;
  1021   1021     FuncDef *pNew;
  1022   1022     int rc = 0;
  1023   1023     char *zLowerName;
  1024   1024     unsigned char *z;
  1025   1025   
  1026   1026   
................................................................................
  1040   1040     ** to see if the implementation wants to overload this function 
  1041   1041     */
  1042   1042     zLowerName = sqlite3DbStrDup(db, pDef->zName);
  1043   1043     if( zLowerName ){
  1044   1044       for(z=(unsigned char*)zLowerName; *z; z++){
  1045   1045         *z = sqlite3UpperToLower[*z];
  1046   1046       }
  1047         -    rc = pMod->xFindFunction(pVtab, nArg, zLowerName, &xFunc, &pArg);
         1047  +    rc = pMod->xFindFunction(pVtab, nArg, zLowerName, &xSFunc, &pArg);
  1048   1048       sqlite3DbFree(db, zLowerName);
  1049   1049     }
  1050   1050     if( rc==0 ){
  1051   1051       return pDef;
  1052   1052     }
  1053   1053   
  1054   1054     /* Create a new ephemeral function definition for the overloaded
................................................................................
  1057   1057                                + sqlite3Strlen30(pDef->zName) + 1);
  1058   1058     if( pNew==0 ){
  1059   1059       return pDef;
  1060   1060     }
  1061   1061     *pNew = *pDef;
  1062   1062     pNew->zName = (char *)&pNew[1];
  1063   1063     memcpy(pNew->zName, pDef->zName, sqlite3Strlen30(pDef->zName)+1);
  1064         -  pNew->xFunc = xFunc;
         1064  +  pNew->xSFunc = xSFunc;
  1065   1065     pNew->pUserData = pArg;
  1066   1066     pNew->funcFlags |= SQLITE_FUNC_EPHEM;
  1067   1067     return pNew;
  1068   1068   }
  1069   1069   
  1070   1070   /*
  1071   1071   ** Make sure virtual table pTab is contained in the pParse->apVirtualLock[]

Changes to src/wherecode.c.

   323    323     }
   324    324     while( n>1 && zAff[n-1]==SQLITE_AFF_BLOB ){
   325    325       n--;
   326    326     }
   327    327   
   328    328     /* Code the OP_Affinity opcode if there is anything left to do. */
   329    329     if( n>0 ){
   330         -    sqlite3VdbeAddOp2(v, OP_Affinity, base, n);
   331         -    sqlite3VdbeChangeP4(v, -1, zAff, n);
          330  +    sqlite3VdbeAddOp4(v, OP_Affinity, base, n, 0, zAff, n);
   332    331       sqlite3ExprCacheAffinityChange(pParse, base, n);
   333    332     }
   334    333   }
   335    334   
   336    335   
   337    336   /*
   338    337   ** Generate code for a single equality term of the WHERE clause.  An equality

Changes to src/whereexpr.c.

   198    198     ExprList *pList;           /* List of operands to the LIKE operator */
   199    199     int c;                     /* One character in z[] */
   200    200     int cnt;                   /* Number of non-wildcard prefix characters */
   201    201     char wc[3];                /* Wildcard characters */
   202    202     sqlite3 *db = pParse->db;  /* Database connection */
   203    203     sqlite3_value *pVal = 0;
   204    204     int op;                    /* Opcode of pRight */
          205  +  int rc;                    /* Result code to return */
   205    206   
   206    207     if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){
   207    208       return 0;
   208    209     }
   209    210   #ifdef SQLITE_EBCDIC
   210    211     if( *pnoCase ) return 0;
   211    212   #endif
................................................................................
   263    264           }
   264    265         }
   265    266       }else{
   266    267         z = 0;
   267    268       }
   268    269     }
   269    270   
          271  +  rc = (z!=0);
   270    272     sqlite3ValueFree(pVal);
   271         -  return (z!=0);
          273  +  return rc;
   272    274   }
   273    275   #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */
   274    276   
   275    277   
   276    278   #ifndef SQLITE_OMIT_VIRTUALTABLE
   277    279   /*
   278    280   ** Check to see if the given expression is of the form

Changes to test/threadtest3.c.

   876    876   */
   877    877   static double timelimit = 0.0;
   878    878   
   879    879   static double currentTime(void){
   880    880     double t;
   881    881     static sqlite3_vfs *pTimelimitVfs = 0;
   882    882     if( pTimelimitVfs==0 ) pTimelimitVfs = sqlite3_vfs_find(0);
   883         -  if( pTimelimitVfs->iVersion>=1 && pTimelimitVfs->xCurrentTimeInt64!=0 ){
          883  +  if( pTimelimitVfs->iVersion>=2 && pTimelimitVfs->xCurrentTimeInt64!=0 ){
   884    884       sqlite3_int64 tm;
   885    885       pTimelimitVfs->xCurrentTimeInt64(pTimelimitVfs, &tm);
   886    886       t = tm/86400000.0;
   887    887     }else{
   888    888       pTimelimitVfs->xCurrentTime(pTimelimitVfs, &t);
   889    889     }
   890    890     return t;