Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -592,10 +592,16 @@ $(TOP)/src/shell.c libsqlite3.la \ $(LIBREADLINE) $(TLIBS) -rpath "$(libdir)" sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) + +srcck1$(BEXE): $(TOP)/tool/srcck1.c + $(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c + +sourcetest: srcck1$(BEXE) sqlite3.c + ./srcck1 sqlite3.c fuzzershell$(TEXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(FUZZERSHELL_OPT) \ $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS) @@ -1093,11 +1099,11 @@ ./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS) # This is the common case. Run many tests that do not take too long, # including fuzzcheck, sqlite3_analyzer, and sqldiff tests. # -test: $(TESTPROGS) fastfuzztest +test: $(TESTPROGS) sourcetest fastfuzztest ./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS) # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. # @@ -1229,10 +1235,11 @@ rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c rm -f sqlite-*-output.vsix rm -f mptester mptester.exe rm -f rbu rbu.exe + rm -f srcck1 srcck1.exe rm -f fuzzershell fuzzershell.exe rm -f fuzzcheck fuzzcheck.exe rm -f sqldiff sqldiff.exe rm -f fts5.* fts5parse.* Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -264,10 +264,19 @@ !ENDIF ############################################################################### ############################### END OF OPTIONS ################################ ############################################################################### + +# When compiling for the Windows 10 platform, the PLATFORM macro must be set +# to an appropriate value (e.g. x86, x64, arm, arm64, etc). +# +!IF $(FOR_WIN10)!=0 +!IFNDEF PLATFORM +!ERROR Using the FOR_WIN10 option requires a value for PLATFORM. +!ENDIF +!ENDIF # This assumes that MSVC is always installed in 32-bit Program Files directory # and sets the variable for use in locating other 32-bit installs accordingly. # PROGRAMFILES_X86 = $(VCINSTALLDIR)\..\.. @@ -295,11 +304,11 @@ # !IFNDEF RC RC = rc.exe !ENDIF -# Check for the MSVC runtime library path macro. Othertise, this value will +# Check for the MSVC runtime library path macro. Otherwise, this value will # default to the 'lib' directory underneath the MSVC installation directory. # !IFNDEF CRTLIBPATH CRTLIBPATH = $(VCINSTALLDIR)\lib !ENDIF @@ -332,21 +341,21 @@ NCC = $(NCC:\\=\) !ELSE NCC = $(CC) !ENDIF -# Check for the MSVC native runtime library path macro. Othertise, +# Check for the MSVC native runtime library path macro. Otherwise, # this value will default to the 'lib' directory underneath the MSVC # installation directory. # !IFNDEF NCRTLIBPATH NCRTLIBPATH = $(VCINSTALLDIR)\lib !ENDIF NCRTLIBPATH = $(NCRTLIBPATH:\\=\) -# Check for the Platform SDK library path macro. Othertise, this +# Check for the Platform SDK library path macro. Otherwise, this # value will default to the 'lib' directory underneath the Windows # SDK installation directory (the environment variable used appears # to be available when using Visual C++ 2008 or later via the # command line). # @@ -353,10 +362,20 @@ !IFNDEF NSDKLIBPATH NSDKLIBPATH = $(WINDOWSSDKDIR)\lib !ENDIF NSDKLIBPATH = $(NSDKLIBPATH:\\=\) + +# Check for the UCRT library path macro. Otherwise, this value will +# default to the version-specific, platform-specific 'lib' directory +# underneath the Windows SDK installation directory. +# +!IFNDEF UCRTLIBPATH +UCRTLIBPATH = $(WINDOWSSDKDIR)\lib\$(WINDOWSSDKLIBVERSION)\ucrt\$(PLATFORM) +!ENDIF + +UCRTLIBPATH = $(UCRTLIBPATH:\\=\) # C compiler and options for use in building executables that # will run on the platform that is doing the build. # !IF $(USE_FULLWARN)!=0 @@ -539,11 +558,11 @@ # Also, we need to dynamically link to the correct MSVC runtime # when compiling for WinRT (e.g. debug or release) OR if the # USE_CRT_DLL option is set to force dynamically linking to the # MSVC runtime library. # -!IF $(FOR_WINRT)!=0 || $(FOR_WIN10)!=0 || $(USE_CRT_DLL)!=0 +!IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0 !IF $(DEBUG)>1 TCC = $(TCC) -MDd BCC = $(BCC) -MDd !ELSE TCC = $(TCC) -MD @@ -886,19 +905,29 @@ LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib !ENDIF -# When compiling for UAP, some extra linker options are also required. +# When compiling for UAP or the Windows 10 platform, some extra linker +# options are also required. # !IF $(FOR_UAP)!=0 || $(FOR_WIN10)!=0 LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE /NODEFAULTLIB:kernel32.lib LTLINKOPTS = $(LTLINKOPTS) mincore.lib !IFDEF PSDKLIBPATH LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(PSDKLIBPATH)" !ENDIF !ENDIF + +!IF $(FOR_WIN10)!=0 +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(UCRTLIBPATH)" +!IF $(DEBUG)>1 +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrtd.lib /DEFAULTLIB:ucrtd.lib +!ELSE +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib +!ENDIF +!ENDIF # If either debugging or symbols are enabled, enable PDBs. # !IF $(DEBUG)>1 || $(SYMBOLS)!=0 LDFLAGS = /DEBUG $(LDOPTS) @@ -1390,10 +1419,16 @@ /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) # <> sqldiff.exe: $(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + +srcck1.exe: $(TOP)\tool\srcck1.c + $(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c + +sourcetest: srcck1.exe sqlite3.c + srcck1.exe sqlite3.c fuzzershell.exe: $(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) fuzzcheck.exe: $(TOP)\test\fuzzcheck.c $(SQLITE3C) $(SQLITE3H) @@ -1914,18 +1949,18 @@ fastfuzztest: fuzzcheck.exe .\fuzzcheck.exe --limit-mem 100M $(FUZZDATA) # Minimal testing that runs in less than 3 minutes (on a fast machine) # -quicktest: testfixture.exe +quicktest: testfixture.exe sourcetest @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\extraquick.test $(TESTOPTS) # This is the common case. Run many tests that do not take too long, # including fuzzcheck, sqlite3_analyzer, and sqldiff tests. # -test: $(TESTPROGS) fastfuzztest +test: $(TESTPROGS) sourcetest fastfuzztest @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\veryquick.test $(TESTOPTS) smoketest: $(TESTPROGS) @set PATH=$(LIBTCLPATH);$(PATH) @@ -2008,15 +2043,15 @@ del /Q tclsqlite3.exe 2>NUL del /Q testloadext.dll 2>NUL del /Q testfixture.exe test.db 2>NUL del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe 2>NUL del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL - del /Q mptester.exe wordcount.exe rbu.exe 2>NUL + del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL del /Q $(SQLITE3EXE) $(SQLITE3DLL) sqlite3.def 2>NUL del /Q sqlite3.c sqlite3-*.c 2>NUL del /Q sqlite3rc.h 2>NUL del /Q shell.c sqlite3ext.h 2>NUL del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL del /Q sqlite-*-output.vsix 2>NUL del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL del /Q fts5.* fts5parse.* 2>NUL # <> Index: autoconf/Makefile.msc ================================================================== --- autoconf/Makefile.msc +++ autoconf/Makefile.msc @@ -249,10 +249,19 @@ !ENDIF ############################################################################### ############################### END OF OPTIONS ################################ ############################################################################### + +# When compiling for the Windows 10 platform, the PLATFORM macro must be set +# to an appropriate value (e.g. x86, x64, arm, arm64, etc). +# +!IF $(FOR_WIN10)!=0 +!IFNDEF PLATFORM +!ERROR Using the FOR_WIN10 option requires a value for PLATFORM. +!ENDIF +!ENDIF # This assumes that MSVC is always installed in 32-bit Program Files directory # and sets the variable for use in locating other 32-bit installs accordingly. # PROGRAMFILES_X86 = $(VCINSTALLDIR)\..\.. @@ -280,11 +289,11 @@ # !IFNDEF RC RC = rc.exe !ENDIF -# Check for the MSVC runtime library path macro. Othertise, this value will +# Check for the MSVC runtime library path macro. Otherwise, this value will # default to the 'lib' directory underneath the MSVC installation directory. # !IFNDEF CRTLIBPATH CRTLIBPATH = $(VCINSTALLDIR)\lib !ENDIF @@ -317,21 +326,21 @@ NCC = $(NCC:\\=\) !ELSE NCC = $(CC) !ENDIF -# Check for the MSVC native runtime library path macro. Othertise, +# Check for the MSVC native runtime library path macro. Otherwise, # this value will default to the 'lib' directory underneath the MSVC # installation directory. # !IFNDEF NCRTLIBPATH NCRTLIBPATH = $(VCINSTALLDIR)\lib !ENDIF NCRTLIBPATH = $(NCRTLIBPATH:\\=\) -# Check for the Platform SDK library path macro. Othertise, this +# Check for the Platform SDK library path macro. Otherwise, this # value will default to the 'lib' directory underneath the Windows # SDK installation directory (the environment variable used appears # to be available when using Visual C++ 2008 or later via the # command line). # @@ -338,10 +347,20 @@ !IFNDEF NSDKLIBPATH NSDKLIBPATH = $(WINDOWSSDKDIR)\lib !ENDIF NSDKLIBPATH = $(NSDKLIBPATH:\\=\) + +# Check for the UCRT library path macro. Otherwise, this value will +# default to the version-specific, platform-specific 'lib' directory +# underneath the Windows SDK installation directory. +# +!IFNDEF UCRTLIBPATH +UCRTLIBPATH = $(WINDOWSSDKDIR)\lib\$(WINDOWSSDKLIBVERSION)\ucrt\$(PLATFORM) +!ENDIF + +UCRTLIBPATH = $(UCRTLIBPATH:\\=\) # C compiler and options for use in building executables that # will run on the platform that is doing the build. # !IF $(USE_FULLWARN)!=0 @@ -524,11 +543,11 @@ # Also, we need to dynamically link to the correct MSVC runtime # when compiling for WinRT (e.g. debug or release) OR if the # USE_CRT_DLL option is set to force dynamically linking to the # MSVC runtime library. # -!IF $(FOR_WINRT)!=0 || $(FOR_WIN10)!=0 || $(USE_CRT_DLL)!=0 +!IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0 !IF $(DEBUG)>1 TCC = $(TCC) -MDd BCC = $(BCC) -MDd !ELSE TCC = $(TCC) -MD @@ -781,19 +800,29 @@ LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib !ENDIF -# When compiling for UAP, some extra linker options are also required. +# When compiling for UAP or the Windows 10 platform, some extra linker +# options are also required. # !IF $(FOR_UAP)!=0 || $(FOR_WIN10)!=0 LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE /NODEFAULTLIB:kernel32.lib LTLINKOPTS = $(LTLINKOPTS) mincore.lib !IFDEF PSDKLIBPATH LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(PSDKLIBPATH)" !ENDIF !ENDIF + +!IF $(FOR_WIN10)!=0 +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(UCRTLIBPATH)" +!IF $(DEBUG)>1 +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrtd.lib /DEFAULTLIB:ucrtd.lib +!ELSE +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib +!ENDIF +!ENDIF # If either debugging or symbols are enabled, enable PDBs. # !IF $(DEBUG)>1 || $(SYMBOLS)!=0 LDFLAGS = /DEBUG $(LDOPTS) Index: ext/fts3/fts3_tokenizer.c ================================================================== --- ext/fts3/fts3_tokenizer.c +++ ext/fts3/fts3_tokenizer.c @@ -65,10 +65,11 @@ zName = sqlite3_value_text(argv[0]); nName = sqlite3_value_bytes(argv[0])+1; if( argc==2 ){ +#ifdef SQLITE_ENABLE_FTS3_TOKENIZER void *pOld; int n = sqlite3_value_bytes(argv[1]); if( zName==0 || n!=sizeof(pPtr) ){ sqlite3_result_error(context, "argument type mismatch", -1); return; @@ -77,11 +78,18 @@ pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); if( pOld==pPtr ){ sqlite3_result_error(context, "out of memory", -1); return; } - }else{ +#else + sqlite3_result_error(context, "fts3tokenize: " + "disabled - rebuild with -DSQLITE_ENABLE_FTS3_TOKENIZER", -1 + ); + return; +#endif /* SQLITE_ENABLE_FTS3_TOKENIZER */ + }else + { if( zName ){ pPtr = sqlite3Fts3HashFind(pHash, zName, nName); } if( !pPtr ){ char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); @@ -326,10 +334,11 @@ sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT); } Tcl_DecrRefCount(pRet); } +#ifdef SQLITE_ENABLE_FTS3_TOKENIZER static int registerTokenizer( sqlite3 *db, char *zName, const sqlite3_tokenizer_module *p @@ -347,10 +356,12 @@ sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); sqlite3_step(pStmt); return sqlite3_finalize(pStmt); } +#endif /* SQLITE_ENABLE_FTS3_TOKENIZER */ + static int queryTokenizer( sqlite3 *db, char *zName, @@ -418,15 +429,17 @@ assert( rc==SQLITE_ERROR ); assert( p2==0 ); assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); /* Test the storage function */ +#ifdef SQLITE_ENABLE_FTS3_TOKENIZER rc = registerTokenizer(db, "nosuchtokenizer", p1); assert( rc==SQLITE_OK ); rc = queryTokenizer(db, "nosuchtokenizer", &p2); assert( rc==SQLITE_OK ); assert( p2==p1 ); +#endif sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); } #endif Index: ext/fts5/fts5Int.h ================================================================== --- ext/fts5/fts5Int.h +++ ext/fts5/fts5Int.h @@ -280,10 +280,11 @@ typedef struct Fts5PoslistWriter Fts5PoslistWriter; struct Fts5PoslistWriter { i64 iPrev; }; int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64); +void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64); int sqlite3Fts5PoslistNext64( const u8 *a, int n, /* Buffer containing poslist */ int *pi, /* IN/OUT: Offset within a[] */ i64 *piOff /* IN/OUT: Current offset */ Index: ext/fts5/fts5_buffer.c ================================================================== --- ext/fts5/fts5_buffer.c +++ ext/fts5/fts5_buffer.c @@ -14,22 +14,24 @@ #include "fts5Int.h" int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){ - u32 nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64; - u8 *pNew; - while( nNewp, nNew); - if( pNew==0 ){ - *pRc = SQLITE_NOMEM; - return 1; - }else{ - pBuf->nSpace = nNew; - pBuf->p = pNew; + if( (u32)pBuf->nSpacenSpace ? pBuf->nSpace : 64; + u8 *pNew; + while( nNewp, nNew); + if( pNew==0 ){ + *pRc = SQLITE_NOMEM; + return 1; + }else{ + pBuf->nSpace = nNew; + pBuf->p = pNew; + } } return 0; } @@ -205,28 +207,41 @@ pIter->a = a; pIter->n = n; sqlite3Fts5PoslistReaderNext(pIter); return pIter->bEof; } + +/* +** Append position iPos to the position list being accumulated in buffer +** pBuf, which must be already be large enough to hold the new data. +** The previous position written to this list is *piPrev. *piPrev is set +** to iPos before returning. +*/ +void sqlite3Fts5PoslistSafeAppend( + Fts5Buffer *pBuf, + i64 *piPrev, + i64 iPos +){ + static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32; + if( (iPos & colmask) != (*piPrev & colmask) ){ + pBuf->p[pBuf->n++] = 1; + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32)); + *piPrev = (iPos & colmask); + } + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2); + *piPrev = iPos; +} int sqlite3Fts5PoslistWriterAppend( Fts5Buffer *pBuf, Fts5PoslistWriter *pWriter, i64 iPos ){ - static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32; - int rc = SQLITE_OK; - if( 0==fts5BufferGrow(&rc, pBuf, 5+5+5) ){ - if( (iPos & colmask) != (pWriter->iPrev & colmask) ){ - pBuf->p[pBuf->n++] = 1; - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32)); - pWriter->iPrev = (iPos & colmask); - } - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-pWriter->iPrev)+2); - pWriter->iPrev = iPos; - } - return rc; + int rc; + if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc; + sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos); + return SQLITE_OK; } void *sqlite3Fts5MallocZero(int *pRc, int nByte){ void *pRet = 0; if( *pRc==SQLITE_OK ){ Index: ext/fts5/fts5_index.c ================================================================== --- ext/fts5/fts5_index.c +++ ext/fts5/fts5_index.c @@ -516,11 +516,10 @@ void (*xSetOutputs)(Fts5Iter*, Fts5SegIter*); int nSeg; /* Size of aSeg[] array */ int bRev; /* True to iterate in reverse order */ u8 bSkipEmpty; /* True to skip deleted entries */ - u8 bFiltered; /* True if column-filter already applied */ i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ Fts5CResult *aFirst; /* Current merge state (see above) */ Fts5SegIter aSeg[1]; /* Array of segment iterators */ }; @@ -2028,13 +2027,10 @@ iPoslist = pIter->iTermLeafOffset; }else{ iPoslist = 4; } fts5IndexSkipVarint(pLeaf->p, iPoslist); - assert( p->pConfig->eDetail==FTS5_DETAIL_NONE || iPoslist==( - pIter->iLeafOffset - sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel) - )); pIter->iLeafOffset = iPoslist; /* If this condition is true then the largest rowid for the current ** term may not be stored on the current page. So search forward to ** see where said rowid really is. */ @@ -3093,11 +3089,11 @@ static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ pIter->base.iRowid = pSeg->iRowid; pIter->base.nData = pSeg->nPos; assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE ); - assert( pIter->pColset==0 || pIter->bFiltered ); + assert( pIter->pColset==0 ); if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ /* All data is stored on the current page. Populate the output ** variables to point into the body of the page object. */ pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset]; @@ -3210,11 +3206,11 @@ Fts5Config *pConfig = pIter->pIndex->pConfig; if( pConfig->eDetail==FTS5_DETAIL_NONE ){ pIter->xSetOutputs = fts5IterSetOutputs_None; } - else if( pIter->pColset==0 || pIter->bFiltered ){ + else if( pIter->pColset==0 ){ pIter->xSetOutputs = fts5IterSetOutputs_Nocolset; } else if( pConfig->eDetail==FTS5_DETAIL_FULL ){ pIter->xSetOutputs = fts5IterSetOutputs_Full; @@ -3357,11 +3353,10 @@ Fts5Iter *pNew; pNew = fts5MultiIterAlloc(p, 2); if( pNew ){ Fts5SegIter *pIter = &pNew->aSeg[1]; - pNew->bFiltered = 1; pIter->flags = FTS5_SEGITER_ONETERM; if( pData->szLeaf>0 ){ pIter->pLeaf = pData; pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid); pIter->iEndofDoclist = pData->nn; @@ -4703,98 +4698,114 @@ Fts5Index *p, /* FTS5 backend object */ Fts5Buffer *p1, /* First list to merge */ Fts5Buffer *p2 /* Second list to merge */ ){ if( p2->n ){ - if( p1->n==0 ){ - fts5BufferSwap(p1, p2); - }else{ - i64 iLastRowid = 0; - Fts5DoclistIter i1; - Fts5DoclistIter i2; - Fts5Buffer out = {0, 0, 0}; - Fts5Buffer tmp = {0, 0, 0}; - - if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return; - fts5DoclistIterInit(p1, &i1); - fts5DoclistIterInit(p2, &i2); - - while( 1 ){ - - if( i1.iRowidrc, &out, p1->n + p2->n) ) return; + fts5DoclistIterInit(p1, &i1); + fts5DoclistIterInit(p2, &i2); + + while( 1 ){ + if( i1.iRowidrc, &tmp, i1.nPoslist + i2.nPoslist); + if( p->rc ) break; + + sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1); + sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); + assert( iPos1>=0 && iPos2>=0 ); + + if( iPos1=0 || iPos2>=0 ){ - i64 iNew; - if( iPos2<0 || (iPos1>=0 && iPos1=0 && iPos2>=0 ){ + while( 1 ){ + if( iPos1rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew); - if( p->rc ) goto error_out; + sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1); + if( iPos1<0 ) break; + }else{ + assert( iPos2!=iPrev ); + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2); + sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); + if( iPos2<0 ) break; } } + } - /* WRITEPOSLISTSIZE */ - fts5BufferSafeAppendVarint(&out, tmp.n * 2); - fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); - fts5DoclistIterNext(&i1); - fts5DoclistIterNext(&i2); - if( i1.aPoslist==0 || i2.aPoslist==0 ) break; + if( iPos1>=0 ){ + if( iPos1!=iPrev ){ + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1); + } + fts5BufferSafeAppendBlob(&tmp, &a1[iOff1], i1.nPoslist-iOff1); + }else{ + assert( iPos2>=0 && iPos2!=iPrev ); + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2); + fts5BufferSafeAppendBlob(&tmp, &a2[iOff2], i2.nPoslist-iOff2); } + + /* WRITEPOSLISTSIZE */ + fts5BufferSafeAppendVarint(&out, tmp.n * 2); + fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); + fts5DoclistIterNext(&i1); + fts5DoclistIterNext(&i2); + if( i1.aPoslist==0 || i2.aPoslist==0 ) break; } + } - if( i1.aPoslist ){ - fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid); - fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.aEof - i1.aPoslist); - } - else if( i2.aPoslist ){ - fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); - fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist); - } + if( i1.aPoslist ){ + fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid); + fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.aEof - i1.aPoslist); + } + else if( i2.aPoslist ){ + fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); + fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist); + } - error_out: - fts5BufferSet(&p->rc, p1, out.n, out.p); - fts5BufferFree(&tmp); - fts5BufferFree(&out); - } + fts5BufferSet(&p->rc, p1, out.n, out.p); + fts5BufferFree(&tmp); + fts5BufferFree(&out); } } static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ @@ -4934,11 +4945,11 @@ ** records must be invalidated. */ int sqlite3Fts5IndexRollback(Fts5Index *p){ fts5CloseReader(p); fts5IndexDiscardData(p); - assert( p->rc==SQLITE_OK ); + /* assert( p->rc==SQLITE_OK ); */ return SQLITE_OK; } /* ** The %_data table is completely empty when this function is called. This @@ -5154,14 +5165,15 @@ }else{ /* Scan multiple terms in the main index */ int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; buf.p[0] = FTS5_MAIN_PREFIX; fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet); + assert( pRet->pColset==0 ); fts5IterSetOutputCb(&p->rc, pRet); if( p->rc==SQLITE_OK ){ Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst]; - if( p->rc==SQLITE_OK && pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg); + if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg); } } if( p->rc ){ sqlite3Fts5IterClose(&pRet->base); Index: ext/fts5/fts5_main.c ================================================================== --- ext/fts5/fts5_main.c +++ ext/fts5/fts5_main.c @@ -801,11 +801,11 @@ ** even if we reach end-of-file. The fts5EofMethod() will be called ** subsequently to determine whether or not an EOF was hit. */ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; - int rc = SQLITE_OK; + int rc; assert( (pCsr->ePlan<3)== (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) ); assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) ); @@ -818,10 +818,11 @@ fts5CsrNewrow(pCsr); }else{ switch( pCsr->ePlan ){ case FTS5_PLAN_SPECIAL: { CsrFlagSet(pCsr, FTS5CSR_EOF); + rc = SQLITE_OK; break; } case FTS5_PLAN_SORTED_MATCH: { rc = fts5SorterNext(pCsr); Index: ext/fts5/fts5_varint.c ================================================================== --- ext/fts5/fts5_varint.c +++ ext/fts5/fts5_varint.c @@ -331,12 +331,15 @@ return fts5PutVarint64(p,v); } int sqlite3Fts5GetVarintLen(u32 iVal){ +#if 0 if( iVal<(1 << 7 ) ) return 1; +#endif + assert( iVal>=(1 << 7) ); if( iVal<(1 << 14) ) return 2; if( iVal<(1 << 21) ) return 3; if( iVal<(1 << 28) ) return 4; return 5; } Index: ext/fts5/test/fts5ad.test ================================================================== --- ext/fts5/test/fts5ad.test +++ ext/fts5/test/fts5ad.test @@ -9,10 +9,12 @@ # #************************************************************************* # This file implements regression tests for SQLite library. The # focus of this script is testing the FTS5 module. # +# More specifically, the focus is on testing prefix queries, both with and +# without prefix indexes. # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ad Index: ext/fts5/test/fts5config.test ================================================================== --- ext/fts5/test/fts5config.test +++ ext/fts5/test/fts5config.test @@ -241,7 +241,11 @@ } { set res [list 1 {malformed detail=... directive}] do_catchsql_test 11.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res } +do_catchsql_test 12.1 { + INSERT INTO t1(t1, rank) VALUES('rank', NULL);; +} {1 {SQL logic error or missing database}} + finish_test Index: ext/fts5/test/fts5detail.test ================================================================== --- ext/fts5/test/fts5detail.test +++ ext/fts5/test/fts5detail.test @@ -82,10 +82,14 @@ } do_execsql_test 2.2 { SELECT fts5_test_poslist(t2) FROM t2('aa'); } {0.0.0} + +do_execsql_test 2.3 { + SELECT fts5_test_collist(t2) FROM t2('aa'); +} {0.0} set ::pc 0 #puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*] #exit Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -464,10 +464,16 @@ $(TOP)/src/shell.c libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB) sqldiff$(EXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h $(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB) + +srcck1$(EXE): $(TOP)/tool/srcck1.c + $(BCC) -o srcck1$(EXE) $(TOP)/tool/srcck1.c + +sourcetest: srcck1$(EXE) sqlite3.c + ./srcck1 sqlite3.c fuzzershell$(EXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h $(TCCX) -o fuzzershell$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(FUZZERSHELL_OPT) $(TOP)/tool/fuzzershell.c sqlite3.c \ $(TLIBS) $(THREADLIB) @@ -753,11 +759,11 @@ ./testfixture$(EXE) $(TOP)/test/extraquick.test $(TESTOPTS) # The default test case. Runs most of the faster standard TCL tests, # and fuzz tests, and sqlite3_analyzer and sqldiff tests. # -test: $(TESTPROGS) fastfuzztest +test: $(TESTPROGS) sourcetest fastfuzztest ./testfixture$(EXE) $(TOP)/test/veryquick.test $(TESTOPTS) # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. # @@ -878,10 +884,11 @@ rm -f showstat4 showstat4.exe rm -f showwal showwal.exe rm -f speedtest1 speedtest1.exe rm -f wordcount wordcount.exe rm -f rbu rbu.exe + rm -f srcck1 srcck1.exe rm -f sqlite3.c sqlite3-*.c fts?amal.c tclsqlite3.c rm -f sqlite3rc.h rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c rm -f sqlite-*-output.vsix Index: src/alter.c ================================================================== --- src/alter.c +++ src/alter.c @@ -693,11 +693,11 @@ sqlite3_value *pVal = 0; int rc; rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_BLOB, &pVal); assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); if( rc!=SQLITE_OK ){ - assert( db->mallocFailed = 1 ); + assert( db->mallocFailed == 1 ); return; } if( !pVal ){ sqlite3ErrorMsg(pParse, "Cannot add a column with non-constant default"); return; Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -6173,11 +6173,11 @@ */ #if SQLITE_DEBUG { CellInfo info; pPage->xParseCell(pPage, pCell, &info); - assert( nHeader=(int)(info.pPayload - pCell) ); + assert( nHeader==(int)(info.pPayload - pCell) ); assert( info.nKey==nKey ); assert( *pnSize == info.nSize ); assert( spaceLeft == info.nLocal ); } #endif @@ -7832,12 +7832,12 @@ int rc = SQLITE_OK; const int nMin = pCur->pBt->usableSize * 2 / 3; u8 aBalanceQuickSpace[13]; u8 *pFree = 0; - TESTONLY( int balance_quick_called = 0 ); - TESTONLY( int balance_deeper_called = 0 ); + VVA_ONLY( int balance_quick_called = 0 ); + VVA_ONLY( int balance_deeper_called = 0 ); do { int iPage = pCur->iPage; MemPage *pPage = pCur->apPage[iPage]; @@ -7846,11 +7846,12 @@ /* The root page of the b-tree is overfull. In this case call the ** balance_deeper() function to create a new child for the root-page ** and copy the current contents of the root-page to it. The ** next iteration of the do-loop will balance the child page. */ - assert( (balance_deeper_called++)==0 ); + assert( balance_deeper_called==0 ); + VVA_ONLY( balance_deeper_called++ ); rc = balance_deeper(pPage, &pCur->apPage[1]); if( rc==SQLITE_OK ){ pCur->iPage = 1; pCur->aiIdx[0] = 0; pCur->aiIdx[1] = 0; @@ -7885,11 +7886,12 @@ ** The purpose of the following assert() is to check that only a ** single call to balance_quick() is made for each call to this ** function. If this were not verified, a subtle bug involving reuse ** of the aBalanceQuickSpace[] might sneak in. */ - assert( (balance_quick_called++)==0 ); + assert( balance_quick_called==0 ); + VVA_ONLY( balance_quick_called++ ); rc = balance_quick(pParent, pPage, aBalanceQuickSpace); }else #endif { /* In this case, call balance_nonroot() to redistribute cells @@ -9352,11 +9354,12 @@ char zErr[100]; VVA_ONLY( int nRef ); sqlite3BtreeEnter(p); assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); - assert( (nRef = sqlite3PagerRefcount(pBt->pPager))>=0 ); + VVA_ONLY( nRef = sqlite3PagerRefcount(pBt->pPager) ); + assert( nRef>=0 ); sCheck.pBt = pBt; sCheck.pPager = pBt->pPager; sCheck.nPage = btreePagecount(sCheck.pBt); sCheck.mxErr = mxErr; sCheck.nErr = 0; Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -1952,11 +1952,11 @@ pParse->nTab = 2; addrTop = sqlite3VdbeCurrentAddr(v) + 1; sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); sqlite3Select(pParse, pSelect, &dest); - sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield); + sqlite3VdbeEndCoroutine(v, regYield); sqlite3VdbeJumpHere(v, addrTop - 1); if( pParse->nErr ) return; pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); if( pSelTab==0 ) return; assert( p->aCol==0 ); Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -683,11 +683,11 @@ dest.iSdst = bIdListInOrder ? regData : 0; dest.nSdst = pTab->nCol; rc = sqlite3Select(pParse, pSelect, &dest); regFromSelect = dest.iSdst; if( rc || db->mallocFailed || pParse->nErr ) goto insert_cleanup; - sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield); + sqlite3VdbeEndCoroutine(v, regYield); sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ assert( pSelect->pEList ); nColumn = pSelect->pEList->nExpr; /* Set useTempTable to TRUE if the result of the SELECT statement Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -3692,11 +3692,11 @@ ** process aborts. If X is false and assert() is disabled, then the ** return value is zero. */ case SQLITE_TESTCTRL_ASSERT: { volatile int x = 0; - assert( (x = va_arg(ap,int))!=0 ); + assert( /*side-effects-ok*/ (x = va_arg(ap,int))!=0 ); rc = x; break; } Index: src/malloc.c ================================================================== --- src/malloc.c +++ src/malloc.c @@ -624,15 +624,15 @@ p = sqlite3Malloc(n); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); return p; } void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){ +#ifndef SQLITE_OMIT_LOOKASIDE + LookasideSlot *pBuf; assert( db!=0 ); assert( sqlite3_mutex_held(db->mutex) ); assert( db->pnBytesFreed==0 ); -#ifndef SQLITE_OMIT_LOOKASIDE - LookasideSlot *pBuf; if( db->lookaside.bDisable==0 ){ assert( db->mallocFailed==0 ); if( n>db->lookaside.sz ){ db->lookaside.anStat[1]++; }else if( (pBuf = db->lookaside.pFree)==0 ){ @@ -648,10 +648,13 @@ } }else if( db->mallocFailed ){ return 0; } #else + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( db->pnBytesFreed==0 ); if( db->mallocFailed ){ return 0; } #endif return dbMallocRawFinish(db, n); Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -2950,11 +2950,11 @@ addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); VdbeComment((v, "left SELECT")); pPrior->iLimit = regLimitA; explainSetInteger(iSub1, pParse->iNextSelectId); sqlite3Select(pParse, pPrior, &destA); - sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrA); + sqlite3VdbeEndCoroutine(v, regAddrA); sqlite3VdbeJumpHere(v, addr1); /* Generate a coroutine to evaluate the SELECT statement on ** the right - the "B" select */ @@ -2967,11 +2967,11 @@ p->iOffset = 0; explainSetInteger(iSub2, pParse->iNextSelectId); sqlite3Select(pParse, p, &destB); p->iLimit = savedLimit; p->iOffset = savedOffset; - sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrB); + sqlite3VdbeEndCoroutine(v, regAddrB); /* Generate a subroutine that outputs the current row of the A ** select as the next output row of the compound select. */ VdbeNoopComment((v, "Output routine for A")); @@ -4988,11 +4988,11 @@ explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); pItem->fg.viaCoroutine = 1; pItem->regResult = dest.iSdst; - sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn); + sqlite3VdbeEndCoroutine(v, pItem->regReturn); sqlite3VdbeJumpHere(v, addrTop-1); sqlite3ClearTempRegCache(pParse); }else{ /* Generate a subroutine that will fill an ephemeral table with ** the content of this subquery. pItem->addrFillSub will point Index: src/test_config.c ================================================================== --- src/test_config.c +++ src/test_config.c @@ -367,10 +367,16 @@ #ifdef SQLITE_ENABLE_FTS3 Tcl_SetVar2(interp, "sqlite_options", "fts3", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "fts3", "0", TCL_GLOBAL_ONLY); #endif + +#ifdef SQLITE_ENABLE_FTS3_TOKENIZER + Tcl_SetVar2(interp, "sqlite_options", "fts3_tokenizer", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "fts3_tokenizer", "0", TCL_GLOBAL_ONLY); +#endif #ifdef SQLITE_ENABLE_FTS5 Tcl_SetVar2(interp, "sqlite_options", "fts5", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "fts5", "0", TCL_GLOBAL_ONLY); Index: src/tokenize.c ================================================================== --- src/tokenize.c +++ src/tokenize.c @@ -16,16 +16,96 @@ ** parser for analysis. */ #include "sqliteInt.h" #include +/* Character classes for tokenizing +** +** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented +** using a lookup table, whereas a switch() directly on c uses a binary search. +** The lookup table is much faster. To maximize speed, and to ensure that +** a lookup table is used, all of the classes need to be small integers and +** all of them need to be used within the switch. +*/ +#define CC_X 0 /* The letter 'x', or start of BLOB literal */ +#define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */ +#define CC_ID 2 /* unicode characters usable in IDs */ +#define CC_DIGIT 3 /* Digits */ +#define CC_DOLLAR 4 /* '$' */ +#define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */ +#define CC_VARNUM 6 /* '?'. Numeric SQL variables */ +#define CC_SPACE 7 /* Space characters */ +#define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */ +#define CC_QUOTE2 9 /* '['. [...] style quoted ids */ +#define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */ +#define CC_MINUS 11 /* '-'. Minus or SQL-style comment */ +#define CC_LT 12 /* '<'. Part of < or <= or <> */ +#define CC_GT 13 /* '>'. Part of > or >= */ +#define CC_EQ 14 /* '='. Part of = or == */ +#define CC_BANG 15 /* '!'. Part of != */ +#define CC_SLASH 16 /* '/'. / or c-style comment */ +#define CC_LP 17 /* '(' */ +#define CC_RP 18 /* ')' */ +#define CC_SEMI 19 /* ';' */ +#define CC_PLUS 20 /* '+' */ +#define CC_STAR 21 /* '*' */ +#define CC_PERCENT 22 /* '%' */ +#define CC_COMMA 23 /* ',' */ +#define CC_AND 24 /* '&' */ +#define CC_TILDA 25 /* '~' */ +#define CC_DOT 26 /* '.' */ +#define CC_ILLEGAL 27 /* Illegal character */ + +static const unsigned char aiClass[] = { +#ifdef SQLITE_ASCII +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ +/* 0x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27, +/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16, +/* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6, +/* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 9, 27, 27, 27, 1, +/* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27, +/* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +#endif +#ifdef SQLITE_EBCDIC +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ +/* 0x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 7, 7, 27, 27, +/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 2x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 3x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 4x */ 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 12, 17, 20, 10, +/* 5x */ 24, 27, 27, 27, 27, 27, 27, 27, 27, 27, 15, 4, 21, 18, 19, 27, +/* 6x */ 11, 16, 27, 27, 27, 27, 27, 27, 27, 27, 27, 23, 22, 1, 13, 7, +/* 7x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 8, 5, 5, 5, 8, 14, 8, +/* 8x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, +/* 9x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, +/* 9x */ 25, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27, +/* Bx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 9, 27, 27, 27, 27, 27, +/* Cx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, +/* Dx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, +/* Ex */ 27, 27, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27, +/* Fx */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 27, 27, 27, 27, 27, 27, +#endif +}; + /* -** The charMap() macro maps alphabetic characters into their +** The charMap() macro maps alphabetic characters (only) into their ** lower-case ASCII equivalent. On ASCII machines, this is just ** an upper-to-lower case map. On EBCDIC machines we also need -** to adjust the encoding. Only alphabetic characters and underscores -** need to be translated. +** to adjust the encoding. The mapping is only valid for alphabetics +** which are the only characters for which this feature is used. +** +** Used by keywordhash.h */ #ifdef SQLITE_ASCII # define charMap(X) sqlite3UpperToLower[(unsigned char)X] #endif #ifdef SQLITE_EBCDIC @@ -55,11 +135,11 @@ ** The sqlite3KeywordCode function looks up an identifier to determine if ** it is a keyword. If it is a keyword, the token code of that keyword is ** returned. If the input is not a keyword, TK_ID is returned. ** ** The implementation of this routine was generated by a program, -** mkkeywordhash.h, located in the tool subdirectory of the distribution. +** mkkeywordhash.c, located in the tool subdirectory of the distribution. ** The output of the mkkeywordhash.c program is written into a file ** named keywordhash.h and then included into this source file by ** the #include below. */ #include "keywordhash.h" @@ -108,74 +188,76 @@ int sqlite3IsIdChar(u8 c){ return IdChar(c); } #endif /* -** Return the length of the token that begins at z[0]. +** Return the length (in bytes) of the token that begins at z[0]. ** Store the token type in *tokenType before returning. */ int sqlite3GetToken(const unsigned char *z, int *tokenType){ int i, c; - switch( *z ){ - case ' ': case '\t': case '\n': case '\f': case '\r': { + switch( aiClass[*z] ){ /* Switch on the character-class of the first byte + ** of the token. See the comment on the CC_ defines + ** above. */ + case CC_SPACE: { testcase( z[0]==' ' ); testcase( z[0]=='\t' ); testcase( z[0]=='\n' ); testcase( z[0]=='\f' ); testcase( z[0]=='\r' ); for(i=1; sqlite3Isspace(z[i]); i++){} *tokenType = TK_SPACE; return i; } - case '-': { + case CC_MINUS: { if( z[1]=='-' ){ for(i=2; (c=z[i])!=0 && c!='\n'; i++){} *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ return i; } *tokenType = TK_MINUS; return 1; } - case '(': { + case CC_LP: { *tokenType = TK_LP; return 1; } - case ')': { + case CC_RP: { *tokenType = TK_RP; return 1; } - case ';': { + case CC_SEMI: { *tokenType = TK_SEMI; return 1; } - case '+': { + case CC_PLUS: { *tokenType = TK_PLUS; return 1; } - case '*': { + case CC_STAR: { *tokenType = TK_STAR; return 1; } - case '/': { + case CC_SLASH: { if( z[1]!='*' || z[2]==0 ){ *tokenType = TK_SLASH; return 1; } for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} if( c ) i++; *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ return i; } - case '%': { + case CC_PERCENT: { *tokenType = TK_REM; return 1; } - case '=': { + case CC_EQ: { *tokenType = TK_EQ; return 1 + (z[1]=='='); } - case '<': { + case CC_LT: { if( (c=z[1])=='=' ){ *tokenType = TK_LE; return 2; }else if( c=='>' ){ *tokenType = TK_NE; @@ -186,11 +268,11 @@ }else{ *tokenType = TK_LT; return 1; } } - case '>': { + case CC_GT: { if( (c=z[1])=='=' ){ *tokenType = TK_GE; return 2; }else if( c=='>' ){ *tokenType = TK_RSHIFT; @@ -198,43 +280,41 @@ }else{ *tokenType = TK_GT; return 1; } } - case '!': { + case CC_BANG: { if( z[1]!='=' ){ *tokenType = TK_ILLEGAL; return 2; }else{ *tokenType = TK_NE; return 2; } } - case '|': { + case CC_PIPE: { if( z[1]!='|' ){ *tokenType = TK_BITOR; return 1; }else{ *tokenType = TK_CONCAT; return 2; } } - case ',': { + case CC_COMMA: { *tokenType = TK_COMMA; return 1; } - case '&': { + case CC_AND: { *tokenType = TK_BITAND; return 1; } - case '~': { + case CC_TILDA: { *tokenType = TK_BITNOT; return 1; } - case '`': - case '\'': - case '"': { + case CC_QUOTE: { int delim = z[0]; testcase( delim=='`' ); testcase( delim=='\'' ); testcase( delim=='"' ); for(i=1; (c=z[i])!=0; i++){ @@ -255,11 +335,11 @@ }else{ *tokenType = TK_ILLEGAL; return i; } } - case '.': { + case CC_DOT: { #ifndef SQLITE_OMIT_FLOATING_POINT if( !sqlite3Isdigit(z[1]) ) #endif { *tokenType = TK_DOT; @@ -266,12 +346,11 @@ return 1; } /* If the next character is a digit, this is a floating point ** number that begins with ".". Fall thru into the next case */ } - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': { + case CC_DIGIT: { testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' ); testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' ); testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); testcase( z[0]=='9' ); *tokenType = TK_INTEGER; @@ -302,26 +381,22 @@ *tokenType = TK_ILLEGAL; i++; } return i; } - case '[': { + case CC_QUOTE2: { for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} *tokenType = c==']' ? TK_ID : TK_ILLEGAL; return i; } - case '?': { + case CC_VARNUM: { *tokenType = TK_VARIABLE; for(i=1; sqlite3Isdigit(z[i]); i++){} return i; } -#ifndef SQLITE_OMIT_TCL_VARIABLE - case '$': -#endif - case '@': /* For compatibility with MS SQL Server */ - case '#': - case ':': { + case CC_DOLLAR: + case CC_VARALPHA: { int n = 0; testcase( z[0]=='$' ); testcase( z[0]=='@' ); testcase( z[0]==':' ); testcase( z[0]=='#' ); *tokenType = TK_VARIABLE; for(i=1; (c=z[i])!=0; i++){ @@ -346,12 +421,24 @@ } } if( n==0 ) *tokenType = TK_ILLEGAL; return i; } + case CC_KYWD: { + for(i=1; aiClass[z[i]]<=CC_KYWD; i++){} + if( IdChar(z[i]) ){ + /* This token started out using characters that can appear in keywords, + ** but z[i] is a character not allowed within keywords, so this must + ** be an identifier instead */ + i++; + break; + } + *tokenType = TK_ID; + return keywordCode((char*)z, i, tokenType); + } #ifndef SQLITE_OMIT_BLOB_LITERAL - case 'x': case 'X': { + case CC_X: { testcase( z[0]=='x' ); testcase( z[0]=='X' ); if( z[1]=='\'' ){ *tokenType = TK_BLOB; for(i=2; sqlite3Isxdigit(z[i]); i++){} if( z[i]!='\'' || i%2 ){ @@ -359,24 +446,26 @@ while( z[i] && z[i]!='\'' ){ i++; } } if( z[i] ) i++; return i; } - /* Otherwise fall through to the next case */ + /* If it is not a BLOB literal, then it must be an ID, since no + ** SQL keywords start with the letter 'x'. Fall through */ } #endif + case CC_ID: { + i = 1; + break; + } default: { - if( !IdChar(*z) ){ - break; - } - for(i=1; IdChar(z[i]); i++){} - *tokenType = TK_ID; - return keywordCode((char*)z, i, tokenType); + *tokenType = TK_ILLEGAL; + return 1; } } - *tokenType = TK_ILLEGAL; - return 1; + while( IdChar(z[i]) ){ i++; } + *tokenType = TK_ID; + return i; } /* ** Run the parser on the given SQL string. The parser structure is ** passed in. An SQLITE_ status code is returned. If an error occurs Index: src/vdbe.h ================================================================== --- src/vdbe.h +++ src/vdbe.h @@ -178,10 +178,11 @@ void sqlite3VdbeMultiLoad(Vdbe*,int,const char*,...); int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); int sqlite3VdbeAddOp4Dup8(Vdbe*,int,int,int,int,const u8*,int); int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); +void sqlite3VdbeEndCoroutine(Vdbe*,int); #if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N); #else # define sqlite3VdbeVerifyNoMallocRequired(A,B) #endif Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -321,10 +321,25 @@ ){ int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3); sqlite3VdbeChangeP4(p, addr, SQLITE_INT_TO_PTR(p4), P4_INT32); return addr; } + +/* Insert the end of a co-routine +*/ +void sqlite3VdbeEndCoroutine(Vdbe *v, int regYield){ + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield); + + /* Clear the temporary register cache, thereby ensuring that each + ** co-routine has its own independent set of registers, because co-routines + ** might expect their registers to be preserved across an OP_Yield, and + ** that could cause problems if two or more co-routines are using the same + ** temporary register. + */ + v->pParse->nTempReg = 0; + v->pParse->nRangeReg = 0; +} /* ** Create a new symbolic label for an instruction that has yet to be ** coded. The symbolic label is really just a negative number. The ** label can be used as the P2 value of an operation. Later, when Index: test/analyze9.test ================================================================== --- test/analyze9.test +++ test/analyze9.test @@ -1242,8 +1242,5 @@ 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x=? AND y>?)} } finish_test - - - Index: test/analyzeB.test ================================================================== --- test/analyzeB.test +++ test/analyzeB.test @@ -678,6 +678,5 @@ SELECT count(*) FROM sqlite_stat3 WHERE sample=$val } {1} } finish_test - Index: test/analyzeD.test ================================================================== --- test/analyzeD.test +++ test/analyzeD.test @@ -112,6 +112,5 @@ } { 0 0 0 {SEARCH TABLE t1 USING INDEX t1_c (c=?)} } finish_test - Index: test/cacheflush.test ================================================================== --- test/cacheflush.test +++ test/cacheflush.test @@ -319,6 +319,5 @@ } } {a b} test_restore_config_pagecache finish_test - Index: test/cffault.test ================================================================== --- test/cffault.test +++ test/cffault.test @@ -154,6 +154,5 @@ catchsql ROLLBACK faultsim_integrity_check } finish_test - Index: test/corruptH.test ================================================================== --- test/corruptH.test +++ test/corruptH.test @@ -172,6 +172,5 @@ } } msg] $msg } {1 {database disk image is malformed}} finish_test - Index: test/corruptI.test ================================================================== --- test/corruptI.test +++ test/corruptI.test @@ -204,11 +204,10 @@ } {} do_test 6.1 { db close hexio_write test.db 616 8FFFFFFF7F02 sqlite3 db test.db - breakpoint execsql { DELETE FROM t1 WHERE rowid=2 } } {} #------------------------------------------------------------------------- # See what happens if the sqlite_master entry associated with a PRIMARY Index: test/cost.test ================================================================== --- test/cost.test +++ test/cost.test @@ -285,8 +285,5 @@ 0 0 0 {SEARCH TABLE t6 USING INDEX t6i1 (a=? AND b=?)} } } finish_test - - - Index: test/e_blobbytes.test ================================================================== --- test/e_blobbytes.test +++ test/e_blobbytes.test @@ -70,7 +70,5 @@ list [catch { sqlite3_blob_write $B 86 "1" 1 } msg] $msg } {1 SQLITE_ERROR} sqlite3_blob_close $B finish_test - - Index: test/e_blobclose.test ================================================================== --- test/e_blobclose.test +++ test/e_blobclose.test @@ -166,6 +166,5 @@ # a harmless no-op. # do_test 4.0 { sqlite3_blob_close 0 } {} finish_test - Index: test/e_blobopen.test ================================================================== --- test/e_blobopen.test +++ test/e_blobopen.test @@ -544,6 +544,5 @@ [string repeat [binary format c 1] 45] \ ] finish_test - Index: test/e_blobwrite.test ================================================================== --- test/e_blobwrite.test +++ test/e_blobwrite.test @@ -199,6 +199,5 @@ } finish_test - Index: test/e_walckpt.test ================================================================== --- test/e_walckpt.test +++ test/e_walckpt.test @@ -749,6 +749,5 @@ } {0 0 0} finish_test - Index: test/fkey8.test ================================================================== --- test/fkey8.test +++ test/fkey8.test @@ -101,6 +101,5 @@ } $use_stmt } finish_test - Index: test/fordelete.test ================================================================== --- test/fordelete.test +++ test/fordelete.test @@ -205,6 +205,5 @@ DELETE FROM t1 WHERE a = 2; } {} finish_test - Index: test/fts3atoken.test ================================================================== --- test/fts3atoken.test +++ test/fts3atoken.test @@ -22,11 +22,11 @@ ifcapable !fts3 { finish_test return } -set ::testprefix fts3token +set ::testprefix fts3atoken proc escape_string {str} { set out "" foreach char [split $str ""] { scan $char %c i @@ -38,11 +38,11 @@ } set out } #-------------------------------------------------------------------------- -# Test cases fts3token-1.* are the warm-body test for the SQL scalar +# Test cases fts3atoken-1.* are the warm-body test for the SQL scalar # function fts3_tokenizer(). The procedure is as follows: # # 1: Verify that there is no such fts3 tokenizer as 'blah'. # # 2: Query for the built-in tokenizer 'simple'. Insert a copy of the @@ -54,74 +54,80 @@ # 4: Test that it is now possible to create an fts3 table using # tokenizer 'blah' (it was not possible in step 1). # # 5: Test that the table created to use tokenizer 'blah' is usable. # -do_test fts3token-1.1 { - catchsql { - CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah); - } -} {1 {unknown tokenizer: blah}} -do_test fts3token-1.2 { - execsql { +ifcapable fts3_tokenizer { + do_test fts3atoken-1.1 { + catchsql { + CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah); + } + } {1 {unknown tokenizer: blah}} + do_test fts3atoken-1.2 { + execsql { + SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL; + } + } {0} + do_test fts3atoken-1.3 { + execsql { + SELECT fts3_tokenizer('blah') == fts3_tokenizer('simple'); + } + } {1} + do_test fts3atoken-1.4 { + catchsql { + CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah); + } + } {0 {}} + do_test fts3atoken-1.5 { + execsql { + INSERT INTO t1(content) VALUES('There was movement at the station'); + INSERT INTO t1(content) VALUES('For the word has passed around'); + INSERT INTO t1(content) VALUES('That the colt from ol regret had got'); + SELECT content FROM t1 WHERE content MATCH 'movement' + } + } {{There was movement at the station}} +} else { + do_catchsql_test 1.6 { SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL; - } -} {0} -do_test fts3token-1.3 { - execsql { - SELECT fts3_tokenizer('blah') == fts3_tokenizer('simple'); - } -} {1} -do_test fts3token-1.4 { - catchsql { - CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah); - } -} {0 {}} -do_test fts3token-1.5 { - execsql { - INSERT INTO t1(content) VALUES('There was movement at the station'); - INSERT INTO t1(content) VALUES('For the word has passed around'); - INSERT INTO t1(content) VALUES('That the colt from ol regret had got away'); - SELECT content FROM t1 WHERE content MATCH 'movement' - } -} {{There was movement at the station}} + } {1 {fts3tokenize: disabled - rebuild with -DSQLITE_ENABLE_FTS3_TOKENIZER}} +} #-------------------------------------------------------------------------- -# Test cases fts3token-2.* test error cases in the scalar function based +# Test cases fts3atoken-2.* test error cases in the scalar function based # API for getting and setting tokenizers. # -do_test fts3token-2.1 { +do_test fts3atoken-2.1 { catchsql { SELECT fts3_tokenizer('nosuchtokenizer'); } } {1 {unknown tokenizer: nosuchtokenizer}} #-------------------------------------------------------------------------- -# Test cases fts3token-3.* test the three built-in tokenizers with a +# Test cases fts3atoken-3.* test the three built-in tokenizers with a # simple input string via the built-in test function. This is as much # to test the test function as the tokenizer implementations. # -do_test fts3token-3.1 { +do_test fts3atoken-3.1 { execsql { SELECT fts3_tokenizer_test('simple', 'I don''t see how'); } } {{0 i I 1 don don 2 t t 3 see see 4 how how}} -do_test fts3token-3.2 { +do_test fts3atoken-3.2 { execsql { SELECT fts3_tokenizer_test('porter', 'I don''t see how'); } } {{0 i I 1 don don 2 t t 3 see see 4 how how}} ifcapable icu { - do_test fts3token-3.3 { + do_test fts3atoken-3.3 { execsql { SELECT fts3_tokenizer_test('icu', 'I don''t see how'); } } {{0 i I 1 don't don't 2 see see 3 how how}} } #-------------------------------------------------------------------------- -# Test cases fts3token-4.* test the ICU tokenizer. In practice, this +# Test cases fts3atoken-4.* test the ICU tokenizer. In practice, this # tokenizer only has two modes - "thai" and "everybody else". Some other # Asian languages (Lao, Khmer etc.) require the same special treatment as # Thai, but ICU doesn't support them yet. # ifcapable icu { @@ -131,12 +137,12 @@ do_test $name { lindex $::out 0 } $output } - do_icu_test fts3token-4.1 en_US {} {} - do_icu_test fts3token-4.2 en_US {Test cases fts3} [list \ + do_icu_test fts3atoken-4.1 en_US {} {} + do_icu_test fts3atoken-4.2 en_US {Test cases fts3} [list \ 0 test Test 1 cases cases 2 fts3 fts3 ] # The following test shows that ICU is smart enough to recognise # Thai chararacters, even when the locale is set to English/United @@ -145,16 +151,16 @@ set input "\u0e2d\u0e30\u0e44\u0e23\u0e19\u0e30\u0e04\u0e23\u0e31\u0e1a" set output "0 \u0e2d\u0e30\u0e44\u0e23 \u0e2d\u0e30\u0e44\u0e23 " append output "1 \u0e19\u0e30 \u0e19\u0e30 " append output "2 \u0e04\u0e23\u0e31\u0e1a \u0e04\u0e23\u0e31\u0e1a" - do_icu_test fts3token-4.3 th_TH $input $output - do_icu_test fts3token-4.4 en_US $input $output + do_icu_test fts3atoken-4.3 th_TH $input $output + do_icu_test fts3atoken-4.4 en_US $input $output # ICU handles an unknown locale by falling back to the default. # So this is not an error. - do_icu_test fts3token-4.5 MiddleOfTheOcean $input $output + do_icu_test fts3atoken-4.5 MiddleOfTheOcean $input $output set longtoken "AReallyReallyLongTokenOneThatWillSurelyRequire" append longtoken "AReallocInTheIcuTokenizerCode" set input "short tokens then " @@ -162,13 +168,13 @@ set output "0 short short " append output "1 tokens tokens " append output "2 then then " append output "3 [string tolower $longtoken] $longtoken" - do_icu_test fts3token-4.6 MiddleOfTheOcean $input $output - do_icu_test fts3token-4.7 th_TH $input $output - do_icu_test fts3token-4.8 en_US $input $output + do_icu_test fts3atoken-4.6 MiddleOfTheOcean $input $output + do_icu_test fts3atoken-4.7 th_TH $input $output + do_icu_test fts3atoken-4.8 en_US $input $output do_execsql_test 5.1 { CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US); insert into x1 (name) values (NULL); insert into x1 (name) values (NULL); @@ -184,11 +190,11 @@ set str [cp_to_str {19968 26085 32822 32645 27874 23433 20986}] execsql { INSERT INTO x1 VALUES($str) } } {} } -do_test fts3token-internal { +do_test fts3atoken-internal { execsql { SELECT fts3_tokenizer_internal_test() } } {ok} #------------------------------------------------------------------------- # Test empty tokenizer names. @@ -204,14 +210,16 @@ } {1 {unknown tokenizer: }} do_catchsql_test 6.2.1 { SELECT fts3_tokenizer(NULL); } {1 {unknown tokenizer: }} -do_catchsql_test 6.2.2 { - SELECT fts3_tokenizer(NULL, X'1234567812345678'); -} {1 {argument type mismatch}} -do_catchsql_test 6.2.3 { - SELECT fts3_tokenizer(NULL, X'12345678'); -} {1 {argument type mismatch}} +ifcapable fts3_tokenizer { + do_catchsql_test 6.2.2 { + SELECT fts3_tokenizer(NULL, X'1234567812345678'); + } {1 {argument type mismatch}} + do_catchsql_test 6.2.3 { + SELECT fts3_tokenizer(NULL, X'12345678'); + } {1 {argument type mismatch}} +} finish_test Index: test/fts3conf.test ================================================================== --- test/fts3conf.test +++ test/fts3conf.test @@ -210,6 +210,5 @@ SELECT * FROM t01 WHERE t01 MATCH 'b'; INSERT INTO t01(t01) VALUES('integrity-check'); } {} finish_test - Index: test/fts3expr4.test ================================================================== --- test/fts3expr4.test +++ test/fts3expr4.test @@ -77,6 +77,5 @@ do_icu_expr_test 3.9 { "ab*c" } { PHRASE 3 0 ab+ * c } do_icu_expr_test 3.10 { ab*c } { AND {PHRASE 3 0 ab+} {PHRASE 3 0 c}} finish_test - Index: test/fts3join.test ================================================================== --- test/fts3join.test +++ test/fts3join.test @@ -60,7 +60,5 @@ do_catchsql_test 2.5 { SELECT * FROM ft3, ft2 WHERE y MATCH x AND x MATCH y; } {1 {unable to use function MATCH in the requested context}} finish_test - - Index: test/fts3matchinfo.test ================================================================== --- test/fts3matchinfo.test +++ test/fts3matchinfo.test @@ -522,11 +522,10 @@ } do_execsql_test 11.1.$tn.2 { SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr } $r2 - breakpoint do_execsql_test 11.1.$tn.2 { SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr } $r2 } @@ -550,6 +549,5 @@ SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc'; } [list [list [expr 1<<4] [expr 1<<(45-32)]]] set sqlite_fts3_enable_parentheses 0 finish_test - Index: test/fts3offsets.test ================================================================== --- test/fts3offsets.test +++ test/fts3offsets.test @@ -119,6 +119,5 @@ } set sqlite_fts3_enable_parentheses 0 finish_test - Index: test/fts3snippet.test ================================================================== --- test/fts3snippet.test +++ test/fts3snippet.test @@ -557,6 +557,5 @@ set sqlite_fts3_enable_parentheses 0 finish_test - Index: test/fts4check.test ================================================================== --- test/fts4check.test +++ test/fts4check.test @@ -208,6 +208,5 @@ CREATE VIRTUAL TABLE t5 USING fts4(a, prefix="1,2,3"); INSERT INTO t5(t5) VALUES('integrity-check'); } {} finish_test - Index: test/fts4content.test ================================================================== --- test/fts4content.test +++ test/fts4content.test @@ -634,6 +634,5 @@ CREATE VIRTUAL TABLE x1 USING fts4(content=x1); } {1 {vtable constructor called recursively: x1}} finish_test - Index: test/fts4growth.test ================================================================== --- test/fts4growth.test +++ test/fts4growth.test @@ -432,6 +432,5 @@ } {633507} finish_test - Index: test/fts4growth2.test ================================================================== --- test/fts4growth2.test +++ test/fts4growth2.test @@ -88,6 +88,5 @@ } {1} } finish_test - Index: test/fts4langid.test ================================================================== --- test/fts4langid.test +++ test/fts4langid.test @@ -356,33 +356,35 @@ INSERT INTO t4(docid, content, lid) VALUES($i, 'The Quick Brown Fox', $i) } } } -do_test 4.1.0 { - reset_db - set ptr [fts3_test_tokenizer] - execsql { SELECT fts3_tokenizer('testtokenizer', $ptr) } - build_multilingual_db_2 db -} {} -do_execsql_test 4.1.1 { - SELECT docid FROM t4 WHERE t4 MATCH 'quick'; -} {0} -do_execsql_test 4.1.2 { - SELECT docid FROM t4 WHERE t4 MATCH 'quick' AND lid=1; -} {} -do_execsql_test 4.1.3 { - SELECT docid FROM t4 WHERE t4 MATCH 'Quick' AND lid=1; -} {1} -for {set i 0} {$i < 50} {incr i} { - do_execsql_test 4.1.4.$i { - SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i; - } [expr 0==($i%2)] -} -do_catchsql_test 4.1.5 { - INSERT INTO t4(content, lid) VALUES('hello world', 101) -} {1 {SQL logic error or missing database}} +ifcapable fts3_tokenizer { + do_test 4.1.0 { + reset_db + set ptr [fts3_test_tokenizer] + execsql { SELECT fts3_tokenizer('testtokenizer', $ptr) } + build_multilingual_db_2 db + } {} + do_execsql_test 4.1.1 { + SELECT docid FROM t4 WHERE t4 MATCH 'quick'; + } {0} + do_execsql_test 4.1.2 { + SELECT docid FROM t4 WHERE t4 MATCH 'quick' AND lid=1; + } {} + do_execsql_test 4.1.3 { + SELECT docid FROM t4 WHERE t4 MATCH 'Quick' AND lid=1; + } {1} + for {set i 0} {$i < 50} {incr i} { + do_execsql_test 4.1.4.$i { + SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i; + } [expr 0==($i%2)] + } + do_catchsql_test 4.1.5 { + INSERT INTO t4(content, lid) VALUES('hello world', 101) + } {1 {SQL logic error or missing database}} +} #------------------------------------------------------------------------- # Test cases 5.* # # The following test cases are designed to detect a 32-bit overflow bug Index: test/fts4noti.test ================================================================== --- test/fts4noti.test +++ test/fts4noti.test @@ -226,8 +226,5 @@ SELECT count(*) FROM t2 WHERE t2 MATCH 'yes'; SELECT count(*) FROM t2 WHERE t2 MATCH 'yep'; } {0 1 1 0 1 1} finish_test - - - Index: test/fts4onepass.test ================================================================== --- test/fts4onepass.test +++ test/fts4onepass.test @@ -142,6 +142,5 @@ } eval $tcl2 } finish_test - Index: test/fuzz3.test ================================================================== --- test/fuzz3.test +++ test/fuzz3.test @@ -171,6 +171,5 @@ } $::cksum } test_restore_config_pagecache finish_test - Index: test/incrcorrupt.test ================================================================== --- test/incrcorrupt.test +++ test/incrcorrupt.test @@ -122,6 +122,5 @@ do_test 2.15 { sqlite3_finalize $stmt } {SQLITE_CORRUPT} do_test 2.16 { sqlite3_errcode db } {SQLITE_CORRUPT} do_test 2.17 { sqlite3_errmsg db } {database disk image is malformed} finish_test - Index: test/mallocK.test ================================================================== --- test/mallocK.test +++ test/mallocK.test @@ -170,6 +170,5 @@ faultsim_test_result [list 0 {}] } finish_test - Index: test/mallocL.test ================================================================== --- test/mallocL.test +++ test/mallocL.test @@ -38,6 +38,5 @@ } } finish_test - Index: test/ovfl.test ================================================================== --- test/ovfl.test +++ test/ovfl.test @@ -43,7 +43,5 @@ do_execsql_test 1.2 { SELECT sum(length(c2)) FROM t1; } [expr 2000 * 2000] finish_test - - Index: test/pragma2.test ================================================================== --- test/pragma2.test +++ test/pragma2.test @@ -253,11 +253,10 @@ } {main unlocked temp unknown aux1 exclusive} db close forcedelete test.db sqlite3 db test.db -breakpoint do_execsql_test pragma2-5.1 { PRAGMA page_size=16384; CREATE TABLE t1(x); PRAGMA cache_size=2; PRAGMA cache_spill=YES; Index: test/quota.test ================================================================== --- test/quota.test +++ test/quota.test @@ -19,10 +19,11 @@ return } source $testdir/malloc_common.tcl +forcedelete bak.db unset -nocomplain defaultVfs set defaultVfs [file_control_vfsname db] db close do_test quota-1.1 { sqlite3_quota_initialize nosuchvfs 1 } {SQLITE_ERROR} Index: test/rollback2.test ================================================================== --- test/rollback2.test +++ test/rollback2.test @@ -152,6 +152,5 @@ } -result { 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 } finish_test - Index: test/rollbackfault.test ================================================================== --- test/rollbackfault.test +++ test/rollbackfault.test @@ -78,7 +78,5 @@ } } finish_test - - Index: test/select4.test ================================================================== --- test/select4.test +++ test/select4.test @@ -913,7 +913,27 @@ VALUES(1),(2),(3),(4) UNION ALL SELECT 5 LIMIT 99; } {1 2 3 4 5} do_execsql_test select4-14.17 { VALUES(1),(2),(3),(4) UNION ALL SELECT 5 LIMIT 3; } {1 2 3} + +# Ticket https://www.sqlite.org/src/info/d06a25c84454a372 +# Incorrect answer due to two co-routines using the same registers and expecting +# those register values to be preserved across a Yield. +# +do_execsql_test select4-15.1 { + DROP TABLE IF EXISTS tx; + CREATE TABLE tx(id INTEGER PRIMARY KEY, a, b); + INSERT INTO tx(a,b) VALUES(33,456); + INSERT INTO tx(a,b) VALUES(33,789); + + SELECT DISTINCT t0.id, t0.a, t0.b + FROM tx AS t0, tx AS t1 + WHERE t0.a=t1.a AND t1.a=33 AND t0.b=456 + UNION + SELECT DISTINCT t0.id, t0.a, t0.b + FROM tx AS t0, tx AS t1 + WHERE t0.a=t1.a AND t1.a=33 AND t0.b=789 + ORDER BY 1; +} {1 33 456 2 33 789} finish_test Index: test/select7.test ================================================================== --- test/select7.test +++ test/select7.test @@ -218,7 +218,5 @@ EXPLAIN QUERY PLAN SELECT * FROM v0 WHERE x='0' OR y; } {1 {SELECTs to the left and right of UNION do not have the same number of result columns}} finish_test - - Index: test/shared3.test ================================================================== --- test/shared3.test +++ test/shared3.test @@ -137,6 +137,5 @@ execsql { COMMIT } } {} sqlite3_enable_shared_cache $::enable_shared_cache finish_test - Index: test/snapshot.test ================================================================== --- test/snapshot.test +++ test/snapshot.test @@ -53,11 +53,10 @@ do_execsql_test 2.1.0 { BEGIN; SELECT * FROM t1; } {1 2 3 4 5 6 7 8} -breakpoint do_test 2.1.1 { set snapshot [sqlite3_snapshot_get db main] execsql { COMMIT; INSERT INTO t1 VALUES(9, 10); Index: test/sort3.test ================================================================== --- test/sort3.test +++ test/sort3.test @@ -109,6 +109,5 @@ 440000 440000000 440000 440000000 } finish_test - Index: test/sort5.test ================================================================== --- test/sort5.test +++ test/sort5.test @@ -40,6 +40,5 @@ } db close tvfs delete finish_test - Index: test/spellfix.test ================================================================== --- test/spellfix.test +++ test/spellfix.test @@ -401,6 +401,5 @@ do_test 7.5.2.$tn.3 { sqlite3_get_autocommit db } $bRollback catchsql ROLLBACK } finish_test - Index: test/sqllog.test ================================================================== --- test/sqllog.test +++ test/sqllog.test @@ -108,9 +108,6 @@ sqlite3_shutdown unset ::env(SQLITE_SQLLOG_DIR) unset ::env(SQLITE_SQLLOG_CONDITIONAL) sqlite3_config_sqllog sqlite3_initialize -breakpoint finish_test - - Index: test/tkt-9f2eb3abac.test ================================================================== --- test/tkt-9f2eb3abac.test +++ test/tkt-9f2eb3abac.test @@ -74,6 +74,5 @@ } -test { faultsim_test_result {0 {}} } finish_test - Index: test/tkt-ba7cbfaedc.test ================================================================== --- test/tkt-ba7cbfaedc.test +++ test/tkt-ba7cbfaedc.test @@ -59,7 +59,5 @@ } { 1 2 3 4 5 1 2 3 4 5 5 4 3 2 1 } finish_test - - Index: test/triggerE.test ================================================================== --- test/triggerE.test +++ test/triggerE.test @@ -106,7 +106,5 @@ SELECT * FROM t3; SELECT * FROM t2; } {1 2 x y z z} finish_test - - Index: test/vtab_shared.test ================================================================== --- test/vtab_shared.test +++ test/vtab_shared.test @@ -274,6 +274,5 @@ db2 close } sqlite3_enable_shared_cache 0 finish_test - Index: test/wal6.test ================================================================== --- test/wal6.test +++ test/wal6.test @@ -236,6 +236,5 @@ catchsql { SELECT * FROM t2 } db2 } {1 {database disk image is malformed}} finish_test - Index: test/waloverwrite.test ================================================================== --- test/waloverwrite.test +++ test/waloverwrite.test @@ -129,11 +129,10 @@ for {set i 0} {$i < 5} {incr i} { foreach x [db eval {SELECT x FROM t1}] { execsql { UPDATE t1 SET y = randomblob(797) WHERE x=$x } } } - breakpoint execsql {ROLLBACK TO abc} } set nPg [wal_frame_count test.db-wal 1024] @@ -159,6 +158,5 @@ } ok db2 close } finish_test - Index: test/whereI.test ================================================================== --- test/whereI.test +++ test/whereI.test @@ -87,6 +87,5 @@ } { 2.1 2.2 1.2 } finish_test - Index: test/withM.test ================================================================== --- test/withM.test +++ test/withM.test @@ -70,8 +70,5 @@ faultsim_test_result {0 {1 1 2 4 3 9 4 16 5 25}} db close } finish_test - - - Index: tool/mkkeywordhash.c ================================================================== --- tool/mkkeywordhash.c +++ tool/mkkeywordhash.c @@ -275,11 +275,14 @@ }; /* Number of keywords */ static int nKeyword = (sizeof(aKeywordTable)/sizeof(aKeywordTable[0])); -/* Map all alphabetic characters into the same case */ +/* Map all alphabetic characters into lower-case for hashing. This is +** only valid for alphabetics. In particular it does not work for '_' +** and so the hash cannot be on a keyword position that might be an '_'. +*/ #define charMap(X) (0x20|(X)) /* ** Comparision function for two Keyword records */ @@ -563,24 +566,32 @@ j = 0; } } printf("%s };\n", j==0 ? "" : "\n"); - printf(" int h, i;\n"); + printf(" int i, j;\n"); + printf(" const char *zKW;\n"); printf(" if( n>=2 ){\n"); - printf(" h = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n) %% %d;\n", + printf(" i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n) %% %d;\n", bestSize); - printf(" for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){\n"); - printf(" if( aLen[i]==n &&" - " sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){\n"); + printf(" for(i=((int)aHash[i])-1; i>=0; i=((int)aNext[i])-1){\n"); + printf(" if( aLen[i]!=n ) continue;\n"); + printf(" j = 0;\n"); + printf(" zKW = &zText[aOffset[i]];\n"); + printf("#ifdef SQLITE_ASCII\n"); + printf(" while( j +#include +#include +#include + +/* Read the complete text of a file into memory. Return a pointer to +** the result. Panic if unable to read the file or allocate memory. +*/ +static char *readFile(const char *zFilename){ + FILE *in; + char *z; + long n; + size_t got; + + in = fopen(zFilename, "rb"); + if( in==0 ){ + fprintf(stderr, "unable to open '%s' for reading\n", zFilename); + exit(1); + } + fseek(in, 0, SEEK_END); + n = ftell(in); + rewind(in); + z = malloc( n+1 ); + if( z==0 ){ + fprintf(stderr, "cannot allocate %d bytes to store '%s'\n", + (int)(n+1), zFilename); + exit(1); + } + got = fread(z, 1, n, in); + fclose(in); + if( got!=(size_t)n ){ + fprintf(stderr, "only read %d of %d bytes from '%s'\n", + (int)got, (int)n, zFilename); + exit(1); + } + z[n] = 0; + return z; +} + +/* Change the C code in the argument to see if it might have +** side effects. The only accurate way to know this is to do a full +** parse of the C code, which this routine does not do. This routine +** uses a simple heuristic of looking for: +** +** * '=' not immediately after '>', '<', '!', or '='. +** * '++' +** * '--' +** +** If the code contains the phrase "side-effects-ok" is inside a +** comment, then always return false. This is used to disable checking +** for assert()s with deliberate side-effects, such as used by +** SQLITE_TESTCTRL_ASSERT - a facility that allows applications to +** determine at runtime whether or not assert()s are enabled. +** Obviously, that determination cannot be made unless the assert() +** has some side-effect. +** +** Return true if a side effect is seen. Return false if not. +*/ +static int hasSideEffect(const char *z, unsigned int n){ + unsigned int i; + for(i=0; i0 && z[i-1]!='=' && z[i-1]!='>' + && z[i-1]!='<' && z[i-1]!='!' && z[i+1]!='=' ) return 1; + if( z[i]=='+' && z[i+1]=='+' ) return 1; + if( z[i]=='-' && z[i+1]=='-' ) return 1; + } + return 0; +} + +/* Return the number of bytes in string z[] prior to the first unmatched ')' +** character. +*/ +static unsigned int findCloseParen(const char *z){ + unsigned int nOpen = 0; + unsigned i; + for(i=0; z[i]; i++){ + if( z[i]=='(' ) nOpen++; + if( z[i]==')' ){ + if( nOpen==0 ) break; + nOpen--; + } + } + return i; +} + +/* Search for instances of assert(...), ALWAYS(...), NEVER(...), and/or +** testcase(...) where the argument contains side effects. +** +** Print error messages whenever a side effect is found. Return the number +** of problems seen. +*/ +static unsigned int findAllSideEffects(const char *z){ + unsigned int lineno = 1; /* Line number */ + unsigned int i; + unsigned int nErr = 0; + char c, prevC = 0; + for(i=0; (c = z[i])!=0; prevC=c, i++){ + if( c=='\n' ){ lineno++; continue; } + if( isalpha(c) && !isalpha(prevC) ){ + if( strncmp(&z[i],"assert(",7)==0 + || strncmp(&z[i],"ALWAYS(",7)==0 + || strncmp(&z[i],"NEVER(",6)==0 + || strncmp(&z[i],"testcase(",9)==0 + ){ + unsigned int n; + const char *z2 = &z[i+5]; + while( z2[0]!='(' ){ z2++; } + z2++; + n = findCloseParen(z2); + if( hasSideEffect(z2, n) ){ + nErr++; + fprintf(stderr, "side-effect line %u: %.*s\n", lineno, + (int)(&z2[n+1] - &z[i]), &z[i]); + } + } + } + } + return nErr; +} + +int main(int argc, char **argv){ + char *z; + unsigned int nErr = 0; + if( argc!=2 ){ + fprintf(stderr, "Usage: %s FILENAME\n", argv[0]); + return 1; + } + z = readFile(argv[1]); + nErr = findAllSideEffects(z); + free(z); + if( nErr ){ + fprintf(stderr, "Found %u undesirable side-effects\n", nErr); + return 1; + } + return 0; +}