Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -1062,16 +1062,13 @@ # Rules to build parse.c and parse.h - the outputs of lemon. # parse.h: parse.c -parse.c: $(TOP)/src/parse.y lemon$(BEXE) $(TOP)/tool/addopcodes.tcl +parse.c: $(TOP)/src/parse.y lemon$(BEXE) cp $(TOP)/src/parse.y . - rm -f parse.h ./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y - mv parse.h parse.h.temp - $(TCLSH_CMD) $(TOP)/tool/addopcodes.tcl parse.h.temp >parse.h sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION $(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h keywordhash.h: $(TOP)/tool/mkkeywordhash.c @@ -1386,12 +1383,13 @@ # This target will fail if the SQLite amalgamation contains any exported # symbols that do not begin with "sqlite3_". It is run as part of the # releasetest.tcl script. # VALIDIDS=' sqlite3(changeset|changegroup|session)?_' -checksymbols: sqlite3.lo - nm -g --defined-only sqlite3.lo | egrep -v $(VALIDIDS); test $$? -ne 0 +checksymbols: sqlite3.o + nm -g --defined-only sqlite3.o + nm -g --defined-only sqlite3.o | egrep -v $(VALIDIDS); test $$? -ne 0 echo '0 errors out of 1 tests' # Build the amalgamation-autoconf package. The amalamgation-tarball target builds # a tarball named for the version number. Ex: sqlite-autoconf-3110000.tar.gz. # The snapshot-tarball target builds a tarball named by the SHA1 hash Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -2137,16 +2137,14 @@ # Rules to build parse.c and parse.h - the outputs of lemon. # parse.h: parse.c -parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\tool\addopcodes.tcl +parse.c: $(TOP)\src\parse.y lemon.exe del /Q parse.y parse.h parse.h.temp 2>NUL copy $(TOP)\src\parse.y . .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) parse.y - move parse.h parse.h.temp - $(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h $(SQLITE3H): $(TOP)\src\sqlite.h.in $(TOP)\manifest mksourceid.exe $(TOP)\VERSION $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H) $(MKSQLITE3H_ARGS) sqlite3ext.h: .target_source Index: README.md ================================================================== --- README.md +++ README.md @@ -173,15 +173,12 @@ The SQL language parser is **parse.c** which is generate from a grammar in the src/parse.y file. The conversion of "parse.y" into "parse.c" is done by the [lemon](./doc/lemon.html) LALR(1) parser generator. The source code for lemon is at tool/lemon.c. Lemon uses the tool/lempar.c file as a template for generating its parser. - Lemon also generates the **parse.h** header file, at the same time it -generates parse.c. But the parse.h header file is -modified further (to add additional symbols) using the ./addopcodes.tcl -Tcl script. +generates parse.c. The **opcodes.h** header file contains macros that define the numbers corresponding to opcodes in the "VDBE" virtual machine. The opcodes.h file is generated by the scanning the src/vdbe.c source file. The Tcl script at ./mkopcodeh.tcl does this scan and generates opcodes.h. Index: ext/fts3/fts3_snippet.c ================================================================== --- ext/fts3/fts3_snippet.c +++ ext/fts3/fts3_snippet.c @@ -126,21 +126,23 @@ */ /* ** Allocate a two-slot MatchinfoBuffer object. */ -static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){ +static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){ MatchinfoBuffer *pRet; - int nByte = sizeof(u32) * (2*nElem + 1) + sizeof(MatchinfoBuffer); - int nStr = (int)strlen(zMatchinfo); + sqlite3_int64 nByte = sizeof(u32) * (2*(sqlite3_int64)nElem + 1) + + sizeof(MatchinfoBuffer); + sqlite3_int64 nStr = strlen(zMatchinfo); - pRet = sqlite3_malloc(nByte + nStr+1); + pRet = sqlite3_malloc64(nByte + nStr+1); if( pRet ){ memset(pRet, 0, nByte); pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; - pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1); - pRet->nElem = nElem; + pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + + sizeof(u32)*((int)nElem+1); + pRet->nElem = (int)nElem; pRet->zMatchinfo = ((char*)pRet) + nByte; memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1); pRet->aRef[0] = 1; } @@ -997,12 +999,12 @@ } sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo request: %c", cArg); return SQLITE_ERROR; } -static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ - int nVal; /* Number of integers output by cArg */ +static size_t fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ + size_t nVal; /* Number of integers output by cArg */ switch( cArg ){ case FTS3_MATCHINFO_NDOC: case FTS3_MATCHINFO_NPHRASE: case FTS3_MATCHINFO_NCOL: @@ -1282,11 +1284,11 @@ } break; case FTS3_MATCHINFO_LHITS_BM: case FTS3_MATCHINFO_LHITS: { - int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); + size_t nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); memset(pInfo->aMatchinfo, 0, nZero); rc = fts3ExprLHitGather(pCsr->pExpr, pInfo); break; } @@ -1351,11 +1353,11 @@ ** matchinfo function has been called for this query. In this case ** allocate the array used to accumulate the matchinfo data and ** initialize those elements that are constant for every row. */ if( pCsr->pMIBuffer==0 ){ - int nMatchinfo = 0; /* Number of u32 elements in match-info */ + size_t nMatchinfo = 0; /* Number of u32 elements in match-info */ int i; /* Used to iterate through zArg */ /* Determine the number of phrases in the query */ pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr); sInfo.nPhrase = pCsr->nPhrase; Index: ext/fts3/fts3_test.c ================================================================== --- ext/fts3/fts3_test.c +++ ext/fts3/fts3_test.c @@ -446,18 +446,18 @@ if( p==pEnd ){ rc = SQLITE_DONE; }else{ /* Advance to the end of the token */ const char *pToken = p; - int nToken; + sqlite3_int64 nToken; while( ppCsr->nBuffer ){ sqlite3_free(pCsr->aBuffer); - pCsr->aBuffer = sqlite3_malloc(nToken); + pCsr->aBuffer = sqlite3_malloc64(nToken); } if( pCsr->aBuffer==0 ){ rc = SQLITE_NOMEM; }else{ int i; @@ -469,11 +469,11 @@ } pCsr->iToken++; pCsr->iInput = (int)(p - pCsr->aInput); *ppToken = pCsr->aBuffer; - *pnBytes = nToken; + *pnBytes = (int)nToken; *piStartOffset = (int)(pToken - pCsr->aInput); *piEndOffset = (int)(p - pCsr->aInput); *piPosition = pCsr->iToken; } } Index: ext/fts3/fts3_tokenize_vtab.c ================================================================== --- ext/fts3/fts3_tokenize_vtab.c +++ ext/fts3/fts3_tokenize_vtab.c @@ -344,11 +344,11 @@ fts3tokResetCursor(pCsr); if( idxNum==1 ){ const char *zByte = (const char *)sqlite3_value_text(apVal[0]); int nByte = sqlite3_value_bytes(apVal[0]); - pCsr->zInput = sqlite3_malloc(nByte+1); + pCsr->zInput = sqlite3_malloc64(nByte+1); if( pCsr->zInput==0 ){ rc = SQLITE_NOMEM; }else{ memcpy(pCsr->zInput, zByte, nByte); pCsr->zInput[nByte] = 0; Index: ext/fts3/fts3_tokenizer.c ================================================================== --- ext/fts3/fts3_tokenizer.c +++ ext/fts3/fts3_tokenizer.c @@ -194,12 +194,12 @@ }else{ char const **aArg = 0; int iArg = 0; z = &z[n+1]; while( z0 ){ - int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); - pReader = (Fts3SegReader *)sqlite3_malloc(nByte); + sqlite3_int64 nByte; + nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); + pReader = (Fts3SegReader *)sqlite3_malloc64(nByte); if( !pReader ){ rc = SQLITE_NOMEM; }else{ memset(pReader, 0, nByte); pReader->iIdx = 0x7FFFFFFF; @@ -3365,11 +3366,11 @@ int nBlob; /* Number of bytes in the BLOB */ sqlite3_stmt *pStmt; /* Statement used to insert the encoding */ int rc; /* Result code from subfunctions */ if( *pRC ) return; - pBlob = sqlite3_malloc( 10*p->nColumn ); + pBlob = sqlite3_malloc64( 10*(sqlite3_int64)p->nColumn ); if( pBlob==0 ){ *pRC = SQLITE_NOMEM; return; } fts3EncodeIntArray(p->nColumn, aSz, pBlob, &nBlob); @@ -3415,11 +3416,11 @@ int rc; /* Result code from subfunctions */ const int nStat = p->nColumn+2; if( *pRC ) return; - a = sqlite3_malloc( (sizeof(u32)+10)*nStat ); + a = sqlite3_malloc64( (sizeof(u32)+10)*(sqlite3_int64)nStat ); if( a==0 ){ *pRC = SQLITE_NOMEM; return; } pBlob = (char*)&a[nStat]; @@ -3536,12 +3537,12 @@ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); } if( rc==SQLITE_OK ){ - int nByte = sizeof(u32) * (p->nColumn+1)*3; - aSz = (u32 *)sqlite3_malloc(nByte); + sqlite3_int64 nByte = sizeof(u32) * ((sqlite3_int64)p->nColumn+1)*3; + aSz = (u32 *)sqlite3_malloc64(nByte); if( aSz==0 ){ rc = SQLITE_NOMEM; }else{ memset(aSz, 0, nByte); aSzIns = &aSz[p->nColumn+1]; @@ -3603,16 +3604,16 @@ int nSeg, /* Number of segments to merge */ Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ int rc; /* Return Code */ sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ - int nByte; /* Bytes allocated at pCsr->apSegment[] */ + sqlite3_int64 nByte; /* Bytes allocated at pCsr->apSegment[] */ /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ memset(pCsr, 0, sizeof(*pCsr)); nByte = sizeof(Fts3SegReader *) * nSeg; - pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); + pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc64(nByte); if( pCsr->apSegment==0 ){ rc = SQLITE_NOMEM; }else{ memset(pCsr->apSegment, 0, nByte); @@ -5588,11 +5589,11 @@ rc = SQLITE_CONSTRAINT; goto update_out; } /* Allocate space to hold the change in document sizes */ - aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 ); + aSzDel = sqlite3_malloc64(sizeof(aSzDel[0])*((sqlite3_int64)p->nColumn+1)*2); if( aSzDel==0 ){ rc = SQLITE_NOMEM; goto update_out; } aSzIns = &aSzDel[p->nColumn+1]; Index: ext/fts5/fts5_index.c ================================================================== --- ext/fts5/fts5_index.c +++ ext/fts5/fts5_index.c @@ -2641,12 +2641,12 @@ }else if( p2->pLeaf==0 ){ /* If p2 is at EOF */ iRes = i1; }else{ int res = fts5BufferCompare(&p1->term, &p2->term); if( res==0 ){ - assert( i2>i1 ); - assert( i2!=0 ); + assert_nc( i2>i1 ); + assert_nc( i2!=0 ); pRes->bTermEq = 1; if( p1->iRowid==p2->iRowid ){ p1->bDel = p2->bDel; return i2; } @@ -3689,11 +3689,11 @@ pWriter->aDlidx, sizeof(Fts5DlidxWriter) * nLvl ); if( aDlidx==0 ){ p->rc = SQLITE_NOMEM; }else{ - int nByte = sizeof(Fts5DlidxWriter) * (nLvl - pWriter->nDlidx); + size_t nByte = sizeof(Fts5DlidxWriter) * (nLvl - pWriter->nDlidx); memset(&aDlidx[pWriter->nDlidx], 0, nByte); pWriter->aDlidx = aDlidx; pWriter->nDlidx = nLvl; } } Index: ext/fts5/fts5_main.c ================================================================== --- ext/fts5/fts5_main.c +++ ext/fts5/fts5_main.c @@ -2466,18 +2466,18 @@ ){ Fts5Global *pGlobal = (Fts5Global*)pApi; int rc = sqlite3_overload_function(pGlobal->db, zName, -1); if( rc==SQLITE_OK ){ Fts5Auxiliary *pAux; - int nName; /* Size of zName in bytes, including \0 */ - int nByte; /* Bytes of space to allocate */ + sqlite3_int64 nName; /* Size of zName in bytes, including \0 */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ - nName = (int)strlen(zName) + 1; + nName = strlen(zName) + 1; nByte = sizeof(Fts5Auxiliary) + nName; - pAux = (Fts5Auxiliary*)sqlite3_malloc(nByte); + pAux = (Fts5Auxiliary*)sqlite3_malloc64(nByte); if( pAux ){ - memset(pAux, 0, nByte); + memset(pAux, 0, (size_t)nByte); pAux->zFunc = (char*)&pAux[1]; memcpy(pAux->zFunc, zName, nName); pAux->pGlobal = pGlobal; pAux->pUserData = pUserData; pAux->xFunc = xFunc; @@ -2503,19 +2503,19 @@ fts5_tokenizer *pTokenizer, /* Tokenizer implementation */ void(*xDestroy)(void*) /* Destructor for pUserData */ ){ Fts5Global *pGlobal = (Fts5Global*)pApi; Fts5TokenizerModule *pNew; - int nName; /* Size of zName and its \0 terminator */ - int nByte; /* Bytes of space to allocate */ + sqlite3_int64 nName; /* Size of zName and its \0 terminator */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ int rc = SQLITE_OK; - nName = (int)strlen(zName) + 1; + nName = strlen(zName) + 1; nByte = sizeof(Fts5TokenizerModule) + nName; - pNew = (Fts5TokenizerModule*)sqlite3_malloc(nByte); + pNew = (Fts5TokenizerModule*)sqlite3_malloc64(nByte); if( pNew ){ - memset(pNew, 0, nByte); + memset(pNew, 0, (size_t)nByte); pNew->zName = (char*)&pNew[1]; memcpy(pNew->zName, zName, nName); pNew->pUserData = pUserData; pNew->x = *pTokenizer; pNew->xDestroy = xDestroy; Index: ext/fts5/fts5_tokenize.c ================================================================== --- ext/fts5/fts5_tokenize.c +++ ext/fts5/fts5_tokenize.c @@ -367,11 +367,11 @@ int i; memset(p, 0, sizeof(Unicode61Tokenizer)); p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE; p->nFold = 64; - p->aFold = sqlite3_malloc(p->nFold * sizeof(char)); + p->aFold = sqlite3_malloc64(p->nFold * sizeof(char)); if( p->aFold==0 ){ rc = SQLITE_NOMEM; } /* Search for a "categories" argument */ Index: ext/fts5/test/fts5corrupt3.test ================================================================== --- ext/fts5/test/fts5corrupt3.test +++ ext/fts5/test/fts5corrupt3.test @@ -7992,13 +7992,143 @@ | 0: 0d 00 00 00 03 0f f2 00 0f fc 0f f7 0f f2 00 00 ................ | 4080: 00 00 03 03 02 01 03 03 02 02 01 02 02 01 02 09 ................ | end crash-2acc487d09f033.db }]} {} -do_catchsql_test 56.1 { - INSERT INTO t1(b) VALUES(randomblob(250)); - INSERT INTO t1(b) VALUES(randomblob(250)); +do_test 56.1 { + set res [catchsql { + INSERT INTO t1(b) VALUES(randomblob(250)); + INSERT INTO t1(b) VALUES(randomblob(250)); + }] + + # For some permutations - those that use the page-cache - this test + # may return SQLITE_CONSTRAINT instead of SQLITE_CORRUPT. This is because + # the corrupt db in the test over-reads the page buffer slightly, with + # different results depending on whether or not the page-cache is in use. + if {$res=="1 {constraint failed}"} { + set res "1 {database disk image is malformed}" + } + set res +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 57.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename x.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 01 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ................ +| 96: 00 2e 34 20 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ..4 ...........m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 61 6b 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 aket1_configt1_c +| 3600: 6f 7e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 o~fig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 1d !.wtablet1_cont. +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 59 64 78 03 43 52 45 t1_idxt1_Ydx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 64 61 02 43 52 45 41 54 45 tat1_dada.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 ARY KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 0e b4 00 06 0e 35 00 0f e8 0e 35 0f bd 0f 4e ......5....5...N +| 16: 0e cb 0e 4f 00 00 00 00 00 00 00 00 00 00 00 00 ...O............ +| 3632: 00 00 00 00 00 18 0a 03 00 36 00 00 00 00 01 04 .........6...... +| 3648: 04 00 04 01 01 01 02 01 01 03 01 01 04 01 01 5e ...............^ +| 3664: 90 80 80 80 80 01 04 00 81 40 00 00 00 51 06 30 .........@...Q.0 +| 3680: 61 62 61 63 6b 01 01 04 04 6e 64 6f 6e 01 01 02 aback....ndon... +| 3696: 04 63 69 76 65 01 01 02 04 6c 70 68 61 01 01 02 .cive....lpha... +| 3712: 03 74 6f 6d 01 01 01 06 62 61 63 6b 75 70 01 01 .tom....backup.. +| 3728: 02 05 6f 6f 6d 65 72 01 01 01 06 63 68 61 6e 6e ..oomer....chann +| 3744: 65 01 01 01 04 74 65 73 74 01 01 04 09 08 08 08 e....test....... +| 3760: 07 0a 09 0a 0f 3a 00 17 30 00 00 00 00 01 03 03 .....:..0....... +| 3776: 00 03 01 01 01 02 01 01 03 01 01 68 8c 80 80 80 ...........h.... +| 3792: 80 01 04 00 81 54 00 00 00 5b 06 30 61 62 61 63 .....T...[.0abac +| 3808: 6b 02 02 07 04 04 6e 64 6f 6e 02 02 05 02 04 63 k.....ndon.....c +| 3824: 69 76 65 02 02 0b 02 04 6c 70 68 61 02 04 02 0a ive.....lpha.... +| 3840: 02 03 74 6f 6d 02 02 09 01 06 62 61 63 6b 75 70 ..tom.....backup +| 3856: 02 02 04 02 05 6f 6f 6d 65 72 02 02 08 01 06 63 .....oomer.....c +| 3872: 68 61 6e 6e 65 02 02 03 01 04 74 65 73 74 02 02 hanne.....test.. +| 3888: 06 04 0a 09 09 0a 08 0b 0a 0b 0f ef 00 14 2a 00 ..............*. +| 3904: 00 00 00 01 02 02 00 02 01 01 01 02 01 01 68 88 ..............h. +| 3920: 80 80 80 80 01 04 00 81 54 00 00 00 5b 06 30 61 ........T...[.0a +| 3936: 62 61 63 6b 01 02 07 04 04 6e 64 6f 6e 01 02 05 back.....ndon... +| 3952: 02 04 63 69 76 65 01 02 0b 02 04 6c 70 68 61 01 ..cive.....lpha. +| 3968: 04 02 0a 02 03 74 6f 6d 01 02 09 01 06 62 61 63 .....tom.....bac +| 3984: 6b 75 70 01 02 04 02 05 6f 6f 6d 65 72 01 02 08 kup.....oomer... +| 4000: 01 06 63 68 61 6e 6e 65 01 02 03 01 04 74 65 73 ..channe.....tes +| 4016: 74 01 02 06 04 0a 09 09 0a 08 0b 0a 0b 24 84 80 t............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 02 02 05 42 66 74 02 02 02 04 04 6e 64 ck....Bft.....nd +| 4064: 6f 6e 03 02 02 04 0a 07 05 01 03 00 10 04 0d 00 on.............. +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 04 0f e5 00 00 00 0f f3 0f ec 0f e5 ................ +| 4064: 00 00 00 00 00 06 04 01 0c 01 04 02 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0d 01 02 ................ +| page 4 offset 12288 +| 0: 0d 0e bc 00 04 0e 78 00 00 00 00 00 00 00 0e 78 ......x........x +| 16: 0e 78 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .x.............. +| 3696: 00 00 00 00 00 00 00 00 42 02 04 00 81 09 61 6c ........B.....al +| 3712: 70 68 61 20 63 68 61 6e 6e 65 20 62 61 63 6b 75 pha channe backu +| 3728: 70 20 61 62 61 6e 64 6f 6e 20 74 65 73 74 20 61 p abandon test a +| 3744: 62 61 63 6b 20 62 6f 6f 6d 65 72 20 61 74 6f 6d back boomer atom +| 3760: 20 61 6c 70 68 61 20 61 63 69 76 65 00 00 00 44 alpha acive...D +| 3776: 81 09 61 6c 70 68 61 20 63 68 61 6e 6e 65 20 62 ..alpha channe b +| 3792: 61 63 6b 75 70 20 61 62 61 6e 64 6f 6e 20 74 65 ackup abandon te +| 3808: 73 74 20 61 62 61 63 6b 20 62 6f 6f 6d 65 72 20 st aback boomer +| 3824: 61 74 6f 6d 20 61 6c 70 68 61 20 61 63 69 76 65 atom alpha acive +| 4064: 0a 03 03 00 1b 61 4e 61 6e 64 6f 6e 08 02 03 00 .....aNandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 71 63 6b .abaft.....abqck +| page 5 offset 16384 +| 0: 0d 0f e8 00 04 0f e2 00 00 00 00 00 00 00 0f e2 ................ +| 16: 0f e2 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4064: 00 00 04 02 03 00 0e 0a 00 00 00 06 0e 0a 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 10 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 00 00 00 00 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 69 7a 65 heck....optimize +| end x.db +}]} {} + +do_catchsql_test 57.1 { + INSERT INTO t1(t1) VALUES('optimize') } {1 {database disk image is malformed}} + sqlite3_fts5_may_be_corrupt 0 finish_test ADDED ext/rbu/rbupartial.test Index: ext/rbu/rbupartial.test ================================================================== --- /dev/null +++ ext/rbu/rbupartial.test @@ -0,0 +1,86 @@ +# 2019 April 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +set ::testprefix rbupartial + +db close +sqlite3_shutdown +sqlite3_config_uri 1 + +foreach {tn without_rowid a b c d} { + 1 "" a b c d + 2 "WITHOUT ROWID" aaa bbb ccc ddd + 3 "WITHOUT ROWID" "\"hello\"" {"one'two"} {[c]} ddd + 4 "WITHOUT ROWID" {`a b`} {"one'two"} {[c c c]} ddd + 5 "" a b c {"d""d"} + 6 "" {'one''two'} b {"c""c"} {"d""d"} +} { + eval [string map [list \ + %WITHOUT_ROWID% $without_rowid %A% $a %B% $b %C% $c %D% $d + ] { + reset_db + do_execsql_test $tn.1.0 { + CREATE TABLE t1(%A% PRIMARY KEY, %B%, %C%, %D%) %WITHOUT_ROWID% ; + CREATE INDEX i1b ON t1(%B%); + CREATE INDEX i1b2 ON t1(%B%) WHERE %C%<5; + CREATE INDEX i1b3 ON t1(%B%) WHERE %C%>=5; + + CREATE INDEX i1c ON t1(%C%); + CREATE INDEX i1c2 ON t1(%C%) WHERE %C% IS NULL; + CREATE INDEX i1c3 ON t1(%C%) WHERE %C% IS NOT NULL; + + CREATE INDEX i1c4 ON t1(%C%) WHERE %D% < 'd'; + } + + do_execsql_test $tn.1.1 { + INSERT INTO t1 VALUES(0, NULL, NULL, 'a'); + INSERT INTO t1 VALUES(1, 2, 3, 'b'); + INSERT INTO t1 VALUES(4, 5, 6, 'c'); + INSERT INTO t1 VALUES(7, 8, 9, 'd'); + } + + forcedelete rbu.db + do_test $tn.1.2 { + sqlite3 rbu rbu.db + rbu eval { + CREATE TABLE data_t1(%A%, %B%, %C%, %D%, rbu_control); + + INSERT INTO data_t1 VALUES(10, 11, 12, 'e', 0); + INSERT INTO data_t1 VALUES(13, 14, NULL, 'f', 0); + + INSERT INTO data_t1 VALUES(0, NULL, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(4, NULL, NULL, NULL, 1); + + INSERT INTO data_t1 VALUES(7, NULL, 4, NULL, '..x.'); + INSERT INTO data_t1 VALUES(1, 10, NULL, NULL, '.xx.'); + } + rbu close + } {} + + do_test $tn.1.3 { + run_rbu test.db rbu.db + execsql { PRAGMA integrity_check } + } {ok} + + do_execsql_test $tn.1.4 { + SELECT * FROM t1 ORDER BY %A%; + } { + 1 10 {} b 7 8 4 d 10 11 12 e 13 14 {} f + } + + set step 0 + do_rbu_vacuum_test $tn.1.5 0 + }] +} + +finish_test Index: ext/rbu/sqlite3rbu.c ================================================================== --- ext/rbu/sqlite3rbu.c +++ ext/rbu/sqlite3rbu.c @@ -238,10 +238,15 @@ ** abIndexed: ** If the table has no indexes on it, abIndexed is set to NULL. Otherwise, ** it points to an array of flags nTblCol elements in size. The flag is ** set for each column that is either a part of the PK or a part of an ** index. Or clear otherwise. +** +** If there are one or more partial indexes on the table, all fields of +** this array set set to 1. This is because in that case, the module has +** no way to tell which fields will be required to add and remove entries +** from the partial indexes. ** */ struct RbuObjIter { sqlite3_stmt *pTblIter; /* Iterate through tables */ sqlite3_stmt *pIdxIter; /* Index iterator */ @@ -1033,11 +1038,11 @@ ** error code in the rbu handle passed as the first argument. Or, if an ** error has already occurred when this function is called, return NULL ** immediately without attempting the allocation or modifying the stored ** error code. */ -static void *rbuMalloc(sqlite3rbu *p, int nByte){ +static void *rbuMalloc(sqlite3rbu *p, sqlite3_int64 nByte){ void *pRet = 0; if( p->rc==SQLITE_OK ){ assert( nByte>0 ); pRet = sqlite3_malloc64(nByte); if( pRet==0 ){ @@ -1054,11 +1059,11 @@ ** Allocate and zero the pIter->azTblCol[] and abTblPk[] arrays so that ** there is room for at least nCol elements. If an OOM occurs, store an ** error code in the RBU handle passed as the first argument. */ static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){ - int nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol; + sqlite3_int64 nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol; char **azNew; azNew = (char**)rbuMalloc(p, nByte); if( azNew ){ pIter->azTblCol = azNew; @@ -1248,12 +1253,16 @@ } pIter->nIndex = 0; while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ const char *zIdx = (const char*)sqlite3_column_text(pList, 1); + int bPartial = sqlite3_column_int(pList, 4); sqlite3_stmt *pXInfo = 0; if( zIdx==0 ) break; + if( bPartial ){ + memset(pIter->abIndexed, 0x01, sizeof(u8)*pIter->nTblCol); + } p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) ); while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ int iCid = sqlite3_column_int(pXInfo, 1); @@ -1694,11 +1703,11 @@ ** when this function is called, NULL is returned immediately, without ** attempting the allocation or modifying the stored error code. */ static char *rbuObjIterGetBindlist(sqlite3rbu *p, int nBind){ char *zRet = 0; - int nByte = nBind*2 + 1; + sqlite3_int64 nByte = 2*(sqlite3_int64)nBind + 1; zRet = (char*)rbuMalloc(p, nByte); if( zRet ){ int i; for(i=0; irc; + char *zRet = 0; + + if( rc==SQLITE_OK ){ + rc = prepareAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + "SELECT trim(sql) FROM sqlite_master WHERE type='index' AND name=?" + ); + } + if( rc==SQLITE_OK ){ + int rc2; + rc = sqlite3_bind_text(pStmt, 1, pIter->zIdx, -1, SQLITE_STATIC); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zSql = (const char*)sqlite3_column_text(pStmt, 0); + if( zSql ){ + int nParen = 0; /* Number of open parenthesis */ + int i; + for(i=0; zSql[i]; i++){ + char c = zSql[i]; + if( c=='(' ){ + nParen++; + } + else if( c==')' ){ + nParen--; + if( nParen==0 ){ + i++; + break; + } + }else if( c=='"' || c=='\'' || c=='`' ){ + for(i++; 1; i++){ + if( zSql[i]==c ){ + if( zSql[i+1]!=c ) break; + i++; + } + } + }else if( c=='[' ){ + for(i++; 1; i++){ + if( zSql[i]==']' ) break; + } + } + } + if( zSql[i] ){ + zRet = rbuStrndup(&zSql[i], &rc); + } + } + } + + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + + p->rc = rc; + return zRet; +} /* ** Ensure that the SQLite statement handles required to update the ** target database object currently indicated by the iterator passed ** as the second argument are available. @@ -1985,17 +2050,19 @@ const char *zTbl = pIter->zTbl; char *zImposterCols = 0; /* Columns for imposter table */ char *zImposterPK = 0; /* Primary key declaration for imposter */ char *zWhere = 0; /* WHERE clause on PK columns */ char *zBind = 0; + char *zPart = 0; int nBind = 0; assert( pIter->eType!=RBU_PK_VTAB ); zCollist = rbuObjIterGetIndexCols( p, pIter, &zImposterCols, &zImposterPK, &zWhere, &nBind ); zBind = rbuObjIterGetBindlist(p, nBind); + zPart = rbuObjIterGetIndexWhere(p, pIter); /* Create the imposter table used to write to this index. */ sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1); sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1,tnum); rbuMPrintfExec(p, p->dbMain, @@ -2024,32 +2091,34 @@ /* Create the SELECT statement to read keys in sorted order */ if( p->rc==SQLITE_OK ){ char *zSql; if( rbuIsVacuum(p) ){ zSql = sqlite3_mprintf( - "SELECT %s, 0 AS rbu_control FROM '%q' ORDER BY %s%s", + "SELECT %s, 0 AS rbu_control FROM '%q' %s ORDER BY %s%s", zCollist, pIter->zDataTbl, - zCollist, zLimit + zPart, zCollist, zLimit ); }else if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zSql = sqlite3_mprintf( - "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s", + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' %s ORDER BY %s%s", zCollist, p->zStateDb, pIter->zDataTbl, - zCollist, zLimit + zPart, zCollist, zLimit ); }else{ zSql = sqlite3_mprintf( - "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' " + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' %s " "UNION ALL " "SELECT %s, rbu_control FROM '%q' " - "WHERE typeof(rbu_control)='integer' AND rbu_control!=1 " + "%s %s typeof(rbu_control)='integer' AND rbu_control!=1 " "ORDER BY %s%s", - zCollist, p->zStateDb, pIter->zDataTbl, + zCollist, p->zStateDb, pIter->zDataTbl, zPart, zCollist, pIter->zDataTbl, + zPart, + (zPart ? "AND" : "WHERE"), zCollist, zLimit ); } p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, zSql); } @@ -2056,10 +2125,11 @@ sqlite3_free(zImposterCols); sqlite3_free(zImposterPK); sqlite3_free(zWhere); sqlite3_free(zBind); + sqlite3_free(zPart); }else{ int bRbuRowid = (pIter->eType==RBU_PK_VTAB) ||(pIter->eType==RBU_PK_NONE) ||(pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)); const char *zTbl = pIter->zTbl; /* Table this step applies to */ @@ -4489,11 +4559,11 @@ ** rbu is in the RBU_STAGE_OAL state, use heap memory for *-shm space ** instead of a file on disk. */ assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) ); if( eStage==RBU_STAGE_OAL || eStage==RBU_STAGE_MOVE ){ if( iRegion<=p->nShm ){ - int nByte = (iRegion+1) * sizeof(char*); + sqlite3_int64 nByte = (iRegion+1) * sizeof(char*); char **apNew = (char**)sqlite3_realloc64(p->apShm, nByte); if( apNew==0 ){ rc = SQLITE_NOMEM; }else{ memset(&apNew[p->nShm], 0, sizeof(char*) * (1 + iRegion - p->nShm)); Index: ext/rtree/geopoly.c ================================================================== --- ext/rtree/geopoly.c +++ ext/rtree/geopoly.c @@ -267,11 +267,11 @@ && (s.z++, geopolySkipSpace(&s)==0) ){ GeoPoly *pOut; int x = 1; s.nVertex--; /* Remove the redundant vertex at the end */ - pOut = sqlite3_malloc64( GEOPOLY_SZ(s.nVertex) ); + pOut = sqlite3_malloc64( GEOPOLY_SZ((sqlite3_int64)s.nVertex) ); x = 1; if( pOut==0 ) goto parse_json_err; pOut->nVertex = s.nVertex; memcpy(pOut->a, s.a, s.nVertex*2*sizeof(GeoCoord)); pOut->hdr[0] = *(unsigned char*)&x; @@ -653,11 +653,11 @@ else if( r>mxY ) mxY = (float)r; } if( pRc ) *pRc = SQLITE_OK; if( aCoord==0 ){ geopolyBboxFill: - pOut = sqlite3_realloc(p, GEOPOLY_SZ(4)); + pOut = sqlite3_realloc64(p, GEOPOLY_SZ(4)); if( pOut==0 ){ sqlite3_free(p); if( context ) sqlite3_result_error_nomem(context); if( pRc ) *pRc = SQLITE_NOMEM; return 0; @@ -1049,13 +1049,13 @@ /* ** Determine the overlap between two polygons */ static int geopolyOverlap(GeoPoly *p1, GeoPoly *p2){ - int nVertex = p1->nVertex + p2->nVertex + 2; + sqlite3_int64 nVertex = p1->nVertex + p2->nVertex + 2; GeoOverlap *p; - int nByte; + sqlite3_int64 nByte; GeoEvent *pThisEvent; double rX; int rc = 0; int needSort = 0; GeoSegment *pActive = 0; @@ -1063,11 +1063,11 @@ unsigned char aOverlap[4]; nByte = sizeof(GeoEvent)*nVertex*2 + sizeof(GeoSegment)*nVertex + sizeof(GeoOverlap); - p = sqlite3_malloc( nByte ); + p = sqlite3_malloc64( nByte ); if( p==0 ) return -1; p->aEvent = (GeoEvent*)&p[1]; p->aSegment = (GeoSegment*)&p->aEvent[nVertex*2]; p->nEvent = p->nSegment = 0; geopolyAddSegments(p, p1, 1); @@ -1222,22 +1222,22 @@ char **pzErr, /* OUT: Error message, if any */ int isCreate /* True for xCreate, false for xConnect */ ){ int rc = SQLITE_OK; Rtree *pRtree; - int nDb; /* Length of string argv[1] */ - int nName; /* Length of string argv[2] */ + sqlite3_int64 nDb; /* Length of string argv[1] */ + sqlite3_int64 nName; /* Length of string argv[2] */ sqlite3_str *pSql; char *zSql; int ii; sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); /* Allocate the sqlite3_vtab structure */ - nDb = (int)strlen(argv[1]); - nName = (int)strlen(argv[2]); - pRtree = (Rtree *)sqlite3_malloc(sizeof(Rtree)+nDb+nName+2); + nDb = strlen(argv[1]); + nName = strlen(argv[2]); + pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName+2); if( !pRtree ){ return SQLITE_NOMEM; } memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2); pRtree->nBusy = 1; Index: ext/session/sqlite3session.c ================================================================== --- ext/session/sqlite3session.c +++ ext/session/sqlite3session.c @@ -900,11 +900,11 @@ */ static int sessionGrowHash(int bPatchset, SessionTable *pTab){ if( pTab->nChange==0 || pTab->nEntry>=(pTab->nChange/2) ){ int i; SessionChange **apNew; - int nNew = (pTab->nChange ? pTab->nChange : 128) * 2; + sqlite3_int64 nNew = 2*(sqlite3_int64)(pTab->nChange ? pTab->nChange : 128); apNew = (SessionChange **)sqlite3_malloc64(sizeof(SessionChange *) * nNew); if( apNew==0 ){ if( pTab->nChange==0 ){ return SQLITE_ERROR; @@ -1827,11 +1827,11 @@ ** If not, use sqlite3_realloc() to grow the buffer so that there is. ** ** If successful, return zero. Otherwise, if an OOM condition is encountered, ** set *pRc to SQLITE_NOMEM and return non-zero. */ -static int sessionBufferGrow(SessionBuffer *p, int nByte, int *pRc){ +static int sessionBufferGrow(SessionBuffer *p, size_t nByte, int *pRc){ if( *pRc==SQLITE_OK && p->nAlloc-p->nBufnAlloc ? p->nAlloc : 128; do { nNew = nNew*2; @@ -2945,11 +2945,11 @@ rc = SQLITE_CORRUPT_BKPT; } } if( rc==SQLITE_OK ){ - int iPK = sizeof(sqlite3_value*)*p->nCol*2; + size_t iPK = sizeof(sqlite3_value*)*p->nCol*2; memset(p->tblhdr.aBuf, 0, iPK); memcpy(&p->tblhdr.aBuf[iPK], &p->in.aData[p->in.iNext], nCopy); p->in.iNext += nCopy; } @@ -4249,11 +4249,11 @@ SessionBuffer cons = pApply->constraints; memset(&pApply->constraints, 0, sizeof(SessionBuffer)); rc = sessionChangesetStart(&pIter2, 0, 0, cons.nBuf, cons.aBuf, 0); if( rc==SQLITE_OK ){ - int nByte = 2*pApply->nCol*sizeof(sqlite3_value*); + size_t nByte = 2*pApply->nCol*sizeof(sqlite3_value*); int rc2; pIter2->bPatchset = bPatchset; pIter2->zTab = (char*)zTab; pIter2->nCol = pApply->nCol; pIter2->abPK = pApply->abPK; Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -379,11 +379,10 @@ $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/totype.c \ $(TOP)/ext/misc/unionvtab.c \ $(TOP)/ext/misc/wholenumber.c \ - $(TOP)/ext/misc/vfslog.c \ $(TOP)/ext/misc/zipfile.c \ $(TOP)/ext/fts5/fts5_tcl.c \ $(TOP)/ext/fts5/fts5_test_mi.c \ $(TOP)/ext/fts5/fts5_test_tok.c @@ -719,16 +718,13 @@ # Rules to build parse.c and parse.h - the outputs of lemon. # parse.h: parse.c -parse.c: $(TOP)/src/parse.y lemon $(TOP)/tool/addopcodes.tcl +parse.c: $(TOP)/src/parse.y lemon cp $(TOP)/src/parse.y . - rm -f parse.h ./lemon -s $(OPTS) parse.y - mv parse.h parse.h.temp - tclsh $(TOP)/tool/addopcodes.tcl parse.h.temp >parse.h sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid $(TOP)/VERSION $(TOP)/ext/rtree/sqlite3rtree.h tclsh $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h keywordhash.h: $(TOP)/tool/mkkeywordhash.c Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -6174,11 +6174,11 @@ MemPage *pTrunk = 0; /* Free-list trunk page */ Pgno iTrunk = 0; /* Page number of free-list trunk page */ MemPage *pPage1 = pBt->pPage1; /* Local reference to page 1 */ MemPage *pPage; /* Page being freed. May be NULL. */ int rc; /* Return Code */ - int nFree; /* Initial number of pages on free-list */ + u32 nFree; /* Initial number of pages on free-list */ assert( sqlite3_mutex_held(pBt->mutex) ); assert( CORRUPT_DB || iPage>1 ); assert( !pMemPage || pMemPage->pgno==iPage ); Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -3766,23 +3766,22 @@ int szEntry, /* Size of each object in the array */ int *pnEntry, /* Number of objects currently in use */ int *pIdx /* Write the index of a new slot here */ ){ char *z; - int n = *pnEntry; + sqlite3_int64 n = *pIdx = *pnEntry; if( (n & (n-1))==0 ){ - int sz = (n==0) ? 1 : 2*n; + sqlite3_int64 sz = (n==0) ? 1 : 2*n; void *pNew = sqlite3DbRealloc(db, pArray, sz*szEntry); if( pNew==0 ){ *pIdx = -1; return pArray; } pArray = pNew; } z = (char*)pArray; memset(&z[n * szEntry], 0, szEntry); - *pIdx = n; ++*pnEntry; return pArray; } /* @@ -3889,11 +3888,11 @@ assert( iStart<=pSrc->nSrc ); /* Allocate additional space if needed */ if( (u32)pSrc->nSrc+nExtra>pSrc->nAlloc ){ SrcList *pNew; - int nAlloc = pSrc->nSrc*2+nExtra; + sqlite3_int64 nAlloc = 2*(sqlite3_int64)pSrc->nSrc+nExtra; sqlite3 *db = pParse->db; if( pSrc->nSrc+nExtra>=SQLITE_MAX_SRCLIST ){ sqlite3ErrorMsg(pParse, "too many FROM clause terms, max: %d", SQLITE_MAX_SRCLIST); @@ -4396,11 +4395,12 @@ char *zErr; int j; StrAccum errMsg; Table *pTab = pIdx->pTable; - sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, 200); + sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, + pParse->db->aLimit[SQLITE_LIMIT_LENGTH]); if( pIdx->aColExpr ){ sqlite3_str_appendf(&errMsg, "index '%q'", pIdx->zName); }else{ for(j=0; jnKeyCol; j++){ char *zCol; @@ -4645,11 +4645,11 @@ } } } if( pWith ){ - int nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); + sqlite3_int64 nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); pNew = sqlite3DbRealloc(db, pWith, nByte); }else{ pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); } assert( (pNew!=0 && zName!=0) || db->mallocFailed ); Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -855,11 +855,11 @@ p = sqlite3ExprAnd(pParse->db, pLeft, pRight); }else{ p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)); if( p ){ memset(p, 0, sizeof(Expr)); - p->op = op & TKFLG_MASK; + p->op = op & 0xff; p->iAgg = -1; } sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); } if( p ) { @@ -1320,11 +1320,11 @@ */ #ifndef SQLITE_OMIT_CTE static With *withDup(sqlite3 *db, With *p){ With *pRet = 0; if( p ){ - int nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); + sqlite3_int64 nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); pRet = sqlite3DbMallocZero(db, nByte); if( pRet ){ int i; pRet->nCte = p->nCte; for(i=0; inCte; i++){ @@ -1585,11 +1585,11 @@ } pList->nExpr = 0; }else if( (pList->nExpr & (pList->nExpr-1))==0 ){ ExprList *pNew; pNew = sqlite3DbRealloc(db, pList, - sizeof(*pList)+(2*pList->nExpr - 1)*sizeof(pList->a[0])); + sizeof(*pList)+(2*(sqlite3_int64)pList->nExpr-1)*sizeof(pList->a[0])); if( pNew==0 ){ goto no_mem; } pList = pNew; } Index: src/hash.c ================================================================== --- src/hash.c +++ src/hash.c @@ -148,11 +148,11 @@ const Hash *pH, /* The pH to be searched */ const char *pKey, /* The key we are searching for */ unsigned int *pHash /* Write the hash value here */ ){ HashElem *elem; /* Used to loop thru the element list */ - int count; /* Number of elements left to test */ + unsigned int count; /* Number of elements left to test */ unsigned int h; /* The computed hash */ static HashElem nullElement = { 0, 0, 0, 0 }; if( pH->ht ){ /*OPTIMIZATION-IF-TRUE*/ struct _ht *pEntry; @@ -196,12 +196,12 @@ if( pH->ht ){ pEntry = &pH->ht[h]; if( pEntry->chain==elem ){ pEntry->chain = elem->next; } + assert( pEntry->count>0 ); pEntry->count--; - assert( pEntry->count>=0 ); } sqlite3_free( elem ); pH->count--; if( pH->count==0 ){ assert( pH->first==0 ); Index: src/hash.h ================================================================== --- src/hash.h +++ src/hash.h @@ -43,11 +43,11 @@ struct Hash { unsigned int htsize; /* Number of buckets in the hash table */ unsigned int count; /* Number of entries in this table */ HashElem *first; /* The first element of the array */ struct _ht { /* the hash table */ - int count; /* Number of entries with this hash */ + unsigned int count; /* Number of entries with this hash */ HashElem *chain; /* Pointer to first entry with this hash */ } *ht; }; /* Each element in the hash table is an instance of the following Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -2272,10 +2272,17 @@ for(pSrcIdx=pSrc->pIndex; pSrcIdx; pSrcIdx=pSrcIdx->pNext){ if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; } if( pSrcIdx==0 ){ return 0; /* pDestIdx has no corresponding index in pSrc */ + } + if( pSrcIdx->tnum==pDestIdx->tnum && pSrc->pSchema==pDest->pSchema + && sqlite3FaultSim(411)==SQLITE_OK ){ + /* The sqlite3FaultSim() call allows this corruption test to be + ** bypassed during testing, in order to exercise other corruption tests + ** further downstream. */ + return 0; /* Corrupt schema - two indexes on the same btree */ } } #ifndef SQLITE_OMIT_CHECK if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) ){ return 0; /* Tables have different CHECK constraints. Ticket #2252 */ Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -706,11 +706,11 @@ if( sz==0 || cnt==0 ){ sz = 0; pStart = 0; }else if( pBuf==0 ){ sqlite3BeginBenignMalloc(); - pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ + pStart = sqlite3Malloc( sz*(sqlite3_int64)cnt ); /* IMP: R-61949-35727 */ sqlite3EndBenignMalloc(); if( pStart ) cnt = sqlite3MallocSize(pStart)/sz; }else{ pStart = pBuf; } Index: src/parse.y ================================================================== --- src/parse.y +++ src/parse.y @@ -191,13 +191,11 @@ // Declare some tokens early in order to influence their values, to // improve performance and reduce the executable size. The goal here is // to get the "jump" operations in ISNULL through ESCAPE to have numeric // values that are early enough so that all jump operations are clustered -// at the beginning, but also so that the comparison tokens NE through GE -// are as large as possible so that they are near to FUNCTION, which is a -// token synthesized by addopcodes.tcl. +// at the beginning. // %token ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST. %token CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL. %token OR AND NOT IS MATCH LIKE_KW BETWEEN IN ISNULL NOTNULL NE EQ. %token GT LE LT GE ESCAPE. @@ -1742,5 +1740,45 @@ } filter_opt(A) ::= . { A = 0; } filter_opt(A) ::= FILTER LP WHERE expr(X) RP. { A = X; } %endif /* SQLITE_OMIT_WINDOWFUNC */ + +/* +** The code generator needs some extra TK_ token values for tokens that +** are synthesized and do not actually appear in the grammar: +*/ +%token + TRUEFALSE /* True or false keyword */ + ISNOT /* Combination of IS and NOT */ + FUNCTION /* A function invocation */ + COLUMN /* Reference to a table column */ + AGG_FUNCTION /* An aggregate function */ + AGG_COLUMN /* An aggregated column */ + UMINUS /* Unary minus */ + UPLUS /* Unary plus */ + TRUTH /* IS TRUE or IS FALSE or IS NOT TRUE or IS NOT FALSE */ + REGISTER /* Reference to a VDBE register */ + VECTOR /* Vector */ + SELECT_COLUMN /* Choose a single column from a multi-column SELECT */ + IF_NULL_ROW /* the if-null-row operator */ + ASTERISK /* The "*" in count(*) and similar */ + SPAN /* The span operator */ +. +/* There must be no more than 255 tokens defined above. If this grammar +** is extended with new rules and tokens, they must either be so few in +** number that TK_SPAN is no more than 255, or else the new tokens must +** appear after this line. +*/ +%include { +#if TK_SPAN>255 +# error too many tokens in the grammar +#endif +} + +/* +** The TK_SPACE and TK_ILLEGAL tokens must be the last two tokens. The +** parser depends on this. Those tokens are not used in any grammar rule. +** They are only used by the tokenizer. Declare them last so that they +** are guaranteed to be the last two tokens +*/ +%token SPACE ILLEGAL. Index: src/pcache1.c ================================================================== --- src/pcache1.c +++ src/pcache1.c @@ -486,13 +486,11 @@ ** Malloc function used by SQLite to obtain space from the buffer configured ** using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no such buffer ** exists, this function falls back to sqlite3Malloc(). */ void *sqlite3PageMalloc(int sz){ - /* During rebalance operations on a corrupt database file, it is sometimes - ** (rarely) possible to overread the temporary page buffer by a few bytes. - ** Enlarge the allocation slightly so that this does not cause problems. */ + assert( sz<=65536+8 ); /* These allocations are never very large */ return pcache1Alloc(sz); } /* ** Free an allocated buffer obtained from sqlite3PageMalloc(). Index: src/printf.c ================================================================== --- src/printf.c +++ src/printf.c @@ -135,10 +135,11 @@ */ static void setStrAccumError(StrAccum *p, u8 eError){ assert( eError==SQLITE_NOMEM || eError==SQLITE_TOOBIG ); p->accError = eError; if( p->mxAlloc ) sqlite3_str_reset(p); + if( eError==SQLITE_TOOBIG ) sqlite3ErrorToParser(p->db, eError); } /* ** Extra argument values from a PrintfArguments object */ Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -3848,10 +3848,11 @@ #endif void sqlite3SetString(char **, sqlite3*, const char*); void sqlite3ErrorMsg(Parse*, const char*, ...); +int sqlite3ErrorToParser(sqlite3*,int); void sqlite3Dequote(char*); void sqlite3DequoteExpr(Expr*); void sqlite3TokenInit(Token*,char*); int sqlite3KeywordCode(const unsigned char*, int); int sqlite3RunParser(Parse*, const char*, char **); Index: src/test4.c ================================================================== --- src/test4.c +++ src/test4.c @@ -62,11 +62,11 @@ /* ** The main loop for a thread. Threads use busy waiting. */ -static void *thread_main(void *pArg){ +static void *test_thread_main(void *pArg){ Thread *p = (Thread*)pArg; if( p->db ){ sqlite3_close(p->db); } sqlite3_open(p->zFilename, &p->db); @@ -149,11 +149,11 @@ threadset[i].busy = 1; sqlite3_free(threadset[i].zFilename); threadset[i].zFilename = sqlite3_mprintf("%s", argv[2]); threadset[i].opnum = 1; threadset[i].completed = 0; - rc = pthread_create(&x, 0, thread_main, &threadset[i]); + rc = pthread_create(&x, 0, test_thread_main, &threadset[i]); if( rc ){ Tcl_AppendResult(interp, "failed to create the thread", 0); sqlite3_free(threadset[i].zFilename); threadset[i].busy = 0; return TCL_ERROR; @@ -163,11 +163,11 @@ } /* ** Wait for a thread to reach its idle state. */ -static void thread_wait(Thread *p){ +static void test_thread_wait(Thread *p){ while( p->opnum>p->completed ) sched_yield(); } /* ** Usage: thread_wait ID @@ -191,22 +191,22 @@ if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ Tcl_AppendResult(interp, "no such thread", 0); return TCL_ERROR; } - thread_wait(&threadset[i]); + test_thread_wait(&threadset[i]); return TCL_OK; } /* ** Stop a thread. */ -static void stop_thread(Thread *p){ - thread_wait(p); +static void test_stop_thread(Thread *p){ + test_thread_wait(p); p->xOp = 0; p->opnum++; - thread_wait(p); + test_thread_wait(p); sqlite3_free(p->zArg); p->zArg = 0; sqlite3_free(p->zFilename); p->zFilename = 0; p->busy = 0; @@ -231,20 +231,20 @@ " ID", 0); return TCL_ERROR; } if( argv[1][0]=='*' && argv[1][1]==0 ){ for(i=0; i=threadset[i].argc ){ Tcl_AppendResult(interp, "column number out of range", 0); return TCL_ERROR; } Tcl_AppendResult(interp, threadset[i].argv[n], 0); @@ -340,11 +340,11 @@ if( !threadset[i].busy ){ Tcl_AppendResult(interp, "no such thread", 0); return TCL_ERROR; } if( Tcl_GetInt(interp, argv[2], &n) ) return TCL_ERROR; - thread_wait(&threadset[i]); + test_thread_wait(&threadset[i]); if( n<0 || n>=threadset[i].argc ){ Tcl_AppendResult(interp, "column number out of range", 0); return TCL_ERROR; } Tcl_AppendResult(interp, threadset[i].colv[n], 0); @@ -375,11 +375,11 @@ if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ Tcl_AppendResult(interp, "no such thread", 0); return TCL_ERROR; } - thread_wait(&threadset[i]); + test_thread_wait(&threadset[i]); zName = sqlite3ErrName(threadset[i].rc); Tcl_AppendResult(interp, zName, 0); return TCL_OK; } @@ -406,11 +406,11 @@ if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ Tcl_AppendResult(interp, "no such thread", 0); return TCL_ERROR; } - thread_wait(&threadset[i]); + test_thread_wait(&threadset[i]); Tcl_AppendResult(interp, threadset[i].zErr, 0); return TCL_OK; } /* @@ -450,11 +450,11 @@ if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ Tcl_AppendResult(interp, "no such thread", 0); return TCL_ERROR; } - thread_wait(&threadset[i]); + test_thread_wait(&threadset[i]); threadset[i].xOp = do_compile; sqlite3_free(threadset[i].zArg); threadset[i].zArg = sqlite3_mprintf("%s", argv[2]); threadset[i].opnum++; return TCL_OK; @@ -503,11 +503,11 @@ if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ Tcl_AppendResult(interp, "no such thread", 0); return TCL_ERROR; } - thread_wait(&threadset[i]); + test_thread_wait(&threadset[i]); threadset[i].xOp = do_step; threadset[i].opnum++; return TCL_OK; } @@ -545,11 +545,11 @@ if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ Tcl_AppendResult(interp, "no such thread", 0); return TCL_ERROR; } - thread_wait(&threadset[i]); + test_thread_wait(&threadset[i]); threadset[i].xOp = do_finalize; sqlite3_free(threadset[i].zArg); threadset[i].zArg = 0; threadset[i].opnum++; return TCL_OK; @@ -577,18 +577,18 @@ if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ Tcl_AppendResult(interp, "no such thread", 0); return TCL_ERROR; } - thread_wait(&threadset[i]); + test_thread_wait(&threadset[i]); j = parse_thread_id(interp, argv[2]); if( j<0 ) return TCL_ERROR; if( !threadset[j].busy ){ Tcl_AppendResult(interp, "no such thread", 0); return TCL_ERROR; } - thread_wait(&threadset[j]); + test_thread_wait(&threadset[j]); temp = threadset[i].db; threadset[i].db = threadset[j].db; threadset[j].db = temp; return TCL_OK; } @@ -618,11 +618,11 @@ if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ Tcl_AppendResult(interp, "no such thread", 0); return TCL_ERROR; } - thread_wait(&threadset[i]); + test_thread_wait(&threadset[i]); sqlite3TestMakePointerStr(interp, zBuf, threadset[i].db); threadset[i].db = 0; Tcl_AppendResult(interp, zBuf, (char*)0); return TCL_OK; } @@ -649,11 +649,11 @@ if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ Tcl_AppendResult(interp, "no such thread", 0); return TCL_ERROR; } - thread_wait(&threadset[i]); + test_thread_wait(&threadset[i]); assert( !threadset[i].db ); threadset[i].db = (sqlite3*)sqlite3TestTextToPtr(argv[2]); return TCL_OK; } @@ -681,11 +681,11 @@ if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ Tcl_AppendResult(interp, "no such thread", 0); return TCL_ERROR; } - thread_wait(&threadset[i]); + test_thread_wait(&threadset[i]); sqlite3TestMakePointerStr(interp, zBuf, threadset[i].pStmt); threadset[i].pStmt = 0; Tcl_AppendResult(interp, zBuf, (char*)0); return TCL_OK; } Index: src/test_fs.c ================================================================== --- src/test_fs.c +++ src/test_fs.c @@ -738,11 +738,11 @@ fd = open(zFile, O_RDONLY); if( fd<0 ) return SQLITE_IOERR; fstat(fd, &sbuf); if( sbuf.st_size>=pCur->nAlloc ){ - int nNew = sbuf.st_size*2; + sqlite3_int64 nNew = sbuf.st_size*2; char *zNew; if( nNew<1024 ) nNew = 1024; zNew = sqlite3Realloc(pCur->zBuf, nNew); if( zNew==0 ){ Index: src/test_vfs.c ================================================================== --- src/test_vfs.c +++ src/test_vfs.c @@ -233,10 +233,11 @@ { SQLITE_IOERR, "SQLITE_IOERR" }, { SQLITE_LOCKED, "SQLITE_LOCKED" }, { SQLITE_BUSY, "SQLITE_BUSY" }, { SQLITE_READONLY, "SQLITE_READONLY" }, { SQLITE_READONLY_CANTINIT, "SQLITE_READONLY_CANTINIT" }, + { -1, "SQLITE_OMIT" }, }; const char *z; int i; @@ -380,10 +381,11 @@ tvfsExecTcl(p, "xWrite", Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, Tcl_NewWideIntObj(iOfst), Tcl_NewIntObj(iAmt) ); tvfsResultCode(p, &rc); + if( rc<0 ) return SQLITE_OK; } if( rc==SQLITE_OK && tvfsInjectFullerr(p) ){ rc = SQLITE_FULL; } Index: src/utf.c ================================================================== --- src/utf.c +++ src/utf.c @@ -198,15 +198,15 @@ ** This routine transforms the internal text encoding used by pMem to ** desiredEnc. It is an error if the string is already of the desired ** encoding, or if *pMem does not contain a string value. */ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ - int len; /* Maximum length of output string in bytes */ - unsigned char *zOut; /* Output buffer */ - unsigned char *zIn; /* Input iterator */ - unsigned char *zTerm; /* End of input */ - unsigned char *z; /* Output iterator */ + sqlite3_int64 len; /* Maximum length of output string in bytes */ + unsigned char *zOut; /* Output buffer */ + unsigned char *zIn; /* Input iterator */ + unsigned char *zTerm; /* End of input */ + unsigned char *z; /* Output iterator */ unsigned int c; assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( pMem->flags&MEM_Str ); assert( pMem->enc!=desiredEnc ); @@ -251,18 +251,18 @@ ** translating a 2-byte character to a 4-byte UTF-8 character. ** A single byte is required for the output string ** nul-terminator. */ pMem->n &= ~1; - len = pMem->n * 2 + 1; + len = 2 * (sqlite3_int64)pMem->n + 1; }else{ /* When converting from UTF-8 to UTF-16 the maximum growth is caused ** when a 1-byte UTF-8 character is translated into a 2-byte UTF-16 ** character. Two bytes are required in the output buffer for the ** nul-terminator. */ - len = pMem->n * 2 + 2; + len = 2 * (sqlite3_int64)pMem->n + 2; } /* Set zIn to point at the start of the input buffer and zTerm to point 1 ** byte past the end. ** Index: src/util.c ================================================================== --- src/util.c +++ src/util.c @@ -30,19 +30,27 @@ dummy += (unsigned)x; } #endif /* -** Give a callback to the test harness that can be used to simulate faults -** in places where it is difficult or expensive to do so purely by means -** of inputs. -** -** The intent of the integer argument is to let the fault simulator know -** which of multiple sqlite3FaultSim() calls has been hit. -** -** Return whatever integer value the test callback returns, or return -** SQLITE_OK if no test callback is installed. +** Calls to sqlite3FaultSim() are used to simulate a failure during testing, +** or to bypass normal error detection during testing in order to let +** execute proceed futher downstream. +** +** In deployment, sqlite3FaultSim() *always* return SQLITE_OK (0). The +** sqlite3FaultSim() function only returns non-zero during testing. +** +** During testing, if the test harness has set a fault-sim callback using +** a call to sqlite3_test_control(SQLITE_TESTCTRL_FAULT_INSTALL), then +** each call to sqlite3FaultSim() is relayed to that application-supplied +** callback and the integer return value form the application-supplied +** callback is returned by sqlite3FaultSim(). +** +** The integer argument to sqlite3FaultSim() is a code to identify which +** sqlite3FaultSim() instance is being invoked. Each call to sqlite3FaultSim() +** should have a unique code. To prevent legacy testing applications from +** breaking, the codes should not be changed or reused. */ #ifndef SQLITE_UNTESTABLE int sqlite3FaultSim(int iTest){ int (*xCallback)(int) = sqlite3GlobalConfig.xTestCallback; return xCallback ? xCallback(iTest) : SQLITE_OK; @@ -222,10 +230,23 @@ sqlite3DbFree(db, pParse->zErrMsg); pParse->zErrMsg = zMsg; pParse->rc = SQLITE_ERROR; } } + +/* +** If database connection db is currently parsing SQL, then transfer +** error code errCode to that parser if the parser has not already +** encountered some other kind of error. +*/ +int sqlite3ErrorToParser(sqlite3 *db, int errCode){ + Parse *pParse; + if( db==0 || (pParse = db->pParse)==0 ) return errCode; + pParse->rc = errCode; + pParse->nErr++; + return errCode; +} /* ** Convert an SQL-style quoted string into a normal string by removing ** the quote characters. The conversion is done in-place. If the ** input does not begin with a quote character, then this routine @@ -1574,11 +1595,11 @@ nInt = nName/4 + 3; assert( pIn==0 || pIn[0]>=3 ); /* Verify ok to add new elements */ if( pIn==0 || pIn[1]+nInt > pIn[0] ){ /* Enlarge the allocation */ - int nAlloc = (pIn ? pIn[0]*2 : 10) + nInt; + sqlite3_int64 nAlloc = (pIn ? 2*(sqlite3_int64)pIn[0] : 10) + nInt; VList *pOut = sqlite3DbRealloc(db, pIn, nAlloc*sizeof(int)); if( pOut==0 ) return pIn; if( pIn==0 ) pOut[1] = 2; pIn = pOut; pIn[0] = nAlloc; Index: src/vdbeInt.h ================================================================== --- src/vdbeInt.h +++ src/vdbeInt.h @@ -280,10 +280,16 @@ ** Clear any existing type flags from a Mem and replace them with f */ #define MemSetTypeFlag(p, f) \ ((p)->flags = ((p)->flags&~(MEM_TypeMask|MEM_Zero))|f) +/* +** True if Mem X is a NULL-nochng type. +*/ +#define MemNullNochng(X) \ + ((X)->flags==(MEM_Null|MEM_Zero) && (X)->n==0 && (X)->u.nZero==0) + /* ** Return true if a memory cell is not marked as invalid. This macro ** is for use inside assert() statements only. */ #ifdef SQLITE_DEBUG Index: src/vdbeapi.c ================================================================== --- src/vdbeapi.c +++ src/vdbeapi.c @@ -68,11 +68,11 @@ assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 ); assert( db->init.busy==0 ); assert( p->zSql!=0 ); sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); iElapse = (iNow - p->startTime)*1000000; -#ifndef SQLITE_OMIT_DEPRECATED +#ifndef SQLITE_OMIT_DEPRECATED if( db->xProfile ){ db->xProfile(db->pProfileArg, p->zSql, iElapse); } #endif if( db->mTrace & SQLITE_TRACE_PROFILE ){ Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -153,13 +153,15 @@ ** during testing only. With SQLITE_TEST_REALLOC_STRESS grow the op array ** by the minimum* amount required until the size reaches 512. Normal ** operation (without SQLITE_TEST_REALLOC_STRESS) is to double the current ** size of the op array or add 1KB of space, whichever is smaller. */ #ifdef SQLITE_TEST_REALLOC_STRESS - int nNew = (v->nOpAlloc>=512 ? v->nOpAlloc*2 : v->nOpAlloc+nOp); + sqlite3_int64 nNew = (v->nOpAlloc>=512 ? 2*(sqlite3_int64)v->nOpAlloc + : (sqlite3_int64)v->nOpAlloc+nOp); #else - int nNew = (v->nOpAlloc ? v->nOpAlloc*2 : (int)(1024/sizeof(Op))); + sqlite3_int64 nNew = (v->nOpAlloc ? 2*(sqlite3_int64)v->nOpAlloc + : (sqlite3_int64)(1024/sizeof(Op))); UNUSED_PARAMETER(nOp); #endif /* Ensure that the size of a VDBE does not grow too large */ if( nNew > p->db->aLimit[SQLITE_LIMIT_VDBE_OP] ){ @@ -943,11 +945,11 @@ int addrLoop, /* Address of loop counter */ int addrVisit, /* Address of rows visited counter */ LogEst nEst, /* Estimated number of output rows */ const char *zName /* Name of table or index being scanned */ ){ - int nByte = (p->nScan+1) * sizeof(ScanStatus); + sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); ScanStatus *aNew; aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); if( aNew ){ ScanStatus *pNew = &aNew[p->nScan++]; pNew->addrExplain = addrExplain; @@ -2064,13 +2066,13 @@ /* An instance of this object describes bulk memory available for use ** by subcomponents of a prepared statement. Space is allocated out ** of a ReusableSpace object by the allocSpace() routine below. */ struct ReusableSpace { - u8 *pSpace; /* Available memory */ - int nFree; /* Bytes of available memory */ - int nNeeded; /* Total bytes that could not be allocated */ + u8 *pSpace; /* Available memory */ + sqlite3_int64 nFree; /* Bytes of available memory */ + sqlite3_int64 nNeeded; /* Total bytes that could not be allocated */ }; /* Try to allocate nByte bytes of 8-byte aligned bulk memory for pBuf ** from the ReusableSpace object. Return a pointer to the allocated ** memory on success. If insufficient memory is available in the @@ -2086,11 +2088,11 @@ ** statement. */ static void *allocSpace( struct ReusableSpace *p, /* Bulk memory available for allocation */ void *pBuf, /* Pointer to a prior allocation */ - int nByte /* Bytes of memory needed */ + sqlite3_int64 nByte /* Bytes of memory needed */ ){ assert( EIGHT_BYTE_ALIGNMENT(p->pSpace) ); if( pBuf==0 ){ nByte = ROUND8(nByte); if( nByte <= p->nFree ){ Index: src/vdbemem.c ================================================================== --- src/vdbemem.c +++ src/vdbemem.c @@ -296,17 +296,19 @@ */ #ifndef SQLITE_OMIT_INCRBLOB int sqlite3VdbeMemExpandBlob(Mem *pMem){ int nByte; assert( pMem->flags & MEM_Zero ); - assert( pMem->flags&MEM_Blob ); + assert( (pMem->flags&MEM_Blob)!=0 || MemNullNochng(pMem) ); + testcase( sqlite3_value_nochange(pMem) ); assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); /* Set nByte to the number of bytes required to store the expanded blob. */ nByte = pMem->n + pMem->u.nZero; if( nByte<=0 ){ + if( (pMem->flags & MEM_Blob)==0 ) return SQLITE_OK; nByte = 1; } if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){ return SQLITE_NOMEM_BKPT; } @@ -1059,11 +1061,11 @@ u32 nAlloc = nByte; if( flags&MEM_Term ){ nAlloc += (enc==SQLITE_UTF8?1:2); } if( nByte>iLimit ){ - return SQLITE_TOOBIG; + return sqlite3ErrorToParser(pMem->db, SQLITE_TOOBIG); } testcase( nAlloc==0 ); testcase( nAlloc==31 ); testcase( nAlloc==32 ); if( sqlite3VdbeMemClearAndResize(pMem, (int)MAX(nAlloc,32)) ){ Index: src/vdbesort.c ================================================================== --- src/vdbesort.c +++ src/vdbesort.c @@ -535,11 +535,11 @@ int nRem; /* Bytes remaining to copy */ /* Extend the p->aAlloc[] allocation if required. */ if( p->nAllocnAlloc*2); + sqlite3_int64 nNew = MAX(128, 2*(sqlite3_int64)p->nAlloc); while( nByte>nNew ) nNew = nNew*2; aNew = sqlite3Realloc(p->aAlloc, nNew); if( !aNew ) return SQLITE_NOMEM_BKPT; p->nAlloc = nNew; p->aAlloc = aNew; @@ -1827,11 +1827,11 @@ int nMin = pSorter->iMemory + nReq; if( nMin>pSorter->nMemory ){ u8 *aNew; int iListOff = (u8*)pSorter->list.pList - pSorter->list.aMemory; - int nNew = pSorter->nMemory * 2; + sqlite3_int64 nNew = 2 * (sqlite3_int64)pSorter->nMemory; while( nNew < nMin ) nNew = nNew*2; if( nNew > pSorter->mxPmaSize ) nNew = pSorter->mxPmaSize; if( nNew < nMin ) nNew = nMin; aNew = sqlite3Realloc(pSorter->list.aMemory, nNew); Index: src/vtab.c ================================================================== --- src/vtab.c +++ src/vtab.c @@ -300,13 +300,17 @@ ** Add a new module argument to pTable->azModuleArg[]. ** The string is not copied - the pointer is stored. The ** string will be freed automatically when the table is ** deleted. */ -static void addModuleArgument(sqlite3 *db, Table *pTable, char *zArg){ - int nBytes = sizeof(char *)*(2+pTable->nModuleArg); +static void addModuleArgument(Parse *pParse, Table *pTable, char *zArg){ + sqlite3_int64 nBytes = sizeof(char *)*(2+pTable->nModuleArg); char **azModuleArg; + sqlite3 *db = pParse->db; + if( pTable->nModuleArg+3>=db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many columns on %s", pTable->zName); + } azModuleArg = sqlite3DbRealloc(db, pTable->azModuleArg, nBytes); if( azModuleArg==0 ){ sqlite3DbFree(db, zArg); }else{ int i = pTable->nModuleArg++; @@ -337,13 +341,13 @@ assert( 0==pTable->pIndex ); db = pParse->db; assert( pTable->nModuleArg==0 ); - addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); - addModuleArgument(db, pTable, 0); - addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); + addModuleArgument(pParse, pTable, sqlite3NameFromToken(db, pModuleName)); + addModuleArgument(pParse, pTable, 0); + addModuleArgument(pParse, pTable, sqlite3DbStrDup(db, pTable->zName)); assert( (pParse->sNameToken.z==pName2->z && pName2->z!=0) || (pParse->sNameToken.z==pName1->z && pName2->z==0) ); pParse->sNameToken.n = (int)( &pModuleName->z[pModuleName->n] - pParse->sNameToken.z @@ -372,11 +376,11 @@ static void addArgumentToVtab(Parse *pParse){ if( pParse->sArg.z && pParse->pNewTable ){ const char *z = (const char*)pParse->sArg.z; int n = pParse->sArg.n; sqlite3 *db = pParse->db; - addModuleArgument(db, pParse->pNewTable, sqlite3DbStrNDup(db, z, n)); + addModuleArgument(pParse, pParse->pNewTable, sqlite3DbStrNDup(db, z, n)); } } /* ** The parser calls this routine after the CREATE VIRTUAL TABLE statement @@ -661,11 +665,12 @@ const int ARRAY_INCR = 5; /* Grow the sqlite3.aVTrans array if required */ if( (db->nVTrans%ARRAY_INCR)==0 ){ VTable **aVTrans; - int nBytes = sizeof(sqlite3_vtab *) * (db->nVTrans + ARRAY_INCR); + sqlite3_int64 nBytes = sizeof(sqlite3_vtab*)* + ((sqlite3_int64)db->nVTrans + ARRAY_INCR); aVTrans = sqlite3DbRealloc(db, (void *)db->aVTrans, nBytes); if( !aVTrans ){ return SQLITE_NOMEM_BKPT; } memset(&aVTrans[db->nVTrans], 0, sizeof(sqlite3_vtab *)*ARRAY_INCR); @@ -1157,13 +1162,13 @@ pMod->pEpoTab = pTab; pTab->nTabRef = 1; pTab->pSchema = db->aDb[0].pSchema; assert( pTab->nModuleArg==0 ); pTab->iPKey = -1; - addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); - addModuleArgument(db, pTab, 0); - addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); + addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName)); + addModuleArgument(pParse, pTab, 0); + addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName)); rc = vtabCallConstructor(db, pTab, pMod, pModule->xConnect, &zErr); if( rc ){ sqlite3ErrorMsg(pParse, "%s", zErr); sqlite3DbFree(db, zErr); sqlite3VtabEponymousTableClear(db, pMod); Index: src/wal.c ================================================================== --- src/wal.c +++ src/wal.c @@ -573,11 +573,11 @@ ){ int rc = SQLITE_OK; /* Enlarge the pWal->apWiData[] array if required */ if( pWal->nWiData<=iPage ){ - int nByte = sizeof(u32*)*(iPage+1); + sqlite3_int64 nByte = sizeof(u32*)*(iPage+1); volatile u32 **apNew; apNew = (volatile u32 **)sqlite3_realloc64((void *)pWal->apWiData, nByte); if( !apNew ){ *ppPage = 0; return SQLITE_NOMEM_BKPT; @@ -677,10 +677,11 @@ s1 = s2 = 0; } assert( nByte>=8 ); assert( (nByte&0x00000007)==0 ); + assert( nByte<=65536 ); if( nativeCksum ){ do { s1 += *aData++ + s2; s2 += *aData++ + s1; @@ -1619,11 +1620,11 @@ */ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ WalIterator *p; /* Return value */ int nSegment; /* Number of segments to merge */ u32 iLast; /* Last frame in log */ - int nByte; /* Number of bytes to allocate */ + sqlite3_int64 nByte; /* Number of bytes to allocate */ int i; /* Iterator variable */ ht_slot *aTmp; /* Temp space used by merge-sort */ int rc = SQLITE_OK; /* Return Code */ /* This routine only runs while holding the checkpoint lock. And Index: src/wherecode.c ================================================================== --- src/wherecode.c +++ src/wherecode.c @@ -1965,11 +1965,16 @@ testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); pExpr = sqlite3ExprDup(db, pExpr, 0); pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); } if( pAndExpr ){ - pAndExpr = sqlite3PExpr(pParse, TK_AND|TKFLG_DONTFOLD, 0, pAndExpr); + /* The extra 0x10000 bit on the opcode is masked off and does not + ** become part of the new Expr.op. However, it does make the + ** op==TK_AND comparison inside of sqlite3PExpr() false, and this + ** prevents sqlite3PExpr() from implementing AND short-circuit + ** optimization, which we do not want here. */ + pAndExpr = sqlite3PExpr(pParse, TK_AND|0x10000, 0, pAndExpr); } } /* Run a separate WHERE clause for each term of the OR clause. After ** eliminating duplicates from other WHERE clauses, the action for each Index: test/altertab3.test ================================================================== --- test/altertab3.test +++ test/altertab3.test @@ -97,12 +97,13 @@ } {1 {error in trigger tr1: no such table: main.t2}} do_execsql_test 4.1.2 { COMMIT; } do_execsql_test 4.1.3 { - SELECT * FROM sqlite_master WHERE type='table' AND name!='t1'; -} {table t3 t3 3 {CREATE TABLE t3(e, f)}} + SELECT type, name, tbl_name, sql + FROM sqlite_master WHERE type='table' AND name!='t1'; +} {table t3 t3 {CREATE TABLE t3(e, f)}} do_catchsql_test 4.2.1 { BEGIN; ALTER TABLE t3 RENAME e TO eee; @@ -109,12 +110,13 @@ } {1 {error in trigger tr1: no such table: main.t2}} do_execsql_test 4.2.2 { COMMIT; } do_execsql_test 4.2.3 { - SELECT * FROM sqlite_master WHERE type='table' AND name!='t1'; -} {table t3 t3 3 {CREATE TABLE t3(e, f)}} + SELECT type, name, tbl_name, sql + FROM sqlite_master WHERE type='table' AND name!='t1'; +} {table t3 t3 {CREATE TABLE t3(e, f)}} #------------------------------------------------------------------------- reset_db do_execsql_test 5.0 { CREATE TABLE t1 ( Index: test/badutf2.test ================================================================== --- test/badutf2.test +++ test/badutf2.test @@ -96,16 +96,22 @@ set res [ sqlite3_exec db $sql ] lindex [ lindex $res 1] 1 } $uval } - do_test badutf2-4.1.$i { - sqlite3_reset $S - sqlite3_bind_text $S 1 $xstr $len - sqlite3_step $S - utf8_to_ustr2 [ sqlite3_column_text $S 0 ] - } $ustr + # Tcl 8.7 and later do automatic bad-utf8 correction for + # characters 0x80 thru 0x9f so test case 5 does not work here. + if {$i==5 && $tcl_version>=8.7} { + # no-op + } else { + do_test badutf2-4.1.$i { + sqlite3_reset $S + sqlite3_bind_text $S 1 $xstr $len + sqlite3_step $S + utf8_to_ustr2 [ sqlite3_column_text $S 0 ] + } $ustr + } ifcapable debug { do_test badutf2-5.1.$i { utf8_to_utf8 $uval } $u2u Index: test/corruptC.test ================================================================== --- test/corruptC.test +++ test/corruptC.test @@ -95,11 +95,12 @@ # insert corrupt byte(s) hexio_write test.db 2053 [format %02x 0x04] sqlite3 db test.db catchsql {PRAGMA integrity_check} -} {1 {database disk image is malformed}} +} {0 {{*** in database main *** +Page 3: free space corruption}}} # test that a corrupt content offset size is handled (seed 5649) # # Update 2016-12-27: As of check-in [0b86fbca66] "In sqlite3BtreeInsert() when # replacing a re-existing row, try to overwrite the cell directly rather than Index: test/dbfuzz001.test ================================================================== --- test/dbfuzz001.test +++ test/dbfuzz001.test @@ -187,15 +187,13 @@ # Prior to a certain fix to sqlite3BtreeDelete() and because of the # corruption to the freeblock list on page 8, this would fail to # cause a rebalance operation, which would leave the btree in a weird # state that would lead to segfaults and or assertion faults. # -set res {0 {}} -ifcapable oversize_cell_check { set res {1 {database disk image is malformed}} } -do_catchsql_test dbfuzz001-110 { +do_execsql_test dbfuzz001-110 { DELETE FROM t3 WHERE x IS NOT NULL AND +rowid=6; -} $res +} {} # This is a dbfuzz2-generate test case that can cause a page with # pPage->nCell==0 to enter the balancer. # do_test dbfuzz001-200 { Index: test/fts3varint.test ================================================================== --- test/fts3varint.test +++ test/fts3varint.test @@ -108,11 +108,14 @@ 1152921504606846975 1152921504606846976 1152921504606846977 } do_fts3_varint_test 2.61 { 2305843009213693951 2305843009213693952 2305843009213693953 } do_fts3_varint_test 2.62 { 4611686018427387903 4611686018427387904 4611686018427387905 } -do_fts3_varint_test 2.63 { - 9223372036854775807 9223372036854775808 9223372036854775809 } + +if {![catch {fts3_test_varint 18446744073709551615}]} { + do_fts3_varint_test 2.63 { + 9223372036854775807 9223372036854775808 9223372036854775809 } -do_fts3_varint_test 3.0 { 18446744073709551615 -18446744073709551615 } + do_fts3_varint_test 3.0 { 18446744073709551615 -18446744073709551615 } +} finish_test Index: test/fuzzdata8.db ================================================================== --- test/fuzzdata8.db +++ test/fuzzdata8.db cannot compute difference between binary files Index: test/journal3.test ================================================================== --- test/journal3.test +++ test/journal3.test @@ -36,10 +36,13 @@ 3 00600 4 00755 } { db close #set effective [format %.5o [expr $permissions & ~$umask]] + if {$tcl_version>=8.7} { + regsub {^00} $permissions {0o} permissions + } set effective $permissions do_test journal3-1.2.$tn.1 { catch { forcedelete test.db-journal } file attributes test.db -permissions $permissions file attributes test.db -permissions Index: test/memdb1.test ================================================================== --- test/memdb1.test +++ test/memdb1.test @@ -184,21 +184,23 @@ set rc [catch {db serialize a b} msg] lappend rc $msg } {1 {wrong # args: should be "db serialize ?DATABASE?"}} #------------------------------------------------------------------------- -reset_db -do_execsql_test 700 { - CREATE TABLE t1(a, b); - PRAGMA schema_version = 0; -} -do_test 710 { - set ser [db serialize main] - db close - sqlite3 db - db deserialize main $ser - catchsql { - CREATE VIRTUAL TABLE t1 USING rtree(id, a, b, c, d); - } -} {1 {table t1 already exists}} +ifcapable vtab { + reset_db + do_execsql_test 700 { + CREATE TABLE t1(a, b); + PRAGMA schema_version = 0; + } + do_test 710 { + set ser [db serialize main] + db close + sqlite3 db + db deserialize main $ser + catchsql { + CREATE VIRTUAL TABLE t1 USING rtree(id, a, b, c, d); + } + } {1 {table t1 already exists}} +} finish_test Index: test/pragma4.test ================================================================== --- test/pragma4.test +++ test/pragma4.test @@ -174,10 +174,11 @@ sqlite3 db3 test.db sqlite3 db2 test.db2 execsql { DROP INDEX i1 } db3 execsql { DROP INDEX i2 } db2 } {} +if {[permutation]=="prepare"} { catchsql { SELECT * FROM sqlite_master } } ifcapable vtab { do_execsql_test 4.3.5 { SELECT * FROM pragma_index_info('i1') } do_execsql_test 4.3.6 { SELECT * FROM pragma_index_info('i2') } } @@ -192,10 +193,13 @@ } do_test 4.4.3 { execsql { DROP INDEX i1 } db3 execsql { DROP INDEX i2 } db2 } {} +if {[permutation]=="prepare"} { + catchsql { SELECT * FROM sqlite_master, aux.sqlite_master } +} ifcapable vtab { do_execsql_test 4.4.5 { SELECT * FROM pragma_index_list('t1') } {} do_execsql_test 4.4.6 { SELECT * FROM pragma_index_list('t2') } {} } execsql {SELECT * FROM main.sqlite_master, aux.sqlite_master} @@ -216,10 +220,13 @@ } do_test 4.5.3 { execsql { DROP TABLE c1 } db3 execsql { DROP TABLE c2 } db2 } {} +if {[permutation]=="prepare"} { + catchsql { SELECT * FROM sqlite_master, aux.sqlite_master } +} ifcapable vtab { do_execsql_test 4.5.4 { SELECT * FROM pragma_foreign_key_list('c1') } do_execsql_test 4.5.5 { SELECT * FROM pragma_foreign_key_list('c2') } } execsql {SELECT * FROM main.sqlite_master, aux.sqlite_master} ADDED test/releasetest_data.tcl Index: test/releasetest_data.tcl ================================================================== --- /dev/null +++ test/releasetest_data.tcl @@ -0,0 +1,412 @@ + +# This file contains Configuration data used by "wapptest.tcl" and +# "releasetest.tcl". +# + +# Omit comments (text between # and \n) in a long multi-line string. +# +proc strip_comments {in} { + regsub -all {#[^\n]*\n} $in {} out + return $out +} + +array set ::Configs [strip_comments { + "Default" { + -O2 + --disable-amalgamation --disable-shared + --enable-session + -DSQLITE_ENABLE_DESERIALIZE + } + "Sanitize" { + CC=clang -fsanitize=undefined + -DSQLITE_ENABLE_STAT4 + --enable-session + } + "Stdcall" { + -DUSE_STDCALL=1 + -O2 + } + "Have-Not" { + # The "Have-Not" configuration sets all possible -UHAVE_feature options + # in order to verify that the code works even on platforms that lack + # these support services. + -DHAVE_FDATASYNC=0 + -DHAVE_GMTIME_R=0 + -DHAVE_ISNAN=0 + -DHAVE_LOCALTIME_R=0 + -DHAVE_LOCALTIME_S=0 + -DHAVE_MALLOC_USABLE_SIZE=0 + -DHAVE_STRCHRNUL=0 + -DHAVE_USLEEP=0 + -DHAVE_UTIME=0 + } + "Unlock-Notify" { + -O2 + -DSQLITE_ENABLE_UNLOCK_NOTIFY + -DSQLITE_THREADSAFE + -DSQLITE_TCL_DEFAULT_FULLMUTEX=1 + } + "User-Auth" { + -O2 + -DSQLITE_USER_AUTHENTICATION=1 + } + "Secure-Delete" { + -O2 + -DSQLITE_SECURE_DELETE=1 + -DSQLITE_SOUNDEX=1 + } + "Update-Delete-Limit" { + -O2 + -DSQLITE_DEFAULT_FILE_FORMAT=4 + -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1 + -DSQLITE_ENABLE_STMT_SCANSTATUS + -DSQLITE_LIKE_DOESNT_MATCH_BLOBS + -DSQLITE_ENABLE_CURSOR_HINTS + --enable-json1 + } + "Check-Symbols" { + -DSQLITE_MEMDEBUG=1 + -DSQLITE_ENABLE_FTS3_PARENTHESIS=1 + -DSQLITE_ENABLE_FTS3=1 + -DSQLITE_ENABLE_RTREE=1 + -DSQLITE_ENABLE_MEMSYS5=1 + -DSQLITE_ENABLE_MEMSYS3=1 + -DSQLITE_ENABLE_COLUMN_METADATA=1 + -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1 + -DSQLITE_SECURE_DELETE=1 + -DSQLITE_SOUNDEX=1 + -DSQLITE_ENABLE_ATOMIC_WRITE=1 + -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1 + -DSQLITE_ENABLE_OVERSIZE_CELL_CHECK=1 + -DSQLITE_ENABLE_STAT4 + -DSQLITE_ENABLE_STMT_SCANSTATUS + --enable-json1 --enable-fts5 --enable-session + } + "Debug-One" { + --disable-shared + -O2 -funsigned-char + -DSQLITE_DEBUG=1 + -DSQLITE_MEMDEBUG=1 + -DSQLITE_MUTEX_NOOP=1 + -DSQLITE_TCL_DEFAULT_FULLMUTEX=1 + -DSQLITE_ENABLE_FTS3=1 + -DSQLITE_ENABLE_RTREE=1 + -DSQLITE_ENABLE_MEMSYS5=1 + -DSQLITE_ENABLE_COLUMN_METADATA=1 + -DSQLITE_ENABLE_STAT4 + -DSQLITE_ENABLE_HIDDEN_COLUMNS + -DSQLITE_MAX_ATTACHED=125 + -DSQLITE_MUTATION_TEST + --enable-fts5 --enable-json1 + } + "Fast-One" { + -O6 + -DSQLITE_ENABLE_FTS4=1 + -DSQLITE_ENABLE_RTREE=1 + -DSQLITE_ENABLE_STAT4 + -DSQLITE_ENABLE_RBU + -DSQLITE_MAX_ATTACHED=125 + -DLONGDOUBLE_TYPE=double + --enable-session + } + "Device-One" { + -O2 + -DSQLITE_DEBUG=1 + -DSQLITE_DEFAULT_AUTOVACUUM=1 + -DSQLITE_DEFAULT_CACHE_SIZE=64 + -DSQLITE_DEFAULT_PAGE_SIZE=1024 + -DSQLITE_DEFAULT_TEMP_CACHE_SIZE=32 + -DSQLITE_DISABLE_LFS=1 + -DSQLITE_ENABLE_ATOMIC_WRITE=1 + -DSQLITE_ENABLE_IOTRACE=1 + -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1 + -DSQLITE_MAX_PAGE_SIZE=4096 + -DSQLITE_OMIT_LOAD_EXTENSION=1 + -DSQLITE_OMIT_PROGRESS_CALLBACK=1 + -DSQLITE_OMIT_VIRTUALTABLE=1 + -DSQLITE_ENABLE_HIDDEN_COLUMNS + -DSQLITE_TEMP_STORE=3 + --enable-json1 + } + "Device-Two" { + -DSQLITE_4_BYTE_ALIGNED_MALLOC=1 + -DSQLITE_DEFAULT_AUTOVACUUM=1 + -DSQLITE_DEFAULT_CACHE_SIZE=1000 + -DSQLITE_DEFAULT_LOCKING_MODE=0 + -DSQLITE_DEFAULT_PAGE_SIZE=1024 + -DSQLITE_DEFAULT_TEMP_CACHE_SIZE=1000 + -DSQLITE_DISABLE_LFS=1 + -DSQLITE_ENABLE_FTS3=1 + -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1 + -DSQLITE_ENABLE_RTREE=1 + -DSQLITE_MAX_COMPOUND_SELECT=50 + -DSQLITE_MAX_PAGE_SIZE=32768 + -DSQLITE_OMIT_TRACE=1 + -DSQLITE_TEMP_STORE=3 + -DSQLITE_THREADSAFE=2 + -DSQLITE_ENABLE_DESERIALIZE=1 + --enable-json1 --enable-fts5 --enable-session + } + "Locking-Style" { + -O2 + -DSQLITE_ENABLE_LOCKING_STYLE=1 + } + "Apple" { + -Os + -DHAVE_GMTIME_R=1 + -DHAVE_ISNAN=1 + -DHAVE_LOCALTIME_R=1 + -DHAVE_PREAD=1 + -DHAVE_PWRITE=1 + -DHAVE_USLEEP=1 + -DHAVE_USLEEP=1 + -DHAVE_UTIME=1 + -DSQLITE_DEFAULT_CACHE_SIZE=1000 + -DSQLITE_DEFAULT_CKPTFULLFSYNC=1 + -DSQLITE_DEFAULT_MEMSTATUS=1 + -DSQLITE_DEFAULT_PAGE_SIZE=1024 + -DSQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS=1 + -DSQLITE_ENABLE_API_ARMOR=1 + -DSQLITE_ENABLE_AUTO_PROFILE=1 + -DSQLITE_ENABLE_FLOCKTIMEOUT=1 + -DSQLITE_ENABLE_FTS3=1 + -DSQLITE_ENABLE_FTS3_PARENTHESIS=1 + -DSQLITE_ENABLE_FTS3_TOKENIZER=1 + if:os=="Darwin" -DSQLITE_ENABLE_LOCKING_STYLE=1 + -DSQLITE_ENABLE_PERSIST_WAL=1 + -DSQLITE_ENABLE_PURGEABLE_PCACHE=1 + -DSQLITE_ENABLE_RTREE=1 + -DSQLITE_ENABLE_SNAPSHOT=1 + # -DSQLITE_ENABLE_SQLLOG=1 + -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1 + -DSQLITE_MAX_LENGTH=2147483645 + -DSQLITE_MAX_VARIABLE_NUMBER=500000 + # -DSQLITE_MEMDEBUG=1 + -DSQLITE_NO_SYNC=1 + -DSQLITE_OMIT_AUTORESET=1 + -DSQLITE_OMIT_LOAD_EXTENSION=1 + -DSQLITE_PREFER_PROXY_LOCKING=1 + -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 + -DSQLITE_THREADSAFE=2 + -DSQLITE_USE_URI=1 + -DSQLITE_WRITE_WALFRAME_PREBUFFERED=1 + -DUSE_GUARDED_FD=1 + -DUSE_PREAD=1 + --enable-json1 --enable-fts5 + } + "Extra-Robustness" { + -DSQLITE_ENABLE_OVERSIZE_CELL_CHECK=1 + -DSQLITE_MAX_ATTACHED=62 + } + "Devkit" { + -DSQLITE_DEFAULT_FILE_FORMAT=4 + -DSQLITE_MAX_ATTACHED=30 + -DSQLITE_ENABLE_COLUMN_METADATA + -DSQLITE_ENABLE_FTS4 + -DSQLITE_ENABLE_FTS5 + -DSQLITE_ENABLE_FTS4_PARENTHESIS + -DSQLITE_DISABLE_FTS4_DEFERRED + -DSQLITE_ENABLE_RTREE + --enable-json1 --enable-fts5 + } + "No-lookaside" { + -DSQLITE_TEST_REALLOC_STRESS=1 + -DSQLITE_OMIT_LOOKASIDE=1 + -DHAVE_USLEEP=1 + } + "Valgrind" { + -DSQLITE_ENABLE_STAT4 + -DSQLITE_ENABLE_FTS4 + -DSQLITE_ENABLE_RTREE + -DSQLITE_ENABLE_HIDDEN_COLUMNS + --enable-json1 + } + + # The next group of configurations are used only by the + # Failure-Detection platform. They are all the same, but we need + # different names for them all so that they results appear in separate + # subdirectories. + # + Fail0 {-O0} + Fail2 {-O0} + Fail3 {-O0} + Fail4 {-O0} + FuzzFail1 {-O0} + FuzzFail2 {-O0} +}] + +array set ::Platforms [strip_comments { + Linux-x86_64 { + "Check-Symbols" checksymbols + "Fast-One" "fuzztest test" + "Debug-One" "mptest test" + "Have-Not" test + "Secure-Delete" test + "Unlock-Notify" "QUICKTEST_INCLUDE=notify2.test test" + "User-Auth" tcltest + "Update-Delete-Limit" test + "Extra-Robustness" test + "Device-Two" test + "No-lookaside" test + "Devkit" test + "Apple" test + "Sanitize" {QUICKTEST_OMIT=func4.test,nan.test test} + "Device-One" fulltest + "Default" "threadtest fulltest" + "Valgrind" valgrindtest + } + Linux-i686 { + "Devkit" test + "Have-Not" test + "Unlock-Notify" "QUICKTEST_INCLUDE=notify2.test test" + "Device-One" test + "Device-Two" test + "Default" "threadtest fulltest" + } + Darwin-i386 { + "Locking-Style" "mptest test" + "Have-Not" test + "Apple" "threadtest fulltest" + } + Darwin-x86_64 { + "Locking-Style" "mptest test" + "Have-Not" test + "Apple" "threadtest fulltest" + } + "Windows NT-intel" { + "Stdcall" test + "Have-Not" test + "Default" "mptest fulltestonly" + } + "Windows NT-amd64" { + "Stdcall" test + "Have-Not" test + "Default" "mptest fulltestonly" + } + + # The Failure-Detection platform runs various tests that deliberately + # fail. This is used as a test of this script to verify that this script + # correctly identifies failures. + # + Failure-Detection { + Fail0 "TEST_FAILURE=0 test" + Sanitize "TEST_FAILURE=1 test" + Fail2 "TEST_FAILURE=2 valgrindtest" + Fail3 "TEST_FAILURE=3 valgrindtest" + Fail4 "TEST_FAILURE=4 test" + FuzzFail1 "TEST_FAILURE=5 test" + FuzzFail2 "TEST_FAILURE=5 valgrindtest" + } +}] + +proc make_test_suite {msvc withtcl name testtarget config} { + + # Tcl variable $opts is used to build up the value used to set the + # OPTS Makefile variable. Variable $cflags holds the value for + # CFLAGS. The makefile will pass OPTS to both gcc and lemon, but + # CFLAGS is only passed to gcc. + # + set makeOpts "" + set cflags [expr {$msvc ? "-Zi" : "-g"}] + set opts "" + set title ${name}($testtarget) + set configOpts $withtcl + set skip 0 + + regsub -all {#[^\n]*\n} $config \n config + foreach arg $config { + if {$skip} { + set skip 0 + continue + } + if {[regexp {^-[UD]} $arg]} { + lappend opts $arg + } elseif {[regexp {^[A-Z]+=} $arg]} { + lappend testtarget $arg + } elseif {[regexp {^if:([a-z]+)(.*)} $arg all key tail]} { + # Arguments of the form 'if:os=="Linux"' will cause the subsequent + # argument to be skipped if the $tcl_platform(os) is not "Linux", for + # example... + set skip [expr !(\$::tcl_platform($key)$tail)] + } elseif {[regexp {^--(enable|disable)-} $arg]} { + if {$msvc} { + if {$arg eq "--disable-amalgamation"} { + lappend makeOpts USE_AMALGAMATION=0 + continue + } + if {$arg eq "--disable-shared"} { + lappend makeOpts USE_CRT_DLL=0 DYNAMIC_SHELL=0 + continue + } + if {$arg eq "--enable-fts5"} { + lappend opts -DSQLITE_ENABLE_FTS5 + continue + } + if {$arg eq "--enable-json1"} { + lappend opts -DSQLITE_ENABLE_JSON1 + continue + } + if {$arg eq "--enable-shared"} { + lappend makeOpts USE_CRT_DLL=1 DYNAMIC_SHELL=1 + continue + } + } + lappend configOpts $arg + } else { + if {$msvc} { + if {$arg eq "-g"} { + lappend cflags -Zi + continue + } + if {[regexp -- {^-O(\d+)$} $arg all level]} then { + lappend makeOpts OPTIMIZATIONS=$level + continue + } + } + lappend cflags $arg + } + } + + # Disable sync to make testing faster. + # + lappend opts -DSQLITE_NO_SYNC=1 + + # Some configurations already set HAVE_USLEEP; in that case, skip it. + # + if {[lsearch -regexp $opts {^-DHAVE_USLEEP(?:=|$)}]==-1} { + lappend opts -DHAVE_USLEEP=1 + } + + # Add the define for this platform. + # + if {$::tcl_platform(platform)=="windows"} { + lappend opts -DSQLITE_OS_WIN=1 + } else { + lappend opts -DSQLITE_OS_UNIX=1 + } + + # Set the sub-directory to use. + # + set dir [string tolower [string map {- _ " " _ "(" _ ")" _} $name]] + + # Join option lists into strings, using space as delimiter. + # + set makeOpts [join $makeOpts " "] + set cflags [join $cflags " "] + set opts [join $opts " "] + + return [list $title $dir $configOpts $testtarget $makeOpts $cflags $opts] +} + +# Configuration verification: Check that each entry in the list of configs +# specified for each platforms exists. +# +foreach {key value} [array get ::Platforms] { + foreach {v t} $value { + if {0==[info exists ::Configs($v)]} { + puts stderr "No such configuration: \"$v\"" + exit -1 + } + } +} + Index: test/shell1.test ================================================================== --- test/shell1.test +++ test/shell1.test @@ -1020,10 +1020,12 @@ # cannot be used here. # if {$i==0x0D || ($tcl_platform(platform)=="windows" && $i==0x1A)} { continue } + # Tcl 8.7 maps 0x80 through 0x9f into valid UTF8. So skip those tests. + if {$i>=0x80 && $i<=0x9f} continue if {$i>=0xE0 && $tcl_platform(os)=="OpenBSD"} continue if {$i>=0xE0 && $i<=0xEF && $tcl_platform(os)=="Linux"} continue set hex [format %02X $i] set char [subst \\x$hex]; set oldChar $char set escapes [list] Index: test/skipscan1.test ================================================================== --- test/skipscan1.test +++ test/skipscan1.test @@ -342,7 +342,35 @@ do_execsql_test skipscan1-9.3 { EXPLAIN QUERY PLAN SELECT * FROM t9a WHERE b IN (SELECT x FROM t9b WHERE y!=5); } {/{SCAN TABLE t9a}/} optimization_control db skip-scan 1 + +do_execsql_test skipscan1-2.1 { + CREATE TABLE t6(a TEXT, b INT, c INT, d INT); + CREATE INDEX t6abc ON t6(a,b,c); + INSERT INTO t6 VALUES('abc',123,4,5); + + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1 VALUES('t6','t6abc','10000 5000 2000 10'); + ANALYZE sqlite_master; + DELETE FROM t6; +} {} + +do_execsql_test skipscan1-2.2eqp { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t6 WHERE d<>99 AND b=345 ORDER BY a; +} {/* USING INDEX t6abc (ANY(a) AND b=?)*/} +do_execsql_test skipscan1-2.2 { + SELECT a,b,c,d,'|' FROM t6 WHERE d<>99 AND b=345 ORDER BY a; +} {} + +do_execsql_test skipscan1-2.3eqp { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t6 WHERE d<>99 AND b=345 ORDER BY a DESC; +} {/* USING INDEX t6abc (ANY(a) AND b=?)*/} +do_execsql_test skipscan1-2.3 { + SELECT a,b,c,d,'|' FROM t6 WHERE d<>99 AND b=345 ORDER BY a DESC; +} {} finish_test Index: test/sqllimits1.test ================================================================== --- test/sqllimits1.test +++ test/sqllimits1.test @@ -887,6 +887,16 @@ foreach {key value} [array get saved] { catch {set $key $value} } + +#------------------------------------------------------------------------- +# At one point the following caused an assert() to fail. +# +sqlite3_limit db SQLITE_LIMIT_LENGTH 10000 +set nm [string repeat x 10000] +do_catchsql_test sqllimits1-17.1 " + CREATE TABLE $nm (x PRIMARY KEY) +" {1 {string or blob too big}} + finish_test Index: test/wal2.test ================================================================== --- test/wal2.test +++ test/wal2.test @@ -1107,11 +1107,11 @@ 4 00755 } { set effective [format %.5o [expr $permissions & ~$umask]] do_test wal2-12.2.$tn.1 { file attributes test.db -permissions $permissions - file attributes test.db -permissions + string map {o 0} [file attributes test.db -permissions] } $permissions do_test wal2-12.2.$tn.2 { list [file exists test.db-wal] [file exists $shmpath] } {0 0} do_test wal2-12.2.$tn.3 { @@ -1118,11 +1118,12 @@ sqlite3 db test.db execsql { INSERT INTO tx DEFAULT VALUES } list [file exists test.db-wal] [file exists $shmpath] } {1 1} do_test wal2-12.2.$tn.4 { - list [file attr test.db-wal -perm] [file attr $shmpath -perm] + set x [list [file attr test.db-wal -perm] [file attr $shmpath -perm]] + string map {o 0} $x } [list $effective $effective] do_test wal2-12.2.$tn.5 { ifcapable enable_persist_wal { file_control_persist_wal db 0 } @@ -1189,10 +1190,11 @@ file attr $shmpath -perm $shm_perm set L [file attr test.db -perm] lappend L [file attr test.db-wal -perm] lappend L [file attr $shmpath -perm] + string map {o 0} $L } [list $db_perm $wal_perm $shm_perm] # If $can_open is true, then it should be possible to open a database # handle. Otherwise, if $can_open is 0, attempting to open the db # handle throws an "unable to open database file" exception. ADDED test/wapp.tcl Index: test/wapp.tcl ================================================================== --- /dev/null +++ test/wapp.tcl @@ -0,0 +1,987 @@ +# Copyright (c) 2017 D. Richard Hipp +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the Simplified BSD License (also +# known as the "2-Clause License" or "FreeBSD License".) +# +# This program is distributed in the hope that it will be useful, +# but without any warranty; without even the implied warranty of +# merchantability or fitness for a particular purpose. +# +#--------------------------------------------------------------------------- +# +# Design rules: +# +# (1) All identifiers in the global namespace begin with "wapp" +# +# (2) Indentifiers intended for internal use only begin with "wappInt" +# +package require Tcl 8.6 + +# Add text to the end of the HTTP reply. No interpretation or transformation +# of the text is performs. The argument should be enclosed within {...} +# +proc wapp {txt} { + global wapp + dict append wapp .reply $txt +} + +# Add text to the page under construction. Do no escaping on the text. +# +# Though "unsafe" in general, there are uses for this kind of thing. +# For example, if you want to return the complete, unmodified content of +# a file: +# +# set fd [open content.html rb] +# wapp-unsafe [read $fd] +# close $fd +# +# You could do the same thing using ordinary "wapp" instead of "wapp-unsafe". +# The difference is that wapp-safety-check will complain about the misuse +# of "wapp", but it assumes that the person who write "wapp-unsafe" understands +# the risks. +# +# Though occasionally necessary, the use of this interface should be minimized. +# +proc wapp-unsafe {txt} { + global wapp + dict append wapp .reply $txt +} + +# Add text to the end of the reply under construction. The following +# substitutions are made: +# +# %html(...) Escape text for inclusion in HTML +# %url(...) Escape text for use as a URL +# %qp(...) Escape text for use as a URI query parameter +# %string(...) Escape text for use within a JSON string +# %unsafe(...) No transformations of the text +# +# The substitutions above terminate at the first ")" character. If the +# text of the TCL string in ... contains ")" characters itself, use instead: +# +# %html%(...)% +# %url%(...)% +# %qp%(...)% +# %string%(...)% +# %unsafe%(...)% +# +# In other words, use "%(...)%" instead of "(...)" to include the TCL string +# to substitute. +# +# The %unsafe substitution should be avoided whenever possible, obviously. +# In addition to the substitutions above, the text also does backslash +# escapes. +# +# The wapp-trim proc works the same as wapp-subst except that it also removes +# whitespace from the left margin, so that the generated HTML/CSS/Javascript +# does not appear to be indented when delivered to the client web browser. +# +if {$tcl_version>=8.7} { + proc wapp-subst {txt} { + global wapp + regsub -all -command \ + {%(html|url|qp|string|unsafe){1,1}?(|%)\((.+)\)\2} $txt wappInt-enc txt + dict append wapp .reply [subst -novariables -nocommand $txt] + } + proc wapp-trim {txt} { + global wapp + regsub -all {\n\s+} [string trim $txt] \n txt + regsub -all -command \ + {%(html|url|qp|string|unsafe){1,1}?(|%)\((.+)\)\2} $txt wappInt-enc txt + dict append wapp .reply [subst -novariables -nocommand $txt] + } + proc wappInt-enc {all mode nu1 txt} { + return [uplevel 2 "wappInt-enc-$mode \"$txt\""] + } +} else { + proc wapp-subst {txt} { + global wapp + regsub -all {%(html|url|qp|string|unsafe){1,1}?(|%)\((.+)\)\2} $txt \ + {[wappInt-enc-\1 "\3"]} txt + dict append wapp .reply [uplevel 1 [list subst -novariables $txt]] + } + proc wapp-trim {txt} { + global wapp + regsub -all {\n\s+} [string trim $txt] \n txt + regsub -all {%(html|url|qp|string|unsafe){1,1}?(|%)\((.+)\)\2} $txt \ + {[wappInt-enc-\1 "\3"]} txt + dict append wapp .reply [uplevel 1 [list subst -novariables $txt]] + } +} + +# There must be a wappInt-enc-NAME routine for each possible substitution +# in wapp-subst. Thus there are routines for "html", "url", "qp", and "unsafe". +# +# wappInt-enc-html Escape text so that it is safe to use in the +# body of an HTML document. +# +# wappInt-enc-url Escape text so that it is safe to pass as an +# argument to href= and src= attributes in HTML. +# +# wappInt-enc-qp Escape text so that it is safe to use as the +# value of a query parameter in a URL or in +# post data or in a cookie. +# +# wappInt-enc-string Escape ", ', \, and < for using inside of a +# javascript string literal. The < character +# is escaped to prevent "" from causing +# problems in embedded javascript. +# +# wappInt-enc-unsafe Perform no encoding at all. Unsafe. +# +proc wappInt-enc-html {txt} { + return [string map {& & < < > > \" " \\ \} $txt] +} +proc wappInt-enc-unsafe {txt} { + return $txt +} +proc wappInt-enc-url {s} { + if {[regsub -all {[^-{}@~?=#_.:/a-zA-Z0-9]} $s {[wappInt-%HHchar {&}]} s]} { + set s [subst -novar -noback $s] + } + if {[regsub -all {[{}]} $s {[wappInt-%HHchar \\&]} s]} { + set s [subst -novar -noback $s] + } + return $s +} +proc wappInt-enc-qp {s} { + if {[regsub -all {[^-{}_.a-zA-Z0-9]} $s {[wappInt-%HHchar {&}]} s]} { + set s [subst -novar -noback $s] + } + if {[regsub -all {[{}]} $s {[wappInt-%HHchar \\&]} s]} { + set s [subst -novar -noback $s] + } + return $s +} +proc wappInt-enc-string {s} { + return [string map {\\ \\\\ \" \\\" ' \\' < \\u003c} $s] +} + +# This is a helper routine for wappInt-enc-url and wappInt-enc-qp. It returns +# an appropriate %HH encoding for the single character c. If c is a unicode +# character, then this routine might return multiple bytes: %HH%HH%HH +# +proc wappInt-%HHchar {c} { + if {$c==" "} {return +} + return [regsub -all .. [binary encode hex [encoding convertto utf-8 $c]] {%&}] +} + + +# Undo the www-url-encoded format. +# +# HT: This code stolen from ncgi.tcl +# +proc wappInt-decode-url {str} { + set str [string map [list + { } "\\" "\\\\" \[ \\\[ \] \\\]] $str] + regsub -all -- \ + {%([Ee][A-Fa-f0-9])%([89ABab][A-Fa-f0-9])%([89ABab][A-Fa-f0-9])} \ + $str {[encoding convertfrom utf-8 [binary decode hex \1\2\3]]} str + regsub -all -- \ + {%([CDcd][A-Fa-f0-9])%([89ABab][A-Fa-f0-9])} \ + $str {[encoding convertfrom utf-8 [binary decode hex \1\2]]} str + regsub -all -- {%([0-7][A-Fa-f0-9])} $str {\\u00\1} str + return [subst -novar $str] +} + +# Reset the document back to an empty string. +# +proc wapp-reset {} { + global wapp + dict set wapp .reply {} +} + +# Change the mime-type of the result document. +# +proc wapp-mimetype {x} { + global wapp + dict set wapp .mimetype $x +} + +# Change the reply code. +# +proc wapp-reply-code {x} { + global wapp + dict set wapp .reply-code $x +} + +# Set a cookie +# +proc wapp-set-cookie {name value} { + global wapp + dict lappend wapp .new-cookies $name $value +} + +# Unset a cookie +# +proc wapp-clear-cookie {name} { + wapp-set-cookie $name {} +} + +# Add extra entries to the reply header +# +proc wapp-reply-extra {name value} { + global wapp + dict lappend wapp .reply-extra $name $value +} + +# Specifies how the web-page under construction should be cached. +# The argument should be one of: +# +# no-cache +# max-age=N (for some integer number of seconds, N) +# private,max-age=N +# +proc wapp-cache-control {x} { + wapp-reply-extra Cache-Control $x +} + +# Redirect to a different web page +# +proc wapp-redirect {uri} { + wapp-reply-code {307 Redirect} + wapp-reply-extra Location $uri +} + +# Return the value of a wapp parameter +# +proc wapp-param {name {dflt {}}} { + global wapp + if {![dict exists $wapp $name]} {return $dflt} + return [dict get $wapp $name] +} + +# Return true if a and only if the wapp parameter $name exists +# +proc wapp-param-exists {name} { + global wapp + return [dict exists $wapp $name] +} + +# Set the value of a wapp parameter +# +proc wapp-set-param {name value} { + global wapp + dict set wapp $name $value +} + +# Return all parameter names that match the GLOB pattern, or all +# names if the GLOB pattern is omitted. +# +proc wapp-param-list {{glob {*}}} { + global wapp + return [dict keys $wapp $glob] +} + +# By default, Wapp does not decode query parameters and POST parameters +# for cross-origin requests. This is a security restriction, designed to +# help prevent cross-site request forgery (CSRF) attacks. +# +# As a consequence of this restriction, URLs for sites generated by Wapp +# that contain query parameters will not work as URLs found in other +# websites. You cannot create a link from a second website into a Wapp +# website if the link contains query planner, by default. +# +# Of course, it is sometimes desirable to allow query parameters on external +# links. For URLs for which this is safe, the application should invoke +# wapp-allow-xorigin-params. This procedure tells Wapp that it is safe to +# go ahead and decode the query parameters even for cross-site requests. +# +# In other words, for Wapp security is the default setting. Individual pages +# need to actively disable the cross-site request security if those pages +# are safe for cross-site access. +# +proc wapp-allow-xorigin-params {} { + global wapp + if {![dict exists $wapp .qp] && ![dict get $wapp SAME_ORIGIN]} { + wappInt-decode-query-params + } +} + +# Set the content-security-policy. +# +# The default content-security-policy is very strict: "default-src 'self'" +# The default policy prohibits the use of in-line javascript or CSS. +# +# Provide an alternative CSP as the argument. Or use "off" to disable +# the CSP completely. +# +proc wapp-content-security-policy {val} { + global wapp + if {$val=="off"} { + dict unset wapp .csp + } else { + dict set wapp .csp $val + } +} + +# Examine the bodys of all procedures in this program looking for +# unsafe calls to various Wapp interfaces. Return a text string +# containing warnings. Return an empty string if all is ok. +# +# This routine is advisory only. It misses some constructs that are +# dangerous and flags others that are safe. +# +proc wapp-safety-check {} { + set res {} + foreach p [info procs] { + set ln 0 + foreach x [split [info body $p] \n] { + incr ln + if {[regexp {^[ \t]*wapp[ \t]+([^\n]+)} $x all tail] + && [string index $tail 0]!="\173" + && [regexp {[[$]} $tail] + } { + append res "$p:$ln: unsafe \"wapp\" call: \"[string trim $x]\"\n" + } + if {[regexp {^[ \t]*wapp-(subst|trim)[ \t]+[^\173]} $x all cx]} { + append res "$p:$ln: unsafe \"wapp-$cx\" call: \"[string trim $x]\"\n" + } + } + } + return $res +} + +# Return a string that descripts the current environment. Applications +# might find this useful for debugging. +# +proc wapp-debug-env {} { + global wapp + set out {} + foreach var [lsort [dict keys $wapp]] { + if {[string index $var 0]=="."} continue + append out "$var = [list [dict get $wapp $var]]\n" + } + append out "\[pwd\] = [list [pwd]]\n" + return $out +} + +# Tracing function for each HTTP request. This is overridden by wapp-start +# if tracing is enabled. +# +proc wappInt-trace {} {} + +# Start up a listening socket. Arrange to invoke wappInt-new-connection +# for each inbound HTTP connection. +# +# port Listen on this TCP port. 0 means to select a port +# that is not currently in use +# +# wappmode One of "scgi", "remote-scgi", "server", or "local". +# +# fromip If not {}, then reject all requests from IP addresses +# other than $fromip +# +proc wappInt-start-listener {port wappmode fromip} { + if {[string match *scgi $wappmode]} { + set type SCGI + set server [list wappInt-new-connection \ + wappInt-scgi-readable $wappmode $fromip] + } else { + set type HTTP + set server [list wappInt-new-connection \ + wappInt-http-readable $wappmode $fromip] + } + if {$wappmode=="local" || $wappmode=="scgi"} { + set x [socket -server $server -myaddr 127.0.0.1 $port] + } else { + set x [socket -server $server $port] + } + set coninfo [chan configure $x -sockname] + set port [lindex $coninfo 2] + if {$wappmode=="local"} { + wappInt-start-browser http://127.0.0.1:$port/ + } elseif {$fromip!=""} { + puts "Listening for $type requests on TCP port $port from IP $fromip" + } else { + puts "Listening for $type requests on TCP port $port" + } +} + +# Start a web-browser and point it at $URL +# +proc wappInt-start-browser {url} { + global tcl_platform + if {$tcl_platform(platform)=="windows"} { + exec cmd /c start $url & + } elseif {$tcl_platform(os)=="Darwin"} { + exec open $url & + } elseif {[catch {exec xdg-open $url}]} { + exec firefox $url & + } +} + +# This routine is a "socket -server" callback. The $chan, $ip, and $port +# arguments are added by the socket command. +# +# Arrange to invoke $callback when content is available on the new socket. +# The $callback will process inbound HTTP or SCGI content. Reject the +# request if $fromip is not an empty string and does not match $ip. +# +proc wappInt-new-connection {callback wappmode fromip chan ip port} { + upvar #0 wappInt-$chan W + if {$fromip!="" && ![string match $fromip $ip]} { + close $chan + return + } + set W [dict create REMOTE_ADDR $ip REMOTE_PORT $port WAPP_MODE $wappmode \ + .header {}] + fconfigure $chan -blocking 0 -translation binary + fileevent $chan readable [list $callback $chan] +} + +# Close an input channel +# +proc wappInt-close-channel {chan} { + if {$chan=="stdout"} { + # This happens after completing a CGI request + exit 0 + } else { + unset ::wappInt-$chan + close $chan + } +} + +# Process new text received on an inbound HTTP request +# +proc wappInt-http-readable {chan} { + if {[catch [list wappInt-http-readable-unsafe $chan] msg]} { + puts stderr "$msg\n$::errorInfo" + wappInt-close-channel $chan + } +} +proc wappInt-http-readable-unsafe {chan} { + upvar #0 wappInt-$chan W wapp wapp + if {![dict exists $W .toread]} { + # If the .toread key is not set, that means we are still reading + # the header + set line [string trimright [gets $chan]] + set n [string length $line] + if {$n>0} { + if {[dict get $W .header]=="" || [regexp {^\s+} $line]} { + dict append W .header $line + } else { + dict append W .header \n$line + } + if {[string length [dict get $W .header]]>100000} { + error "HTTP request header too big - possible DOS attack" + } + } elseif {$n==0} { + # We have reached the blank line that terminates the header. + global argv0 + set a0 [file normalize $argv0] + dict set W SCRIPT_FILENAME $a0 + dict set W DOCUMENT_ROOT [file dir $a0] + if {[wappInt-parse-header $chan]} { + catch {close $chan} + return + } + set len 0 + if {[dict exists $W CONTENT_LENGTH]} { + set len [dict get $W CONTENT_LENGTH] + } + if {$len>0} { + # Still need to read the query content + dict set W .toread $len + } else { + # There is no query content, so handle the request immediately + set wapp $W + wappInt-handle-request $chan 0 + } + } + } else { + # If .toread is set, that means we are reading the query content. + # Continue reading until .toread reaches zero. + set got [read $chan [dict get $W .toread]] + dict append W CONTENT $got + dict set W .toread [expr {[dict get $W .toread]-[string length $got]}] + if {[dict get $W .toread]<=0} { + # Handle the request as soon as all the query content is received + set wapp $W + wappInt-handle-request $chan 0 + } + } +} + +# Decode the HTTP request header. +# +# This routine is always running inside of a [catch], so if +# any problems arise, simply raise an error. +# +proc wappInt-parse-header {chan} { + upvar #0 wappInt-$chan W + set hdr [split [dict get $W .header] \n] + if {$hdr==""} {return 1} + set req [lindex $hdr 0] + dict set W REQUEST_METHOD [set method [lindex $req 0]] + if {[lsearch {GET HEAD POST} $method]<0} { + error "unsupported request method: \"[dict get $W REQUEST_METHOD]\"" + } + set uri [lindex $req 1] + set split_uri [split $uri ?] + set uri0 [lindex $split_uri 0] + if {![regexp {^/[-.a-z0-9_/]*$} $uri0]} { + error "invalid request uri: \"$uri0\"" + } + dict set W REQUEST_URI $uri0 + dict set W PATH_INFO $uri0 + set uri1 [lindex $split_uri 1] + dict set W QUERY_STRING $uri1 + set n [llength $hdr] + for {set i 1} {$i<$n} {incr i} { + set x [lindex $hdr $i] + if {![regexp {^(.+): +(.*)$} $x all name value]} { + error "invalid header line: \"$x\"" + } + set name [string toupper $name] + switch -- $name { + REFERER {set name HTTP_REFERER} + USER-AGENT {set name HTTP_USER_AGENT} + CONTENT-LENGTH {set name CONTENT_LENGTH} + CONTENT-TYPE {set name CONTENT_TYPE} + HOST {set name HTTP_HOST} + COOKIE {set name HTTP_COOKIE} + ACCEPT-ENCODING {set name HTTP_ACCEPT_ENCODING} + default {set name .hdr:$name} + } + dict set W $name $value + } + return 0 +} + +# Decode the QUERY_STRING parameters from a GET request or the +# application/x-www-form-urlencoded CONTENT from a POST request. +# +# This routine sets the ".qp" element of the ::wapp dict as a signal +# that query parameters have already been decoded. +# +proc wappInt-decode-query-params {} { + global wapp + dict set wapp .qp 1 + if {[dict exists $wapp QUERY_STRING]} { + foreach qterm [split [dict get $wapp QUERY_STRING] &] { + set qsplit [split $qterm =] + set nm [lindex $qsplit 0] + if {[regexp {^[a-z][a-z0-9]*$} $nm]} { + dict set wapp $nm [wappInt-decode-url [lindex $qsplit 1]] + } + } + } + if {[dict exists $wapp CONTENT_TYPE] && [dict exists $wapp CONTENT]} { + set ctype [dict get $wapp CONTENT_TYPE] + if {$ctype=="application/x-www-form-urlencoded"} { + foreach qterm [split [string trim [dict get $wapp CONTENT]] &] { + set qsplit [split $qterm =] + set nm [lindex $qsplit 0] + if {[regexp {^[a-z][-a-z0-9_]*$} $nm]} { + dict set wapp $nm [wappInt-decode-url [lindex $qsplit 1]] + } + } + } elseif {[string match multipart/form-data* $ctype]} { + regexp {^(.*?)\r\n(.*)$} [dict get $wapp CONTENT] all divider body + set ndiv [string length $divider] + while {[string length $body]} { + set idx [string first $divider $body] + set unit [string range $body 0 [expr {$idx-3}]] + set body [string range $body [expr {$idx+$ndiv+2}] end] + if {[regexp {^Content-Disposition: form-data; (.*?)\r\n\r\n(.*)$} \ + $unit unit hdr content]} { + if {[regexp {name="(.*)"; filename="(.*)"\r\nContent-Type: (.*?)$}\ + $hdr hr name filename mimetype]} { + dict set wapp $name.filename \ + [string map [list \\\" \" \\\\ \\] $filename] + dict set wapp $name.mimetype $mimetype + dict set wapp $name.content $content + } elseif {[regexp {name="(.*)"} $hdr hr name]} { + dict set wapp $name $content + } + } + } + } + } +} + +# Invoke application-supplied methods to generate a reply to +# a single HTTP request. +# +# This routine always runs within [catch], so handle exceptions by +# invoking [error]. +# +proc wappInt-handle-request {chan useCgi} { + global wapp + dict set wapp .reply {} + dict set wapp .mimetype {text/html; charset=utf-8} + dict set wapp .reply-code {200 Ok} + dict set wapp .csp {default-src 'self'} + + # Set up additional CGI environment values + # + if {![dict exists $wapp HTTP_HOST]} { + dict set wapp BASE_URL {} + } elseif {[dict exists $wapp HTTPS]} { + dict set wapp BASE_URL https://[dict get $wapp HTTP_HOST] + } else { + dict set wapp BASE_URL http://[dict get $wapp HTTP_HOST] + } + if {![dict exists $wapp REQUEST_URI]} { + dict set wapp REQUEST_URI / + } elseif {[regsub {\?.*} [dict get $wapp REQUEST_URI] {} newR]} { + # Some servers (ex: nginx) append the query parameters to REQUEST_URI. + # These need to be stripped off + dict set wapp REQUEST_URI $newR + } + if {[dict exists $wapp SCRIPT_NAME]} { + dict append wapp BASE_URL [dict get $wapp SCRIPT_NAME] + } else { + dict set wapp SCRIPT_NAME {} + } + if {![dict exists $wapp PATH_INFO]} { + # If PATH_INFO is missing (ex: nginx) then construct it + set URI [dict get $wapp REQUEST_URI] + set skip [string length [dict get $wapp SCRIPT_NAME]] + dict set wapp PATH_INFO [string range $URI $skip end] + } + if {[regexp {^/([^/]+)(.*)$} [dict get $wapp PATH_INFO] all head tail]} { + dict set wapp PATH_HEAD $head + dict set wapp PATH_TAIL [string trimleft $tail /] + } else { + dict set wapp PATH_INFO {} + dict set wapp PATH_HEAD {} + dict set wapp PATH_TAIL {} + } + dict set wapp SELF_URL [dict get $wapp BASE_URL]/[dict get $wapp PATH_HEAD] + + # Parse query parameters from the query string, the cookies, and + # POST data + # + if {[dict exists $wapp HTTP_COOKIE]} { + foreach qterm [split [dict get $wapp HTTP_COOKIE] {;}] { + set qsplit [split [string trim $qterm] =] + set nm [lindex $qsplit 0] + if {[regexp {^[a-z][-a-z0-9_]*$} $nm]} { + dict set wapp $nm [wappInt-decode-url [lindex $qsplit 1]] + } + } + } + set same_origin 0 + if {[dict exists $wapp HTTP_REFERER]} { + set referer [dict get $wapp HTTP_REFERER] + set base [dict get $wapp BASE_URL] + if {$referer==$base || [string match $base/* $referer]} { + set same_origin 1 + } + } + dict set wapp SAME_ORIGIN $same_origin + if {$same_origin} { + wappInt-decode-query-params + } + + # Invoke the application-defined handler procedure for this page + # request. If an error occurs while running that procedure, generate + # an HTTP reply that contains the error message. + # + wapp-before-dispatch-hook + wappInt-trace + set mname [dict get $wapp PATH_HEAD] + if {[catch { + if {$mname!="" && [llength [info proc wapp-page-$mname]]>0} { + wapp-page-$mname + } else { + wapp-default + } + } msg]} { + if {[wapp-param WAPP_MODE]=="local" || [wapp-param WAPP_MODE]=="server"} { + puts "ERROR: $::errorInfo" + } + wapp-reset + wapp-reply-code "500 Internal Server Error" + wapp-mimetype text/html + wapp-trim { +

Wapp Application Error

+
%html($::errorInfo)
+ } + dict unset wapp .new-cookies + } + + # Transmit the HTTP reply + # + if {$chan=="stdout"} { + puts $chan "Status: [dict get $wapp .reply-code]\r" + } else { + puts $chan "HTTP/1.1 [dict get $wapp .reply-code]\r" + puts $chan "Server: wapp\r" + puts $chan "Connection: close\r" + } + if {[dict exists $wapp .reply-extra]} { + foreach {name value} [dict get $wapp .reply-extra] { + puts $chan "$name: $value\r" + } + } + if {[dict exists $wapp .csp]} { + puts $chan "Content-Security-Policy: [dict get $wapp .csp]\r" + } + set mimetype [dict get $wapp .mimetype] + puts $chan "Content-Type: $mimetype\r" + if {[dict exists $wapp .new-cookies]} { + foreach {nm val} [dict get $wapp .new-cookies] { + if {[regexp {^[a-z][-a-z0-9_]*$} $nm]} { + if {$val==""} { + puts $chan "Set-Cookie: $nm=; HttpOnly; Path=/; Max-Age=1\r" + } else { + set val [wappInt-enc-url $val] + puts $chan "Set-Cookie: $nm=$val; HttpOnly; Path=/\r" + } + } + } + } + if {[string match text/* $mimetype]} { + set reply [encoding convertto utf-8 [dict get $wapp .reply]] + if {[regexp {\ygzip\y} [wapp-param HTTP_ACCEPT_ENCODING]]} { + catch { + set x [zlib gzip $reply] + set reply $x + puts $chan "Content-Encoding: gzip\r" + } + } + } else { + set reply [dict get $wapp .reply] + } + puts $chan "Content-Length: [string length $reply]\r" + puts $chan \r + puts -nonewline $chan $reply + flush $chan + wappInt-close-channel $chan +} + +# This routine runs just prior to request-handler dispatch. The +# default implementation is a no-op, but applications can override +# to do additional transformations or checks. +# +proc wapp-before-dispatch-hook {} {return} + +# Process a single CGI request +# +proc wappInt-handle-cgi-request {} { + global wapp env + foreach key { + CONTENT_LENGTH + CONTENT_TYPE + DOCUMENT_ROOT + HTTP_ACCEPT_ENCODING + HTTP_COOKIE + HTTP_HOST + HTTP_REFERER + HTTP_USER_AGENT + HTTPS + PATH_INFO + QUERY_STRING + REMOTE_ADDR + REQUEST_METHOD + REQUEST_URI + REMOTE_USER + SCRIPT_FILENAME + SCRIPT_NAME + SERVER_NAME + SERVER_PORT + SERVER_PROTOCOL + } { + if {[info exists env($key)]} { + dict set wapp $key $env($key) + } + } + set len 0 + if {[dict exists $wapp CONTENT_LENGTH]} { + set len [dict get $wapp CONTENT_LENGTH] + } + if {$len>0} { + fconfigure stdin -translation binary + dict set wapp CONTENT [read stdin $len] + } + dict set wapp WAPP_MODE cgi + fconfigure stdout -translation binary + wappInt-handle-request stdout 1 +} + +# Process new text received on an inbound SCGI request +# +proc wappInt-scgi-readable {chan} { + if {[catch [list wappInt-scgi-readable-unsafe $chan] msg]} { + puts stderr "$msg\n$::errorInfo" + wappInt-close-channel $chan + } +} +proc wappInt-scgi-readable-unsafe {chan} { + upvar #0 wappInt-$chan W wapp wapp + if {![dict exists $W .toread]} { + # If the .toread key is not set, that means we are still reading + # the header. + # + # An SGI header is short. This implementation assumes the entire + # header is available all at once. + # + dict set W .remove_addr [dict get $W REMOTE_ADDR] + set req [read $chan 15] + set n [string length $req] + scan $req %d:%s len hdr + incr len [string length "$len:,"] + append hdr [read $chan [expr {$len-15}]] + foreach {nm val} [split $hdr \000] { + if {$nm==","} break + dict set W $nm $val + } + set len 0 + if {[dict exists $W CONTENT_LENGTH]} { + set len [dict get $W CONTENT_LENGTH] + } + if {$len>0} { + # Still need to read the query content + dict set W .toread $len + } else { + # There is no query content, so handle the request immediately + dict set W SERVER_ADDR [dict get $W .remove_addr] + set wapp $W + wappInt-handle-request $chan 0 + } + } else { + # If .toread is set, that means we are reading the query content. + # Continue reading until .toread reaches zero. + set got [read $chan [dict get $W .toread]] + dict append W CONTENT $got + dict set W .toread [expr {[dict get $W .toread]-[string length $got]}] + if {[dict get $W .toread]<=0} { + # Handle the request as soon as all the query content is received + dict set W SERVER_ADDR [dict get $W .remove_addr] + set wapp $W + wappInt-handle-request $chan 0 + } + } +} + +# Start up the wapp framework. Parameters are a list passed as the +# single argument. +# +# -server $PORT Listen for HTTP requests on this TCP port $PORT +# +# -local $PORT Listen for HTTP requests on 127.0.0.1:$PORT +# +# -scgi $PORT Listen for SCGI requests on 127.0.0.1:$PORT +# +# -remote-scgi $PORT Listen for SCGI requests on TCP port $PORT +# +# -cgi Handle a single CGI request +# +# With no arguments, the behavior is called "auto". In "auto" mode, +# if the GATEWAY_INTERFACE environment variable indicates CGI, then run +# as CGI. Otherwise, start an HTTP server bound to the loopback address +# only, on an arbitrary TCP port, and automatically launch a web browser +# on that TCP port. +# +# Additional options: +# +# -fromip GLOB Reject any incoming request where the remote +# IP address does not match the GLOB pattern. This +# value defaults to '127.0.0.1' for -local and -scgi. +# +# -nowait Do not wait in the event loop. Return immediately +# after all event handlers are established. +# +# -trace "puts" each request URL as it is handled, for +# debugging +# +# -lint Run wapp-safety-check on the application instead +# of running the application itself +# +# -Dvar=value Set TCL global variable "var" to "value" +# +# +proc wapp-start {arglist} { + global env + set mode auto + set port 0 + set nowait 0 + set fromip {} + set n [llength $arglist] + for {set i 0} {$i<$n} {incr i} { + set term [lindex $arglist $i] + if {[string match --* $term]} {set term [string range $term 1 end]} + switch -glob -- $term { + -server { + incr i; + set mode "server" + set port [lindex $arglist $i] + } + -local { + incr i; + set mode "local" + set fromip 127.0.0.1 + set port [lindex $arglist $i] + } + -scgi { + incr i; + set mode "scgi" + set fromip 127.0.0.1 + set port [lindex $arglist $i] + } + -remote-scgi { + incr i; + set mode "remote-scgi" + set port [lindex $arglist $i] + } + -cgi { + set mode "cgi" + } + -fromip { + incr i + set fromip [lindex $arglist $i] + } + -nowait { + set nowait 1 + } + -trace { + proc wappInt-trace {} { + set q [wapp-param QUERY_STRING] + set uri [wapp-param BASE_URL][wapp-param PATH_INFO] + if {$q!=""} {append uri ?$q} + puts $uri + } + } + -lint { + set res [wapp-safety-check] + if {$res!=""} { + puts "Potential problems in this code:" + puts $res + exit 1 + } else { + exit + } + } + -D*=* { + if {[regexp {^.D([^=]+)=(.*)$} $term all var val]} { + set ::$var $val + } + } + default { + error "unknown option: $term" + } + } + } + if {$mode=="auto"} { + if {[info exists env(GATEWAY_INTERFACE)] + && [string match CGI/1.* $env(GATEWAY_INTERFACE)]} { + set mode cgi + } else { + set mode local + } + } + if {$mode=="cgi"} { + wappInt-handle-cgi-request + } else { + wappInt-start-listener $port $mode $fromip + if {!$nowait} { + vwait ::forever + } + } +} + +# Call this version 1.0 +package provide wapp 1.0 ADDED test/wapptest.tcl Index: test/wapptest.tcl ================================================================== --- /dev/null +++ test/wapptest.tcl @@ -0,0 +1,674 @@ +#!/bin/sh +# \ +exec wapptclsh "$0" ${1+"$@"} + +# package required wapp +source [file join [file dirname [info script]] wapp.tcl] + +# Read the data from the releasetest_data.tcl script. +# +source [file join [file dirname [info script]] releasetest_data.tcl] + +# Variables set by the "control" form: +# +# G(platform) - User selected platform. +# G(test) - Set to "Normal", "Veryquick", "Smoketest" or "Build-Only". +# G(keep) - Boolean. True to delete no files after each test. +# G(msvc) - Boolean. True to use MSVC as the compiler. +# G(tcl) - Use Tcl from this directory for builds. +# G(jobs) - How many sub-processes to run simultaneously. +# +set G(platform) $::tcl_platform(os)-$::tcl_platform(machine) +set G(test) Normal +set G(keep) 0 +set G(msvc) 0 +set G(tcl) [::tcl::pkgconfig get libdir,install] +set G(jobs) 3 +set G(debug) 0 + +proc wapptest_init {} { + global G + + set lSave [list platform test keep msvc tcl jobs debug] + foreach k $lSave { set A($k) $G($k) } + array unset G + foreach k $lSave { set G($k) $A($k) } + + # The root of the SQLite source tree. + set G(srcdir) [file dirname [file dirname [info script]]] + + # releasetest.tcl script + set G(releaseTest) [file join [file dirname [info script]] releasetest.tcl] + + set G(sqlite_version) "unknown" + + # Either "config", "running" or "stopped": + set G(state) "config" + + set G(hostname) "(unknown host)" + catch { set G(hostname) [exec hostname] } + set G(host) $G(hostname) + append G(host) " $::tcl_platform(os) $::tcl_platform(osVersion)" + append G(host) " $::tcl_platform(machine) $::tcl_platform(byteOrder)" +} + +# Check to see if there are uncommitted changes in the SQLite source +# directory. Return true if there are, or false otherwise. +# +proc check_uncommitted {} { + global G + set ret 0 + set pwd [pwd] + cd $G(srcdir) + if {[catch {exec fossil changes} res]==0 && [string trim $res]!=""} { + set ret 1 + } + cd $pwd + return $ret +} + +proc generate_fossil_info {} { + global G + set pwd [pwd] + cd $G(srcdir) + if {[catch {exec fossil info} r1]} return + if {[catch {exec fossil changes} r2]} return + cd $pwd + + foreach line [split $r1 "\n"] { + if {[regexp {^checkout: *(.*)$} $line -> co]} { + wapp-trim {
%html($co) } + } + } + + if {[string trim $r2]!=""} { + wapp-trim { +
+ WARNING: Uncommitted changes in checkout + + } + } +} + +# If the application is in "config" state, set the contents of the +# ::G(test_array) global to reflect the tests that will be run. If the +# app is in some other state ("running" or "stopped"), this command +# is a no-op. +# +proc set_test_array {} { + global G + if { $G(state)=="config" } { + set G(test_array) [list] + foreach {config target} $::Platforms($G(platform)) { + + # If using MSVC, do not run sanitize or valgrind tests. Or the + # checksymbols test. + if {$G(msvc) && ( + "Sanitize" == $config + || "checksymbols" in $target + || "valgrindtest" in $target + )} { + continue + } + + # If the test mode is not "Normal", override the target. + # + if {$target!="checksymbols" && $G(platform)!="Failure-Detection"} { + switch -- $G(test) { + Veryquick { set target quicktest } + Smoketest { set target smoketest } + Build-Only { + set target testfixture + if {$::tcl_platform(platform)=="windows"} { + set target testfixture.exe + } + } + } + } + + lappend G(test_array) [dict create config $config target $target] + + set exclude [list checksymbols valgrindtest fuzzoomtest] + if {$G(debug) && !($target in $exclude)} { + set debug_idx [lsearch -glob $::Configs($config) -DSQLITE_DEBUG*] + set xtarget $target + regsub -all {fulltest[a-z]*} $xtarget test xtarget + if {$debug_idx<0} { + lappend G(test_array) [ + dict create config $config-(Debug) target $xtarget + ] + } else { + lappend G(test_array) [ + dict create config $config-(NDebug) target $xtarget + ] + } + } + } + } +} + +proc count_tests_and_errors {name logfile} { + global G + + set fd [open $logfile rb] + set seen 0 + while {![eof $fd]} { + set line [gets $fd] + if {[regexp {(\d+) errors out of (\d+) tests} $line all nerr ntest]} { + incr G(test.$name.nError) $nerr + incr G(test.$name.nTest) $ntest + set seen 1 + if {$nerr>0} { + set G(test.$name.errmsg) $line + } + } + if {[regexp {runtime error: +(.*)} $line all msg]} { + # skip over "value is outside range" errors + if {[regexp {value .* is outside the range of representable} $line]} { + # noop + } else { + incr G(test.$name.nError) + if {$G(test.$name.errmsg)==""} { + set G(test.$name.errmsg) $msg + } + } + } + if {[regexp {fatal error +(.*)} $line all msg]} { + incr G(test.$name.nError) + if {$G(test.$name.errmsg)==""} { + set G(test.$name.errmsg) $msg + } + } + if {[regexp {ERROR SUMMARY: (\d+) errors.*} $line all cnt] && $cnt>0} { + incr G(test.$name.nError) + if {$G(test.$name.errmsg)==""} { + set G(test.$name.errmsg) $all + } + } + if {[regexp {^VERSION: 3\.\d+.\d+} $line]} { + set v [string range $line 9 end] + if {$G(sqlite_version) eq "unknown"} { + set G(sqlite_version) $v + } elseif {$G(sqlite_version) ne $v} { + set G(test.$name.errmsg) "version conflict: {$G(sqlite_version)} vs. {$v}" + } + } + } + close $fd + if {$G(test) == "Build-Only"} { + incr G(test.$name.nTest) + if {$G(test.$name.nError)>0} { + set errmsg "Build failed" + } + } elseif {!$seen} { + set G(test.$name.errmsg) "Test did not complete" + if {[file readable core]} { + append G(test.$name.errmsg) " - core file exists" + } + } +} + +proc slave_test_done {name rc} { + global G + set G(test.$name.done) [clock seconds] + set G(test.$name.nError) 0 + set G(test.$name.nTest) 0 + set G(test.$name.errmsg) "" + if {$rc} { + incr G(test.$name.nError) + } + if {[file exists $G(test.$name.log)]} { + count_tests_and_errors $name $G(test.$name.log) + } +} + +proc slave_fileevent {name} { + global G + set fd $G(test.$name.channel) + + if {[eof $fd]} { + fconfigure $fd -blocking 1 + set rc [catch { close $fd }] + unset G(test.$name.channel) + slave_test_done $name $rc + } else { + set line [gets $fd] + if {[string trim $line] != ""} { puts "Trace : $name - \"$line\"" } + } + + do_some_stuff +} + +proc do_some_stuff {} { + global G + + # Count the number of running jobs. A running job has an entry named + # "channel" in its dictionary. + set nRunning 0 + set bFinished 1 + foreach j $G(test_array) { + set name [dict get $j config] + if { [info exists G(test.$name.channel)]} { incr nRunning } + if {![info exists G(test.$name.done)]} { set bFinished 0 } + } + + if {$bFinished} { + set nError 0 + set nTest 0 + set nConfig 0 + foreach j $G(test_array) { + set name [dict get $j config] + incr nError $G(test.$name.nError) + incr nTest $G(test.$name.nTest) + incr nConfig + } + set G(result) "$nError errors from $nTest tests in $nConfig configurations." + catch { + append G(result) " SQLite version $G(sqlite_version)" + } + set G(state) "stopped" + } else { + set nLaunch [expr $G(jobs) - $nRunning] + foreach j $G(test_array) { + if {$nLaunch<=0} break + set name [dict get $j config] + if { ![info exists G(test.$name.channel)] + && ![info exists G(test.$name.done)] + } { + set target [dict get $j target] + set G(test.$name.start) [clock seconds] + set fd [open "|[info nameofexecutable] $G(releaseTest) --slave" r+] + set G(test.$name.channel) $fd + fconfigure $fd -blocking 0 + fileevent $fd readable [list slave_fileevent $name] + + puts $fd [list 0 $G(msvc) 0 $G(keep)] + + set wtcl "" + if {$G(tcl)!=""} { set wtcl "--with-tcl=$G(tcl)" } + + # If this configuration is named -(Debug) or -(NDebug), + # then add or remove the SQLITE_DEBUG option from the base + # configuration before running the test. + if {[regexp -- {(.*)-(\(.*\))} $name -> head tail]} { + set opts $::Configs($head) + if {$tail=="(Debug)"} { + append opts " -DSQLITE_DEBUG=1 -DSQLITE_EXTRA_IFNULLROW=1" + } else { + regsub { *-DSQLITE_MEMDEBUG[^ ]* *} $opts { } opts + regsub { *-DSQLITE_DEBUG[^ ]* *} $opts { } opts + } + } else { + set opts $::Configs($name) + } + + set L [make_test_suite $G(msvc) $wtcl $name $target $opts] + puts $fd $L + flush $fd + set G(test.$name.log) [file join [lindex $L 1] test.log] + incr nLaunch -1 + } + } + } +} + +proc generate_select_widget {label id lOpt opt} { + wapp-trim { + + } +} + +proc generate_main_page {{extra {}}} { + global G + set_test_array + + set hostname $G(hostname) + wapp-trim { + + + %html($hostname): wapptest.tcl + + + + } + + set host $G(host) + wapp-trim { +
%string($host) + } + generate_fossil_info + wapp-trim { +
+
+
+ } + + # Build the "platform" select widget. + set lOpt [array names ::Platforms] + generate_select_widget Platform control_platform $lOpt $G(platform) + + # Build the "test" select widget. + set lOpt [list Normal Veryquick Smoketest Build-Only] + generate_select_widget Test control_test $lOpt $G(test) + + # Build the "jobs" select widget. Options are 1 to 8. + generate_select_widget Jobs control_jobs {1 2 3 4 5 6 7 8} $G(jobs) + + switch $G(state) { + config { + set txt "Run Tests!" + set id control_run + } + running { + set txt "STOP Tests!" + set id control_stop + } + stopped { + set txt "Reset!" + set id control_reset + } + } + wapp-trim { +
+ + +
+ } + + wapp-trim { +

+ + + + + + + + + + + } + wapp-trim { +
+ } + wapp-trim { +
+
+ } + wapp-page-tests + + set script "script/$G(state).js" + wapp-trim { +
+ + + + } +} + +proc wapp-default {} { + generate_main_page +} + +proc wapp-page-tests {} { + global G + wapp-trim { } + foreach t $G(test_array) { + set config [dict get $t config] + set target [dict get $t target] + + set class "testwait" + set seconds "" + + if {[info exists G(test.$config.log)]} { + if {[info exists G(test.$config.channel)]} { + set class "testrunning" + set seconds [expr [clock seconds] - $G(test.$config.start)] + } elseif {[info exists G(test.$config.done)]} { + if {$G(test.$config.nError)>0} { + set class "testfail" + } else { + set class "testdone" + } + set seconds [expr $G(test.$config.done) - $G(test.$config.start)] + } + + set min [format %.2d [expr ($seconds / 60) % 60]] + set hr [format %.2d [expr $seconds / 3600]] + set sec [format %.2d [expr $seconds % 60]] + set seconds "$hr:$min:$sec" + } + + wapp-trim { + + +
%html($config) + %html($target) + %html($seconds) + + } + if {[info exists G(test.$config.log)]} { + set log $G(test.$config.log) + set uri "log/$log" + wapp-trim { + %html($log) + } + } + if {[info exists G(test.$config.errmsg)] && $G(test.$config.errmsg)!=""} { + set errmsg $G(test.$config.errmsg) + wapp-trim { +
%html($errmsg) + } + } + } + + wapp-trim {
} + + if {[info exists G(result)]} { + set res $G(result) + wapp-trim { +
%string($res)
+ } + } +} + +# URI: /control +# +# Whenever the form at the top of the application page is submitted, it +# is submitted here. +# +proc wapp-page-control {} { + global G + if {$::G(state)=="config"} { + set lControls [list platform test tcl jobs keep msvc debug] + set G(msvc) 0 + set G(keep) 0 + set G(debug) 0 + } else { + set lControls [list jobs] + } + foreach v $lControls { + if {[wapp-param-exists control_$v]} { + set G($v) [wapp-param control_$v] + } + } + + if {[wapp-param-exists control_run]} { + # This is a "run test" command. + set_test_array + set ::G(state) "running" + } + + if {[wapp-param-exists control_stop]} { + # A "STOP tests" command. + set G(state) "stopped" + set G(result) "Test halted by user" + foreach j $G(test_array) { + set name [dict get $j config] + if { [info exists G(test.$name.channel)] } { + close $G(test.$name.channel) + unset G(test.$name.channel) + slave_test_done $name 1 + } + } + } + + if {[wapp-param-exists control_reset]} { + # A "reset app" command. + set G(state) "config" + wapptest_init + } + + if {$::G(state) == "running"} { + do_some_stuff + } + wapp-redirect / +} + +# URI: /style.css +# +# Return the stylesheet for the application main page. +# +proc wapp-page-style.css {} { + wapp-subst { + + /* The boxes with black borders use this class */ + .border { + border: 3px groove #444444; + padding: 1em; + margin-top: 1em; + margin-bottom: 1em; + } + + /* Float to the right (used for the Run/Stop/Reset button) */ + .right { float: right; } + + /* Style for the large red warning at the top of the page */ + .warning { + color: red; + font-weight: bold; + } + + /* Styles used by cells in the test table */ + .padleft { padding-left: 5ex; } + .nowrap { white-space: nowrap; } + + /* Styles for individual tests, depending on the outcome */ + .testwait { } + .testrunning { color: blue } + .testdone { color: green } + .testfail { color: red } + } +} + +# URI: /script/${state}.js +# +# The last part of this URI is always "config.js", "running.js" or +# "stopped.js", depending on the state of the application. It returns +# the javascript part of the front-end for the requested state to the +# browser. +# +proc wapp-page-script {} { + regexp {[^/]*$} [wapp-param REQUEST_URI] script + + set tcl $::G(tcl) + set keep $::G(keep) + set msvc $::G(msvc) + set debug $::G(debug) + + wapp-subst { + var lElem = \["control_platform", "control_test", "control_msvc", + "control_jobs", "control_debug" + \]; + lElem.forEach(function(e) { + var elem = document.getElementById(e); + elem.addEventListener("change", function() { control.submit() } ); + }) + + elem = document.getElementById("control_tcl"); + elem.value = "%string($tcl)" + + elem = document.getElementById("control_keep"); + elem.checked = %string($keep); + + elem = document.getElementById("control_msvc"); + elem.checked = %string($msvc); + + elem = document.getElementById("control_debug"); + elem.checked = %string($debug); + } + + if {$script != "config.js"} { + wapp-subst { + var lElem = \["control_platform", "control_test", + "control_tcl", "control_keep", "control_msvc", + "control_debug" + \]; + lElem.forEach(function(e) { + var elem = document.getElementById(e); + elem.disabled = true; + }) + } + } + + if {$script == "running.js"} { + wapp-subst { + function reload_tests() { + fetch('tests') + .then( data => data.text() ) + .then( data => { + document.getElementById("tests").innerHTML = data; + }) + .then( data => { + if( document.getElementById("result") ){ + document.location = document.location; + } else { + setTimeout(reload_tests, 1000) + } + }); + } + + setTimeout(reload_tests, 1000) + } + } +} + +# URI: /env +# +# This is for debugging only. Serves no other purpose. +# +proc wapp-page-env {} { + wapp-allow-xorigin-params + wapp-trim { +

Wapp Environment

\n
+    
%html([wapp-debug-env])
+ } +} + +# URI: /log/dirname/test.log +# +# This URI reads file "dirname/test.log" from disk, wraps it in a
+# block, and returns it to the browser. Use for viewing log files.
+#
+proc wapp-page-log {} {
+  set log [string range [wapp-param REQUEST_URI] 5 end]
+  set fd [open $log]
+  set data [read $fd]
+  close $fd
+  wapp-trim {
+    
+    %html($data)
+    
+ } +} + +wapptest_init +wapp-start $argv + Index: test/where.test ================================================================== --- test/where.test +++ test/where.test @@ -1436,6 +1436,107 @@ CREATE TABLE t2(x INTEGER PRIMARY KEY, y INT); INSERT INTO t2(y) VALUES(2),(3); SELECT * FROM t1, t2 WHERE a=y AND y=3; } {3 2 3} +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test where-24.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + INSERT INTO t1 VALUES(3, 'three'); + INSERT INTO t1 VALUES(4, 'four'); +} + +foreach {tn sql res} { + 1 "SELECT b FROM t1" {one two three four} + 2 "SELECT b FROM t1 WHERE a<4" {one two three} + 3 "SELECT b FROM t1 WHERE a>1" {two three four} + 4 "SELECT b FROM t1 WHERE a>1 AND a<4" {two three} + + 5 "SELECT b FROM t1 WHERE a>? AND a<4" {} + 6 "SELECT b FROM t1 WHERE a>1 AND a? AND a=? AND a<=4" {} + 8 "SELECT b FROM t1 WHERE a>=1 AND a<=?" {} + 9 "SELECT b FROM t1 WHERE a>=? AND a<=?" {} +} { + set rev [list] + foreach r $res { set rev [concat $r $rev] } + + do_execsql_test where-24.$tn.1 "$sql" $res + do_execsql_test where-24.$tn.2 "$sql ORDER BY rowid" $res + do_execsql_test where-24.$tn.3 "$sql ORDER BY rowid DESC" $rev + + do_execsql_test where-24-$tn.4 " + BEGIN; + DELETE FROM t1; + $sql; + $sql ORDER BY rowid; + $sql ORDER BY rowid DESC; + ROLLBACK; + " +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test where-25.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE UNIQUE INDEX i1 ON t1(c); + INSERT INTO t1 VALUES(1, 'one', 'i'); + INSERT INTO t1 VALUES(2, 'two', 'ii'); + + CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c); + CREATE UNIQUE INDEX i2 ON t2(c); + INSERT INTO t2 VALUES(1, 'one', 'i'); + INSERT INTO t2 VALUES(2, 'two', 'ii'); + INSERT INTO t2 VALUES(3, 'three', 'iii'); + + PRAGMA writable_schema = 1; + UPDATE sqlite_master SET rootpage = ( + SELECT rootpage FROM sqlite_master WHERE name = 'i2' + ) WHERE name = 'i1'; +} +db close +sqlite3 db test.db +do_catchsql_test where-25.1 { + DELETE FROM t1 WHERE c='iii' +} {1 {database disk image is malformed}} +do_catchsql_test where-25.2 { + INSERT INTO t1 VALUES(4, 'four', 'iii') + ON CONFLICT(c) DO UPDATE SET b=NULL +} {1 {database disk image is malformed}} + +reset_db +do_execsql_test where-25.3 { + CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE UNIQUE INDEX i1 ON t1(c); + INSERT INTO t1 VALUES(1, 'one', 'i'); + INSERT INTO t1 VALUES(2, 'two', 'ii'); + + CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c); + CREATE UNIQUE INDEX i2 ON t2(c); + INSERT INTO t2 VALUES(1, 'one', 'i'); + INSERT INTO t2 VALUES(2, 'two', 'ii'); + INSERT INTO t2 VALUES(3, 'three', 'iii'); + + PRAGMA writable_schema = 1; + UPDATE sqlite_master SET rootpage = ( + SELECT rootpage FROM sqlite_master WHERE name = 'i2' + ) WHERE name = 'i1'; +} +db close +sqlite3 db test.db +do_catchsql_test where-25.4 { + SELECT * FROM t1 WHERE c='iii' +} {0 {}} +do_catchsql_test where-25.5 { + INSERT INTO t1 VALUES(4, 'four', 'iii') + ON CONFLICT(c) DO UPDATE SET b=NULL +} {1 {corrupt database}} + finish_test + DELETED tool/addopcodes.tcl Index: tool/addopcodes.tcl ================================================================== --- tool/addopcodes.tcl +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/tclsh -# -# This script appends additional token codes to the end of the -# parse.h file that lemon generates. These extra token codes are -# not used by the parser. But they are used by the tokenizer and/or -# the code generator. -# -# -set in [open [lindex $argv 0] rb] -set max 0 -while {![eof $in]} { - set line [gets $in] - if {[regexp {^#define TK_} $line]} { - puts $line - set x [lindex $line 2] - if {$x>$max} {set max $x} - } -} -close $in - -# The following are the extra token codes to be added. SPACE and -# ILLEGAL *must* be the last two token codes and they must be in that order. -# -set extras { - TRUEFALSE - ISNOT - FUNCTION - COLUMN - AGG_FUNCTION - AGG_COLUMN - UMINUS - UPLUS - TRUTH - REGISTER - VECTOR - SELECT_COLUMN - IF_NULL_ROW - ASTERISK - SPAN - END_OF_FILE - UNCLOSED_STRING - SPACE - ILLEGAL -} -if {[lrange $extras end-1 end]!="SPACE ILLEGAL"} { - error "SPACE and ILLEGAL must be the last two token codes and they\ - must be in that order" -} -foreach x $extras { - incr max - puts [format "#define TK_%-29s %4d" $x $max] -} - -# Some additional #defines related to token codes. -# -puts "\n/* The token codes above must all fit in 8 bits */" -puts [format "#define %-20s %-6s" TKFLG_MASK 0xff] -puts "\n/* Flags that can be added to a token code when it is not" -puts "** being stored in a u8: */" -foreach {fg val comment} { - TKFLG_DONTFOLD 0x100 {/* Omit constant folding optimizations */} -} { - puts [format "#define %-20s %-6s %s" $fg $val $comment] -} Index: tool/omittest.tcl ================================================================== --- tool/omittest.tcl +++ tool/omittest.tcl @@ -190,53 +190,62 @@ SQLITE_OMIT_AUTORESET \ SQLITE_OMIT_AUTOVACUUM \ SQLITE_OMIT_BETWEEN_OPTIMIZATION \ SQLITE_OMIT_BLOB_LITERAL \ SQLITE_OMIT_BTREECOUNT \ - SQLITE_OMIT_BUILTIN_TEST \ SQLITE_OMIT_CAST \ SQLITE_OMIT_CHECK \ SQLITE_OMIT_COMPILEOPTION_DIAGS \ SQLITE_OMIT_COMPLETE \ SQLITE_OMIT_COMPOUND_SELECT \ + SQLITE_OMIT_CONFLICT_CLAUSE \ SQLITE_OMIT_CTE \ SQLITE_OMIT_DATETIME_FUNCS \ SQLITE_OMIT_DECLTYPE \ SQLITE_OMIT_DEPRECATED \ + SQLITE_OMIT_DISKIO \ SQLITE_OMIT_EXPLAIN \ SQLITE_OMIT_FLAG_PRAGMAS \ SQLITE_OMIT_FLOATING_POINT \ SQLITE_OMIT_FOREIGN_KEY \ SQLITE_OMIT_GET_TABLE \ + SQLITE_OMIT_HEX_INTEGER \ SQLITE_OMIT_INCRBLOB \ SQLITE_OMIT_INTEGRITY_CHECK \ SQLITE_OMIT_LIKE_OPTIMIZATION \ SQLITE_OMIT_LOAD_EXTENSION \ SQLITE_OMIT_LOCALTIME \ SQLITE_OMIT_LOOKASIDE \ SQLITE_OMIT_MEMORYDB \ + SQLITE_OMIT_MEMORY_ALLOCATION \ SQLITE_OMIT_OR_OPTIMIZATION \ SQLITE_OMIT_PAGER_PRAGMAS \ + SQLITE_OMIT_PARSER_TRACE \ + SQLITE_OMIT_POPEN \ SQLITE_OMIT_PRAGMA \ SQLITE_OMIT_PROGRESS_CALLBACK \ SQLITE_OMIT_QUICKBALANCE \ + SQLITE_OMIT_RANDOMNESS \ SQLITE_OMIT_REINDEX \ SQLITE_OMIT_SCHEMA_PRAGMAS \ SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS \ SQLITE_OMIT_SHARED_CACHE \ + SQLITE_OMIT_SHUTDOWN_DIRECTORIES \ SQLITE_OMIT_SUBQUERY \ SQLITE_OMIT_TCL_VARIABLE \ SQLITE_OMIT_TEMPDB \ + SQLITE_OMIT_TEST_CONTROL \ SQLITE_OMIT_TRACE \ SQLITE_OMIT_TRIGGER \ SQLITE_OMIT_TRUNCATE_OPTIMIZATION \ - SQLITE_OMIT_UNIQUE_ENFORCEMENT \ + SQLITE_OMIT_UPSERT \ SQLITE_OMIT_UTF16 \ SQLITE_OMIT_VACUUM \ SQLITE_OMIT_VIEW \ SQLITE_OMIT_VIRTUALTABLE \ SQLITE_OMIT_WAL \ + SQLITE_OMIT_WINDOWFUNC \ SQLITE_OMIT_WSD \ SQLITE_OMIT_XFER_OPT \ ] set ::ENABLE_SYMBOLS [list \ Index: tool/symbols.sh ================================================================== --- tool/symbols.sh +++ tool/symbols.sh @@ -3,21 +3,23 @@ # Run this script in a directory that contains a valid SQLite makefile in # order to verify that unintentionally exported symbols. # make sqlite3.c -echo '****** Exported symbols from a build including RTREE, FTS4 & ICU ******' +echo '****** Exported symbols from a build including RTREE, FTS4 & FTS5 ******' gcc -c -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \ -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT3 \ -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ - -DSQLITE_ENABLE_ICU -DSQLITE_ENABLE_PREUPDATE_HOOK -DSQLITE_ENABLE_SESSION \ + -DSQLITE_ENABLE_PREUPDATE_HOOK -DSQLITE_ENABLE_SESSION \ + -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_GEOPOLY \ sqlite3.c nm sqlite3.o | grep ' [TD] ' | sort -k 3 -echo '****** Surplus symbols from a build including RTREE, FTS4 & ICU ******' -nm sqlite3.o | grep ' [TD] ' | grep -v ' .*sqlite3_' +echo '****** Surplus symbols from a build including RTREE, FTS4 & FTS5 ******' +nm sqlite3.o | grep ' [TD] ' | + egrep -v ' .*sqlite3(session|rebaser|changeset|changegroup)?_' echo '****** Dependencies of the core. No extensions. No OS interface *******' gcc -c -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT3 \ -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \