Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -2075,10 +2075,28 @@ $(TOP)\ext\fts5\fts5_tokenize.c \ $(TOP)\ext\fts5\fts5_unicode2.c \ $(TOP)\ext\fts5\fts5_varint.c \ $(TOP)\ext\fts5\fts5_vocab.c +LSM1_SRC = \ + $(TOP)\ext\lsm1\lsm.h \ + $(TOP)\ext\lsm1\lsmInt.h \ + $(TOP)\ext\lsm1\lsm_ckpt.c \ + $(TOP)\ext\lsm1\lsm_file.c \ + $(TOP)\ext\lsm1\lsm_log.c \ + $(TOP)\ext\lsm1\lsm_main.c \ + $(TOP)\ext\lsm1\lsm_mem.c \ + $(TOP)\ext\lsm1\lsm_mutex.c \ + $(TOP)\ext\lsm1\lsm_shared.c \ + $(TOP)\ext\lsm1\lsm_sorted.c \ + $(TOP)\ext\lsm1\lsm_str.c \ + $(TOP)\ext\lsm1\lsm_tree.c \ + $(TOP)\ext\lsm1\lsm_unix.c \ + $(TOP)\ext\lsm1\lsm_varint.c \ + $(TOP)\ext\lsm1\lsm_vtab.c \ + $(TOP)\ext\lsm1\lsm_win32.c + fts5parse.c: $(TOP)\ext\fts5\fts5parse.y lemon.exe copy $(TOP)\ext\fts5\fts5parse.y . del /Q fts5parse.h 2>NUL .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) fts5parse.y @@ -2086,10 +2104,14 @@ fts5.c: $(FTS5_SRC) $(TCLSH_CMD) $(TOP)\ext\fts5\tool\mkfts5c.tcl copy $(TOP)\ext\fts5\fts5.h . +lsm1.c: $(LSM1_SRC) + $(TCLSH_CMD) $(TOP)\ext\lsm1\tool\mklsm1c.tcl + copy $(TOP)\ext\lsm1\lsm.h . + fts5.lo: fts5.c $(HDR) $(EXTHDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c fts5.c fts5_ext.lo: fts5.c $(HDR) $(EXTHDR) $(LTCOMPILE) $(NO_WARN) -c fts5.c @@ -2318,6 +2340,7 @@ del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL del /Q sqlite-*-output.vsix 2>NUL del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL del /Q fts5.* fts5parse.* 2>NUL + del /Q lsm.h lsm1.c 2>NUL # <> Index: README.md ================================================================== --- README.md +++ README.md @@ -205,12 +205,12 @@ The amalgamation source file is more than 200K lines long. Some symbolic debuggers (most notably MSVC) are unable to deal with files longer than 64K lines. To work around this, a separate Tcl script, tool/split-sqlite3c.tcl, can be run on the amalgamation to break it up into a single small C file -called **sqlite3-all.c** that does #include on about five other files -named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-5.c**. In this way, +called **sqlite3-all.c** that does #include on about seven other files +named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-7.c**. In this way, all of the source code is contained within a single translation unit so that the compiler can do extra cross-procedure optimization, but no individual source file exceeds 32K lines in length. ## How It All Fits Together @@ -235,11 +235,12 @@ * **sqlite.h.in** - This file defines the public interface to the SQLite library. Readers will need to be familiar with this interface before trying to understand how the library works internally. * **sqliteInt.h** - this header file defines many of the data objects - used internally by SQLite. + used internally by SQLite. In addition to "sqliteInt.h", some + subsystems have their own header files. * **parse.y** - This file describes the LALR(1) grammar that SQLite uses to parse SQL statements, and the actions that are taken at each step in the parsing process. @@ -247,32 +248,47 @@ prepared statements. There are various helper files whose names begin with "vdbe". The VDBE has access to the vdbeInt.h header file which defines internal data objects. The rest of SQLite interacts with the VDBE through an interface defined by vdbe.h. - * **where.c** - This file analyzes the WHERE clause and generates + * **where.c** - This file (together with its helper files named + by "where*.c") analyzes the WHERE clause and generates virtual machine code to run queries efficiently. This file is sometimes called the "query optimizer". It has its own private header file, whereInt.h, that defines data objects used internally. * **btree.c** - This file contains the implementation of the B-Tree - storage engine used by SQLite. + storage engine used by SQLite. The interface to the rest of the system + is defined by "btree.h". The "btreeInt.h" header defines objects + used internally by btree.c and not published to the rest of the system. * **pager.c** - This file contains the "pager" implementation, the - module that implements transactions. + module that implements transactions. The "pager.h" header file + defines the interface between pager.c and the rest of the system. * **os_unix.c** and **os_win.c** - These two files implement the interface between SQLite and the underlying operating system using the run-time pluggable VFS interface. - * **shell.c** - This file is not part of the core SQLite library. This + * **shell.c.in** - This file is not part of the core SQLite library. This is the file that, when linked against sqlite3.a, generates the - "sqlite3.exe" command-line shell. + "sqlite3.exe" command-line shell. The "shell.c.in" file is transformed + into "shell.c" as part of the build process. * **tclsqlite.c** - This file implements the Tcl bindings for SQLite. It is not part of the core SQLite library. But as most of the tests in this repository are written in Tcl, the Tcl language bindings are important. + + * **test*.c** - Files in the src/ folder that begin with "test" go into + building the "testfixture.exe" program. The testfixture.exe program is + an enhanced TCL shell. The testfixture.exe program runs scripts in the + test/ folder to validate the core SQLite code. The testfixture program + (and some other test programs too) is build and run when you type + "make test". + + * **ext/misc/json1.c** - This file implements the various JSON functions + that are build into SQLite. There are many other source files. Each has a succinct header comment that describes its purpose and role within the larger system. Index: ext/fts5/fts5_index.c ================================================================== --- ext/fts5/fts5_index.c +++ ext/fts5/fts5_index.c @@ -4907,11 +4907,17 @@ Fts5DoclistIter i1; Fts5DoclistIter i2; Fts5Buffer out = {0, 0, 0}; Fts5Buffer tmp = {0, 0, 0}; - if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return; + /* The maximum size of the output is equal to the sum of the two + ** input sizes + 1 varint (9 bytes). The extra varint is because if the + ** first rowid in one input is a large negative number, and the first in + ** the other a non-negative number, the delta for the non-negative + ** number will be larger on disk than the literal integer value + ** was. */ + if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n + 9) ) return; fts5DoclistIterInit(p1, &i1); fts5DoclistIterInit(p2, &i2); while( 1 ){ if( i1.iRowidn+p2->n+9) ); fts5BufferSet(&p->rc, p1, out.n, out.p); fts5BufferFree(&tmp); fts5BufferFree(&out); } Index: ext/fts5/test/fts5query.test ================================================================== --- ext/fts5/test/fts5query.test +++ ext/fts5/test/fts5query.test @@ -62,11 +62,11 @@ execsql { INSERT INTO t1 VALUES($doc) } } execsql COMMIT } {} - do_execsql_test 1.$tn.2 { + do_execsql_test 2.$tn.2 { INSERT INTO t1(t1) VALUES('integrity-check'); } set ret 1 foreach x [list a c e g i k m o q s u] { @@ -75,7 +75,17 @@ } {} incr ret } } +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a); + INSERT INTO x1(rowid, a) VALUES(-1000000000000, 'toyota'); + INSERT INTO x1(rowid, a) VALUES(1, 'tarago'); +} +do_execsql_test 3.1 { + SELECT rowid FROM x1('t*'); +} {-1000000000000 1} + finish_test Index: ext/icu/icu.c ================================================================== --- ext/icu/icu.c +++ ext/icu/icu.c @@ -26,11 +26,13 @@ ** ** * An implementation of the LIKE operator that uses ICU to ** provide case-independent matching. */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) +#if !defined(SQLITE_CORE) \ + || defined(SQLITE_ENABLE_ICU) \ + || defined(SQLITE_ENABLE_ICU_COLLATIONS) /* Include ICU headers */ #include #include #include @@ -43,10 +45,30 @@ SQLITE_EXTENSION_INIT1 #else #include "sqlite3.h" #endif +/* +** This function is called when an ICU function called from within +** the implementation of an SQL scalar function returns an error. +** +** The scalar function context passed as the first argument is +** loaded with an error message based on the following two args. +*/ +static void icuFunctionError( + sqlite3_context *pCtx, /* SQLite scalar function context */ + const char *zName, /* Name of ICU function that failed */ + UErrorCode e /* Error code returned by ICU function */ +){ + char zBuf[128]; + sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); + zBuf[127] = '\0'; + sqlite3_result_error(pCtx, zBuf, -1); +} + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) + /* ** Maximum length (in bytes) of the pattern in a LIKE or GLOB ** operator. */ #ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH @@ -222,28 +244,10 @@ if( zA && zB ){ sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc)); } } -/* -** This function is called when an ICU function called from within -** the implementation of an SQL scalar function returns an error. -** -** The scalar function context passed as the first argument is -** loaded with an error message based on the following two args. -*/ -static void icuFunctionError( - sqlite3_context *pCtx, /* SQLite scalar function context */ - const char *zName, /* Name of ICU function that failed */ - UErrorCode e /* Error code returned by ICU function */ -){ - char zBuf[128]; - sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); - zBuf[127] = '\0'; - sqlite3_result_error(pCtx, zBuf, -1); -} - /* ** Function to delete compiled regexp objects. Registered as ** a destructor function with sqlite3_set_auxdata(). */ static void icuRegexpDelete(void *p){ @@ -405,10 +409,12 @@ return; } assert( 0 ); /* Unreachable */ } +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ + /* ** Collation sequence destructor function. The pCtx argument points to ** a UCollator structure previously allocated using ucol_open(). */ static void icuCollationDel(void *pCtx){ @@ -499,10 +505,11 @@ unsigned short enc; /* Optimal text encoding */ unsigned char iContext; /* sqlite3_user_data() context */ void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } scalars[] = { {"icu_load_collation", 2, SQLITE_UTF8, 1, icuLoadCollation}, +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC, 0, icuRegexpFunc}, {"lower", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, {"lower", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, {"upper", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, {"upper", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, @@ -510,14 +517,14 @@ {"lower", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, {"upper", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, {"upper", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, {"like", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, {"like", 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ }; int rc = SQLITE_OK; int i; - for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){ const struct IcuScalar *p = &scalars[i]; rc = sqlite3_create_function( db, p->zName, p->nArg, p->enc, Index: ext/lsm1/lsmInt.h ================================================================== --- ext/lsm1/lsmInt.h +++ ext/lsm1/lsmInt.h @@ -108,11 +108,11 @@ typedef lsm_i64 i64; typedef unsigned long long int u64; #endif /* A page number is a 64-bit integer. */ -typedef i64 Pgno; +typedef i64 LsmPgno; #ifdef LSM_DEBUG int lsmErrorBkpt(int); #else # define lsmErrorBkpt(x) (x) @@ -400,13 +400,13 @@ TreeHeader treehdr; /* Local copy of tree-header */ u32 aSnapshot[LSM_META_PAGE_SIZE / sizeof(u32)]; }; struct Segment { - Pgno iFirst; /* First page of this run */ - Pgno iLastPg; /* Last page of this run */ - Pgno iRoot; /* Root page number (if any) */ + LsmPgno iFirst; /* First page of this run */ + LsmPgno iLastPg; /* Last page of this run */ + LsmPgno iRoot; /* Root page number (if any) */ int nSize; /* Size of this run in pages */ Redirect *pRedirect; /* Block redirects (or NULL) */ }; @@ -454,20 +454,20 @@ ** iOutputOff: ** The byte offset to write to next within the last page of the ** output segment. */ struct MergeInput { - Pgno iPg; /* Page on which next input is stored */ + LsmPgno iPg; /* Page on which next input is stored */ int iCell; /* Cell containing next input to merge */ }; struct Merge { int nInput; /* Number of input runs being merged */ MergeInput *aInput; /* Array nInput entries in size */ MergeInput splitkey; /* Location in file of current splitkey */ int nSkip; /* Number of separators entries to skip */ int iOutputOff; /* Write offset on output page */ - Pgno iCurrentPtr; /* Current pointer value */ + LsmPgno iCurrentPtr; /* Current pointer value */ }; /* ** The first argument to this macro is a pointer to a Segment structure. ** Returns true if the structure instance indicates that the separators @@ -577,14 +577,14 @@ i64 iId; /* Snapshot id */ i64 iLogOff; /* Log file offset */ Redirect redirect; /* Block redirection array */ /* Used by worker snapshots only */ - int nBlock; /* Number of blocks in database file */ - Pgno aiAppend[LSM_APPLIST_SZ]; /* Append point list */ - Freelist freelist; /* Free block list */ - u32 nWrite; /* Total number of pages written to disk */ + int nBlock; /* Number of blocks in database file */ + LsmPgno aiAppend[LSM_APPLIST_SZ]; /* Append point list */ + Freelist freelist; /* Free block list */ + u32 nWrite; /* Total number of pages written to disk */ }; #define LSM_INITIAL_SNAPSHOT_ID 11 /* ** Functions from file "lsm_ckpt.c". @@ -708,11 +708,11 @@ void lsmFsSetPageSize(FileSystem *, int); int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId); /* Creating, populating, gobbling and deleting sorted runs. */ -void lsmFsGobble(lsm_db *, Segment *, Pgno *, int); +void lsmFsGobble(lsm_db *, Segment *, LsmPgno *, int); int lsmFsSortedDelete(FileSystem *, Snapshot *, int, Segment *); int lsmFsSortedFinish(FileSystem *, Segment *); int lsmFsSortedAppend(FileSystem *, Snapshot *, Level *, int, Page **); int lsmFsSortedPadding(FileSystem *, Snapshot *, Segment *); @@ -725,18 +725,18 @@ void lsmSortedSplitkey(lsm_db *, Level *, int *); /* Reading sorted run content. */ int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg); -int lsmFsDbPageGet(FileSystem *, Segment *, Pgno, Page **); +int lsmFsDbPageGet(FileSystem *, Segment *, LsmPgno, Page **); int lsmFsDbPageNext(Segment *, Page *, int eDir, Page **); u8 *lsmFsPageData(Page *, int *); int lsmFsPageRelease(Page *); int lsmFsPagePersist(Page *); void lsmFsPageRef(Page *); -Pgno lsmFsPageNumber(Page *); +LsmPgno lsmFsPageNumber(Page *); int lsmFsNRead(FileSystem *); int lsmFsNWrite(FileSystem *); int lsmFsMetaPageGet(FileSystem *, int, int, MetaPage **); @@ -746,11 +746,11 @@ #ifdef LSM_DEBUG int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg); int lsmFsIntegrityCheck(lsm_db *); #endif -Pgno lsmFsRedirectPage(FileSystem *, Redirect *, Pgno); +LsmPgno lsmFsRedirectPage(FileSystem *, Redirect *, LsmPgno); int lsmFsPageWritable(Page *); /* Functions to read, write and sync the log file. */ int lsmFsWriteLog(FileSystem *pFS, i64 iOff, LsmString *pStr); @@ -766,12 +766,12 @@ int lsmFsSyncDb(FileSystem *, int); void lsmFsFlushWaiting(FileSystem *, int *); /* Used by lsm_info(ARRAY_STRUCTURE) and lsm_config(MMAP) */ -int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, Pgno iFirst, char **pzOut); -int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut); +int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, LsmPgno iFirst, char **pz); +int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut); int lsmConfigMmap(lsm_db *pDb, int *piParam); int lsmEnvOpen(lsm_env *, const char *, int, lsm_file **); int lsmEnvClose(lsm_env *pEnv, lsm_file *pFile); int lsmEnvLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int eLock); @@ -783,11 +783,11 @@ void lsmEnvSleep(lsm_env *, int); int lsmFsReadSyncedId(lsm_db *db, int, i64 *piVal); -int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, Pgno, int *); +int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, LsmPgno, int *); void lsmFsPurgeCache(FileSystem *); /* ** End of functions from "lsm_file.c". @@ -794,11 +794,11 @@ **************************************************************************/ /* ** Functions from file "lsm_sorted.c". */ -int lsmInfoPageDump(lsm_db *, Pgno, int, char **); +int lsmInfoPageDump(lsm_db *, LsmPgno, int, char **); void lsmSortedCleanup(lsm_db *); int lsmSortedAutoWork(lsm_db *, int nUnit); int lsmSortedWalkFreelist(lsm_db *, int, int (*)(void *, int, i64), void *); Index: ext/lsm1/lsm_ckpt.c ================================================================== --- ext/lsm1/lsm_ckpt.c +++ ext/lsm1/lsm_ckpt.c @@ -387,11 +387,11 @@ CkptBuffer *p, /* Checkpoint buffer to write to */ int *piOut, /* IN/OUT: Offset within checkpoint buffer */ int *pRc /* IN/OUT: Error code */ ){ int i; - Pgno *aiAppend = db->pWorker->aiAppend; + LsmPgno *aiAppend = db->pWorker->aiAppend; for(i=0; inPagesize <= pFS->nMapLimit); } /* ** Given that there are currently nHash slots in the hash table, return ** the hash key for file iFile, page iPg. */ -static int fsHashKey(int nHash, Pgno iPg){ +static int fsHashKey(int nHash, LsmPgno iPg){ return (iPg % nHash); } /* ** This is a helper function for lsmFsOpen(). It opens a single file on @@ -878,17 +878,17 @@ ** ** For a compressed database, page numbers are byte offsets. The first ** page on each block is the byte offset immediately following the 4-byte ** "previous block" pointer at the start of each block. */ -static Pgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){ - Pgno iPg; +static LsmPgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){ + LsmPgno iPg; if( pFS->pCompress ){ if( iBlock==1 ){ iPg = pFS->nMetasize * 2 + 4; }else{ - iPg = pFS->nBlocksize * (Pgno)(iBlock-1) + 4; + iPg = pFS->nBlocksize * (LsmPgno)(iBlock-1) + 4; } }else{ const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); if( iBlock==1 ){ iPg = 1 + ((pFS->nMetasize*2 + pFS->nPagesize - 1) / pFS->nPagesize); @@ -905,13 +905,13 @@ ** ** For a compressed database, page numbers are byte offsets. The first ** page on each block is the byte offset of the byte immediately before ** the 4-byte "next block" pointer at the end of each block. */ -static Pgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){ +static LsmPgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){ if( pFS->pCompress ){ - return pFS->nBlocksize * (Pgno)iBlock - 1 - 4; + return pFS->nBlocksize * (LsmPgno)iBlock - 1 - 4; }else{ const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); return iBlock * nPagePerBlock; } } @@ -918,11 +918,11 @@ /* ** Return the block number of the block that page iPg is located on. ** Blocks are numbered starting from 1. */ -static int fsPageToBlock(FileSystem *pFS, Pgno iPg){ +static int fsPageToBlock(FileSystem *pFS, LsmPgno iPg){ if( pFS->pCompress ){ return (int)((iPg / pFS->nBlocksize) + 1); }else{ return (int)(1 + ((iPg-1) / (pFS->nBlocksize / pFS->nPagesize))); } @@ -931,11 +931,11 @@ /* ** Return true if page iPg is the last page on its block. ** ** This function is only called in non-compressed database mode. */ -static int fsIsLast(FileSystem *pFS, Pgno iPg){ +static int fsIsLast(FileSystem *pFS, LsmPgno iPg){ const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); assert( !pFS->pCompress ); return ( iPg && (iPg % nPagePerBlock)==0 ); } @@ -942,11 +942,11 @@ /* ** Return true if page iPg is the first page on its block. ** ** This function is only called in non-compressed database mode. */ -static int fsIsFirst(FileSystem *pFS, Pgno iPg){ +static int fsIsFirst(FileSystem *pFS, LsmPgno iPg){ const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); assert( !pFS->pCompress ); return ( (iPg % nPagePerBlock)==1 || (iPgflags & PAGE_DIRTY)==0 ); */ return pPage ? pPage->iPg : 0; } /* @@ -1056,11 +1056,11 @@ ** to it. Otherwise, return NULL. ** ** Either way, if argument piHash is not NULL set *piHash to the hash slot ** number that page iPg would be stored in before returning. */ -static Page *fsPageFindInHash(FileSystem *pFS, Pgno iPg, int *piHash){ +static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash){ Page *p; /* Return value */ int iHash = fsHashKey(pFS->nHash, iPg); if( piHash ) *piHash = iHash; for(p=pFS->apHash[iHash]; p; p=p->pHashNext){ @@ -1187,12 +1187,12 @@ /* ** If page iPg has been redirected according to the redirections in the ** object passed as the second argument, return the destination page to ** which it is redirected. Otherwise, return a copy of iPg. */ -Pgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, Pgno iPg){ - Pgno iReal = iPg; +LsmPgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, LsmPgno iPg){ + LsmPgno iReal = iPg; if( pRedir ){ const int nPagePerBlock = ( pFS->pCompress ? pFS->nBlocksize : (pFS->nBlocksize / pFS->nPagesize) ); @@ -1201,11 +1201,11 @@ for(i=0; in; i++){ int iFrom = pRedir->a[i].iFrom; if( iFrom>iBlk ) break; if( iFrom==iBlk ){ int iTo = pRedir->a[i].iTo; - iReal = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock; + iReal = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock; if( iTo==1 ){ iReal += (fsFirstPageOnBlock(pFS, 1)-1); } break; } @@ -1215,11 +1215,11 @@ assert( iReal!=0 ); return iReal; } /* Required by the circular fsBlockNext<->fsPageGet dependency. */ -static int fsPageGet(FileSystem *, Segment *, Pgno, int, Page **, int *); +static int fsPageGet(FileSystem *, Segment *, LsmPgno, int, Page **, int *); /* ** Parameter iBlock is a database file block. This function reads the value ** stored in the blocks "next block" pointer and stores it in *piNext. ** LSM_OK is returned if everything is successful, or an LSM error code @@ -1267,11 +1267,11 @@ } /* ** Return the page number of the last page on the same block as page iPg. */ -Pgno fsLastPageOnPagesBlock(FileSystem *pFS, Pgno iPg){ +LsmPgno fsLastPageOnPagesBlock(FileSystem *pFS, LsmPgno iPg){ return fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iPg)); } /* ** Read nData bytes of data from offset iOff of the database file into @@ -1535,11 +1535,11 @@ ** If no error occurs, LSM_OK is returned. Otherwise, an lsm error code. */ static int fsPageGet( FileSystem *pFS, /* File-system handle */ Segment *pSeg, /* Block redirection to use (or NULL) */ - Pgno iPg, /* Page id */ + LsmPgno iPg, /* Page id */ int noContent, /* True to not load content from disk */ Page **ppPg, /* OUT: New page handle */ int *pnSpace /* OUT: Bytes of free space */ ){ Page *p; @@ -1547,11 +1547,11 @@ int rc = LSM_OK; /* In most cases iReal is the same as iPg. Except, if pSeg->pRedirect is ** not NULL, and the block containing iPg has been redirected, then iReal ** is the page number after redirection. */ - Pgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg); + LsmPgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg); assert_lists_are_ok(pFS); assert( iPg>=fsFirstPageOnBlock(pFS, 1) ); assert( iReal>=fsFirstPageOnBlock(pFS, 1) ); *ppPg = 0; @@ -1687,12 +1687,12 @@ ** and iLast, inclusive, and pRun is not equal to pIgnore. */ static int fsRunEndsBetween( Segment *pRun, Segment *pIgnore, - Pgno iFirst, - Pgno iLast + LsmPgno iFirst, + LsmPgno iLast ){ return (pRun!=pIgnore && ( (pRun->iFirst>=iFirst && pRun->iFirst<=iLast) || (pRun->iLastPg>=iFirst && pRun->iLastPg<=iLast) )); @@ -1703,12 +1703,12 @@ ** which the first or last page is between iFirst and iLast, inclusive. */ static int fsLevelEndsBetween( Level *pLevel, Segment *pIgnore, - Pgno iFirst, - Pgno iLast + LsmPgno iFirst, + LsmPgno iLast ){ int i; if( fsRunEndsBetween(&pLevel->lhs, pIgnore, iFirst, iLast) ){ return 1; @@ -1731,17 +1731,17 @@ Snapshot *pSnapshot, /* Worker snapshot */ Segment *pIgnore, /* Ignore this run when searching */ int iBlk /* Block number of block to free */ ){ int rc = LSM_OK; /* Return code */ - Pgno iFirst; /* First page on block iBlk */ - Pgno iLast; /* Last page on block iBlk */ + LsmPgno iFirst; /* First page on block iBlk */ + LsmPgno iLast; /* Last page on block iBlk */ Level *pLevel; /* Used to iterate through levels */ int iIn; /* Used to iterate through append points */ int iOut = 0; /* Used to output append points */ - Pgno *aApp = pSnapshot->aiAppend; + LsmPgno *aApp = pSnapshot->aiAppend; iFirst = fsFirstPageOnBlock(pFS, iBlk); iLast = fsLastPageOnBlock(pFS, iBlk); /* Check if any other run in the snapshot has a start or end page @@ -1809,15 +1809,20 @@ /* ** aPgno is an array containing nPgno page numbers. Return the smallest page ** number from the array that falls on block iBlk. Or, if none of the pages ** in aPgno[] fall on block iBlk, return 0. */ -static Pgno firstOnBlock(FileSystem *pFS, int iBlk, Pgno *aPgno, int nPgno){ - Pgno iRet = 0; +static LsmPgno firstOnBlock( + FileSystem *pFS, + int iBlk, + LsmPgno *aPgno, + int nPgno +){ + LsmPgno iRet = 0; int i; for(i=0; ipRedirect, iPg)); } /* ** Return true if the second argument is not NULL and any of the first @@ -1852,11 +1857,11 @@ ** the new first page of the run). */ void lsmFsGobble( lsm_db *pDb, Segment *pRun, - Pgno *aPgno, + LsmPgno *aPgno, int nPgno ){ int rc = LSM_OK; FileSystem *pFS = pDb->pFS; Snapshot *pSnapshot = pDb->pWorker; @@ -1869,11 +1874,11 @@ iBlk = fsPageToBlock(pFS, pRun->iFirst); pRun->nSize += (int)(pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk)); while( rc==LSM_OK ){ int iNext = 0; - Pgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno); + LsmPgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno); if( iFirst ){ pRun->iFirst = iFirst; break; } rc = fsBlockNext(pFS, pRun, iBlk, &iNext); @@ -1903,15 +1908,15 @@ ** But take block overflow and redirection into account. */ static int fsNextPageOffset( FileSystem *pFS, /* File system object */ Segment *pSeg, /* Segment to move within */ - Pgno iPg, /* Offset of current page */ + LsmPgno iPg, /* Offset of current page */ int nByte, /* Size of current page including headers */ - Pgno *piNext /* OUT: Offset of next page. Or zero (EOF) */ + LsmPgno *piNext /* OUT: Offset of next page. Or zero (EOF) */ ){ - Pgno iNext; + LsmPgno iNext; int rc; assert( pFS->pCompress ); rc = fsAddOffset(pFS, pSeg, iPg, nByte-1, &iNext); @@ -1937,12 +1942,12 @@ ** *piPrev is undefined. */ static int fsGetPageBefore( FileSystem *pFS, Segment *pSeg, - Pgno iPg, - Pgno *piPrev + LsmPgno iPg, + LsmPgno *piPrev ){ u8 aSz[3]; int rc; i64 iRead; @@ -1988,11 +1993,11 @@ ** caller using lsmFsPageRelease(). */ int lsmFsDbPageNext(Segment *pRun, Page *pPg, int eDir, Page **ppNext){ int rc = LSM_OK; FileSystem *pFS = pPg->pFS; - Pgno iPg = pPg->iPg; + LsmPgno iPg = pPg->iPg; assert( 0==fsSegmentRedirects(pFS, pRun) ); if( pFS->pCompress ){ int nSpace = pPg->nCompress + 2*3; @@ -2060,14 +2065,14 @@ ** ** If argument pLvl is not NULL, then this function will not attempt to ** start the new segment immediately following any segment that is part ** of the right-hand-side of pLvl. */ -static Pgno findAppendPoint(FileSystem *pFS, Level *pLvl){ +static LsmPgno findAppendPoint(FileSystem *pFS, Level *pLvl){ int i; - Pgno *aiAppend = pFS->pDb->pWorker->aiAppend; - Pgno iRet = 0; + LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend; + LsmPgno iRet = 0; for(i=LSM_APPLIST_SZ-1; iRet==0 && i>=0; i--){ if( (iRet = aiAppend[i]) ){ if( pLvl ){ int iBlk = fsPageToBlock(pFS, iRet); @@ -2096,14 +2101,14 @@ int bDefer, Page **ppOut ){ int rc = LSM_OK; Page *pPg = 0; - Pgno iApp = 0; - Pgno iNext = 0; + LsmPgno iApp = 0; + LsmPgno iNext = 0; Segment *p = &pLvl->lhs; - Pgno iPrev = p->iLastPg; + LsmPgno iPrev = p->iLastPg; *ppOut = 0; assert( p->pRedirect==0 ); if( pFS->pCompress || bDefer ){ @@ -2193,11 +2198,11 @@ ** Otherwise, add the first free page in the last block used by the run ** to the lAppend list. */ if( fsLastPageOnPagesBlock(pFS, p->iLastPg)!=p->iLastPg ){ int i; - Pgno *aiAppend = pFS->pDb->pWorker->aiAppend; + LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend; for(i=0; iiLastPg+1; break; } @@ -2224,11 +2229,11 @@ /* ** Obtain a reference to page number iPg. ** ** Return LSM_OK if successful, or an lsm error code if an error occurs. */ -int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, Pgno iPg, Page **ppPg){ +int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, LsmPgno iPg, Page **ppPg){ return fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0); } /* ** Obtain a reference to the last page in the segment passed as the @@ -2236,11 +2241,11 @@ ** ** Return LSM_OK if successful, or an lsm error code if an error occurs. */ int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg){ int rc; - Pgno iPg = pSeg->iLastPg; + LsmPgno iPg = pSeg->iLastPg; if( pFS->pCompress ){ int nSpace; iPg++; do { nSpace = 0; @@ -2364,18 +2369,18 @@ */ static void fsMovePage( FileSystem *pFS, /* File system object */ int iTo, /* Destination block */ int iFrom, /* Source block */ - Pgno *piPg /* IN/OUT: Page number */ + LsmPgno *piPg /* IN/OUT: Page number */ ){ - Pgno iPg = *piPg; + LsmPgno iPg = *piPg; if( iFrom==fsPageToBlock(pFS, iPg) ){ const int nPagePerBlock = ( pFS->pCompress ? pFS ->nBlocksize : (pFS->nBlocksize / pFS->nPagesize) ); - *piPg = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock; + *piPg = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock; } } /* ** Copy the contents of block iFrom to block iTo. @@ -2455,25 +2460,25 @@ ** data is written to (this may be used as the page number if the data ** being appended is a new page record). ** ** This function is only used in compressed database mode. */ -static Pgno fsAppendData( +static LsmPgno fsAppendData( FileSystem *pFS, /* File-system handle */ Segment *pSeg, /* Segment to append to */ const u8 *aData, /* Buffer containing data to write */ int nData, /* Size of buffer aData[] in bytes */ int *pRc /* IN/OUT: Error code */ ){ - Pgno iRet = 0; + LsmPgno iRet = 0; int rc = *pRc; assert( pFS->pCompress ); if( rc==LSM_OK ){ int nRem = 0; int nWrite = 0; - Pgno iLastOnBlock; - Pgno iApp = pSeg->iLastPg+1; + LsmPgno iLastOnBlock; + LsmPgno iApp = pSeg->iLastPg+1; /* If this is the first data written into the segment, find an append-point ** or allocate a new block. */ if( iApp==1 ){ pSeg->iFirst = iApp = findAppendPoint(pFS, 0); @@ -2517,11 +2522,11 @@ rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, aPtr, sizeof(aPtr)); } /* Set the "prev" pointer on the new block */ if( rc==LSM_OK ){ - Pgno iWrite; + LsmPgno iWrite; lsmPutU32(aPtr, fsPageToBlock(pFS, iApp)); iWrite = fsFirstPageOnBlock(pFS, iBlk); rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iWrite-4, aPtr, sizeof(aPtr)); if( nRem>0 ) iApp = iWrite; } @@ -2586,15 +2591,15 @@ ** output variables is undefined. */ static int fsAppendPage( FileSystem *pFS, Segment *pSeg, - Pgno *piNew, + LsmPgno *piNew, int *piPrev, int *piNext ){ - Pgno iPrev = pSeg->iLastPg; + LsmPgno iPrev = pSeg->iLastPg; int rc; assert( iPrev!=0 ); *piPrev = 0; *piNext = 0; @@ -2648,11 +2653,11 @@ } /* ** If there exists a hash-table entry associated with page iPg, remove it. */ -static void fsRemoveHashEntry(FileSystem *pFS, Pgno iPg){ +static void fsRemoveHashEntry(FileSystem *pFS, LsmPgno iPg){ Page *p; int iHash = fsHashKey(pFS->nHash, iPg); for(p=pFS->apHash[iHash]; p && p->iPg!=iPg; p=p->pHashNext); @@ -2802,12 +2807,12 @@ Snapshot *pSnapshot, Segment *pSeg ){ int rc = LSM_OK; if( pFS->pCompress && pSeg->iFirst ){ - Pgno iLast2; - Pgno iLast = pSeg->iLastPg; /* Current last page of segment */ + LsmPgno iLast2; + LsmPgno iLast = pSeg->iLastPg; /* Current last page of segment */ int nPad; /* Bytes of padding required */ u8 aSz[3]; iLast2 = (1 + iLast/pFS->szSector) * pFS->szSector - 1; assert( fsPageToBlock(pFS, iLast)==fsPageToBlock(pFS, iLast2) ); @@ -2933,19 +2938,19 @@ } /* ** Helper function for lsmInfoArrayStructure(). */ -static Segment *startsWith(Segment *pRun, Pgno iFirst){ +static Segment *startsWith(Segment *pRun, LsmPgno iFirst){ return (iFirst==pRun->iFirst) ? pRun : 0; } /* ** Return the segment that starts with page iFirst, if any. If no such segment ** can be found, return NULL. */ -static Segment *findSegment(Snapshot *pWorker, Pgno iFirst){ +static Segment *findSegment(Snapshot *pWorker, LsmPgno iFirst){ Level *pLvl; /* Used to iterate through db levels */ Segment *pSeg = 0; /* Pointer to segment to return */ for(pLvl=lsmDbSnapshotLevel(pWorker); pLvl && pSeg==0; pLvl=pLvl->pNext){ if( 0==(pSeg = startsWith(&pLvl->lhs, iFirst)) ){ @@ -2968,11 +2973,11 @@ ** If an error occurs, *pzOut is set to NULL and an LSM error code returned. */ int lsmInfoArrayStructure( lsm_db *pDb, int bBlock, /* True for block numbers only */ - Pgno iFirst, + LsmPgno iFirst, char **pzOut ){ int rc = LSM_OK; Snapshot *pWorker; /* Worker snapshot */ Segment *pArray = 0; /* Array to report on */ @@ -3033,11 +3038,11 @@ } int lsmFsSegmentContainsPg( FileSystem *pFS, Segment *pSeg, - Pgno iPg, + LsmPgno iPg, int *pbRes ){ Redirect *pRedir = pSeg->pRedirect; int rc = LSM_OK; int iBlk; @@ -3062,11 +3067,11 @@ ** containing the array structure and LSM_OK is returned. The caller should ** eventually free the string using lsmFree(). ** ** If an error occurs, *pzOut is set to NULL and an LSM error code returned. */ -int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut){ +int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut){ int rc = LSM_OK; Snapshot *pWorker; /* Worker snapshot */ Segment *pSeg = 0; /* Array to report on */ int bUnlock = 0; @@ -3295,13 +3300,13 @@ ** Return true if pPg happens to be the last page in segment pSeg. Or false ** otherwise. This function is only invoked as part of assert() conditions. */ int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg){ if( pPg->pFS->pCompress ){ - Pgno iNext = 0; + LsmPgno iNext = 0; int rc; rc = fsNextPageOffset(pPg->pFS, pSeg, pPg->iPg, pPg->nCompress+6, &iNext); return (rc!=LSM_OK || iNext==0); } return (pPg->iPg==pSeg->iLastPg); } #endif Index: ext/lsm1/lsm_main.c ================================================================== --- ext/lsm1/lsm_main.c +++ ext/lsm1/lsm_main.c @@ -581,26 +581,26 @@ rc = lsmStructList(pDb, pzVal); break; } case LSM_INFO_ARRAY_STRUCTURE: { - Pgno pgno = va_arg(ap, Pgno); + LsmPgno pgno = va_arg(ap, LsmPgno); char **pzVal = va_arg(ap, char **); rc = lsmInfoArrayStructure(pDb, 0, pgno, pzVal); break; } case LSM_INFO_ARRAY_PAGES: { - Pgno pgno = va_arg(ap, Pgno); + LsmPgno pgno = va_arg(ap, LsmPgno); char **pzVal = va_arg(ap, char **); rc = lsmInfoArrayPages(pDb, pgno, pzVal); break; } case LSM_INFO_PAGE_HEX_DUMP: case LSM_INFO_PAGE_ASCII_DUMP: { - Pgno pgno = va_arg(ap, Pgno); + LsmPgno pgno = va_arg(ap, LsmPgno); char **pzVal = va_arg(ap, char **); int bUnlock = 0; rc = infoGetWorker(pDb, 0, &bUnlock); if( rc==LSM_OK ){ int bHex = (eParam==LSM_INFO_PAGE_HEX_DUMP); Index: ext/lsm1/lsm_sorted.c ================================================================== --- ext/lsm1/lsm_sorted.c +++ ext/lsm1/lsm_sorted.c @@ -102,13 +102,13 @@ #ifndef LSM_SEGMENTPTR_FREE_THRESHOLD # define LSM_SEGMENTPTR_FREE_THRESHOLD 1024 #endif typedef struct SegmentPtr SegmentPtr; -typedef struct Blob Blob; +typedef struct LsmBlob LsmBlob; -struct Blob { +struct LsmBlob { lsm_env *pEnv; void *pData; int nData; int nAlloc; }; @@ -127,22 +127,22 @@ /* Current page. See segmentPtrLoadPage(). */ Page *pPg; /* Current page */ u16 flags; /* Copy of page flags field */ int nCell; /* Number of cells on pPg */ - Pgno iPtr; /* Base cascade pointer */ + LsmPgno iPtr; /* Base cascade pointer */ /* Current cell. See segmentPtrLoadCell() */ int iCell; /* Current record within page pPg */ int eType; /* Type of current record */ - Pgno iPgPtr; /* Cascade pointer offset */ + LsmPgno iPgPtr; /* Cascade pointer offset */ void *pKey; int nKey; /* Key associated with current record */ void *pVal; int nVal; /* Current record value (eType==WRITE only) */ /* Blobs used to allocate buffers for pKey and pVal as required */ - Blob blob1; - Blob blob2; + LsmBlob blob1; + LsmBlob blob2; }; /* ** Used to iterate through the keys stored in a b-tree hierarchy from start ** to finish. Only First() and Next() operations are required. @@ -169,14 +169,14 @@ /* Cache of current entry. pKey==0 for EOF. */ void *pKey; int nKey; int eType; - Pgno iPtr; + LsmPgno iPtr; /* Storage for key, if not local */ - Blob blob; + LsmBlob blob; }; /* ** A cursor used for merged searches or iterations through up to one @@ -201,12 +201,12 @@ lsm_db *pDb; /* Connection that owns this cursor */ MultiCursor *pNext; /* Next cursor owned by connection pDb */ int flags; /* Mask of CURSOR_XXX flags */ int eType; /* Cache of current key type */ - Blob key; /* Cache of current key (or NULL) */ - Blob val; /* Cache of current value */ + LsmBlob key; /* Cache of current key (or NULL) */ + LsmBlob val; /* Cache of current value */ /* All the component cursors: */ TreeCursor *apTreeCsr[2]; /* Up to two tree cursors */ int iFree; /* Next element of free-list (-ve for eof) */ SegmentPtr *aPtr; /* Array of segment pointers */ @@ -219,11 +219,11 @@ /* Used by cursors flushing the in-memory tree only */ void *pSystemVal; /* Pointer to buffer to free */ /* Used by worker cursors only */ - Pgno *pPrevMergePtr; + LsmPgno *pPrevMergePtr; }; /* ** The following constants are used to assign integers to each component ** cursor of a multi-cursor. @@ -293,15 +293,15 @@ MultiCursor *pCsr; /* Cursor to read new segment contents from */ int bFlush; /* True if this is an in-memory tree flush */ Hierarchy hier; /* B-tree hierarchy under construction */ Page *pPage; /* Current output page */ int nWork; /* Number of calls to mergeWorkerNextPage() */ - Pgno *aGobble; /* Gobble point for each input segment */ + LsmPgno *aGobble; /* Gobble point for each input segment */ - Pgno iIndirect; + LsmPgno iIndirect; struct SavedPgno { - Pgno iPgno; + LsmPgno iPgno; int bStore; } aSave[2]; }; #ifdef LSM_DEBUG_EXPENSIVE @@ -369,11 +369,11 @@ aOut[5] = (u8)((nVal>>16) & 0xFF); aOut[6] = (u8)((nVal>> 8) & 0xFF); aOut[7] = (u8)((nVal ) & 0xFF); } -static int sortedBlobGrow(lsm_env *pEnv, Blob *pBlob, int nData){ +static int sortedBlobGrow(lsm_env *pEnv, LsmBlob *pBlob, int nData){ assert( pBlob->pEnv==pEnv || (pBlob->pEnv==0 && pBlob->pData==0) ); if( pBlob->nAllocpData = lsmReallocOrFree(pEnv, pBlob->pData, nData); if( !pBlob->pData ) return LSM_NOMEM_BKPT; pBlob->nAlloc = nData; @@ -380,36 +380,36 @@ pBlob->pEnv = pEnv; } return LSM_OK; } -static int sortedBlobSet(lsm_env *pEnv, Blob *pBlob, void *pData, int nData){ +static int sortedBlobSet(lsm_env *pEnv, LsmBlob *pBlob, void *pData, int nData){ if( sortedBlobGrow(pEnv, pBlob, nData) ) return LSM_NOMEM; memcpy(pBlob->pData, pData, nData); pBlob->nData = nData; return LSM_OK; } #if 0 -static int sortedBlobCopy(Blob *pDest, Blob *pSrc){ +static int sortedBlobCopy(LsmBlob *pDest, LsmBlob *pSrc){ return sortedBlobSet(pDest, pSrc->pData, pSrc->nData); } #endif -static void sortedBlobFree(Blob *pBlob){ +static void sortedBlobFree(LsmBlob *pBlob){ assert( pBlob->pEnv || pBlob->pData==0 ); if( pBlob->pData ) lsmFree(pBlob->pEnv, pBlob->pData); - memset(pBlob, 0, sizeof(Blob)); + memset(pBlob, 0, sizeof(LsmBlob)); } static int sortedReadData( Segment *pSeg, Page *pPg, int iOff, int nByte, void **ppData, - Blob *pBlob + LsmBlob *pBlob ){ int rc = LSM_OK; int iEnd; int nData; int nCell; @@ -479,12 +479,12 @@ static int pageGetNRec(u8 *aData, int nData){ return (int)lsmGetU16(&aData[SEGMENT_NRECORD_OFFSET(nData)]); } -static Pgno pageGetPtr(u8 *aData, int nData){ - return (Pgno)lsmGetU64(&aData[SEGMENT_POINTER_OFFSET(nData)]); +static LsmPgno pageGetPtr(u8 *aData, int nData){ + return (LsmPgno)lsmGetU64(&aData[SEGMENT_POINTER_OFFSET(nData)]); } static int pageGetFlags(u8 *aData, int nData){ return (int)lsmGetU16(&aData[SEGMENT_FLAGS_OFFSET(nData)]); } @@ -504,12 +504,12 @@ /* ** Return the decoded (possibly relative) pointer value stored in cell ** iCell from page aData/nData. */ -static Pgno pageGetRecordPtr(u8 *aData, int nData, int iCell){ - Pgno iRet; /* Return value */ +static LsmPgno pageGetRecordPtr(u8 *aData, int nData, int iCell){ + LsmPgno iRet; /* Return value */ u8 *aCell; /* Pointer to cell iCell */ assert( iCell=0 ); aCell = pageGetCell(aData, nData, iCell); lsmVarintGet64(&aCell[1], &iRet); @@ -520,11 +520,11 @@ Segment *pSeg, /* Segment pPg belongs to */ Page *pPg, /* Page to read from */ int iCell, /* Index of cell on page to read */ int *piTopic, /* OUT: Topic associated with this key */ int *pnKey, /* OUT: Size of key in bytes */ - Blob *pBlob /* If required, use this for dynamic memory */ + LsmBlob *pBlob /* If required, use this for dynamic memory */ ){ u8 *pKey; int nDummy; int eType; u8 *aData; @@ -552,11 +552,11 @@ lsm_env *pEnv, /* Environment handle */ Segment *pSeg, /* Segment pPg belongs to */ Page *pPg, /* Page to read from */ int iCell, /* Index of cell on page to read */ int *piTopic, /* OUT: Topic associated with this key */ - Blob *pBlob /* If required, use this for dynamic memory */ + LsmBlob *pBlob /* If required, use this for dynamic memory */ ){ int rc = LSM_OK; int nKey; u8 *aKey; @@ -567,12 +567,12 @@ } return rc; } -static Pgno pageGetBtreeRef(Page *pPg, int iKey){ - Pgno iRef; +static LsmPgno pageGetBtreeRef(Page *pPg, int iKey){ + LsmPgno iRef; u8 *aData; int nData; u8 *aCell; aData = fsPageData(pPg, &nData); @@ -590,15 +590,15 @@ static int pageGetBtreeKey( Segment *pSeg, /* Segment page pPg belongs to */ Page *pPg, int iKey, - Pgno *piPtr, + LsmPgno *piPtr, int *piTopic, void **ppKey, int *pnKey, - Blob *pBlob + LsmBlob *pBlob ){ u8 *aData; int nData; u8 *aCell; int eType; @@ -611,11 +611,11 @@ eType = *aCell++; aCell += GETVARINT64(aCell, *piPtr); if( eType==0 ){ int rc; - Pgno iRef; /* Page number of referenced page */ + LsmPgno iRef; /* Page number of referenced page */ Page *pRef; aCell += GETVARINT64(aCell, iRef); rc = lsmFsDbPageGet(lsmPageFS(pPg), pSeg, iRef, &pRef); if( rc!=LSM_OK ) return rc; pageGetKeyCopy(lsmPageEnv(pPg), pSeg, pRef, 0, &eType, pBlob); @@ -636,11 +636,11 @@ if( pCsr->iPg<0 ){ pCsr->pKey = 0; pCsr->nKey = 0; pCsr->eType = 0; }else{ - Pgno dummy; + LsmPgno dummy; int iPg = pCsr->iPg; int iCell = pCsr->aPg[iPg].iCell; while( iCell<0 && (--iPg)>=0 ){ iCell = pCsr->aPg[iPg].iCell-1; } @@ -681,11 +681,11 @@ aData = fsPageData(pPg->pPage, &nData); nCell = pageGetNRec(aData, nData); assert( pPg->iCell<=nCell ); pPg->iCell++; if( pPg->iCell==nCell ){ - Pgno iLoad; + LsmPgno iLoad; /* Up to parent. */ lsmFsPageRelease(pPg->pPage); pPg->pPage = 0; pCsr->iPg--; @@ -840,11 +840,11 @@ int rc = LSM_OK; if( p->iPg ){ lsm_env *pEnv = lsmFsEnv(pCsr->pFS); int iCell; /* Current cell number on leaf page */ - Pgno iLeaf; /* Page number of current leaf page */ + LsmPgno iLeaf; /* Page number of current leaf page */ int nDepth; /* Depth of b-tree structure */ Segment *pSeg = pCsr->pSeg; /* Decode the MergeInput structure */ iLeaf = p->iPg; @@ -864,11 +864,11 @@ rc = lsmFsDbPageGet(pCsr->pFS, pSeg, iLeaf, pp); } /* Populate any other aPg[] array entries */ if( rc==LSM_OK && nDepth>1 ){ - Blob blob = {0,0,0}; + LsmBlob blob = {0,0,0}; void *pSeek; int nSeek; int iTopicSeek; int iPg = 0; int iLoad = (int)pSeg->iRoot; @@ -881,11 +881,11 @@ assert( iCell==-1 ); iTopicSeek = 1000; pSeek = 0; nSeek = 0; }else{ - Pgno dummy; + LsmPgno dummy; rc = pageGetBtreeKey(pSeg, pPg, 0, &dummy, &iTopicSeek, &pSeek, &nSeek, &pCsr->blob ); } @@ -910,11 +910,11 @@ while( iMax>=iMin ){ int iTry = (iMin+iMax)/2; void *pKey; int nKey; /* Key for cell iTry */ int iTopic; /* Topic for key pKeyT/nKeyT */ - Pgno iPtr; /* Pointer for cell iTry */ + LsmPgno iPtr; /* Pointer for cell iTry */ int res; /* (pSeek - pKeyT) */ rc = pageGetBtreeKey( pSeg, pPg2, iTry, &iPtr, &iTopic, &pKey, &nKey, &blob ); @@ -953,11 +953,11 @@ pBtreePg = &pCsr->aPg[pCsr->iPg]; aData = fsPageData(pBtreePg->pPage, &nData); pCsr->iPtr = btreeCursorPtr(aData, nData, pBtreePg->iCell+1); if( pBtreePg->iCell<0 ){ - Pgno dummy; + LsmPgno dummy; int i; for(i=pCsr->iPg-1; i>=0; i--){ if( pCsr->aPg[i].iCell>0 ) break; } assert( i>=0 ); @@ -1028,11 +1028,11 @@ static int segmentPtrReadData( SegmentPtr *pPtr, int iOff, int nByte, void **ppData, - Blob *pBlob + LsmBlob *pBlob ){ return sortedReadData(pPtr->pSeg, pPtr->pPg, iOff, nByte, ppData, pBlob); } static int segmentPtrNextPage( @@ -1121,19 +1121,19 @@ if( rc==LSM_OK ){ rc = lsmFsDbPageGet(pDb->pFS, pSeg, pMerge->splitkey.iPg, &pPg); } if( rc==LSM_OK ){ int iTopic; - Blob blob = {0, 0, 0, 0}; + LsmBlob blob = {0, 0, 0, 0}; u8 *aData; int nData; aData = lsmFsPageData(pPg, &nData); if( pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG ){ void *pKey; int nKey; - Pgno dummy; + LsmPgno dummy; rc = pageGetBtreeKey(pSeg, pPg, pMerge->splitkey.iCell, &dummy, &iTopic, &pKey, &nKey, &blob ); if( rc==LSM_OK && blob.pData!=pKey ){ rc = sortedBlobSet(pEnv, &blob, pKey, nKey); @@ -1340,11 +1340,11 @@ MultiCursor *pCsr, SegmentPtr *pPtr, void *pKey, int nKey ){ lsm_env *pEnv = lsmFsEnv(pCsr->pDb->pFS); - Blob blob = {0, 0, 0}; + LsmBlob blob = {0, 0, 0}; int eDir; int iTopic = 0; /* TODO: Fix me */ for(eDir=-1; eDir<=1; eDir+=2){ Page *pTest = pPtr->pPg; @@ -1486,11 +1486,11 @@ static int ptrFwdPointer( Page *pPage, int iCell, Segment *pSeg, - Pgno *piPtr, + LsmPgno *piPtr, int *pbFound ){ Page *pPg = pPage; int iFirst = iCell; int rc = LSM_OK; @@ -1571,18 +1571,18 @@ ** the big range-delete. */ static int segmentPtrFwdPointer( MultiCursor *pCsr, /* Multi-cursor pPtr belongs to */ SegmentPtr *pPtr, /* Segment-pointer to extract FC ptr from */ - Pgno *piPtr /* OUT: FC pointer value */ + LsmPgno *piPtr /* OUT: FC pointer value */ ){ Level *pLvl = pPtr->pLevel; Level *pNext = pLvl->pNext; Page *pPg = pPtr->pPg; int rc; int bFound; - Pgno iOut = 0; + LsmPgno iOut = 0; if( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[pLvl->nRight-1] ){ if( pNext==0 || (pNext->nRight==0 && pNext->lhs.iRoot) || (pNext->nRight!=0 && pNext->aRhs[0].iRoot) @@ -1639,11 +1639,11 @@ int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; int res = 0; /* Result of comparison operation */ int rc = LSM_OK; int iMin; int iMax; - Pgno iPtrOut = 0; + LsmPgno iPtrOut = 0; /* If the current page contains an oversized entry, then there are no ** pointers to one or more of the subsequent pages in the sorted run. ** The following call ensures that the segment-ptr points to the correct ** page in this case. */ @@ -1766,22 +1766,22 @@ static int seekInBtree( MultiCursor *pCsr, /* Multi-cursor object */ Segment *pSeg, /* Seek within this segment */ int iTopic, void *pKey, int nKey, /* Key to seek to */ - Pgno *aPg, /* OUT: Page numbers */ + LsmPgno *aPg, /* OUT: Page numbers */ Page **ppPg /* OUT: Leaf (sorted-run) page reference */ ){ int i = 0; int rc; int iPg; Page *pPg = 0; - Blob blob = {0, 0, 0}; + LsmBlob blob = {0, 0, 0}; iPg = (int)pSeg->iRoot; do { - Pgno *piFirst = 0; + LsmPgno *piFirst = 0; if( aPg ){ aPg[i++] = iPg; piFirst = &aPg[i]; } @@ -1806,11 +1806,11 @@ iMax = nRec-1; while( iMax>=iMin ){ int iTry = (iMin+iMax)/2; void *pKeyT; int nKeyT; /* Key for cell iTry */ int iTopicT; /* Topic for key pKeyT/nKeyT */ - Pgno iPtr; /* Pointer associated with cell iTry */ + LsmPgno iPtr; /* Pointer associated with cell iTry */ int res; /* (pKey - pKeyT) */ rc = pageGetBtreeKey( pSeg, pPg, iTry, &iPtr, &iTopicT, &pKeyT, &nKeyT, &blob ); @@ -1897,11 +1897,11 @@ MultiCursor *pCsr, /* Sorted cursor object to seek */ SegmentPtr *aPtr, /* Pointer to array of (nRhs+1) SPs */ int eSeek, /* Search bias - see above */ int iTopic, /* Key topic to search for */ void *pKey, int nKey, /* Key to search for */ - Pgno *piPgno, /* IN/OUT: fraction cascade pointer (or 0) */ + LsmPgno *piPgno, /* IN/OUT: fraction cascade pointer (or 0) */ int *pbStop /* OUT: See above */ ){ Level *pLvl = aPtr[0].pLevel; /* Level to seek within */ int rc = LSM_OK; /* Return code */ int iOut = 0; /* Pointer to return to caller */ @@ -3053,11 +3053,11 @@ ){ int eESeek = eSeek; /* Effective eSeek parameter */ int bStop = 0; /* Set to true to halt search operation */ int rc = LSM_OK; /* Return code */ int iPtr = 0; /* Used to iterate through pCsr->aPtr[] */ - Pgno iPgno = 0; /* FC pointer value */ + LsmPgno iPgno = 0; /* FC pointer value */ assert( pCsr->apTreeCsr[0]==0 || iTopic==0 ); assert( pCsr->apTreeCsr[1]==0 || iTopic==0 ); if( eESeek==LSM_SEEK_LEFAST ) eESeek = LSM_SEEK_LE; @@ -3535,11 +3535,11 @@ ** 1. The record format is (usually, see below) as follows: ** ** + Type byte (always SORTED_SEPARATOR or SORTED_SYSTEM_SEPARATOR), ** + Absolute pointer value (varint), ** + Number of bytes in key (varint), -** + Blob containing key data. +** + LsmBlob containing key data. ** ** 2. All pointer values are stored as absolute values (not offsets ** relative to the footer pointer value). ** ** 3. Each pointer that is part of a record points to a page that @@ -3569,12 +3569,12 @@ */ static int mergeWorkerBtreeWrite( MergeWorker *pMW, u8 eType, - Pgno iPtr, - Pgno iKeyPg, + LsmPgno iPtr, + LsmPgno iKeyPg, void *pKey, int nKey ){ Hierarchy *p = &pMW->hier; lsm_db *pDb = pMW->pDb; /* Database handle */ @@ -3680,11 +3680,11 @@ } static int mergeWorkerBtreeIndirect(MergeWorker *pMW){ int rc = LSM_OK; if( pMW->iIndirect ){ - Pgno iKeyPg = pMW->aSave[1].iPgno; + LsmPgno iKeyPg = pMW->aSave[1].iPgno; rc = mergeWorkerBtreeWrite(pMW, 0, pMW->iIndirect, iKeyPg, 0, 0); pMW->iIndirect = 0; } return rc; } @@ -3701,11 +3701,11 @@ int iTopic, /* Topic value for this key */ void *pKey, /* Pointer to key buffer */ int nKey /* Size of pKey buffer in bytes */ ){ int rc = LSM_OK; /* Return Code */ - Pgno iPtr; /* Pointer value to accompany pKey/nKey */ + LsmPgno iPtr; /* Pointer value to accompany pKey/nKey */ assert( pMW->aSave[0].bStore==0 ); assert( pMW->aSave[1].bStore==0 ); rc = mergeWorkerBtreeIndirect(pMW); @@ -3732,11 +3732,11 @@ static int mergeWorkerFinishHierarchy( MergeWorker *pMW /* Merge worker object */ ){ int i; /* Used to loop through apHier[] */ int rc = LSM_OK; /* Return code */ - Pgno iPtr; /* New right-hand-child pointer value */ + LsmPgno iPtr; /* New right-hand-child pointer value */ iPtr = pMW->aSave[0].iPgno; for(i=0; ihier.nHier && rc==LSM_OK; i++){ Page *pPg = pMW->hier.apHier[i]; int nData; /* Size of aData[] in bytes */ @@ -3828,11 +3828,11 @@ ** ** If successful, LSM_OK is returned. Otherwise, an error code. */ static int mergeWorkerNextPage( MergeWorker *pMW, /* Merge worker object to append page to */ - Pgno iFPtr /* Pointer value for footer of new page */ + LsmPgno iFPtr /* Pointer value for footer of new page */ ){ int rc = LSM_OK; /* Return code */ Page *pNext = 0; /* New page appended to run */ lsm_db *pDb = pMW->pDb; /* Database handle */ @@ -4216,11 +4216,11 @@ lsm_db *pDb = pMW->pDb; /* Database handle */ MultiCursor *pCsr; /* Cursor to read input data from */ int rc = LSM_OK; /* Return code */ int eType; /* SORTED_SEPARATOR, WRITE or DELETE */ void *pKey; int nKey; /* Key */ - Pgno iPtr; + LsmPgno iPtr; int iVal; pCsr = pMW->pCsr; /* Pull the next record out of the source cursor. */ @@ -4369,11 +4369,11 @@ } if( rc!=LSM_OK ){ lsmMCursorClose(pCsr, 0); }else{ - Pgno iLeftPtr = 0; + LsmPgno iLeftPtr = 0; Merge merge; /* Merge object used to create new level */ MergeWorker mergeworker; /* MergeWorker object for the same purpose */ memset(&merge, 0, sizeof(Merge)); memset(&mergeworker, 0, sizeof(MergeWorker)); @@ -4546,11 +4546,11 @@ assert( pLevel->nRight>0 ); memset(pMW, 0, sizeof(MergeWorker)); pMW->pDb = pDb; pMW->pLevel = pLevel; - pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(Pgno) * pLevel->nRight, &rc); + pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*pLevel->nRight,&rc); /* Create a multi-cursor to read the data to write to the new ** segment. The new segment contains: ** ** 1. Records from LHS of each of the nMerge levels being merged. @@ -4628,20 +4628,20 @@ int iGobble /* pCsr->aPtr[] entry to operate on */ ){ int rc = LSM_OK; if( rtTopic(pCsr->eType)==0 ){ Segment *pSeg = pCsr->aPtr[iGobble].pSeg; - Pgno *aPg; + LsmPgno *aPg; int nPg; /* Seek from the root of the b-tree to the segment leaf that may contain ** a key equal to the one multi-cursor currently points to. Record the ** page number of each b-tree page and the leaf. The segment may be ** gobbled up to (but not including) the first of these page numbers. */ assert( pSeg->iRoot>0 ); - aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(Pgno)*32, &rc); + aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*32, &rc); if( rc==LSM_OK ){ rc = seekInBtree(pCsr, pSeg, rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, aPg, 0 ); } @@ -5464,13 +5464,13 @@ ** Space for the returned string is allocated using lsmMalloc(), and should ** be freed by the caller using lsmFree(). */ static char *segToString(lsm_env *pEnv, Segment *pSeg, int nMin){ int nSize = pSeg->nSize; - Pgno iRoot = pSeg->iRoot; - Pgno iFirst = pSeg->iFirst; - Pgno iLast = pSeg->iLastPg; + LsmPgno iRoot = pSeg->iRoot; + LsmPgno iFirst = pSeg->iFirst; + LsmPgno iLast = pSeg->iLastPg; char *z; char *z1; char *z2; int nPad; @@ -5525,11 +5525,11 @@ return i; } void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){ - Blob blob = {0, 0, 0}; /* Blob used for keys */ + LsmBlob blob = {0, 0, 0}; /* LsmBlob used for keys */ LsmString s; int i; int nRec; int iPtr; @@ -5561,11 +5561,11 @@ eType = *aCell++; assert( (flags & SEGMENT_BTREE_FLAG) || eType!=0 ); aCell += lsmVarintGet32(aCell, &iPgPtr); if( eType==0 ){ - Pgno iRef; /* Page number of referenced page */ + LsmPgno iRef; /* Page number of referenced page */ aCell += lsmVarintGet64(aCell, &iRef); lsmFsDbPageGet(pDb->pFS, pRun, iRef, &pRef); aKey = pageGetKey(pRun, pRef, 0, &iTopic, &nKey, &blob); }else{ aCell += lsmVarintGet32(aCell, &nKey); @@ -5605,11 +5605,11 @@ int iCell, int *peType, int *piPgPtr, u8 **paKey, int *pnKey, u8 **paVal, int *pnVal, - Blob *pBlob + LsmBlob *pBlob ){ u8 *aData; int nData; /* Page data */ u8 *aKey; int nKey = 0; /* Key */ u8 *aVal = 0; int nVal = 0; /* Value */ int eType; @@ -5623,11 +5623,11 @@ eType = *aCell++; aCell += lsmVarintGet32(aCell, &iPgPtr); if( eType==0 ){ int dummy; - Pgno iRef; /* Page number of referenced page */ + LsmPgno iRef; /* Page number of referenced page */ aCell += lsmVarintGet64(aCell, &iRef); if( bIndirect ){ lsmFsDbPageGet(pDb->pFS, pSeg, iRef, &pRef); pageGetKeyCopy(pDb->pEnv, pSeg, pRef, 0, &dummy, pBlob); aKey = (u8 *)pBlob->pData; @@ -5669,11 +5669,11 @@ #define INFO_PAGE_DUMP_HEX 0x04 #define INFO_PAGE_DUMP_INDIRECT 0x08 static int infoPageDump( lsm_db *pDb, /* Database handle */ - Pgno iPg, /* Page number of page to dump */ + LsmPgno iPg, /* Page number of page to dump */ int flags, char **pzOut /* OUT: lsmMalloc'd string */ ){ int rc = LSM_OK; /* Return code */ Page *pPg = 0; /* Handle for page iPg */ @@ -5710,11 +5710,11 @@ if( rc==LSM_OK ){ rc = lsmFsDbPageGet(pDb->pFS, 0, iPg, &pPg); } if( rc==LSM_OK ){ - Blob blob = {0, 0, 0, 0}; + LsmBlob blob = {0, 0, 0, 0}; int nKeyWidth = 0; LsmString str; int nRec; int iPtr; int flags2; @@ -5745,11 +5745,11 @@ for(iCell=0; iCellpFS, pSeg, pSeg->iFirst, &pPg); while( pPg ){ u8 *aData; int nData; Page *pNext; @@ -6050,11 +6050,11 @@ int bRhs /* True if pTwo may have been Gobble()d */ ){ int rc = LSM_OK; /* Error code */ SegmentPtr ptr1; /* Iterates through pOne */ SegmentPtr ptr2; /* Iterates through pTwo */ - Pgno iPrev; + LsmPgno iPrev; assert( pOne && pTwo ); memset(&ptr1, 0, sizeof(ptr1)); memset(&ptr2, 0, sizeof(ptr1)); @@ -6073,11 +6073,11 @@ if( rc==LSM_OK && ptr1.nCell>0 ){ rc = segmentPtrLoadCell(&ptr1, 0); } while( rc==LSM_OK && ptr2.pPg ){ - Pgno iThis; + LsmPgno iThis; /* Advance to the next page of segment pTwo that contains at least ** one cell. Break out of the loop if the iterator reaches EOF. */ do{ rc = segmentPtrNextPage(&ptr2, 1); @@ -6135,11 +6135,11 @@ lsm_db *pDb, Segment *pSeg ){ int rc = LSM_OK; /* Return code */ if( pSeg->iRoot ){ - Blob blob = {0, 0, 0}; /* Buffer used to cache overflow keys */ + LsmBlob blob = {0, 0, 0}; /* Buffer used to cache overflow keys */ FileSystem *pFS = pDb->pFS; /* File system to read from */ Page *pPg = 0; /* Main run page */ BtreeCursor *pCsr = 0; /* Btree cursor */ rc = btreeCursorNew(pDb, pSeg, &pCsr); ADDED ext/lsm1/tool/mklsm1c.tcl Index: ext/lsm1/tool/mklsm1c.tcl ================================================================== --- ext/lsm1/tool/mklsm1c.tcl +++ ext/lsm1/tool/mklsm1c.tcl @@ -0,0 +1,88 @@ +#!/bin/sh +# restart with tclsh \ +exec tclsh "$0" "$@" + +set srcdir [file dirname [file dirname [info script]]] +set G(src) [string map [list %dir% $srcdir] { + %dir%/lsm.h + %dir%/lsmInt.h + %dir%/lsm_vtab.c + %dir%/lsm_ckpt.c + %dir%/lsm_file.c + %dir%/lsm_log.c + %dir%/lsm_main.c + %dir%/lsm_mem.c + %dir%/lsm_mutex.c + %dir%/lsm_shared.c + %dir%/lsm_sorted.c + %dir%/lsm_str.c + %dir%/lsm_tree.c + %dir%/lsm_unix.c + %dir%/lsm_varint.c + %dir%/lsm_win32.c +}] + +set G(hdr) { + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) + +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif +#if defined(NDEBUG) && defined(SQLITE_DEBUG) +# undef NDEBUG +#endif + +} + +set G(footer) { + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) */ +} + +#------------------------------------------------------------------------- +# Read and return the entire contents of text file $zFile from disk. +# +proc readfile {zFile} { + set fd [open $zFile] + set data [read $fd] + close $fd + return $data +} + +proc lsm1c_init {zOut} { + global G + set G(fd) stdout + set G(fd) [open $zOut w] + + puts -nonewline $G(fd) $G(hdr) +} + +proc lsm1c_printfile {zIn} { + global G + set data [readfile $zIn] + set zTail [file tail $zIn] + puts $G(fd) "#line 1 \"$zTail\"" + + foreach line [split $data "\n"] { + if {[regexp {^# *include.*lsm} $line]} { + set line "/* $line */" + } elseif { [regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?lsm[^_]} $line] } { + set line "static $line" + } + puts $G(fd) $line + } +} + +proc lsm1c_close {} { + global G + puts -nonewline $G(fd) $G(footer) + if {$G(fd)!="stdout"} { + close $G(fd) + } +} + + +lsm1c_init lsm1.c +foreach f $G(src) { lsm1c_printfile $f } +lsm1c_close Index: ext/misc/rot13.c ================================================================== --- ext/misc/rot13.c +++ ext/misc/rot13.c @@ -45,21 +45,21 @@ sqlite3_value **argv ){ const unsigned char *zIn; int nIn; unsigned char *zOut; - char *zToFree = 0; + unsigned char *zToFree = 0; int i; - char zTemp[100]; + unsigned char zTemp[100]; assert( argc==1 ); if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; zIn = (const unsigned char*)sqlite3_value_text(argv[0]); nIn = sqlite3_value_bytes(argv[0]); if( nIn ** USING swarmvtab(, ); @@ -64,17 +66,82 @@ ** The difference is that for a swarmvtab table, the first column returned ** by the must return a path or URI that can be used to open ** the database file containing the source table. The option ** is optional. If included, it is the name of an application-defined ** SQL function that is invoked with the URI of the file, if the file -** does not already exist on disk. +** does not already exist on disk when required by swarmvtab. +** +** NEW SYNTAX: +** +** Using the new syntax, a swarmvtab table is created with: +** +** CREATE VIRTUAL TABLE USING swarmvtab( +** [, ] +** ); +** +** where valid are: +** +** missing= +** openclose= +** maxopen= +** = +** +** The must return the same 4 columns as for a swarmvtab +** table in legacy mode. However, it may also return a 5th column - the +** "context" column. The text value returned in this column is not used +** at all by the swarmvtab implementation, except that it is passed as +** an additional argument to the two UDF functions that may be invoked +** (see below). +** +** The "missing" option, if present, specifies the name of an SQL UDF +** function to be invoked if a database file is not already present on +** disk when required by swarmvtab. If the did not provide +** a context column, it is invoked as: +** +** SELECT (); +** +** Or, if there was a context column: +** +** SELECT (, ); +** +** The "openclose" option may also specify a UDF function. This function +** is invoked right before swarmvtab opens a database, and right after +** it closes one. The first argument - or first two arguments, if +** supplied the context column - is the same as for +** the "missing" UDF. Following this, the UDF is passed integer value +** 0 before a db is opened, and 1 right after it is closed. If both +** a missing and openclose UDF is supplied, the application should expect +** the following sequence of calls (for a single database): +** +** SELECT (, , 0); +** if( db not already on disk ){ +** SELECT (, ); +** } +** ... swarmvtab uses database ... +** SELECT (, , 1); +** +** The "maxopen" option is used to configure the maximum number of +** database files swarmvtab will hold open simultaneously (default 9). +** +** If an option name begins with a ":" character, then it is assumed +** to be an SQL parameter. In this case, the specified text value is +** bound to the same variable of the before it is +** executed. It is an error of the named SQL parameter does not exist. +** For example: +** +** CREATE VIRTUAL TABLE swarm USING swarmvtab( +** 'SELECT :path || localfile, tbl, min, max FROM swarmdir', +** :path='/home/user/databases/' +** missing='missing_func' +** ); */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 #include #include +#include #ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Largest and smallest possible 64-bit signed integers. These macros @@ -126,10 +193,11 @@ sqlite3_int64 iMin; /* Minimum rowid */ sqlite3_int64 iMax; /* Maximum rowid */ /* Fields used by swarmvtab only */ char *zFile; /* Database file containing table zTab */ + char *zContext; /* Context string, if any */ int nUser; /* Current number of users */ sqlite3 *db; /* Database handle */ UnionSrc *pNextClosable; /* Next in list of closable sources */ }; @@ -143,12 +211,15 @@ int iPK; /* INTEGER PRIMARY KEY column, or -1 */ int nSrc; /* Number of elements in the aSrc[] array */ UnionSrc *aSrc; /* Array of source tables, sorted by rowid */ /* Used by swarmvtab only */ + int bHasContext; /* Has context strings */ char *zSourceStr; /* Expected unionSourceToStr() value */ - char *zNotFoundCallback; /* UDF to invoke if file not found on open */ + sqlite3_stmt *pNotFound; /* UDF to invoke if file not found on open */ + sqlite3_stmt *pOpenClose; /* UDF to invoke on open and close */ + UnionSrc *pClosable; /* First in list of closable sources */ int nOpen; /* Current number of open sources */ int nMaxOpen; /* Maximum number of open sources */ }; @@ -348,25 +419,61 @@ if( rc ){ *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); } } } + +/* +** If an "openclose" UDF was supplied when this virtual table was created, +** invoke it now. The first argument passed is the name of the database +** file for source pSrc. The second is integer value bClose. +** +** If successful, return SQLITE_OK. Otherwise an SQLite error code. In this +** case if argument pzErr is not NULL, also set (*pzErr) to an English +** language error message. The caller is responsible for eventually freeing +** any error message using sqlite3_free(). +*/ +static int unionInvokeOpenClose( + UnionTab *pTab, + UnionSrc *pSrc, + int bClose, + char **pzErr +){ + int rc = SQLITE_OK; + if( pTab->pOpenClose ){ + sqlite3_bind_text(pTab->pOpenClose, 1, pSrc->zFile, -1, SQLITE_STATIC); + if( pTab->bHasContext ){ + sqlite3_bind_text(pTab->pOpenClose, 2, pSrc->zContext, -1, SQLITE_STATIC); + } + sqlite3_bind_int(pTab->pOpenClose, 2+pTab->bHasContext, bClose); + sqlite3_step(pTab->pOpenClose); + if( SQLITE_OK!=(rc = sqlite3_reset(pTab->pOpenClose)) ){ + if( pzErr ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); + } + } + } + return rc; +} /* ** This function is a no-op for unionvtab. For swarmvtab, it attempts to ** close open database files until at most nMax are open. An SQLite error ** code is returned if an error occurs, or SQLITE_OK otherwise. */ static void unionCloseSources(UnionTab *pTab, int nMax){ while( pTab->pClosable && pTab->nOpen>nMax ){ + UnionSrc *p; UnionSrc **pp; for(pp=&pTab->pClosable; (*pp)->pNextClosable; pp=&(*pp)->pNextClosable); - assert( (*pp)->db ); - sqlite3_close((*pp)->db); - (*pp)->db = 0; + p = *pp; + assert( p->db ); + sqlite3_close(p->db); + p->db = 0; *pp = 0; pTab->nOpen--; + unionInvokeOpenClose(pTab, p, 1, 0); } } /* ** xDisconnect method. @@ -375,17 +482,22 @@ if( pVtab ){ UnionTab *pTab = (UnionTab*)pVtab; int i; for(i=0; inSrc; i++){ UnionSrc *pSrc = &pTab->aSrc[i]; + if( pSrc->db ){ + unionInvokeOpenClose(pTab, pSrc, 1, 0); + } sqlite3_free(pSrc->zDb); sqlite3_free(pSrc->zTab); sqlite3_free(pSrc->zFile); + sqlite3_free(pSrc->zContext); sqlite3_close(pSrc->db); } + sqlite3_finalize(pTab->pNotFound); + sqlite3_finalize(pTab->pOpenClose); sqlite3_free(pTab->zSourceStr); - sqlite3_free(pTab->zNotFoundCallback); sqlite3_free(pTab->aSrc); sqlite3_free(pTab); } return SQLITE_OK; } @@ -494,33 +606,35 @@ sqlite3_free(z0); return rc; } - /* ** Try to open the swarmvtab database. If initially unable, invoke the ** not-found callback UDF and then try again. */ static int unionOpenDatabaseInner(UnionTab *pTab, UnionSrc *pSrc, char **pzErr){ - int rc = SQLITE_OK; - static const int openFlags = - SQLITE_OPEN_READONLY | SQLITE_OPEN_URI; + static const int openFlags = SQLITE_OPEN_READONLY | SQLITE_OPEN_URI; + int rc; + + rc = unionInvokeOpenClose(pTab, pSrc, 0, pzErr); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3_open_v2(pSrc->zFile, &pSrc->db, openFlags, 0); if( rc==SQLITE_OK ) return rc; - if( pTab->zNotFoundCallback ){ - char *zSql = sqlite3_mprintf("SELECT \"%w\"(%Q);", - pTab->zNotFoundCallback, pSrc->zFile); + if( pTab->pNotFound ){ sqlite3_close(pSrc->db); pSrc->db = 0; - if( zSql==0 ){ - *pzErr = sqlite3_mprintf("out of memory"); - return SQLITE_NOMEM; + sqlite3_bind_text(pTab->pNotFound, 1, pSrc->zFile, -1, SQLITE_STATIC); + if( pTab->bHasContext ){ + sqlite3_bind_text(pTab->pNotFound, 2, pSrc->zContext, -1, SQLITE_STATIC); } - rc = sqlite3_exec(pTab->db, zSql, 0, 0, pzErr); - sqlite3_free(zSql); - if( rc ) return rc; + sqlite3_step(pTab->pNotFound); + if( SQLITE_OK!=(rc = sqlite3_reset(pTab->pNotFound)) ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); + return rc; + } rc = sqlite3_open_v2(pSrc->zFile, &pSrc->db, openFlags, 0); } if( rc!=SQLITE_OK ){ *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pSrc->db)); } @@ -570,10 +684,11 @@ pTab->pClosable = pSrc; pTab->nOpen++; }else{ sqlite3_close(pSrc->db); pSrc->db = 0; + unionInvokeOpenClose(pTab, pSrc, 1, 0); } } return rc; } @@ -624,10 +739,136 @@ unionCloseSources(pTab, pTab->nMaxOpen); } } return rc; } + +/* +** Return true if the argument is a space, tab, CR or LF character. +*/ +static int union_isspace(char c){ + return (c==' ' || c=='\n' || c=='\r' || c=='\t'); +} + +/* +** Return true if the argument is an alphanumeric character in the +** ASCII range. +*/ +static int union_isidchar(char c){ + return ((c>='a' && c<='z') || (c>='A' && c<'Z') || (c>='0' && c<='9')); +} + +/* +** This function is called to handle all arguments following the first +** (the SQL statement) passed to a swarmvtab (not unionvtab) CREATE +** VIRTUAL TABLE statement. It may bind parameters to the SQL statement +** or configure members of the UnionTab object passed as the second +** argument. +** +** Refer to header comments at the top of this file for a description +** of the arguments parsed. +** +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, if an error occurs, *pRc is set to an SQLite error +** code. In this case *pzErr may be set to point to a buffer containing +** an English language error message. It is the responsibility of the +** caller to eventually free the buffer using sqlite3_free(). +*/ +static void unionConfigureVtab( + int *pRc, /* IN/OUT: Error code */ + UnionTab *pTab, /* Table to configure */ + sqlite3_stmt *pStmt, /* SQL statement to find sources */ + int nArg, /* Number of entries in azArg[] array */ + const char * const *azArg, /* Array of arguments to consider */ + char **pzErr /* OUT: Error message */ +){ + int rc = *pRc; + int i; + if( rc==SQLITE_OK ){ + pTab->bHasContext = (sqlite3_column_count(pStmt)>4); + } + for(i=0; rc==SQLITE_OK && inMaxOpen = atoi(zVal); + if( pTab->nMaxOpen<=0 ){ + *pzErr = sqlite3_mprintf("swarmvtab: illegal maxopen value"); + rc = SQLITE_ERROR; + } + }else if( nOpt==7 && 0==sqlite3_strnicmp(zOpt, "missing", 7) ){ + if( pTab->pNotFound ){ + *pzErr = sqlite3_mprintf( + "swarmvtab: duplicate \"missing\" option"); + rc = SQLITE_ERROR; + }else{ + pTab->pNotFound = unionPreparePrintf(&rc, pzErr, pTab->db, + "SELECT \"%w\"(?%s)", zVal, pTab->bHasContext ? ",?" : "" + ); + } + }else if( nOpt==9 && 0==sqlite3_strnicmp(zOpt, "openclose", 9) ){ + if( pTab->pOpenClose ){ + *pzErr = sqlite3_mprintf( + "swarmvtab: duplicate \"openclose\" option"); + rc = SQLITE_ERROR; + }else{ + pTab->pOpenClose = unionPreparePrintf(&rc, pzErr, pTab->db, + "SELECT \"%w\"(?,?%s)", zVal, pTab->bHasContext ? ",?" : "" + ); + } + }else{ + *pzErr = sqlite3_mprintf("swarmvtab: unrecognized option: %s",zOpt); + rc = SQLITE_ERROR; + } + sqlite3_free(zVal); + } + }else{ + if( i==0 && nArg==1 ){ + pTab->pNotFound = unionPreparePrintf(&rc, pzErr, pTab->db, + "SELECT \"%w\"(?)", zArg + ); + }else{ + *pzErr = sqlite3_mprintf( "swarmvtab: parse error: %s", azArg[i]); + rc = SQLITE_ERROR; + } + } + sqlite3_free(zArg); + } + } + *pRc = rc; +} /* ** xConnect/xCreate method. ** ** The argv[] array contains the following: @@ -652,11 +893,11 @@ if( sqlite3_stricmp("temp", argv[1]) ){ /* unionvtab tables may only be created in the temp schema */ *pzErr = sqlite3_mprintf("%s tables must be created in TEMP schema", zVtab); rc = SQLITE_ERROR; - }else if( argc!=4 && argc!=5 ){ + }else if( argc<4 || (argc>4 && bSwarm==0) ){ *pzErr = sqlite3_mprintf("wrong number of arguments for %s", zVtab); rc = SQLITE_ERROR; }else{ int nAlloc = 0; /* Allocated size of pTab->aSrc[] */ sqlite3_stmt *pStmt = 0; /* Argument statement */ @@ -671,10 +912,21 @@ "SELECT * FROM (%z) ORDER BY 3", zArg ); /* Allocate the UnionTab structure */ pTab = unionMalloc(&rc, sizeof(UnionTab)); + if( pTab ){ + assert( rc==SQLITE_OK ); + pTab->db = db; + pTab->bSwarm = bSwarm; + pTab->nMaxOpen = SWARMVTAB_MAX_OPEN; + } + + /* Parse other CVT arguments, if any */ + if( bSwarm ){ + unionConfigureVtab(&rc, pTab, pStmt, argc-4, &argv[4], pzErr); + } /* Iterate through the rows returned by the SQL statement specified ** as an argument to the CREATE VIRTUAL TABLE statement. */ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ const char *zDb = (const char*)sqlite3_column_text(pStmt, 0); @@ -713,21 +965,19 @@ if( bSwarm ){ pSrc->zFile = unionStrdup(&rc, zDb); }else{ pSrc->zDb = unionStrdup(&rc, zDb); } + if( pTab->bHasContext ){ + const char *zContext = (const char*)sqlite3_column_text(pStmt, 4); + pSrc->zContext = unionStrdup(&rc, zContext); + } } } unionFinalize(&rc, pStmt, pzErr); pStmt = 0; - /* Capture the not-found callback UDF name */ - if( rc==SQLITE_OK && argc>=5 ){ - pTab->zNotFoundCallback = unionStrdup(&rc, argv[4]); - unionDequote(pTab->zNotFoundCallback); - } - /* It is an error if the SELECT statement returned zero rows. If only ** because there is no way to determine the schema of the virtual ** table in this case. */ if( rc==SQLITE_OK && pTab->nSrc==0 ){ *pzErr = sqlite3_mprintf("no source tables configured"); @@ -736,13 +986,10 @@ /* For unionvtab, verify that all source tables exist and have ** compatible schemas. For swarmvtab, attach the first database and ** check that the first table is a rowid table only. */ if( rc==SQLITE_OK ){ - pTab->db = db; - pTab->bSwarm = bSwarm; - pTab->nMaxOpen = SWARMVTAB_MAX_OPEN; if( bSwarm ){ rc = unionOpenDatabase(pTab, 0, pzErr); }else{ rc = unionSourceCheck(pTab, pzErr); } Index: ext/rtree/rtree.c ================================================================== --- ext/rtree/rtree.c +++ ext/rtree/rtree.c @@ -2019,11 +2019,11 @@ int iHeight, /* Height of sub-tree rooted at pCell */ RtreeNode **ppLeaf /* OUT: Selected leaf page */ ){ int rc; int ii; - RtreeNode *pNode; + RtreeNode *pNode = 0; rc = nodeAcquire(pRtree, 1, 0, &pNode); for(ii=0; rc==SQLITE_OK && ii<(pRtree->iDepth-iHeight); ii++){ int iCell; sqlite3_int64 iBest = 0; @@ -2894,11 +2894,11 @@ ** the root node (the operation that Gutman's paper says to perform ** in this scenario). */ if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ int rc2; - RtreeNode *pChild; + RtreeNode *pChild = 0; i64 iChild = nodeGetRowid(pRtree, pRoot, 0); rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); if( rc==SQLITE_OK ){ rc = removeNode(pRtree, pChild, pRtree->iDepth-1); } Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -261,10 +261,28 @@ $(TOP)/ext/fts5/fts5_tokenize.c \ $(TOP)/ext/fts5/fts5_unicode2.c \ $(TOP)/ext/fts5/fts5_varint.c \ $(TOP)/ext/fts5/fts5_vocab.c \ +LSM1_SRC = \ + $(TOP)/ext/lsm1/lsm.h \ + $(TOP)/ext/lsm1/lsmInt.h \ + $(TOP)/ext/lsm1/lsm_ckpt.c \ + $(TOP)/ext/lsm1/lsm_file.c \ + $(TOP)/ext/lsm1/lsm_log.c \ + $(TOP)/ext/lsm1/lsm_main.c \ + $(TOP)/ext/lsm1/lsm_mem.c \ + $(TOP)/ext/lsm1/lsm_mutex.c \ + $(TOP)/ext/lsm1/lsm_shared.c \ + $(TOP)/ext/lsm1/lsm_sorted.c \ + $(TOP)/ext/lsm1/lsm_str.c \ + $(TOP)/ext/lsm1/lsm_tree.c \ + $(TOP)/ext/lsm1/lsm_unix.c \ + $(TOP)/ext/lsm1/lsm_varint.c \ + $(TOP)/ext/lsm1/lsm_vtab.c \ + $(TOP)/ext/lsm1/lsm_win32.c + # Generated source code files # SRC += \ keywordhash.h \ @@ -766,10 +784,14 @@ fts5.c: $(FTS5_SRC) $(FTS5_HDR) tclsh $(TOP)/ext/fts5/tool/mkfts5c.tcl cp $(TOP)/ext/fts5/fts5.h . +lsm1.c: $(LSM1_SRC) + tclsh $(TOP)/ext/lsm1/tool/mklsm1c.tcl + cp $(TOP)/ext/lsm1/lsm.h . + userauth.o: $(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c sqlite3session.o: $(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c @@ -1023,5 +1045,6 @@ rm -f mptester mptester.exe rm -f fuzzershell fuzzershell.exe rm -f fuzzcheck fuzzcheck.exe rm -f sqldiff sqldiff.exe rm -f fts5.* fts5parse.* + rm -f lsm.h lsm1.c Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -4362,10 +4362,22 @@ pKey->aColl[i] = zColl==sqlite3StrBINARY ? 0 : sqlite3LocateCollSeq(pParse, zColl); pKey->aSortOrder[i] = pIdx->aSortOrder[i]; } if( pParse->nErr ){ + assert( pParse->rc==SQLITE_ERROR_MISSING_COLLSEQ ); + if( pIdx->bNoQuery==0 ){ + /* Deactivate the index because it contains an unknown collating + ** sequence. The only way to reactive the index is to reload the + ** schema. Adding the missing collating sequence later does not + ** reactive the index. The application had the chance to register + ** the missing index using the collation-needed callback. For + ** simplicity, SQLite will not give the application a second chance. + */ + pIdx->bNoQuery = 1; + pParse->rc = SQLITE_ERROR_RETRY; + } sqlite3KeyInfoUnref(pKey); pKey = 0; } } return pKey; Index: src/callback.c ================================================================== --- src/callback.c +++ src/callback.c @@ -103,10 +103,11 @@ p = 0; } assert( !p || p->xCmp ); if( p==0 ){ sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); + pParse->rc = SQLITE_ERROR_MISSING_COLLSEQ; } return p; } /* Index: src/date.c ================================================================== --- src/date.c +++ src/date.c @@ -37,11 +37,11 @@ ** The conversion algorithms are implemented based on descriptions ** in the following text: ** ** Jean Meeus ** Astronomical Algorithms, 2nd Edition, 1998 -** ISBM 0-943396-61-1 +** ISBN 0-943396-61-1 ** Willmann-Bell, Inc ** Richmond, Virginia (USA) */ #include "sqliteInt.h" #include Index: src/delete.c ================================================================== --- src/delete.c +++ src/delete.c @@ -281,15 +281,15 @@ ** deleted from is a view */ #ifndef SQLITE_OMIT_TRIGGER pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); isView = pTab->pSelect!=0; - bComplex = pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0); #else # define pTrigger 0 # define isView 0 #endif + bComplex = pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0); #ifdef SQLITE_OMIT_VIEW # undef isView # define isView 0 #endif Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -20,11 +20,11 @@ # include "fts3.h" #endif #ifdef SQLITE_ENABLE_RTREE # include "rtree.h" #endif -#ifdef SQLITE_ENABLE_ICU +#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) # include "sqliteicu.h" #endif #ifdef SQLITE_ENABLE_JSON1 int sqlite3Json1Init(sqlite3*); #endif @@ -3063,11 +3063,11 @@ if( !db->mallocFailed && rc==SQLITE_OK ){ rc = sqlite3Fts3Init(db); } #endif -#ifdef SQLITE_ENABLE_ICU +#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) if( !db->mallocFailed && rc==SQLITE_OK ){ rc = sqlite3IcuInit(db); } #endif Index: src/mutex.c ================================================================== --- src/mutex.c +++ src/mutex.c @@ -132,11 +132,11 @@ assert( SQLITE_MUTEX_RECURSIVE<2 ); assert( SQLITE_MUTEX_FAST<2 ); assert( SQLITE_MUTEX_WARNONCONTENTION<2 ); #if SQLITE_ENABLE_API_ARMOR - if( p->iType<2 ){ + if( ((CheckMutex*)p)->iType<2 ) #endif { CheckMutex *pCheck = (CheckMutex*)p; pGlobalMutexMethods->xMutexFree(pCheck->mutex); sqlite3_free(pCheck); Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -481,11 +481,11 @@ #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, #else { "munmap", (sqlite3_syscall_ptr)0, 0 }, #endif -#define osMunmap ((void*(*)(void*,size_t))aSyscall[23].pCurrent) +#define osMunmap ((int(*)(void*,size_t))aSyscall[23].pCurrent) #if HAVE_MREMAP && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, #else { "mremap", (sqlite3_syscall_ptr)0, 0 }, @@ -4163,11 +4163,11 @@ struct flock f; /* The posix advisory locking structure */ int rc = SQLITE_OK; /* Result code form fcntl() */ /* Access to the unixShmNode object is serialized by the caller */ pShmNode = pFile->pInode->pShmNode; - assert( sqlite3_mutex_held(pShmNode->mutex) || pShmNode->nRef==0 ); + assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->mutex) ); /* Shared locks never span more than one byte */ assert( n==1 || lockType!=F_RDLCK ); /* Locks are within range */ @@ -5797,11 +5797,11 @@ /* If creating a master or main-file journal, this function will open ** a file-descriptor on the directory too. The first time unixSync() ** is called the directory file descriptor will be fsync()ed and close()d. */ - int syncDir = (isCreate && ( + int isNewJrnl = (isCreate && ( eType==SQLITE_OPEN_MASTER_JOURNAL || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_WAL )); @@ -5867,11 +5867,11 @@ ** sqlite3_uri_parameter(). */ assert( (flags & SQLITE_OPEN_URI) || zName[strlen(zName)+1]==0 ); }else if( !zName ){ /* If zName is NULL, the upper layer is requesting a temp file. */ - assert(isDelete && !syncDir); + assert(isDelete && !isNewJrnl); rc = unixGetTempname(pVfs->mxPathname, zTmpname); if( rc!=SQLITE_OK ){ return rc; } zName = zTmpname; @@ -5913,10 +5913,13 @@ isReadonly = 1; fd = robust_open(zName, openFlags, openMode); } if( fd<0 ){ rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName); + /* If unable to create a journal, change the error code to + ** indicate that the directory permissions are wrong. */ + if( isNewJrnl && osAccess(zName, F_OK) ) rc = SQLITE_READONLY_DIRECTORY; goto open_finished; } /* If this process is running as root and if creating a new rollback ** journal or WAL file, set the ownership of the journal or WAL to be @@ -5972,11 +5975,11 @@ /* Set up appropriate ctrlFlags */ if( isDelete ) ctrlFlags |= UNIXFILE_DELETE; if( isReadonly ) ctrlFlags |= UNIXFILE_RDONLY; noLock = eType!=SQLITE_OPEN_MAIN_DB; if( noLock ) ctrlFlags |= UNIXFILE_NOLOCK; - if( syncDir ) ctrlFlags |= UNIXFILE_DIRSYNC; + if( isNewJrnl ) ctrlFlags |= UNIXFILE_DIRSYNC; if( flags & SQLITE_OPEN_URI ) ctrlFlags |= UNIXFILE_URI; #if SQLITE_ENABLE_LOCKING_STYLE #if SQLITE_PREFER_PROXY_LOCKING isAutoProxy = 1; Index: src/os_win.c ================================================================== --- src/os_win.c +++ src/os_win.c @@ -3740,11 +3740,11 @@ int nByte /* Number of bytes to lock or unlock */ ){ int rc = 0; /* Result code form Lock/UnlockFileEx() */ /* Access to the winShmNode object is serialized by the caller */ - assert( sqlite3_mutex_held(pFile->mutex) || pFile->nRef==0 ); + assert( pFile->nRef==0 || sqlite3_mutex_held(pFile->mutex) ); OSTRACE(("SHM-LOCK file=%p, lock=%d, offset=%d, size=%d\n", pFile->hFile.h, lockType, ofst, nByte)); /* Release/Acquire the system-level lock */ Index: src/prepare.c ================================================================== --- src/prepare.c +++ src/prepare.c @@ -653,12 +653,10 @@ } end_prepare: sqlite3ParserReset(&sParse); - rc = sqlite3ApiExit(db, rc); - assert( (rc&db->errMask)==rc ); return rc; } static int sqlite3LockAndPrepare( sqlite3 *db, /* Database handle. */ const char *zSql, /* UTF-8 encoded SQL statement. */ @@ -667,10 +665,11 @@ Vdbe *pOld, /* VM being reprepared */ sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ const char **pzTail /* OUT: End of parsed string */ ){ int rc; + int cnt = 0; #ifdef SQLITE_ENABLE_API_ARMOR if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; #endif *ppStmt = 0; @@ -677,19 +676,22 @@ if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); sqlite3BtreeEnterAll(db); - rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail); - if( rc==SQLITE_SCHEMA ){ - sqlite3ResetOneSchema(db, -1); - sqlite3_finalize(*ppStmt); + do{ + /* Make multiple attempts to compile the SQL, until it either succeeds + ** or encounters a permanent error. A schema problem after one schema + ** reset is considered a permanent error. */ rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail); - } + assert( rc==SQLITE_OK || *ppStmt==0 ); + }while( rc==SQLITE_ERROR_RETRY + || (rc==SQLITE_SCHEMA && (sqlite3ResetOneSchema(db,-1), cnt++)==0) ); sqlite3BtreeLeaveAll(db); + rc = sqlite3ApiExit(db, rc); + assert( (rc&db->errMask)==rc ); sqlite3_mutex_leave(db->mutex); - assert( rc==SQLITE_OK || *ppStmt==0 ); return rc; } /* ** Rerun the compilation of a statement after a schema change. Index: src/shell.c.in ================================================================== --- src/shell.c.in +++ src/shell.c.in @@ -3893,14 +3893,14 @@ ** This SELECT statement returns one row for each foreign key constraint ** in the schema of the main database. The column values are: ** ** 0. The text of an SQL statement similar to: ** - ** "EXPLAIN QUERY PLAN SELECT rowid FROM child_table WHERE child_key=?" + ** "EXPLAIN QUERY PLAN SELECT 1 FROM child_table WHERE child_key=?" ** - ** This is the same SELECT that the foreign keys implementation needs - ** to run internally on child tables. If there is an index that can + ** This SELECT is similar to the one that the foreign keys implementation + ** needs to run internally on child tables. If there is an index that can ** be used to optimize this query, then it can also be used by the FK ** implementation to optimize DELETE or UPDATE statements on the parent ** table. ** ** 1. A GLOB pattern suitable for sqlite3_strglob(). If the plan output by @@ -3924,11 +3924,11 @@ ** ** These six values are used by the C logic below to generate the report. */ const char *zSql = "SELECT " - " 'EXPLAIN QUERY PLAN SELECT rowid FROM ' || quote(s.name) || ' WHERE '" + " 'EXPLAIN QUERY PLAN SELECT 1 FROM ' || quote(s.name) || ' WHERE '" " || group_concat(quote(s.name) || '.' || quote(f.[from]) || '=?' " " || fkey_collate_clause(" " f.[table], COALESCE(f.[to], p.[name]), s.name, f.[from]),' AND ')" ", " " 'SEARCH TABLE ' || s.name || ' USING COVERING INDEX*('" Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -468,10 +468,12 @@ ** on a per database connection basis using the ** [sqlite3_extended_result_codes()] API. Or, the extended code for ** the most recent error can be obtained using ** [sqlite3_extended_errcode()]. */ +#define SQLITE_ERROR_MISSING_COLLSEQ (SQLITE_ERROR | (1<<8)) +#define SQLITE_ERROR_RETRY (SQLITE_ERROR | (2<<8)) #define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) #define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) #define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) #define SQLITE_IOERR_FSYNC (SQLITE_IOERR | (4<<8)) #define SQLITE_IOERR_DIR_FSYNC (SQLITE_IOERR | (5<<8)) @@ -513,10 +515,11 @@ #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) #define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) #define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) #define SQLITE_READONLY_DBMOVED (SQLITE_READONLY | (4<<8)) #define SQLITE_READONLY_CANTINIT (SQLITE_READONLY | (5<<8)) +#define SQLITE_READONLY_DIRECTORY (SQLITE_READONLY | (6<<8)) #define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) #define SQLITE_CONSTRAINT_CHECK (SQLITE_CONSTRAINT | (1<<8)) #define SQLITE_CONSTRAINT_COMMITHOOK (SQLITE_CONSTRAINT | (2<<8)) #define SQLITE_CONSTRAINT_FOREIGNKEY (SQLITE_CONSTRAINT | (3<<8)) #define SQLITE_CONSTRAINT_FUNCTION (SQLITE_CONSTRAINT | (4<<8)) Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -2172,10 +2172,11 @@ unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ unsigned isResized:1; /* True if resizeIndexObject() has been called */ unsigned isCovering:1; /* True if this is a covering index */ unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ + unsigned bNoQuery:1; /* Do not use this index to optimize queries */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ @@ -2984,11 +2985,11 @@ int nErr; /* Number of errors seen */ int nTab; /* Number of previously allocated VDBE cursors */ int nMem; /* Number of memory cells used so far */ int nOpAlloc; /* Number of slots allocated for Vdbe.aOp[] */ int szOpAlloc; /* Bytes of memory space allocated for Vdbe.aOp[] */ - int iSelfTab; /* Table for associated with an index on expr, or negative + int iSelfTab; /* Table associated with an index on expr, or negative ** of the base register during check-constraint eval */ int iCacheLevel; /* ColCache valid when aColCache[].iLevel<=iCacheLevel */ int iCacheCnt; /* Counter used to generate aColCache[].lru values */ int nLabel; /* Number of labels used */ int *aLabel; /* Space to hold the labels */ Index: src/test_config.c ================================================================== --- src/test_config.c +++ src/test_config.c @@ -426,10 +426,16 @@ #ifdef SQLITE_ENABLE_ICU Tcl_SetVar2(interp, "sqlite_options", "icu", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "icu", "0", TCL_GLOBAL_ONLY); #endif + +#ifdef SQLITE_ENABLE_ICU_COLLATIONS + Tcl_SetVar2(interp, "sqlite_options", "icu_collations", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "icu_collations", "0", TCL_GLOBAL_ONLY); +#endif #ifdef SQLITE_OMIT_INCRBLOB Tcl_SetVar2(interp, "sqlite_options", "incrblob", "0", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "incrblob", "1", TCL_GLOBAL_ONLY); Index: src/vdbemem.c ================================================================== --- src/vdbemem.c +++ src/vdbemem.c @@ -1319,11 +1319,15 @@ const char *zNeg = ""; int rc = SQLITE_OK; assert( pExpr!=0 ); while( (op = pExpr->op)==TK_UPLUS || op==TK_SPAN ) pExpr = pExpr->pLeft; +#if defined(SQLITE_ENABLE_STAT3_OR_STAT4) if( op==TK_REGISTER ) op = pExpr->op2; +#else + if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; +#endif /* Compressed expressions only appear when parsing the DEFAULT clause ** on a table column definition, and hence only when pCtx==0. This ** check ensures that an EP_TokenOnly expression is never passed down ** into valueFromFunction(). */ @@ -1414,11 +1418,14 @@ *ppVal = pVal; return rc; no_mem: - sqlite3OomFault(db); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( pCtx==0 || pCtx->pParse->nErr==0 ) +#endif + sqlite3OomFault(db); sqlite3DbFree(db, zVal); assert( *ppVal==0 ); #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( pCtx==0 ) sqlite3ValueFree(pVal); #else Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -2877,10 +2877,11 @@ if( pProbe->pPartIdxWhere!=0 && !whereUsablePartialIndex(pSrc->iCursor, pWC, pProbe->pPartIdxWhere) ){ testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */ continue; /* Partial index inappropriate for this query */ } + if( pProbe->bNoQuery ) continue; rSize = pProbe->aiRowLogEst[0]; pNew->u.btree.nEq = 0; pNew->u.btree.nBtm = 0; pNew->u.btree.nTop = 0; pNew->nSkip = 0; Index: src/wherecode.c ================================================================== --- src/wherecode.c +++ src/wherecode.c @@ -1688,10 +1688,11 @@ }else{ endEq = 1; } }else if( bStopAtNull ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); endEq = 0; nConstraint++; } sqlite3DbFree(db, zStartAff); sqlite3DbFree(db, zEndAff); Index: test/icu.test ================================================================== --- test/icu.test +++ test/icu.test @@ -13,11 +13,11 @@ # set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !icu { +ifcapable !icu&&!icu_collations { finish_test return } # Create a table to work with. @@ -33,58 +33,61 @@ ROLLBACK; }] 0 } $settings $expr] $result } -# Tests of the REGEXP operator. -# -test_expr icu-1.1 {i1='hello'} {i1 REGEXP 'hello'} 1 -test_expr icu-1.2 {i1='hello'} {i1 REGEXP '.ello'} 1 -test_expr icu-1.3 {i1='hello'} {i1 REGEXP '.ell'} 0 -test_expr icu-1.4 {i1='hello'} {i1 REGEXP '.ell.*'} 1 -test_expr icu-1.5 {i1=NULL} {i1 REGEXP '.ell.*'} {} +ifcapable icu { + + # Tests of the REGEXP operator. + # + test_expr icu-1.1 {i1='hello'} {i1 REGEXP 'hello'} 1 + test_expr icu-1.2 {i1='hello'} {i1 REGEXP '.ello'} 1 + test_expr icu-1.3 {i1='hello'} {i1 REGEXP '.ell'} 0 + test_expr icu-1.4 {i1='hello'} {i1 REGEXP '.ell.*'} 1 + test_expr icu-1.5 {i1=NULL} {i1 REGEXP '.ell.*'} {} + + # Some non-ascii characters with defined case mappings + # + set ::EGRAVE "\xC8" + set ::egrave "\xE8" + + set ::OGRAVE "\xD2" + set ::ograve "\xF2" + + # That German letter that looks a bit like a B. The + # upper-case version of which is "SS" (two characters). + # + set ::szlig "\xDF" -# Some non-ascii characters with defined case mappings -# -set ::EGRAVE "\xC8" -set ::egrave "\xE8" + # Tests of the upper()/lower() functions. + # + test_expr icu-2.1 {i1='HellO WorlD'} {upper(i1)} {HELLO WORLD} + test_expr icu-2.2 {i1='HellO WorlD'} {lower(i1)} {hello world} + test_expr icu-2.3 {i1=$::egrave} {lower(i1)} $::egrave + test_expr icu-2.4 {i1=$::egrave} {upper(i1)} $::EGRAVE + test_expr icu-2.5 {i1=$::ograve} {lower(i1)} $::ograve + test_expr icu-2.6 {i1=$::ograve} {upper(i1)} $::OGRAVE + test_expr icu-2.3 {i1=$::EGRAVE} {lower(i1)} $::egrave + test_expr icu-2.4 {i1=$::EGRAVE} {upper(i1)} $::EGRAVE + test_expr icu-2.5 {i1=$::OGRAVE} {lower(i1)} $::ograve + test_expr icu-2.6 {i1=$::OGRAVE} {upper(i1)} $::OGRAVE -set ::OGRAVE "\xD2" -set ::ograve "\xF2" - -# That German letter that looks a bit like a B. The -# upper-case version of which is "SS" (two characters). -# -set ::szlig "\xDF" + test_expr icu-2.7 {i1=$::szlig} {upper(i1)} "SS" + test_expr icu-2.8 {i1='SS'} {lower(i1)} "ss" -# Tests of the upper()/lower() functions. -# -test_expr icu-2.1 {i1='HellO WorlD'} {upper(i1)} {HELLO WORLD} -test_expr icu-2.2 {i1='HellO WorlD'} {lower(i1)} {hello world} -test_expr icu-2.3 {i1=$::egrave} {lower(i1)} $::egrave -test_expr icu-2.4 {i1=$::egrave} {upper(i1)} $::EGRAVE -test_expr icu-2.5 {i1=$::ograve} {lower(i1)} $::ograve -test_expr icu-2.6 {i1=$::ograve} {upper(i1)} $::OGRAVE -test_expr icu-2.3 {i1=$::EGRAVE} {lower(i1)} $::egrave -test_expr icu-2.4 {i1=$::EGRAVE} {upper(i1)} $::EGRAVE -test_expr icu-2.5 {i1=$::OGRAVE} {lower(i1)} $::ograve -test_expr icu-2.6 {i1=$::OGRAVE} {upper(i1)} $::OGRAVE + do_execsql_test icu-2.9 { + SELECT upper(char(0xfb04,0xfb04,0xfb04,0xfb04)); + } {FFLFFLFFLFFL} -test_expr icu-2.7 {i1=$::szlig} {upper(i1)} "SS" -test_expr icu-2.8 {i1='SS'} {lower(i1)} "ss" - -do_execsql_test icu-2.9 { - SELECT upper(char(0xfb04,0xfb04,0xfb04,0xfb04)); -} {FFLFFLFFLFFL} - -# In turkish (locale="tr_TR"), the lower case version of I -# is "small dotless i" (code point 0x131 (decimal 305)). -# -set ::small_dotless_i "\u0131" -test_expr icu-3.1 {i1='I'} {lower(i1)} "i" -test_expr icu-3.2 {i1='I'} {lower(i1, 'tr_tr')} $::small_dotless_i -test_expr icu-3.3 {i1='I'} {lower(i1, 'en_AU')} "i" + # In turkish (locale="tr_TR"), the lower case version of I + # is "small dotless i" (code point 0x131 (decimal 305)). + # + set ::small_dotless_i "\u0131" + test_expr icu-3.1 {i1='I'} {lower(i1)} "i" + test_expr icu-3.2 {i1='I'} {lower(i1, 'tr_tr')} $::small_dotless_i + test_expr icu-3.3 {i1='I'} {lower(i1, 'en_AU')} "i" +} #-------------------------------------------------------------------- # Test the collation sequence function. # do_test icu-4.1 { @@ -122,24 +125,25 @@ # Test that it is not possible to call the ICU regex() function with # anything other than exactly two arguments. See also: # # http://src.chromium.org/viewvc/chrome/trunk/src/third_party/sqlite/icu-regexp.patch?revision=34807&view=markup # -do_catchsql_test icu-5.1 { SELECT regexp('a[abc]c.*', 'abc') } {0 1} -do_catchsql_test icu-5.2 { - SELECT regexp('a[abc]c.*') -} {1 {wrong number of arguments to function regexp()}} -do_catchsql_test icu-5.3 { - SELECT regexp('a[abc]c.*', 'abc', 'c') -} {1 {wrong number of arguments to function regexp()}} -do_catchsql_test icu-5.4 { - SELECT 'abc' REGEXP 'a[abc]c.*' -} {0 1} -do_catchsql_test icu-5.4 { SELECT 'abc' REGEXP } {1 {near " ": syntax error}} -do_catchsql_test icu-5.5 { SELECT 'abc' REGEXP, 1 } {1 {near ",": syntax error}} - - -do_malloc_test icu-6.10 -sqlbody { - SELECT upper(char(0xfb04,0xdf,0xfb04,0xe8,0xfb04)); +ifcapable icu { + do_catchsql_test icu-5.1 { SELECT regexp('a[abc]c.*', 'abc') } {0 1} + do_catchsql_test icu-5.2 { + SELECT regexp('a[abc]c.*') + } {1 {wrong number of arguments to function regexp()}} + do_catchsql_test icu-5.3 { + SELECT regexp('a[abc]c.*', 'abc', 'c') + } {1 {wrong number of arguments to function regexp()}} + do_catchsql_test icu-5.4 { + SELECT 'abc' REGEXP 'a[abc]c.*' + } {0 1} + do_catchsql_test icu-5.4 {SELECT 'abc' REGEXP } {1 {near " ": syntax error}} + do_catchsql_test icu-5.5 {SELECT 'abc' REGEXP, 1} {1 {near ",": syntax error}} + + do_malloc_test icu-6.10 -sqlbody { + SELECT upper(char(0xfb04,0xdf,0xfb04,0xe8,0xfb04)); + } } finish_test Index: test/limit2.test ================================================================== --- test/limit2.test +++ test/limit2.test @@ -147,7 +147,24 @@ INSERT INTO t502 VALUES(3, 3); INSERT INTO t502 VALUES(4, 6); INSERT INTO t502 VALUES(5, 1); SELECT j FROM t502 WHERE i IN (1,2,3,4,5) ORDER BY j LIMIT 3; } {1 3 4} + +# Ticket https://www.sqlite.org/src/info/123c9ba32130a6c9 2017-12-13 +# Incorrect result when an idnex is used for an ordered join. +# +# This test case is in the limit2.test module because the problem was first +# exposed by check-in https://www.sqlite.org/src/info/559733b09e which +# implemented the ORDER BY LIMIT optimization that limit2.test strives to +# test. +# +do_execsql_test 600 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a, b); INSERT INTO t1 VALUES(1,2); + DROP TABLE IF EXISTS t2; + CREATE TABLE t2(x, y); INSERT INTO t2 VALUES(1,3); + CREATE INDEX t1ab ON t1(a,b); + SELECT y FROM t1, t2 WHERE a=x AND b<=y ORDER BY b DESC; +} {3} finish_test Index: test/mjournal.test ================================================================== --- test/mjournal.test +++ test/mjournal.test @@ -12,10 +12,15 @@ # set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix mjournal + +if {[permutation]=="inmemory_journal"} { + finish_test + return +} # Test that nothing bad happens if a journal file contains a pointer to # a master journal file that does not have a "-" in the name. At one point # this was causing a segfault on unix. # Index: test/shell6.test ================================================================== --- test/shell6.test +++ test/shell6.test @@ -90,10 +90,18 @@ 9 { CREATE TABLE p1(a, b UNIQUE); CREATE TABLE c1(x INTEGER PRIMARY KEY REFERENCES p1(b)); } { } + + 10 { + CREATE TABLE parent (id INTEGER PRIMARY KEY); + CREATE TABLE child2 (id INT PRIMARY KEY, parentID INT REFERENCES parent) + WITHOUT ROWID; + } { + CREATE INDEX 'child2_parentID' ON 'child2'('parentID'); --> parent(id) + } } { forcedelete test.db sqlite3 db test.db execsql $schema Index: test/swarmvtab.test ================================================================== --- test/swarmvtab.test +++ test/swarmvtab.test @@ -211,11 +211,11 @@ 'VALUES ("test.db1", "t1", 1, 10), ("test.db2", "t1", 11, 20) ', 'fetch_db_no_such_function' ); -} {1 {no such function: fetch_db_no_such_function}} +} {1 {sql error: no such function: fetch_db_no_such_function}} do_catchsql_test 3.2 { CREATE VIRTUAL TABLE temp.xyz USING swarmvtab( 'VALUES ("test.db1", "t1", 1, 10), Index: test/swarmvtab2.test ================================================================== --- test/swarmvtab2.test +++ test/swarmvtab2.test @@ -12,11 +12,11 @@ # focus of this file is the "swarmvtab" extension # set testdir [file dirname $argv0] source $testdir/tester.tcl -set testprefix swarmvtab +set testprefix swarmvtab2 do_not_use_codec ifcapable !vtab { finish_test return ADDED test/swarmvtab3.test Index: test/swarmvtab3.test ================================================================== --- test/swarmvtab3.test +++ test/swarmvtab3.test @@ -0,0 +1,233 @@ +# 2017-07-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is the "swarmvtab" extension +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix swarmvtab3 +do_not_use_codec + +ifcapable !vtab { + finish_test + return +} + +load_static_extension db unionvtab + +set nFile $sqlite_open_file_count + +do_execsql_test 1.0 { + CREATE TEMP TABLE swarm(id, tbl, minval, maxval); +} + +# Set up 100 databases with filenames "remote_test.dbN", where N is between +# 0 and 99. +do_test 1.1 { + for {set i 0} {$i < 100} {incr i} { + set file remote_test.db$i + forcedelete $file + forcedelete test.db$i + sqlite3 rrr $file + rrr eval { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES($i, $i); + } + rrr close + db eval { + INSERT INTO swarm VALUES($i, 't1', $i, $i); + } + set ::dbcache(test.db$i) 0 + } +} {} + +proc missing_db {filename} { + set remote "remote_$filename" + forcedelete $filename + file copy $remote $filename +} +db func missing_db missing_db + +proc openclose_db {filename bClose} { + if {$bClose} { + incr ::dbcache($filename) -1 + } else { + incr ::dbcache($filename) 1 + } + if {$::dbcache($filename)==0} { + forcedelete $filename + } +} +db func openclose_db openclose_db + +proc check_dbcache {} { + set n 0 + for {set i 0} {$i<100} {incr i} { + set exists [file exists test.db$i] + if {$exists!=($::dbcache(test.db$i)!=0)} { + error "inconsistent ::dbcache and disk" + } + incr n $exists + } + return $n +} + +foreach {tn nMaxOpen cvt} { + 1 5 { + CREATE VIRTUAL TABLE temp.s USING swarmvtab( + 'SELECT :prefix || id, tbl, minval, minval FROM swarm', + :prefix='test.db', + missing=missing_db, + openclose=openclose_db, + maxopen=5 + ) + } + + 2 3 { + CREATE VIRTUAL TABLE temp.s USING swarmvtab( + 'SELECT :prefix || id, tbl, minval, minval FROM swarm', + :prefix='test.db', + missing = 'missing_db', + openclose=[openclose_db], + maxopen = 3 + ) + } + + 3 1 { + CREATE VIRTUAL TABLE temp.s USING swarmvtab( + 'SELECT :prefix||''.''||:suffix||id, tbl, minval, minval FROM swarm', + :prefix=test, :suffix=db, + missing = 'missing_db', + openclose=[openclose_db], + maxopen = 1 + ) + } + +} { + execsql { DROP TABLE IF EXISTS s } + + do_execsql_test 1.$tn.1 $cvt + + do_execsql_test 1.$tn.2 { + SELECT b FROM s WHERE a<10; + } {0 1 2 3 4 5 6 7 8 9} + + do_test 1.$tn.3 { check_dbcache } $nMaxOpen + + do_execsql_test 1.$tn.4 { + SELECT b FROM s WHERE (b%10)=0; + } {0 10 20 30 40 50 60 70 80 90} + + do_test 1.$tn.5 { check_dbcache } $nMaxOpen +} + +execsql { DROP TABLE IF EXISTS s } +for {set i 0} {$i < 100} {incr i} { + forcedelete remote_test.db$i +} + +#---------------------------------------------------------------------------- +# +do_execsql_test 2.0 { + DROP TABLE IF EXISTS swarm; + CREATE TEMP TABLE swarm(file, tbl, minval, maxval, ctx); +} + +catch { array unset ::dbcache } + +# Set up 100 databases with filenames "remote_test.dbN", where N is a +# random integer between 0 and 1,000,000 +# 0 and 99. +do_test 2.1 { + for {set i 0} {$i < 100} {incr i} { + while 1 { + set ctx [expr abs(int(rand() *1000000))] + if {[info exists ::dbcache($ctx)]==0} break + } + + set file test_remote.db$ctx + forcedelete $file + forcedelete test.db$i + sqlite3 rrr $file + rrr eval { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES($i, $i); + } + rrr close + db eval { + INSERT INTO swarm VALUES('test.db' || $i, 't1', $i, $i, $file) + } + set ::dbcache(test.db$i) 0 + } +} {} + +proc missing_db {filename ctx} { + file copy $ctx $filename +} +db func missing_db missing_db + +proc openclose_db {filename ctx bClose} { + if {$bClose} { + incr ::dbcache($filename) -1 + } else { + incr ::dbcache($filename) 1 + } + if {$::dbcache($filename)==0} { + forcedelete $filename + } +} +db func openclose_db openclose_db + +proc check_dbcache {} { + set n 0 + foreach k [array names ::dbcache] { + set exists [file exists $k] + if {$exists!=($::dbcache($k)!=0)} { + error "inconsistent ::dbcache and disk ($k)" + } + incr n $exists + } + return $n +} + +foreach {tn nMaxOpen cvt} { + 2 5 { + CREATE VIRTUAL TABLE temp.s USING swarmvtab( + 'SELECT file, tbl, minval, minval, ctx FROM swarm', + missing=missing_db, + openclose=openclose_db, + maxopen=5 + ) + } +} { + execsql { DROP TABLE IF EXISTS s } + + do_execsql_test 1.$tn.1 $cvt + + do_execsql_test 1.$tn.2 { + SELECT b FROM s WHERE a<10; + } {0 1 2 3 4 5 6 7 8 9} + + do_test 1.$tn.3 { check_dbcache } $nMaxOpen + + do_execsql_test 1.$tn.4 { + SELECT b FROM s WHERE (b%10)=0; + } {0 10 20 30 40 50 60 70 80 90} + + do_test 1.$tn.5 { check_dbcache } $nMaxOpen +} + +db close +forcedelete {*}[glob test_remote.db*] + +finish_test + Index: test/walprotocol.test ================================================================== --- test/walprotocol.test +++ test/walprotocol.test @@ -50,25 +50,25 @@ T filter xShmLock T script lock_callback set ::locks [list] sqlite3 db test.db -vfs T execsql { SELECT * FROM x } - lrange $::locks 0 3 -} [list {0 1 lock exclusive} {1 7 lock exclusive} \ - {1 7 unlock exclusive} {0 1 unlock exclusive} \ + lrange $::locks 0 5 +} [list {0 1 lock exclusive} {1 2 lock exclusive} {4 4 lock exclusive} \ + {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive} \ ] do_test 1.2 { db close set ::locks [list] sqlite3 db test.db -vfs T execsql { SELECT * FROM x } - lrange $::locks 0 3 -} [list {0 1 lock exclusive} {1 7 lock exclusive} \ - {1 7 unlock exclusive} {0 1 unlock exclusive} \ + lrange $::locks 0 5 +} [list {0 1 lock exclusive} {1 2 lock exclusive} {4 4 lock exclusive} \ + {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive} \ ] proc lock_callback {method filename handle lock} { - if {$lock == "1 7 lock exclusive"} { return SQLITE_BUSY } + if {$lock == "1 2 lock exclusive"} { return SQLITE_BUSY } return SQLITE_OK } puts "# Warning: This next test case causes SQLite to call xSleep(1) 100 times." puts "# Normally this equates to a delay of roughly 10 seconds, but if SQLite" puts "# is built on unix without HAVE_USLEEP defined, it may be much longer." @@ -83,10 +83,22 @@ proc lock_callback {method filename handle lock} { if {$lock == "0 1 lock exclusive"} { return SQLITE_BUSY } return SQLITE_OK } do_test 1.4 { + db close + set ::locks [list] + sqlite3 db test.db -vfs T + catchsql { SELECT * FROM x } +} {1 {locking protocol}} + +puts "# Warning: Third time!" +proc lock_callback {method filename handle lock} { + if {$lock == "4 4 lock exclusive"} { return SQLITE_BUSY } + return SQLITE_OK +} +do_test 1.5 { db close set ::locks [list] sqlite3 db test.db -vfs T catchsql { SELECT * FROM x } } {1 {locking protocol}} @@ -133,17 +145,18 @@ faultsim_restore_and_reopen T filter xShmLock T script lock_callback proc lock_callback {method file handle spec} { - if {$spec == "1 7 unlock exclusive"} { + if {$spec == "1 2 unlock exclusive"} { T filter {} set ::r [catchsql { SELECT * FROM b } db2] } } sqlite3 db test.db sqlite3 db2 test.db +puts "# Warning: Another slow test!" do_test 2.5 { execsql { SELECT * FROM b } } {Tehran Qom Markazi Qazvin Gilan Ardabil} do_test 2.6 { set ::r @@ -155,16 +168,17 @@ faultsim_restore_and_reopen sqlite3 db2 test.db T filter xShmLock T script lock_callback proc lock_callback {method file handle spec} { - if {$spec == "1 7 unlock exclusive"} { + if {$spec == "1 2 unlock exclusive"} { T filter {} set ::r [catchsql { SELECT * FROM b } db2] } } unset ::r +puts "# Warning: Last one!" do_test 2.7 { execsql { SELECT * FROM b } } {Tehran Qom Markazi Qazvin Gilan Ardabil} do_test 2.8 { set ::r Index: test/walro2.test ================================================================== --- test/walro2.test +++ test/walro2.test @@ -260,12 +260,13 @@ INSERT INTO t2 SELECT randomblob(500) FROM s; SELECT count(*) FROM t2; } } {500} do_test $TN.4.2.2 { - file size test.db-wal - } {461152} + set sz [file size test.db-wal] + expr {$sz>400000 && $sz<500000} + } {1} do_test $TN.4.2.4 { file_control_persist_wal db 1; db close copy_to_test2 $bZeroShm code2 { sqlite3 db2 file:test.db2?readonly_shm=1 } Index: tool/lempar.c ================================================================== --- tool/lempar.c +++ tool/lempar.c @@ -649,14 +649,22 @@ static void yy_accept(yyParser*); /* Forward Declaration */ /* ** Perform a reduce action and the shift that must immediately ** follow the reduce. +** +** The yyLookahead and yyLookaheadToken parameters provide reduce actions +** access to the lookahead token (if any). The yyLookahead will be YYNOCODE +** if the lookahead token has already been consumed. As this procedure is +** only called from one place, optimizing compilers will in-line it, which +** means that the extra parameters have no performance impact. */ static void yy_reduce( yyParser *yypParser, /* The parser */ - unsigned int yyruleno /* Number of the rule by which to reduce */ + unsigned int yyruleno, /* Number of the rule by which to reduce */ + int yyLookahead, /* Lookahead token, or YYNOCODE if none */ + ParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ ){ int yygoto; /* The next state */ int yyact; /* The next action */ yyStackEntry *yymsp; /* The top of the parser's stack */ int yysize; /* Amount to pop the stack */ @@ -851,11 +859,11 @@ #ifndef YYNOERRORRECOVERY yypParser->yyerrcnt--; #endif yymajor = YYNOCODE; }else if( yyact <= YY_MAX_REDUCE ){ - yy_reduce(yypParser,yyact-YY_MIN_REDUCE); + yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor,yyminor); }else{ assert( yyact == YY_ERROR_ACTION ); yyminorunion.yy0 = yyminor; #ifdef YYERRORSYMBOL int yymx;