Index: lsm-test/README ================================================================== --- lsm-test/README +++ lsm-test/README @@ -27,13 +27,8 @@ The difference from lsmtest2.c is that this file tests live-recovery (recovery from a failure that occurs while other clients are still running) whereas lsmtest2.c tests recovery from a system or power failure. - lsmtest9.c: More data tests. These focus on testing that calling - lsm_work(nMerge=1) to compact the database does not corrupt it. - In other words, that databases containing block-redirects - can be read and written. - Index: lsm-test/lsmtest.h ================================================================== --- lsm-test/lsmtest.h +++ lsm-test/lsmtest.h @@ -63,25 +63,15 @@ */ int test_kc_open(const char *zFilename, int bClear, TestDb **ppDb); int test_kc_close(TestDb *); int test_kc_write(TestDb *, void *, int , void *, int); int test_kc_delete(TestDb *, void *, int); -int test_kc_delete_range(TestDb *, void *, int, void *, int); int test_kc_fetch(TestDb *, void *, int, void **, int *); int test_kc_scan(TestDb *, void *, int, void *, int, void *, int, void (*)(void *, void *, int , void *, int) ); -int test_mdb_open(const char *zFilename, int bClear, TestDb **ppDb); -int test_mdb_close(TestDb *); -int test_mdb_write(TestDb *, void *, int , void *, int); -int test_mdb_delete(TestDb *, void *, int); -int test_mdb_fetch(TestDb *, void *, int, void **, int *); -int test_mdb_scan(TestDb *, void *, int, void *, int, void *, int, - void (*)(void *, void *, int , void *, int) -); - /* ** Functions in wrapper3.c. This file contains the tdb wrapper for lsm. ** The wrapper for lsm is a bit more involved than the others, as it ** includes code for a couple of different lsm configurations, and for ** various types of fault injection and robustness testing. @@ -90,13 +80,10 @@ int test_lsm_lomem_open(const char *zFilename, int bClear, TestDb **ppDb); int test_lsm_zip_open(const char *zFilename, int bClear, TestDb **ppDb); int test_lsm_small_open(const char *zFilename, int bClear, TestDb **ppDb); int test_lsm_mt2(const char *zFilename, int bClear, TestDb **ppDb); int test_lsm_mt3(const char *zFilename, int bClear, TestDb **ppDb); - -int tdb_lsm_configure(lsm_db *, const char *); - /* Functions in testutil.c. */ int testPrngInit(void); u32 testPrngValue(u32 iVal); void testPrngArray(u32 iVal, u32 *aOut, int nOut); @@ -121,11 +108,10 @@ void testClose(TestDb **ppDb); void testFetch(TestDb *, void *, int, void *, int, int *); void testWrite(TestDb *, void *, int, void *, int, int *); void testDelete(TestDb *, void *, int, int *); -void testDeleteRange(TestDb *, void *, int, void *, int, int *); void testWriteStr(TestDb *, const char *, const char *zVal, int *pRc); void testFetchStr(TestDb *, const char *, const char *, int *pRc); void testBegin(TestDb *pDb, int iTrans, int *pRc); void testCommit(TestDb *pDb, int iTrans, int *pRc); @@ -212,18 +198,14 @@ /* test1.c */ void test_data_1(const char *, const char *, int *pRc); void test_data_2(const char *, const char *, int *pRc); -void test_data_3(const char *, const char *, int *pRc); void testDbContents(TestDb *, Datasource *, int, int, int, int, int, int *); void testCaseProgress(int, int, int, int *); int testCaseNDot(void); -void testCompareDb(Datasource *, int, int, TestDb *, TestDb *, int *); -int testControlDb(TestDb **ppDb); - typedef struct CksumDb CksumDb; CksumDb *testCksumArrayNew(Datasource *, int, int, int); char *testCksumArrayGet(CksumDb *, int); void testCksumArrayFree(CksumDb *); void testCaseStart(int *pRc, char *zFmt, ...); @@ -235,13 +217,10 @@ int testCksumDatabase(TestDb *pDb, char *zOut); int testCountDatabase(TestDb *pDb); void testCompareInt(int, int, int *); void testCompareStr(const char *z1, const char *z2, int *pRc); -/* lsmtest9.c */ -void test_data_4(const char *, const char *, int *pRc); - /* ** Similar to the Tcl_GetIndexFromObjStruct() Tcl library function. */ #define testArgSelect(w,x,y,z) testArgSelectX(w,x,sizeof(w[0]),y,z) Index: lsm-test/lsmtest1.c ================================================================== --- lsm-test/lsmtest1.c +++ lsm-test/lsmtest1.c @@ -90,11 +90,11 @@ ); testFree(zData); return zRet; } -int testControlDb(TestDb **ppDb){ +static int testControlDb(TestDb **ppDb){ #ifdef HAVE_KYOTOCABINET return tdb_open("kyotocabinet", "tmp.db", 1, ppDb); #else return tdb_open("sqlite3", ":memory:", 1, ppDb); #endif @@ -161,11 +161,11 @@ int *pRc /* IN/OUT: Error code */ ){ int j; int rc = *pRc; - if( 0 && rc==0 && nScanTest ){ + if( rc==0 && nScanTest ){ TestDb *pDb2 = 0; /* Open a control db (i.e. one that we assume works) */ rc = testControlDb(&pDb2); @@ -346,11 +346,11 @@ } testFree(zName); } } -void testCompareDb( +static void testCompareDb( Datasource *pData, int nData, int iSeed, TestDb *pControl, TestDb *pDb, @@ -407,11 +407,11 @@ pData = testDatasourceNew(&p->defn); rc = testControlDb(&pControl); if( tdb_lsm(pDb) ){ int nBuf = 32 * 1024 * 1024; - lsm_config(tdb_lsm(pDb), LSM_CONFIG_AUTOFLUSH, &nBuf); + lsm_config(tdb_lsm(pDb), LSM_CONFIG_WRITE_BUFFER, &nBuf); } for(i=0; rc==0 && inIter; i++){ void *pKey1; int nKey1; void *pKey2; int nKey2; @@ -483,139 +483,6 @@ doDataTest2(zSystem, &aTest[i], pRc); } testFree(zName); } } - -/************************************************************************* -** Test case data3.* -*/ - -typedef struct Datatest3 Datatest3; -struct Datatest3 { - int nRange; /* Keys are between 1 and this value, incl. */ - int nIter; /* Number of iterations */ - int nWrite; /* Number of writes per iteration */ - int nDelete; /* Number of deletes per iteration */ - - int nValMin; /* Minimum value size for writes */ - int nValMax; /* Maximum value size for writes */ -}; - -void testPutU32(u8 *aBuf, u32 iVal){ - aBuf[0] = (iVal >> 24) & 0xFF; - aBuf[1] = (iVal >> 16) & 0xFF; - aBuf[2] = (iVal >> 8) & 0xFF; - aBuf[3] = (iVal >> 0) & 0xFF; -} - -void dt3PutKey(u8 *aBuf, int iKey){ - assert( iKey<100000 && iKey>=0 ); - sprintf((char *)aBuf, "%.5d", iKey); -} - -static void doDataTest3( - const char *zSystem, /* Database system to test */ - Datatest3 *p, /* Structure containing test parameters */ - int *pRc /* OUT: Error code */ -){ - int iDot = 0; - int rc = *pRc; - TestDb *pDb; - u8 *abPresent; /* Array of boolean */ - char *aVal; /* Buffer to hold values */ - int i; - u32 iSeq = 10; /* prng counter */ - - abPresent = (u8 *)testMalloc(p->nRange+1); - aVal = (char *)testMalloc(p->nValMax+1); - pDb = testOpen(zSystem, 1, &rc); - - for(i=0; inIter && rc==0; i++){ - int ii; - - testCaseProgress(i, p->nIter, testCaseNDot(), &iDot); - - /* Perform nWrite inserts */ - for(ii=0; iinWrite; ii++){ - u8 aKey[6]; - u32 iKey; - int nVal; - - iKey = (testPrngValue(iSeq++) % p->nRange) + 1; - nVal = (testPrngValue(iSeq++) % (p->nValMax - p->nValMin)) + p->nValMin; - testPrngString(testPrngValue(iSeq++), aVal, nVal); - dt3PutKey(aKey, iKey); - - testWrite(pDb, aKey, sizeof(aKey)-1, aVal, nVal, &rc); - abPresent[iKey] = 1; - } - - /* Perform nDelete deletes */ - for(ii=0; iinDelete; ii++){ - u8 aKey1[6]; - u8 aKey2[6]; - u32 iKey; - - iKey = (testPrngValue(iSeq++) % p->nRange) + 1; - dt3PutKey(aKey1, iKey-1); - dt3PutKey(aKey2, iKey+1); - - testDeleteRange(pDb, aKey1, sizeof(aKey1)-1, aKey2, sizeof(aKey2)-1, &rc); - abPresent[iKey] = 0; - } - - testReopen(&pDb, &rc); - - for(ii=1; rc==0 && ii<=p->nRange; ii++){ - int nDbVal; - void *pDbVal; - u8 aKey[6]; - int dbrc; - - dt3PutKey(aKey, ii); - dbrc = tdb_fetch(pDb, aKey, sizeof(aKey)-1, &pDbVal, &nDbVal); - testCompareInt(0, dbrc, &rc); - - if( abPresent[ii] ){ - testCompareInt(1, (nDbVal>0), &rc); - }else{ - testCompareInt(1, (nDbVal<0), &rc); - } - } - } - - testClose(&pDb); - testCaseFinish(rc); - *pRc = rc; -} - -static char *getName3(const char *zSystem, Datatest3 *p){ - return testMallocPrintf("data3.%s.%d.%d.%d.%d.(%d..%d)", - zSystem, p->nRange, p->nIter, p->nWrite, p->nDelete, - p->nValMin, p->nValMax - ); -} - -void test_data_3( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - Datatest3 aTest[] = { - /* nRange, nIter, nWrite, nDelete, nValMin, nValMax */ - { 100, 1000, 5, 5, 50, 100 }, - { 100, 1000, 2, 2, 5, 10 }, - }; - - int i; - - for(i=0; *pRc==LSM_OK && inRec / 3); - int iData = 0; - - /* Start the test case, open a database and allocate the datasource. */ - rc = testControlDb(&pControl); - pDb = testOpen(zSystem, 1, &rc); - pData = testDatasourceNew(&p->defn); - if( rc==0 ) db = tdb_lsm(pDb); - - testWriteDatasourceRange(pControl, pData, iData, nRecOn3*3, &rc); - testWriteDatasourceRange(pDb, pData, iData, nRecOn3*3, &rc); - - for(i=0; rc==0 && inRepeat; i++){ - - testDeleteDatasourceRange(pControl, pData, iData, nRecOn3*2, &rc); - testDeleteDatasourceRange(pDb, pData, iData, nRecOn3*2, &rc); - - if( db ){ - int nDone; -#if 0 - fprintf(stderr, "lsm_work() start...\n"); fflush(stderr); -#endif - do { - nDone = 0; - rc = lsm_work(db, 1, (1<<30), &nDone); - }while( rc==0 && nDone>0 ); -#if 0 - fprintf(stderr, "lsm_work() done...\n"); fflush(stderr); -#endif - } - -if( i+1nRepeat ){ - iData += (nRecOn3*2); - testWriteDatasourceRange(pControl, pData, iData+nRecOn3, nRecOn3*2, &rc); - testWriteDatasourceRange(pDb, pData, iData+nRecOn3, nRecOn3*2, &rc); - - testCompareDb(pData, nRecOn3*3, iData, pControl, pDb, &rc); - - /* If Datatest4.bReopen is true, close and reopen the database */ - if( p->bReopen ){ - testReopen(&pDb, &rc); - if( rc==0 ) db = tdb_lsm(pDb); - } -} - - /* Update the progress dots... */ - testCaseProgress(i, p->nRepeat, testCaseNDot(), &iDot); - } - - testClose(&pDb); - testClose(&pControl); - testDatasourceFree(pData); - testCaseFinish(rc); - *pRc = rc; -} - -static char *getName4(const char *zSystem, Datatest4 *pTest){ - char *zRet; - char *zData; - zData = testDatasourceName(&pTest->defn); - zRet = testMallocPrintf("data4.%s.%s.%d.%d.%d", - zSystem, zData, pTest->nRec, pTest->nRepeat, pTest->bReopen - ); - testFree(zData); - return zRet; -} - -void test_data_4( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - Datatest4 aTest[] = { - /* defn, nRec, nRepeat, bReopen */ - { {DATA_RANDOM, 20,25, 500,600}, 10000, 10, 0 }, - { {DATA_RANDOM, 20,25, 500,600}, 10000, 10, 1 }, - }; - - int i; - - for(i=0; *pRc==LSM_OK && i2 && strlen(azArg[0])>1 - && memcmp(azArg[0], "-config", strlen(azArg[0]))==0 - ){ - zConfig = azArg[1]; - iDb = 2; - } - if( nArg<(iDb+1) ) goto usage; - - if( nArg>(iDb+1) ){ - rc = testArgSelect(aOpt, "option", azArg[iDb+1], &eOpt); - if( rc!=0 ) return rc; - bConfig = aOpt[eOpt].bConfig; - eOpt = aOpt[eOpt].eOpt; - if( (bConfig==0 && eOpt==LSM_INFO_FREELIST) - || (bConfig==1 && eOpt==LSM_CONFIG_BLOCK_SIZE) - || (bConfig==1 && eOpt==LSM_CONFIG_PAGE_SIZE) - ){ - if( nArg!=(iDb+2) ) goto usage; - }else{ - if( nArg!=(iDb+3) ) goto usage; - iPg = atoi(azArg[iDb+2]); - } - } - zDb = azArg[iDb]; - - rc = lsm_new(0, &pDb); - tdb_lsm_configure(pDb, zConfig); + char *z = 0; + + if( nArg<1 || nArg>3 ) goto usage; + + if( nArg>1 ){ + rc = testArgSelect(aOpt, "option", azArg[1], &eOpt); + if( rc!=0 ) return rc; + eOpt = aOpt[eOpt].eOpt; + if( eOpt==LSM_INFO_FREELIST ){ + if( nArg!=2 ) goto usage; + }else{ + if( nArg!=3 ) goto usage; + iPg = atoi(azArg[2]); + } + } + zDb = azArg[0]; + + rc = lsm_new(0, &pDb); if( rc!=LSM_OK ){ testPrintError("lsm_new(): rc=%d\n", rc); }else{ rc = lsm_open(pDb, zDb); if( rc!=LSM_OK ){ @@ -137,41 +111,33 @@ testPrintError("lsm_open(): rc=%d\n", rc); } } if( rc==LSM_OK ){ - if( bConfig==0 ){ - switch( eOpt ){ - case LSM_INFO_DB_STRUCTURE: - case LSM_INFO_FREELIST: - rc = lsm_info(pDb, eOpt, &z); - break; - case LSM_INFO_ARRAY_STRUCTURE: - case LSM_INFO_ARRAY_PAGES: - case LSM_INFO_PAGE_ASCII_DUMP: - case LSM_INFO_PAGE_HEX_DUMP: - rc = lsm_info(pDb, eOpt, iPg, &z); - break; - default: - assert( !"no chance" ); - } - - if( rc==LSM_OK ){ - printf("%s\n", z ? z : ""); - fflush(stdout); - } - lsm_free(lsm_get_env(pDb), z); - }else{ - int iRes = -1; - lsm_config(pDb, eOpt, &iRes); - printf("%d\n", iRes); - fflush(stdout); - } + switch( eOpt ){ + case LSM_INFO_DB_STRUCTURE: + case LSM_INFO_FREELIST: + rc = lsm_info(pDb, eOpt, &z); + break; + case LSM_INFO_ARRAY_STRUCTURE: + case LSM_INFO_PAGE_ASCII_DUMP: + case LSM_INFO_PAGE_HEX_DUMP: + rc = lsm_info(pDb, eOpt, iPg, &z); + break; + default: + assert( !"no chance" ); + } + + if( rc==LSM_OK ){ + printf("%s\n", z ? z : ""); + fflush(stdout); + } + lsm_free(lsm_get_env(pDb), z); } lsm_close(pDb); return rc; usage: testPrintUsage("DATABASE ?array|page-ascii|page-hex PGNO?"); return -1; } Index: lsm-test/lsmtest_main.c ================================================================== --- lsm-test/lsmtest_main.c +++ lsm-test/lsmtest_main.c @@ -460,12 +460,10 @@ for(j=0; tdb_system_name(j); j++){ rc = 0; test_data_1(tdb_system_name(j), zPattern, &rc); test_data_2(tdb_system_name(j), zPattern, &rc); - test_data_3(tdb_system_name(j), zPattern, &rc); - test_data_4(tdb_system_name(j), zPattern, &rc); test_rollback(tdb_system_name(j), zPattern, &rc); test_mc(tdb_system_name(j), zPattern, &rc); test_mt(tdb_system_name(j), zPattern, &rc); if( rc ) nFail++; @@ -492,55 +490,15 @@ static lsm_db *configure_lsm_db(TestDb *pDb){ lsm_db *pLsm; pLsm = tdb_lsm(pDb); if( pLsm ){ - tdb_lsm_config_str(pDb, "mmap=1 autowork=1 automerge=4 worker_automerge=4"); + tdb_lsm_config_str(pDb, "mmap=1 autowork=1 nmerge=4 worker_nmerge=4"); } return pLsm; } -typedef struct WriteHookEvent WriteHookEvent; -struct WriteHookEvent { - i64 iOff; - int nData; - int nUs; -}; -WriteHookEvent prev = {0, 0, 0}; - -static void flushPrev(FILE *pOut){ - if( prev.nData ){ - fprintf(pOut, "w %s %lld %d %d\n", "d", prev.iOff, prev.nData, prev.nUs); - prev.nData = 0; - } -} - -static void do_speed_write_hook2( - void *pCtx, - int bLog, - i64 iOff, - int nData, - int nUs -){ - FILE *pOut = (FILE *)pCtx; - if( bLog ) return; - - if( prev.nData && nData && iOff==prev.iOff+prev.nData ){ - prev.nData += nData; - prev.nUs += nUs; - }else{ - flushPrev(pOut); - if( nData==0 ){ - fprintf(pOut, "s %s 0 0 %d\n", (bLog ? "l" : "d"), nUs); - }else{ - prev.iOff = iOff; - prev.nData = nData; - prev.nUs = nUs; - } - } -} - #define ST_REPEAT 0 #define ST_WRITE 1 #define ST_PAUSE 2 #define ST_FETCH 3 #define ST_SCAN 4 @@ -599,11 +557,10 @@ TestDb *pDb; Datasource *pData; DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 0, 0, 0, 0 }; char *zSystem = ""; int bLsm = 1; - FILE *pLog = 0; #ifdef NDEBUG /* If NDEBUG is defined, disable the dynamic memory related checks in ** lsmtest_mem.c. They slow things down. */ testMallocUninstall(tdb_lsm_env()); @@ -669,15 +626,10 @@ if( rc!=0 ) return rc; if( bReadonly ){ nContent = testCountDatabase(pDb); } -#if 0 - pLog = fopen("/tmp/speed.log", "w"); - tdb_lsm_write_hook(pDb, do_speed_write_hook2, (void *)pLog); -#endif - for(i=0; ipMethods = &KcdbMethods; - *ppDb = pTestDb; - return 0; -} -#endif /* HAVE_MDB */ - /************************************************************************* ** Begin wrapper for SQLite. */ /* @@ -702,14 +623,11 @@ #endif #ifdef HAVE_LEVELDB { "leveldb", "testdb.leveldb", test_leveldb_open }, #endif #ifdef HAVE_KYOTOCABINET - { "kyotocabinet", "testdb.kc", kc_open }, -#endif -#ifdef HAVE_MDB - { "mdb", "./testdb.mdb", mdb_open } + { "kyotocabinet", "testdb.kc", kc_open } #endif }; const char *tdb_system_name(int i){ if( i<0 || i>=ArraySize(aLib) ) return 0; Index: lsm-test/lsmtest_tdb2.cc ================================================================== --- lsm-test/lsmtest_tdb2.cc +++ lsm-test/lsmtest_tdb2.cc @@ -1,9 +1,155 @@ #include "lsmtest.h" #include + +#ifdef HAVE_LEVELDB +#include "leveldb/db.h" +extern "C" { + struct LevelDb { + TestDb base; + leveldb::DB* db; + std::string *pRes; + }; +} + +int test_leveldb_open(const char *zFilename, int bClear, TestDb **ppDb){ + LevelDb *pLevelDb; + + if( bClear ){ + char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); + system(zCmd); + sqlite3_free(zCmd); + } + + pLevelDb = (LevelDb *)malloc(sizeof(LevelDb)); + memset(pLevelDb, 0, sizeof(LevelDb)); + + leveldb::Options options; + options.create_if_missing = 1; + leveldb::Status s = leveldb::DB::Open(options, zFilename, &pLevelDb->db); + + if( s.ok() ){ + *ppDb = (TestDb *)pLevelDb; + pLevelDb->pRes = new std::string(""); + }else{ + test_leveldb_close((TestDb *)pLevelDb); + *ppDb = 0; + return 1; + } + return 0; +} + +int test_leveldb_close(TestDb *pDb){ + LevelDb *pLevelDb = (LevelDb *)pDb; + if( pDb ){ + if( pLevelDb->db ) delete pLevelDb->db; + } + free((void *)pDb); + return 0; +} + +int test_leveldb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ + LevelDb *pLevelDb = (LevelDb *)pDb; + leveldb::Status s = pLevelDb->db->Put(leveldb::WriteOptions(), + leveldb::Slice((const char *)pKey, nKey), + leveldb::Slice((const char *)pVal, nVal) + ); + return !s.ok(); +} + +int test_leveldb_delete(TestDb *pDb, void *pKey, int nKey){ + LevelDb *pLevelDb = (LevelDb *)pDb; + leveldb::Status s = pLevelDb->db->Delete(leveldb::WriteOptions(), + leveldb::Slice((const char *)pKey, nKey) + ); + return !s.ok(); +} + +int test_leveldb_fetch( + TestDb *pDb, + void *pKey, + int nKey, + void **ppVal, + int *pnVal +){ + LevelDb *pLevelDb = (LevelDb *)pDb; + pLevelDb->pRes->assign(""); + leveldb::Status s = pLevelDb->db->Get( + leveldb::ReadOptions(), + leveldb::Slice((const char *)pKey, nKey), + pLevelDb->pRes + ); + if( s.ok() ){ + *ppVal = (void *)pLevelDb->pRes->data(); + *pnVal = (int)pLevelDb->pRes->length(); + }else{ + *pnVal = -1; + *ppVal = 0; + } + return 0; +} + +int test_leveldb_scan( + TestDb *pDb, /* Database handle */ + void *pCtx, /* Context pointer to pass to xCallback */ + int bReverse, /* True to iterate in reverse order */ + void *pKey1, int nKey1, /* Start of search */ + void *pKey2, int nKey2, /* End of search */ + void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) +){ + LevelDb *pLevelDb = (LevelDb *)pDb; + leveldb::Iterator *pIter; + + leveldb::Slice sKey1((const char *)pKey1, nKey1); + leveldb::Slice sKey2((const char *)pKey2, nKey2); + + pIter = pLevelDb->db->NewIterator(leveldb::ReadOptions()); + if( bReverse==0 ){ + if( pKey1 ){ + pIter->Seek(sKey1); + }else{ + pIter->SeekToFirst(); + } + }else{ + if( pKey2 ){ + pIter->Seek(sKey2); + if( pIter->Valid()==0 ){ + pIter->SeekToLast(); + }else{ + int res = pIter->key().compare(sKey2); + assert( res>=0 ); + if( res>0 ){ + pIter->Prev(); + } + assert( pIter->Valid()==0 || pIter->key().compare(sKey2)<=0 ); + } + }else{ + pIter->SeekToLast(); + } + } + + while( pIter->Valid() ){ + if( (bReverse==0 && pKey2 && pIter->key().compare(sKey2)>0) ) break; + if( (bReverse!=0 && pKey1 && pIter->key().compare(sKey1)<0) ) break; + + leveldb::Slice k = pIter->key(); + leveldb::Slice v = pIter->value(); + xCallback(pCtx, (void *)k.data(), k.size(), (void *)v.data(), v.size()); + + if( bReverse==0 ){ + pIter->Next(); + }else{ + pIter->Prev(); + } + } + + delete pIter; + return LSM_OK; +} +#endif #ifdef HAVE_KYOTOCABINET #include "kcpolydb.h" extern "C" { struct KcDb { @@ -71,54 +217,10 @@ ok = pKcDb->db->remove((const char *)pKey, nKey); return (ok ? 0 : 1); } -int test_kc_delete_range( - TestDb *pDb, - void *pKey1, int nKey1, - void *pKey2, int nKey2 -){ - int res; - KcDb *pKcDb = (KcDb *)pDb; - kyotocabinet::DB::Cursor* pCur = pKcDb->db->cursor(); - - if( pKey1 ){ - res = pCur->jump((const char *)pKey1, nKey1); - }else{ - res = pCur->jump(); - } - - while( 1 ){ - const char *pKey; size_t nKey; - const char *pVal; size_t nVal; - - pKey = pCur->get(&nKey, &pVal, &nVal); - if( pKey==0 ) break; - -#ifndef NDEBUG - if( pKey1 ){ - res = memcmp(pKey, pKey1, MIN((size_t)nKey1, nKey)); - assert( res>0 || (res==0 && nKey>nKey1) ); - } -#endif - - if( pKey2 ){ - res = memcmp(pKey, pKey2, MIN((size_t)nKey2, nKey)); - if( res>0 || (res==0 && (size_t)nKey2remove(); - delete [] pKey; - } - - delete pCur; - return 0; -} - int test_kc_fetch( TestDb *pDb, void *pKey, int nKey, void **ppVal, @@ -200,166 +302,7 @@ } delete pCur; return 0; } -#endif /* HAVE_KYOTOCABINET */ - -#ifdef HAVE_MDB -#include "lmdb.h" - -extern "C" { - struct MdbDb { - TestDb base; - MDB_env *env; - MDB_dbi dbi; - }; -} - -int test_mdb_open(const char *zFilename, int bClear, TestDb **ppDb){ - MDB_txn *txn; - MdbDb *pMdb; - int rc; - - if( bClear ){ - char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); - system(zCmd); - sqlite3_free(zCmd); - } - - pMdb = (MdbDb *)malloc(sizeof(MdbDb)); - memset(pMdb, 0, sizeof(MdbDb)); - - rc = mdb_env_create(&pMdb->env); - if( rc==0 ) rc = mdb_env_set_mapsize(pMdb->env, 1*1024*1024*1024); - if( rc==0 ) rc = mdb_env_open(pMdb->env, zFilename, MDB_NOSYNC|MDB_NOSUBDIR, 0600); - if( rc==0 ) rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); - if( rc==0 ){ - rc = mdb_open(txn, NULL, 0, &pMdb->dbi); - mdb_txn_commit(txn); - } - - *ppDb = (TestDb *)pMdb; - return rc; -} - -int test_mdb_close(TestDb *pDb){ - MdbDb *pMdb = (MdbDb *)pDb; - - mdb_close(pMdb->env, pMdb->dbi); - mdb_env_close(pMdb->env); - free(pMdb); - return 0; -} - -int test_mdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ - int rc; - MdbDb *pMdb = (MdbDb *)pDb; - MDB_val val; - MDB_val key; - MDB_txn *txn; - - val.mv_size = nVal; - val.mv_data = pVal; - key.mv_size = nKey; - key.mv_data = pKey; - - rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); - if( rc==0 ){ - rc = mdb_put(txn, pMdb->dbi, &key, &val, 0); - if( rc==0 ){ - rc = mdb_txn_commit(txn); - }else{ - mdb_txn_abort(txn); - } - } - - return rc; -} - -int test_mdb_delete(TestDb *pDb, void *pKey, int nKey){ - int rc; - MdbDb *pMdb = (MdbDb *)pDb; - MDB_val key; - MDB_txn *txn; - - key.mv_size = nKey; - key.mv_data = pKey; - rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); - if( rc==0 ){ - rc = mdb_del(txn, pMdb->dbi, &key, 0); - if( rc==0 ){ - rc = mdb_txn_commit(txn); - }else{ - mdb_txn_abort(txn); - } - } - - return rc; -} - -int test_mdb_fetch( - TestDb *pDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - int rc; - MdbDb *pMdb = (MdbDb *)pDb; - MDB_val key; - MDB_txn *txn; - - key.mv_size = nKey; - key.mv_data = pKey; - - rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn); - if( rc==0 ){ - MDB_val val = {0, 0}; - rc = mdb_get(txn, pMdb->dbi, &key, &val); - if( rc==MDB_NOTFOUND ){ - rc = 0; - *ppVal = 0; - *pnVal = -1; - }else{ - *ppVal = val.mv_data; - *pnVal = val.mv_size; - } - mdb_txn_commit(txn); - } - - return rc; -} - -int test_mdb_scan( - TestDb *pDb, /* Database handle */ - void *pCtx, /* Context pointer to pass to xCallback */ - int bReverse, /* True for a reverse order scan */ - void *pKey1, int nKey1, /* Start of search */ - void *pKey2, int nKey2, /* End of search */ - void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) -){ - MdbDb *pMdb = (MdbDb *)pDb; - int rc; - MDB_cursor_op op = bReverse ? MDB_PREV : MDB_NEXT; - MDB_txn *txn; - - rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn); - if( rc==0 ){ - MDB_cursor *csr; - MDB_val key = {0, 0}; - MDB_val val = {0, 0}; - - rc = mdb_cursor_open(txn, pMdb->dbi, &csr); - if( rc==0 ){ - while( mdb_cursor_get(csr, &key, &val, op)==0 ){ - xCallback(pCtx, key.mv_data, key.mv_size, val.mv_data, val.mv_size); - } - mdb_cursor_close(csr); - } - } - - return rc; -} - -#endif /* HAVE_MDB */ +#endif Index: lsm-test/lsmtest_tdb3.c ================================================================== --- lsm-test/lsmtest_tdb3.c +++ lsm-test/lsmtest_tdb3.c @@ -1,8 +1,9 @@ #include "lsmtest_tdb.h" #include "lsm.h" + #include "lsmtest.h" #include #include #include @@ -13,41 +14,27 @@ typedef struct LsmDb LsmDb; typedef struct LsmWorker LsmWorker; typedef struct LsmFile LsmFile; -#define LSMTEST_DFLT_MT_MAX_CKPT (8*1024) -#define LSMTEST_DFLT_MT_MIN_CKPT (2*1024) - #ifdef LSM_MUTEX_PTHREADS #include - -#define LSMTEST_THREAD_CKPT 1 -#define LSMTEST_THREAD_WORKER 2 -#define LSMTEST_THREAD_WORKER_AC 3 - -/* -** There are several different types of worker threads that run in different -** test configurations, depending on the value of LsmWorker.eType. -** -** 1. Checkpointer. -** 2. Worker with auto-checkpoint. -** 3. Worker without auto-checkpoint. -*/ struct LsmWorker { LsmDb *pDb; /* Main database structure */ lsm_db *pWorker; /* Worker database handle */ pthread_t worker_thread; /* Worker thread */ pthread_cond_t worker_cond; /* Condition var the worker waits on */ pthread_mutex_t worker_mutex; /* Mutex used with worker_cond */ int bDoWork; /* Set to true by client when there is work */ int worker_rc; /* Store error code here */ - int eType; /* LSMTEST_THREAD_XXX constant */ - int bBlock; + + int lsm_work_flags; /* Flags to pass to lsm_work() */ + int lsm_work_npage; /* nPage parameter to pass to lsm_work() */ + int bCkpt; /* True to call lsm_checkpoint() */ }; #else -struct LsmWorker { int worker_rc; int bBlock; }; +struct LsmWorker { int worker_rc; }; #endif static void mt_shutdown(LsmDb *); lsm_env *tdb_lsm_env(void){ @@ -118,22 +105,14 @@ /* IO logging hook */ void (*xWriteHook)(void *, int, lsm_i64, int, int); void *pWriteCtx; /* Worker threads (for lsm_mt) */ - int nMtMinCkpt; - int nMtMaxCkpt; - int eMode; int nWorker; LsmWorker *aWorker; }; -#define LSMTEST_MODE_SINGLETHREAD 1 -#define LSMTEST_MODE_BACKGROUND_CKPT 2 -#define LSMTEST_MODE_BACKGROUND_WORK 3 -#define LSMTEST_MODE_BACKGROUND_BOTH 4 - /************************************************************************* ************************************************************************** ** Begin test VFS code. */ @@ -452,12 +431,12 @@ return (rc==Z_OK ? 0 : LSM_ERROR); } static int testConfigureCompression(lsm_db *pDb){ static lsm_compress zip = { + 1, sizeof(lsm_compress), 0, /* Context pointer (unused) */ - 1, /* Id value */ testZipBound, /* xBound method */ testZipCompress, /* xCompress method */ testZipUncompress /* xUncompress method */ }; return lsm_config(pDb, LSM_CONFIG_SET_COMPRESSION, &zip); @@ -496,75 +475,37 @@ testFree((char *)pDb->pBuf); testFree((char *)pDb); return rc; } -static int waitOnCheckpointer(LsmDb *pDb, lsm_db *db){ - int nSleep = 0; - int nKB; - int rc; - - do { - nKB = 0; - rc = lsm_info(db, LSM_INFO_CHECKPOINT_SIZE, &nKB); - if( rc!=LSM_OK || nKBnMtMaxCkpt ) break; - usleep(5000); - nSleep += 5; - }while( 1 ); - -#if 0 - if( nSleep ) printf("# waitOnCheckpointer(): nSleep=%d\n", nSleep); -#endif - - return rc; -} - -static int waitOnWorker(LsmDb *pDb){ - int rc; - int nLimit = -1; - int nSleep = 0; - - rc = lsm_config(pDb->db, LSM_CONFIG_AUTOFLUSH, &nLimit); - do { - int nOld, nNew, rc; - rc = lsm_info(pDb->db, LSM_INFO_TREE_SIZE, &nOld, &nNew); - if( rc!=LSM_OK ) return rc; - if( nOld==0 || nNew<(nLimit/2) ) break; - usleep(5000); - nSleep += 5; - }while( 1 ); - -#if 0 - if( nSleep ) printf("# waitOnWorker(): nSleep=%d\n", nSleep); -#endif - - return rc; -} - static int test_lsm_write( TestDb *pTestDb, void *pKey, int nKey, - void *pVal, + void *pVal, int nVal ){ LsmDb *pDb = (LsmDb *)pTestDb; - int rc = LSM_OK; - - if( pDb->eMode==LSMTEST_MODE_BACKGROUND_CKPT ){ - rc = waitOnCheckpointer(pDb, pDb->db); - }else if( - pDb->eMode==LSMTEST_MODE_BACKGROUND_WORK - || pDb->eMode==LSMTEST_MODE_BACKGROUND_BOTH - ){ - rc = waitOnWorker(pDb); - } - - if( rc==LSM_OK ){ - rc = lsm_insert(pDb->db, pKey, nKey, pVal, nVal); - } - return rc; + + if( pDb->aWorker ){ + int nLimit = -1; + int nSleep = 0; + lsm_config(pDb->db, LSM_CONFIG_WRITE_BUFFER, &nLimit); + do { + int bOld, nNew, rc; + rc = lsm_tree_size(pDb->db, &bOld, &nNew); + if( rc!=LSM_OK ) return rc; + if( bOld==0 || nNewdb, pKey, nKey, pVal, nVal); } static int test_lsm_delete(TestDb *pTestDb, void *pKey, int nKey){ LsmDb *pDb = (LsmDb *)pTestDb; return lsm_delete(pDb->db, pKey, nKey); @@ -729,17 +670,14 @@ LsmDb *p = (LsmDb *)pArg; if( p->xWork ) p->xWork(db, p->pWorkCtx); } #define TEST_NO_RECOVERY -1 +#define TEST_THREADS -2 #define TEST_COMPRESSION -3 -#define TEST_MT_MODE -2 -#define TEST_MT_MIN_CKPT -4 -#define TEST_MT_MAX_CKPT -5 - -int test_lsm_config_str( +static int test_lsm_config_str( LsmDb *pLsm, lsm_db *db, int bWorker, const char *zStr, int *pnThread @@ -747,29 +685,25 @@ struct CfgParam { const char *zParam; int bWorker; int eParam; } aParam[] = { - { "autoflush", 0, LSM_CONFIG_AUTOFLUSH }, + { "write_buffer", 0, LSM_CONFIG_WRITE_BUFFER }, { "page_size", 0, LSM_CONFIG_PAGE_SIZE }, { "block_size", 0, LSM_CONFIG_BLOCK_SIZE }, { "safety", 0, LSM_CONFIG_SAFETY }, { "autowork", 0, LSM_CONFIG_AUTOWORK }, { "autocheckpoint", 0, LSM_CONFIG_AUTOCHECKPOINT }, + { "log_size", 0, LSM_CONFIG_LOG_SIZE }, { "mmap", 0, LSM_CONFIG_MMAP }, { "use_log", 0, LSM_CONFIG_USE_LOG }, - { "automerge", 0, LSM_CONFIG_AUTOMERGE }, + { "nmerge", 0, LSM_CONFIG_NMERGE }, { "max_freelist", 0, LSM_CONFIG_MAX_FREELIST }, { "multi_proc", 0, LSM_CONFIG_MULTIPLE_PROCESSES }, - { "worker_automerge", 1, LSM_CONFIG_AUTOMERGE }, + { "worker_nmerge", 1, LSM_CONFIG_NMERGE }, { "test_no_recovery", 0, TEST_NO_RECOVERY }, - { "bg_min_ckpt", 0, TEST_NO_RECOVERY }, - - { "mt_mode", 0, TEST_MT_MODE }, - { "mt_min_ckpt", 0, TEST_MT_MIN_CKPT }, - { "mt_max_ckpt", 0, TEST_MT_MAX_CKPT }, - + { "threads", 0, TEST_THREADS }, #ifdef HAVE_ZLIB { "compression", 0, TEST_COMPRESSION }, #endif { 0, 0 } }; @@ -787,11 +721,10 @@ while( *z && *z!='=' ) z++; if( *z ){ int eParam; int i; int iVal; - int iMul = 1; int rc; char zParam[32]; int nParam = z-zStart; if( nParam==0 || nParam>sizeof(zParam)-1 ) goto syntax_error; @@ -802,58 +735,43 @@ eParam = aParam[i].eParam; z++; zStart = z; while( *z>='0' && *z<='9' ) z++; - if( *z=='k' || *z=='K' ){ - iMul = 1024; - z++; - }else if( *z=='M' || *z=='M' ){ - iMul = 1024 * 1024; - z++; - } nParam = z-zStart; if( nParam==0 || nParam>sizeof(zParam)-1 ) goto syntax_error; memcpy(zParam, zStart, nParam); zParam[nParam] = '\0'; - iVal = atoi(zParam) * iMul; + iVal = atoi(zParam); if( eParam>0 ){ if( bWorker || aParam[i].bWorker==0 ){ lsm_config(db, eParam, &iVal); } }else{ - switch( eParam ){ - case TEST_NO_RECOVERY: - if( pLsm ) pLsm->bNoRecovery = iVal; - break; - case TEST_MT_MODE: - if( pLsm ) nThread = iVal; - break; - case TEST_MT_MIN_CKPT: - if( pLsm && iVal>0 ) pLsm->nMtMinCkpt = iVal; - break; - case TEST_MT_MAX_CKPT: - if( pLsm && iVal>0 ) pLsm->nMtMaxCkpt = iVal; - break; + if( pLsm ){ + switch( eParam ){ + case TEST_NO_RECOVERY: + pLsm->bNoRecovery = iVal; + break; + case TEST_THREADS: + nThread = iVal; + break; #ifdef HAVE_ZLIB - case TEST_COMPRESSION: - testConfigureCompression(db); - break; + case TEST_COMPRESSION: + testConfigureCompression(db); + break; #endif + } } } }else if( z!=zStart ){ goto syntax_error; } } if( pnThread ) *pnThread = nThread; - if( pLsm && pLsm->nMtMaxCkpt < pLsm->nMtMinCkpt ){ - pLsm->nMtMinCkpt = pLsm->nMtMaxCkpt; - } - return 0; syntax_error: testPrintError("syntax error at: \"%s\"\n", z); return 1; } @@ -872,14 +790,10 @@ #endif } return rc; } -int tdb_lsm_configure(lsm_db *db, const char *zConfig){ - return test_lsm_config_str(0, db, 0, zConfig, 0); -} - static int testLsmStartWorkers(LsmDb *, int, const char *, const char *); static int testLsmOpen( const char *zCfg, const char *zFilename, @@ -918,14 +832,10 @@ ** there is in SQLite. For now, LSM assumes that it is smaller than ** the page size (default 4KB). */ pDb->szSector = 256; - /* Default values for the mt_min_ckpt and mt_max_ckpt parameters. */ - pDb->nMtMinCkpt = LSMTEST_DFLT_MT_MIN_CKPT; - pDb->nMtMaxCkpt = LSMTEST_DFLT_MT_MAX_CKPT; - memcpy(&pDb->env, tdb_lsm_env(), sizeof(lsm_env)); pDb->env.pVfsCtx = (void *)pDb; pDb->env.xFullpath = testEnvFullpath; pDb->env.xOpen = testEnvOpen; pDb->env.xRead = testEnvRead; @@ -950,14 +860,13 @@ lsm_config_work_hook(pDb->db, xWorkHook, (void *)pDb); rc = test_lsm_config_str(pDb, pDb->db, 0, zCfg, &nThread); if( rc==LSM_OK ) rc = lsm_open(pDb->db, zFilename); - pDb->eMode = nThread; #ifdef LSM_MUTEX_PTHREADS - if( rc==LSM_OK && nThread>1 ){ - testLsmStartWorkers(pDb, nThread, zFilename, zCfg); + if( rc==LSM_OK && (nThread==2 || nThread==3) ){ + testLsmStartWorkers(pDb, nThread-1, zFilename, zCfg); } #endif if( rc!=LSM_OK ){ test_lsm_close((TestDb *)pDb); @@ -976,23 +885,22 @@ int test_lsm_small_open( const char *zFile, int bClear, TestDb **ppDb ){ - const char *zCfg = "page_size=256 block_size=64"; + const char *zCfg = "page_size=256 block_size=65536"; return testLsmOpen(zCfg, zFile, bClear, ppDb); } int test_lsm_lomem_open( const char *zFilename, int bClear, TestDb **ppDb ){ - /* "max_freelist=4 autocheckpoint=32" */ const char *zCfg = - "page_size=256 block_size=64 autoflush=16 " - "autocheckpoint=32" + "page_size=256 block_size=65536 write_buffer=16384 " + "max_freelist=4 autocheckpoint=32768 " "mmap=0 " ; return testLsmOpen(zCfg, zFilename, bClear, ppDb); } @@ -1000,12 +908,13 @@ const char *zFilename, int bClear, TestDb **ppDb ){ const char *zCfg = - "page_size=256 block_size=64 autoflush=16 " - "autocheckpoint=32 compression=1 mmap=0 " + "page_size=256 block_size=65536 write_buffer=16384 " + "max_freelist=4 autocheckpoint=32768 compression=1" + "mmap=0 " ; return testLsmOpen(zCfg, zFilename, bClear, ppDb); } lsm_db *tdb_lsm(TestDb *pDb){ @@ -1016,10 +925,12 @@ } void tdb_lsm_enable_log(TestDb *pDb, int bEnable){ lsm_db *db = tdb_lsm(pDb); if( db ){ + LsmDb *p = (LsmDb *)pDb; + int i; lsm_config_log(db, (bEnable ? xLog : 0), (void *)"client"); } } void tdb_lsm_application_crash(TestDb *pDb){ @@ -1104,13 +1015,10 @@ p->bDoWork = 1; pthread_cond_signal(&p->worker_cond); pthread_mutex_unlock(&p->worker_mutex); } -/* -** This routine is used as the main() for all worker threads. -*/ static void *worker_main(void *pArg){ LsmWorker *p = (LsmWorker *)pArg; lsm_db *pWorker; /* Connection to access db through */ pthread_mutex_lock(&p->worker_mutex); @@ -1117,43 +1025,49 @@ while( (pWorker = p->pWorker) ){ int rc = LSM_OK; int nCkpt = -1; /* Do some work. If an error occurs, exit. */ - pthread_mutex_unlock(&p->worker_mutex); - if( p->eType==LSMTEST_THREAD_CKPT ){ - int nKB = 0; - rc = lsm_info(pWorker, LSM_INFO_CHECKPOINT_SIZE, &nKB); - if( rc==LSM_OK && nKB>=p->pDb->nMtMinCkpt ){ - rc = lsm_checkpoint(pWorker, 0); - } + + if( p->bCkpt ){ + rc = lsm_checkpoint(pWorker, 0); }else{ - int nWrite; + int nWrite = 0; /* Pages written by lsm_work() call */ + int nAuto = -1; /* Configured AUTOCHECKPOINT value */ + int nLimit = -1; /* Configured WRITE_BUFFER value */ + + lsm_config(pWorker, LSM_CONFIG_WRITE_BUFFER, &nLimit); + lsm_config(pWorker, LSM_CONFIG_AUTOCHECKPOINT, &nAuto); do { - - if( p->eType==LSMTEST_THREAD_WORKER ){ - waitOnCheckpointer(p->pDb, pWorker); - } - - nWrite = 0; - rc = lsm_work(pWorker, 0, 256, &nWrite); - - if( p->eType==LSMTEST_THREAD_WORKER && nWrite ){ - mt_signal_worker(p->pDb, 1); - } + int nSleep = 0; + lsm_ckpt_size(pWorker, &nCkpt); + while( nAuto==0 && nCkpt>(nLimit*4) ){ + usleep(1000); + mt_signal_worker(p->pDb, 1); + nSleep++; + lsm_ckpt_size(pWorker, &nCkpt); + } +#if 0 + if( nSleep ) printf("nLimit=%d nSleep=%d (worker)\n", nLimit, nSleep); +#endif + + rc = lsm_work(pWorker, p->lsm_work_flags, p->lsm_work_npage, &nWrite); + if( nAuto==0 && nWrite && rc==LSM_OK ) mt_signal_worker(p->pDb, 1); }while( nWrite && p->pWorker ); } pthread_mutex_lock(&p->worker_mutex); if( rc!=LSM_OK && rc!=LSM_BUSY ){ p->worker_rc = rc; break; } - /* The thread will wake up when it is signaled either because another - ** thread has created some work for this one or because the connection + /* If the call to lsm_work() indicates that there is nothing more + ** to do at this point, wait on the condition variable. The thread will + ** wake up when it is signaled either because the client thread has + ** flushed an in-memory tree into the db file or when the connection ** is being closed. */ if( p->pWorker && p->bDoWork==0 ){ pthread_cond_wait(&p->worker_cond, &p->worker_mutex); } p->bDoWork = 0; @@ -1200,15 +1114,16 @@ ** This implies there may be work to do on the database, so signal ** the worker threads. */ static void mt_client_work_hook(lsm_db *db, void *pArg){ LsmDb *pDb = (LsmDb *)pArg; /* LsmDb database handle */ + int i; /* Iterator variable */ /* Invoke the user level work-hook, if any. */ if( pDb->xWork ) pDb->xWork(db, pDb->pWorkCtx); - /* Wake up worker thread 0. */ + /* Signal the lsm_work() thread */ mt_signal_worker(pDb, 0); } static void mt_worker_work_hook(lsm_db *db, void *pArg){ LsmDb *pDb = (LsmDb *)pArg; /* LsmDb database handle */ @@ -1222,24 +1137,24 @@ */ static int mt_start_worker( LsmDb *pDb, /* Main database structure */ int iWorker, /* Worker number to start */ const char *zFilename, /* File name of database to open */ - const char *zCfg, /* Connection configuration string */ - int eType /* Type of worker thread */ + const char *zCfg, + int flags, /* flags parameter to lsm_work() */ + int nPage, /* nPage parameter to lsm_work() */ + int bCkpt /* True to call lsm_checkpoint() */ ){ int rc = 0; /* Return code */ LsmWorker *p; /* Object to initialize */ assert( iWorkernWorker ); - assert( eType==LSMTEST_THREAD_CKPT - || eType==LSMTEST_THREAD_WORKER - || eType==LSMTEST_THREAD_WORKER_AC - ); p = &pDb->aWorker[iWorker]; - p->eType = eType; + p->lsm_work_flags = flags; + p->lsm_work_npage = nPage; + p->bCkpt = bCkpt; p->pDb = pDb; /* Open the worker connection */ if( rc==0 ) rc = lsm_new(&pDb->env, &p->pWorker); if( zCfg ){ @@ -1251,14 +1166,10 @@ /* Configure the work-hook */ if( rc==0 ){ lsm_config_work_hook(p->pWorker, mt_worker_work_hook, (void *)pDb); } - if( eType==LSMTEST_THREAD_WORKER ){ - test_lsm_config_str(0, p->pWorker, 1, "autocheckpoint=0", 0); - } - /* Kick off the worker thread. */ if( rc==0 ) rc = pthread_cond_init(&p->worker_cond, 0); if( rc==0 ) rc = pthread_mutex_init(&p->worker_mutex, 0); if( rc==0 ) rc = pthread_create(&p->worker_thread, 0, worker_main, (void *)p); @@ -1265,60 +1176,45 @@ return rc; } static int testLsmStartWorkers( - LsmDb *pDb, int eModel, const char *zFilename, const char *zCfg + LsmDb *pDb, int nWorker, const char *zFilename, const char *zCfg ){ int rc; - - if( eModel<1 || eModel>4 ) return 1; - if( eModel==1 ) return 0; + int bAutowork = 0; + assert( nWorker==1 || nWorker==2 ); - /* Configure a work-hook for the client connection. Worker 0 is signalled - ** every time the users connection writes to the database. */ + /* Configure a work-hook for the client connection. */ lsm_config_work_hook(pDb->db, mt_client_work_hook, (void *)pDb); - /* Allocate space for two worker connections. They may not both be - ** used, but both are allocated. */ - pDb->aWorker = (LsmWorker *)testMalloc(sizeof(LsmWorker) * 2); - memset(pDb->aWorker, 0, sizeof(LsmWorker) * 2); - - switch( eModel ){ - case LSMTEST_MODE_BACKGROUND_CKPT: - pDb->nWorker = 1; - test_lsm_config_str(0, pDb->db, 0, "autocheckpoint=0", 0); - rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_CKPT); - break; - - case LSMTEST_MODE_BACKGROUND_WORK: - pDb->nWorker = 1; - test_lsm_config_str(0, pDb->db, 0, "autowork=0", 0); - rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_WORKER_AC); - break; - - case LSMTEST_MODE_BACKGROUND_BOTH: - pDb->nWorker = 2; - test_lsm_config_str(0, pDb->db, 0, "autowork=0", 0); - rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_WORKER); - if( rc==0 ){ - rc = mt_start_worker(pDb, 1, zFilename, zCfg, LSMTEST_THREAD_CKPT); - } - break; + pDb->aWorker = (LsmWorker *)testMalloc(sizeof(LsmWorker) * nWorker); + memset(pDb->aWorker, 0, sizeof(LsmWorker) * nWorker); + pDb->nWorker = nWorker; + + if( nWorker==1 ){ + int flags = LSM_WORK_FLUSH; + rc = mt_start_worker(pDb, 0, zFilename, zCfg, flags, 256, 0); + }else{ + int flags = LSM_WORK_FLUSH; + rc = mt_start_worker(pDb, 0, zFilename, zCfg, flags, 256, 0); + if( rc==LSM_OK ){ + rc = mt_start_worker(pDb, 1, zFilename, zCfg, 0, 0, 1); + } } return rc; } int test_lsm_mt2(const char *zFilename, int bClear, TestDb **ppDb){ - const char *zCfg = "mt_mode=2"; + const char *zCfg = "threads=2"; return testLsmOpen(zCfg, zFilename, bClear, ppDb); } int test_lsm_mt3(const char *zFilename, int bClear, TestDb **ppDb){ - const char *zCfg = "mt_mode=4"; + const char *zCfg = "threads=3"; return testLsmOpen(zCfg, zFilename, bClear, ppDb); } #else static void mt_shutdown(LsmDb *pDb) { Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -62,31 +62,36 @@ # TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 TCCX += -I$(TOP)/ext/async -TCPPX = g++ -Wall -g -I. -I$(TOP)/src $(OPTS) +# Object files for the SQLite library. +# +FTS3_OBJ = fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ + fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o \ + fts3_write.o +# To remove fts3 (if it won't compile for you), unset FTS3_OBJ: +# FTS3_OBJ = -LIBOBJ+= vdbe.o parse.o \ - alter.o analyze.o attach.o auth.o \ +LIBOBJ+= alter.o analyze.o attach.o auth.o \ build.o \ callback.o complete.o ctime.o date.o delete.o expr.o fault.o fkey.o \ - fts5.o fts5func.o \ + $(FTS3_OBJ) \ func.o global.o hash.o \ - icu.o insert.o kv.o kvlsm.o kvmem.o legacy.o \ + icu.o insert.o kvlsm.o kvmem.o legacy.o \ lsm_ckpt.o lsm_file.o lsm_log.o lsm_main.o lsm_mem.o lsm_mutex.o \ lsm_shared.o lsm_str.o lsm_sorted.o lsm_tree.o \ lsm_unix.o lsm_varint.o \ main.o malloc.o math.o mem0.o mem1.o mem2.o mem3.o mem5.o \ mutex.o mutex_noop.o mutex_unix.o mutex_w32.o \ opcodes.o os.o \ - pragma.o prepare.o printf.o \ - random.o resolve.o rowset.o rtree.o select.o status.o \ + parse.o pragma.o prepare.o printf.o \ + random.o resolve.o rowset.o rtree.o select.o status.o storage.o \ tokenize.o trigger.o \ update.o util.o varint.o \ - vdbeapi.o vdbeaux.o vdbecodec.o vdbecursor.o \ + vdbe.o vdbeapi.o vdbeaux.o vdbecodec.o vdbecursor.o \ vdbemem.o vdbetrace.o \ walker.o where.o utf.o # All of the source code files. # @@ -102,20 +107,16 @@ $(TOP)/src/date.c \ $(TOP)/src/delete.c \ $(TOP)/src/expr.c \ $(TOP)/src/fault.c \ $(TOP)/src/fkey.c \ - $(TOP)/src/fts5.c \ - $(TOP)/src/fts5func.c \ $(TOP)/src/func.c \ $(TOP)/src/global.c \ $(TOP)/src/hash.c \ $(TOP)/src/hash.h \ $(TOP)/src/hwtime.h \ $(TOP)/src/insert.c \ - $(TOP)/src/kv.c \ - $(TOP)/src/kv.h \ $(TOP)/src/kvlsm.c \ $(TOP)/src/kvmem.c \ $(TOP)/src/legacy.c \ $(TOP)/src/lsm.h \ $(TOP)/src/lsmInt.h \ @@ -157,10 +158,12 @@ $(TOP)/src/shell.c \ $(TOP)/src/sqlite.h.in \ $(TOP)/src/sqliteInt.h \ $(TOP)/src/sqliteLimit.h \ $(TOP)/src/status.c \ + $(TOP)/src/storage.c \ + $(TOP)/src/storage.h \ $(TOP)/src/tclsqlite.c \ $(TOP)/src/tokenize.c \ $(TOP)/src/trigger.c \ $(TOP)/src/utf.c \ $(TOP)/src/update.c \ @@ -215,23 +218,25 @@ # Source code to the test files. # TESTSRC = \ + $(TOP)/ext/fts3/fts3_term.c \ + $(TOP)/ext/fts3/fts3_test.c \ $(TOP)/test/test_main.c \ $(TOP)/test/test_thread0.c \ $(TOP)/test/test_utf.c \ $(TOP)/test/test_misc1.c \ $(TOP)/test/test_config.c \ $(TOP)/test/test_func.c \ $(TOP)/test/test_hexio.c \ $(TOP)/test/test_lsm.c \ - $(TOP)/test/test_kv.c \ - $(TOP)/test/test_kv2.c \ $(TOP)/test/test_malloc.c \ $(TOP)/test/test_mem.c \ $(TOP)/test/test_mutex.c \ + $(TOP)/test/test_storage.c \ + $(TOP)/test/test_storage2.c \ $(TOP)/test/test_thread.c \ $(TOP)/test/test_wsd.c #TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c #TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c @@ -239,12 +244,10 @@ TESTSRC2 = \ $(TOP)/src/attach.c \ $(TOP)/src/build.c \ $(TOP)/src/date.c \ $(TOP)/src/expr.c \ - $(TOP)/src/fts5.c \ - $(TOP)/src/fts5func.c \ $(TOP)/src/func.c \ $(TOP)/src/insert.c \ $(TOP)/src/mem5.c \ $(TOP)/src/os.c \ $(TOP)/src/pragma.c \ @@ -259,27 +262,32 @@ $(TOP)/src/vdbeaux.c \ $(TOP)/src/vdbe.c \ $(TOP)/src/vdbemem.c \ $(TOP)/src/where.c \ parse.c \ + $(TOP)/ext/fts3/fts3.c \ + $(TOP)/ext/fts3/fts3_aux.c \ + $(TOP)/ext/fts3/fts3_expr.c \ + $(TOP)/ext/fts3/fts3_tokenizer.c \ + $(TOP)/ext/fts3/fts3_write.c # Header files used by all library source files. # HDR = \ $(TOP)/src/hash.h \ $(TOP)/src/hwtime.h \ keywordhash.h \ - $(TOP)/src/kv.h \ $(TOP)/src/lsm.h \ $(TOP)/src/lsmInt.h \ $(TOP)/src/mutex.h \ opcodes.h \ $(TOP)/src/os.h \ parse.h \ sqlite4.h \ $(TOP)/src/sqliteInt.h \ $(TOP)/src/sqliteLimit.h \ + $(TOP)/src/storage.h \ $(TOP)/src/vdbe.h \ $(TOP)/src/vdbeInt.h EXTHDR = \ $(TOP)/ext/fts3/fts3.h \ @@ -293,11 +301,10 @@ LSMTESTSRC = $(TOP)/lsm-test/lsmtest1.c $(TOP)/lsm-test/lsmtest2.c \ $(TOP)/lsm-test/lsmtest3.c $(TOP)/lsm-test/lsmtest4.c \ $(TOP)/lsm-test/lsmtest5.c $(TOP)/lsm-test/lsmtest6.c \ $(TOP)/lsm-test/lsmtest7.c $(TOP)/lsm-test/lsmtest8.c \ - $(TOP)/lsm-test/lsmtest9.c \ $(TOP)/lsm-test/lsmtest_datasource.c \ $(TOP)/lsm-test/lsmtest_func.c $(TOP)/lsm-test/lsmtest_io.c \ $(TOP)/lsm-test/lsmtest_main.c $(TOP)/lsm-test/lsmtest_mem.c \ $(TOP)/lsm-test/lsmtest_tdb.c $(TOP)/lsm-test/lsmtest_tdb3.c \ $(TOP)/lsm-test/lsmtest_util.c @@ -512,12 +519,11 @@ ./testfixture$(EXE) $(TOP)/test/src4.test # Rules to build the 'lsmtest' application. # lsmtest$(EXE): libsqlite4.a $(LSMTESTSRC) $(LSMTESTHDR) - $(TCPPX) -c $(TOP)/lsm-test/lsmtest_tdb2.cc - $(TCCX) $(LSMTESTSRC) lsmtest_tdb2.o libsqlite4.a -o lsmtest$(EXE) $(THREADLIB) -lsqlite3 + $(TCCX) $(LSMTESTSRC) libsqlite4.a -o lsmtest$(EXE) $(THREADLIB) -lsqlite3 varint$(EXE): $(TOP)/src/varint.c $(TCCX) -DVARINT_TOOL -o varint$(EXE) $(TOP)/src/varint.c Index: src/alter.c ================================================================== --- src/alter.c +++ src/alter.c @@ -77,11 +77,11 @@ assert( len>0 ); } while( token!=TK_LP && token!=TK_USING ); zRet = sqlite4MPrintf(db, "%.*s\"%w\"%s", ((u8*)tname.z) - zSql, zSql, zTableName, tname.z+tname.n); - sqlite4_result_text(context, zRet, -1, SQLITE4_TRANSIENT, 0); + sqlite4_result_text(context, zRet, -1, SQLITE4_TRANSIENT); sqlite4DbFree(db, zRet); } } /* @@ -127,11 +127,11 @@ }while( token==TK_SPACE ); zParent = sqlite4DbStrNDup(db, (const char *)z, n); if( zParent==0 ) break; sqlite4Dequote(zParent); - if( 0==sqlite4_stricmp((const char *)zOld, zParent) ){ + if( 0==sqlite4StrICmp((const char *)zOld, zParent) ){ char *zOut = sqlite4MPrintf(db, "%s%.*s\"%w\"", (zOutput?zOutput:""), z-zInput, zInput, (const char *)zNew ); sqlite4DbFree(db, zOutput); zOutput = zOut; @@ -140,11 +140,11 @@ sqlite4DbFree(db, zParent); } } zResult = sqlite4MPrintf(db, "%s%s", (zOutput?zOutput:""), zInput), - sqlite4_result_text(context, zResult, -1, SQLITE4_TRANSIENT, 0); + sqlite4_result_text(context, zResult, -1, SQLITE4_TRANSIENT); sqlite4DbFree(db, zOutput); sqlite4DbFree(db, zResult); } #endif @@ -218,11 +218,11 @@ /* Variable tname now contains the token that is the old table-name ** in the CREATE TRIGGER statement. */ zRet = sqlite4MPrintf(db, "%.*s\"%w\"%s", ((u8*)tname.z) - zSql, zSql, zTableName, tname.z+tname.n); - sqlite4_result_text(context, zRet, -1, SQLITE4_TRANSIENT, 0); + sqlite4_result_text(context, zRet, -1, SQLITE4_TRANSIENT); sqlite4DbFree(db, zRet); } } #endif /* !SQLITE4_OMIT_TRIGGER */ @@ -379,11 +379,11 @@ ** in pParse->zErr (system tables may not be altered) and returns non-zero. ** ** Or, if zName is not a system table, zero is returned. */ static int isSystemTable(Parse *pParse, const char *zName){ - if( sqlite4Strlen30(zName)>6 && 0==sqlite4_strnicmp(zName, "sqlite_", 7) ){ + if( sqlite4Strlen30(zName)>6 && 0==sqlite4StrNICmp(zName, "sqlite_", 7) ){ sqlite4ErrorMsg(pParse, "table %s may not be altered", zName); return 1; } return 0; } Index: src/attach.c ================================================================== --- src/attach.c +++ src/attach.c @@ -100,11 +100,11 @@ goto attach_error; } for(i=0; inDb; i++){ char *z = db->aDb[i].zName; assert( z && zName ); - if( sqlite4_stricmp(z, zName)==0 ){ + if( sqlite4StrICmp(z, zName)==0 ){ zErrDyn = sqlite4MPrintf(db, "database %s is already in use", zName); goto attach_error; } } @@ -218,11 +218,11 @@ if( zName==0 ) zName = ""; for(i=0; inDb; i++){ pDb = &db->aDb[i]; if( pDb->pKV==0 ) continue; - if( sqlite4_stricmp(pDb->zName, zName)==0 ) break; + if( sqlite4StrICmp(pDb->zName, zName)==0 ) break; } if( i>=db->nDb ){ sqlite4_snprintf(zErr,sizeof(zErr), "no such database: %s", zName); goto detach_error; @@ -241,12 +241,10 @@ goto detach_error; } sqlite4KVStoreClose(pDb->pKV); pDb->pKV = 0; - sqlite4SchemaClear(db->pEnv, pDb->pSchema); - sqlite4DbFree(db, pDb->pSchema); pDb->pSchema = 0; sqlite4ResetInternalSchema(db, -1); return; detach_error: @@ -422,11 +420,11 @@ if( NEVER(pList==0) ) return 0; zDb = pFix->zDb; for(i=0, pItem=pList->a; inSrc; i++, pItem++){ if( pItem->zDatabase==0 ){ pItem->zDatabase = sqlite4DbStrDup(pFix->pParse->db, zDb); - }else if( sqlite4_stricmp(pItem->zDatabase,zDb)!=0 ){ + }else if( sqlite4StrICmp(pItem->zDatabase,zDb)!=0 ){ sqlite4ErrorMsg(pFix->pParse, "%s %T cannot reference objects in database %s", pFix->zType, pFix->pName, pItem->zDatabase); return 1; } Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -215,11 +215,11 @@ assert( zName!=0 ); nName = sqlite4Strlen30(zName); /* All mutexes are required for schema access. Make sure we hold them. */ for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ - if( zDatabase!=0 && sqlite4_stricmp(zDatabase, db->aDb[j].zName) ) continue; + if( zDatabase!=0 && sqlite4StrICmp(zDatabase, db->aDb[j].zName) ) continue; p = sqlite4HashFind(&db->aDb[j].pSchema->tblHash, zName, nName); if( p ) break; } return p; } @@ -280,11 +280,11 @@ /* All mutexes are required for schema access. Make sure we hold them. */ for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ Schema *pSchema = db->aDb[j].pSchema; assert( pSchema ); - if( zDb && sqlite4_stricmp(zDb, db->aDb[j].zName) ) continue; + if( zDb && sqlite4StrICmp(zDb, db->aDb[j].zName) ) continue; p = sqlite4HashFind(&pSchema->idxHash, zName, nName); if( p ) break; } return p; } @@ -294,11 +294,10 @@ */ static void freeIndex(sqlite4 *db, Index *p){ #ifndef SQLITE4_OMIT_ANALYZE sqlite4DeleteIndexSamples(db, p); #endif - sqlite4Fts5IndexFree(db, p); sqlite4DbFree(db, p->zColAff); sqlite4DbFree(db, p); } /* @@ -548,11 +547,11 @@ if( zName ){ Db *pDb; int n = sqlite4Strlen30(zName); for(i=(db->nDb-1), pDb=&db->aDb[i]; i>=0; i--, pDb--){ if( (!OMIT_TEMPDB || i!=1 ) && n==sqlite4Strlen30(pDb->zName) && - 0==sqlite4_stricmp(pDb->zName, zName) ){ + 0==sqlite4StrICmp(pDb->zName, zName) ){ break; } } } return i; @@ -627,11 +626,11 @@ ** is reserved for internal use. */ int sqlite4CheckObjectName(Parse *pParse, const char *zName){ if( !pParse->db->init.busy && pParse->nested==0 && (pParse->db->flags & SQLITE4_WriteSchema)==0 - && 0==sqlite4_strnicmp(zName, "sqlite_", 7) ){ + && 0==sqlite4StrNICmp(zName, "sqlite_", 7) ){ sqlite4ErrorMsg(pParse, "object name reserved for internal use: %s", zName); return SQLITE4_ERROR; } return SQLITE4_OK; } @@ -838,20 +837,20 @@ return; } /* ** This macro is used to compare two strings in a case-insensitive manner. -** It is slightly faster than calling sqlite4_stricmp() directly, but +** It is slightly faster than calling sqlite4StrICmp() directly, but ** produces larger code. ** ** WARNING: This macro is not compatible with the strcmp() family. It ** returns true if the two strings are equal, otherwise false. */ #define STRICMP(x, y) (\ sqlite4UpperToLower[*(unsigned char *)(x)]== \ sqlite4UpperToLower[*(unsigned char *)(y)] \ -&& sqlite4_stricmp((x)+1,(y)+1)==0 ) +&& sqlite4StrICmp((x)+1,(y)+1)==0 ) /* ** Add a new column to the table currently being constructed. ** ** The parser calls this routine once for each column declaration @@ -1038,21 +1037,31 @@ ** most recently added column of the table is the primary key. ** ** A table can have at most one primary key. If the table already has ** a primary key (and this is the second primary key) then create an ** error. +** +** If the PRIMARY KEY is on a single column whose datatype is INTEGER, +** then we will try to use that column as the rowid. Set the Table.iPKey +** field of the table under construction to be the index of the +** INTEGER PRIMARY KEY column. Table.iPKey is set to -1 if there is +** no INTEGER PRIMARY KEY. +** +** If the key is not an INTEGER PRIMARY KEY, then create a unique +** index for the key. No index is created for INTEGER PRIMARY KEYs. */ void sqlite4AddPrimaryKey( Parse *pParse, /* Parsing context */ ExprList *pList, /* List of field names to be indexed */ int onError, /* What to do with a uniqueness conflict */ int autoInc, /* True if the AUTOINCREMENT keyword is present */ int sortOrder /* SQLITE4_SO_ASC or SQLITE4_SO_DESC */ ){ Table *pTab = pParse->pNewTable; - Index *pPk; /* Primary key index */ - char *zType = 0; /* Primary key column type */ +#if 0 + char *zType = 0; +#endif int iCol = -1, i; if( pTab==0 || IN_DECLARE_VTAB ) goto primary_key_exit; if( pTab->tabFlags & TF_HasPrimaryKey ){ sqlite4ErrorMsg(pParse, "table \"%s\" has more than one primary key", pTab->zName); @@ -1064,39 +1073,47 @@ pTab->aCol[iCol].isPrimKey = 1; pTab->aCol[iCol].notNull = 1; }else{ for(i=0; inExpr; i++){ for(iCol=0; iColnCol; iCol++){ - if( sqlite4_stricmp(pList->a[i].zName, pTab->aCol[iCol].zName)==0 ){ + if( sqlite4StrICmp(pList->a[i].zName, pTab->aCol[iCol].zName)==0 ){ break; } } if( iColnCol ){ - pTab->aCol[iCol].isPrimKey = i+1; + pTab->aCol[iCol].isPrimKey = 1; pTab->aCol[iCol].notNull = 1; } } if( pList->nExpr>1 ) iCol = -1; } - pPk = sqlite4CreateIndex( - pParse, 0, 0, 0, pList, onError, 0, 0, sortOrder, 0, 1 - ); - - if( iCol>=0 && iColnCol - && (zType = pTab->aCol[iCol].zType)!=0 - && sqlite4_stricmp(zType, "INTEGER")==0 - && sortOrder==SQLITE4_SO_ASC - && pPk - ){ - pPk->fIndex |= IDX_IntPK; + +#if 0 + if( iCol>=0 && iColnCol ){ + zType = pTab->aCol[iCol].zType; + } + + if( zType && sqlite4StrICmp(zType, "INTEGER")==0 + && sortOrder==SQLITE4_SO_ASC ){ + pTab->iPKey = iCol; + pTab->keyConf = (u8)onError; assert( autoInc==0 || autoInc==1 ); - pTab->tabFlags |= (-autoInc)&TF_Autoincrement; + pTab->tabFlags |= autoInc*TF_Autoincrement; }else if( autoInc ){ - sqlite4ErrorMsg(pParse, - "AUTOINCREMENT permitted on INTEGER PRIMARY KEY ASC only"); +#ifndef SQLITE4_OMIT_AUTOINCREMENT + sqlite4ErrorMsg(pParse, "AUTOINCREMENT is only allowed on an " + "INTEGER PRIMARY KEY"); +#endif + }else +#endif + + { + sqlite4CreateIndex( + pParse, 0, 0, 0, pList, onError, 0, 0, sortOrder, 0, 1 + ); + pList = 0; } - pList = 0; primary_key_exit: sqlite4ExprListDelete(pParse->db, pList); return; } @@ -2045,12 +2062,12 @@ if( sqlite4AuthCheck(pParse, SQLITE4_DELETE, pTab->zName, 0, zDb) ){ goto exit_drop_table; } } #endif - if( sqlite4_strnicmp(pTab->zName, "sqlite_", 7)==0 - && sqlite4_strnicmp(pTab->zName, "sqlite_stat", 11)!=0 ){ + if( sqlite4StrNICmp(pTab->zName, "sqlite_", 7)==0 + && sqlite4StrNICmp(pTab->zName, "sqlite_stat", 11)!=0 ){ sqlite4ErrorMsg(pParse, "table %s may not be dropped", pTab->zName); goto exit_drop_table; } #ifndef SQLITE4_OMIT_VIEW @@ -2158,11 +2175,11 @@ pFKey->aCol[0].iFrom = p->nCol-1; }else{ for(i=0; inCol; j++){ - if( sqlite4_stricmp(p->aCol[j].zName, pFromCol->a[i].zName)==0 ){ + if( sqlite4StrICmp(p->aCol[j].zName, pFromCol->a[i].zName)==0 ){ pFKey->aCol[i].iFrom = j; break; } } if( j>=p->nCol ){ @@ -2269,306 +2286,30 @@ if( bCreate ) sqlite4VdbeChangeP5(v, 1); /* Loop through the contents of the PK index. At each row, insert the ** corresponding entry into the auxiliary index. */ addr1 = sqlite4VdbeAddOp2(v, OP_Rewind, iTab, 0); - - if( pIdx->eIndexType==SQLITE4_INDEX_FTS5 ){ - int regData; - int i; - - regKey = sqlite4GetTempRange(pParse, pTab->nCol+1); - regData = regKey+1; - - sqlite4VdbeAddOp2(v, OP_RowKey, iTab, regKey); - for(i=0; inCol; i++){ - sqlite4VdbeAddOp3(v, OP_Column, iTab, i, regData+i); - } - sqlite4Fts5CodeUpdate(pParse, pIdx, pParse->iNewidxReg, regKey, regData, 0); - }else{ - sqlite4GetTempRange(pParse,2); - regKey = sqlite4GetTempReg(pParse); - sqlite4EncodeIndexKey(pParse, pPk, iTab, pIdx, iIdx, 0, regKey); - if( pIdx->onError!=OE_None ){ - const char *zErr = "indexed columns are not unique"; - int addrTest; - - addrTest = sqlite4VdbeAddOp4Int(v, OP_IsUnique, iIdx, 0, regKey, 0); - sqlite4HaltConstraint(pParse, OE_Abort, (char *)zErr, P4_STATIC); - sqlite4VdbeJumpHere(v, addrTest); - } - sqlite4VdbeAddOp3(v, OP_IdxInsert, iIdx, 0, regKey); - } - + sqlite4GetTempRange(pParse,2); + regKey = sqlite4GetTempReg(pParse); + sqlite4EncodeIndexKey(pParse, pPk, iTab, pIdx, iIdx, 0, regKey); + if( pIdx->onError!=OE_None ){ + const char *zErr = "indexed columns are not unique"; + int addrTest; + + addrTest = sqlite4VdbeAddOp4Int(v, OP_IsUnique, iIdx, 0, regKey, 0); + sqlite4HaltConstraint(pParse, OE_Abort, (char *)zErr, P4_STATIC); + sqlite4VdbeJumpHere(v, addrTest); + } + sqlite4VdbeAddOp3(v, OP_IdxInsert, iIdx, 0, regKey); sqlite4VdbeAddOp2(v, OP_Next, iTab, addr1+1); sqlite4VdbeJumpHere(v, addr1); sqlite4ReleaseTempReg(pParse, regKey); sqlite4VdbeAddOp1(v, OP_Close, iTab); sqlite4VdbeAddOp1(v, OP_Close, iIdx); } -/* -** The CreateIndex structure indicated by the first argument contains the -** results of parsing the first part of a CREATE INDEX statement. -** Specifically, everything up to and including the "ON tblname" clause. -** The index may be an ordinary index, or it may be a "USING fts5" index. -** This function performs processing common to both. -*/ -static Table *createIndexFindTable( - Parse *pParse, /* Parsing context */ - CreateIndex *p, /* First part of CREATE INDEX statement */ - Token **ppIdxName, /* OUT: Pointer to index name token */ - char **pzIdxName, /* OUT: DbMalloc'd copy of index name */ - int *piDb /* OUT: Database to create index in */ -){ - DbFixer sFix; /* For assigning database names to pTblName */ - sqlite4 *db = pParse->db; /* Database handle */ - Token *pName = 0; /* Token containing unqualified index name */ - char *zName; /* Name of index being created */ - int iDb; /* Index of database in db->aDb[] */ - Table *pTab; /* Table object to return */ - - /* Use the two-part index name to determine the database - ** to search for the table. 'Fix' the table name to this db - ** before looking up the table. */ - iDb = sqlite4TwoPartName(pParse, &p->tName1, &p->tName2, &pName); - if( iDb<0 ) return 0; - assert( pName && pName->z ); - -#ifndef SQLITE4_OMIT_TEMPDB - /* If the index name was unqualified, check if the the table - ** is a temp table. If so, set the database to 1. Do not do this - ** if initialising a database schema. */ - if( !db->init.busy ){ - pTab = sqlite4SrcListLookup(pParse, p->pTblName); - if( p->tName2.n==0 && pTab && pTab->pSchema==db->aDb[1].pSchema ){ - iDb = 1; - } - } -#endif - - if( sqlite4FixInit(&sFix, pParse, iDb, "index", pName) && - sqlite4FixSrcList(&sFix, p->pTblName) - ){ - /* Because the parser constructs pTblName from a single identifier, - ** sqlite4FixSrcList can never fail. */ - assert(0); - } - - pTab = sqlite4SrcListLookup(pParse, p->pTblName); - if( !pTab || db->mallocFailed ) return 0; - assert( db->aDb[iDb].pSchema==pTab->pSchema ); - assert( pParse->nErr==0 ); - - /* TODO: We will need to reinstate this block when sqlite_master is - ** modified to use an implicit primary key. */ -#if 0 - if( sqlite4_strnicmp(pTab->zName, "sqlite_", 7)==0 - && memcmp(&pTab->zName[7],"altertab_",9)!=0 ){ - sqlite4ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); - goto exit_create_index; - } -#endif - - /* Verify that this is not an attempt to create an index on a view or - ** virtual table. */ - if( IsView(pTab) ){ - sqlite4ErrorMsg(pParse, "views may not be indexed"); - return 0; - } - if( IsVirtual(pTab) ){ - sqlite4ErrorMsg(pParse, "virtual tables may not be indexed"); - return 0; - } - - /* Ensure that the proposed index name is not reserved. */ - assert( pName->z!=0 ); - zName = sqlite4NameFromToken(db, pName); - if( zName==0 || sqlite4CheckObjectName(pParse, zName) ) return 0; - - /* Unless SQLite is currently parsing an existing database schema, check - ** that there is not already an index or table using the proposed name. */ - if( !db->init.busy ){ - char *zDb = db->aDb[iDb].zName; - if( sqlite4FindTable(db, zName, zDb)!=0 ){ - sqlite4ErrorMsg(pParse, "there is already a table named %s", zName); - } - else if( sqlite4FindIndex(db, zName, zDb)!=0 ){ - if( p->bIfnotexist ){ - assert( !db->init.busy ); - sqlite4CodeVerifySchema(pParse, iDb); - pTab = 0; - }else{ - sqlite4ErrorMsg(pParse, "index %s already exists", zName); - } - } - } - - if( pParse->nErr || pTab==0 ){ - sqlite4DbFree(db, zName); - pTab = 0; - zName = 0; - } - *ppIdxName = pName; - *pzIdxName = zName; - *piDb = iDb; - return pTab; -} - -#ifndef SQLITE4_OMIT_AUTHORIZATION -/* -** Check for authorization to create index zIdx on table pTab. If -** authorization is granted, return zero. Otherwise, return non-zero -** and leave an error in pParse. -*/ -static int createIndexAuth( - Parse *pParse, /* Parser context */ - Table *pTab, /* Table index is being created on */ - const char *zIdx /* Name of index being created */ -){ - sqlite4 *db; - int iDb; - int iOp; - const char *zDb; - - db = pParse->db; - iDb = sqlite4SchemaToIndex(db, pTab->pSchema); - zDb = db->aDb[iDb].zName; - - if( sqlite4AuthCheck(pParse, SQLITE4_INSERT, SCHEMA_TABLE(iDb), 0, zDb) ){ - return 1; - } - - iOp = (!OMIT_TEMPDB && iDb==1)?SQLITE4_CREATE_TEMP_INDEX:SQLITE4_CREATE_INDEX; - if( sqlite4AuthCheck(pParse, iOp, zIdx, pTab->zName, zDb) ){ - return 1; - } - - return 0; -} -#else -# define createIndexAuth(a, b, c) 0 -#endif // SQLITE4_OMIT_AUTHORIZATION - -/* -** This function is called when parsing a CREATE statement executed by the -** user (not when parsing the schema of an existing database). It generates -** the VDBE code to assign a root page number to the new index and, if -** required, to write a new entry into the sqlite_master table. -*/ -static void createIndexWriteSchema( - Parse *pParse, /* Parser context */ - Index *pIdx, /* New index object */ - Token *pName, /* Token containing name of new index */ - Token *pEnd /* Token for final closing paren of CREATE */ -){ - sqlite4 *db = pParse->db; - int iDb; - - assert( db->init.busy==0 ); - iDb = sqlite4SchemaToIndex(db, pIdx->pSchema); - pIdx->tnum = ++pParse->nMem; - allocateTableNumber(pParse, iDb, pIdx->tnum); - - if( pIdx->eIndexType!=SQLITE4_INDEX_PRIMARYKEY ){ - Vdbe *v; - char *zStmt; - - v = sqlite4GetVdbe(pParse); - if( v==0 ) return; - - sqlite4BeginWriteOperation(pParse, 1, iDb); - - /* Unless this index is an automatic index created by a UNIQUE - ** constraint, assemble a CREATE INDEX statement to write into the - ** sqlite_master table. */ - if( pIdx->eIndexType!=SQLITE4_INDEX_UNIQUE ){ - int n = (int)(pEnd->z - pName->z) + pEnd->n; - const char *zUnique = (pIdx->onError==OE_None ? "" : " UNIQUE"); - zStmt = sqlite4MPrintf(db, "CREATE%s INDEX %.*s", zUnique, n, pName->z); - }else{ - /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ - zStmt = 0; - } - - /* Add an entry in sqlite_master for this index */ - sqlite4NestedParse(pParse, - "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#%d,%Q);", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), - pIdx->zName, - pIdx->pTable->zName, - pIdx->tnum, - zStmt - ); - sqlite4DbFree(db, zStmt); - - /* Fill the index with data and reparse the schema. Code an OP_Expire - ** to invalidate all pre-compiled statements. - */ - if( pIdx->eIndexType!=SQLITE4_INDEX_UNIQUE ){ - sqlite4RefillIndex(pParse, pIdx, 1); - sqlite4ChangeCookie(pParse, iDb); - sqlite4VdbeAddParseSchemaOp(v, iDb, - sqlite4MPrintf(db, "name='%q' AND type='index'", pIdx->zName)); - sqlite4VdbeAddOp1(v, OP_Expire, 0); - } - } -} - -void sqlite4CreateUsingIndex( - Parse *pParse, /* Parsing context */ - CreateIndex *p, /* First part of CREATE INDEX statement */ - ExprList *pList, - Token *pUsing, /* Token following USING keyword */ - Token *pEnd /* Final '(' in CREATE INDEX */ -){ - sqlite4 *db = pParse->db; - if( p->bUnique ){ - sqlite4ErrorMsg(pParse, "USING %.*s index may not be UNIQUE", - pUsing->n, pUsing->z - ); - }else{ - Index *pIdx = 0; /* New index object */ - Table *pTab; - Token *pIdxName = 0; - char *zIdx = 0; - int iDb = 0; - - pTab = createIndexFindTable(pParse, p, &pIdxName, &zIdx, &iDb); - if( pTab && 0==createIndexAuth(pParse, pTab, zIdx) ){ - int nExtra = sqlite4Fts5IndexSz(); - char *zExtra = 0; - pIdx = newIndex(pParse, pTab, zIdx, 0, 0, nExtra, &zExtra); - if( pIdx ){ - pIdx->pFts = (Fts5Index *)zExtra; - sqlite4Fts5IndexInit(pParse, pIdx, pList); - pIdx->eIndexType = SQLITE4_INDEX_FTS5; - } - } - - if( pIdx ){ - if( db->init.busy ){ - db->flags |= SQLITE4_InternChanges; - pIdx->tnum = db->init.newTnum; - pIdx->pNext = pTab->pIndex; - pTab->pIndex = pIdx; - addIndexToHash(db, pIdx); - pIdx = 0; - }else{ - createIndexWriteSchema(pParse, pIdx, pIdxName, pEnd); - } - } - - if( pIdx ) freeIndex(db, pIdx); - sqlite4DbFree(db, zIdx); - } - - sqlite4ExprListDelete(db, pList); - sqlite4SrcListDelete(db, p->pTblName); -} - /* ** Create a new index for an SQL table. pName1.pName2 is the name of the index ** and pTblList is the name of the table that is to be indexed. Both will ** be NULL for a primary key or an index that is created to satisfy a ** UNIQUE constraint. If pTable and pIndex are NULL, use pParse->pNewTable @@ -2622,23 +2363,46 @@ /* ** Find the table that is to be indexed. Return early if not found. */ if( pTblName!=0 ){ - CreateIndex sCreate; - sCreate.bUnique = onError; - sCreate.bIfnotexist = ifNotExist; - sCreate.tCreate = *pStart; - sCreate.tName1 = *pName1; - sCreate.tName2 = *pName2; - sCreate.pTblName = pTblName; - - pTab = createIndexFindTable(pParse, &sCreate, &pName, &zName, &iDb); - if( !pTab ) goto exit_create_index; - - }else{ - + + /* Use the two-part index name to determine the database + ** to search for the table. 'Fix' the table name to this db + ** before looking up the table. + */ + assert( !bPrimaryKey ); + assert( pName1 && pName2 ); + iDb = sqlite4TwoPartName(pParse, pName1, pName2, &pName); + if( iDb<0 ) goto exit_create_index; + assert( pName && pName->z ); + +#ifndef SQLITE4_OMIT_TEMPDB + /* If the index name was unqualified, check if the the table + ** is a temp table. If so, set the database to 1. Do not do this + ** if initialising a database schema. + */ + if( !db->init.busy ){ + pTab = sqlite4SrcListLookup(pParse, pTblName); + if( pName2->n==0 && pTab && pTab->pSchema==db->aDb[1].pSchema ){ + iDb = 1; + } + } +#endif + + if( sqlite4FixInit(&sFix, pParse, iDb, "index", pName) && + sqlite4FixSrcList(&sFix, pTblName) + ){ + /* Because the parser constructs pTblName from a single identifier, + ** sqlite4FixSrcList can never fail. */ + assert(0); + } + pTab = sqlite4LocateTable(pParse, 0, pTblName->a[0].zName, + pTblName->a[0].zDatabase); + if( !pTab || db->mallocFailed ) goto exit_create_index; + assert( db->aDb[iDb].pSchema==pTab->pSchema ); + }else{ assert( pName==0 ); assert( pStart==0 ); pTab = pParse->pNewTable; if( !pTab ) goto exit_create_index; iDb = sqlite4SchemaToIndex(db, pTab->pSchema); @@ -2645,32 +2409,99 @@ } pDb = &db->aDb[iDb]; assert( pTab!=0 ); assert( pParse->nErr==0 ); - assert( IsVirtual(pTab)==0 && IsView(pTab)==0 ); - - /* If pName==0 it means that we are dealing with a primary key or - ** UNIQUE constraint. We have to invent our own name. */ - if( pName==0 ){ - if( !bPrimaryKey ){ - int n; - Index *pLoop; - for(pLoop=pTab->pIndex, n=1; pLoop; pLoop=pLoop->pNext, n++){} - zName = sqlite4MPrintf(db, "sqlite_%s_unique%d", pTab->zName, n); - }else{ - zName = sqlite4MPrintf(db, "%s", pTab->zName); - } - if( zName==0 ){ - goto exit_create_index; - } - } - - /* Check for authorization to create the index. */ - if( bPrimaryKey==0 && createIndexAuth(pParse, pTab, zName) ){ - goto exit_create_index; - } + + /* TODO: We will need to reinstate this block when sqlite_master is + ** modified to use an implicit primary key. */ +#if 0 + if( sqlite4StrNICmp(pTab->zName, "sqlite_", 7)==0 + && memcmp(&pTab->zName[7],"altertab_",9)!=0 ){ + sqlite4ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); + goto exit_create_index; + } +#endif + +#ifndef SQLITE4_OMIT_VIEW + if( pTab->pSelect ){ + assert( !bPrimaryKey ); + sqlite4ErrorMsg(pParse, "views may not be indexed"); + goto exit_create_index; + } +#endif +#ifndef SQLITE4_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + assert( !bPrimaryKey ); + sqlite4ErrorMsg(pParse, "virtual tables may not be indexed"); + goto exit_create_index; + } +#endif + + /* + ** Find the name of the index. Make sure there is not already another + ** index or table with the same name. + ** + ** Exception: If we are reading the names of permanent indices from the + ** sqlite_master table (because some other process changed the schema) and + ** one of the index names collides with the name of a temporary table or + ** index, then we will continue to process this index. + ** + ** If pName==0 it means that we are + ** dealing with a primary key or UNIQUE constraint. We have to invent our + ** own name. + */ + if( pName ){ + assert( !bPrimaryKey ); + zName = sqlite4NameFromToken(db, pName); + if( zName==0 ) goto exit_create_index; + assert( pName->z!=0 ); + if( SQLITE4_OK!=sqlite4CheckObjectName(pParse, zName) ){ + goto exit_create_index; + } + if( !db->init.busy ){ + if( sqlite4FindTable(db, zName, 0)!=0 ){ + sqlite4ErrorMsg(pParse, "there is already a table named %s", zName); + goto exit_create_index; + } + } + if( sqlite4FindIndex(db, zName, pDb->zName)!=0 ){ + if( !ifNotExist ){ + sqlite4ErrorMsg(pParse, "index %s already exists", zName); + }else{ + assert( !db->init.busy ); + sqlite4CodeVerifySchema(pParse, iDb); + } + goto exit_create_index; + } + }else if( !bPrimaryKey ){ + int n; + Index *pLoop; + for(pLoop=pTab->pIndex, n=1; pLoop; pLoop=pLoop->pNext, n++){} + zName = sqlite4MPrintf(db, "sqlite_autoindex_%s_%d", pTab->zName, n); + }else{ + zName = sqlite4MPrintf(db, "%s", pTab->zName); + } + if( zName==0 ){ + goto exit_create_index; + } + + /* Check for authorization to create an index. + */ +#ifndef SQLITE4_OMIT_AUTHORIZATION + if( bPrimaryKey==0 ){ + const char *zDb = pDb->zName; + if( sqlite4AuthCheck(pParse, SQLITE4_INSERT, SCHEMA_TABLE(iDb), 0, zDb) ){ + goto exit_create_index; + } + i = SQLITE4_CREATE_INDEX; + if( !OMIT_TEMPDB && iDb==1 ) i = SQLITE4_CREATE_TEMP_INDEX; + if( sqlite4AuthCheck(pParse, i, zName, pTab->zName, zDb) ){ + goto exit_create_index; + } + } +#endif /* If pList==0, it means this routine was called as a result of a PRIMARY ** KEY or UNIQUE constraint attached to the last column added to the table ** under construction. So create a fake list to simulate this. ** @@ -2718,28 +2549,35 @@ ** load the column indices into the Index structure. Report an error ** if any column is not found. ** ** TODO: Add a test to make sure that the same column is not named ** more than once within the same index. Only the first instance of - ** the column will ever be used by the optimizer. + ** the column will ever be used by the optimizer. Note that using the + ** same column more than once cannot be an error because that would + ** break backwards compatibility - it needs to be a warning. */ for(i=0, pListItem=pList->a; inExpr; i++, pListItem++){ const char *zColName = pListItem->zName; Column *pTabCol; char *zColl; /* Collation sequence name */ for(j=0, pTabCol=pTab->aCol; jnCol; j++, pTabCol++){ - if( sqlite4_stricmp(zColName, pTabCol->zName)==0 ) break; + if( sqlite4StrICmp(zColName, pTabCol->zName)==0 ) break; } if( j>=pTab->nCol ){ sqlite4ErrorMsg(pParse, "table %s has no column named %s", pTab->zName, zColName); pParse->checkSchema = 1; goto exit_create_index; } pIndex->aiColumn[i] = j; - if( pListItem->pExpr && pListItem->pExpr->pColl ){ + /* Justification of the ALWAYS(pListItem->pExpr->pColl): Because of + ** the way the "idxlist" non-terminal is constructed by the parser, + ** if pListItem->pExpr is not null then either pListItem->pExpr->pColl + ** must exist or else there must have been an OOM error. But if there + ** was an OOM error, we would never reach this point. */ + if( pListItem->pExpr && ALWAYS(pListItem->pExpr->pColl) ){ int nColl; zColl = pListItem->pExpr->pColl->zName; nColl = sqlite4Strlen30(zColl) + 1; assert( nExtra>=nColl ); memcpy(zExtra, zColl, nColl); @@ -2794,11 +2632,11 @@ const char *z1; const char *z2; if( pIdx->aiColumn[k]!=pIndex->aiColumn[k] ) break; z1 = pIdx->azColl[k]; z2 = pIndex->azColl[k]; - if( z1!=z2 && sqlite4_stricmp(z1, z2) ) break; + if( z1!=z2 && sqlite4StrICmp(z1, z2) ) break; } if( k==pIdx->nColumn ){ if( pIdx->onError!=pIndex->onError ){ /* This constraint creates the same index as a previous ** constraint specified somewhere in the CREATE TABLE statement. @@ -2852,11 +2690,62 @@ ** or UNIQUE constraint of a CREATE TABLE statement. Since the table ** has just been created, it contains no data and the index initialization ** step can be skipped. */ else{ - createIndexWriteSchema(pParse, pIndex, pName, pEnd); + pIndex->tnum = ++pParse->nMem; + allocateTableNumber(pParse, iDb, pIndex->tnum); + if( bPrimaryKey==0 ){ + Vdbe *v; + char *zStmt; + + v = sqlite4GetVdbe(pParse); + if( v==0 ) goto exit_create_index; + + /* Create the rootpage for the index + */ + sqlite4BeginWriteOperation(pParse, 1, iDb); + + /* Gather the complete text of the CREATE INDEX statement into + ** the zStmt variable + */ + if( pStart ){ + assert( pEnd!=0 ); + /* A named index with an explicit CREATE INDEX statement */ + zStmt = sqlite4MPrintf(db, "CREATE%s INDEX %.*s", + onError==OE_None ? "" : " UNIQUE", + (int)(pEnd->z - pName->z) + 1, + pName->z); + }else{ + /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ + /* zStmt = sqlite4MPrintf(""); */ + zStmt = 0; + } + + /* Add an entry in sqlite_master for this index + */ + sqlite4NestedParse(pParse, + "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#%d,%Q);", + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + pIndex->zName, + pTab->zName, + pIndex->tnum, + zStmt + ); + sqlite4DbFree(db, zStmt); + + /* Fill the index with data and reparse the schema. Code an OP_Expire + ** to invalidate all pre-compiled statements. + */ + if( pTblName ){ + sqlite4RefillIndex(pParse, pIndex, 1); + sqlite4ChangeCookie(pParse, iDb); + sqlite4VdbeAddParseSchemaOp(v, iDb, + sqlite4MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); + sqlite4VdbeAddOp1(v, OP_Expire, 0); + } + } } /* When adding an index to the list of indices for a table, make ** sure all indices labeled OE_Replace come after all those labeled ** OE_Ignore. This is necessary for the correct constraint check @@ -2953,13 +2842,11 @@ sqlite4CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); } pParse->checkSchema = 1; goto exit_drop_index; } - if( pIndex->eIndexType!=SQLITE4_INDEX_USER - && pIndex->eIndexType!=SQLITE4_INDEX_FTS5 - ){ + if( pIndex->eIndexType!=SQLITE4_INDEX_USER ){ sqlite4ErrorMsg(pParse, "index associated with UNIQUE " "or PRIMARY KEY constraint cannot be dropped", 0); goto exit_drop_index; } iDb = sqlite4SchemaToIndex(db, pIndex->pSchema); @@ -3090,11 +2977,11 @@ */ int sqlite4IdListIndex(IdList *pList, const char *zName){ int i; if( pList==0 ) return -1; for(i=0; inId; i++){ - if( sqlite4_stricmp(pList->a[i].zName, zName)==0 ) return i; + if( sqlite4StrICmp(pList->a[i].zName, zName)==0 ) return i; } return -1; } /* @@ -3513,11 +3400,11 @@ void sqlite4CodeVerifyNamedSchema(Parse *pParse, const char *zDb){ sqlite4 *db = pParse->db; int i; for(i=0; inDb; i++){ Db *pDb = &db->aDb[i]; - if( pDb->pKV && (!zDb || 0==sqlite4_stricmp(zDb, pDb->zName)) ){ + if( pDb->pKV && (!zDb || 0==sqlite4StrICmp(zDb, pDb->zName)) ){ sqlite4CodeVerifySchema(pParse, i); } } } @@ -3596,11 +3483,11 @@ int i; assert( zColl!=0 ); for(i=0; inColumn; i++){ const char *z = pIndex->azColl[i]; assert( z!=0 ); - if( 0==sqlite4_stricmp(z, zColl) ){ + if( 0==sqlite4StrICmp(z, zColl) ){ return 1; } } return 0; } Index: src/callback.c ================================================================== --- src/callback.c +++ src/callback.c @@ -30,11 +30,11 @@ } #ifndef SQLITE4_OMIT_UTF16 if( db->xCollNeeded16 ){ char const *zExternal; sqlite4_value *pTmp = sqlite4ValueNew(db); - sqlite4ValueSetStr(pTmp, -1, zName, SQLITE4_UTF8, SQLITE4_STATIC, 0); + sqlite4ValueSetStr(pTmp, -1, zName, SQLITE4_UTF8, SQLITE4_STATIC); zExternal = sqlite4ValueText(pTmp, SQLITE4_UTF16NATIVE); if( zExternal ){ db->xCollNeeded16(db->pCollNeededArg, db, (int)ENC(db), zExternal); } sqlite4ValueFree(pTmp); @@ -268,11 +268,11 @@ int nFunc /* Number of bytes in zFunc */ ){ FuncDef *p; if( nFunc<0 ) nFunc = sqlite4Strlen30(zFunc); for(p=pFuncTab->pFirst; p; p=p->pNextName){ - if( sqlite4_strnicmp(p->zName, zFunc, nFunc)==0 && p->zName[nFunc]==0 ){ + if( sqlite4StrNICmp(p->zName, zFunc, nFunc)==0 && p->zName[nFunc]==0 ){ return p; } } return 0; } @@ -297,11 +297,11 @@ if( pFuncTab->pFirst==0 ){ pFuncTab->pFirst = pDef; pFuncTab->pLast = pDef; pFuncTab->pSame = pDef; }else if( isBuiltIn - && sqlite4_stricmp(pDef->zName, pFuncTab->pLast->zName)==0 ){ + && sqlite4StrICmp(pDef->zName, pFuncTab->pLast->zName)==0 ){ assert( pFuncTab->pSame->pSameName==0 || pFuncTab->pSame->pSameName==pDef ); pFuncTab->pSame->pSameName = pDef; pFuncTab->pSame = pDef; }else if( !isBuiltIn && (pOther=functionSearch(pFuncTab,pDef->zName,-1))!=0 ){ pDef->pSameName = pOther->pSameName; @@ -422,17 +422,17 @@ Hash temp2; HashElem *pElem; temp1 = pSchema->tblHash; temp2 = pSchema->trigHash; - sqlite4HashInit(pEnv, &pSchema->trigHash, 0); + sqlite4HashInit(pEnv, &pSchema->trigHash); sqlite4HashClear(&pSchema->idxHash); for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){ sqlite4DeleteTrigger(0, (Trigger*)sqliteHashData(pElem)); } sqlite4HashClear(&temp2); - sqlite4HashInit(pEnv, &pSchema->tblHash, 0); + sqlite4HashInit(pEnv, &pSchema->tblHash); for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ Table *pTab = sqliteHashData(pElem); sqlite4DeleteTable(0, pTab); } sqlite4HashClear(&temp1); @@ -452,13 +452,13 @@ Schema * p; p = (Schema *)sqlite4DbMallocZero(0, sizeof(Schema)); if( !p ){ db->mallocFailed = 1; }else if ( 0==p->file_format ){ - sqlite4HashInit(db->pEnv, &p->tblHash, 0); - sqlite4HashInit(db->pEnv, &p->idxHash, 0); - sqlite4HashInit(db->pEnv, &p->trigHash, 0); - sqlite4HashInit(db->pEnv, &p->fkeyHash, 0); + sqlite4HashInit(db->pEnv, &p->tblHash); + sqlite4HashInit(db->pEnv, &p->idxHash); + sqlite4HashInit(db->pEnv, &p->trigHash); + sqlite4HashInit(db->pEnv, &p->fkeyHash); p->enc = SQLITE4_UTF8; } return p; } Index: src/complete.c ================================================================== --- src/complete.c +++ src/complete.c @@ -198,35 +198,35 @@ #ifdef SQLITE4_OMIT_TRIGGER token = tkOTHER; #else switch( *zSql ){ case 'c': case 'C': { - if( nId==6 && sqlite4_strnicmp(zSql, "create", 6)==0 ){ + if( nId==6 && sqlite4StrNICmp(zSql, "create", 6)==0 ){ token = tkCREATE; }else{ token = tkOTHER; } break; } case 't': case 'T': { - if( nId==7 && sqlite4_strnicmp(zSql, "trigger", 7)==0 ){ + if( nId==7 && sqlite4StrNICmp(zSql, "trigger", 7)==0 ){ token = tkTRIGGER; - }else if( nId==4 && sqlite4_strnicmp(zSql, "temp", 4)==0 ){ + }else if( nId==4 && sqlite4StrNICmp(zSql, "temp", 4)==0 ){ token = tkTEMP; - }else if( nId==9 && sqlite4_strnicmp(zSql, "temporary", 9)==0 ){ + }else if( nId==9 && sqlite4StrNICmp(zSql, "temporary", 9)==0 ){ token = tkTEMP; }else{ token = tkOTHER; } break; } case 'e': case 'E': { - if( nId==3 && sqlite4_strnicmp(zSql, "end", 3)==0 ){ + if( nId==3 && sqlite4StrNICmp(zSql, "end", 3)==0 ){ token = tkEND; }else #ifndef SQLITE4_OMIT_EXPLAIN - if( nId==7 && sqlite4_strnicmp(zSql, "explain", 7)==0 ){ + if( nId==7 && sqlite4StrNICmp(zSql, "explain", 7)==0 ){ token = tkEXPLAIN; }else #endif { token = tkOTHER; @@ -267,11 +267,11 @@ #ifndef SQLITE4_OMIT_AUTOINIT rc = sqlite4_initialize(0); if( rc ) return rc; #endif pVal = sqlite4ValueNew(0); - sqlite4ValueSetStr(pVal, -1, zSql, SQLITE4_UTF16NATIVE, SQLITE4_STATIC, 0); + sqlite4ValueSetStr(pVal, -1, zSql, SQLITE4_UTF16NATIVE, SQLITE4_STATIC); zSql8 = sqlite4ValueText(pVal, SQLITE4_UTF8); if( zSql8 ){ rc = sqlite4_complete(zSql8); }else{ rc = SQLITE4_NOMEM; Index: src/ctime.c ================================================================== --- src/ctime.c +++ src/ctime.c @@ -344,17 +344,17 @@ ** The name can optionally begin with "SQLITE4_" but the "SQLITE4_" prefix ** is not required for a match. */ int sqlite4_compileoption_used(const char *zOptName){ int i, n; - if( sqlite4_strnicmp(zOptName, "SQLITE4_", 8)==0 ) zOptName += 8; + if( sqlite4StrNICmp(zOptName, "SQLITE4_", 8)==0 ) zOptName += 8; n = sqlite4Strlen30(zOptName); /* Since ArraySize(azCompileOpt) is normally in single digits, a ** linear search is adequate. No need for a binary search. */ for(i=0; iiJD = (sqlite4_int64)(r*86400000.0 + 0.5); p->validJD = 1; return 0; @@ -812,11 +812,11 @@ if( isDate(context, argc, argv, &x)==0 ){ char zBuf[100]; computeYMD_HMS(&x); sqlite4_snprintf(zBuf,sizeof(zBuf), "%04d-%02d-%02d %02d:%02d:%02d", x.Y, x.M, x.D, x.h, x.m, (int)(x.s)); - sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT, 0); + sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT); } } /* ** time( TIMESTRING, MOD, MOD, ...) @@ -831,11 +831,11 @@ DateTime x; if( isDate(context, argc, argv, &x)==0 ){ char zBuf[100]; computeHMS(&x); sqlite4_snprintf(zBuf,sizeof(zBuf), "%02d:%02d:%02d", x.h, x.m, (int)x.s); - sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT, 0); + sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT); } } /* ** date( TIMESTRING, MOD, MOD, ...) @@ -850,11 +850,11 @@ DateTime x; if( isDate(context, argc, argv, &x)==0 ){ char zBuf[100]; computeYMD(&x); sqlite4_snprintf(zBuf,sizeof(zBuf), "%04d-%02d-%02d", x.Y, x.M, x.D); - sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT, 0); + sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT); } } /* ** strftime( FORMAT, TIMESTRING, MOD, MOD, ...) @@ -998,11 +998,11 @@ } } } z[j] = 0; sqlite4_result_text(context, z, -1, - z==zBuf ? SQLITE4_TRANSIENT : SQLITE4_DYNAMIC, 0); + z==zBuf ? SQLITE4_TRANSIENT : SQLITE4_DYNAMIC); } /* ** current_time() ** @@ -1085,11 +1085,11 @@ if( pTm ) memcpy(&sNow, pTm, sizeof(sNow)); sqlite4_mutex_leave(sqlite4MutexAlloc(SQLITE4_MUTEX_STATIC_MASTER)); #endif if( pTm ){ strftime(zBuf, 20, zFormat, &sNow); - sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT, 0); + sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT); } } #endif /* Index: src/delete.c ================================================================== --- src/delete.c +++ src/delete.c @@ -551,11 +551,11 @@ } /* Build the index key. If bAddSeq is true, append a sequence number to ** the end of the key to ensure it is unique. */ sqlite4VdbeAddOp3(v, OP_MakeIdxKey, iIdxCsr, regTmp, regOut); - if( bAddSeq ) sqlite4VdbeChangeP5(v, OPFLAG_SEQCOUNT); + if( bAddSeq ) sqlite4VdbeChangeP5(v, 1); /* Release temp registers */ sqlite4ReleaseTempRange(pParse, regTmp, nTmpReg); } @@ -583,32 +583,21 @@ int *aRegIdx /* Only delete if (aRegIdx && aRegIdx[i]>0) */ ){ Vdbe *v = pParse->pVdbe; Index *pPk; int iPk; - int iPkCsr; int i; int regKey; Index *pIdx; regKey = sqlite4GetTempReg(pParse); pPk = sqlite4FindPrimaryKey(pTab, &iPk); - iPkCsr = baseCur+iPk; for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ - if( pIdx->eIndexType==SQLITE4_INDEX_FTS5 ){ - int iCol; - int iReg = pParse->nMem+1; - pParse->nMem += (1 + pTab->nCol); - for(iCol=0; iColnCol; iCol++){ - sqlite4VdbeAddOp3(v, OP_Column, iPkCsr, iCol, iReg+iCol); - } - sqlite4VdbeAddOp2(v, OP_RowKey, iPkCsr, iReg+pTab->nCol); - sqlite4Fts5CodeUpdate(pParse, pIdx, 0, iReg+pTab->nCol, iReg, 1); - }else if( pIdx!=pPk && (aRegIdx==0 || aRegIdx[i]>0) ){ + if( pIdx!=pPk && (aRegIdx==0 || aRegIdx[i]>0) ){ int addrNotFound; - sqlite4EncodeIndexKey(pParse, pPk, baseCur+iPk,pIdx,baseCur+i,0,regKey); + sqlite4EncodeIndexKey(pParse, pPk, baseCur+iPk, pIdx, baseCur+i,0,regKey); addrNotFound = sqlite4VdbeAddOp4(v, OP_NotFound, baseCur+i, 0, regKey, 0, P4_INT32 ); sqlite4VdbeAddOp1(v, OP_Delete, baseCur+i); sqlite4VdbeJumpHere(v, addrNotFound); Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -364,11 +364,12 @@ ** is responsible for making sure the node eventually gets freed. ** ** If dequote is true, then the token (if it exists) is dequoted. ** If dequote is false, no dequoting is performance. The deQuote ** parameter is ignored if pToken is NULL or if the token does not -** appear to be quoted. +** appear to be quoted. If the quotes were of the form "..." (double-quotes) +** then the EP_DblQuoted flag is set on the expression node. ** ** Special case: If op==TK_INTEGER and pToken points to a string that ** can be translated into a 32-bit integer, then the token is not ** stored in u.zToken. Instead, the integer values is written ** into u.iValue and the EP_IntValue flag is set. No extra storage @@ -404,12 +405,13 @@ pNew->u.zToken = (char*)&pNew[1]; assert( pToken->z!=0 || pToken->n==0 ); if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); pNew->u.zToken[pToken->n] = 0; if( dequote && nExtra>=3 - && ((c = pToken->z[0])=='\'' || c=='"' || c=='[') ){ + && ((c = pToken->z[0])=='\'' || c=='"' || c=='[' || c=='`') ){ sqlite4Dequote(pNew->u.zToken); + if( c=='"' ) pNew->flags |= EP_DblQuoted; } } } #if SQLITE4_MAX_EXPR_DEPTH>0 pNew->nHeight = 1; @@ -1130,11 +1132,10 @@ ** and pWalker->u.i==2 */ case TK_FUNCTION: if( pWalker->u.i==2 ) return 0; /* Fall through */ case TK_ID: - case TK_MATCH: case TK_COLUMN: case TK_AGG_FUNCTION: case TK_AGG_COLUMN: testcase( pExpr->op==TK_ID ); testcase( pExpr->op==TK_COLUMN ); @@ -2586,13 +2587,10 @@ } if( pDef->flags & SQLITE4_FUNC_NEEDCOLL ){ if( !pColl ) pColl = db->pDfltColl; sqlite4VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); } - if( pDef->bMatchinfo ){ - sqlite4VdbeAddOp1(v, OP_Mifunction, pFarg->a[0].pExpr->iTable); - } sqlite4VdbeAddOp4(v, OP_Function, constMask, r1, target, (char*)pDef, P4_FUNCDEF); sqlite4VdbeChangeP5(v, (u8)nFarg); if( nFarg ){ sqlite4ReleaseTempRange(pParse, r1, nFarg); @@ -2716,15 +2714,10 @@ } #endif break; } - case TK_MATCH: { - sqlite4ErrorMsg(pParse, "no index to process MATCH operator"); - return 0; - } - /* ** Form A: ** CASE x WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END ** @@ -3190,11 +3183,10 @@ */ static int isAppropriateForFactoring(Expr *p){ if( !sqlite4ExprIsConstantNotJoin(p) ){ return 0; /* Only constant expressions are appropriate for factoring */ } - if( p->op==TK_MATCH || p->op==TK_TABLE ) return 0; if( (p->flags & EP_FixedDest)==0 ){ return 1; /* Any constant without a fixed destination is appropriate */ } while( p->op==TK_UPLUS ) p = p->pLeft; switch( p->op ){ Index: src/fkey.c ================================================================== --- src/fkey.c +++ src/fkey.c @@ -239,15 +239,15 @@ ** unusable. Bail out early in this case. */ zDfltColl = pParent->aCol[iCol].zColl; if( !zDfltColl ){ zDfltColl = "BINARY"; } - if( sqlite4_stricmp(pIdx->azColl[i], zDfltColl) ) break; + if( sqlite4StrICmp(pIdx->azColl[i], zDfltColl) ) break; zIdxCol = pParent->aCol[iCol].zName; for(j=0; jaCol[j].zCol, zIdxCol)==0 ){ + if( sqlite4StrICmp(pFKey->aCol[j].zCol, zIdxCol)==0 ){ if( aiCol ) aiCol[i] = pFKey->aCol[j].iFrom; break; } } if( j==nCol ) break; @@ -866,11 +866,11 @@ for(i=0; inCol; i++){ char *zKey = p->aCol[i].zCol; int iKey; for(iKey=0; iKeynCol; iKey++){ Column *pCol = &pTab->aCol[iKey]; - if( (zKey ? !sqlite4_stricmp(pCol->zName, zKey) : pCol->isPrimKey) ){ + if( (zKey ? !sqlite4StrICmp(pCol->zName, zKey) : pCol->isPrimKey) ){ if( aChange[iKey]>=0 ) return 1; } } } } DELETED src/fts5.c Index: src/fts5.c ================================================================== --- src/fts5.c +++ /dev/null @@ -1,3425 +0,0 @@ -/* -** 2012 December 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -*/ - -#include "sqliteInt.h" -#include "vdbeInt.h" - -/* -** Stream numbers must be lower than this. -** -** For optimization purposes, it is assumed that a given tokenizer uses -** a set of contiguous stream numbers starting with 0. And that most -** tokens belong to stream 0. -** -** The hard limit is 63 (due to the format of "row size" records). -*/ -#define SQLITE4_FTS5_NSTREAM 32 - -/* -** Records stored within the index: -** -** Row size record: -** There is one "row size" record in the index for each row in the -** indexed table. The "row size" record contains the number of tokens -** in the associated row for each combination of a stream and column -** number (i.e. contains the data required to find the number of -** tokens associated with stream S present in column C of the row for -** all S and C). -** -** The key for the row size record is a single 0x00 byte followed by -** a copy of the PK blob for the table row. -** -** The value is a series of varints. Each column of the table is -** represented by one or more varints packed into the array. -** -** If a column contains only stream 0 tokens, then it is represented -** by a single varint - (nToken << 1), where nToken is the number of -** stream 0 tokens stored in the column. -** -** Or, if the column contains tokens from multiple streams, the first -** varint contains a bitmask indicating which of the streams are present -** (stored as ((bitmask << 1) | 0x01)). Following the bitmask is a -** varint containing the number of tokens for each stream present, in -** ascending order of stream number. -** -** TODO: The format above is not currently implemented! Instead, there -** is a simpler place-holder format (which consumes more space). -** -** Global size record: -** There is a single "global size" record stored in the database. The -** database key for this record is a single byte - 0x00. -** -** The data for this record is a series of varint values. The first -** varint is the total number of rows in the table. The subsequent -** varints make up a "row size" record containing the total number of -** tokens for each S/C combination in all rows of the table. -** -** FTS index records: -** The FTS index records implement the following mapping: -** -** (token, document-pk) -> (list of instances) -** -** The key for each index record is in the same format as the keys for -** regular text indexes. An 0x24 byte, followed by the utf-8 representation -** of the token, followed by 0x00, followed by the PK blob for the table -** row. -** -** TODO: Describe value format. -*/ - -/* -** Default distance value for NEAR operators. -*/ -#define FTS5_DEFAULT_NEAR 10 - -/* -** Token types used by expression parser. -*/ -#define TOKEN_EOF 0 /* end of expression - no more tokens */ -#define TOKEN_PRIMITIVE 1 /* quoted string or non-keyword */ -#define TOKEN_STAR 2 /* * */ -#define TOKEN_PLUS 3 /* + */ -#define TOKEN_NEAR 4 /* NEAR/nnn */ -#define TOKEN_COLON 5 /* : */ -#define TOKEN_NOT 6 /* NOT */ -#define TOKEN_AND 7 /* AND */ -#define TOKEN_OR 8 /* OR */ -#define TOKEN_LP 9 /* ( */ -#define TOKEN_RP 10 /* ) */ - -/* -** Each tokenizer registered with a database handle is stored as an object -** of the following type. All objects associated with a single database -** connection are stored in the singly-linked list starting at -** sqlite4.pTokenizer and connected by Fts5Tokenizer.pNext. -*/ -struct Fts5Tokenizer { - char *zName; /* Name of tokenizer (nul-terminated) */ - void *pCtx; - int (*xCreate)(void*, const char**, int, sqlite4_tokenizer**); - int (*xTokenize)(void*, sqlite4_tokenizer*, - const char*, int, int(*x)(void*, int, int, const char*, int, int, int) - ); - int (*xDestroy)(sqlite4_tokenizer *); - Fts5Tokenizer *pNext; -}; - -/* -** FTS5 specific index data. -** -** This object is part of a database schema, so it may be shared between -** multiple connections. -*/ -struct Fts5Index { - int nTokenizer; /* Elements in azTokenizer[] array */ - char **azTokenizer; /* Name and arguments for tokenizer */ -}; - -/* -** Expression grammar: -** -** phrase := PRIMITIVE -** phrase := PRIMITIVE * -** phrase := phrase + phrase -** phrase := phrase NEAR phrase -** -** expr := phrase -** expr := PRIMITIVE COLON phrase -** -** expr := expr NOT expr -** expr := expr AND expr -** expr := expr OR expr -** expr := LP expr RP -*/ - -/* -** Structure types used by this module. -*/ -typedef struct Fts5Expr Fts5Expr; -typedef struct Fts5ExprNode Fts5ExprNode; -typedef struct Fts5List Fts5List; -typedef struct Fts5MatchIter Fts5MatchIter; -typedef struct Fts5Parser Fts5Parser; -typedef struct Fts5ParserToken Fts5ParserToken; -typedef struct Fts5Phrase Fts5Phrase; -typedef struct Fts5Prefix Fts5Prefix; -typedef struct Fts5Size Fts5Size; -typedef struct Fts5Str Fts5Str; -typedef struct Fts5Token Fts5Token; - - -struct Fts5ParserToken { - int eType; /* Token type */ - int n; /* Size of z[] in bytes */ - const char *z; /* Token value */ -}; - -struct Fts5Parser { - Fts5Tokenizer *pTokenizer; - sqlite4_tokenizer *p; - sqlite4 *db; /* Database handle */ - - char *zErr; /* Error message (or NULL) */ - - const char *zExpr; /* Pointer to expression text (nul-term) */ - int iExpr; /* Current offset in zExpr */ - Fts5ParserToken next; /* Next token */ - - char **azCol; /* Column names of indexed table */ - int nCol; /* Size of azCol[] in bytes */ - int iRoot; /* Root page number of FTS index */ - - /* Space for dequoted copies of strings */ - char *aSpace; - int iSpace; - int nSpace; /* Total size of aSpace in bytes */ -}; - -struct Fts5List { - u8 *aData; - int nData; -}; - -struct Fts5Prefix { - u8 *aPk; /* Buffer containing PK */ - int nPk; /* Size of PK in bytes */ - Fts5Prefix *pNext; /* Next entry in query-time list */ - u8 *aList; - int nList; - int nAlloc; -}; - -struct Fts5Token { - /* TODO: The first three members are redundant in some senses, since the - ** same information is encoded in the aPrefix[]/nPrefix key. */ - int bPrefix; /* True for a prefix search */ - int n; /* Size of z[] in bytes */ - char *z; /* Token value */ - - KVByteArray *aPrefix; /* KV prefix to iterate through */ - KVSize nPrefix; /* Size of aPrefix in bytes */ - KVCursor *pCsr; /* Cursor to iterate thru entries for token */ - Fts5Prefix *pPrefix; /* Head of prefix list */ -}; - -struct Fts5Str { - Fts5Token *aToken; - int nToken; - u8 *aList; - int nList; - int nListAlloc; -}; - -struct Fts5Phrase { - int iCol; /* Column of table to search (-1 -> all) */ - int nStr; - Fts5Str *aStr; - int *aiNear; -}; - -struct Fts5ExprNode { - int eType; - Fts5Phrase *pPhrase; - Fts5ExprNode *pLeft; - Fts5ExprNode *pRight; - const u8 *aPk; /* Primary key of current entry (or null) */ - int nPk; /* Size of aPk[] in bytes */ -}; - -struct Fts5Expr { - Fts5ExprNode *pRoot; /* Root node of expression */ - int nPhrase; /* Number of Fts5Str objects in query */ - Fts5Str **apPhrase; /* All Fts5Str objects */ -}; - -/* -** FTS5 specific cursor data. -*/ -struct Fts5Cursor { - sqlite4 *db; - Fts5Info *pInfo; - Fts5Expr *pExpr; /* MATCH expression for this cursor */ - char *zExpr; /* Full text of MATCH expression */ - KVByteArray *aKey; /* Buffer for primary key */ - int nKeyAlloc; /* Bytes allocated at aKey[] */ - - KVCursor *pCsr; /* Cursor used to retrive values */ - Mem *aMem; /* Array of column values */ - int bMemValid; /* True if contents of aMem[] are valid */ - - Fts5Size *pSz; /* Local size data */ - Fts5Size *pGlobal; /* Global size data */ - i64 nGlobal; /* Total number of rows in table */ - - /* Arrays used by sqlite4_mi_row_count(). */ - int *anRowCS; - int *anRowC; - int *anRowS; - int *anRow; - - Fts5MatchIter *pIter; /* Used by mi_match_detail() */ -}; - -/* -** A deserialized 'size record' (see above). -*/ -struct Fts5Size { - int nCol; /* Number of columns in indexed table */ - int nStream; /* Number of streams */ - i64 *aSz; /* Token count for each C/S */ -}; - -/* -** This type is used when reading (decoding) an instance-list. -*/ -typedef struct InstanceList InstanceList; -struct InstanceList { - u8 *aList; - int nList; - int iList; - - /* The current entry */ - int iCol; - int iStream; - int iOff; -}; - -/* -** An instance of this structure is used by the sqlite4_mi_match_detail() -** API to iterate through matches. -*/ -struct Fts5MatchIter { - int bValid; /* True if aList[] is current row */ - int iCurrent; /* Current index in aList[] (or -1) */ - int iMatch; /* Current iMatch value */ - InstanceList *aList; /* One iterator for each phrase in expr */ -}; - -/* -** Return true for EOF, or false if the next entry is valid. -*/ -static int fts5InstanceListNext(InstanceList *p){ - int i = p->iList; - int bRet = 1; - - while( bRet && inList ){ - u32 iVal; - i += getVarint32(&p->aList[i], iVal); - if( (iVal & 0x03)==0x01 ){ - p->iCol = (iVal>>2); - p->iOff = 0; - } - else if( (iVal & 0x03)==0x03 ){ - p->iStream = (iVal>>2); - } - else{ - p->iOff += (iVal>>1); - bRet = 0; - } - } - if( bRet ){ - p->aList = 0; - } - - p->iList = i; - return bRet; -} - -static int fts5InstanceListEof(InstanceList *p){ - return (p->aList==0); -} - -static void fts5InstanceListAppend( - InstanceList *p, /* Instance list to append to */ - int iCol, /* Column of new entry */ - int iStream, /* Weight of new entry */ - int iOff /* Offset of new entry */ -){ - assert( iCol>=p->iCol ); - assert( iCol>p->iCol || iOff>=p->iOff ); - - if( iCol!=p->iCol ){ - p->iList += putVarint32(&p->aList[p->iList], (iCol<<2)|0x01); - p->iCol = iCol; - p->iOff = 0; - } - - if( iStream!=p->iStream ){ - p->iList += putVarint32(&p->aList[p->iList], (iStream<<2)|0x03); - p->iStream = iStream; - } - - p->iList += putVarint32(&p->aList[p->iList], (iOff-p->iOff)<<1); - p->iOff = iOff; - - assert( p->iList<=p->nList ); -} - -static void fts5InstanceListInit(u8 *aList, int nList, InstanceList *p){ - memset(p, 0, sizeof(InstanceList)); - p->aList = aList; - p->nList = nList; -} - -/* -** Return true if argument c is one of the special non-whitespace -** characters that ends an unquoted expression token. -*/ -static int fts5IsSpecial(char c){ - return (c==':' || c=='(' || c==')' || c=='+' || c=='"' || c=='*'); -} - -static int fts5NextToken( - Fts5Parser *pParse, /* Parser context */ - Fts5ParserToken *p /* OUT: Populate this object */ -){ - const char *z = pParse->zExpr; - char c; - - memset(p, 0, sizeof(Fts5ParserToken)); - - /* Skip past any whitespace */ - while( sqlite4Isspace(z[pParse->iExpr]) ) pParse->iExpr++; - - c = z[pParse->iExpr]; - if( c=='\0' ){ - p->eType = TOKEN_EOF; - } - - else if( c=='(' ){ - pParse->iExpr++; - p->eType = TOKEN_LP; - } - - else if( c==')' ){ - pParse->iExpr++; - p->eType = TOKEN_RP; - } - - else if( c==':' ){ - pParse->iExpr++; - p->eType = TOKEN_COLON; - } - - else if( c=='+' ){ - pParse->iExpr++; - p->eType = TOKEN_PLUS; - } - - else if( c=='*' ){ - pParse->iExpr++; - p->eType = TOKEN_STAR; - } - - else if( c=='"' ){ - char *zOut = &pParse->aSpace[pParse->iSpace]; - const char *zPrimitive = zOut; - int i = pParse->iExpr+1; - - while( z[i] ){ - if( z[i]=='"' ){ - if( z[i+1]=='"' ){ - i++; - }else{ - break; - } - } - *zOut++ = z[i]; - i++; - } - if( z[i]!='"' ){ - /* Mismatched quotation mark */ - return SQLITE4_ERROR; - } - - pParse->iExpr = i+1; - p->eType = TOKEN_PRIMITIVE; - p->z = zPrimitive; - p->n = (zOut - zPrimitive); - pParse->iSpace += (zOut - zPrimitive); - } - - else{ - const char *zPrimitive = &z[pParse->iExpr]; - int n = 0; - while( zPrimitive[n] - && fts5IsSpecial(zPrimitive[n])==0 - && sqlite4Isspace(zPrimitive[n])==0 - ){ - n++; - } - pParse->iExpr += n; - - if( n>=4 && memcmp(zPrimitive, "NEAR", 4)==0 ){ - int nNear = FTS5_DEFAULT_NEAR; - if( n>4 ){ - int i; - nNear = 0; - for(i=5; ieType = TOKEN_NEAR; - p->n = nNear; - p->z = 0; - }else if( n==3 && memcmp(zPrimitive, "NOT", 3)==0 ){ - p->eType = TOKEN_NOT; - } - else if( n==2 && memcmp(zPrimitive, "OR", 2)==0 ){ - p->eType = TOKEN_OR; - } - else if( n==3 && memcmp(zPrimitive, "AND", 3)==0 ){ - p->eType = TOKEN_AND; - }else{ - p->eType = TOKEN_PRIMITIVE; - p->z = zPrimitive; - p->n = n; - } - } - - return SQLITE4_OK; -} - -static int fts5NextToken2( - Fts5Parser *pParse, - Fts5ParserToken *p -){ - int rc = SQLITE4_OK; - if( pParse->iExpr==0 ){ - rc = fts5NextToken(pParse, p); - }else{ - *p = pParse->next; - } - - if( rc==SQLITE4_OK && p->eType!=TOKEN_EOF ){ - rc = fts5NextToken(pParse, &pParse->next); - } - - return rc; -} - -static int fts5PhraseNewStr( - Fts5Parser *pParse, /* Expression parsing context */ - Fts5Phrase *pPhrase, /* Phrase to add a new Fts5Str to */ - int nNear /* Value of nnn in NEAR/nnn operator */ -){ - const int nIncr = 4; - - if( (pPhrase->nStr % nIncr)==0 ){ - Fts5Str *aNew; - aNew = (Fts5Str *)sqlite4DbRealloc(pParse->db, - pPhrase->aStr, (pPhrase->nStr+nIncr)*sizeof(Fts5Str) - ); - if( !aNew ) return SQLITE4_NOMEM; - memset(&aNew[pPhrase->nStr], 0, nIncr*sizeof(Fts5Str)); - pPhrase->aStr = aNew; - } - if( pPhrase->nStr>0 ){ - if( ((pPhrase->nStr-1) % nIncr)==0 ){ - int *aNew; - aNew = (int *)sqlite4DbRealloc(pParse->db, - pPhrase->aiNear, (pPhrase->nStr+nIncr-1)*sizeof(int) - ); - if( !aNew ) return SQLITE4_NOMEM; - pPhrase->aiNear = aNew; - } - pPhrase->aiNear[pPhrase->nStr-1] = nNear; - } - - pPhrase->nStr++; - return SQLITE4_OK; -} - -/* -** Callback for fts5CountTokens(). -*/ -static int fts5CountTokensCb( - void *pCtx, - int iStream, - int iOff, - const char *z, int n, - int iSrc, int nSrc -){ - (*((int *)pCtx))++; - return 0; -} - -/* -** Count the number of tokens in document zDoc/nDoc using the tokenizer and -** tokenizer instance supplied as the first two arguments. Set *pnToken to -** the result before returning. -*/ -static int fts5CountTokens( - Fts5Tokenizer *pTokenizer, - sqlite4_tokenizer *p, - const char *zDoc, - int nDoc, - int *pnToken -){ - int nToken = 0; - int rc; - rc = pTokenizer->xTokenize((void *)&nToken, p, zDoc, nDoc, fts5CountTokensCb); - *pnToken = nToken; - return rc; -} - -struct AppendTokensCtx { - Fts5Parser *pParse; - Fts5Str *pStr; -}; - -static int fts5AppendTokensCb( - void *pCtx, - int iStream, - int iOff, - const char *z, int n, - int iSrc, int nSrc -){ - struct AppendTokensCtx *p = (struct AppendTokensCtx *)pCtx; - Fts5Parser *pParse = p->pParse; - Fts5Token *pToken; - char *zSpace; - int nUsed; - - pToken = &p->pStr->aToken[p->pStr->nToken]; - - zSpace = &pParse->aSpace[pParse->iSpace]; - nUsed = putVarint32((u8 *)zSpace, pParse->iRoot); - zSpace[nUsed++] = 0x24; - pToken->bPrefix = 0; - pToken->pPrefix = 0; - pToken->z = &zSpace[nUsed]; - pToken->n = n; - memcpy(pToken->z, z, n); - pToken->z[n] = '\0'; - - nUsed += (n+1); - pToken->aPrefix = (u8 *)zSpace; - pToken->nPrefix = nUsed; - pToken->pCsr = 0; - pParse->iSpace += nUsed; - p->pStr->nToken++; - - assert( pParse->iSpace<=pParse->nSpace ); - return 0; -} - -static int fts5AppendTokens( - Fts5Parser *pParse, - Fts5Str *pStr, const char *zPrim, - int nPrim -){ - struct AppendTokensCtx ctx; - ctx.pParse = pParse; - ctx.pStr = pStr; - - return pParse->pTokenizer->xTokenize( - (void *)&ctx, pParse->p , zPrim, nPrim, fts5AppendTokensCb - ); -} - -/* -** Append a new token to the current phrase. -*/ -static int fts5PhraseAppend( - Fts5Parser *pParse, - Fts5Phrase *pPhrase, - const char *zPrim, - int nPrim -){ - Fts5Tokenizer *pTok = pParse->pTokenizer; - int nToken; - int rc; - - rc = fts5CountTokens(pTok, pParse->p, zPrim, nPrim, &nToken); - if( rc==SQLITE4_OK && nToken>0 ){ - /* Extend the size of the token array by nToken entries */ - Fts5Str *pStr = &pPhrase->aStr[pPhrase->nStr-1]; - - pStr->aToken = sqlite4DbReallocOrFree(pParse->db, pStr->aToken, - (pStr->nToken + nToken) * sizeof(Fts5Token) - ); - if( !pStr->aToken ){ - rc = SQLITE4_NOMEM; - }else{ - rc = fts5AppendTokens(pParse, pStr, zPrim, nPrim); - } - } - - return rc; -} - -static int fts5PhraseAppendStar( - Fts5Parser *pParse, - Fts5Phrase *pPhrase -){ - Fts5Str *pStr = &pPhrase->aStr[pPhrase->nStr-1]; - Fts5Token *p = &pStr->aToken[pStr->nToken-1]; - - if( p->bPrefix ){ - return SQLITE4_ERROR; - } - p->bPrefix = 1; - p->nPrefix--; - return SQLITE4_OK; -} - -static void fts5PhraseFree(sqlite4 *db, Fts5Phrase *p){ - if( p ){ - int i; - for(i=0; inStr; i++){ - int iTok; - for(iTok=0; iTokaStr[i].nToken; iTok++){ - sqlite4KVCursorClose(p->aStr[i].aToken[iTok].pCsr); - } - sqlite4DbFree(db, p->aStr[i].aToken); - sqlite4DbFree(db, p->aStr[i].aList); - } - sqlite4DbFree(db, p->aiNear); - sqlite4DbFree(db, p->aStr); - sqlite4DbFree(db, p); - } -} - -static int fts5NextTokenOrPhrase( - Fts5Parser *pParse, /* Parser context */ - int *peType, /* OUT: Token type */ - Fts5Phrase **ppPhrase /* OUT: New phrase object */ -){ - int rc; - Fts5Phrase *pPhrase = 0; - Fts5ParserToken t; - - rc = fts5NextToken2(pParse, &t); - *peType = t.eType; - if( rc==SQLITE4_OK && t.eType==TOKEN_PRIMITIVE ){ - - /* Allocate the Fts5Phrase object */ - pPhrase = sqlite4DbMallocZero(pParse->db, sizeof(Fts5Phrase)); - if( pPhrase==0 ){ - rc = SQLITE4_NOMEM; - goto token_or_phrase_out; - } - pPhrase->iCol = -1; - - /* Check if this first primitive is a column name or not. */ - if( pParse->next.eType==TOKEN_COLON ){ - int iCol; - for(iCol=0; iColnCol; iCol++){ - if( sqlite4_strnicmp(pParse->azCol[iCol], t.z, t.n)==0 ) break; - } - if( iCol==pParse->nCol ){ - pParse->zErr = sqlite4MPrintf(pParse->db, - "fts5: no such column: %.*s", t.n, t.z - ); - rc = SQLITE4_ERROR; - goto token_or_phrase_out; - } - pPhrase->iCol = iCol; - - rc = fts5NextToken2(pParse, &t); - if( rc==SQLITE4_OK ) rc = fts5NextToken2(pParse, &t); - if( rc==SQLITE4_OK && t.eType!=TOKEN_PRIMITIVE ){ - rc = SQLITE4_ERROR; - } - if( rc!=SQLITE4_OK ) goto token_or_phrase_out; - } - - /* Add the first Fts5Str to the new phrase object. Populate it with the - ** results of tokenizing t.z/t.n. */ - rc = fts5PhraseNewStr(pParse, pPhrase, 0); - if( rc==SQLITE4_OK ){ - rc = fts5PhraseAppend(pParse, pPhrase, t.z, t.n); - } - if( rc==SQLITE4_OK && pParse->next.eType==TOKEN_STAR ){ - fts5NextToken2(pParse, &t); - rc = fts5PhraseAppendStar(pParse, pPhrase); - } - - /* Add any further primitives connected by "+" or NEAR operators. */ - while( rc==SQLITE4_OK && - (pParse->next.eType==TOKEN_PLUS || pParse->next.eType==TOKEN_NEAR) - ){ - rc = fts5NextToken2(pParse, &t); - if( rc==SQLITE4_OK ){ - if( t.eType==TOKEN_NEAR ){ - rc = fts5PhraseNewStr(pParse, pPhrase, t.n); - if( rc!=SQLITE4_OK ) goto token_or_phrase_out; - } - rc = fts5NextToken2(pParse, &t); - if( rc!=SQLITE4_OK ) goto token_or_phrase_out; - if( t.eType!=TOKEN_PRIMITIVE ){ - rc = SQLITE4_ERROR; - }else{ - rc = fts5PhraseAppend(pParse, pPhrase, t.z, t.n); - if( rc==SQLITE4_OK && pParse->next.eType==TOKEN_STAR ){ - fts5NextToken2(pParse, &t); - rc = fts5PhraseAppendStar(pParse, pPhrase); - } - } - } - } - } - - token_or_phrase_out: - if( rc!=SQLITE4_OK ){ - fts5PhraseFree(pParse->db, pPhrase); - }else{ - *ppPhrase = pPhrase; - } - return rc; -} - -static void fts5FreeExprNode(sqlite4 *db, Fts5ExprNode *pNode){ - if( pNode ){ - fts5PhraseFree(db, pNode->pPhrase); - fts5FreeExprNode(db, pNode->pLeft); - fts5FreeExprNode(db, pNode->pRight); - sqlite4DbFree(db, pNode); - } -} - -static void fts5ExpressionFree(sqlite4 *db, Fts5Expr *pExpr){ - if( pExpr ){ - fts5FreeExprNode(db, pExpr->pRoot); - sqlite4DbFree(db, pExpr->apPhrase); - sqlite4DbFree(db, pExpr); - } -} - -typedef struct ExprHier ExprHier; -struct ExprHier { - Fts5ExprNode **ppNode; - int nOpen; -}; - -static int fts5GrowExprHier( - sqlite4 *db, - int *pnAlloc, - ExprHier **paHier, - int nReq -){ - int rc = SQLITE4_OK; - int nAlloc = *pnAlloc; - if( nAlloceType = eType; - - pp = (*paHier)[*pnHier-1].ppNode; - pNode->pLeft = *pp; - *pp = pNode; - (*paHier)[*pnHier].ppNode = &pNode->pRight; - (*paHier)[*pnHier].nOpen = 0; - (*pnHier)++; - - return SQLITE4_OK; -} - -static void fts5FindStrings(Fts5ExprNode *p, Fts5Str ***papStr){ - if( p ){ - if( p->eType==TOKEN_PRIMITIVE ){ - int i; - Fts5Str *aStr = p->pPhrase->aStr; - for(i=0; ipPhrase->nStr; i++){ - **papStr = &aStr[i]; - (*papStr)++; - } - } - fts5FindStrings(p->pLeft, papStr); - fts5FindStrings(p->pRight, papStr); - } -} - -static int fts5ParseExpression( - sqlite4 *db, /* Database handle */ - Fts5Tokenizer *pTokenizer, /* Tokenizer module */ - sqlite4_tokenizer *p, /* Tokenizer instance */ - int iRoot, /* Root page number of FTS index */ - char **azCol, /* Array of column names (nul-term'd) */ - int nCol, /* Size of array azCol[] */ - const char *zExpr, /* FTS expression text */ - Fts5Expr **ppExpr, /* OUT: Expression object */ - char **pzErr /* OUT: Error message */ -){ - int rc = SQLITE4_OK; - Fts5Parser sParse; - int nStr = 0; - int nExpr; - int i; - Fts5Expr *pExpr; - - int nHier = 0; - int nHierAlloc = 0; - ExprHier *aHier = 0; - - nExpr = sqlite4Strlen30(zExpr); - memset(&sParse, 0, sizeof(Fts5Parser)); - sParse.zExpr = zExpr; - sParse.azCol = azCol; - sParse.nCol = nCol; - sParse.pTokenizer = pTokenizer; - sParse.p = p; - sParse.db = db; - sParse.iRoot = iRoot; - - pExpr = sqlite4DbMallocZero(db, sizeof(Fts5Expr) + nExpr*4); - if( !pExpr ) return SQLITE4_NOMEM; - sParse.aSpace = (char *)&pExpr[1]; - sParse.nSpace = nExpr*4; - - rc = fts5GrowExprHier(db, &nHierAlloc, &aHier, 1); - if( rc==SQLITE4_OK ){ - aHier[0].ppNode = &pExpr->pRoot; - aHier[0].nOpen = 0; - nHier = 1; - } - - while( rc==SQLITE4_OK ){ - int eType = 0; - Fts5Phrase *pPhrase = 0; - Fts5ExprNode *pNode = 0; - - rc = fts5NextTokenOrPhrase(&sParse, &eType, &pPhrase); - if( rc!=SQLITE4_OK || eType==TOKEN_EOF ) break; - - switch( eType ){ - case TOKEN_PRIMITIVE: { - Fts5ExprNode **pp = aHier[nHier-1].ppNode; - if( *pp ){ - rc = fts5AddBinary(db, TOKEN_AND, &nHier, &nHierAlloc, &aHier); - pp = aHier[nHier-1].ppNode; - } - if( rc==SQLITE4_OK ){ - pNode = sqlite4DbMallocZero(db, sizeof(Fts5ExprNode)); - if( pNode==0 ){ - rc = SQLITE4_NOMEM; - }else{ - pNode->eType = TOKEN_PRIMITIVE; - pNode->pPhrase = pPhrase; - *pp = pNode; - } - } - nStr += pPhrase->nStr; - break; - } - - case TOKEN_AND: - case TOKEN_OR: - case TOKEN_NOT: { - Fts5ExprNode **pp = aHier[nHier-1].ppNode; - - if( *pp==0 ){ - rc = SQLITE4_ERROR; - }else{ - while( nHier>1 - && aHier[nHier-1].nOpen==0 - && (*aHier[nHier-2].ppNode)->eType < eType - ){ - nHier--; - } - - rc = fts5AddBinary(db, eType, &nHier, &nHierAlloc, &aHier); - } - break; - } - - case TOKEN_LP: { - Fts5ExprNode **pp = aHier[nHier-1].ppNode; - if( *pp ){ - rc = SQLITE4_ERROR; - }else{ - aHier[nHier-1].nOpen++; - } - break; - } - - case TOKEN_RP: { - Fts5ExprNode **pp = aHier[nHier-1].ppNode; - if( *pp==0 ){ - rc = SQLITE4_ERROR; - }else{ - for(i=nHier-1; i>=0; i--){ - if( aHier[i].nOpen>0 ) break; - } - if( i<0 ){ - rc = SQLITE4_ERROR; - }else{ - aHier[i].nOpen--; - nHier = i+1; - } - } - break; - } - - default: - rc = SQLITE4_ERROR; - break; - } - - if( rc!=SQLITE4_OK ){ - sqlite4DbFree(db, pNode); - break; - } - } - - if( rc==SQLITE4_OK && *aHier[nHier-1].ppNode==0 ){ - rc = SQLITE4_ERROR; - } - for(i=0; rc==SQLITE4_OK && i0 ) rc = SQLITE4_ERROR; - } - - if( rc==SQLITE4_OK ){ - pExpr->nPhrase = nStr; - pExpr->apPhrase = (Fts5Str**)sqlite4DbMallocZero(db, sizeof(Fts5Str*)*nStr); - if( pExpr->apPhrase==0 ){ - rc = SQLITE4_NOMEM; - }else{ - Fts5Str **a = pExpr->apPhrase; - fts5FindStrings(pExpr->pRoot, &a); - } - } - - if( rc!=SQLITE4_OK ){ - fts5ExpressionFree(db, pExpr); - *pzErr = sParse.zErr; - pExpr = 0; - } - *ppExpr = pExpr; - sqlite4DbFree(db, aHier); - return rc; -} - -/* -** Search for the Fts5Tokenizer object named zName. Return a pointer to it -** if it exists, or NULL otherwise. -*/ -static Fts5Tokenizer *fts5FindTokenizer(sqlite4 *db, const char *zName){ - Fts5Tokenizer *p; - for(p=db->pTokenizer; p; p=p->pNext){ - if( 0==sqlite4_stricmp(zName, p->zName) ) break; - } - return p; -} - -static void fts5TokenizerCreate( - Parse *pParse, - Fts5Index *pFts, - Fts5Tokenizer **ppTokenizer, - sqlite4_tokenizer **pp -){ - Fts5Tokenizer *pTok; - char *zTok; /* Tokenizer name */ - const char **azArg; /* Tokenizer arguments */ - int nArg; /* Number of elements in azArg */ - - if( pFts->nTokenizer ){ - zTok = pFts->azTokenizer[0]; - azArg = (const char **)&pFts->azTokenizer[1]; - nArg = pFts->nTokenizer-1; - }else{ - zTok = "simple"; - azArg = 0; - nArg = 0; - } - - *ppTokenizer = pTok = fts5FindTokenizer(pParse->db, zTok); - if( !pTok ){ - sqlite4ErrorMsg(pParse, "no such tokenizer: \"%s\"", zTok); - }else{ - int rc = pTok->xCreate(pTok->pCtx, azArg, nArg, pp); - if( rc!=SQLITE4_OK ){ - assert( *pp==0 ); - sqlite4ErrorMsg(pParse, "error creating tokenizer"); - } - } -} - -static void fts5TokenizerDestroy(Fts5Tokenizer *pTok, sqlite4_tokenizer *p){ - if( p ) pTok->xDestroy(p); -} - -void sqlite4ShutdownFts5(sqlite4 *db){ - Fts5Tokenizer *p; - Fts5Tokenizer *pNext; - for(p=db->pTokenizer; p; p=pNext){ - pNext = p->pNext; - sqlite4DbFree(db, p); - } -} - -/* -** This function is used to install custom FTS tokenizers. -*/ -int sqlite4_create_tokenizer( - sqlite4 *db, - const char *zName, - void *pCtx, - int (*xCreate)(void*, const char**, int, sqlite4_tokenizer**), - int (*xTokenize)(void*, sqlite4_tokenizer*, - const char*, int, int(*x)(void*, int, int, const char*, int, int, int) - ), - int (*xDestroy)(sqlite4_tokenizer *) -){ - int rc = SQLITE4_OK; - sqlite4_mutex_enter(db->mutex); - - /* It is not possible to override an existing tokenizer */ - if( fts5FindTokenizer(db, zName) ){ - rc = SQLITE4_ERROR; - }else{ - int nName = sqlite4Strlen30(zName); - Fts5Tokenizer *pTokenizer = (Fts5Tokenizer *)sqlite4DbMallocZero(db, - sizeof(Fts5Tokenizer) + nName+1 - ); - if( !pTokenizer ){ - rc = SQLITE4_NOMEM; - }else{ - pTokenizer->pCtx = pCtx; - pTokenizer->xCreate = xCreate; - pTokenizer->xTokenize = xTokenize; - pTokenizer->xDestroy = xDestroy; - pTokenizer->zName = (char *)&pTokenizer[1]; - memcpy(pTokenizer->zName, zName, nName+1); - - pTokenizer->pNext = db->pTokenizer; - db->pTokenizer = pTokenizer; - } - } - - rc = sqlite4ApiExit(db, rc); - sqlite4_mutex_leave(db->mutex); - return rc; -} - -/* -** Return the size of an Fts5Index structure, in bytes. -*/ -int sqlite4Fts5IndexSz(void){ - return sizeof(Fts5Index); -} - -/* -** Initialize the fts5 specific part of the index object passed as the -** second argument. -*/ -void sqlite4Fts5IndexInit(Parse *pParse, Index *pIdx, ExprList *pArgs){ - Fts5Index *pFts = pIdx->pFts; - - if( pArgs ){ - int i; - for(i=0; pParse->nErr==0 && inExpr; i++){ - char *zArg = pArgs->a[i].zName; - char *zVal = pArgs->a[i].pExpr->u.zToken; - - if( zArg && sqlite4_stricmp(zArg, "tokenizer")==0 ){ - /* zVal is the name of the tokenizer to use. Any subsequent arguments - ** that do not contain assignment operators (=) are also passed to - ** the tokenizer. Figure out how many bytes of space are required for - ** all. */ - int j; - char *pSpace; - int nByte = sqlite4Strlen30(zVal) + 1; - for(j=i+1; jnExpr; j++){ - ExprListItem *pItem = &pArgs->a[j]; - if( pItem->zName ) break; - nByte += sqlite4Strlen30(pItem->pExpr->u.zToken) + 1; - } - nByte += sizeof(char *) * (j-i); - pFts->azTokenizer = (char **)sqlite4DbMallocZero(pParse->db, nByte); - if( pFts->azTokenizer==0 ) return; - pFts->nTokenizer = (j-i); - - pSpace = (char *)&pFts->azTokenizer[j-i]; - for(j=i; jnExpr; j++){ - ExprListItem *pItem = &pArgs->a[j]; - if( pItem->zName && j>i ){ - break; - }else{ - int nToken = sqlite4Strlen30(pItem->pExpr->u.zToken); - memcpy(pSpace, pItem->pExpr->u.zToken, nToken+1); - pFts->azTokenizer[j-i] = pSpace; - pSpace += nToken+1; - } - } - - /* If this function is being called as part of a CREATE INDEX statement - ** issued by the user (to create a new index) check if the tokenizer - ** is valid. If not, return an error. Do not do this if this function - ** is being called as part of parsing an existing database schema. - */ - if( pParse->db->init.busy==0 ){ - Fts5Tokenizer *pTok = 0; - sqlite4_tokenizer *t = 0; - - fts5TokenizerCreate(pParse, pFts, &pTok, &t); - fts5TokenizerDestroy(pTok, t); - } - } - else{ - sqlite4ErrorMsg(pParse,"unrecognized argument: \"%s\"", zArg?zArg:zVal); - } - } - } -} - -void sqlite4Fts5IndexFree(sqlite4 *db, Index *pIdx){ - if( pIdx->pFts ){ - sqlite4DbFree(db, pIdx->pFts->azTokenizer); - } -} - - -/* -** Context structure passed to tokenizer callback when tokenizing a document. -** -** The hash table maps between tokens and TokenizeTerm structures. -** -** TokenizeTerm structures are allocated using sqlite4DbMalloc(). Immediately -** following the structure in memory is the token itself (TokenizeTerm.nToken -** bytes of data). Following this is the list of token instances in the same -** format as it is stored in the database. -** -** All of the above is a single allocation, size TokenizeTerm.nAlloc bytes. -** If the initial allocation is too small, it is extended using -** sqlite4DbRealloc(). -*/ -typedef struct TokenizeCtx TokenizeCtx; -typedef struct TokenizeTerm TokenizeTerm; -struct TokenizeCtx { - int rc; - int iCol; - int nCol; /* Number of columns in table */ - sqlite4 *db; - int nMax; - i64 *aSz; /* Number of tokens in each column/stream */ - int nStream; /* Number of streams in document */ - Hash hash; -}; -struct TokenizeTerm { - int iStream; /* Weight of previous entry */ - int iCol; /* Column containing previous entry */ - int iOff; /* Token offset of previous entry */ - int nToken; /* Size of token in bytes */ - int nData; /* Bytes of data in value */ - int nAlloc; /* Bytes of data allocated */ -}; - -TokenizeTerm *fts5TokenizeAppendInt( - TokenizeCtx *p, - TokenizeTerm *pTerm, - int iVal -){ - unsigned char *a; - int nSpace = pTerm->nAlloc-pTerm->nData-pTerm->nToken-sizeof(TokenizeTerm); - - if( nSpace < 5 ){ - int nAlloc = (pTerm->nAlloc<256) ? 256 : pTerm->nAlloc * 2; - pTerm = sqlite4DbReallocOrFree(p->db, pTerm, nAlloc); - if( !pTerm ) return 0; - pTerm->nAlloc = sqlite4DbMallocSize(p->db, pTerm); - } - - a = &(((unsigned char *)&pTerm[1])[pTerm->nToken+pTerm->nData]); - pTerm->nData += putVarint32(a, iVal); - return pTerm; -} - -static int fts5TokenizeCb( - void *pCtx, - int iStream, - int iOff, - const char *zToken, - int nToken, - int iSrc, - int nSrc -){ - TokenizeCtx *p = (TokenizeCtx *)pCtx; - sqlite4 *db = p->db; - TokenizeTerm *pTerm = 0; - - /* TODO: Error here if iStream is out of range */ - - if( nToken>p->nMax ) p->nMax = nToken; - - if( iStream>=p->nStream ){ - int iCol; - int nOld = p->nStream; - int nNew = 4; - i64 *aNew; - - while( nNew<=iStream ) nNew = nNew*2; - aNew = sqlite4DbMallocZero(db, nNew*p->nCol*sizeof(i64)); - if( aNew==0 ) goto tokenize_cb_out; - - for(iCol=0; iColnCol; iCol++){ - int iStr; - for(iStr=0; iStraSz[nOld*iCol + iStr]; - } - } - - sqlite4DbFree(db, p->aSz); - p->aSz = aNew; - p->nStream = nNew; - } - p->aSz[p->iCol * p->nStream + iStream]++; - - pTerm = (TokenizeTerm *)sqlite4HashFind(&p->hash, zToken, nToken); - if( pTerm==0 ){ - /* Size the initial allocation so that it fits in the lookaside buffer */ - int nAlloc = sizeof(TokenizeTerm) + nToken + 32; - - pTerm = sqlite4DbMallocZero(p->db, nAlloc); - if( pTerm ){ - pTerm->nAlloc = sqlite4DbMallocSize(p->db, pTerm); - pTerm->nToken = nToken; - memcpy(&pTerm[1], zToken, nToken); - } - if( pTerm==0 ) goto tokenize_cb_out; - }else{ - sqlite4HashInsert(&p->hash, zToken, nToken, 0); - } - - if( iStream!=pTerm->iStream ){ - pTerm = fts5TokenizeAppendInt(p, pTerm, (iStream << 2) | 0x00000003); - if( !pTerm ) goto tokenize_cb_out; - pTerm->iStream = iStream; - } - - if( pTerm && p->iCol!=pTerm->iCol ){ - pTerm = fts5TokenizeAppendInt(p, pTerm, (p->iCol << 2) | 0x00000001); - if( !pTerm ) goto tokenize_cb_out; - pTerm->iCol = p->iCol; - pTerm->iOff = 0; - } - - pTerm = fts5TokenizeAppendInt(p, pTerm, (iOff-pTerm->iOff) << 1); - if( !pTerm ) goto tokenize_cb_out; - pTerm->iOff = iOff; - -tokenize_cb_out: - sqlite4HashInsert(&p->hash, (char *)&pTerm[1], nToken, pTerm); - if( !pTerm ){ - p->rc = SQLITE4_NOMEM; - return 1; - } - - return 0; -} - -static int fts5LoadSizeRecord( - sqlite4 *db, /* Database handle */ - u8 *aKey, int nKey, /* KVStore key */ - int nMinStream, /* Space for at least this many streams */ - Fts5Info *pInfo, /* Info record */ - i64 *pnRow, /* non-NULL when reading global record */ - Fts5Size **ppSz /* OUT: Loaded size record */ -){ - Fts5Size *pSz = 0; /* Size object */ - KVCursor *pCsr = 0; /* Cursor used to read global record */ - int rc; - - rc = sqlite4KVStoreOpenCursor(db->aDb[pInfo->iDb].pKV, &pCsr); - if( rc==SQLITE4_OK ){ - rc = sqlite4KVCursorSeek(pCsr, aKey, nKey, 0); - if( rc==SQLITE4_NOTFOUND ){ - if( pnRow ){ - int nByte = sizeof(Fts5Size) + sizeof(i64) * pInfo->nCol * nMinStream; - pSz = sqlite4DbMallocZero(db, nByte); - if( pSz==0 ){ - rc = SQLITE4_NOMEM; - }else{ - pSz->aSz = (i64 *)&pSz[1]; - pSz->nStream = nMinStream; - pSz->nCol = pInfo->nCol; - *pnRow = 0; - rc = SQLITE4_OK; - } - }else{ - rc = SQLITE4_CORRUPT_BKPT; - } - }else if( rc==SQLITE4_OK ){ - const u8 *aData = 0; - int nData = 0; - rc = sqlite4KVCursorData(pCsr, 0, -1, &aData, &nData); - if( rc==SQLITE4_OK ){ - int iOff = 0; - int nStream = 0; - int nAlloc; - - /* If pnRow is not NULL, then this is the global record. Read the - ** number of documents in the table from the start of the record. */ - if( pnRow ){ - iOff += sqlite4GetVarint(&aData[iOff], (u64 *)pnRow); - } - iOff += getVarint32(&aData[iOff], nStream); - nAlloc = (nStream < nMinStream ? nMinStream : nStream); - - pSz = sqlite4DbMallocZero(db, - sizeof(Fts5Size) + sizeof(i64) * pInfo->nCol * nAlloc - ); - if( pSz==0 ){ - rc = SQLITE4_NOMEM; - }else{ - int iCol = 0; - pSz->aSz = (i64 *)&pSz[1]; - pSz->nCol = pInfo->nCol; - pSz->nStream = nAlloc; - while( iOffaSz[iCol*nAlloc]; - for(i=0; i=0 ){ - iOff += sqlite4PutVarint(&a[iOff], nRow); - } - iOff += sqlite4PutVarint(&a[iOff], pSz->nStream); - - for(iCol=0; iColnCol; iCol++){ - int i; - for(i=0; inStream; i++){ - iOff += sqlite4PutVarint(&a[iOff], pSz->aSz[iCol*pSz->nStream+i]); - } - } - - return sqlite4KVStoreReplace(p, aKey, nKey, a, iOff); -} - -static int fts5CsrLoadGlobal(Fts5Cursor *pCsr){ - int rc = SQLITE4_OK; - if( pCsr->pGlobal==0 ){ - int nKey; - u8 aKey[10]; - nKey = putVarint32(aKey, pCsr->pInfo->iRoot); - aKey[nKey++] = 0x00; - rc = fts5LoadSizeRecord( - pCsr->db, aKey, nKey, 0, pCsr->pInfo, &pCsr->nGlobal, &pCsr->pGlobal - ); - } - return rc; -} - -static int fts5CsrLoadSz(Fts5Cursor *pCsr){ - int rc = SQLITE4_OK; - if( pCsr->pSz==0 ){ - sqlite4 *db = pCsr->db; - Fts5Info *pInfo = pCsr->pInfo; - u8 *aKey; - int nKey = 0; - int nPk = pCsr->pExpr->pRoot->nPk; - - aKey = (u8 *)sqlite4DbMallocZero(db, 10 + nPk); - if( !aKey ) return SQLITE4_NOMEM; - - nKey = putVarint32(aKey, pInfo->iRoot); - aKey[nKey++] = 0x00; - memcpy(&aKey[nKey], pCsr->pExpr->pRoot->aPk, nPk); - nKey += nPk; - - rc = fts5LoadSizeRecord(pCsr->db, aKey, nKey, 0, pInfo, 0, &pCsr->pSz); - sqlite4DbFree(db, aKey); - } - - return rc; -} - - -/* -** Update an fts index. -*/ -int sqlite4Fts5Update( - sqlite4 *db, /* Database handle */ - Fts5Info *pInfo, /* Description of fts index to update */ - int iRoot, - Mem *pKey, /* Primary key blob */ - Mem *aArg, /* Array of arguments (see above) */ - int bDel, /* True for a delete, false for insert */ - char **pzErr /* OUT: Error message */ -){ - int i; - int rc = SQLITE4_OK; - KVStore *pStore; - TokenizeCtx sCtx; - int nTnum = 0; - u32 dummy = 0; - - u8 *aSpace = 0; - int nSpace = 0; - - const u8 *pPK; - int nPK; - HashElem *pElem; - - if( iRoot==0 ) iRoot = pInfo->iRoot; - pStore = db->aDb[pInfo->iDb].pKV; - - memset(&sCtx, 0, sizeof(sCtx)); - sCtx.db = db; - sCtx.nCol = pInfo->nCol; - sqlite4HashInit(db->pEnv, &sCtx.hash, 1); - - pPK = (const u8 *)sqlite4_value_blob(pKey); - nPK = sqlite4_value_bytes(pKey); - - nTnum = getVarint32(pPK, dummy); - nPK -= nTnum; - pPK += nTnum; - - for(i=0; rc==SQLITE4_OK && inCol; i++){ - sqlite4_value *pArg = (sqlite4_value *)(&aArg[i]); - if( pArg->flags & MEM_Str ){ - const char *zText; - int nText; - - zText = (const char *)sqlite4_value_text(pArg); - nText = sqlite4_value_bytes(pArg); - sCtx.iCol = i; - rc = pInfo->pTokenizer->xTokenize( - &sCtx, pInfo->p, zText, nText, fts5TokenizeCb - ); - } - } - - /* Allocate enough space to serialize all the stuff that needs to - ** be inserted into the database. Specifically: - ** - ** * Space for index record keys, - ** * space for the size record and key for this document, and - ** * space for the updated global size record for the document set. - ** - ** To make it easier, the below allocates enough space to simultaneously - ** store the largest index record key and the largest possible global - ** size record. - */ - nSpace = (sqlite4VarintLen(iRoot) + 2 + sCtx.nMax + nPK) + - (9 * (2 + pInfo->nCol * sCtx.nStream)); - aSpace = sqlite4DbMallocRaw(db, nSpace); - if( aSpace==0 ) rc = SQLITE4_NOMEM; - - for(pElem=sqliteHashFirst(&sCtx.hash); pElem; pElem=sqliteHashNext(pElem)){ - TokenizeTerm *pTerm = (TokenizeTerm *)sqliteHashData(pElem); - if( rc==SQLITE4_OK ){ - int nToken = sqliteHashKeysize(pElem); - char *zToken = (char *)sqliteHashKey(pElem); - u8 *aKey = aSpace; - int nKey; - - nKey = putVarint32(aKey, iRoot); - aKey[nKey++] = 0x24; - memcpy(&aKey[nKey], zToken, nToken); - nKey += nToken; - aKey[nKey++] = 0x00; - memcpy(&aKey[nKey], pPK, nPK); - nKey += nPK; - - if( bDel ){ - /* delete key aKey/nKey from the index */ - rc = sqlite4KVStoreReplace(pStore, aKey, nKey, 0, -1); - }else{ - /* Insert a new entry for aKey/nKey into the fts index */ - const KVByteArray *aData = (const KVByteArray *)&pTerm[1]; - aData += pTerm->nToken; - rc = sqlite4KVStoreReplace(pStore, aKey, nKey, aData, pTerm->nData); - } - } - sqlite4DbFree(db, pTerm); - } - - /* Write the size record into the db */ - if( rc==SQLITE4_OK ){ - u8 *aKey = aSpace; - int nKey; - - nKey = putVarint32(aKey, iRoot); - aKey[nKey++] = 0x00; - memcpy(&aKey[nKey], pPK, nPK); - nKey += nPK; - - if( bDel==0 ){ - Fts5Size sSz; - sSz.nCol = pInfo->nCol; - sSz.nStream = sCtx.nStream; - sSz.aSz = sCtx.aSz; - rc = fts5StoreSizeRecord(pStore, aKey, nKey, &sSz, -1, &aKey[nKey]); - }else{ - rc = sqlite4KVStoreReplace(pStore, aKey, nKey, 0, -1); - } - } - - /* Update the global record */ - if( rc==SQLITE4_OK ){ - Fts5Size *pSz; /* Deserialized global size record */ - i64 nRow; /* Number of rows in indexed table */ - u8 *aKey = aSpace; /* Space to format the global record key */ - int nKey; /* Size of global record key in bytes */ - - nKey = putVarint32(aKey, iRoot); - aKey[nKey++] = 0x00; - rc = fts5LoadSizeRecord(db, aKey, nKey, sCtx.nStream, pInfo, &nRow, &pSz); - assert( rc!=SQLITE4_OK || pSz->nStream>=sCtx.nStream ); - - if( rc==SQLITE4_OK ){ - int iCol; - for(iCol=0; iColnCol; iCol++){ - int iStr; - i64 *aIn = &sCtx.aSz[iCol * sCtx.nStream]; - i64 *aOut = &pSz->aSz[iCol * pSz->nStream]; - for(iStr=0; iStrdb; - Fts5Info *pInfo; /* p4 argument for FtsUpdate opcode */ - int nByte; - - nByte = sizeof(Fts5Info); - if( bCol ){ - int i; - int nCol = pIdx->pTable->nCol; - for(i=0; ipTable->aCol[i].zName; - nByte += sqlite4Strlen30(zCol) + 1; - } - nByte += nCol * sizeof(char *); - } - - pInfo = sqlite4DbMallocZero(db, nByte); - if( pInfo ){ - pInfo->iDb = sqlite4SchemaToIndex(db, pIdx->pSchema); - pInfo->iRoot = pIdx->tnum; - pInfo->iTbl = sqlite4FindPrimaryKey(pIdx->pTable, 0)->tnum; - pInfo->nCol = pIdx->pTable->nCol; - fts5TokenizerCreate(pParse, pIdx->pFts, &pInfo->pTokenizer, &pInfo->p); - - if( pInfo->p==0 ){ - assert( pParse->nErr ); - sqlite4DbFree(db, pInfo); - pInfo = 0; - } - else if( bCol ){ - int i; - char *p; - int nCol = pIdx->pTable->nCol; - - pInfo->azCol = (char **)(&pInfo[1]); - p = (char *)(&pInfo->azCol[nCol]); - for(i=0; ipTable->aCol[i].zName; - int n = sqlite4Strlen30(zCol) + 1; - pInfo->azCol[i] = p; - memcpy(p, zCol, n); - p += n; - } - } - } - - return pInfo; -} - -void sqlite4Fts5CodeUpdate( - Parse *pParse, - Index *pIdx, - int iRegRoot, - int iRegPk, - int iRegData, - int bDel -){ - Vdbe *v; - Fts5Info *pInfo; /* p4 argument for FtsUpdate opcode */ - - if( 0==(pInfo = fts5InfoCreate(pParse, pIdx, 0)) ) return; - - v = sqlite4GetVdbe(pParse); - sqlite4VdbeAddOp3(v, OP_FtsUpdate, iRegPk, iRegRoot, iRegData); - sqlite4VdbeChangeP4(v, -1, (const char *)pInfo, P4_FTS5INFO); - sqlite4VdbeChangeP5(v, (u8)bDel); -} - -void sqlite4Fts5CodeQuery( - Parse *pParse, - Index *pIdx, - int iCsr, - int iJump, - int iRegMatch -){ - Vdbe *v; - Fts5Info *pInfo; /* p4 argument for FtsOpen opcode */ - - if( 0==(pInfo = fts5InfoCreate(pParse, pIdx, 1)) ) return; - - v = sqlite4GetVdbe(pParse); - sqlite4VdbeAddOp3(v, OP_FtsOpen, iCsr, iJump, iRegMatch); - sqlite4VdbeChangeP4(v, -1, (const char *)pInfo, P4_FTS5INFO); -} - -void sqlite4Fts5FreeInfo(sqlite4 *db, Fts5Info *p){ - if( db->pnBytesFreed==0 ){ - if( p->p ) p->pTokenizer->xDestroy(p->p); - sqlite4DbFree(db, p); - } -} - -void sqlite4Fts5CodeCksum( - Parse *pParse, - Index *pIdx, - int iCksum, - int iReg, - int bIdx /* True for fts index, false for table */ -){ - Vdbe *v; - Fts5Info *pInfo; /* p4 argument for FtsCksum opcode */ - - if( 0==(pInfo = fts5InfoCreate(pParse, pIdx, 0)) ) return; - - v = sqlite4GetVdbe(pParse); - sqlite4VdbeAddOp3(v, OP_FtsCksum, iCksum, 0, iReg); - sqlite4VdbeChangeP4(v, -1, (const char *)pInfo, P4_FTS5INFO); - sqlite4VdbeChangeP5(v, bIdx); -} - -/* -** Calculate a 64-bit checksum for a term instance. The index checksum is -** the XOR of the checksum for each term instance in the table. A term -** instance checksum is calculated based on: -** -** * the term itself, -** * the pk of the row the instance appears in, -** * the weight assigned to the instance, -** * the column number, and -** * the term offset. -*/ -static i64 fts5TermInstanceCksum( - const u8 *aTerm, int nTerm, - const u8 *aPk, int nPk, - int iStream, - int iCol, - int iOff -){ - int i; - i64 cksum = 0; - - /* Add the term to the checksum */ - for(i=0; ipPK, p->nPK, - (const u8 *)zToken, nToken, iStream, p->iCol, iOff - ); - - p->cksum = (p->cksum ^ cksum); - return 0; -} - -int sqlite4Fts5RowCksum( - sqlite4 *db, /* Database handle */ - Fts5Info *pInfo, /* Index description */ - Mem *pKey, /* Primary key blob */ - Mem *aArg, /* Array of column values */ - i64 *piCksum /* OUT: Checksum value */ -){ - int i; - int rc = SQLITE4_OK; - CksumCtx sCtx; - int nTnum = 0; - u32 dummy = 0; - - sCtx.cksum = 0; - - sCtx.pPK = (const u8 *)sqlite4_value_blob(pKey); - sCtx.nPK = sqlite4_value_bytes(pKey); - nTnum = getVarint32(sCtx.pPK, dummy); - sCtx.nPK -= nTnum; - sCtx.pPK += nTnum; - - for(i=0; rc==SQLITE4_OK && inCol; i++){ - sqlite4_value *pArg = (sqlite4_value *)(&aArg[i]); - if( pArg->flags & MEM_Str ){ - const char *zText; - int nText; - - zText = (const char *)sqlite4_value_text(pArg); - nText = sqlite4_value_bytes(pArg); - sCtx.iCol = i; - rc = pInfo->pTokenizer->xTokenize( - &sCtx, pInfo->p, zText, nText, fts5CksumCb - ); - } - } - - *piCksum = sCtx.cksum; - return rc; -} - -/* -** Obtain the primary key value from the entry cursor pToken->pCsr currently -** points to. Set *paPk to point to a buffer containing the PK, and *pnPk -** to the size of the buffer in bytes before returning. -** -** Return SQLITE4_OK if everything goes according to plan, or an error code -** if an error occurs. If an error occurs the final values of *paPk and *pnPk -** are undefined. -*/ -static int fts5TokenPk(Fts5Token *p, const u8 **paPk, int *pnPk){ - int rc; - - if( p->pCsr ){ - const u8 *aKey; - int nKey; - - rc = sqlite4KVCursorKey(p->pCsr, &aKey, &nKey); - if( rc==SQLITE4_OK ){ - if( nKeynPrefix || memcmp(p->aPrefix, aKey, p->nPrefix) ){ - rc = SQLITE4_NOTFOUND; - }else if( p->bPrefix==0 ){ - *paPk = &aKey[p->nPrefix]; - *pnPk = nKey - p->nPrefix; - }else{ - const u8 *z = &aKey[p->nPrefix]; - while( *(z++)!='\0' ); - *paPk = z; - *pnPk = nKey - (z-aKey); - } - } - }else{ - if( p->pPrefix ){ - *paPk = p->pPrefix->aPk; - *pnPk = p->pPrefix->nPk; - rc = SQLITE4_OK; - }else{ - rc = SQLITE4_NOTFOUND; - } - } - - return rc; -} - -static int fts5TokenAdvance(sqlite4 *db, Fts5Token *pToken){ - int rc; - if( pToken->pCsr ){ - rc = sqlite4KVCursorNext(pToken->pCsr); - }else if( pToken->pPrefix ){ - Fts5Prefix *pDel = pToken->pPrefix; - pToken->pPrefix = pDel->pNext; - sqlite4DbFree(db, pDel->aList); - sqlite4DbFree(db, pDel); - rc = SQLITE4_OK; - }else{ - rc = SQLITE4_NOTFOUND; - } - return rc; -} - -static int fts5TokenData(Fts5Token *pToken, const u8 **paData, int *pnData){ - int rc; - if( pToken->pCsr ){ - rc = sqlite4KVCursorData(pToken->pCsr, 0, -1, paData, pnData); - }else if( pToken->pPrefix ){ - *paData = pToken->pPrefix->aList; - *pnData = pToken->pPrefix->nList; - rc = SQLITE4_OK; - }else{ - rc = SQLITE4_NOTFOUND; - } - - return rc; -} - -/* -** Compare keys (aLeft/nLeft) and (aRight/nRight) using the ordinary memcmp() -** method. Except, if either aLeft or aRight are NULL, consider them larger -** than all other values. -*/ -static int fts5KeyCompare( - const u8 *aLeft, int nLeft, - const u8 *aRight, int nRight -){ - int res; - int nMin; - - res = (aLeft==0) - (aRight==0); - if( res==0 ){ - nMin = (nLeft > nRight) ? nRight : nLeft; - res = memcmp(aLeft, aRight, nMin); - } - return (res ? res : (nLeft-nRight)); -} - -static int fts5ListMerge(sqlite4 *db, Fts5List *p1, Fts5List *p2, int bFree){ - InstanceList in1; - InstanceList in2; - InstanceList out; - - memset(&out, 0, sizeof(InstanceList)); - if( p1->nData==0 && p2->nData==0 ) return SQLITE4_OK; - out.nList = p1->nData+p2->nData; - out.aList = sqlite4DbMallocRaw(db, out.nList); - if( !out.aList ) return SQLITE4_NOMEM; - fts5InstanceListInit(p1->aData, p1->nData, &in1); - fts5InstanceListInit(p2->aData, p2->nData, &in2); - - fts5InstanceListNext(&in1); - fts5InstanceListNext(&in2); - - while( fts5InstanceListEof(&in1)==0 || fts5InstanceListEof(&in2)==0 ){ - InstanceList *pAdv; - - if( fts5InstanceListEof(&in1) ){ - pAdv = &in2; - }else if( fts5InstanceListEof(&in2) ){ - pAdv = &in1; - }else if( in1.iCol==in2.iCol && in1.iOff==in2.iOff ){ - pAdv = &in1; - fts5InstanceListNext(&in2); - }else if( in1.iColiCol, pAdv->iStream, pAdv->iOff); - fts5InstanceListNext(pAdv); - } - - if( bFree ){ - sqlite4DbFree(db, p1->aData); - sqlite4DbFree(db, p2->aData); - } - memset(p2, 0, sizeof(Fts5List)); - p1->aData = out.aList; - p1->nData = out.iList; - return SQLITE4_OK; -} - -static void fts5PrefixMerge(Fts5Prefix **pp, Fts5Prefix *p2){ - Fts5Prefix *p1 = *pp; - Fts5Prefix *pRet = 0; - Fts5Prefix **ppWrite = &pRet; - - while( p1 || p2 ){ - Fts5Prefix **ppAdv = 0; - if( p1==0 ){ - ppAdv = &p2; - }else if( p2==0 ){ - ppAdv = &p1; - }else{ - int res = fts5KeyCompare(p1->aPk, p1->nPk, p2->aPk, p2->nPk); - assert( res!=0 ); - if( res<0 ){ - ppAdv = &p1; - }else{ - ppAdv = &p2; - } - } - - *ppWrite = *ppAdv; - ppWrite = &((*ppWrite)->pNext); - *ppAdv = (*ppAdv)->pNext; - *ppWrite = 0; - } - - *pp = pRet; -} - -static int fts5FindPrefixes(sqlite4 *db, Fts5Info *pInfo, Fts5Token *pToken){ - int rc = SQLITE4_OK; - HashElem *pElem; - Hash hash; - - assert( pToken->bPrefix ); - assert( pToken->aPrefix[pToken->nPrefix-1]!='\0' ); - sqlite4HashInit(db->pEnv, &hash, 1); - - do { - const u8 *aData; - int nData; - const u8 *aPk; - int nPk; - - rc = fts5TokenPk(pToken, &aPk, &nPk); - if( rc==SQLITE4_OK ){ - rc = fts5TokenData(pToken, &aData, &nData); - } - if( rc==SQLITE4_OK ){ - Fts5Prefix *p; - - p = (Fts5Prefix *)sqlite4HashFind(&hash, (const char *)aPk, nPk); - if( !p ){ - p = (Fts5Prefix *)sqlite4DbMallocZero(db, sizeof(Fts5Prefix) + nPk); - if( !p ){ - rc = SQLITE4_NOMEM; - }else{ - void *pFree; - p->aPk = (u8 *)&p[1]; - p->nPk = nPk; - memcpy(p->aPk, aPk, nPk); - pFree = sqlite4HashInsert(&hash, (const char *)p->aPk, p->nPk, p); - if( pFree ){ - assert( pFree==(void *)p ); - rc = SQLITE4_NOMEM; - sqlite4DbFree(db, pFree); - } - } - } - - if( rc==SQLITE4_OK ){ - int nReq = nData + sqlite4VarintLen(nData); - while( (p->nList + nReq) > p->nAlloc ){ - int nAlloc = (p->nAlloc ? p->nAlloc*2 : 64); - p->aList = sqlite4DbReallocOrFree(db, p->aList, nAlloc); - if( !p->aList ){ - rc = SQLITE4_NOMEM; - break; - } - p->nAlloc = nAlloc; - } - } - - if( rc==SQLITE4_OK ){ - p->nList += putVarint32(&p->aList[p->nList], nData); - memcpy(&p->aList[p->nList], aData, nData); - p->nList += nData; - } - - if( rc==SQLITE4_OK ){ - rc = fts5TokenAdvance(db, pToken); - } - } - }while( rc==SQLITE4_OK ); - if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK; - - if( rc==SQLITE4_OK ){ - Fts5List *aMerge; - aMerge = (Fts5List *)sqlite4DbMallocZero(db, sizeof(Fts5List) * 32); - if( !aMerge ) rc = SQLITE4_NOMEM; - - for(pElem=sqliteHashFirst(&hash); pElem; pElem=sqliteHashNext(pElem)){ - Fts5Prefix *p = (Fts5Prefix *)sqliteHashData(pElem); - Fts5List list = {0, 0}; - int i = 0; - int iLevel; - - memset(aMerge, 0, sizeof(Fts5List)*32); - while( inList && rc==SQLITE4_OK ){ - u32 n; - i += getVarint32(&p->aList[i], n); - list.aData = &p->aList[i]; - list.nData = n; - i += n; - - for(iLevel=0; rc==SQLITE4_OK && iLevel<32; iLevel++){ - if( aMerge[iLevel].aData==0 ){ - aMerge[iLevel] = list; - break; - }else{ - rc = fts5ListMerge(db, &list, &aMerge[iLevel], (iLevel>0)); - } - } - assert( iLevel<32 ); - } - - list.aData = 0; - list.nData = 0; - for(iLevel=0; rc==SQLITE4_OK && iLevel<32; iLevel++){ - rc = fts5ListMerge(db, &list, &aMerge[iLevel], (iLevel>0)); - } - - if( rc==SQLITE4_OK ){ - sqlite4DbFree(db, p->aList); - p->aList = list.aData; - p->nAlloc = p->nList = list.nData; - }else{ - sqlite4DbFree(db, list.aData); - } - } - - sqlite4DbFree(db, aMerge); - } - - if( rc==SQLITE4_OK ){ - Fts5Prefix **aMerge; - Fts5Prefix *pPrefix = 0; - - aMerge = (Fts5Prefix **)sqlite4DbMallocZero(db, sizeof(Fts5List) * 32); - if( !aMerge ){ - rc = SQLITE4_NOMEM; - }else{ - int iLevel; - for(pElem=sqliteHashFirst(&hash); pElem; pElem=sqliteHashNext(pElem)){ - pPrefix = (Fts5Prefix *)sqliteHashData(pElem); - for(iLevel=0; iLevel<32; iLevel++){ - if( aMerge[iLevel] ){ - fts5PrefixMerge(&pPrefix, aMerge[iLevel]); - aMerge[iLevel] = 0; - }else{ - aMerge[iLevel] = pPrefix; - break; - } - } - assert( iLevel<32 ); - } - pPrefix = 0; - for(iLevel=0; iLevel<32; iLevel++){ - fts5PrefixMerge(&pPrefix, aMerge[iLevel]); - } - sqlite4HashClear(&hash); - sqlite4DbFree(db, aMerge); - } - pToken->pPrefix = pPrefix; - } - - for(pElem=sqliteHashFirst(&hash); pElem; pElem=sqliteHashNext(pElem)){ - Fts5Prefix *pPrefix = (Fts5Prefix *)sqliteHashData(pElem); - sqlite4DbFree(db, pPrefix->aList); - sqlite4DbFree(db, pPrefix); - } - sqlite4KVCursorClose(pToken->pCsr); - pToken->pCsr = 0; - - return rc; -} - -static int fts5OpenExprCursors(sqlite4 *db, Fts5Info *pInfo, Fts5ExprNode *p){ - int rc = SQLITE4_OK; - if( p ){ - if( p->eType==TOKEN_PRIMITIVE ){ - KVStore *pStore = db->aDb[pInfo->iDb].pKV; - Fts5Phrase *pPhrase = p->pPhrase; - int iStr; - - for(iStr=0; rc==SQLITE4_OK && iStrnStr; iStr++){ - Fts5Str *pStr = &pPhrase->aStr[iStr]; - int i; - for(i=0; rc==SQLITE4_OK && inToken; i++){ - Fts5Token *pToken = &pStr->aToken[i]; - rc = sqlite4KVStoreOpenCursor(pStore, &pToken->pCsr); - rc = sqlite4KVCursorSeek( - pToken->pCsr, pToken->aPrefix, pToken->nPrefix, 1 - ); - if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK; - if( rc==SQLITE4_OK && pToken->bPrefix ){ - rc = fts5FindPrefixes(db, pInfo, pToken); - } - } - } - } - if( rc==SQLITE4_OK ) rc = fts5OpenExprCursors(db, pInfo, p->pLeft); - if( rc==SQLITE4_OK ) rc = fts5OpenExprCursors(db, pInfo, p->pRight); - } - - return rc; -} - -/* -** Open a cursor for each token in the expression. -*/ -static int fts5OpenCursors(sqlite4 *db, Fts5Info *pInfo, Fts5Cursor *pCsr){ - return fts5OpenExprCursors(db, pInfo, pCsr->pExpr->pRoot); -} - -void sqlite4Fts5Close(Fts5Cursor *pCsr){ - if( pCsr ){ - sqlite4 *db = pCsr->db; - if( pCsr->aMem ){ - int i; - for(i=0; ipInfo->nCol; i++){ - sqlite4DbFree(db, pCsr->aMem[i].zMalloc); - } - sqlite4DbFree(db, pCsr->aMem); - } - - sqlite4KVCursorClose(pCsr->pCsr); - fts5ExpressionFree(db, pCsr->pExpr); - sqlite4DbFree(db, pCsr->pIter); - sqlite4DbFree(db, pCsr->aKey); - sqlite4DbFree(db, pCsr->anRow); - sqlite4DbFree(db, pCsr); - } -} - -static int fts5TokenAdvanceToMatch( - InstanceList *p, - InstanceList *pFirst, - int iOff, - int *pbEof -){ - int iReq = pFirst->iOff + iOff; - - while( p->iColiCol || (p->iCol==pFirst->iCol && p->iOff < iReq) ){ - int bEof = fts5InstanceListNext(p); - if( bEof ){ - *pbEof = 1; - return 0; - } - } - - return (p->iCol==pFirst->iCol && p->iOff==iReq); -} - -static int fts5StringFindInstances(sqlite4 *db, int iCol, Fts5Str *pStr){ - int i; - int rc = SQLITE4_OK; - int bEof = 0; - int nByte = sizeof(InstanceList) * pStr->nToken; - InstanceList *aIn; - InstanceList out; - - pStr->nList = 0; - memset(&out, 0, sizeof(InstanceList)); - - aIn = (InstanceList *)sqlite4DbMallocZero(db, nByte); - if( !aIn ) rc = SQLITE4_NOMEM; - for(i=0; rc==SQLITE4_OK && inToken; i++){ - const u8 *aData; - int nData; - rc = fts5TokenData(&pStr->aToken[i], &aData, &nData); - if( rc==SQLITE4_OK ){ - fts5InstanceListInit((u8 *)aData, nData, &aIn[i]); - fts5InstanceListNext(&aIn[i]); - } - } - - /* Allocate the output list */ - if( rc==SQLITE4_OK ){ - int nReq = aIn[0].nList; - if( nReq<=pStr->nListAlloc ){ - out.aList = pStr->aList; - out.nList = pStr->nListAlloc; - }else{ - pStr->aList = out.aList = sqlite4DbReallocOrFree(db, pStr->aList, nReq*2); - pStr->nListAlloc = out.nList = nReq*2; - if( out.aList==0 ) rc = SQLITE4_NOMEM; - } - } - - while( rc==SQLITE4_OK && bEof==0 ){ - for(i=1; inToken; i++){ - int bMatch = fts5TokenAdvanceToMatch(&aIn[i], &aIn[0], i, &bEof); - if( bMatch==0 || bEof ) break; - } - if( i==pStr->nToken && (iCol<0 || aIn[0].iCol==iCol) ){ - /* Record a match here */ - fts5InstanceListAppend(&out, aIn[0].iCol, aIn[0].iStream, aIn[0].iOff); - } - bEof = fts5InstanceListNext(&aIn[0]); - } - - pStr->nList = out.iList; - sqlite4DbFree(db, aIn); - - return rc; -} - -static int fts5IsNear(InstanceList *p1, InstanceList *p2, int nNear){ - if( p1->iCol==p2->iCol && p1->iOffiOff && (p1->iOff+nNear)>=p2->iOff ){ - return 1; - } - return 0; -} - -static int fts5StringNearTrim( - Fts5Str *pTrim, /* Trim this instance list */ - Fts5Str *pNext, /* According to this one */ - int nNear -){ - if( pNext->nList==0 ){ - pTrim->nList = 0; - }else{ - int bEof = 0; - int nTrail = nNear + (pNext->nToken-1) + 1; - int nLead = nNear + (pTrim->nToken-1) + 1; - - InstanceList lNear; - InstanceList in; - InstanceList out; - - fts5InstanceListInit(pNext->aList, pNext->nList, &lNear); - fts5InstanceListInit(pTrim->aList, pTrim->nList, &in); - fts5InstanceListInit(pTrim->aList, pTrim->nList, &out); - fts5InstanceListNext(&lNear); - fts5InstanceListNext(&in); - - while( bEof==0 ){ - if( fts5IsNear(&lNear, &in, nTrail) - || fts5IsNear(&in, &lNear, nLead) - ){ - /* The current position is a match. Append an entry to the output - ** and advance the input cursor. */ - fts5InstanceListAppend(&out, in.iCol, in.iStream, in.iOff); - bEof = fts5InstanceListNext(&in); - }else{ - if( lNear.iColnList = out.iList; - } - return SQLITE4_OK; -} - -/* -** This function tests if the cursors embedded in the Fts5Phrase object -** currently point to a match for the entire phrase. If so, *pbMatch -** is set to true before returning. -** -** If the cursors do not point to a match, then *ppAdvance is set to -** the token of the individual cursor that should be advanced before -** retrying this function. -*/ -static int fts5PhraseIsMatch( - sqlite4 *db, /* Database handle */ - Fts5Phrase *pPhrase, /* Phrase to test */ - int *pbMatch, /* OUT: True for a match, false otherwise */ - Fts5Token **ppAdvance /* OUT: Token to advance before retrying */ -){ - const u8 *aPk1 = 0; - int nPk1 = 0; - int rc = SQLITE4_OK; - int i; - - *pbMatch = 0; - *ppAdvance = &pPhrase->aStr[0].aToken[0]; - - rc = fts5TokenPk(*ppAdvance, &aPk1, &nPk1); - for(i=0; rc==SQLITE4_OK && inStr; i++){ - int j; - for(j=(i==0); jaStr[i].nToken; j++){ - const u8 *aPk = 0; - int nPk = 0; - Fts5Token *pToken = &pPhrase->aStr[i].aToken[j]; - rc = fts5TokenPk(pToken, &aPk, &nPk); - if( rc==SQLITE4_OK ){ - int res = fts5KeyCompare(aPk1, nPk1, aPk, nPk); - if( res<0 ){ - return SQLITE4_OK; - } - if( res>0 ){ - *ppAdvance = pToken; - return SQLITE4_OK; - } - } - } - } - - /* At this point, it is established that all of the token cursors in the - ** phrase point to an entry with the same primary key. Now figure out if - ** the various string constraints are met. Along the way, synthesize a - ** position list for each Fts5Str object. */ - for(i=0; rc==SQLITE4_OK && inStr; i++){ - Fts5Str *pStr = &pPhrase->aStr[i]; - rc = fts5StringFindInstances(db, pPhrase->iCol, pStr); - } - - /* Trim the instance lists according to any NEAR constraints. */ - for(i=1; rc==SQLITE4_OK && inStr; i++){ - int n = pPhrase->aiNear[i-1]; - rc = fts5StringNearTrim(&pPhrase->aStr[i], &pPhrase->aStr[i-1], n); - } - for(i=pPhrase->nStr-1; rc==SQLITE4_OK && i>0; i--){ - int n = pPhrase->aiNear[i-1]; - rc = fts5StringNearTrim(&pPhrase->aStr[i-1], &pPhrase->aStr[i], n); - } - - *pbMatch = (pPhrase->aStr[0].nList>0); - return rc; -} - -static int fts5PhraseAdvanceToMatch(sqlite4 *db, Fts5Phrase *pPhrase){ - int rc; - do { - int bMatch; - Fts5Token *pAdvance = 0; - rc = fts5PhraseIsMatch(db, pPhrase, &bMatch, &pAdvance); - if( rc!=SQLITE4_OK || bMatch ) break; - rc = fts5TokenAdvance(db, pAdvance); - }while( rc==SQLITE4_OK ); - return rc; -} - -static int fts5ExprAdvance(sqlite4 *db, Fts5ExprNode *p, int bFirst){ - int rc = SQLITE4_OK; - - switch( p->eType ){ - case TOKEN_PRIMITIVE: { - Fts5Phrase *pPhrase = p->pPhrase; - if( bFirst==0 ){ - rc = fts5TokenAdvance(db, &pPhrase->aStr[0].aToken[0]); - } - if( rc==SQLITE4_OK ) rc = fts5PhraseAdvanceToMatch(db, pPhrase); - if( rc==SQLITE4_OK ){ - rc = fts5TokenPk(&pPhrase->aStr[0].aToken[0], &p->aPk, &p->nPk); - }else{ - p->aPk = 0; - p->nPk = 0; - if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK; - } - break; - } - - case TOKEN_AND: - p->aPk = 0; - p->nPk = 0; - rc = fts5ExprAdvance(db, p->pLeft, bFirst); - if( rc==SQLITE4_OK ) rc = fts5ExprAdvance(db, p->pRight, bFirst); - while( rc==SQLITE4_OK && p->pLeft->aPk && p->pRight->aPk ){ - int res = fts5KeyCompare( - p->pLeft->aPk, p->pLeft->nPk, p->pRight->aPk, p->pRight->nPk - ); - if( res<0 ){ - rc = fts5ExprAdvance(db, p->pLeft, 0); - }else if( res>0 ){ - rc = fts5ExprAdvance(db, p->pRight, 0); - }else{ - p->aPk = p->pLeft->aPk; - p->nPk = p->pLeft->nPk; - break; - } - } - break; - - case TOKEN_OR: { - int res = 0; - if( bFirst==0 ){ - res = fts5KeyCompare( - p->pLeft->aPk, p->pLeft->nPk, p->pRight->aPk, p->pRight->nPk - ); - } - - if( res<=0 ) rc = fts5ExprAdvance(db, p->pLeft, bFirst); - if( rc==SQLITE4_OK && res>=0 ){ - rc = fts5ExprAdvance(db, p->pRight, bFirst); - } - - res = fts5KeyCompare( - p->pLeft->aPk, p->pLeft->nPk, p->pRight->aPk, p->pRight->nPk - ); - if( res>0 ){ - p->aPk = p->pRight->aPk; - p->nPk = p->pRight->nPk; - }else{ - p->aPk = p->pLeft->aPk; - p->nPk = p->pLeft->nPk; - } - assert( p->aPk!=0 || (p->pLeft->aPk==0 && p->pRight->aPk==0) ); - break; - } - - - default: assert( p->eType==TOKEN_NOT ); - - p->aPk = 0; - p->nPk = 0; - - rc = fts5ExprAdvance(db, p->pLeft, bFirst); - if( bFirst && rc==SQLITE4_OK ){ - rc = fts5ExprAdvance(db, p->pRight, bFirst); - } - - while( rc==SQLITE4_OK && p->pLeft->aPk && p->pRight->aPk ){ - int res = fts5KeyCompare( - p->pLeft->aPk, p->pLeft->nPk, p->pRight->aPk, p->pRight->nPk - ); - if( res<0 ){ - break; - }else if( res>0 ){ - rc = fts5ExprAdvance(db, p->pRight, 0); - }else{ - rc = fts5ExprAdvance(db, p->pLeft, 0); - } - } - - p->aPk = p->pLeft->aPk; - p->nPk = p->pLeft->nPk; - break; - } - - assert( rc!=SQLITE4_NOTFOUND ); - return rc; -} - -int sqlite4Fts5Next(Fts5Cursor *pCsr){ - sqlite4DbFree(pCsr->db, pCsr->pSz); - pCsr->pSz = 0; - pCsr->bMemValid = 0; - return fts5ExprAdvance(pCsr->db, pCsr->pExpr->pRoot, 0); -} - -int sqlite4Fts5Open( - sqlite4 *db, /* Database handle */ - Fts5Info *pInfo, /* Index description */ - const char *zMatch, /* Match expression */ - int bDesc, /* True to iterate in desc. order of PK */ - Fts5Cursor **ppCsr, /* OUT: New FTS cursor object */ - char **pzErr /* OUT: Error message */ -){ - int rc = SQLITE4_OK; - Fts5Cursor *pCsr; - int nMatch = sqlite4Strlen30(zMatch); - - pCsr = sqlite4DbMallocZero(db, sizeof(Fts5Cursor) + nMatch + 1); - - if( !pCsr ){ - rc = SQLITE4_NOMEM; - }else{ - pCsr->zExpr = (char *)&pCsr[1]; - memcpy(pCsr->zExpr, zMatch, nMatch); - pCsr->pInfo = pInfo; - pCsr->db = db; - rc = fts5ParseExpression(db, pInfo->pTokenizer, pInfo->p, - pInfo->iRoot, pInfo->azCol, pInfo->nCol, zMatch, &pCsr->pExpr, pzErr - ); - } - - if( rc==SQLITE4_OK ){ - /* Open a KV cursor for each term in the expression. Set each cursor - ** to point to the first entry in the range it will scan. */ - rc = fts5OpenCursors(db, pInfo, pCsr); - } - if( rc!=SQLITE4_OK ){ - sqlite4Fts5Close(pCsr); - pCsr = 0; - }else{ - rc = fts5ExprAdvance(db, pCsr->pExpr->pRoot, 1); - } - *ppCsr = pCsr; - return rc; -} - -/* -** Return true if the cursor passed as the second argument currently points -** to a valid entry, or false otherwise. -*/ -int sqlite4Fts5Valid(Fts5Cursor *pCsr){ - return( pCsr->pExpr->pRoot->aPk!=0 ); -} - -int sqlite4Fts5Pk( - Fts5Cursor *pCsr, - int iTbl, - KVByteArray **paKey, - KVSize *pnKey -){ - int i; - int nReq; - const u8 *aPk; - int nPk; - - aPk = pCsr->pExpr->pRoot->aPk; - nPk = pCsr->pExpr->pRoot->nPk; - - nReq = sqlite4VarintLen(iTbl) + nPk; - if( nReq>pCsr->nKeyAlloc ){ - pCsr->aKey = sqlite4DbReallocOrFree(pCsr->db, pCsr->aKey, nReq*2); - if( !pCsr->aKey ) return SQLITE4_NOMEM; - pCsr->nKeyAlloc = nReq*2; - } - - i = putVarint32(pCsr->aKey, iTbl); - memcpy(&pCsr->aKey[i], aPk, nPk); - - *paKey = pCsr->aKey; - *pnKey = nReq; - return SQLITE4_OK; -} - -int sqlite4_mi_column_count(sqlite4_context *pCtx, int *pn){ - int rc = SQLITE4_OK; - if( pCtx->pFts ){ - *pn = pCtx->pFts->pInfo->nCol; - }else{ - rc = SQLITE4_MISUSE; - } - return rc; -} - -int sqlite4_mi_phrase_count(sqlite4_context *pCtx, int *pn){ - int rc = SQLITE4_OK; - if( pCtx->pFts ){ - *pn = pCtx->pFts->pExpr->nPhrase; - }else{ - rc = SQLITE4_MISUSE; - } - return rc; -} - -int sqlite4_mi_phrase_token_count(sqlite4_context *pCtx, int iP, int *pn){ - int rc = SQLITE4_OK; - if( pCtx->pFts ){ - Fts5Expr *pExpr = pCtx->pFts->pExpr; - if( iP>pExpr->nPhrase || iP<0 ){ - *pn = 0; - }else{ - *pn = pExpr->apPhrase[iP]->nToken; - } - }else{ - rc = SQLITE4_MISUSE; - } - return rc; -} - -int sqlite4_mi_stream_count(sqlite4_context *pCtx, int *pn){ - int rc = SQLITE4_OK; - Fts5Cursor *pCsr = pCtx->pFts; - if( pCsr ){ - rc = fts5CsrLoadGlobal(pCtx->pFts); - if( rc==SQLITE4_OK ) *pn = pCsr->pGlobal->nStream; - }else{ - rc = SQLITE4_MISUSE; - } - return rc; -} - -static int fts5GetSize(Fts5Size *pSz, int iC, int iS){ - int nToken = 0; - int i; - - if( iC<0 && iS<0 ){ - int nFin = pSz->nCol * pSz->nStream; - for(i=0; iaSz[i]; - }else if( iC<0 ){ - for(i=0; inCol; i++) nToken += pSz->aSz[i*pSz->nStream + iS]; - }else if( iS<0 ){ - for(i=0; inStream; i++) nToken += pSz->aSz[pSz->nStream*iC + i]; - }else if( iCnCol && iSnStream ){ - nToken = pSz->aSz[iC * pSz->nStream + iS]; - } - - return nToken; -} - -int sqlite4_mi_size(sqlite4_context *pCtx, int iC, int iS, int *pn){ - int rc = SQLITE4_OK; - Fts5Cursor *pCsr = pCtx->pFts; - - if( pCsr==0 ){ - rc = SQLITE4_MISUSE; - }else{ - rc = fts5CsrLoadSz(pCsr); - if( rc==SQLITE4_OK ){ - *pn = fts5GetSize(pCsr->pSz, iC, iS); - } - } - return rc; -} - -int sqlite4_mi_total_size(sqlite4_context *pCtx, int iC, int iS, int *pn){ - int rc = SQLITE4_OK; - Fts5Cursor *pCsr = pCtx->pFts; - - if( pCsr==0 ){ - rc = SQLITE4_MISUSE; - }else{ - rc = fts5CsrLoadGlobal(pCsr); - if( rc==SQLITE4_OK ){ - *pn = fts5GetSize(pCsr->pGlobal, iC, iS); - } - } - return rc; -} - -int sqlite4_mi_total_rows(sqlite4_context *pCtx, int *pn){ - int rc = SQLITE4_OK; - Fts5Cursor *pCsr = pCtx->pFts; - if( pCsr==0 ){ - rc = SQLITE4_MISUSE; - }else{ - rc = fts5CsrLoadGlobal(pCsr); - if( rc==SQLITE4_OK ) *pn = pCsr->nGlobal; - } - return rc; -} - -int sqlite4_mi_column_value( - sqlite4_context *pCtx, - int iCol, - sqlite4_value **ppVal -){ - int rc = SQLITE4_OK; - Fts5Cursor *pCsr = pCtx->pFts; - if( pCsr==0 ){ - rc = SQLITE4_MISUSE; - }else{ - if( pCsr->bMemValid==0 ){ - sqlite4 *db = pCsr->db; - - Fts5Info *pInfo = pCsr->pInfo; - if( pCsr->aMem==0 ){ - int nByte = sizeof(Mem) * pInfo->nCol; - pCsr->aMem = (Mem *)sqlite4DbMallocZero(db, nByte); - if( pCsr->aMem==0 ){ - rc = SQLITE4_NOMEM; - }else{ - int i; - for(i=0; inCol; i++){ - pCsr->aMem[i].db = db; - } - } - } - - if( pCsr->pCsr==0 && rc==SQLITE4_OK ){ - KVStore *pStore = db->aDb[pInfo->iDb].pKV; - rc = sqlite4KVStoreOpenCursor(pStore, &pCsr->pCsr); - } - - if( rc==SQLITE4_OK ){ - u8 *aKey = 0; int nKey; /* Primary key for current row */ - const u8 *aData; int nData; /* Data record for current row */ - - rc = sqlite4Fts5Pk(pCsr, pInfo->iTbl, &aKey, &nKey); - if( rc==SQLITE4_OK ){ - rc = sqlite4KVCursorSeek(pCsr->pCsr, aKey, nKey, 0); - if( rc==SQLITE4_NOTFOUND ){ - rc = SQLITE4_CORRUPT_BKPT; - } - } - - if( rc==SQLITE4_OK ){ - rc = sqlite4KVCursorData(pCsr->pCsr, 0, -1, &aData, &nData); - } - - if( rc==SQLITE4_OK ){ - int i; - ValueDecoder *pCodec; /* The decoder object */ - - rc = sqlite4VdbeCreateDecoder(db, aData, nData, pInfo->nCol, &pCodec); - for(i=0; rc==SQLITE4_OK && inCol; i++){ - rc = sqlite4VdbeDecodeValue(pCodec, i, 0, &pCsr->aMem[i]); - } - sqlite4VdbeDestroyDecoder(pCodec); - } - - if( rc==SQLITE4_OK ) pCsr->bMemValid = 1; - } - } - - if( rc==SQLITE4_OK ){ - assert( pCsr->bMemValid ); - *ppVal = &pCsr->aMem[iCol]; - } - } - - return rc; -} - -int sqlite4_mi_tokenize( - sqlite4_context *pCtx, - const char *zText, - int nText, - void *p, - int(*x)(void *, int, int, const char *, int, int, int) -){ - int rc = SQLITE4_OK; - Fts5Cursor *pCsr = pCtx->pFts; - - if( pCsr==0 ){ - rc = SQLITE4_MISUSE; - }else{ - Fts5Info *pInfo = pCsr->pInfo; - rc = pInfo->pTokenizer->xTokenize(p, pInfo->p, zText, nText, x); - } - return rc; -} - -static Fts5Str *fts5FindStr( - const u8 *aPk, int nPk, - Fts5ExprNode *p, - int *piStr -){ - Fts5Str *pRet = 0; - if( p->eType==TOKEN_PRIMITIVE ){ - int iStr = *piStr; - if( iStrpPhrase->nStr && iStr>=0 - && p->nPk==nPk && 0==memcmp(p->aPk, aPk, nPk) - ){ - pRet = &p->pPhrase->aStr[iStr]; - }else{ - *piStr = iStr - p->pPhrase->nStr; - } - }else{ - pRet = fts5FindStr(aPk, nPk, p->pLeft, piStr); - if( pRet==0 ) pRet = fts5FindStr(aPk, nPk, p->pRight, piStr); - } - return pRet; -} - -int sqlite4_mi_match_count( - sqlite4_context *pCtx, - int iC, - int iS, - int iPhrase, - int *pnMatch -){ - int rc = SQLITE4_OK; - Fts5Cursor *pCsr = pCtx->pFts; - if( pCsr ){ - Fts5ExprNode *pRoot = pCsr->pExpr->pRoot; - int nMatch = 0; - Fts5Str *pStr; - int iCopy = iPhrase; - InstanceList sList; - - pStr = fts5FindStr(pRoot->aPk, pRoot->nPk, pRoot, &iCopy); - if( pStr ){ - fts5InstanceListInit(pStr->aList, pStr->nList, &sList); - while( 0==fts5InstanceListNext(&sList) ){ - if( (iC<0 || sList.iCol==iC) && (iS<0 || sList.iStream==iS) ) nMatch++; - } - } - *pnMatch = nMatch; - }else{ - rc = SQLITE4_MISUSE; - } - return rc; -} - -int sqlite4_mi_match_offset( - sqlite4_context *pCtx, - int iCol, - int iPhrase, - int iMatch, - int *piOff -){ - return SQLITE4_OK; -} - -int sqlite4_mi_total_match_count( - sqlite4_context *pCtx, - int iCol, - int iPhrase, - int *pnMatch, - int *pnDoc, - int *pnRelevant -){ - return SQLITE4_OK; -} - -static void fts5StrLoadRowcounts( - Fts5Str *pStr, - int nStream, - int *anRow, - int *anRowC, - int *anRowS, - int *pnRowCS -){ - u32 mask = 0; - int iPrevCol = -1; - InstanceList sList; - - fts5InstanceListInit(pStr->aList, pStr->nList, &sList); - while( 0==fts5InstanceListNext(&sList) ){ - if( iPrevCol<0 ) (*pnRowCS)++; - if( sList.iCol!=iPrevCol ){ - mask = 0; - anRowC[sList.iCol]++; - } - if( (mask & (1<pInfo; - - if( pNode->eType==TOKEN_PRIMITIVE ){ - Fts5Phrase *pPhrase = pNode->pPhrase; - int iStr = *piStr; - - rc = fts5ExprAdvance(db, pNode, 1); - while( rc==SQLITE4_OK && pNode->aPk ){ - int i; - for(i=0; inStr; i++){ - int *anRow = &pCsr->anRow[(iStr+i) * pInfo->nCol * nStream]; - int *anRowC = &pCsr->anRowC[(iStr+i) * pInfo->nCol]; - int *anRowS = &pCsr->anRowS[(iStr+i) * nStream]; - int *pnRowCS = &pCsr->anRowCS[iStr+i]; - fts5StrLoadRowcounts( - &pPhrase->aStr[i], nStream, anRow, anRowC, anRowS, pnRowCS - ); - } - rc = fts5ExprAdvance(db, pNode, 0); - } - - *piStr = iStr + pPhrase->nStr; - } - - if( rc==SQLITE4_OK ){ - rc = fts5ExprLoadRowcounts(db, pCsr, nStream, pNode->pLeft, piStr); - } - if( rc==SQLITE4_OK ){ - rc = fts5ExprLoadRowcounts(db, pCsr, nStream, pNode->pRight, piStr); - } - } - - return rc; -} - -static int fts5CsrLoadRowcounts(Fts5Cursor *pCsr){ - int rc = SQLITE4_OK; - - if( pCsr->anRow==0 ){ - int nStream = pCsr->pGlobal->nStream; - sqlite4 *db = pCsr->db; - Fts5Expr *pCopy; - Fts5Expr *pExpr = pCsr->pExpr; - Fts5Info *pInfo = pCsr->pInfo; - int *anRow; - int iPhrase = 0; - - pCsr->anRow = anRow = (int *)sqlite4DbMallocZero(db, sizeof(int) * ( - pExpr->nPhrase * pInfo->nCol * pCsr->pGlobal->nStream - + pExpr->nPhrase * pInfo->nCol - + pExpr->nPhrase * pCsr->pGlobal->nStream - + pExpr->nPhrase - )); - if( !anRow ) return SQLITE4_NOMEM; - pCsr->anRowC = &anRow[pExpr->nPhrase*pInfo->nCol*pCsr->pGlobal->nStream]; - pCsr->anRowS = &pCsr->anRowC[pExpr->nPhrase * pInfo->nCol]; - pCsr->anRowCS = &pCsr->anRowS[pExpr->nPhrase * pCsr->pGlobal->nStream]; - - rc = fts5ParseExpression(db, pInfo->pTokenizer, pInfo->p, - pInfo->iRoot, pInfo->azCol, pInfo->nCol, pCsr->zExpr, &pCopy, 0 - ); - if( rc==SQLITE4_OK ){ - rc = fts5OpenExprCursors(db, pInfo, pCopy->pRoot); - } - if( rc==SQLITE4_OK ){ - rc = fts5ExprLoadRowcounts(db, pCsr, nStream, pCopy->pRoot, &iPhrase); - } - - fts5ExpressionFree(db, pCopy); - } - - return rc; -} - -int sqlite4_mi_row_count( - sqlite4_context *pCtx, /* Context object passed to mi function */ - int iC, /* Specific column (or -ve for all columns) */ - int iS, /* Specific stream (or -ve for all streams) */ - int iP, /* Specific phrase */ - int *pn /* Total number of rows containing C/S/P */ -){ - int rc = SQLITE4_OK; - Fts5Cursor *pCsr = pCtx->pFts; - if( pCsr==0 ){ - rc = SQLITE4_MISUSE; - }else{ - rc = fts5CsrLoadGlobal(pCsr); - if( rc==SQLITE4_OK ) rc = fts5CsrLoadRowcounts(pCsr); - - if( rc==SQLITE4_OK ){ - int i; - int nRow = 0; - int nStream = pCsr->pGlobal->nStream; - int nCol = pCsr->pInfo->nCol; - int *aRow = &pCsr->anRow[iP * nStream * nCol]; - - if( iC<0 && iS<0 ){ - nRow = pCsr->anRowCS[iP]; - }else if( iC<0 ){ - for(i=0; ianRowC[iP*nCol + iC]; - }else if( iCaList[i]; - if( fts5InstanceListEof(p)==0 ){ - if( (pBest==0) - || (p->iColiCol) - || (p->iCol==pBest->iCol && p->iOffiOff) - ){ - pBest = p; - } - } - } - - if( pBest==0 ){ - pIter->iCurrent = -1; - }else{ - pIter->iCurrent = pBest - pIter->aList; - } -} - -static void fts5InitExprIterator( - const u8 *aPk, - int nPk, - Fts5ExprNode *p, - Fts5MatchIter *pIter -){ - if( p ){ - if( p->eType==TOKEN_PRIMITIVE ){ - if( p->nPk==nPk && 0==memcmp(aPk, p->aPk, nPk) ){ - int i; - for(i=0; ipPhrase->nStr; i++){ - Fts5Str *pStr = &p->pPhrase->aStr[i]; - InstanceList *pList = &pIter->aList[pIter->iCurrent++]; - fts5InstanceListInit(pStr->aList, pStr->nList, pList); - fts5InstanceListNext(pList); - } - }else{ - memset(&pIter->aList[pIter->iCurrent], 0, sizeof(InstanceList)); - pIter->iCurrent += p->pPhrase->nStr; - } - } - fts5InitExprIterator(aPk, nPk, p->pLeft, pIter); - fts5InitExprIterator(aPk, nPk, p->pRight, pIter); - } -} - -static void fts5InitIterator(Fts5Cursor *pCsr){ - Fts5MatchIter *pIter = pCsr->pIter; - Fts5ExprNode *pRoot = pCsr->pExpr->pRoot; - - pIter->iCurrent = 0; - fts5InitExprIterator(pRoot->aPk, pRoot->nPk, pRoot, pIter); - pIter->iMatch = 0; - pIter->bValid = 1; - fts5IterSetCurrent(pIter, pCsr->pExpr->nPhrase); -} - -int sqlite4_mi_match_detail( - sqlite4_context *pCtx, /* Context object passed to mi function */ - int iMatch, /* Index of match */ - int *piOff, /* OUT: Token offset of match */ - int *piC, /* OUT: Column number of match iMatch */ - int *piS, /* OUT: Stream number of match iMatch */ - int *piP /* OUT: Phrase number of match iMatch */ -){ - int rc = SQLITE4_OK; - Fts5Cursor *pCsr = pCtx->pFts; - if( pCsr==0 ){ - rc = SQLITE4_MISUSE; - }else{ - int nPhrase = pCsr->pExpr->nPhrase; - Fts5MatchIter *pIter = pCsr->pIter; - if( pIter==0 ){ - pCsr->pIter = pIter = (Fts5MatchIter *)sqlite4DbMallocZero( - pCsr->db, sizeof(Fts5MatchIter) + sizeof(InstanceList)*nPhrase - ); - if( pIter ){ - pIter->aList = (InstanceList *)&pIter[1]; - }else{ - rc = SQLITE4_NOMEM; - } - } - - if( rc==SQLITE4_OK && (pIter->bValid==0 || iMatchiMatch) ){ - fts5InitIterator(pCsr); -#if 0 - int i; - for(i=0; ipExpr->nPhrase; i++){ - Fts5Str *pStr = pCsr->pExpr->apPhrase[i]; - fts5InstanceListInit(pStr->aList, pStr->nList, &pIter->aList[i]); - fts5InstanceListNext(&pIter->aList[i]); - } - pIter->iMatch = 0; - fts5IterSetCurrent(pIter, pCsr->pExpr->nPhrase); -#endif - } - - if( rc==SQLITE4_OK ){ - assert( pIter->iMatch<=iMatch ); - while( pIter->iCurrent>=0 && pIter->iMatchaList[pIter->iCurrent]); - fts5IterSetCurrent(pIter, pCsr->pExpr->nPhrase); - pIter->iMatch++; - } - if( pIter->iCurrent<0 ){ - rc = SQLITE4_NOTFOUND; - }else{ - InstanceList *p = &pIter->aList[pIter->iCurrent]; - *piOff = p->iOff; - *piC = p->iCol; - *piS = p->iStream; - *piP = pIter->iCurrent; - } - } - } - return rc; -} - -/************************************************************************** -*************************************************************************** -** Below this point is test code. -*/ -#ifdef SQLITE4_TEST -static int fts5PrintExprNode(sqlite4 *, const char **, Fts5ExprNode *, char **); -static int fts5PrintExprNodeParen( - sqlite4 *db, const char **azCol, - Fts5ExprNode *pNode, - char **pzRet -){ - int bParen = (pNode->eType!=TOKEN_PRIMITIVE || pNode->pPhrase->nStr>1); - sqlite4_env *pEnv = sqlite4_db_env(db); - char *zRet = *pzRet; - - if( bParen ) zRet = sqlite4_mprintf(pEnv, "%z(", zRet); - fts5PrintExprNode(db, azCol, pNode, &zRet); - if( bParen ) zRet = sqlite4_mprintf(pEnv, "%z)", zRet); - - *pzRet = zRet; - return SQLITE4_OK; -} -static int fts5PrintExprNode( - sqlite4 *db, - const char **azCol, - Fts5ExprNode *pNode, - char **pzRet -){ - sqlite4_env *pEnv = sqlite4_db_env(db); - char *zRet = *pzRet; - - assert( - pNode->eType==TOKEN_AND || pNode->eType==TOKEN_OR - || pNode->eType==TOKEN_NOT || pNode->eType==TOKEN_PRIMITIVE - ); - assert( (pNode->eType==TOKEN_PRIMITIVE)==(pNode->pPhrase!=0) ); - - if( pNode->eType==TOKEN_PRIMITIVE ){ - int iStr; - Fts5Phrase *pPhrase = pNode->pPhrase; - if( pPhrase->iCol>=0 ){ - zRet = sqlite4_mprintf(pEnv, "%z\"%s\":", zRet, azCol[pPhrase->iCol]); - } - for(iStr=0; iStrnStr; iStr++){ - int iToken; - Fts5Str *pStr = &pPhrase->aStr[iStr]; - if( iStr>0 ){ - zRet = sqlite4_mprintf( - pEnv, "%z NEAR/%d ", zRet, pPhrase->aiNear[iStr-1] - ); - } - for(iToken=0; iTokennToken; iToken++){ - int nRet = sqlite4Strlen30(zRet); - const char *z = pStr->aToken[iToken].z; - int n = pStr->aToken[iToken].n; - int i; - - zRet = (char *)sqlite4_realloc(pEnv, zRet, nRet + n*2+4); - if( iToken>0 ) zRet[nRet++] = '+'; - zRet[nRet++] = '"'; - - for(i=0; iaToken[iToken].bPrefix ){ - zRet[nRet++] = '*'; - } - zRet[nRet++] = '\0'; - } - } - }else{ - fts5PrintExprNodeParen(db, azCol, pNode->pLeft, &zRet); - switch( pNode->eType ){ - case TOKEN_AND: - zRet = sqlite4_mprintf(pEnv, "%z AND ", zRet); - break; - case TOKEN_OR: - zRet = sqlite4_mprintf(pEnv, "%z OR ", zRet); - break; - case TOKEN_NOT: - zRet = sqlite4_mprintf(pEnv, "%z NOT ", zRet); - break; - } - fts5PrintExprNodeParen(db, azCol, pNode->pRight, &zRet); - } - - *pzRet = zRet; - return SQLITE4_OK; -} -static int fts5PrintExpr( - sqlite4 *db, - const char **azCol, - Fts5Expr *pExpr, - char **pzRet -){ - return fts5PrintExprNode(db, azCol, pExpr->pRoot, pzRet); -} - -/* -** A user defined function used to test the fts5 expression parser. As follows: -** -** fts5_parse_expr(, ); -*/ -static void fts5_parse_expr( - sqlite4_context *pCtx, - int nVal, - sqlite4_value **aVal -){ - int rc; - Fts5Expr *pExpr = 0; - Fts5Tokenizer *pTok; - sqlite4_tokenizer *p = 0; - sqlite4 *db; - - const char *zTokenizer; - const char *zExpr; - const char *zTbl; - char *zErr = 0; - char *zRet = 0; - const char **azCol = 0; - int nCol = 0; - sqlite4_stmt *pStmt = 0; - - db = sqlite4_context_db_handle(pCtx); - assert( nVal==3 ); - zTokenizer = (const char *)sqlite4_value_text(aVal[0]); - zExpr = (const char *)sqlite4_value_text(aVal[1]); - zTbl = (const char *)sqlite4_value_text(aVal[2]); - - if( sqlite4Strlen30(zTbl)>0 ){ - int i; - char *zSql = sqlite4MPrintf(db, "SELECT * FROM '%q'", zTbl); - rc = sqlite4_prepare(db, zSql, -1, &pStmt, 0); - sqlite4DbFree(db, zSql); - if( rc!=SQLITE4_OK ){ - sqlite4_result_error(pCtx, sqlite4_errmsg(db), -1); - sqlite4_result_error_code(pCtx, rc); - return; - } - nCol = sqlite4_column_count(pStmt); - azCol = sqlite4DbMallocZero(db, sizeof(char *)*nCol); - for(i=0; ixCreate(pTok->pCtx, 0, 0, &p); - if( rc!=SQLITE4_OK ){ - zErr = sqlite4MPrintf(db, "error creating tokenizer: %d", rc); - goto fts5_parse_expr_out; - } - } - - rc = fts5ParseExpression( - db, pTok, p, 0, (char **)azCol, nCol, zExpr, &pExpr, &zErr); - if( rc!=SQLITE4_OK ){ - if( zErr==0 ){ - zErr = sqlite4MPrintf(db, "error parsing expression: %d", rc); - } - goto fts5_parse_expr_out; - } - - fts5PrintExpr(db, azCol, pExpr, &zRet); - sqlite4_result_text(pCtx, zRet, -1, SQLITE4_TRANSIENT, 0); - fts5ExpressionFree(db, pExpr); - sqlite4_free(sqlite4_db_env(db), zRet); - - fts5_parse_expr_out: - if( p ) pTok->xDestroy(p); - sqlite4DbFree(db, azCol); - sqlite4_finalize(pStmt); - if( zErr ){ - sqlite4_result_error(pCtx, zErr, -1); - sqlite4DbFree(db, zErr); - } -} -#endif - -/* -** Register the default FTS5 tokenizer and functions with handle db. -*/ -int sqlite4InitFts5(sqlite4 *db){ -#ifdef SQLITE4_TEST - int rc = sqlite4_create_function( - db, "fts5_parse_expr", 3, SQLITE4_UTF8, 0, fts5_parse_expr, 0, 0 - ); - if( rc!=SQLITE4_OK ) return rc; -#endif - return sqlite4InitFts5Func(db); -} DELETED src/fts5func.c Index: src/fts5func.c ================================================================== --- src/fts5func.c +++ /dev/null @@ -1,672 +0,0 @@ -/* -** 2012 December 17 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -*/ - -/* -** The BM25 and BM25F implementations in this file are based on information -** found in: -** -** Stephen Robertson and Hugo Zaragoza: "The Probablistic Relevance -** Framework: BM25 and Beyond", 2009. -*/ - -#include "sqliteInt.h" -#include /* temporary: For log() */ - -static char fts5Tolower(char c){ - if( c>='A' && c<='Z' ) c = c + ('a' - 'A'); - return c; -} - -static int fts5SimpleCreate( - void *pCtx, - const char **azArg, - int nArg, - sqlite4_tokenizer **pp -){ - *pp = (sqlite4_tokenizer *)pCtx; - return SQLITE4_OK; -} - -static int fts5SimpleDestroy(sqlite4_tokenizer *p){ - return SQLITE4_OK; -} - -typedef struct Fts5RankCtx Fts5RankCtx; -struct Fts5RankCtx { - sqlite4 *db; - double *aAvgdl; /* Average document size of each field */ - int nPhrase; /* Number of phrases in query */ - double *aIdf; /* IDF weights for each phrase in query */ -}; - -static void fts5RankFreeCtx(void *pNotUsed, void *pCtx){ - if( pCtx ){ - Fts5RankCtx *p = (Fts5RankCtx *)pCtx; - sqlite4DbFree(p->db, p); - } -} - -#define BM25_EXPLAIN 0x01 -#define BM25_FCOLUMNS 0x02 -#define BM25_FSTREAMS 0x04 - -static int fts5GetSizeFreqScale( - sqlite4_context *pCtx, - int nArg, sqlite4_value **apArg,/* Function arguments */ - int bm25mask, /* bm25 configuration mask */ - int iPhrase, /* Phrase number */ - int iField, /* Field number */ - int *pnSize, /* OUT: Size of field in tokens */ - int *pnFreq, /* OUT: Occurences of phrase in field */ - double *pdScale /* OUT: Scale to use with this field */ -){ - int rc; - double scale = 1.0; - int nSize = 0; - int nFreq = 0; - - if( bm25mask & BM25_FCOLUMNS ){ - rc = sqlite4_mi_match_count(pCtx, iField, -1, iPhrase, &nFreq); - if( rc==SQLITE4_OK ) rc = sqlite4_mi_size(pCtx, iField, -1, &nSize); - if( nArg>iField ) scale = sqlite4_value_double(apArg[iField]); - }else if( bm25mask & BM25_FSTREAMS ){ - rc = sqlite4_mi_match_count(pCtx, -1, iField, iPhrase, &nFreq); - if( rc==SQLITE4_OK ) rc = sqlite4_mi_size(pCtx, -1, iField, &nSize); - if( nArg>iField ) scale = sqlite4_value_double(apArg[iField]); - }else{ - rc = sqlite4_mi_match_count(pCtx, -1, -1, iPhrase, &nFreq); - if( rc==SQLITE4_OK ) rc = sqlite4_mi_size(pCtx, -1, -1, &nSize); - } - - *pnSize = nSize; - *pnFreq = nFreq; - *pdScale = scale; - return rc; -} - -/* -** A BM25(F) based ranking function for fts5. -** -** This is based on the information in the Robertson/Zaragoza paper -** referenced above. As there is no way to provide relevance feedback -** IDF weights (equation 3.3 in R/Z) are used instead of RSJ for each phrase. -** The rest of the implementation is as presented in equations 3.19-21. -** -** R and Z observe that the experimental evidence suggests that reasonable -** values for free parameters "b" and "k1" are often in the ranges -** (0.5 < b < 0.8) and (1.2 < k1 < 2), although the optimal values depend -** on the nature of both the documents and queries. The implementation -** below sets each parameter to the midpoint of the suggested range. -*/ -static void fts5Rank(sqlite4_context *pCtx, int nArg, sqlite4_value **apArg){ - const double b = 0.65; - const double k1 = 1.6; - - sqlite4 *db = sqlite4_context_db_handle(pCtx); - int rc = SQLITE4_OK; /* Error code */ - Fts5RankCtx *p; /* Structure to store reusable values */ - int i; /* Used to iterate through phrases */ - double rank = 0.0; /* UDF return value */ - - int bExplain; /* True to run in explain mode */ - char *zExplain = 0; /* String to return in explain mode */ - int nField = 1; /* Number of fields in collection */ - - int bm25mask = SQLITE4_PTR_TO_INT(sqlite4_user_data(pCtx)); - bExplain = (bm25mask & BM25_EXPLAIN); - - if( bm25mask & BM25_FCOLUMNS ) sqlite4_mi_column_count(pCtx, &nField); - if( bm25mask & BM25_FSTREAMS ) sqlite4_mi_stream_count(pCtx, &nField); - - p = sqlite4_get_auxdata(pCtx, 0); - if( p==0 ){ - int nPhrase; /* Number of phrases in query expression */ - int nByte; /* Number of bytes of data to allocate */ - - sqlite4_mi_phrase_count(pCtx, &nPhrase); - nByte = sizeof(Fts5RankCtx) + (nPhrase+nField) * sizeof(double); - p = (Fts5RankCtx *)sqlite4DbMallocZero(db, nByte); - sqlite4_set_auxdata(pCtx, 0, (void *)p, fts5RankFreeCtx, 0); - p = sqlite4_get_auxdata(pCtx, 0); - - if( !p ){ - rc = SQLITE4_NOMEM; - }else{ - int N; /* Total number of docs in collection */ - int ni; /* Number of docs with phrase i */ - - p->db = db; - p->nPhrase = nPhrase; - p->aIdf = (double *)&p[1]; - p->aAvgdl = &p->aIdf[nPhrase]; - - /* Determine the IDF weight for each phrase in the query. */ - rc = sqlite4_mi_total_rows(pCtx, &N); - for(i=0; rc==SQLITE4_OK && iaIdf[i] = log((0.5 + N - ni) / (0.5 + ni)); - } - } - - /* Determine the average document length. For bm25f, determine the - ** average length of each field. */ - if( rc==SQLITE4_OK ){ - int iField; - for(iField=0; iFieldaAvgdl[iField] = (double)nTotal / (double)N; - } - } - } - } - } - - if( bExplain ){ - int iField; - zExplain = sqlite4MAppendf( - db, zExplain, "%s
StreamScaleavgslsl", - zExplain - ); - for(i=0; inPhrase; i++){ - zExplain = sqlite4MAppendf( - db, zExplain, "%stf%d", zExplain, i - ); - } - for(iField=0; rc==SQLITE4_OK && iField%d%.2f%.2f%d", - zExplain, iField, scale, p->aAvgdl[iField], dl - ); - for(i=0; rc==SQLITE4_OK && inPhrase; i++){ - rc = fts5GetSizeFreqScale( - pCtx, nArg, apArg, bm25mask, i, iField, &dl, &tf, &scale - ); - zExplain = sqlite4MAppendf(db, zExplain, "%s%d", zExplain, tf); - } - } - zExplain = sqlite4MAppendf( - db, zExplain, "%s
PhraseIDF", zExplain - ); - for(i=0; i" - "tfs%d", zExplain, i - ); - } - zExplain = sqlite4MAppendf(db, zExplain, "%srank", zExplain); - } - - for(i=0; rc==SQLITE4_OK && inPhrase; i++){ - int iField; - double tfns = 0.0; /* Sum of tfn for all fields */ - double prank; /* Contribution to rank of this phrase */ - - if( bExplain ){ - zExplain = sqlite4MAppendf( - db, zExplain, "%s
%d%.2f", zExplain, i, p->aIdf[i] - ); - } - - for(iField = 0; iFieldaAvgdl[iField]; /* 3.20 */ - tfn = scale * (double)tf / B; - tfns += tfn; /* 3.19 */ - - - if( bExplain ){ - zExplain = sqlite4MAppendf(db, zExplain, "%s%.2f", zExplain, tfn); - } - } - - prank = p->aIdf[i] * tfns / (k1 + tfns); /* 3.21 */ - if( bExplain ){ - zExplain = sqlite4MAppendf(db, zExplain, "%s%.2f", zExplain, prank); - } - - /* Add it to the overall rank */ - rank += prank; - } - - if( rc==SQLITE4_OK ){ - if( bExplain ){ - zExplain = sqlite4MAppendf( - db, zExplain, "%s
overall rank=%.2f", zExplain, rank - ); - sqlite4_result_text(pCtx, zExplain, -1, SQLITE4_TRANSIENT, 0); - }else{ - sqlite4_result_double(pCtx, rank); - } - }else{ - sqlite4_result_error_code(pCtx, rc); - } - sqlite4DbFree(db, zExplain); -} - -typedef struct Snippet Snippet; -typedef struct SnippetText SnippetText; - -struct Snippet { - int iCol; - int iOff; - u64 hlmask; -}; - -struct SnippetText { - char *zOut; /* Pointer to snippet text */ - int nOut; /* Size of zOut in bytes */ - int nAlloc; /* Bytes of space allocated at zOut */ -}; - -typedef struct SnippetCtx SnippetCtx; -struct SnippetCtx { - sqlite4 *db; /* Database handle */ - int nToken; /* Number of tokens in snippet */ - int iOff; /* First token in snippet */ - u64 mask; /* Snippet mask. Highlight these terms */ - const char *zStart; - const char *zEnd; - const char *zEllipses; - - SnippetText *pOut; - - int iFrom; - int iTo; - const char *zText; /* Document to extract snippet from */ - int rc; /* Set to NOMEM if OOM is encountered */ -}; - -static void fts5SnippetAppend(SnippetCtx *p, const char *z, int n){ - if( p->rc==SQLITE4_OK ){ - SnippetText *pOut = p->pOut; - if( n<0 ) n = strlen(z); - if( (pOut->nOut + n) > pOut->nAlloc ){ - int nNew = (pOut->nOut+n) * 2; - - pOut->zOut = sqlite4DbReallocOrFree(p->db, pOut->zOut, nNew); - if( pOut->zOut==0 ){ - p->rc = SQLITE4_NOMEM; - return; - } - pOut->nAlloc = sqlite4DbMallocSize(p->db, pOut->zOut); - } - - memcpy(&pOut->zOut[pOut->nOut], z, n); - pOut->nOut += n; - } -} - -static int fts5SnippetCb( - void *pCtx, - int iStream, - int iOff, - const char *z, int n, - int iSrc, int nSrc -){ - SnippetCtx *p = (SnippetCtx *)pCtx; - - if( iOffiOff ){ - return 0; - }else if( iOff>=(p->iOff + p->nToken) ){ - fts5SnippetAppend(p, &p->zText[p->iFrom], p->iTo - p->iFrom); - fts5SnippetAppend(p, p->zEllipses, -1); - p->iFrom = -1; - return 1; - }else{ - int bHighlight; /* True to highlight term */ - - bHighlight = (p->mask & ((u64)1 << (iOff-p->iOff))) ? 1 : 0; - - if( p->iFrom==0 && p->iOff!=0 ){ - p->iFrom = iSrc; - if( p->pOut->nOut==0 ) fts5SnippetAppend(p, p->zEllipses, -1); - } - - if( bHighlight ){ - fts5SnippetAppend(p, &p->zText[p->iFrom], iSrc - p->iFrom); - fts5SnippetAppend(p, p->zStart, -1); - fts5SnippetAppend(p, &p->zText[iSrc], nSrc); - fts5SnippetAppend(p, p->zEnd, -1); - p->iTo = p->iFrom = iSrc+nSrc; - }else{ - p->iTo = iSrc + nSrc; - } - } - - return 0; -} - -static int fts5SnippetText( - sqlite4_context *pCtx, - Snippet *pSnip, - SnippetText *pText, - int nToken, - const char *zStart, - const char *zEnd, - const char *zEllipses -){ - int rc; - sqlite4_value *pVal = 0; - - u64 mask = pSnip->hlmask; - int iOff = pSnip->iOff; - int iCol = pSnip->iCol; - - rc = sqlite4_mi_column_value(pCtx, iCol, &pVal); - if( rc==SQLITE4_OK ){ - SnippetCtx sCtx; - int nText; - - nText = sqlite4_value_bytes(pVal); - memset(&sCtx, 0, sizeof(sCtx)); - sCtx.zText = (const char *)sqlite4_value_text(pVal); - sCtx.db = sqlite4_context_db_handle(pCtx); - sCtx.nToken = nToken; - sCtx.iOff = iOff; - sCtx.mask = mask; - sCtx.zStart = zStart; - sCtx.zEnd = zEnd; - sCtx.zEllipses = zEllipses; - sCtx.pOut = pText; - - sqlite4_mi_tokenize(pCtx, sCtx.zText, nText, &sCtx, fts5SnippetCb); - if( sCtx.rc==SQLITE4_OK && sCtx.iFrom>0 ){ - fts5SnippetAppend(&sCtx, &sCtx.zText[sCtx.iFrom], nText - sCtx.iFrom); - } - rc = sCtx.rc; - } - - return rc; -} - -static int fts5BestSnippet( - sqlite4_context *pCtx, /* Context snippet() was called in */ - int iColumn, /* In this column (-1 means any column) */ - u64 *pMask, /* IN/OUT: Mask of high-priority phrases */ - int nToken, /* Number of tokens in requested snippet */ - Snippet *pSnip /* Populate this object */ -){ - sqlite4 *db = sqlite4_context_db_handle(pCtx); - int nPhrase; - int rc = SQLITE4_OK; - int i; - int iPrev = 0; - int iPrevCol = 0; - u64 *aMask; - u64 mask = *pMask; - u64 allmask = 0; - - int iBestOff = nToken-1; - int iBestCol = (iColumn >= 0 ? iColumn : 0); - int nBest = 0; - u64 hlmask = 0; /* Highlight mask associated with iBestOff */ - u64 missmask = 0; /* Mask of missing terms in iBestOff snip. */ - - sqlite4_mi_phrase_count(pCtx, &nPhrase); - aMask = sqlite4DbMallocZero(db, sizeof(u64) * nPhrase); - if( !aMask ) return SQLITE4_NOMEM; - - /* Iterate through all matches for all phrases */ - for(i=0; rc==SQLITE4_OK; i++){ - int iOff; - int iCol; - int iStream; - int iPhrase; - - rc = sqlite4_mi_match_detail(pCtx, i, &iOff, &iCol, &iStream, &iPhrase); - if( rc==SQLITE4_OK ){ - u64 tmask = 0; - u64 miss = 0; - int iMask; - int nShift; - int nScore = 0; - - int nPTok; - int iPTok; - - if( iColumn>=0 && iColumn!=iCol ) continue; - - allmask |= ((u64)1 << iPhrase); - - nShift = ((iPrevCol==iCol) ? (iOff-iPrev) : 100); - - for(iMask=0; iMask> nShift; - }else{ - aMask[iMask] = 0; - } - } - sqlite4_mi_phrase_token_count(pCtx, iPhrase, &nPTok); - for(iPTok=0; iPToknBest ){ - hlmask = tmask; - missmask = miss; - nBest = nScore; - iBestOff = iOff; - iBestCol = iCol; - } - - iPrev = iOff; - iPrevCol = iCol; - } - } - if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK; - - pSnip->iOff = iBestOff-nToken+1; - pSnip->iCol = iBestCol; - pSnip->hlmask = hlmask; - *pMask = mask & missmask & allmask; - - sqlite4DbFree(db, aMask); - return rc; -} - -static void fts5SnippetImprove( - sqlite4_context *pCtx, - int nToken, /* Size of required snippet */ - int nSz, /* Total size of column in tokens */ - Snippet *pSnip -){ - int i; - int nLead = 0; - int nShift = 0; - - u64 mask = pSnip->hlmask; - int iOff = pSnip->iOff; - - if( mask==0 ) return; - assert( mask & ((u64)1 << (nToken-1)) ); - - for(i=0; (mask & ((u64)1 << i))==0; i++); - nLead = i; - - nShift = (nLead/2); - if( iOff+nShift > nSz-nToken ) nShift = (nSz-nToken) - iOff; - if( iOff+nShift < 0 ) nShift = -1 * iOff; - - iOff += nShift; - mask = mask >> nShift; - - pSnip->iOff = iOff; - pSnip->hlmask = mask; -} - -static void fts5Snippet(sqlite4_context *pCtx, int nArg, sqlite4_value **apArg){ - Snippet aSnip[4]; - int nSnip; - int iCol = -1; - int nToken = -15; - int rc; - int nPhrase; - - const char *zStart = ""; - const char *zEnd = ""; - const char *zEllipses = "..."; - - if( nArg>0 ) zStart = (const char *)sqlite4_value_text(apArg[0]); - if( nArg>1 ) zEnd = (const char *)sqlite4_value_text(apArg[1]); - if( nArg>2 ) zEllipses = (const char *)sqlite4_value_text(apArg[2]); - if( nArg>3 ) iCol = sqlite4_value_int(apArg[3]); - if( nArg>4 ) nToken = sqlite4_value_int(apArg[4]); - - rc = sqlite4_mi_phrase_count(pCtx, &nPhrase); - for(nSnip=1; rc==SQLITE4_OK && nSnip<5; nSnip = ((nSnip==2) ? 3 : (nSnip+1))){ - int nTok; - int i; - u64 mask = ((u64)1 << nPhrase) - 1; - - if( nToken<0 ){ - nTok = nToken * -1; - }else{ - nTok = (nToken + (nSnip-1)) / nSnip; - } - - memset(aSnip, 0, sizeof(aSnip)); - for(i=0; rc==SQLITE4_OK && i0 ){ - brk = x(pCtx, 0, iOff++, aBuf, iBuf, i-iBuf, iBuf); - iBuf = 0; - } - } - if( iBuf>0 ) x(pCtx, 0, iOff++, aBuf, iBuf, i-iBuf, iBuf); - - sqlite4_free(pEnv, aBuf); - return SQLITE4_OK; -} - -int sqlite4InitFts5Func(sqlite4 *db){ - int rc; - int i; - sqlite4_env *pEnv = sqlite4_db_env(db); - - struct RankFunction { - const char *zName; - int mask; - } aRank[] = { - { "rank", 0 }, - { "erank", BM25_EXPLAIN }, - { "rankc", BM25_FCOLUMNS }, - { "erankc", BM25_FCOLUMNS|BM25_EXPLAIN }, - { "ranks", BM25_FSTREAMS }, - { "eranks", BM25_FSTREAMS|BM25_EXPLAIN } - }; - - rc = sqlite4_create_tokenizer(db, "simple", (void *)pEnv, - fts5SimpleCreate, fts5SimpleTokenize, fts5SimpleDestroy - ); - if( rc==SQLITE4_OK ){ - rc = sqlite4_create_mi_function( - db, "snippet", -1, SQLITE4_UTF8, 0, fts5Snippet, 0); - } - - for(i=0; rc==SQLITE4_OK && ilen ){ p2 = len-p1; if( p2<0 ) p2 = 0; } - sqlite4_result_blob(context, (char*)&z[p1], (int)p2, SQLITE4_TRANSIENT, 0); + sqlite4_result_blob(context, (char*)&z[p1], (int)p2, SQLITE4_TRANSIENT); } } /* ** Implementation of the round() function @@ -334,11 +334,11 @@ z1 = contextMalloc(context, ((i64)n)+1); if( z1 ){ for(i=0; i>4)&0x0F]; zText[(i*2)+3] = hexdigits[(zBlob[i])&0x0F]; } zText[(nBlob*2)+2] = '\''; zText[(nBlob*2)+3] = '\0'; - zText[0] = 'x'; + zText[0] = 'X'; zText[1] = '\''; - sqlite4_result_text(context, zText, -1, SQLITE4_TRANSIENT, 0); + sqlite4_result_text(context, zText, -1, SQLITE4_TRANSIENT); sqlite4_free(sqlite4_context_env(context), zText); } break; } case SQLITE4_TEXT: { @@ -900,17 +899,17 @@ z[j++] = '\''; } } z[j++] = '\''; z[j] = 0; - sqlite4_result_text(context, z, j, SQLITE4_DYNAMIC, 0); + sqlite4_result_text(context, z, j, SQLITE4_DYNAMIC); } break; } default: { assert( sqlite4_value_type(argv[0])==SQLITE4_NULL ); - sqlite4_result_text(context, "NULL", 4, SQLITE4_STATIC, 0); + sqlite4_result_text(context, "NULL", 4, SQLITE4_STATIC); break; } } } @@ -937,11 +936,11 @@ unsigned char c = *pBlob; *(z++) = hexdigits[(c>>4)&0xf]; *(z++) = hexdigits[c&0xf]; } *z = 0; - sqlite4_result_text(context, zHex, n*2, SQLITE4_DYNAMIC, 0); + sqlite4_result_text(context, zHex, n*2, SQLITE4_DYNAMIC); } } /* ** The zeroblob(N) function returns a zero-filled blob of size N bytes. @@ -1046,11 +1045,11 @@ assert( j+nStr-i+1==nOut ); memcpy(&zOut[j], &zStr[i], nStr-i); j += nStr - i; assert( j<=nOut ); zOut[j] = 0; - sqlite4_result_text(context, (char*)zOut, j, SQLITE4_DYNAMIC, 0); + sqlite4_result_text(context, (char*)zOut, j, SQLITE4_DYNAMIC); } /* ** Implementation of the TRIM(), LTRIM(), and RTRIM() functions. ** The userdata is 0x1 for left trim, 0x2 for right trim, 0x3 for both. @@ -1130,11 +1129,11 @@ } if( zCharSet ){ sqlite4_free(sqlite4_context_env(context), azChar); } } - sqlite4_result_text(context, (char*)zIn, nIn, SQLITE4_TRANSIENT, 0); + sqlite4_result_text(context, (char*)zIn, nIn, SQLITE4_TRANSIENT); } /* IMP: R-25361-16150 This function is omitted from SQLite by default. It ** is only available if the SQLITE4_SOUNDEX compile-time option is used @@ -1185,15 +1184,15 @@ } while( j<4 ){ zResult[j++] = '0'; } zResult[j] = 0; - sqlite4_result_text(context, zResult, 4, SQLITE4_TRANSIENT, 0); + sqlite4_result_text(context, zResult, 4, SQLITE4_TRANSIENT); }else{ /* IMP: R-64894-50321 The string "?000" is returned if the argument ** is NULL or contains no ASCII alphabetic characters. */ - sqlite4_result_text(context, "?000", 4, SQLITE4_STATIC, 0); + sqlite4_result_text(context, "?000", 4, SQLITE4_STATIC); } } #endif /* SQLITE4_SOUNDEX */ #if 0 /*ndef SQLITE4_OMIT_LOAD_EXTENSION*/ @@ -1417,11 +1416,11 @@ sqlite4_result_error_toobig(context); }else if( pAccum->mallocFailed ){ sqlite4_result_error_nomem(context); }else{ sqlite4_result_text(context, sqlite4StrAccumFinish(pAccum), -1, - SQLITE4_DYNAMIC, 0); + SQLITE4_DYNAMIC); } } } /* Index: src/hash.c ================================================================== --- src/hash.c +++ src/hash.c @@ -13,60 +13,22 @@ ** used in SQLite. */ #include "sqliteInt.h" #include -/* -** The hashing function. -*/ -static unsigned int strHash(const char *z, int nKey){ - int h = 0; - assert( nKey>=0 ); - while( nKey > 0 ){ - h = (h<<3) ^ h ^ sqlite4UpperToLower[(unsigned char)*z++]; - nKey--; - } - return h; -} - -static int strCmp(const char *z1, const char *z2, int n){ - return sqlite4_strnicmp(z1, z2, n); -} - -static unsigned int binHash(const char *z, int nKey){ - int h = 0; - assert( nKey>=0 ); - while( nKey > 0 ){ - h = (h<<3) ^ h ^ ((unsigned char)*z++); - nKey--; - } - return h; -} - -static int binCmp(const char *z1, const char *z2, int n){ - return memcmp(z1, z2, n); -} - /* Turn bulk memory into a hash table object by initializing the ** fields of the Hash structure. ** ** "pNew" is a pointer to the hash table that is to be initialized. */ -void sqlite4HashInit(sqlite4_env *pEnv, Hash *pNew, int bBin){ +void sqlite4HashInit(sqlite4_env *pEnv, Hash *pNew){ assert( pNew!=0 ); pNew->first = 0; pNew->count = 0; pNew->htsize = 0; pNew->ht = 0; pNew->pEnv = pEnv; - if( bBin ){ - pNew->xHash = binHash; - pNew->xCmp = binCmp; - }else{ - pNew->xHash = strHash; - pNew->xCmp = strCmp; - } } /* Remove all entries from a hash table. Reclaim all memory. ** Call this routine to delete a hash table or to reset a hash table ** to the empty state. @@ -85,10 +47,24 @@ sqlite4_free(pH->pEnv, elem); elem = next_elem; } pH->count = 0; } + +/* +** The hashing function. +*/ +static unsigned int strHash(const char *z, int nKey){ + int h = 0; + assert( nKey>=0 ); + while( nKey > 0 ){ + h = (h<<3) ^ h ^ sqlite4UpperToLower[(unsigned char)*z++]; + nKey--; + } + return h; +} + /* Link pNew element into the hash table pH. If pEntry!=0 then also ** insert pNew into the pEntry hash bucket. */ static void insertElement( @@ -148,11 +124,11 @@ sqlite4_free(pH->pEnv, pH->ht); pH->ht = new_ht; pH->htsize = new_size = sqlite4MallocSize(pH->pEnv, new_ht)/sizeof(struct _ht); memset(new_ht, 0, new_size*sizeof(struct _ht)); for(elem=pH->first, pH->first=0; elem; elem = next_elem){ - unsigned int h = pH->xHash(elem->pKey, elem->nKey) % new_size; + unsigned int h = strHash(elem->pKey, elem->nKey) % new_size; next_elem = elem->next; insertElement(pH, &new_ht[h], elem); } return 1; } @@ -177,11 +153,11 @@ }else{ elem = pH->first; count = pH->count; } while( count-- && ALWAYS(elem) ){ - if( elem->nKey==nKey && pH->xCmp(elem->pKey,pKey,nKey)==0 ){ + if( elem->nKey==nKey && sqlite4StrNICmp(elem->pKey,pKey,nKey)==0 ){ return elem; } elem = elem->next; } return 0; @@ -231,11 +207,11 @@ assert( pH!=0 ); assert( pKey!=0 ); assert( nKey>=0 ); if( pH->ht ){ - h = pH->xHash(pKey, nKey) % pH->htsize; + h = strHash(pKey, nKey) % pH->htsize; }else{ h = 0; } elem = findElementGivenHash(pH, pKey, nKey, h); return elem ? elem->data : 0; @@ -262,11 +238,11 @@ assert( pH!=0 ); assert( pKey!=0 ); assert( nKey>=0 ); if( pH->htsize ){ - h = pH->xHash(pKey, nKey) % pH->htsize; + h = strHash(pKey, nKey) % pH->htsize; }else{ h = 0; } elem = findElementGivenHash(pH,pKey,nKey,h); if( elem ){ @@ -288,15 +264,15 @@ new_elem->data = data; pH->count++; if( pH->count>=10 && pH->count > 2*pH->htsize ){ if( rehash(pH, pH->count*2) ){ assert( pH->htsize>0 ); - h = pH->xHash(pKey, nKey) % pH->htsize; + h = strHash(pKey, nKey) % pH->htsize; } } if( pH->ht ){ insertElement(pH, &pH->ht[h], new_elem); }else{ insertElement(pH, 0, new_elem); } return 0; } Index: src/hash.h ================================================================== --- src/hash.h +++ src/hash.h @@ -40,12 +40,10 @@ ** in the table, it is faster to do a linear search than to manage ** the hash table. */ struct Hash { sqlite4_env *pEnv; /* Memory allocation environment */ - unsigned int (*xHash)(const char *, int); - int (*xCmp)(const char *, const char *, int); unsigned int htsize; /* Number of buckets in the hash table */ unsigned int count; /* Number of entries in this table */ HashElem *first; /* The first element of the array */ struct _ht { /* the hash table */ int count; /* Number of entries with this hash */ @@ -66,11 +64,11 @@ }; /* ** Access routines. To delete, insert a NULL pointer. */ -void sqlite4HashInit(sqlite4_env *pEnv, Hash*, int); +void sqlite4HashInit(sqlite4_env *pEnv, Hash*); void *sqlite4HashInsert(Hash*, const char *pKey, int nKey, void *pData); void *sqlite4HashFind(const Hash*, const char *pKey, int nKey); void sqlite4HashClear(Hash*); /* @@ -86,14 +84,14 @@ ** } */ #define sqliteHashFirst(H) ((H)->first) #define sqliteHashNext(E) ((E)->next) #define sqliteHashData(E) ((E)->data) -#define sqliteHashKey(E) ((E)->pKey) -#define sqliteHashKeysize(E) ((E)->nKey) +/* #define sqliteHashKey(E) ((E)->pKey) // NOT USED */ +/* #define sqliteHashKeysize(E) ((E)->nKey) // NOT USED */ /* ** Number of entries in a hash table */ /* #define sqliteHashCount(H) ((H)->count) // NOT USED */ #endif /* _SQLITE4_HASH_H_ */ Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -22,12 +22,11 @@ int iCur, /* The cursor number of the table */ int iDb, /* The database index in sqlite4.aDb[] */ Table *pTab, /* The table to be opened */ int opcode /* OP_OpenRead or OP_OpenWrite */ ){ - Index *pPk = sqlite4FindPrimaryKey(pTab, 0); - sqlite4OpenIndex(p, iCur, iDb, pPk, opcode); + assert( 0 ); } /* ** Open VDBE cursor iCur to access index pIdx. pIdx is guaranteed to be ** a part of database iDb. @@ -247,11 +246,11 @@ ** (3) Register to hold the rowid in sqlite_sequence of pTab ** ** The 2nd register is the one that is returned. That is all the ** insert routine needs to know about. */ -static int autoIncBegin( +/*static FIXME: make static when this function gets used. */ int autoIncBegin( Parse *pParse, /* Parsing context */ int iDb, /* Index of the database holding pTab */ Table *pTab /* The table we are writing to */ ){ int memId = 0; /* Register holding maximum rowid */ @@ -321,11 +320,11 @@ ** This routine should be called when the top of the stack holds a ** new rowid that is about to be inserted. If that new rowid is ** larger than the maximum rowid in the memId memory cell, then the ** memory cell is updated. The stack is unchanged. */ -static void autoIncStep(Parse *pParse, int memId, int regRowid){ +/*static FIXME: make static when this function gets used. */ void autoIncStep(Parse *pParse, int memId, int regRowid){ if( memId>0 ){ sqlite4VdbeAddOp2(pParse->pVdbe, OP_MemMax, memId, regRowid); } } @@ -516,22 +515,22 @@ int addrSelect = 0; /* Address of coroutine that implements the SELECT */ SelectDest dest; /* Destination for SELECT on rhs of INSERT */ int iDb; /* Index of database holding TABLE */ Db *pDb; /* The database containing table being inserted into */ int appendFlag = 0; /* True if the insert is likely to be an append */ - int iPk; /* Cursor offset of PK index cursor */ - Index *pPk; /* Primary key for table pTab */ - int iIntPKCol = -1; /* Column of INTEGER PRIMARY KEY or -1 */ - int bImplicitPK; /* True if table pTab has an implicit PK */ /* Register allocations */ int regFromSelect = 0;/* Base register for data coming from SELECT */ int regEof = 0; /* Register recording end of SELECT data */ int *aRegIdx = 0; /* One register allocated to each index */ - int regContent; /* First register in column value array */ - int regRowid; /* If bImplicitPK, register holding IPK */ - int regAutoinc; /* Register holding the AUTOINCREMENT counter */ + + int iPk; /* Cursor offset of PK index cursor */ + Index *pPk; /* Primary key for table pTab */ + int bImplicitPK; /* True if table pTab has an implicit PK */ + int regContent; /* First register in column value array */ + int regRowid; /* If bImplicitPK, register holding IPK */ + #ifndef SQLITE4_OMIT_TRIGGER int isView; /* True if attempting to insert into a view */ Trigger *pTrigger; /* List of triggers on pTab, if required */ int tmask; /* Mask of trigger times */ @@ -563,20 +562,11 @@ ** Also set pPk to point to the primary key, and iPk to the cursor offset ** of the primary key cursor (i.e. so that the cursor opened on the primary ** key index is VDBE cursor (baseCur+iPk). */ pPk = sqlite4FindPrimaryKey(pTab, &iPk); assert( (pPk==0)==IsView(pTab) ); - if( pPk ){ - bImplicitPK = pPk->aiColumn[0]==(-1); - if( pPk->fIndex & IDX_IntPK ){ - assert( pPk->nColumn==1 ); - iIntPKCol = pPk->aiColumn[0]; - } - }else{ - bImplicitPK = 0; - } - + bImplicitPK = (pPk && pPk->aiColumn[0]==-1); /* Figure out if we have any triggers and if the table being ** inserted into is a view. */ #ifndef SQLITE4_OMIT_TRIGGER pTrigger = sqlite4TriggersExist(pParse, pTab, TK_INSERT, 0, &tmask); @@ -628,15 +618,10 @@ assert( pList==0 ); goto insert_end; } #endif /* SQLITE4_OMIT_XFER_OPT */ - /* If this is an AUTOINCREMENT table, look up the sequence number in the - ** sqlite_sequence table and store it in memory cell regAutoinc. - */ - regAutoinc = autoIncBegin(pParse, iDb, pTab); - /* Figure out how many columns of data are supplied. If the data ** is coming from a SELECT statement, then generate a co-routine that ** produces a single row of the SELECT on each invocation. The ** co-routine is the common header to the 3rd and 4th templates. */ @@ -781,11 +766,11 @@ pColumn->a[i].idx = -1; } for(i=0; inId; i++){ char *zTest = pColumn->a[i].zName; for(j=0; jnCol; j++){ - if( sqlite4_stricmp(zTest, pTab->aCol[j].zName)==0 ){ + if( sqlite4StrICmp(zTest, pTab->aCol[j].zName)==0 ){ pColumn->a[i].idx = j; break; } } if( j==pTab->nCol ){ @@ -847,11 +832,11 @@ addrInsTop = sqlite4VdbeAddOp1(v, OP_If, regEof); } /* Allocate an array of registers in which to assemble the values for the ** new row. If the table has an explicit primary key, we need one register - ** for each table column. If the table uses an implicit primary key, then + ** for each table column. If the table uses an implicit primary key, the ** nCol+1 registers are required. */ regRowid = ++pParse->nMem; regContent = pParse->nMem+1; pParse->nMem += pTab->nCol; @@ -863,27 +848,26 @@ } endOfLoop = sqlite4VdbeMakeLabel(v); for(i=0; inCol; i++){ - int regDest = regContent+i; j = i; if( pColumn ){ for(j=0; jnId; j++){ if( pColumn->a[j].idx==i ) break; } } if( nColumn==0 || (pColumn && j>=pColumn->nId) ){ - sqlite4ExprCode(pParse, pTab->aCol[i].pDflt, regDest); + sqlite4ExprCode(pParse, pTab->aCol[i].pDflt, regContent+i); }else if( useTempTable ){ - sqlite4VdbeAddOp3(v, OP_Column, srcTab, j, regDest); + sqlite4VdbeAddOp3(v, OP_Column, srcTab, j, regContent+i); }else if( pSelect ){ - sqlite4VdbeAddOp2(v, OP_SCopy, regFromSelect+j, regDest); + sqlite4VdbeAddOp2(v, OP_SCopy, regFromSelect+j, regContent+i); }else{ assert( pSelect==0 ); /* Otherwise useTempTable is true */ - sqlite4ExprCodeAndCache(pParse, pList->a[j].pExpr, regDest); + sqlite4ExprCodeAndCache(pParse, pList->a[j].pExpr, regContent+i); } } if( !isView ){ sqlite4VdbeAddOp2(v, OP_Affinity, regContent, pTab->nCol); @@ -898,19 +882,10 @@ pParse, pTrigger, TK_INSERT, 0, TRIGGER_BEFORE, pTab, (regRowid - pTab->nCol - 1), onError, endOfLoop ); } - if( iIntPKCol>=0 ){ - int regDest = regContent+iIntPKCol; - int a1; - a1 = sqlite4VdbeAddOp1(v, OP_NotNull, regDest); - sqlite4VdbeAddOp3(v, OP_NewRowid, baseCur, regDest, regAutoinc); - sqlite4VdbeJumpHere(v, a1); - autoIncStep(pParse, regAutoinc, regDest); - } - if( bImplicitPK ){ assert( !isView ); sqlite4VdbeAddOp2(v, OP_NewRowid, baseCur+iPk, regRowid); } @@ -1085,13 +1060,10 @@ } #else /* !defined(SQLITE4_OMIT_CHECK) */ # define generateCheckChecks(a,b,c,d,e) #endif -/* -** Locate the primary key index for a table. -*/ Index *sqlite4FindPrimaryKey( Table *pTab, /* Table to locate primary key for */ int *piPk /* OUT: Index of PRIMARY KEY */ ){ Index *p; @@ -1317,13 +1289,10 @@ int idx = pPk->aiColumn[i]; sqlite4VdbeAddOp2(v, OP_SCopy, regContent+idx, regTmp+i+pIdx->nColumn); } } sqlite4VdbeAddOp3(v, OP_MakeIdxKey, iIdx, regTmp, regKey); - if( pIdx==pPk && (pPk->fIndex & IDX_IntPK)!=0 ){ - sqlite4VdbeChangeP5(v, OPFLAG_LASTROWID); - } VdbeComment((v, "key for %s", pIdx->zName)); /* If Index.onError==OE_None, then pIdx is not a UNIQUE or PRIMARY KEY ** index. In this case there is no need to test the index for uniqueness ** - all that is required is to populate the regKey register. Jump @@ -1448,16 +1417,11 @@ sqlite4ExprCacheAffinityChange(pParse, regContent, pTab->nCol); /* Write the entry to each index. */ for(i=0, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ assert( pIdx->eIndexType!=SQLITE4_INDEX_PRIMARYKEY || aRegIdx[i] ); - if( pIdx->eIndexType==SQLITE4_INDEX_FTS5 ){ - int iPK; - sqlite4FindPrimaryKey(pTab, &iPK); - sqlite4Fts5CodeUpdate(pParse, pIdx, 0, aRegIdx[iPK], regContent, 0); - } - else if( aRegIdx[i] ){ + if( aRegIdx[i] ){ int regData = 0; int flags = 0; if( pIdx->eIndexType==SQLITE4_INDEX_PRIMARYKEY ){ regData = regRec; flags = pik_flags; @@ -1537,11 +1501,11 @@ return z2==0; } if( z2==0 ){ return 0; } - return sqlite4_stricmp(z1, z2)==0; + return sqlite4StrICmp(z1, z2)==0; } /* ** Check to see if index pSrc is compatible as a source of data DELETED src/kv.c Index: src/kv.c ================================================================== --- src/kv.c +++ /dev/null @@ -1,496 +0,0 @@ -/* -** 2012 January 21 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** General wrapper functions around the various KV storage engine -** implementations. It also implements tracing of calls to the KV -** engine and some higher-level ensembles of the low-level storage -** calls. -*/ -#include "sqliteInt.h" - -/* -** Names of error codes used for tracing. -*/ -static const char *kvErrName(int e){ - const char *zName; - switch( e ){ - case SQLITE4_OK: zName = "OK"; break; - case SQLITE4_ERROR: zName = "ERROR"; break; - case SQLITE4_INTERNAL: zName = "INTERNAL"; break; - case SQLITE4_PERM: zName = "PERM"; break; - case SQLITE4_ABORT: zName = "ABORT"; break; - case SQLITE4_BUSY: zName = "BUSY"; break; - case SQLITE4_LOCKED: zName = "LOCKED"; break; - case SQLITE4_NOMEM: zName = "NOMEM"; break; - case SQLITE4_READONLY: zName = "READONLY"; break; - case SQLITE4_INTERRUPT: zName = "INTERRUPT"; break; - case SQLITE4_IOERR: zName = "IOERR"; break; - case SQLITE4_CORRUPT: zName = "CORRUPT"; break; - case SQLITE4_NOTFOUND: zName = "NOTFOUND"; break; - case SQLITE4_FULL: zName = "FULL"; break; - case SQLITE4_CANTOPEN: zName = "CANTOPEN"; break; - case SQLITE4_PROTOCOL: zName = "PROTOCOL"; break; - case SQLITE4_EMPTY: zName = "EMPTY"; break; - case SQLITE4_SCHEMA: zName = "SCHEMA"; break; - case SQLITE4_TOOBIG: zName = "TOOBIG"; break; - case SQLITE4_CONSTRAINT: zName = "CONSTRAINT"; break; - case SQLITE4_MISMATCH: zName = "MISMATCH"; break; - case SQLITE4_MISUSE: zName = "MISUSE"; break; - case SQLITE4_NOLFS: zName = "NOLFS"; break; - case SQLITE4_AUTH: zName = "AUTH"; break; - case SQLITE4_FORMAT: zName = "FORMAT"; break; - case SQLITE4_RANGE: zName = "RANGE"; break; - case SQLITE4_NOTADB: zName = "NOTADB"; break; - case SQLITE4_ROW: zName = "ROW"; break; - case SQLITE4_DONE: zName = "DONE"; break; - case SQLITE4_INEXACT: zName = "INEXACT"; break; - default: zName = "???"; break; - } - return zName; -} - -/* -** Do any requested tracing -*/ -static void kvTrace(KVStore *p, const char *zFormat, ...){ - if( p->fTrace ){ - va_list ap; - char *z; - - va_start(ap, zFormat); - z = sqlite4_vmprintf(p->pEnv, zFormat, ap); - va_end(ap); - printf("%s.%s\n", p->zKVName, z); - fflush(stdout); - sqlite4_free(p->pEnv, z); - } -} - -/* -** Open a storage engine via URI -*/ -int sqlite4KVStoreOpen( - sqlite4 *db, /* The database connection doing the open */ - const char *zName, /* Symbolic name for this database */ - const char *zUri, /* URI for this database */ - KVStore **ppKVStore, /* Write the new KVStore object here */ - unsigned flags /* Option flags */ -){ - KVStore *pNew = 0; - int rc; - sqlite4_env *pEnv = &sqlite4DefaultEnv; /* OR db->pEnv */ - const char *zStorageName; - KVFactory *pMkr; - sqlite4_kvfactory xFactory; - - if( (flags & SQLITE4_KVOPEN_TEMPORARY)!=0 || zUri==0 || zUri[0]==0 ){ - zStorageName = "temp"; - }else{ - zStorageName = sqlite4_uri_parameter(zName, "kv"); - if( zStorageName==0 ){ - if( memcmp(":memory:", zUri, 8)==0 ){ - zStorageName = "temp"; - }else{ - zStorageName = "main"; - } - } - } - *ppKVStore = 0; - sqlite4_mutex_enter(pEnv->pFactoryMutex); - for(pMkr=pEnv->pFactory; pMkr && strcmp(zStorageName,pMkr->zName); - pMkr=pMkr->pNext){} - xFactory = pMkr ? pMkr->xFactory : 0; - sqlite4_mutex_leave(pEnv->pFactoryMutex); - if( xFactory==0 ){ - return SQLITE4_ERROR; - } - rc = xFactory(pEnv, &pNew, zUri, flags); - *ppKVStore = pNew; - if( pNew ){ - sqlite4_randomness(pEnv, sizeof(pNew->kvId), &pNew->kvId); - sqlite4_snprintf(pNew->zKVName, sizeof(pNew->zKVName), - "%s", zName); - pNew->fTrace = (db->flags & SQLITE4_KvTrace)!=0; - kvTrace(pNew, "open(%s,%d,0x%04x)", zUri, pNew->kvId, flags); - } - return rc; -} - -/* Convert binary data to hex for display in trace messages */ -static void binToHex(char *zOut, int mxOut, const KVByteArray *a, KVSize n){ - int i; - if( n>mxOut/2-1 ) n = mxOut/2-1; - for(i=0; i>4)&0xf]; - zOut[i*2+1] = "0123456789abcdef"[a[i]&0xf]; - } - zOut[i*2] = 0; -} - -/* -** The following wrapper functions invoke the underlying methods of -** the storage object and add optional tracing. -*/ -int sqlite4KVStoreReplace( - KVStore *p, - const KVByteArray *pKey, KVSize nKey, - const KVByteArray *pData, KVSize nData -){ - if( p->fTrace ){ - char zKey[52], zData[52]; - binToHex(zKey, sizeof(zKey), pKey, nKey); - binToHex(zData, sizeof(zData), pData, nData); - kvTrace(p, "xReplace(%d,%s,%d,%s,%d)", - p->kvId, zKey, (int)nKey, zData, (int)nData); - } - return p->pStoreVfunc->xReplace(p,pKey,nKey,pData,nData); -} -int sqlite4KVStoreOpenCursor(KVStore *p, KVCursor **ppKVCursor){ - KVCursor *pCur; - int rc; - - rc = p->pStoreVfunc->xOpenCursor(p, &pCur); - *ppKVCursor = pCur; - if( pCur ){ - sqlite4_randomness(pCur->pEnv, sizeof(pCur->curId), &pCur->curId); - pCur->fTrace = p->fTrace; - pCur->pStore = p; - } - kvTrace(p, "xOpenCursor(%d,%d) -> %s", - p->kvId, pCur?pCur->curId:-1, kvErrName(rc)); - return rc; -} -int sqlite4KVCursorSeek( - KVCursor *p, - const KVByteArray *pKey, KVSize nKey, - int dir -){ - int rc; - assert( dir==0 || dir==(+1) || dir==(-1) || dir==(-2) ); - rc = p->pStoreVfunc->xSeek(p,pKey,nKey,dir); - if( p->fTrace ){ - char zKey[52]; - binToHex(zKey, sizeof(zKey), pKey, nKey); - kvTrace(p->pStore, "xSeek(%d,%s,%d,%d) -> %s", - p->curId, zKey, (int)nKey, dir, kvErrName(rc)); - } - return rc; -} -int sqlite4KVCursorNext(KVCursor *p){ - int rc; - rc = p->pStoreVfunc->xNext(p); - kvTrace(p->pStore, "xNext(%d) -> %s", p->curId, kvErrName(rc)); - return rc; -} -int sqlite4KVCursorPrev(KVCursor *p){ - int rc; - rc = p->pStoreVfunc->xPrev(p); - kvTrace(p->pStore, "xPrev(%d) -> %s", p->curId, kvErrName(rc)); - return rc; -} -int sqlite4KVCursorDelete(KVCursor *p){ - int rc; - rc = p->pStoreVfunc->xDelete(p); - kvTrace(p->pStore, "xDelete(%d) -> %s", p->curId, kvErrName(rc)); - return rc; -} -int sqlite4KVCursorReset(KVCursor *p){ - int rc; - rc = p->pStoreVfunc->xReset(p); - kvTrace(p->pStore, "xReset(%d) -> %s", p->curId, kvErrName(rc)); - return rc; -} -int sqlite4KVCursorKey(KVCursor *p, const KVByteArray **ppKey, KVSize *pnKey){ - int rc; - rc = p->pStoreVfunc->xKey(p, ppKey, pnKey); - if( p->fTrace ){ - if( rc==SQLITE4_OK ){ - char zKey[52]; - binToHex(zKey, sizeof(zKey), *ppKey, *pnKey); - kvTrace(p->pStore, "xKey(%d,%s,%d)", p->curId, zKey, (int)*pnKey); - }else{ - kvTrace(p->pStore, "xKey(%d,)", p->curId, rc); - } - } - return rc; -} -int sqlite4KVCursorData( - KVCursor *p, - KVSize ofst, - KVSize n, - const KVByteArray **ppData, - KVSize *pnData -){ - int rc; - rc = p->pStoreVfunc->xData(p, ofst, n, ppData, pnData); - if( p->fTrace ){ - if( rc==SQLITE4_OK ){ - char zData[52]; - binToHex(zData, sizeof(zData), *ppData, *pnData); - kvTrace(p->pStore, "xData(%d,%d,%d,%s,%d)", - p->curId, (int)ofst, (int)n, zData, (int)*pnData); - }else{ - kvTrace(p->pStore, "xData(%d,%d,%d,)", - p->curId, (int)ofst, (int)n, rc); - } - } - return rc; -} -int sqlite4KVCursorClose(KVCursor *p){ - int rc = SQLITE4_OK; - if( p ){ - KVStore *pStore = p->pStore; - int curId = p->curId; - rc = p->pStoreVfunc->xCloseCursor(p); - kvTrace(pStore, "xCloseCursor(%d) -> %s", curId, kvErrName(rc)); - } - return rc; -} -int sqlite4KVStoreBegin(KVStore *p, int iLevel){ - int rc; - rc = p->pStoreVfunc->xBegin(p, iLevel); - kvTrace(p, "xBegin(%d,%d) -> %s", p->kvId, iLevel, kvErrName(rc)); - assert( p->iTransLevel==iLevel || rc!=SQLITE4_OK ); - return rc; -} -int sqlite4KVStoreCommitPhaseOne(KVStore *p, int iLevel){ - int rc; - assert( iLevel>=0 ); - assert( iLevel<=p->iTransLevel ); - if( p->iTransLevel==iLevel ) return SQLITE4_OK; - if( p->pStoreVfunc->xCommitPhaseOne ){ - rc = p->pStoreVfunc->xCommitPhaseOne(p, iLevel); - }else{ - rc = SQLITE4_OK; - } - kvTrace(p, "xCommitPhaseOne(%d,%d) -> %s", p->kvId, iLevel, kvErrName(rc)); - assert( p->iTransLevel>iLevel ); - return rc; -} -int sqlite4KVStoreCommitPhaseTwo(KVStore *p, int iLevel){ - int rc; - assert( iLevel>=0 ); - assert( iLevel<=p->iTransLevel ); - if( p->iTransLevel==iLevel ) return SQLITE4_OK; - rc = p->pStoreVfunc->xCommitPhaseTwo(p, iLevel); - kvTrace(p, "xCommitPhaseTwo(%d,%d) -> %s", p->kvId, iLevel, kvErrName(rc)); - assert( p->iTransLevel==iLevel || rc!=SQLITE4_OK ); - return rc; -} -int sqlite4KVStoreCommit(KVStore *p, int iLevel){ - int rc; - rc = sqlite4KVStoreCommitPhaseOne(p, iLevel); - if( rc==SQLITE4_OK ) rc = sqlite4KVStoreCommitPhaseTwo(p, iLevel); - return rc; -} -int sqlite4KVStoreRollback(KVStore *p, int iLevel){ - int rc; - assert( iLevel>=0 ); - assert( iLevel<=p->iTransLevel ); - rc = p->pStoreVfunc->xRollback(p, iLevel); - kvTrace(p, "xRollback(%d,%d) -> %s", p->kvId, iLevel, kvErrName(rc)); - assert( p->iTransLevel==iLevel || rc!=SQLITE4_OK ); - return rc; -} -int sqlite4KVStoreRevert(KVStore *p, int iLevel){ - int rc; - assert( iLevel>0 ); - assert( iLevel<=p->iTransLevel ); - if( p->pStoreVfunc->xRevert ){ - rc = p->pStoreVfunc->xRevert(p, iLevel); - kvTrace(p, "xRevert(%d,%d) -> %s", p->kvId, iLevel, kvErrName(rc)); - }else{ - rc = sqlite4KVStoreRollback(p, iLevel-1); - if( rc==SQLITE4_OK ){ - rc = sqlite4KVStoreBegin(p, iLevel); - } - } - assert( p->iTransLevel==iLevel || rc!=SQLITE4_OK ); - return rc; -} -int sqlite4KVStoreClose(KVStore *p){ - int rc; - if( p ){ - kvTrace(p, "xClose(%d)", p->kvId); - rc = p->pStoreVfunc->xClose(p); - } - return rc; -} - -/* -** Key for the meta-data -*/ -static const KVByteArray metadataKey[] = { 0x00, 0x00 }; - -static void writeMetaArray(KVByteArray *aMeta, int iElem, u32 iVal){ - int i = sizeof(u32) * iElem; - aMeta[i+0] = (iVal>>24)&0xff; - aMeta[i+1] = (iVal>>16)&0xff; - aMeta[i+2] = (iVal>>8) &0xff; - aMeta[i+3] = (iVal>>0) &0xff; -} - -/* -** Read nMeta unsigned 32-bit integers of metadata beginning at iStart. -*/ -int sqlite4KVStoreGetMeta(KVStore *p, int iStart, int nMeta, unsigned int *a){ - KVCursor *pCur; - int rc; - int i, j; - KVSize nData; - const KVByteArray *aData; - - rc = sqlite4KVStoreOpenCursor(p, &pCur); - if( rc==SQLITE4_OK ){ - rc = sqlite4KVCursorSeek(pCur, metadataKey, sizeof(metadataKey), 0); - if( rc==SQLITE4_NOTFOUND ){ - rc = SQLITE4_OK; - nData = 0; - }else if( rc==SQLITE4_OK ){ - rc = sqlite4KVCursorData(pCur, 0, -1, &aData, &nData); - } - if( rc==SQLITE4_OK ){ - i = 0; - j = iStart*4; - while( i=0 ){ - for(i=0; i<16 && i>4]; - zOut[i*3+1] = base16[v&0xf]; - zOut[16*3+3+i] = (v>=0x20 && v<=0x7e) ? v : '.'; - } - while( i<16 ){ - zOut[i*3] = ' '; - zOut[i*3+1] = ' '; - zOut[16*3+3+i] = ' '; - i++; - } - sqlite4DebugPrintf("%.3s %s\n", zPrefix, zOut); - n -= 16; - if( n<=0 ) break; - a += 16; - zPrefix = " "; - } -} - -/* -** Dump the entire content of a key-value database -*/ -void sqlite4KVStoreDump(KVStore *pStore){ - int rc; - int nRow = 0; - KVCursor *pCur; - KVSize nKey, nData; - KVByteArray const *aKey; - KVByteArray const *aData; - static const KVByteArray aProbe[] = { 0x00 }; - - rc = sqlite4KVStoreOpenCursor(pStore, &pCur); - if( rc==SQLITE4_OK ){ - rc = sqlite4KVCursorSeek(pCur, aProbe, 1, +1); - while( rc!=SQLITE4_NOTFOUND ){ - rc = sqlite4KVCursorKey(pCur, &aKey, &nKey); - if( rc ) break; - if( nRow>0 ) sqlite4DebugPrintf("\n"); - nRow++; - outputBinary(aKey, nKey, "K: "); - rc = sqlite4KVCursorData(pCur, 0, -1, &aData, &nData); - if( rc ) break; - outputBinary(aData, nData, "V: "); - rc = sqlite4KVCursorNext(pCur); - } - sqlite4KVCursorClose(pCur); - } -} -#endif /* SQLITE4_DEBUG */ DELETED src/kv.h Index: src/kv.h ================================================================== --- src/kv.h +++ /dev/null @@ -1,179 +0,0 @@ -/* -** 2012 January 20 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This header file defines the interface to the KV storage engine(s). -** -** Notes on the storage subsystem interface: -** -** The storage subsystem is a key/value database. All keys and values are -** binary with arbitrary content. Keys are unique. Keys compare in -** memcmp() order. Shorter keys appear first. -** -** The xBegin, xCommit, and xRollback methods change the transaction level -** of the store. The transaction level is a non-negative integer that is -** initialized to zero. The transaction level must be at least 1 in order -** for content to be read. The transaction level must be at least 2 for -** content to be modified. -** -** The xBegin method increases transaction level. The increase may be no -** more than 1 unless the transaction level is initially 0 in which case -** it can be increased immediately to 2. Increasing the transaction level -** to 1 or more makes a "snapshot" of the database file such that changes -** made by other connections are not visible. Calls to xBegin may fail -** with SQLITE4_BUSY if the initial transaction level is 0 or 1. -** -** A read-only database will fail an attempt to increase xBegin above 1. An -** implementation that does not support nested transactions will fail any -** attempt to increase the transaction level above 2. -** -** The xCommitPhaseOne and xCommitPhaseTwo methods implement a 2-phase -** commit that lowers the transaction level to the value given in the -** second argument, making all the changes made at higher transaction levels -** permanent. A rollback is still possible following phase one. If -** possible, errors should be reported during phase one so that a -** multiple-database transaction can still be rolled back if the -** phase one fails on a different database. Implementations that do not -** support two-phase commit can implement xCommitPhaseOne as a no-op function -** returning SQLITE4_OK. -** -** The xRollback method lowers the transaction level to the value given in -** its argument and reverts or undoes all changes made at higher transaction -** levels. An xRollback to level N causes the database to revert to the state -** it was in on the most recent xBegin to level N+1. -** -** The xRevert(N) method causes the state of the database file to go back -** to what it was immediately after the most recent xCommit(N). Higher-level -** subtransactions are cancelled. This call is equivalent to xRollback(N-1) -** followed by xBegin(N) but is atomic and might be more efficient. -** -** The xReplace method replaces the value for an existing entry with the -** given key, or creates a new entry with the given key and value if no -** prior entry exists with the given key. The key and value pointers passed -** into xReplace belong to the caller and will likely be destroyed when the -** call to xReplace returns so the xReplace routine must make its own -** copy of that information if it needs to retain it after returning. -** -** A cursor is at all times pointing to ether an entry in the database or -** to EOF. EOF means "no entry". Cursor operations other than xCloseCursor -** will fail if the transaction level is less than 1. -** -** The xSeek method moves a cursor to an entry in the database that matches -** the supplied key as closely as possible. If the dir argument is 0, then -** the match must be exact or else the seek fails and the cursor is left -** pointing to EOF. If dir is negative, then an exact match is -** found if it is available, otherwise the cursor is positioned at the largest -** entry that is less than the search key or to EOF if the store contains no -** entry less than the search key. If dir is positive, then an exist match -** is found if it is available, otherwise the cursor is left pointing the -** the smallest entry that is larger than the search key, or to EOF if there -** are no entries larger than the search key. -** -** The return code from xSeek might be one of the following: -** -** SQLITE4_OK The cursor is left pointing to any entry that -** exactly matchings the probe key. -** -** SQLITE4_INEXACT The cursor is left pointing to the nearest entry -** to the probe it could find, either before or after -** the probe, according to the dir argument. -** -** SQLITE4_NOTFOUND No suitable entry could be found. Either dir==0 and -** there was no exact match, or dir<0 and the probe is -** smaller than every entry in the database, or dir>0 and -** the probe is larger than every entry in the database. -** -** xSeek might also return some error code like SQLITE4_IOERR or -** SQLITE4_NOMEM. -** -** The xNext method will only be called following an xSeek with a positive dir, -** or another xNext. The xPrev method will only be called following an xSeek -** with a negative dir or another xPrev. Both xNext and xPrev will return -** SQLITE4_OK on success and SQLITE4_NOTFOUND if they run off the end of the -** database. Both routines might also return error codes such as -** SQLITE4_IOERR, SQLITE4_CORRUPT, or SQLITE4_NOMEM. -** -** Values returned by xKey and xData are guaranteed to remain stable until -** the next xSeek, xNext, xPrev, xReset, xDelete, or xCloseCursor on the same -** cursor. This is true even if the transaction level is reduced to zero, -** or if the content of the entry is changed by xInsert, xDelete on a different -** cursor, or xRollback. The content returned by repeated calls to xKey and -** xData is allowed (but is not required) to change if xInsert, xDelete, or -** xRollback are invoked in between the calls, but the content returned by -** every call must be stable until the cursor moves, or is reset or closed. -** The cursor owns the values returned by xKey and xData and will take -** responsiblity for freeing memory used to hold those values when appropriate. -** -** The xDelete method deletes the entry that the cursor is currently -** pointing at. However, subsequent xNext or xPrev calls behave as if the -** entries is not actually deleted until the cursor moves. In other words -** it is acceptable to xDelete an entry out from under a cursor. Subsequent -** xNext or xPrev calls on that cursor will work the same as if the entry -** had not been deleted. Two cursors can be pointing to the same entry and -** one cursor can xDelete and the other cursor is expected to continue -** functioning normally, including responding correctly to subsequent -** xNext and xPrev calls. -*/ - -/* Typedefs of datatypes */ -typedef struct sqlite4_kvstore KVStore; -typedef struct sqlite4_kv_methods KVStoreMethods; -typedef struct sqlite4_kvcursor KVCursor; -typedef unsigned char KVByteArray; -typedef sqlite4_kvsize KVSize; - - -int sqlite4KVStoreOpenMem(sqlite4_env*, KVStore**, const char *, unsigned); -int sqlite4KVStoreOpenLsm(sqlite4_env*, KVStore**, const char *, unsigned); -int sqlite4KVStoreOpen( - sqlite4*, - const char *zLabel, - const char *zUri, - KVStore**, - unsigned flags -); -int sqlite4KVStoreReplace( - KVStore*, - const KVByteArray *pKey, KVSize nKey, - const KVByteArray *pData, KVSize nData -); -int sqlite4KVStoreOpenCursor(KVStore *p, KVCursor **ppKVCursor); -int sqlite4KVCursorSeek( - KVCursor *p, - const KVByteArray *pKey, KVSize nKey, - int dir -); -int sqlite4KVCursorNext(KVCursor *p); -int sqlite4KVCursorPrev(KVCursor *p); -int sqlite4KVCursorDelete(KVCursor *p); -int sqlite4KVCursorReset(KVCursor *p); -int sqlite4KVCursorKey(KVCursor *p, const KVByteArray **ppKey, KVSize *pnKey); -int sqlite4KVCursorData( - KVCursor *p, - KVSize ofst, - KVSize n, - const KVByteArray **ppData, - KVSize *pnData -); -int sqlite4KVCursorClose(KVCursor *p); -int sqlite4KVStoreBegin(KVStore *p, int iLevel); -int sqlite4KVStoreCommitPhaseOne(KVStore *p, int iLevel); -int sqlite4KVStoreCommitPhaseTwo(KVStore *p, int iLevel); -int sqlite4KVStoreCommit(KVStore *p, int iLevel); -int sqlite4KVStoreRollback(KVStore *p, int iLevel); -int sqlite4KVStoreRevert(KVStore *p, int iLevel); -int sqlite4KVStoreClose(KVStore *p); - -int sqlite4KVStoreGetMeta(KVStore *p, int, int, unsigned int*); -int sqlite4KVStorePutMeta(sqlite4*, KVStore *p, int, int, unsigned int*); -#ifdef SQLITE4_DEBUG - void sqlite4KVStoreDump(KVStore *p); -#endif Index: src/kvlsm.c ================================================================== --- src/kvlsm.c +++ src/kvlsm.c @@ -9,32 +9,24 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* ** ** An in-memory key/value storage subsystem that presents the interfadce -** defined by kv.h +** defined by storage.h */ #include "sqliteInt.h" #include "lsm.h" -/* Forward declarations of objects */ typedef struct KVLsm KVLsm; typedef struct KVLsmCsr KVLsmCsr; -/* -** An instance of an open connection to an LSM store. A subclass of KVStore. -*/ struct KVLsm { KVStore base; /* Base class, must be first */ lsm_db *pDb; /* LSM database handle */ lsm_cursor *pCsr; /* LSM cursor holding read-trans open */ }; -/* -** An instance of an open cursor pointing into an LSM store. A subclass -** of KVCursor. -*/ struct KVLsmCsr { KVCursor base; /* Base class. Must be first */ lsm_cursor *pCsr; /* LSM cursor handle */ }; @@ -163,11 +155,11 @@ KVStore *pKVStore, const KVByteArray *aKey, KVSize nKey, const KVByteArray *aData, KVSize nData ){ KVLsm *p = (KVLsm *)pKVStore; - return lsm_insert(p->pDb, (void *)aKey, nKey, (void *)aData, nData); + return lsm_write(p->pDb, (void *)aKey, nKey, (void *)aData, nData); } /* ** Create a new cursor object. */ @@ -307,11 +299,12 @@ KVCursor *pKVCursor, /* The cursor whose key is desired */ const KVByteArray **paKey, /* Make this point to the key */ KVSize *pN /* Make this point to the size of the key */ ){ KVLsmCsr *pCsr = (KVLsmCsr *)pKVCursor; - if( 0==lsm_csr_valid(pCsr->pCsr) ) return SQLITE4_DONE; + + assert( lsm_csr_valid(pCsr->pCsr) ); return lsm_csr_key(pCsr->pCsr, (const void **)paKey, (int *)pN); } /* ** Return the data of the node the cursor is pointing to. @@ -381,18 +374,18 @@ *peSafety = eParam-1; break; } case SQLITE4_KVCTRL_LSM_FLUSH: { - lsm_flush(p->pDb); + lsm_work(p->pDb, LSM_WORK_FLUSH, 0, 0); break; } case SQLITE4_KVCTRL_LSM_MERGE: { int nPage = *(int*)pArg; int nWrite = 0; - lsm_work(p->pDb, 0, nPage, &nWrite); + lsm_work(p->pDb, LSM_WORK_OPTIMIZE, nPage, &nWrite); *(int*)pArg = nWrite; break; } case SQLITE4_KVCTRL_LSM_CHECKPOINT: { @@ -451,12 +444,11 @@ }else{ struct Config { const char *zParam; int eParam; } aConfig[] = { - { "lsm_block_size", LSM_CONFIG_BLOCK_SIZE }, - { "lsm_multiple_processes", LSM_CONFIG_MULTIPLE_PROCESSES } + { "lsm_block_size", LSM_CONFIG_BLOCK_SIZE } }; memset(pNew, 0, sizeof(KVLsm)); pNew->base.pStoreVfunc = &kvlsmMethods; pNew->base.pEnv = pEnv; Index: src/kvmem.c ================================================================== --- src/kvmem.c +++ src/kvmem.c @@ -9,11 +9,11 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* ** ** An in-memory key/value storage subsystem that presents the interfadce -** defined by kv.h +** defined by storage.h */ #include "sqliteInt.h" /* Forward declarations of object names */ typedef struct KVMemNode KVMemNode; Index: src/lsm.h ================================================================== --- src/lsm.h +++ src/lsm.h @@ -21,29 +21,28 @@ #endif /* ** Opaque handle types. */ -typedef struct lsm_compress lsm_compress; /* Compression library functions */ -typedef struct lsm_compress_factory lsm_compress_factory; +typedef struct lsm_db lsm_db; /* Database connection handle */ typedef struct lsm_cursor lsm_cursor; /* Database cursor handle */ -typedef struct lsm_db lsm_db; /* Database connection handle */ -typedef struct lsm_env lsm_env; /* Runtime environment */ -typedef struct lsm_file lsm_file; /* OS file handle */ typedef struct lsm_mutex lsm_mutex; /* Mutex handle */ +typedef struct lsm_file lsm_file; /* OS file handle */ /* 64-bit integer type used for file offsets. */ typedef long long int lsm_i64; /* 64-bit signed integer type */ +/* Forward reference */ +typedef struct lsm_env lsm_env; /* Runtime environment */ +typedef struct lsm_compress lsm_compress; /* Compression library functions */ + /* Candidate values for the 3rd argument to lsm_env.xLock() */ #define LSM_LOCK_UNLOCK 0 #define LSM_LOCK_SHARED 1 #define LSM_LOCK_EXCL 2 /* -** CAPI: Database Runtime Environment -** ** Run-time environment used by LSM */ struct lsm_env { int nByte; /* Size of this structure in bytes */ int iVersion; /* Version number of this structure (1) */ @@ -84,19 +83,41 @@ int (*xSleep)(lsm_env*, int microseconds); /* New fields may be added in future releases, in which case the ** iVersion value will increase. */ }; + +/* +** The compression library interface. +*/ +struct lsm_compress { + int nByte; /* Size of this structure in bytes */ + int iVersion; /* Version number of this structure (1) */ + + /* Compression library functions */ + void *pCtx; + int (*xBound)(void *, int nSrc); + int (*xCompress)(void *, char *, int *, const char *, int); + int (*xUncompress)(void *, char *, int *, const char *, int); + + /* New fields may be added in future releases, in which case the + ** iVersion value will increase. */ +}; /* ** Values that may be passed as the second argument to xMutexStatic. */ #define LSM_MUTEX_GLOBAL 1 #define LSM_MUTEX_HEAP 2 /* -** CAPI: LSM Error Codes +** Return a pointer to the default LSM run-time environment +*/ +lsm_env *lsm_default_env(void); + +/* +** Error codes. */ #define LSM_OK 0 #define LSM_ERROR 1 #define LSM_BUSY 5 #define LSM_NOMEM 7 @@ -105,393 +126,293 @@ #define LSM_FULL 13 #define LSM_CANTOPEN 14 #define LSM_PROTOCOL 15 #define LSM_MISUSE 21 -#define LSM_MISMATCH 50 - /* -** CAPI: Creating and Destroying Database Connection Handles -** -** Open and close a database connection handle. +** Open and close a connection to a named database. */ int lsm_new(lsm_env*, lsm_db **ppDb); +int lsm_open(lsm_db *pDb, const char *zFilename); int lsm_close(lsm_db *pDb); -/* -** CAPI: Connecting to a Database -*/ -int lsm_open(lsm_db *pDb, const char *zFilename); - -/* -** CAPI: Obtaining pointers to database environments -** +/* ** Return a pointer to the environment used by the database connection ** passed as the first argument. Assuming the argument is valid, this ** function always returns a valid environment pointer - it cannot fail. */ lsm_env *lsm_get_env(lsm_db *pDb); /* -** The lsm_default_env() function returns a pointer to the default LSM -** environment for the current platform. -*/ -lsm_env *lsm_default_env(void); - - -/* -** CAPI: Configuring a database connection. -** -** The lsm_config() function is used to configure a database connection. +** Configure a database connection. */ int lsm_config(lsm_db *, int, ...); /* ** The following values may be passed as the second argument to lsm_config(). ** -** LSM_CONFIG_AUTOFLUSH: -** A read/write integer parameter. -** -** This value determines the amount of data allowed to accumulate in a -** live in-memory tree before it is marked as old. After committing a -** transaction, a connection checks if the size of the live in-memory tree, -** including data structure overhead, is greater than the value of this -** option in KB. If it is, and there is not already an old in-memory tree, -** the live in-memory tree is marked as old. -** -** The maximum allowable value is 1048576 (1GB). There is no minimum -** value. If this parameter is set to zero, then an attempt is made to -** mark the live in-memory tree as old after each transaction is committed. -** -** The default value is 1024 (1MB). -** -** LSM_CONFIG_PAGE_SIZE: -** A read/write integer parameter. This parameter may only be set before -** lsm_open() has been called. -** -** LSM_CONFIG_BLOCK_SIZE: -** A read/write integer parameter. -** -** This parameter may only be set before lsm_open() has been called. It -** must be set to a power of two between 64 and 65536, inclusive (block -** sizes between 64KB and 64MB). -** -** If the connection creates a new database, the block size of the new -** database is set to the value of this option in KB. After lsm_open() -** has been called, querying this parameter returns the actual block -** size of the opened database. -** -** The default value is 1024 (1MB blocks). -** -** LSM_CONFIG_SAFETY: -** A read/write integer parameter. Valid values are 0, 1 (the default) -** and 2. This parameter determines how robust the database is in the -** face of a system crash (e.g. a power failure or operating system -** crash). As follows: -** -** 0 (off): No robustness. A system crash may corrupt the database. -** -** 1 (normal): Some robustness. A system crash may not corrupt the -** database file, but recently committed transactions may -** be lost following recovery. -** -** 2 (full): Full robustness. A system crash may not corrupt the -** database file. Following recovery the database file -** contains all successfully committed transactions. -** -** LSM_CONFIG_AUTOWORK: -** A read/write integer parameter. -** -** LSM_CONFIG_AUTOCHECKPOINT: -** A read/write integer parameter. -** -** If this option is set to non-zero value N, then a checkpoint is -** automatically attempted after each N KB of data have been written to -** the database file. -** -** The amount of uncheckpointed data already written to the database file -** is a global parameter. After performing database work (writing to the -** database file), the process checks if the total amount of uncheckpointed -** data exceeds the value of this paramter. If so, a checkpoint is performed. -** This means that this option may cause the connection to perform a -** checkpoint even if the current connection has itself written very little -** data into the database file. -** -** The default value is 2048 (checkpoint every 2MB). -** -** LSM_CONFIG_MMAP: -** A read/write integer parameter. True to use mmap() to access the -** database file. False otherwise. -** -** LSM_CONFIG_USE_LOG: -** A read/write boolean parameter. True (the default) to use the log -** file normally. False otherwise. -** -** LSM_CONFIG_AUTOMERGE: -** A read/write integer parameter. The minimum number of segments to -** merge together at a time. Default value 4. -** -** LSM_CONFIG_MAX_FREELIST: -** A read/write integer parameter. The maximum number of free-list -** entries that are stored in a database checkpoint (the others are -** stored elsewhere in the database). -** -** There is no reason for an application to configure or query this -** parameter. It is only present because configuring a small value -** makes certain parts of the lsm code easier to test. -** -** LSM_CONFIG_MULTIPLE_PROCESSES: -** A read/write boolean parameter. This parameter may only be set before -** lsm_open() has been called. If true, the library uses shared-memory -** and posix advisory locks to co-ordinate access by clients from within -** multiple processes. Otherwise, if false, all database clients must be -** located in the same process. The default value is true. -** -** LSM_CONFIG_SET_COMPRESSION: -** Set the compression methods used to compress and decompress database -** content. The argument to this option should be a pointer to a structure -** of type lsm_compress. The lsm_config() method takes a copy of the -** structures contents. -** -** This option may only be used before lsm_open() is called. Invoking it -** after lsm_open() has been called results in an LSM_MISUSE error. -** -** LSM_CONFIG_GET_COMPRESSION: -** Query the compression methods used to compress and decompress database -** content. -** -** LSM_CONFIG_SET_COMPRESSION_FACTORY: -** Configure a factory method to be invoked in case of an LSM_MISMATCH -** error. -*/ -#define LSM_CONFIG_AUTOFLUSH 1 -#define LSM_CONFIG_PAGE_SIZE 2 -#define LSM_CONFIG_SAFETY 3 -#define LSM_CONFIG_BLOCK_SIZE 4 -#define LSM_CONFIG_AUTOWORK 5 -#define LSM_CONFIG_MMAP 7 -#define LSM_CONFIG_USE_LOG 8 -#define LSM_CONFIG_AUTOMERGE 9 -#define LSM_CONFIG_MAX_FREELIST 10 -#define LSM_CONFIG_MULTIPLE_PROCESSES 11 -#define LSM_CONFIG_AUTOCHECKPOINT 12 -#define LSM_CONFIG_SET_COMPRESSION 13 -#define LSM_CONFIG_GET_COMPRESSION 14 -#define LSM_CONFIG_SET_COMPRESSION_FACTORY 15 +** LSM_CONFIG_WRITE_BUFFER +** A read/write integer parameter. This value determines the maximum amount +** of space (in bytes) used to accumulate writes in main-memory before +** they are flushed to a level 0 segment. +** +** LSM_CONFIG_PAGE_SIZE +** A read/write integer parameter. This parameter may only be set before +** lsm_open() has been called. +** +** LSM_CONFIG_BLOCK_SIZE +** A read/write integer parameter. This parameter may only be set before +** lsm_open() has been called. +** +** LSM_CONFIG_LOG_SIZE +** A read/write integer parameter. +** +** LSM_CONFIG_SAFETY +** A read/write integer parameter. Valid values are 0, 1 (the default) +** and 2. This parameter determines how robust the database is in the +** face of a system crash (e.g. a power failure or operating system +** crash). As follows: +** +** 0 (off): No robustness. A system crash may corrupt the database. +** +** 1 (normal): Some robustness. A system crash may not corrupt the +** database file, but recently committed transactions may +** be lost following recovery. +** +** 2 (full): Full robustness. A system crash may not corrupt the +** database file. Following recovery the database file +** contains all successfully committed transactions. +** +** LSM_CONFIG_AUTOWORK +** A read/write integer parameter. +** +** LSM_CONFIG_AUTOCHECKPOINT +** A read/write integer parameter. +** +** LSM_CONFIG_MMAP +** A read/write integer parameter. True to use mmap() to access the +** database file. False otherwise. +** +** LSM_CONFIG_USE_LOG +** A read/write boolean parameter. True (the default) to use the log +** file normally. False otherwise. +** +** LSM_CONFIG_NMERGE +** A read/write integer parameter. The minimum number of segments to +** merge together at a time. Default value 4. +** +** LSM_CONFIG_MAX_FREELIST +** A read/write integer parameter. The maximum number of free-list +** entries that are stored in a database checkpoint (the others are +** stored elsewhere in the database). +** +** There is no reason for an application to configure or query this +** parameter. It is only present because configuring a small value +** makes certain parts of the lsm code easier to test. +** +** LSM_CONFIG_MULTIPLE_PROCESSES +** A read/write boolean parameter. This parameter may only be set before +** lsm_open() has been called. If true, the library uses shared-memory +** and posix advisory locks to co-ordinate access by clients from within +** multiple processes. Otherwise, if false, all database clients must be +** located in the same process. The default value is true. +** +** LSM_CONFIG_SET_COMPRESSION +** Set the compression methods used to compress and decompress database +** content. The argument to this option should be a pointer to a structure +** of type lsm_compress. The lsm_config() method takes a copy of the +** structures contents. +** +** This option may only be used before lsm_open() is called. Invoking it +** after lsm_open() has been called results in an LSM_MISUSE error. +** +** LSM_CONFIG_GET_COMPRESSION +** Query the compression methods used to compress and decompress database +** content. +*/ +#define LSM_CONFIG_WRITE_BUFFER 1 +#define LSM_CONFIG_PAGE_SIZE 2 +#define LSM_CONFIG_SAFETY 3 +#define LSM_CONFIG_BLOCK_SIZE 4 +#define LSM_CONFIG_AUTOWORK 5 +#define LSM_CONFIG_LOG_SIZE 6 +#define LSM_CONFIG_MMAP 7 +#define LSM_CONFIG_USE_LOG 8 +#define LSM_CONFIG_NMERGE 9 +#define LSM_CONFIG_MAX_FREELIST 10 +#define LSM_CONFIG_MULTIPLE_PROCESSES 11 +#define LSM_CONFIG_AUTOCHECKPOINT 12 + +#define LSM_CONFIG_SET_COMPRESSION 13 +#define LSM_CONFIG_GET_COMPRESSION 14 #define LSM_SAFETY_OFF 0 #define LSM_SAFETY_NORMAL 1 #define LSM_SAFETY_FULL 2 -/* -** CAPI: Compression and/or Encryption Hooks -*/ -struct lsm_compress { - void *pCtx; - unsigned int iId; - int (*xBound)(void *, int nSrc); - int (*xCompress)(void *, char *, int *, const char *, int); - int (*xUncompress)(void *, char *, int *, const char *, int); - void (*xFree)(void *pCtx); -}; - -struct lsm_compress_factory { - void *pCtx; - int (*xFactory)(void *, lsm_db *, unsigned int); - void (*xFree)(void *pCtx); -}; - -#define LSM_COMPRESSION_EMPTY 0 -#define LSM_COMPRESSION_NONE 1 - -/* -** CAPI: Allocating and Freeing Memory -** + +/* ** Invoke the memory allocation functions that belong to environment ** pEnv. Or the system defaults if no memory allocation functions have ** been registered. */ void *lsm_malloc(lsm_env*, size_t); void *lsm_realloc(lsm_env*, void *, size_t); void lsm_free(lsm_env*, void *); /* -** CAPI: Querying a Connection For Operational Data -** +** Configure a callback to which debugging and other messages should +** be directed. Only useful for debugging lsm. +*/ +void lsm_config_log(lsm_db *, void (*)(void *, int, const char *), void *); + +/* +** Configure a callback that is invoked if the database connection ever +** writes to the database file. +*/ +void lsm_config_work_hook(lsm_db *, void (*)(lsm_db *, void *), void *); + +/* ** Query a database connection for operational statistics or data. */ int lsm_info(lsm_db *, int, ...); /* ** The following values may be passed as the second argument to lsm_info(). ** -** LSM_INFO_NWRITE: -** The third parameter should be of type (int *). The location pointed -** to by the third parameter is set to the number of 4KB pages written to -** the database file during the lifetime of this connection. -** -** LSM_INFO_NREAD: -** The third parameter should be of type (int *). The location pointed -** to by the third parameter is set to the number of 4KB pages read from -** the database file during the lifetime of this connection. -** -** LSM_INFO_DB_STRUCTURE: -** The third argument should be of type (char **). The location pointed -** to is populated with a pointer to a nul-terminated string containing -** the string representation of a Tcl data-structure reflecting the -** current structure of the database file. Specifically, the current state -** of the worker snapshot. The returned string should be eventually freed -** by the caller using lsm_free(). -** -** The returned list contains one element for each level in the database, -** in order from most to least recent. Each element contains a -** single element for each segment comprising the corresponding level, -** starting with the lhs segment, then each of the rhs segments (if any) -** in order from most to least recent. -** -** Each segment element is itself a list of 4 integer values, as follows: -** -**
  1. First page of segment -**
  2. Last page of segment -**
  3. Root page of segment (if applicable) -**
  4. Total number of pages in segment -**
-** -** LSM_INFO_ARRAY_STRUCTURE: -** There should be two arguments passed following this option (i.e. a -** total of four arguments passed to lsm_info()). The first argument -** should be the page number of the first page in a database array -** (perhaps obtained from an earlier INFO_DB_STRUCTURE call). The second -** trailing argument should be of type (char **). The location pointed -** to is populated with a pointer to a nul-terminated string that must -** be eventually freed using lsm_free() by the caller. -** -** The output string contains the text representation of a Tcl list of -** integers. Each pair of integers represent a range of pages used by -** the identified array. For example, if the array occupies database -** pages 993 to 1024, then pages 2048 to 2777, then the returned string -** will be "993 1024 2048 2777". -** -** If the specified integer argument does not correspond to the first -** page of any database array, LSM_ERROR is returned and the output -** pointer is set to a NULL value. -** -** LSM_INFO_LOG_STRUCTURE: -** The third argument should be of type (char **). The location pointed -** to is populated with a pointer to a nul-terminated string containing -** the string representation of a Tcl data-structure. The returned -** string should be eventually freed by the caller using lsm_free(). -** -** The Tcl structure returned is a list of six integers that describe -** the current structure of the log file. -** -** LSM_INFO_ARRAY_PAGES: -** -** LSM_INFO_PAGE_ASCII_DUMP: -** As with LSM_INFO_ARRAY_STRUCTURE, there should be two arguments passed -** with calls that specify this option - an integer page number and a -** (char **) used to return a nul-terminated string that must be later -** freed using lsm_free(). In this case the output string is populated -** with a human-readable description of the page content. -** -** If the page cannot be decoded, it is not an error. In this case the -** human-readable output message will report the systems failure to -** interpret the page data. -** -** LSM_INFO_PAGE_HEX_DUMP: -** This argument is similar to PAGE_ASCII_DUMP, except that keys and -** values are represented using hexadecimal notation instead of ascii. -** -** LSM_INFO_FREELIST: -** The third argument should be of type (char **). The location pointed -** to is populated with a pointer to a nul-terminated string containing -** the string representation of a Tcl data-structure. The returned -** string should be eventually freed by the caller using lsm_free(). -** -** The Tcl structure returned is a list containing one element for each -** free block in the database. The element itself consists of two -** integers - the block number and the id of the snapshot that freed it. -** -** LSM_INFO_CHECKPOINT_SIZE: -** The third argument should be of type (int *). The location pointed to -** by this argument is populated with the number of KB written to the -** database file since the most recent checkpoint. -** -** LSM_INFO_TREE_SIZE: -** If this value is passed as the second argument to an lsm_info() call, it -** should be followed by two arguments of type (int *) (for a total of four -** arguments). -** -** At any time, there are either one or two tree structures held in shared -** memory that new database clients will access (there may also be additional -** tree structures being used by older clients - this API does not provide -** information on them). One tree structure - the current tree - is used to -** accumulate new data written to the database. The other tree structure - -** the old tree - is a read-only tree holding older data and may be flushed -** to disk at any time. -** -** Assuming no error occurs, the location pointed to by the first of the two -** (int *) arguments is set to the size of the old in-memory tree in KB. -** The second is set to the size of the current, or live in-memory tree. -** -** LSM_INFO_COMPRESSION_ID: -** This value should be followed by a single argument of type -** (unsigned int *). If successful, the location pointed to is populated -** with the database compression id before returning. +** LSM_INFO_NWRITE +** The third parameter should be of type (int *). The location pointed +** to by the third parameter is set to the number of 4KB pages written to +** the database file during the lifetime of this connection. +** +** LSM_INFO_NREAD +** The third parameter should be of type (int *). The location pointed +** to by the third parameter is set to the number of 4KB pages read from +** the database file during the lifetime of this connection. +** +** LSM_INFO_DB_STRUCTURE +** The third argument should be of type (char **). The location pointed +** to is populated with a pointer to a nul-terminated string containing +** the string representation of a Tcl data-structure reflecting the +** current structure of the database file. Specifically, the current state +** of the worker snapshot. The returned string should be eventually freed +** by the caller using lsm_free(). +** +** The returned list contains one element for each level in the database, +** in order from most to least recent. Each element contains a +** single element for each segment comprising the corresponding level, +** starting with the lhs segment, then each of the rhs segments (if any) +** in order from most to least recent. +** +** Each segment element is itself a list of 6 integer values, as follows: +** +** 1. First page of segment +** 2. Last page of segment +** 3. Root page of segment (if applicable). +** 4. Total number of pages in segment. +** +** LSM_INFO_ARRAY_STRUCTURE +** There should be two arguments passed following this option (i.e. a +** total of four arguments passed to lsm_info()). The first argument +** should be the page number of the first page in a database array +** (perhaps obtained from an earlier INFO_DB_STRUCTURE call). The second +** trailing argument should be of type (char **). The location pointed +** to is populated with a pointer to a nul-terminated string that must +** be eventually freed using lsm_free() by the caller. +** +** The output string contains the text representation of a Tcl list of +** integers. Each pair of integers represent a range of pages used by +** the identified array. For example, if the array occupies database +** pages 993 to 1024, then pages 2048 to 2777, then the returned string +** will be "993 1024 2048 2777". +** +** If the specified integer argument does not correspond to the first +** page of any database array, LSM_ERROR is returned and the output +** pointer is set to a NULL value. +** +** LSM_INFO_PAGE_ASCII_DUMP +** As with LSM_INFO_ARRAY_STRUCTURE, there should be two arguments passed +** with calls that specify this option - an integer page number and a +** (char **) used to return a nul-terminated string that must be later +** freed using lsm_free(). In this case the output string is populated +** with a human-readable description of the page content. +** +** If the page cannot be decoded, it is not an error. In this case the +** human-readable output message will report the systems failure to +** interpret the page data. +** +** LSM_INFO_PAGE_HEX_DUMP +** This argument is similar to PAGE_ASCII_DUMP, except that keys and +** values are represented using hexadecimal notation instead of ascii. +** +** LSM_INFO_LOG_STRUCTURE +** The third argument should be of type (char **). The location pointed +** to is populated with a pointer to a nul-terminated string containing +** the string representation of a Tcl data-structure. The returned +** string should be eventually freed by the caller using lsm_free(). +** +** The Tcl structure returned is a list of six integers that describe +** the current structure of the log file. +** +** LSM_INFO_FREELIST +** The third argument should be of type (char **). The location pointed +** to is populated with a pointer to a nul-terminated string containing +** the string representation of a Tcl data-structure. The returned +** string should be eventually freed by the caller using lsm_free(). +** +** The Tcl structure returned is a list containing one element for each +** free block in the database. The element itself consists of two +** integers - the block number and the id of the snapshot that freed it. */ #define LSM_INFO_NWRITE 1 #define LSM_INFO_NREAD 2 #define LSM_INFO_DB_STRUCTURE 3 #define LSM_INFO_LOG_STRUCTURE 4 #define LSM_INFO_ARRAY_STRUCTURE 5 #define LSM_INFO_PAGE_ASCII_DUMP 6 #define LSM_INFO_PAGE_HEX_DUMP 7 #define LSM_INFO_FREELIST 8 -#define LSM_INFO_ARRAY_PAGES 9 -#define LSM_INFO_CHECKPOINT_SIZE 10 -#define LSM_INFO_TREE_SIZE 11 -#define LSM_INFO_FREELIST_SIZE 12 -#define LSM_INFO_COMPRESSION_ID 13 /* -** CAPI: Opening and Closing Write Transactions -** -** These functions are used to open and close transactions and nested -** sub-transactions. -** -** The lsm_begin() function is used to open transactions and sub-transactions. -** A successful call to lsm_begin() ensures that there are at least iLevel -** nested transactions open. To open a top-level transaction, pass iLevel=1. -** To open a sub-transaction within the top-level transaction, iLevel=2. -** Passing iLevel=0 is a no-op. -** -** lsm_commit() is used to commit transactions and sub-transactions. A -** successful call to lsm_commit() ensures that there are at most iLevel -** nested transactions open. To commit a top-level transaction, pass iLevel=0. -** To commit all sub-transactions inside the main transaction, pass iLevel=1. -** -** Function lsm_rollback() is used to roll back transactions and -** sub-transactions. A successful call to lsm_rollback() restores the database -** to the state it was in when the iLevel'th nested sub-transaction (if any) -** was first opened. And then closes transactions to ensure that there are -** at most iLevel nested transactions open. Passing iLevel=0 rolls back and -** closes the top-level transaction. iLevel=1 also rolls back the top-level -** transaction, but leaves it open. iLevel=2 rolls back the sub-transaction -** nested directly inside the top-level transaction (and leaves it open). +** Open and close transactions and nested transactions. +** +** lsm_begin(): +** Used to open transactions and sub-transactions. A successful call to +** lsm_begin() ensures that there are at least iLevel nested transactions +** open. To open a top-level transaction, pass iLevel==1. To open a +** sub-transaction within the top-level transaction, iLevel==2. Passing +** iLevel==0 is a no-op. +** +** lsm_commit(): +** Used to commit transactions and sub-transactions. A successful call +** to lsm_commit() ensures that there are at most iLevel nested +** transactions open. To commit a top-level transaction, pass iLevel==0. +** To commit all sub-transactions inside the main transaction, pass +** iLevel==1. +** +** lsm_rollback(): +** Used to roll back transactions and sub-transactions. A successful call +** to lsm_rollback() restores the database to the state it was in when +** the iLevel'th nested sub-transaction (if any) was first opened. And then +** closes transactions to ensure that there are at most iLevel nested +** transactions open. +** +** Passing iLevel==0 rolls back and closes the top-level transaction. +** iLevel==1 also rolls back the top-level transaction, but leaves it +** open. iLevel==2 rolls back the sub-transaction nested directly inside +** the top-level transaction (and leaves it open). */ int lsm_begin(lsm_db *pDb, int iLevel); int lsm_commit(lsm_db *pDb, int iLevel); int lsm_rollback(lsm_db *pDb, int iLevel); /* -** CAPI: Writing to a Database -** ** Write a new value into the database. If a value with a duplicate key ** already exists it is replaced. */ -int lsm_insert(lsm_db*, const void *pKey, int nKey, const void *pVal, int nVal); +int lsm_write(lsm_db *, const void *pKey, int nKey, const void *pVal, int nVal); /* ** Delete a value from the database. No error is returned if the specified ** key value does not exist in the database. */ @@ -507,83 +428,133 @@ int lsm_delete_range(lsm_db *, const void *pKey1, int nKey1, const void *pKey2, int nKey2 ); /* -** CAPI: Explicit Database Work and Checkpointing +** The lsm_tree_size() function reports on the current state of the +** in-memory tree data structure. +** +** At any time, there are either one or two tree structures held in shared +** memory that new database clients will access (there may also be additional +** tree structures being used by older clients - this API does not provide +** information on them). One tree structure - the current tree - is used to +** accumulate new data written to the database. The other tree structure - the +** old tree - is a read-only tree holding older data and may be flushed to disk +** at any time. +** +** If successful, this function sets *pnNew to the number of bytes of shared +** memory space used by the current tree. *pbOld is set to true if the old +** tree exists, or false if it does not. +** +** If no error occurs, LSM_OK is returned. Otherwise an LSM error code. +** +** RACE CONDITION: +** Describe the race condition this function is subject to. +*/ +int lsm_tree_size(lsm_db *, int *pbOld, int *pnNew); + +/* +** This function is used to query the amount of data that has been written +** to the database file but not checkpointed (synced). If successful, *pnByte +** is set to the number of bytes before returning. +** +** LSM_OK is returned if successful. Or if an error occurs, an LSM error +** code is returned. ** +** RACE CONDITION: +** Describe the race condition this function is subject to. Or remove +** it somehow. +*/ +int lsm_ckpt_size(lsm_db *, int *pnByte); + +/* ** This function is called by a thread to work on the database structure. +** The actual operations performed by this function depend on the value +** passed as the "flags" parameter: +** +** LSM_WORK_FLUSH: +** Attempt to flush the contents of the in-memory tree to disk. +** +** LSM_WORK_OPTIMIZE: +** If nMerge suitable arrays cannot be found, where nMerge is as +** configured by LSM_CONFIG_NMERGE, merge together any arrays that +** can be found. This is usually used to optimize the database by +** merging the whole thing into one big array. */ -int lsm_work(lsm_db *pDb, int nMerge, int nKB, int *pnWrite); +int lsm_work(lsm_db *pDb, int flags, int nPage, int *pnWrite); -int lsm_flush(lsm_db *pDb); +#define LSM_WORK_FLUSH 0x00000001 +#define LSM_WORK_OPTIMIZE 0x00000002 /* ** Attempt to checkpoint the current database snapshot. Return an LSM ** error code if an error occurs or LSM_OK otherwise. ** ** If the current snapshot has already been checkpointed, calling this -** function is a no-op. In this case if pnKB is not NULL, *pnKB is +** function is a no-op. In this case if pnByte is not NULL, *pnByte is ** set to 0. Or, if the current snapshot is successfully checkpointed -** by this function and pbKB is not NULL, *pnKB is set to the number +** by this function and pbCkpt is not NULL, *pnByte is set to the number ** of bytes written to the database file since the previous checkpoint -** (the same measure as returned by the LSM_INFO_CHECKPOINT_SIZE query). +** (the same measure as returned by lsm_ckpt_size()). */ -int lsm_checkpoint(lsm_db *pDb, int *pnKB); +int lsm_checkpoint(lsm_db *pDb, int *pnByte); /* -** CAPI: Opening and Closing Database Cursors -** ** Open and close a database cursor. */ int lsm_csr_open(lsm_db *pDb, lsm_cursor **ppCsr); int lsm_csr_close(lsm_cursor *pCsr); /* -** CAPI: Positioning Database Cursors -** ** If the fourth parameter is LSM_SEEK_EQ, LSM_SEEK_GE or LSM_SEEK_LE, ** this function searches the database for an entry with key (pKey/nKey). ** If an error occurs, an LSM error code is returned. Otherwise, LSM_OK. ** ** If no error occurs and the requested key is present in the database, the ** cursor is left pointing to the entry with the specified key. Or, if the ** specified key is not present in the database the state of the cursor ** depends on the value passed as the final parameter, as follows: ** -** LSM_SEEK_EQ: -** The cursor is left at EOF (invalidated). A call to lsm_csr_valid() -** returns non-zero. -** -** LSM_SEEK_LE: -** The cursor is left pointing to the largest key in the database that -** is smaller than (pKey/nKey). If the database contains no keys smaller -** than (pKey/nKey), the cursor is left at EOF. -** -** LSM_SEEK_GE: -** The cursor is left pointing to the smallest key in the database that -** is larger than (pKey/nKey). If the database contains no keys larger -** than (pKey/nKey), the cursor is left at EOF. +** LSM_SEEK_EQ: +** The cursor is left at EOF (invalidated). A call to lsm_csr_valid() +** returns non-zero. +** +** LSM_SEEK_LE: +** The cursor is left pointing to the largest key in the database that +** is smaller than (pKey/nKey). If the database contains no keys smaller +** than (pKey/nKey), the cursor is left at EOF. +** +** LSM_SEEK_GE: +** The cursor is left pointing to the smallest key in the database that +** is larger than (pKey/nKey). If the database contains no keys larger +** than (pKey/nKey), the cursor is left at EOF. ** ** If the fourth parameter is LSM_SEEK_LEFAST, this function searches the ** database in a similar manner to LSM_SEEK_LE, with two differences: ** -**
  1. Even if a key can be found (the cursor is not left at EOF), the -** lsm_csr_value() function may not be used (attempts to do so return -** LSM_MISUSE). +** 1) Even if a key can be found (the cursor is not left at EOF), the +** lsm_csr_value() function may not be used (attempts to do so return +** LSM_MISUSE). ** -**
  2. The key that the cursor is left pointing to may be one that has -** been recently deleted from the database. In this case it is -** guaranteed that the returned key is larger than any key currently -** in the database that is less than or equal to (pKey/nKey). -**
+** 2) The key that the cursor is left pointing to may be one that has +** been recently deleted from the database. In this case it is guaranteed +** that the returned key is larger than any key currently in the database +** that is less than or equal to (pKey/nKey). ** ** LSM_SEEK_LEFAST requests are intended to be used to allocate database ** keys. */ int lsm_csr_seek(lsm_cursor *pCsr, const void *pKey, int nKey, int eSeek); +/* +** Values that may be passed as the fourth argument to lsm_csr_seek(). +*/ +#define LSM_SEEK_LEFAST -2 +#define LSM_SEEK_LE -1 +#define LSM_SEEK_EQ 0 +#define LSM_SEEK_GE 1 + int lsm_csr_first(lsm_cursor *pCsr); int lsm_csr_last(lsm_cursor *pCsr); /* ** Advance the specified cursor to the next or previous key in the database. @@ -592,71 +563,33 @@ ** Functions lsm_csr_seek(), lsm_csr_first() and lsm_csr_last() are "seek" ** functions. Whether or not lsm_csr_next and lsm_csr_prev may be called ** successfully also depends on the most recent seek function called on ** the cursor. Specifically: ** -**
    -**
  • At least one seek function must have been called on the cursor. -**
  • To call lsm_csr_next(), the most recent call to a seek function must -** have been either lsm_csr_first() or a call to lsm_csr_seek() specifying -** LSM_SEEK_GE. -**
  • To call lsm_csr_prev(), the most recent call to a seek function must -** have been either lsm_csr_first() or a call to lsm_csr_seek() specifying -** LSM_SEEK_GE. -**
+** * At least one seek function must have been called on the cursor. +** +** * To call lsm_csr_next(), the most recent call to a seek function must +** have been either lsm_csr_first() or a call to lsm_csr_seek() specifying +** LSM_SEEK_GE. +** +** * To call lsm_csr_prev(), the most recent call to a seek function must +** have been either lsm_csr_first() or a call to lsm_csr_seek() specifying +** LSM_SEEK_GE. ** ** Otherwise, if the above conditions are not met when lsm_csr_next or ** lsm_csr_prev is called, LSM_MISUSE is returned and the cursor position ** remains unchanged. */ int lsm_csr_next(lsm_cursor *pCsr); int lsm_csr_prev(lsm_cursor *pCsr); -/* -** Values that may be passed as the fourth argument to lsm_csr_seek(). -*/ -#define LSM_SEEK_LEFAST -2 -#define LSM_SEEK_LE -1 -#define LSM_SEEK_EQ 0 -#define LSM_SEEK_GE 1 - -/* -** CAPI: Extracting Data From Database Cursors -** +/* ** Retrieve data from a database cursor. */ int lsm_csr_valid(lsm_cursor *pCsr); int lsm_csr_key(lsm_cursor *pCsr, const void **ppKey, int *pnKey); int lsm_csr_value(lsm_cursor *pCsr, const void **ppVal, int *pnVal); -/* -** If no error occurs, this function compares the database key passed via -** the pKey/nKey arguments with the key that the cursor passed as the first -** argument currently points to. If the cursors key is less than, equal to -** or greater than pKey/nKey, *piRes is set to less than, equal to or greater -** than zero before returning. LSM_OK is returned in this case. -** -** Or, if an error occurs, an LSM error code is returned and the final -** value of *piRes is undefined. If the cursor does not point to a valid -** key when this function is called, LSM_MISUSE is returned. -*/ -int lsm_csr_cmp(lsm_cursor *pCsr, const void *pKey, int nKey, int *piRes); - -/* -** CAPI: Change these!! -** -** Configure a callback to which debugging and other messages should -** be directed. Only useful for debugging lsm. -*/ -void lsm_config_log(lsm_db *, void (*)(void *, int, const char *), void *); - -/* -** Configure a callback that is invoked if the database connection ever -** writes to the database file. -*/ -void lsm_config_work_hook(lsm_db *, void (*)(lsm_db *, void *), void *); - -/* ENDOFAPI */ #ifdef __cplusplus } /* End of the 'extern "C"' block */ #endif #endif /* ifndef _LSM_H */ Index: src/lsmInt.h ================================================================== --- src/lsmInt.h +++ src/lsmInt.h @@ -40,21 +40,16 @@ /* ** Default values for various data structure parameters. These may be ** overridden by calls to lsm_config(). */ -#define LSM_DFLT_PAGE_SIZE (4 * 1024) -#define LSM_DFLT_BLOCK_SIZE (1 * 1024 * 1024) -#define LSM_DFLT_AUTOFLUSH (1 * 1024 * 1024) -#define LSM_DFLT_AUTOCHECKPOINT (i64)(2 * 1024 * 1024) -#define LSM_DFLT_AUTOWORK 1 -#define LSM_DFLT_LOG_SIZE (128*1024) -#define LSM_DFLT_AUTOMERGE 4 -#define LSM_DFLT_SAFETY LSM_SAFETY_NORMAL -#define LSM_DFLT_MMAP LSM_IS_64_BIT -#define LSM_DFLT_MULTIPLE_PROCESSES 1 -#define LSM_DFLT_USE_LOG 1 +#define LSM_DFLT_PAGE_SIZE (4 * 1024) +#define LSM_DFLT_BLOCK_SIZE (2 * 1024 * 1024) +#define LSM_DFLT_WRITE_BUFFER (2 * 1024 * 1024) +#define LSM_DFLT_AUTOCHECKPOINT (4 * 1024 * 1024) +#define LSM_DFLT_LOG_SIZE (128*1024) +#define LSM_DFLT_NMERGE 4 /* Initial values for log file checksums. These are only used if the ** database file does not contain a valid checkpoint. */ #define LSM_CKSUM0_INIT 42 #define LSM_CKSUM1_INIT 42 @@ -65,15 +60,21 @@ ** spaces. The following macro is used to test for this. */ #define LSM_IS_64_BIT (sizeof(void*)==8) #define LSM_AUTOWORK_QUANT 32 +/* Minimum number of free-list entries to store in the checkpoint, assuming +** the free-list contains this many entries. i.e. if overflow is required, +** the first LSM_CKPT_MIN_FREELIST entries are stored in the checkpoint and +** the remainder in an LSM system entry. */ +#define LSM_CKPT_MIN_FREELIST 6 +#define LSM_CKPT_MAX_REFREE 2 +#define LSM_CKPT_MIN_NONLSM (LSM_CKPT_MIN_FREELIST - LSM_CKPT_MAX_REFREE) + typedef struct Database Database; typedef struct DbLog DbLog; typedef struct FileSystem FileSystem; -typedef struct Freelist Freelist; -typedef struct FreelistEntry FreelistEntry; typedef struct Level Level; typedef struct LogMark LogMark; typedef struct LogRegion LogRegion; typedef struct LogWriter LogWriter; typedef struct LsmString LsmString; @@ -81,11 +82,10 @@ typedef struct Merge Merge; typedef struct MergeInput MergeInput; typedef struct MetaPage MetaPage; typedef struct MultiCursor MultiCursor; typedef struct Page Page; -typedef struct Redirect Redirect; typedef struct Segment Segment; typedef struct SegmentMerger SegmentMerger; typedef struct ShmChunk ShmChunk; typedef struct ShmHeader ShmHeader; typedef struct ShmReader ShmReader; @@ -95,17 +95,15 @@ typedef struct TreeCursor TreeCursor; typedef struct TreeHeader TreeHeader; typedef struct TreeMark TreeMark; typedef struct TreeRoot TreeRoot; -#ifndef _SQLITEINT_H_ typedef unsigned char u8; typedef unsigned short int u16; typedef unsigned int u32; typedef lsm_i64 i64; typedef unsigned long long int u64; -#endif /* A page number is a 64-bit integer. */ typedef i64 Pgno; #ifdef LSM_DEBUG @@ -145,12 +143,10 @@ ** a checkpoint (the remainder are stored as a system record in the LSM). ** See also LSM_CONFIG_MAX_FREELIST. */ #define LSM_MAX_FREELIST_ENTRIES 24 -#define LSM_MAX_BLOCK_REDIRECTS 16 - #define LSM_ATTEMPTS_BEFORE_PROTOCOL 10000 /* ** Each entry stored in the LSM (or in-memory tree structure) has an @@ -194,18 +190,10 @@ int nAlloc; int nArray; u32 *aArray; }; -struct Redirect { - int n; /* Number of redirects */ - struct RedirectEntry { - int iFrom; - int iTo; - } *a; -}; - /* ** An instance of this structure represents a point in the history of the ** tree structure to roll back to. Refer to comments in lsm_tree.c for ** details. */ @@ -253,11 +241,10 @@ }; struct TreeRoot { u32 iRoot; u32 nHeight; - u32 nByte; /* Total size of this tree in bytes */ u32 iTransId; }; /* ** Tree header structure. @@ -267,10 +254,11 @@ u32 iNextShmid; /* Shm-id of next chunk allocated */ u32 iFirst; /* Chunk number of smallest shm-id */ u32 nChunk; /* Number of chunks in shared-memory file */ TreeRoot root; /* Root and height of current tree */ u32 iWrite; /* Write offset in shm file */ + u32 nByte; /* Size of current tree structure in bytes */ TreeRoot oldroot; /* Root and height of the previous tree */ u32 iOldShmid; /* Last shm-id used by previous tree */ i64 iOldLog; /* Log offset associated with old tree */ u32 oldcksum0; u32 oldcksum1; @@ -304,44 +292,38 @@ int (*xCmp)(void *, int, void *, int); /* Compare function */ /* Values configured by calls to lsm_config */ int eSafety; /* LSM_SAFETY_OFF, NORMAL or FULL */ int bAutowork; /* Configured by LSM_CONFIG_AUTOWORK */ - int nTreeLimit; /* Configured by LSM_CONFIG_AUTOFLUSH */ - int nMerge; /* Configured by LSM_CONFIG_AUTOMERGE */ + int nTreeLimit; /* Configured by LSM_CONFIG_WRITE_BUFFER */ + int nMerge; /* Configured by LSM_CONFIG_NMERGE */ + int nLogSz; /* Configured by LSM_CONFIG_LOG_SIZE */ int bUseLog; /* Configured by LSM_CONFIG_USE_LOG */ int nDfltPgsz; /* Configured by LSM_CONFIG_PAGE_SIZE */ int nDfltBlksz; /* Configured by LSM_CONFIG_BLOCK_SIZE */ int nMaxFreelist; /* Configured by LSM_CONFIG_MAX_FREELIST */ int bMmap; /* Configured by LSM_CONFIG_MMAP */ - i64 nAutockpt; /* Configured by LSM_CONFIG_AUTOCHECKPOINT */ + int nAutockpt; /* Configured by LSM_CONFIG_AUTOCHECKPOINT */ int bMultiProc; /* Configured by L_C_MULTIPLE_PROCESSES */ lsm_compress compress; /* Compression callbacks */ - lsm_compress_factory factory; /* Compression callback factory */ /* Sub-system handles */ FileSystem *pFS; /* On-disk portion of database */ Database *pDatabase; /* Database shared data */ /* Client transaction context */ - Snapshot *pClient; /* Client snapshot */ + Snapshot *pClient; /* Client snapshot (non-NULL in read trans) */ int iReader; /* Read lock held (-1 == unlocked) */ MultiCursor *pCsr; /* List of all open cursors */ LogWriter *pLogWriter; /* Context for writing to the log file */ int nTransOpen; /* Number of opened write transactions */ int nTransAlloc; /* Allocated size of aTrans[] array */ TransMark *aTrans; /* Array of marks for transaction rollback */ IntArray rollback; /* List of tree-nodes to roll back */ - int bDiscardOld; /* True if lsmTreeDiscardOld() was called */ /* Worker context */ Snapshot *pWorker; /* Worker snapshot (or NULL) */ - Freelist *pFreelist; /* See sortedNewToplevel() */ - int bUseFreelist; /* True to use pFreelist */ - int bIncrMerge; /* True if currently doing a merge */ - - int bInFactory; /* True if within factory.xFactory() */ /* Debugging message callback */ void (*xLog)(void *, int, const char *); void *pLogCtx; @@ -362,12 +344,10 @@ struct Segment { Pgno iFirst; /* First page of this run */ Pgno iLastPg; /* Last page of this run */ Pgno iRoot; /* Root page number (if any) */ int nSize; /* Size of this run in pages */ - - Redirect *pRedirect; /* Block redirects (or NULL) */ }; /* ** iSplitTopic/pSplitKey/nSplitKey: ** If nRight>0, this buffer contains a copy of the largest key that has @@ -379,29 +359,14 @@ Segment *aRhs; /* Old segments being merged into this */ int iSplitTopic; /* Split key topic (if nRight>0) */ void *pSplitKey; /* Pointer to split-key (if nRight>0) */ int nSplitKey; /* Number of bytes in split-key */ - u16 iAge; /* Number of times data has been written */ - u16 flags; /* Mask of LEVEL_XXX bits */ + int iAge; /* Number of times data has been written */ Merge *pMerge; /* Merge operation currently underway */ Level *pNext; /* Next level in tree */ }; - -/* -** The Level.flags field is set to a combination of the following bits. -** -** LEVEL_FREELIST_ONLY: -** Set if the level consists entirely of free-list entries. -** -** LEVEL_INCOMPLETE: -** This is set while a new toplevel level is being constructed. It is -** never set for any level other than a new toplevel. -*/ -#define LEVEL_FREELIST_ONLY 0x0001 -#define LEVEL_INCOMPLETE 0x0002 - /* ** A structure describing an ongoing merge. There is an instance of this ** structure for every Level currently undergoing a merge in the worker ** snapshot. @@ -478,42 +443,23 @@ struct ShmChunk { u32 iShmid; u32 iNext; }; -/* -** Maximum number of shared-memory chunks allowed in the *-shm file. Since -** each shared-memory chunk is 32KB in size, this is a theoretical limit only. -*/ -#define LSM_MAX_SHMCHUNKS (1<<30) - /* Return true if shm-sequence "a" is larger than or equal to "b" */ -#define shm_sequence_ge(a, b) (((u32)a-(u32)b) < LSM_MAX_SHMCHUNKS) +#define shm_sequence_ge(a, b) (((u32)a-(u32)b) < (1<<30)) #define LSM_APPLIST_SZ 4 +typedef struct Freelist Freelist; +typedef struct FreelistEntry FreelistEntry; + /* -** An instance of the following structure stores the in-memory part of -** the current free block list. This structure is to the free block list -** as the in-memory tree is to the users database content. The contents -** of the free block list is found by merging the in-memory components -** with those stored in the LSM, just as the contents of the database is -** found by merging the in-memory tree with the user data entries in the -** LSM. -** -** Each FreelistEntry structure in the array represents either an insert -** or delete operation on the free-list. For deletes, the FreelistEntry.iId -** field is set to -1. For inserts, it is set to zero or greater. -** -** The array of FreelistEntry structures is always sorted in order of -** block number (ascending). -** -** When the in-memory free block list is written into the LSM, each insert -** operation is written separately. The entry key is the bitwise inverse -** of the block number as a 32-bit big-endian integer. This is done so that -** the entries in the LSM are sorted in descending order of block id. -** The associated value is the snapshot id, formated as a varint. +** An instance of the following structure stores the current database free +** block list. The free list is a list of blocks that are not currently +** used by the worker snapshot. Assocated with each block in the list is the +** snapshot id of the most recent snapshot that did actually use the block. */ struct Freelist { FreelistEntry *aEntry; /* Free list entries */ int nEntry; /* Number of valid slots in aEntry[] */ int nAlloc; /* Allocated size of aEntry[] */ @@ -528,30 +474,33 @@ ** to read or write a database file on disk. See the description of struct ** Database below for futher details. */ struct Snapshot { Database *pDatabase; /* Database this snapshot belongs to */ - u32 iCmpId; /* Id of compression scheme */ Level *pLevel; /* Pointer to level 0 of snapshot (or NULL) */ i64 iId; /* Snapshot id */ i64 iLogOff; /* Log file offset */ - Redirect redirect; /* Block redirection array */ /* Used by worker snapshots only */ int nBlock; /* Number of blocks in database file */ Pgno aiAppend[LSM_APPLIST_SZ]; /* Append point list */ Freelist freelist; /* Free block list */ + int nFreelistOvfl; /* Number of extra free-list entries in LSM */ u32 nWrite; /* Total number of pages written to disk */ }; #define LSM_INITIAL_SNAPSHOT_ID 11 /* ** Functions from file "lsm_ckpt.c". */ -int lsmCheckpointWrite(lsm_db *, int, u32 *); +int lsmCheckpointWrite(lsm_db *, u32 *); int lsmCheckpointLevels(lsm_db *, int, void **, int *); int lsmCheckpointLoadLevels(lsm_db *pDb, void *pVal, int nVal); + +int lsmCheckpointOverflow(lsm_db *pDb, void **, int *, int *); +int lsmCheckpointOverflowRequired(lsm_db *pDb); +int lsmCheckpointOverflowLoad(lsm_db *pDb, Freelist *); int lsmCheckpointRecover(lsm_db *); int lsmCheckpointDeserialize(lsm_db *, int, u32 *, Snapshot **); int lsmCheckpointLoadWorker(lsm_db *pDb); @@ -559,32 +508,29 @@ int lsmCheckpointLoad(lsm_db *pDb, int *); int lsmCheckpointLoadOk(lsm_db *pDb, int); int lsmCheckpointClientCacheOk(lsm_db *); -u32 lsmCheckpointNBlock(u32 *); i64 lsmCheckpointId(u32 *, int); u32 lsmCheckpointNWrite(u32 *, int); i64 lsmCheckpointLogOffset(u32 *); int lsmCheckpointPgsz(u32 *); int lsmCheckpointBlksz(u32 *); void lsmCheckpointLogoffset(u32 *aCkpt, DbLog *pLog); void lsmCheckpointZeroLogoffset(lsm_db *); -int lsmCheckpointSaveWorker(lsm_db *pDb, int); +int lsmCheckpointSaveWorker(lsm_db *pDb, int, int); int lsmDatabaseFull(lsm_db *pDb); int lsmCheckpointSynced(lsm_db *pDb, i64 *piId, i64 *piLog, u32 *pnWrite); -int lsmCheckpointSize(lsm_db *db, int *pnByte); - -int lsmInfoCompressionId(lsm_db *db, u32 *piCmpId); /* ** Functions from file "lsm_tree.c". */ int lsmTreeNew(lsm_env *, int (*)(void *, int, void *, int), Tree **ppTree); void lsmTreeRelease(lsm_env *, Tree *); +void lsmTreeClear(lsm_db *); int lsmTreeInit(lsm_db *); int lsmTreeRepair(lsm_db *); void lsmTreeMakeOld(lsm_db *pDb); void lsmTreeDiscardOld(lsm_db *pDb); @@ -594,11 +540,10 @@ int lsmTreeEndTransaction(lsm_db *pDb, int bCommit); int lsmTreeLoadHeader(lsm_db *pDb, int *); int lsmTreeLoadHeaderOk(lsm_db *, int); int lsmTreeInsert(lsm_db *pDb, void *pKey, int nKey, void *pVal, int nVal); -int lsmTreeDelete(lsm_db *db, void *pKey1, int nKey1, void *pKey2, int nKey2); void lsmTreeRollback(lsm_db *pDb, TreeMark *pMark); void lsmTreeMark(lsm_db *pDb, TreeMark *pMark); int lsmTreeCursorNew(lsm_db *pDb, int, TreeCursor **); void lsmTreeCursorDestroy(TreeCursor *); @@ -617,10 +562,19 @@ void lsmFlagsToString(int flags, char *zFlags); /* ** Functions from file "mem.c". */ +int lsmPoolNew(lsm_env *pEnv, Mempool **ppPool); +void lsmPoolDestroy(lsm_env *pEnv, Mempool *pPool); +void *lsmPoolMalloc(lsm_env *pEnv, Mempool *pPool, int nByte); +void *lsmPoolMallocZero(lsm_env *pEnv, Mempool *pPool, int nByte); +int lsmPoolUsed(Mempool *pPool); + +void lsmPoolMark(Mempool *pPool, void **, int *); +void lsmPoolRollback(lsm_env *pEnv, Mempool *pPool, void *, int); + void *lsmMalloc(lsm_env*, size_t); void lsmFree(lsm_env*, void *); void *lsmRealloc(lsm_env*, void *, size_t); void *lsmReallocOrFree(lsm_env*, void *, size_t); void *lsmReallocOrFreeRc(lsm_env *, void *, size_t, int *); @@ -650,12 +604,10 @@ ** Start of functions from "lsm_file.c". */ int lsmFsOpen(lsm_db *, const char *); void lsmFsClose(FileSystem *); -int lsmFsConfigure(lsm_db *db); - int lsmFsBlockSize(FileSystem *); void lsmFsSetBlockSize(FileSystem *, int); int lsmFsPageSize(FileSystem *); void lsmFsSetPageSize(FileSystem *, int); @@ -664,11 +616,11 @@ /* Creating, populating, gobbling and deleting sorted runs. */ void lsmFsGobble(lsm_db *, Segment *, Pgno *, int); int lsmFsSortedDelete(FileSystem *, Snapshot *, int, Segment *); int lsmFsSortedFinish(FileSystem *, Segment *); -int lsmFsSortedAppend(FileSystem *, Snapshot *, Level *, int, Page **); +int lsmFsSortedAppend(FileSystem *, Snapshot *, Segment *, Page **); int lsmFsSortedPadding(FileSystem *, Snapshot *, Segment *); /* Functions to retrieve the lsm_env pointer from a FileSystem or Page object */ lsm_env *lsmFsEnv(FileSystem *); lsm_env *lsmPageEnv(Page *); @@ -678,11 +630,11 @@ void lsmSortedSplitkey(lsm_db *, Level *, int *); /* Reading sorted run content. */ int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg); -int lsmFsDbPageGet(FileSystem *, Segment *, Pgno, Page **); +int lsmFsDbPageGet(FileSystem *, Pgno, Page **); int lsmFsDbPageNext(Segment *, Page *, int eDir, Page **); u8 *lsmFsPageData(Page *, int *); int lsmFsPageRelease(Page *); int lsmFsPagePersist(Page *); @@ -706,23 +658,19 @@ /* Functions to read, write and sync the log file. */ int lsmFsWriteLog(FileSystem *pFS, i64 iOff, LsmString *pStr); int lsmFsSyncLog(FileSystem *pFS); int lsmFsReadLog(FileSystem *pFS, i64 iOff, int nRead, LsmString *pStr); int lsmFsTruncateLog(FileSystem *pFS, i64 nByte); -int lsmFsTruncateDb(FileSystem *pFS, i64 nByte); int lsmFsCloseAndDeleteLog(FileSystem *pFS); void lsmFsDeferClose(FileSystem *pFS, LsmFile **pp); /* And to sync the db file */ -int lsmFsSyncDb(FileSystem *, int); - -void lsmFsFlushWaiting(FileSystem *, int *); +int lsmFsSyncDb(FileSystem *); /* Used by lsm_info(ARRAY_STRUCTURE) and lsm_config(MMAP) */ -int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, Pgno iFirst, char **pzOut); -int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut); +int lsmInfoArrayStructure(lsm_db *pDb, Pgno iFirst, char **pzOut); int lsmConfigMmap(lsm_db *pDb, int *piParam); int lsmEnvOpen(lsm_env *, const char *, lsm_file **); int lsmEnvClose(lsm_env *pEnv, lsm_file *pFile); int lsmEnvLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int eLock); @@ -731,15 +679,10 @@ void lsmEnvShmBarrier(lsm_env *); void lsmEnvShmUnmap(lsm_env *, lsm_file *, int); void lsmEnvSleep(lsm_env *, int); -int lsmFsReadSyncedId(lsm_db *db, int, i64 *piVal); - -int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, Pgno, int *); -Pgno lsmFsRedirectPage(FileSystem *, Redirect *, Pgno); - /* ** End of functions from "lsm_file.c". **************************************************************************/ /* @@ -747,14 +690,10 @@ */ int lsmInfoPageDump(lsm_db *, Pgno, int, char **); void lsmSortedCleanup(lsm_db *); int lsmSortedAutoWork(lsm_db *, int nUnit); -int lsmSortedWalkFreelist(lsm_db *, int, int (*)(void *, int, i64), void *); - -int lsmSaveWorker(lsm_db *, int); - int lsmFlushTreeToDisk(lsm_db *pDb); void lsmSortedRemap(lsm_db *pDb); void lsmSortedFreeLevel(lsm_env *pEnv, Level *); @@ -768,11 +707,11 @@ void lsmSortedSaveTreeCursors(lsm_db *); int lsmMCursorNew(lsm_db *, MultiCursor **); void lsmMCursorClose(MultiCursor *); -int lsmMCursorSeek(MultiCursor *, int, void *, int , int); +int lsmMCursorSeek(MultiCursor *, void *, int , int); int lsmMCursorFirst(MultiCursor *); int lsmMCursorPrev(MultiCursor *); int lsmMCursorLast(MultiCursor *); int lsmMCursorValid(MultiCursor *); int lsmMCursorNext(MultiCursor *); @@ -785,15 +724,13 @@ int lsmRestoreCursors(lsm_db *pDb); void lsmSortedDumpStructure(lsm_db *pDb, Snapshot *, int, int, const char *); void lsmFsDumpBlocklists(lsm_db *); -void lsmSortedExpandBtreePage(Page *pPg, int nOrig); void lsmPutU32(u8 *, u32); u32 lsmGetU32(u8 *); -u64 lsmGetU64(u8 *); /* ** Functions from "lsm_varint.c". */ int lsmVarintPut32(u8 *, int); @@ -806,11 +743,10 @@ /* ** Functions from file "main.c". */ void lsmLogMessage(lsm_db *, int, const char *, ...); -int lsmInfoFreelist(lsm_db *pDb, char **pzOut); /* ** Functions from file "lsm_log.c". */ int lsmLogBegin(lsm_db *pDb); @@ -817,14 +753,14 @@ int lsmLogWrite(lsm_db *, void *, int, void *, int); int lsmLogCommit(lsm_db *); void lsmLogEnd(lsm_db *pDb, int bCommit); void lsmLogTell(lsm_db *, LogMark *); void lsmLogSeek(lsm_db *, LogMark *); -void lsmLogClose(lsm_db *); int lsmLogRecover(lsm_db *); int lsmInfoLogStructure(lsm_db *pDb, char **pzVal); +int lsmInfoFreelist(lsm_db *, char **pzVal); /************************************************************************** ** Functions from file "lsm_shared.c". */ @@ -835,11 +771,11 @@ int lsmBeginReadTrans(lsm_db *); int lsmBeginWriteTrans(lsm_db *); int lsmBeginFlush(lsm_db *); int lsmBeginWork(lsm_db *); -void lsmFinishWork(lsm_db *, int, int *); +void lsmFinishWork(lsm_db *, int, int, int *); int lsmFinishRecovery(lsm_db *); void lsmFinishReadTrans(lsm_db *); int lsmFinishWriteTrans(lsm_db *, int); int lsmFinishFlush(lsm_db *, int); @@ -854,11 +790,11 @@ Level *lsmDbSnapshotLevel(Snapshot *); void lsmDbSnapshotSetLevel(Snapshot *, Level *); void lsmDbRecoveryComplete(lsm_db *, int); -int lsmBlockAllocate(lsm_db *, int, int *); +int lsmBlockAllocate(lsm_db *, int *); int lsmBlockFree(lsm_db *, int); int lsmBlockRefree(lsm_db *, int); void lsmFreelistDeltaBegin(lsm_db *); void lsmFreelistDeltaEnd(lsm_db *); @@ -878,11 +814,11 @@ /* Candidate values for the 3rd argument to lsmShmLock() */ #define LSM_LOCK_UNLOCK 0 #define LSM_LOCK_SHARED 1 #define LSM_LOCK_EXCL 2 -int lsmShmCacheChunks(lsm_db *db, int nChunk); +int lsmShmChunk(lsm_db *db, int iChunk, void **ppData); int lsmShmLock(lsm_db *db, int iLock, int eOp, int bBlock); void lsmShmBarrier(lsm_db *db); #ifdef LSM_DEBUG void lsmShmHasLock(lsm_db *db, int iLock, int eOp); @@ -899,14 +835,10 @@ int lsmDbMultiProc(lsm_db *); void lsmDbDeferredClose(lsm_db *, lsm_file *, LsmFile *); LsmFile *lsmDbRecycleFd(lsm_db *); -int lsmWalkFreelist(lsm_db *, int, int (*)(void *, int, i64), void *); - -int lsmCheckCompressionId(lsm_db *, u32); - /************************************************************************** ** functions in lsm_str.c */ void lsmStringInit(LsmString*, lsm_env *pEnv); Index: src/lsm_ckpt.c ================================================================== --- src/lsm_ckpt.c +++ src/lsm_ckpt.c @@ -30,16 +30,15 @@ ** ** 1. The checkpoint id MSW. ** 2. The checkpoint id LSW. ** 3. The number of integer values in the entire checkpoint, including ** the two checksum values. -** 4. The compression scheme id. -** 5. The total number of blocks in the database. -** 6. The block size. -** 7. The number of levels. -** 8. The nominal database page size. -** 9. The number of pages (in total) written to the database file. +** 4. The total number of blocks in the database. +** 5. The block size. +** 6. The number of levels. +** 7. The nominal database page size. +** 8. Flag indicating if there exists a FREELIST record in the database. ** ** Log pointer: ** ** 1. The log offset MSW. ** 2. The log offset LSW. @@ -48,16 +47,15 @@ ** ** See ckptExportLog() and ckptImportLog(). ** ** Append points: ** -** 8 integers (4 * 64-bit page numbers). See ckptExportAppendlist(). +** 4 integers. See ckptExportAppendlist(). ** ** For each level in the database, a level record. Formatted as follows: ** -** 0. Age of the level (least significant 16-bits). And flags mask (most -** significant 16-bits). +** 0. Age of the level. ** 1. The number of right-hand segments (nRight, possibly 0), ** 2. Segment record for left-hand segment (8 integers defined below), ** 3. Segment record for each right-hand segment (8 integers defined below), ** 4. If nRight>0, The number of segments involved in the merge ** 5. if nRight>0, Current nSkip value (see Merge structure defn.), @@ -67,28 +65,22 @@ ** 5b. Cell number of next cell to read during merge ** 7. Page containing current split-key (64-bits - 2 integers). ** 8. Cell within page containing current split-key. ** 9. Current pointer value (64-bits - 2 integers). ** -** The block redirect array: -** -** 1. Number of redirections (maximum LSM_MAX_BLOCK_REDIRECTS). -** 2. For each redirection: -** a. "from" block number -** b. "to" block number -** -** The in-memory freelist entries. Each entry is either an insert or a -** delete. The in-memory freelist is to the free-block-list as the -** in-memory tree is to the users database content. +** The freelist. ** ** 1. Number of free-list entries stored in checkpoint header. -** 2. Number of free blocks (in total). -** 3. Total number of blocks freed during database lifetime. -** 4. For each entry: +** 2. For each entry: ** 2a. Block number of free block. -** 2b. A 64-bit integer (MSW followed by LSW). -1 for a delete entry, -** or the associated checkpoint id for an insert. +** 2b. MSW of associated checkpoint id. +** 2c. LSW of associated checkpoint id. +** +** If the overflow flag is set, then extra free-list entries may be stored +** in the FREELIST record. The FREELIST record contains 3 32-bit integers +** per entry, in the same format as above (without the "number of entries" +** field). ** ** The checksum: ** ** 1. Checksum value 1. ** 2. Checksum value 2. @@ -181,15 +173,15 @@ /* A #define to describe each integer in the checkpoint header. */ #define CKPT_HDR_ID_MSW 0 #define CKPT_HDR_ID_LSW 1 #define CKPT_HDR_NCKPT 2 -#define CKPT_HDR_CMPID 3 -#define CKPT_HDR_NBLOCK 4 -#define CKPT_HDR_BLKSZ 5 -#define CKPT_HDR_NLEVEL 6 -#define CKPT_HDR_PGSZ 7 +#define CKPT_HDR_NBLOCK 3 +#define CKPT_HDR_BLKSZ 4 +#define CKPT_HDR_NLEVEL 5 +#define CKPT_HDR_PGSZ 6 +#define CKPT_HDR_OVFL 7 #define CKPT_HDR_NWRITE 8 #define CKPT_HDR_LO_MSW 9 #define CKPT_HDR_LO_LSW 10 #define CKPT_HDR_LO_CKSUM1 11 @@ -318,11 +310,11 @@ ){ int iOut = *piOut; Merge *pMerge; pMerge = pLevel->pMerge; - ckptSetValue(p, iOut++, (u32)pLevel->iAge + (u32)(pLevel->flags<<16), pRc); + ckptSetValue(p, iOut++, pLevel->iAge, pRc); ckptSetValue(p, iOut++, pLevel->nRight, pRc); ckptExportSegment(&pLevel->lhs, p, &iOut, pRc); assert( (pLevel->nRight>0)==(pMerge!=0) ); if( pMerge ){ @@ -370,11 +362,10 @@ for(; iOut<=CKPT_HDR_LO_CKSUM2; iOut++){ ckptSetValue(p, iOut, pDb->pShmhdr->aSnap2[iOut], pRc); } } - assert( *pRc || iOut==CKPT_HDR_LO_CKSUM2+1 ); *piOut = iOut; } static void ckptExportAppendlist( lsm_db *db, /* Database connection */ @@ -390,10 +381,11 @@ } }; static int ckptExportSnapshot( lsm_db *pDb, /* Connection handle */ + int nOvfl, /* Number of free-list entries in LSM */ int bLog, /* True to update log-offset fields */ i64 iId, /* Checkpoint id */ int bCksum, /* If true, include checksums */ void **ppCkpt, /* OUT: Buffer containing checkpoint */ int *pnCkpt /* OUT: Size of checkpoint in bytes */ @@ -405,10 +397,19 @@ int iLevel; /* Used to count out nLevel levels */ int iOut = 0; /* Current offset in aCkpt[] */ Level *pLevel; /* Level iterator */ int i; /* Iterator used while serializing freelist */ CkptBuffer ckpt; + int nFree; + + nFree = pSnap->freelist.nEntry; + if( nOvfl>=0 ){ + nFree -= nOvfl; + }else{ + assert( 0 ); + nOvfl = pDb->pShmhdr->aSnap2[CKPT_HDR_OVFL]; + } /* Initialize the output buffer */ memset(&ckpt, 0, sizeof(CkptBuffer)); ckpt.pEnv = pDb->pEnv; iOut = CKPT_HDR_SIZE; @@ -427,21 +428,12 @@ for(pLevel=lsmDbSnapshotLevel(pSnap); iLevelpNext){ ckptExportLevel(pLevel, &ckpt, &iOut, &rc); iLevel++; } - /* Write the block-redirect list */ - ckptSetValue(&ckpt, iOut++, pSnap->redirect.n, &rc); - for(i=0; iredirect.n; i++){ - ckptSetValue(&ckpt, iOut++, pSnap->redirect.a[i].iFrom, &rc); - ckptSetValue(&ckpt, iOut++, pSnap->redirect.a[i].iTo, &rc); - } - /* Write the freelist */ - assert( pSnap->freelist.nEntry<=pDb->nMaxFreelist ); if( rc==LSM_OK ){ - int nFree = pSnap->freelist.nEntry; ckptSetValue(&ckpt, iOut++, nFree, &rc); for(i=0; ifreelist.aEntry[i]; ckptSetValue(&ckpt, iOut++, p->iBlk, &rc); ckptSetValue(&ckpt, iOut++, (p->iId >> 32) & 0xFFFFFFFF, &rc); @@ -449,21 +441,18 @@ } } /* Write the checkpoint header */ assert( iId>=0 ); - assert( pSnap->iCmpId==pDb->compress.iId - || pSnap->iCmpId==LSM_COMPRESSION_EMPTY - ); ckptSetValue(&ckpt, CKPT_HDR_ID_MSW, (u32)(iId>>32), &rc); ckptSetValue(&ckpt, CKPT_HDR_ID_LSW, (u32)(iId&0xFFFFFFFF), &rc); ckptSetValue(&ckpt, CKPT_HDR_NCKPT, iOut+2, &rc); - ckptSetValue(&ckpt, CKPT_HDR_CMPID, pDb->compress.iId, &rc); ckptSetValue(&ckpt, CKPT_HDR_NBLOCK, pSnap->nBlock, &rc); ckptSetValue(&ckpt, CKPT_HDR_BLKSZ, lsmFsBlockSize(pFS), &rc); ckptSetValue(&ckpt, CKPT_HDR_NLEVEL, nLevel, &rc); ckptSetValue(&ckpt, CKPT_HDR_PGSZ, lsmFsPageSize(pFS), &rc); + ckptSetValue(&ckpt, CKPT_HDR_OVFL, (nOvfl?nOvfl:pSnap->nFreelistOvfl), &rc); ckptSetValue(&ckpt, CKPT_HDR_NWRITE, pSnap->nWrite, &rc); if( bCksum ){ ckptAddChecksum(&ckpt, iOut, &rc); }else{ @@ -473,19 +462,12 @@ iOut += 2; assert( iOut<=1024 ); #ifdef LSM_LOG_FREELIST lsmLogMessage(pDb, rc, - "ckptExportSnapshot(): id=%lld freelist: %d", iId, pSnap->freelist.nEntry - ); - for(i=0; ifreelist.nEntry; i++){ - lsmLogMessage(pDb, rc, - "ckptExportSnapshot(): iBlk=%d id=%lld", - pSnap->freelist.aEntry[i].iBlk, - pSnap->freelist.aEntry[i].iId - ); - } + "ckptExportSnapshot(): id=%d freelist: %d/%d", (int)iId, nFree, nOvfl + ); #endif *ppCkpt = (void *)ckpt.aCkpt; if( pnCkpt ) *pnCkpt = sizeof(u32)*iOut; return rc; @@ -561,13 +543,11 @@ Level *pLevel; /* Allocate space for the Level structure and Level.apRight[] array */ pLevel = (Level *)lsmMallocZeroRc(pDb->pEnv, sizeof(Level), &rc); if( rc==LSM_OK ){ - pLevel->iAge = (u16)(aIn[iIn] & 0x0000FFFF); - pLevel->flags = (u16)((aIn[iIn]>>16) & 0x0000FFFF); - iIn++; + pLevel->iAge = aIn[iIn++]; pLevel->nRight = aIn[iIn++]; if( pLevel->nRight ){ int nByte = sizeof(Segment) * pLevel->nRight; pLevel->aRhs = (Segment *)lsmMallocZeroRc(pDb->pEnv, nByte, &rc); } @@ -685,10 +665,155 @@ *paVal = 0; } return rc; } + +/* +** The worker lock must be held to call this function. +** +** The function serializes and returns the data that should be stored as +** the FREELIST system record. +*/ +int lsmCheckpointOverflow( + lsm_db *pDb, /* Database handle (must hold worker lock) */ + void **ppVal, /* OUT: lsmMalloc'd buffer */ + int *pnVal, /* OUT: Size of *ppVal in bytes */ + int *pnOvfl /* OUT: Number of freelist entries in buf */ +){ + int rc = LSM_OK; + int nRet; + Snapshot *p = pDb->pWorker; + + assert( lsmShmAssertWorker(pDb) ); + assert( pnOvfl && ppVal && pnVal ); + assert( pDb->nMaxFreelist>=2 && pDb->nMaxFreelist<=LSM_MAX_FREELIST_ENTRIES ); + + if( p->nFreelistOvfl ){ + rc = lsmCheckpointOverflowLoad(pDb, &p->freelist); + if( rc!=LSM_OK ) return rc; + p->nFreelistOvfl = 0; + } + + if( p->freelist.nEntry<=pDb->nMaxFreelist ){ + nRet = 0; + *pnVal = 0; + *ppVal = 0; + }else{ + int i; /* Iterator variable */ + int iOut = 0; /* Current size of blob in ckpt */ + CkptBuffer ckpt; /* Used to build FREELIST blob */ + + nRet = (p->freelist.nEntry - pDb->nMaxFreelist); + + memset(&ckpt, 0, sizeof(CkptBuffer)); + ckpt.pEnv = pDb->pEnv; + for(i=p->freelist.nEntry-nRet; rc==LSM_OK && ifreelist.nEntry; i++){ + FreelistEntry *pEntry = &p->freelist.aEntry[i]; + ckptSetValue(&ckpt, iOut++, pEntry->iBlk, &rc); + ckptSetValue(&ckpt, iOut++, (pEntry->iId >> 32) & 0xFFFFFFFF, &rc); + ckptSetValue(&ckpt, iOut++, pEntry->iId & 0xFFFFFFFF, &rc); + } + ckptChangeEndianness(ckpt.aCkpt, iOut); + + *ppVal = ckpt.aCkpt; + *pnVal = iOut*sizeof(u32); + } + + *pnOvfl = nRet; + return rc; +} + +/* +** The connection must be the worker in order to call this function. +** +** True is returned if there are currently too many free-list entries +** in-memory to store in a checkpoint. Before calling CheckpointSaveWorker() +** to save the current worker snapshot, a new top-level LSM segment must +** be created so that some of them can be written to the LSM. +*/ +int lsmCheckpointOverflowRequired(lsm_db *pDb){ + Snapshot *p = pDb->pWorker; + assert( lsmShmAssertWorker(pDb) ); + return (p->freelist.nEntry > pDb->nMaxFreelist || p->nFreelistOvfl>0); +} + +/* +** Connection pDb must be the worker to call this function. +** +** Load the FREELIST record from the database. Decode it and append the +** results to list pFreelist. +*/ +int lsmCheckpointOverflowLoad( + lsm_db *pDb, + Freelist *pFreelist +){ + int rc; + int nVal = 0; + void *pVal = 0; + assert( lsmShmAssertWorker(pDb) ); + + /* Load the blob of data from the LSM. If that is successful (and the + ** blob is greater than zero bytes in size), decode the contents and + ** merge them into the current contents of *pFreelist. */ + rc = lsmSortedLoadFreelist(pDb, &pVal, &nVal); + if( pVal ){ + u32 *aFree = (u32 *)pVal; + int nFree = nVal / sizeof(int); + ckptChangeEndianness(aFree, nFree); + if( (nFree % 3) ){ + rc = LSM_CORRUPT_BKPT; + }else{ + int iNew = 0; /* Offset of next element in aFree[] */ + int iOld = 0; /* Next element in freelist fl */ + Freelist fl = *pFreelist; /* Original contents of *pFreelist */ + + memset(pFreelist, 0, sizeof(Freelist)); + while( rc==LSM_OK && (iNew=fl.nEntry ){ + iBlk = aFree[iNew]; + iId = ((i64)(aFree[iNew+1])<<32) + (i64)aFree[iNew+2]; + iNew += 3; + }else if( iNew>=nFree ){ + iBlk = fl.aEntry[iOld].iBlk; + iId = fl.aEntry[iOld].iId; + iOld += 1; + }else{ + iId = ((i64)(aFree[iNew+1])<<32) + (i64)aFree[iNew+2]; + if( iIdpEnv, pFreelist, iBlk, iId); + } + lsmFree(pDb->pEnv, fl.aEntry); + +#ifdef LSM_DEBUG + if( rc==LSM_OK ){ + int i; + for(i=1; rc==LSM_OK && inEntry; i++){ + assert( pFreelist->aEntry[i].iId >= pFreelist->aEntry[i-1].iId ); + } + assert( pFreelist->nEntry==(fl.nEntry + nFree/3) ); + } +#endif + } + + lsmFree(pDb->pEnv, pVal); + } + + return rc; +} /* ** Read the checkpoint id from meta-page pPg. */ static i64 ckptLoadId(MetaPage *pPg){ @@ -765,24 +890,23 @@ ** Initialize the shared-memory header with an empty snapshot. This function ** is called when no valid snapshot can be found in the database header. */ static void ckptLoadEmpty(lsm_db *pDb){ u32 aCkpt[] = { - 0, /* CKPT_HDR_ID_MSW */ - 10, /* CKPT_HDR_ID_LSW */ - 0, /* CKPT_HDR_NCKPT */ - LSM_COMPRESSION_EMPTY, /* CKPT_HDR_CMPID */ - 0, /* CKPT_HDR_NBLOCK */ - 0, /* CKPT_HDR_BLKSZ */ - 0, /* CKPT_HDR_NLEVEL */ - 0, /* CKPT_HDR_PGSZ */ - 0, /* CKPT_HDR_OVFL */ - 0, /* CKPT_HDR_NWRITE */ - 0, 0, 1234, 5678, /* The log pointer and initial checksum */ - 0,0,0,0, 0,0,0,0, /* The append list */ - 0, /* The free block list */ - 0, 0 /* Space for checksum values */ + 0, /* CKPT_HDR_ID_MSW */ + 10, /* CKPT_HDR_ID_LSW */ + 0, /* CKPT_HDR_NCKPT */ + 0, /* CKPT_HDR_NBLOCK */ + 0, /* CKPT_HDR_BLKSZ */ + 0, /* CKPT_HDR_NLEVEL */ + 0, /* CKPT_HDR_PGSZ */ + 0, /* CKPT_HDR_OVFL */ + 0, /* CKPT_HDR_NWRITE */ + 0, 0, 1234, 5678, /* The log pointer and initial checksum */ + 0,0,0,0, 0,0,0,0, /* The append list */ + 0, /* The free block list */ + 0, 0 /* Space for checksum values */ }; u32 nCkpt = array_size(aCkpt); ShmHeader *pShm = pDb->pShmhdr; aCkpt[CKPT_HDR_NCKPT] = nCkpt; @@ -884,22 +1008,10 @@ lsmShmBarrier(pDb); } return LSM_PROTOCOL; } -int lsmInfoCompressionId(lsm_db *db, u32 *piCmpId){ - int rc; - - assert( db->pClient==0 && db->pWorker==0 ); - rc = lsmCheckpointLoad(db, 0); - if( rc==LSM_OK ){ - *piCmpId = db->aSnapshot[CKPT_HDR_CMPID]; - } - - return rc; -} - int lsmCheckpointLoadOk(lsm_db *pDb, int iSnap){ u32 *aShm; assert( iSnap==1 || iSnap==2 ); aShm = (iSnap==1) ? pDb->pShmhdr->aSnap1 : pDb->pShmhdr->aSnap2; return (lsmCheckpointId(pDb->aSnapshot, 0)==lsmCheckpointId(aShm, 0) ); @@ -917,15 +1029,12 @@ int rc; ShmHeader *pShm = pDb->pShmhdr; int nInt1; int nInt2; - /* Must be holding the WORKER lock to do this. Or DMS2. */ - assert( - lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL) - || lsmShmAssertLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_EXCL) - ); + /* Must be holding the WORKER lock to do this */ + assert( lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL) ); /* Check that the two snapshots match. If not, repair them. */ nInt1 = pShm->aSnap1[CKPT_HDR_NCKPT]; nInt2 = pShm->aSnap2[CKPT_HDR_NCKPT]; if( nInt1!=nInt2 || memcmp(pShm->aSnap1, pShm->aSnap2, nInt2*sizeof(u32)) ){ @@ -937,19 +1046,11 @@ return LSM_PROTOCOL; } } rc = lsmCheckpointDeserialize(pDb, 1, pShm->aSnap1, &pDb->pWorker); - if( pDb->pWorker ) pDb->pWorker->pDatabase = pDb->pDatabase; - - if( rc==LSM_OK ){ - rc = lsmCheckCompressionId(pDb, pDb->pWorker->iCmpId); - } - -#if 0 assert( rc!=LSM_OK || lsmFsIntegrityCheck(pDb) ); -#endif return rc; } int lsmCheckpointDeserialize( lsm_db *pDb, @@ -960,11 +1061,10 @@ int rc = LSM_OK; Snapshot *pNew; pNew = (Snapshot *)lsmMallocZeroRc(pDb->pEnv, sizeof(Snapshot), &rc); if( rc==LSM_OK ){ - Level *pLvl; int nFree; int i; int nLevel = (int)aCkpt[CKPT_HDR_NLEVEL]; int iIn = CKPT_HDR_SIZE + CKPT_APPENDLIST_SIZE + CKPT_LOGPTR_SIZE; @@ -971,40 +1071,20 @@ pNew->iId = lsmCheckpointId(aCkpt, 0); pNew->nBlock = aCkpt[CKPT_HDR_NBLOCK]; pNew->nWrite = aCkpt[CKPT_HDR_NWRITE]; rc = ckptLoadLevels(pDb, aCkpt, &iIn, nLevel, &pNew->pLevel); pNew->iLogOff = lsmCheckpointLogOffset(aCkpt); - pNew->iCmpId = aCkpt[CKPT_HDR_CMPID]; /* Make a copy of the append-list */ for(i=0; iaiAppend[i] = ckptRead64(a); } - /* Read the block-redirect list */ - pNew->redirect.n = aCkpt[iIn++]; - if( pNew->redirect.n ){ - pNew->redirect.a = lsmMallocZeroRc(pDb->pEnv, - (sizeof(struct RedirectEntry) * LSM_MAX_BLOCK_REDIRECTS), &rc - ); - if( rc==LSM_OK ){ - for(i=0; iredirect.n; i++){ - pNew->redirect.a[i].iFrom = aCkpt[iIn++]; - pNew->redirect.a[i].iTo = aCkpt[iIn++]; - } - } - for(pLvl=pNew->pLevel; pLvl->pNext; pLvl=pLvl->pNext); - if( pLvl->nRight ){ - pLvl->aRhs[pLvl->nRight-1].pRedirect = &pNew->redirect; - }else{ - pLvl->lhs.pRedirect = &pNew->redirect; - } - } - /* Copy the free-list */ - if( rc==LSM_OK && bInclFreelist ){ + if( bInclFreelist ){ + pNew->nFreelistOvfl = aCkpt[CKPT_HDR_OVFL]; nFree = aCkpt[iIn++]; if( nFree ){ pNew->freelist.aEntry = (FreelistEntry *)lsmMallocZeroRc( pDb->pEnv, sizeof(FreelistEntry)*nFree, &rc ); @@ -1065,29 +1145,34 @@ ** the new snapshot produced by the work performed by pDb. ** ** If successful, LSM_OK is returned. Otherwise, if an error occurs, an LSM ** error code is returned. */ -int lsmCheckpointSaveWorker(lsm_db *pDb, int bFlush){ +int lsmCheckpointSaveWorker(lsm_db *pDb, int bFlush, int nOvfl){ Snapshot *pSnap = pDb->pWorker; ShmHeader *pShm = pDb->pShmhdr; void *p = 0; int n = 0; int rc; - pSnap->iId++; - rc = ckptExportSnapshot(pDb, bFlush, pSnap->iId, 1, &p, &n); +#if 0 +if( bFlush ){ + printf("pushing %p tree to %d\n", (void *)pDb, pSnap->iId+1); + fflush(stdout); +} +#endif + assert( lsmFsIntegrityCheck(pDb) ); + rc = ckptExportSnapshot(pDb, nOvfl, bFlush, pSnap->iId+1, 1, &p, &n); if( rc!=LSM_OK ) return rc; assert( ckptChecksumOk((u32 *)p) ); assert( n<=LSM_META_PAGE_SIZE ); memcpy(pShm->aSnap2, p, n); lsmShmBarrier(pDb); memcpy(pShm->aSnap1, p, n); lsmFree(pDb->pEnv, p); - assert( lsmFsIntegrityCheck(pDb) ); return LSM_OK; } /* ** This function is used to determine the snapshot-id of the most recently @@ -1157,14 +1242,10 @@ iId = ((i64)aCkpt[CKPT_HDR_ID_MSW] << 32) + (i64)aCkpt[CKPT_HDR_ID_LSW]; } return iId; } -u32 lsmCheckpointNBlock(u32 *aCkpt){ - return aCkpt[CKPT_HDR_NBLOCK]; -} - u32 lsmCheckpointNWrite(u32 *aCkpt, int bDisk){ if( bDisk ){ return lsmGetU32((u8 *)&aCkpt[CKPT_HDR_NWRITE]); }else{ return aCkpt[CKPT_HDR_NWRITE]; @@ -1208,27 +1289,20 @@ memcpy(pDb->pShmhdr->aSnap1, pDb->aSnapshot, nCkpt*sizeof(u32)); memcpy(pDb->pShmhdr->aSnap2, pDb->aSnapshot, nCkpt*sizeof(u32)); } -/* -** Set the output variable to the number of KB of data written into the -** database file since the most recent checkpoint. -*/ -int lsmCheckpointSize(lsm_db *db, int *pnKB){ +int lsm_ckpt_size(lsm_db *db, int *pnByte){ ShmHeader *pShm = db->pShmhdr; int rc = LSM_OK; u32 nSynced; - /* Set nSynced to the number of pages that had been written when the - ** database was last checkpointed. */ rc = lsmCheckpointSynced(db, 0, 0, &nSynced); - if( rc==LSM_OK ){ u32 nPgsz = db->pShmhdr->aSnap1[CKPT_HDR_PGSZ]; u32 nWrite = db->pShmhdr->aSnap1[CKPT_HDR_NWRITE]; - *pnKB = (int)(( ((i64)(nWrite - nSynced) * nPgsz) + 1023) / 1024); + *pnByte = (int)((nWrite - nSynced) * nPgsz); } return rc; } Index: src/lsm_file.c ================================================================== --- src/lsm_file.c +++ src/lsm_file.c @@ -131,17 +131,11 @@ ** ** Sometimes, in order to avoid touching sectors that contain synced data ** when writing, it is necessary to insert unused space between compressed ** page records. This can be done as follows: ** -** * For less than 6 bytes of empty space, the first and last byte -** of the free space contain the total number of free bytes. For -** example: -** -** Block of 4 free bytes: 0x04 0x?? 0x?? 0x04 -** Block of 2 free bytes: 0x02 0x02 -** A single free byte: 0x01 +** * For less than 6 bytes of empty space, a series of 0x00 bytes. ** ** * For 6 or more bytes of empty space, a record similar to a ** compressed page record is added to the segment. A padding record ** is distinguished from a compressed page record by the most ** significant bit of the second byte of the size field, which is @@ -167,12 +161,11 @@ ** are carrying pointers into the database file mapping (pMap/nMap). If the ** file has to be unmapped and then remapped (required to grow the mapping ** as the file grows), the Page.aData pointers are updated by iterating ** through the contents of this list. ** -** In non-mmap() mode, this list is an LRU list of cached pages with -** nRef==0. +** In non-mmap() mode, this list is an LRU list of cached pages with nRef==0. */ struct FileSystem { lsm_db *pDb; /* Database handle that owns this object */ lsm_env *pEnv; /* Environment pointer */ char *zDb; /* Database file name */ @@ -180,30 +173,27 @@ int nMetasize; /* Size of meta pages in bytes */ int nPagesize; /* Database page-size in bytes */ int nBlocksize; /* Database block-size in bytes */ /* r/w file descriptors for both files. */ - LsmFile *pLsmFile; /* Used after lsm_close() to link into list */ + LsmFile *pLsmFile; lsm_file *fdDb; /* Database file */ lsm_file *fdLog; /* Log file */ int szSector; /* Database file sector size */ /* If this is a compressed database, a pointer to the compression methods. ** For an uncompressed database, a NULL pointer. */ lsm_compress *pCompress; - u8 *aIBuffer; /* Buffer to compress to */ - u8 *aOBuffer; /* Buffer to uncompress from */ + u8 *aBuffer; /* Buffer to compress into */ int nBuffer; /* Allocated size of aBuffer[] in bytes */ /* mmap() mode things */ int bUseMmap; /* True to use mmap() to access db file */ void *pMap; /* Current mapping of database file */ i64 nMap; /* Bytes mapped at pMap */ Page *pFree; - Page *pWaiting; /* b-tree pages waiting to be written */ - /* Statistics */ int nWrite; /* Total number of pages written */ int nRead; /* Total number of pages read */ /* Page cache parameters for non-mmap() mode */ @@ -242,13 +232,10 @@ /* Only used in compressed database mode: */ int nCompress; /* Compressed size (or 0 for uncomp. db) */ int nCompressPrev; /* Compressed size of prev page */ Segment *pSeg; /* Segment this page will be written to */ - - /* Fix this up somehow */ - Page *pNextWaiting; }; /* ** Meta-data page handle. There are two meta-data pages at the start of ** the database file, each FileSystem.nMetasize bytes in size. @@ -261,13 +248,13 @@ }; /* ** Values for LsmPage.flags */ -#define PAGE_DIRTY 0x00000001 /* Set if page is dirty */ -#define PAGE_FREE 0x00000002 /* Set if Page.aData requires lsmFree() */ -#define PAGE_HASPREV 0x00000004 /* Set if page is first on uncomp. block */ +#define PAGE_DIRTY 0x00000001 /* Set if page is dirty */ +#define PAGE_FREE 0x00000002 /* Set if Page.aData requires lsmFree() */ +#define PAGE_SHORT 0x00000004 /* Set if page is 4 bytes short */ /* ** Number of pgsz byte pages omitted from the start of block 1. The start ** of block 1 contains two 4096 byte meta pages (8192 bytes in total). */ @@ -415,18 +402,10 @@ int lsmFsTruncateLog(FileSystem *pFS, i64 nByte){ if( pFS->fdLog==0 ) return LSM_OK; return lsmEnvTruncate(pFS->pEnv, pFS->fdLog, nByte); } -/* -** Truncate the log file to nByte bytes in size. -*/ -int lsmFsTruncateDb(FileSystem *pFS, i64 nByte){ - if( pFS->fdDb==0 ) return LSM_OK; - return lsmEnvTruncate(pFS->pEnv, pFS->fdDb, nByte); -} - /* ** Close the log file. Then delete it from the file-system. This function ** is called during database shutdown only. */ int lsmFsCloseAndDeleteLog(FileSystem *pFS){ @@ -508,10 +487,14 @@ pFS->nPagesize = LSM_DFLT_PAGE_SIZE; pFS->nBlocksize = LSM_DFLT_BLOCK_SIZE; pFS->nMetasize = 4 * 1024; pFS->pDb = pDb; pFS->pEnv = pDb->pEnv; + pFS->bUseMmap = pDb->bMmap; + if( pDb->compress.xCompress ){ + pFS->pCompress = &pDb->compress; + } /* Make a copy of the database and log file names. */ memcpy(pFS->zDb, zDb, nDb+1); memcpy(pFS->zLog, zDb, nDb); memcpy(&pFS->zLog[nDb], "-log", 5); @@ -545,62 +528,10 @@ pDb->pFS = pFS; return rc; } -/* -** Configure the file-system object according to the current values of -** the LSM_CONFIG_MMAP and LSM_CONFIG_SET_COMPRESSION options. -*/ -int lsmFsConfigure(lsm_db *db){ - FileSystem *pFS = db->pFS; - if( pFS ){ - lsm_env *pEnv = pFS->pEnv; - Page *pPg; - - assert( pFS->nOut==0 ); - assert( pFS->pWaiting==0 ); - - /* Reset any compression/decompression buffers already allocated */ - lsmFree(pEnv, pFS->aIBuffer); - lsmFree(pEnv, pFS->aOBuffer); - pFS->nBuffer = 0; - - /* Unmap the file, if it is currently mapped */ - if( pFS->pMap ){ - lsmEnvRemap(pEnv, pFS->fdDb, -1, &pFS->pMap, &pFS->nMap); - pFS->bUseMmap = 0; - } - - /* Free all allocate page structures */ - pPg = pFS->pLruFirst; - while( pPg ){ - Page *pNext = pPg->pLruNext; - if( pPg->flags & PAGE_FREE ) lsmFree(pEnv, pPg->aData); - lsmFree(pEnv, pPg); - pPg = pNext; - } - - /* Zero pointers that point to deleted page objects */ - pFS->nCacheAlloc = 0; - pFS->pLruFirst = 0; - pFS->pLruLast = 0; - pFS->pFree = 0; - - /* Configure the FileSystem object */ - if( db->compress.xCompress ){ - pFS->pCompress = &db->compress; - pFS->bUseMmap = 0; - }else{ - pFS->pCompress = 0; - pFS->bUseMmap = db->bMmap; - } - } - - return LSM_OK; -} - /* ** Close and destroy a FileSystem object. */ void lsmFsClose(FileSystem *pFS){ if( pFS ){ @@ -618,12 +549,11 @@ if( pFS->fdDb ) lsmEnvClose(pFS->pEnv, pFS->fdDb ); if( pFS->fdLog ) lsmEnvClose(pFS->pEnv, pFS->fdLog ); lsmFree(pEnv, pFS->pLsmFile); lsmFree(pEnv, pFS->apHash); - lsmFree(pEnv, pFS->aIBuffer); - lsmFree(pEnv, pFS->aOBuffer); + lsmFree(pEnv, pFS->aBuffer); lsmFree(pEnv, pFS); } } void lsmFsDeferClose(FileSystem *pFS, LsmFile **pp){ @@ -768,10 +698,11 @@ ** This function is only called in non-compressed database mode. */ static int fsIsFirst(FileSystem *pFS, Pgno iPg){ const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); assert( !pFS->pCompress ); + return ( (iPg % nPagePerBlock)==1 || (iPgnData; +#ifndef NDEBUG + int bShort = (pPage->pFS->pCompress==0 && + (fsIsFirst(pPage->pFS, pPage->iPg) || fsIsLast(pPage->pFS, pPage->iPg)) + ); + assert( bShort==!!(pPage->flags & PAGE_SHORT) ); + assert( PAGE_SHORT==4 ); +#endif + *pnData = pPage->pFS->nPagesize - (pPage->flags & PAGE_SHORT); } return pPage->aData; } /* ** Return the page number of a page. */ Pgno lsmFsPageNumber(Page *pPage){ - /* assert( (pPage->flags & PAGE_DIRTY)==0 ); */ + assert( (pPage->flags & PAGE_DIRTY)==0 ); return pPage ? pPage->iPg : 0; } /* ** Page pPg is currently part of the LRU list belonging to pFS. Remove @@ -892,142 +830,70 @@ i64 iSz, int *pRc ){ /* This function won't work with compressed databases yet. */ assert( pFS->pCompress==0 ); - assert( PAGE_HASPREV==4 ); if( *pRc==LSM_OK && iSz>pFS->nMap ){ + Page *pFix; int rc; u8 *aOld = pFS->pMap; rc = lsmEnvRemap(pFS->pEnv, pFS->fdDb, iSz, &pFS->pMap, &pFS->nMap); if( rc==LSM_OK && pFS->pMap!=aOld ){ - Page *pFix; - i64 iOff = (u8 *)pFS->pMap - aOld; + u8 *aData = (u8 *)pFS->pMap; for(pFix=pFS->pLruFirst; pFix; pFix=pFix->pLruNext){ - pFix->aData += iOff; + assert( &aOld[pFS->nPagesize * (i64)(pFix->iPg-1)]==pFix->aData ); + pFix->aData = &aData[pFS->nPagesize * (i64)(pFix->iPg-1)]; } lsmSortedRemap(pFS->pDb); } *pRc = rc; } } - -/* -** fsync() the database file. -*/ -int lsmFsSyncDb(FileSystem *pFS, int nBlock){ - if( nBlock && pFS->bUseMmap ){ - int rc = LSM_OK; - i64 nMin = (i64)nBlock * (i64)pFS->nBlocksize; - fsGrowMapping(pFS, nMin, &rc); - if( rc!=LSM_OK ) return rc; - } - return lsmEnvSync(pFS->pEnv, pFS->fdDb); -} - -static int fsPageGet(FileSystem *, Segment *, Pgno, int, Page **, int *); - -static int fsRedirectBlock(Redirect *p, int iBlk){ - if( p ){ - int i; - for(i=0; in; i++){ - if( iBlk==p->a[i].iFrom ) return p->a[i].iTo; - } - } - assert( iBlk!=0 ); - return iBlk; -} - -Pgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, Pgno iPg){ - Pgno iReal = iPg; - - if( pRedir ){ - const int nPagePerBlock = ( - pFS->pCompress ? pFS->nBlocksize : (pFS->nBlocksize / pFS->nPagesize) - ); - int iBlk = fsPageToBlock(pFS, iPg); - int i; - for(i=0; in; i++){ - int iFrom = pRedir->a[i].iFrom; - if( iFrom>iBlk ) break; - if( iFrom==iBlk ){ - int iTo = pRedir->a[i].iTo; - iReal = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock; - if( iTo==1 ){ - iReal += (fsFirstPageOnBlock(pFS, 1)-1); - } - break; - } - } - } - - assert( iReal!=0 ); - return iReal; -} +static int fsPageGet(FileSystem *, Pgno, int, Page **, int *); /* ** Parameter iBlock is a database file block. This function reads the value ** stored in the blocks "next block" pointer and stores it in *piNext. ** LSM_OK is returned if everything is successful, or an LSM error code ** otherwise. */ static int fsBlockNext( FileSystem *pFS, /* File-system object handle */ - Segment *pSeg, /* Use this segment for block redirects */ int iBlock, /* Read field from this block */ int *piNext /* OUT: Next block in linked list */ ){ int rc; - int iRead; /* Read block from here */ - - if( pSeg ){ - iRead = fsRedirectBlock(pSeg->pRedirect, iBlock); - }else{ - iRead = iBlock; - } assert( pFS->bUseMmap==0 || pFS->pCompress==0 ); if( pFS->pCompress ){ i64 iOff; /* File offset to read data from */ u8 aNext[4]; /* 4-byte pointer read from db file */ - iOff = (i64)iRead * pFS->nBlocksize - sizeof(aNext); + iOff = (i64)iBlock * pFS->nBlocksize - sizeof(aNext); rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aNext, sizeof(aNext)); if( rc==LSM_OK ){ *piNext = (int)lsmGetU32(aNext); } }else{ const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); Page *pLast; - rc = fsPageGet(pFS, 0, iRead*nPagePerBlock, 0, &pLast, 0); + rc = fsPageGet(pFS, iBlock*nPagePerBlock, 0, &pLast, 0); if( rc==LSM_OK ){ - *piNext = lsmGetU32(&pLast->aData[pFS->nPagesize-4]); + *piNext = fsPageToBlock(pFS, lsmGetU32(&pLast->aData[pFS->nPagesize-4])); lsmFsPageRelease(pLast); } } - - if( pSeg ){ - *piNext = fsRedirectBlock(pSeg->pRedirect, *piNext); - } return rc; } -/* -** Return the page number of the last page on the same block as page iPg. -*/ -Pgno fsLastPageOnPagesBlock(FileSystem *pFS, Pgno iPg){ - return fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iPg)); -} - /* ** This function is only called in compressed database mode. */ static int fsReadData( FileSystem *pFS, /* File-system handle */ - Segment *pSeg, /* Block redirection */ i64 iOff, /* Read data from this offset */ u8 *aData, /* Buffer to read data into */ int nData /* Number of bytes to read */ ){ i64 iEob; /* End of block */ @@ -1034,18 +900,18 @@ int nRead; int rc; assert( pFS->pCompress ); - iEob = fsLastPageOnPagesBlock(pFS, iOff) + 1; + iEob = fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iOff)) + 1; nRead = LSM_MIN(iEob - iOff, nData); rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aData, nRead); if( rc==LSM_OK && nRead!=nData ){ int iBlk; - rc = fsBlockNext(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); + rc = fsBlockNext(pFS, fsPageToBlock(pFS, iOff), &iBlk); if( rc==LSM_OK ){ i64 iOff2 = fsFirstPageOnBlock(pFS, iBlk); rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff2, &aData[nRead], nData-nRead); } } @@ -1059,11 +925,10 @@ ** LSM_OK is returned if everything is successful, or an LSM error code ** otherwise. */ static int fsBlockPrev( FileSystem *pFS, /* File-system object handle */ - Segment *pSeg, /* Use this segment for block redirects */ int iBlock, /* Read field from this block */ int *piPrev /* OUT: Previous block in linked list */ ){ int rc = LSM_OK; /* Return code */ @@ -1073,12 +938,11 @@ if( pFS->pCompress ){ i64 iOff = fsFirstPageOnBlock(pFS, iBlock) - 4; u8 aPrev[4]; /* 4-byte pointer read from db file */ rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aPrev, sizeof(aPrev)); if( rc==LSM_OK ){ - Redirect *pRedir = (pSeg ? pSeg->pRedirect : 0); - *piPrev = fsRedirectBlock(pRedir, (int)lsmGetU32(aPrev)); + *piPrev = (int)lsmGetU32(aPrev); } }else{ assert( 0 ); } return rc; @@ -1099,19 +963,13 @@ nByte += (aBuf[2] & 0x7F); *pbFree = !(aBuf[1] & 0x80); return nByte; } -static int fsSubtractOffset( - FileSystem *pFS, - Segment *pSeg, - i64 iOff, - int iSub, - i64 *piRes -){ +static int fsSubtractOffset(FileSystem *pFS, i64 iOff, int iSub, i64 *piRes){ i64 iStart; - int iBlk = 0; + int iBlk; int rc; assert( pFS->pCompress ); iStart = fsFirstPageOnBlock(pFS, fsPageToBlock(pFS, iOff)); @@ -1118,60 +976,43 @@ if( (iOff-iSub)>=iStart ){ *piRes = (iOff-iSub); return LSM_OK; } - rc = fsBlockPrev(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); + rc = fsBlockPrev(pFS, fsPageToBlock(pFS, iOff), &iBlk); *piRes = fsLastPageOnBlock(pFS, iBlk) - iSub + (iOff - iStart + 1); return rc; } -static int fsAddOffset( - FileSystem *pFS, - Segment *pSeg, - i64 iOff, - int iAdd, - i64 *piRes -){ +static int fsAddOffset(FileSystem *pFS, i64 iOff, int iAdd, i64 *piRes){ i64 iEob; int iBlk; int rc; assert( pFS->pCompress ); - iEob = fsLastPageOnPagesBlock(pFS, iOff); + iEob = fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iOff)); if( (iOff+iAdd)<=iEob ){ *piRes = (iOff+iAdd); return LSM_OK; } - rc = fsBlockNext(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); + rc = fsBlockNext(pFS, fsPageToBlock(pFS, iOff), &iBlk); *piRes = fsFirstPageOnBlock(pFS, iBlk) + iAdd - (iEob - iOff + 1); return rc; } -static int fsAllocateBuffer(FileSystem *pFS, int bWrite){ - u8 **pp; /* Pointer to either aIBuffer or aOBuffer */ - +static int fsAllocateBuffer(FileSystem *pFS){ assert( pFS->pCompress ); - - /* If neither buffer has been allocated, figure out how large they - ** should be. Store this value in FileSystem.nBuffer. */ - if( pFS->nBuffer==0 ){ - assert( pFS->aIBuffer==0 && pFS->aOBuffer==0 ); + if( pFS->aBuffer==0 ){ pFS->nBuffer = pFS->pCompress->xBound(pFS->pCompress->pCtx, pFS->nPagesize); if( pFS->nBuffer<(pFS->szSector+6) ){ pFS->nBuffer = pFS->szSector+6; } - } - - pp = (bWrite ? &pFS->aOBuffer : &pFS->aIBuffer); - if( *pp==0 ){ - *pp = lsmMalloc(pFS->pEnv, LSM_MAX(pFS->nBuffer, pFS->nPagesize)); - if( *pp==0 ) return LSM_NOMEM_BKPT; - } - + pFS->aBuffer = lsmMalloc(pFS->pEnv, LSM_MAX(pFS->nBuffer, pFS->nPagesize)); + if( pFS->aBuffer==0 ) return LSM_NOMEM_BKPT; + } return LSM_OK; } /* ** This function is only called in compressed database mode. It reads and @@ -1180,11 +1021,10 @@ ** ** LSM_OK is returned if successful, or an LSM error code otherwise. */ static int fsReadPagedata( FileSystem *pFS, /* File-system handle */ - Segment *pSeg, /* pPg is part of this segment */ Page *pPg, /* Page to read and uncompress data for */ int *pnSpace /* OUT: Total bytes of free space */ ){ lsm_compress *p = pFS->pCompress; i64 iOff = pPg->iPg; @@ -1191,43 +1031,38 @@ u8 aSz[3]; int rc; assert( p && pPg->nCompress==0 ); - if( fsAllocateBuffer(pFS, 0) ) return LSM_NOMEM; + if( fsAllocateBuffer(pFS) ) return LSM_NOMEM; - rc = fsReadData(pFS, pSeg, iOff, aSz, sizeof(aSz)); + rc = fsReadData(pFS, iOff, aSz, sizeof(aSz)); if( rc==LSM_OK ){ int bFree; - if( aSz[0] & 0x80 ){ - pPg->nCompress = (int)getRecordSize(aSz, &bFree); - }else{ - pPg->nCompress = (int)aSz[0] - sizeof(aSz)*2; - bFree = 1; - } + pPg->nCompress = (int)getRecordSize(aSz, &bFree); if( bFree ){ if( pnSpace ){ *pnSpace = pPg->nCompress + sizeof(aSz)*2; }else{ rc = LSM_CORRUPT_BKPT; } }else{ - rc = fsAddOffset(pFS, pSeg, iOff, 3, &iOff); + rc = fsAddOffset(pFS, iOff, 3, &iOff); if( rc==LSM_OK ){ if( pPg->nCompress>pFS->nBuffer ){ rc = LSM_CORRUPT_BKPT; }else{ - rc = fsReadData(pFS, pSeg, iOff, pFS->aIBuffer, pPg->nCompress); + rc = fsReadData(pFS, iOff, pFS->aBuffer, pPg->nCompress); } if( rc==LSM_OK ){ int n = pFS->nPagesize; rc = p->xUncompress(p->pCtx, (char *)pPg->aData, &n, - (const char *)pFS->aIBuffer, pPg->nCompress - ); - if( rc==LSM_OK && n!=pPg->pFS->nPagesize ){ + (const char *)pFS->aBuffer, pPg->nCompress + ); + if( rc==LSM_OK && n!=pPg->nData ){ rc = LSM_CORRUPT_BKPT; } } } } @@ -1238,46 +1073,27 @@ /* ** Return a handle for a database page. */ static int fsPageGet( FileSystem *pFS, /* File-system handle */ - Segment *pSeg, /* Block redirection to use (or NULL) */ Pgno iPg, /* Page id */ int noContent, /* True to not load content from disk */ Page **ppPg, /* OUT: New page handle */ int *pnSpace /* OUT: Bytes of free space */ ){ Page *p; int iHash; int rc = LSM_OK; - /* In most cases iReal is the same as iPg. Except, if pSeg->pRedirect is - ** not NULL, and the block containing iPg has been redirected, then iReal - ** is the page number after redirection. */ - Pgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg); - assert( iPg>=fsFirstPageOnBlock(pFS, 1) ); - assert( iReal>=fsFirstPageOnBlock(pFS, 1) ); *ppPg = 0; - assert( pFS->bUseMmap==0 || pFS->pCompress==0 ); if( pFS->bUseMmap ){ - Page *pTest; - i64 iEnd = (i64)iReal * pFS->nPagesize; + i64 iEnd = (i64)iPg * pFS->nPagesize; fsGrowMapping(pFS, iEnd, &rc); if( rc!=LSM_OK ) return rc; - p = 0; - for(pTest=pFS->pWaiting; pTest; pTest=pTest->pNextWaiting){ - if( pTest->iPg==iReal ){ - assert( iReal==iPg ); - p = pTest; - p->nRef++; - *ppPg = p; - return LSM_OK; - } - } if( pFS->pFree ){ p = pFS->pFree; pFS->pFree = p->pHashNext; assert( p->nRef==0 ); }else{ @@ -1284,39 +1100,49 @@ p = lsmMallocZeroRc(pFS->pEnv, sizeof(Page), &rc); if( rc ) return rc; fsPageAddToLru(pFS, p); p->pFS = pFS; } - p->aData = &((u8 *)pFS->pMap)[pFS->nPagesize * (iReal-1)]; - p->iPg = iReal; + p->aData = &((u8 *)pFS->pMap)[pFS->nPagesize * (i64)(iPg-1)]; + p->iPg = iPg; + if( fsIsLast(pFS, iPg) || fsIsFirst(pFS, iPg) ){ + p->flags = PAGE_SHORT; + }else{ + p->flags = 0; + } + p->nData = pFS->nPagesize - (p->flags & PAGE_SHORT); }else{ /* Search the hash-table for the page */ - iHash = fsHashKey(pFS->nHash, iReal); + iHash = fsHashKey(pFS->nHash, iPg); for(p=pFS->apHash[iHash]; p; p=p->pHashNext){ - if( p->iPg==iReal) break; + if( p->iPg==iPg) break; } if( p==0 ){ rc = fsPageBuffer(pFS, 1, &p); if( rc==LSM_OK ){ int nSpace = 0; - p->iPg = iReal; + p->iPg = iPg; p->nRef = 0; p->pFS = pFS; assert( p->flags==0 || p->flags==PAGE_FREE ); + if( pFS->pCompress==0 && (fsIsLast(pFS, iPg) || fsIsFirst(pFS, iPg)) ){ + p->flags |= PAGE_SHORT; + } + p->nData = pFS->nPagesize - (p->flags & PAGE_SHORT); #ifdef LSM_DEBUG memset(p->aData, 0x56, pFS->nPagesize); #endif assert( p->pLruNext==0 && p->pLruPrev==0 ); if( noContent==0 ){ if( pFS->pCompress ){ - rc = fsReadPagedata(pFS, pSeg, p, &nSpace); + rc = fsReadPagedata(pFS, p, &nSpace); }else{ int nByte = pFS->nPagesize; - i64 iOff = (i64)(iReal-1) * pFS->nPagesize; + i64 iOff = (i64)(iPg-1) * pFS->nPagesize; rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, p->aData, nByte); } pFS->nRead++; } @@ -1338,57 +1164,15 @@ assert( (rc==LSM_OK && (p || (pnSpace && *pnSpace))) || (rc!=LSM_OK && p==0) ); } - if( rc==LSM_OK && p ){ - if( pFS->pCompress==0 && (fsIsLast(pFS, iReal) || fsIsFirst(pFS, iReal)) ){ - p->nData = pFS->nPagesize - 4; - if( fsIsFirst(pFS, iReal) && p->nRef==0 ){ - p->aData += 4; - p->flags |= PAGE_HASPREV; - } - }else{ - p->nData = pFS->nPagesize; - } pFS->nOut += (p->nRef==0); p->nRef++; } *ppPg = p; - return rc; -} - -/* -** Read the 64-bit checkpoint id of the checkpoint currently stored on meta -** page iMeta of the database file. If no error occurs, store the id value -** in *piVal and return LSM_OK. Otherwise, return an LSM error code and leave -** *piVal unmodified. -** -** If a checkpointer connection is currently updating meta-page iMeta, or an -** earlier checkpointer crashed while doing so, the value read into *piVal -** may be garbage. It is the callers responsibility to deal with this. -*/ -int lsmFsReadSyncedId(lsm_db *db, int iMeta, i64 *piVal){ - FileSystem *pFS = db->pFS; - int rc = LSM_OK; - - assert( iMeta==1 || iMeta==2 ); - if( pFS->bUseMmap ){ - fsGrowMapping(pFS, iMeta*LSM_META_PAGE_SIZE, &rc); - if( rc==LSM_OK ){ - *piVal = (i64)lsmGetU64(&((u8 *)pFS->pMap)[(iMeta-1)*LSM_META_PAGE_SIZE]); - } - }else{ - MetaPage *pMeta = 0; - rc = lsmFsMetaPageGet(pFS, 0, iMeta, &pMeta); - if( rc==LSM_OK ){ - *piVal = (i64)lsmGetU64(pMeta->aData); - lsmFsMetaPageRelease(pMeta); - } - } - return rc; } static int fsRunEndsBetween( @@ -1482,23 +1266,18 @@ /* Mark all blocks currently used by this sorted run as free */ while( iBlk && rc==LSM_OK ){ int iNext = 0; if( iBlk!=iLastBlk ){ - rc = fsBlockNext(pFS, pDel, iBlk, &iNext); + rc = fsBlockNext(pFS, iBlk, &iNext); }else if( bZero==0 && pDel->iLastPg!=fsLastPageOnBlock(pFS, iLastBlk) ){ break; } rc = fsFreeBlock(pFS, pSnapshot, pDel, iBlk); iBlk = iNext; } - if( pDel->pRedirect ){ - assert( pDel->pRedirect==&pSnapshot->redirect ); - pSnapshot->redirect.n = 0; - } - if( bZero ) memset(pDel, 0, sizeof(Segment)); } return LSM_OK; } @@ -1512,32 +1291,10 @@ } } return iRet; } -#ifndef NDEBUG -/* -** Return true if page iPg, which is a part of segment p, lies on -** a redirected block. -*/ -static int fsPageRedirects(FileSystem *pFS, Segment *p, Pgno iPg){ - return (iPg!=0 && iPg!=lsmFsRedirectPage(pFS, p->pRedirect, iPg)); -} - -/* -** Return true if the second argument is not NULL and any of the first -** last or root pages lie on a redirected block. -*/ -static int fsSegmentRedirects(FileSystem *pFS, Segment *p){ - return (p && ( - fsPageRedirects(pFS, p, p->iFirst) - || fsPageRedirects(pFS, p, p->iRoot) - || fsPageRedirects(pFS, p, p->iLastPg) - )); -} -#endif - /* ** Argument aPgno is an array of nPgno page numbers. All pages belong to ** the segment pRun. This function gobbles from the start of the run to the ** first page that appears in aPgno[] (i.e. so that the aPgno[] entry is ** the new first page of the run). @@ -1552,13 +1309,10 @@ FileSystem *pFS = pDb->pFS; Snapshot *pSnapshot = pDb->pWorker; int iBlk; assert( pRun->nSize>0 ); - assert( 0==fsSegmentRedirects(pFS, pRun) ); - assert( nPgno>0 && 0==fsPageRedirects(pFS, pRun, aPgno[0]) ); - iBlk = fsPageToBlock(pFS, pRun->iFirst); pRun->nSize += (pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk)); while( rc==LSM_OK ){ int iNext = 0; @@ -1565,11 +1319,11 @@ Pgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno); if( iFirst ){ pRun->iFirst = iFirst; break; } - rc = fsBlockNext(pFS, pRun, iBlk, &iNext); + rc = fsBlockNext(pFS, iBlk, &iNext); if( rc==LSM_OK ) rc = fsFreeBlock(pFS, pSnapshot, pRun, iBlk); pRun->nSize -= ( 1 + fsLastPageOnBlock(pFS, iBlk) - fsFirstPageOnBlock(pFS, iBlk) ); iBlk = iNext; @@ -1577,24 +1331,10 @@ pRun->nSize -= (pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk)); assert( pRun->nSize>0 ); } -/* -** This function is only used in compressed database mode. -** -** Argument iPg is the page number (byte offset) of a page within segment -** pSeg. The page record, including all headers, is nByte bytes in size. -** Before returning, set *piNext to the page number of the next page in -** the segment, or to zero if iPg is the last. -** -** In other words, do: -** -** *piNext = iPg + nByte; -** -** But take block overflow and redirection into account. -*/ static int fsNextPageOffset( FileSystem *pFS, /* File system object */ Segment *pSeg, /* Segment to move within */ Pgno iPg, /* Offset of current page */ int nByte, /* Size of current page including headers */ @@ -1603,44 +1343,32 @@ Pgno iNext; int rc; assert( pFS->pCompress ); - rc = fsAddOffset(pFS, pSeg, iPg, nByte-1, &iNext); + rc = fsAddOffset(pFS, iPg, nByte-1, &iNext); if( pSeg && iNext==pSeg->iLastPg ){ iNext = 0; }else if( rc==LSM_OK ){ - rc = fsAddOffset(pFS, pSeg, iNext, 1, &iNext); + rc = fsAddOffset(pFS, iNext, 1, &iNext); } *piNext = iNext; return rc; } -static int fsGetPageBefore( - FileSystem *pFS, - Segment *pSeg, - i64 iOff, - Pgno *piPrev -){ +static int fsGetPageBefore(FileSystem *pFS, i64 iOff, Pgno *piPrev){ u8 aSz[3]; int rc; i64 iRead; - rc = fsSubtractOffset(pFS, pSeg, iOff, sizeof(aSz), &iRead); - if( rc==LSM_OK ) rc = fsReadData(pFS, pSeg, iRead, aSz, sizeof(aSz)); - + rc = fsSubtractOffset(pFS, iOff, sizeof(aSz), &iRead); + if( rc==LSM_OK ) rc = fsReadData(pFS, iRead, aSz, sizeof(aSz)); if( rc==LSM_OK ){ int bFree; - int nSz; - if( aSz[2] & 0x80 ){ - nSz = getRecordSize(aSz, &bFree) + sizeof(aSz)*2; - }else{ - nSz = (int)(aSz[2] & 0x7F); - bFree = 1; - } - rc = fsSubtractOffset(pFS, pSeg, iOff, nSz, piPrev); + int nSz = getRecordSize(aSz, &bFree); + rc = fsSubtractOffset(pFS, iOff, nSz + sizeof(aSz)*2, piPrev); } return rc; } @@ -1668,11 +1396,10 @@ int lsmFsDbPageNext(Segment *pRun, Page *pPg, int eDir, Page **ppNext){ int rc = LSM_OK; FileSystem *pFS = pPg->pFS; Pgno iPg = pPg->iPg; - assert( 0==fsSegmentRedirects(pFS, pRun) ); if( pFS->pCompress ){ int nSpace = pPg->nCompress + 2*3; do { if( eDir>0 ){ @@ -1679,104 +1406,80 @@ rc = fsNextPageOffset(pFS, pRun, iPg, nSpace, &iPg); }else{ if( iPg==pRun->iFirst ){ iPg = 0; }else{ - rc = fsGetPageBefore(pFS, pRun, iPg, &iPg); + rc = fsGetPageBefore(pFS, iPg, &iPg); } } nSpace = 0; if( iPg!=0 ){ - rc = fsPageGet(pFS, pRun, iPg, 0, ppNext, &nSpace); + rc = fsPageGet(pFS, iPg, 0, ppNext, &nSpace); assert( (*ppNext==0)==(rc!=LSM_OK || nSpace>0) ); }else{ *ppNext = 0; } }while( nSpace>0 && rc==LSM_OK ); }else{ - Redirect *pRedir = pRun ? pRun->pRedirect : 0; assert( eDir==1 || eDir==-1 ); if( eDir<0 ){ if( pRun && iPg==pRun->iFirst ){ *ppNext = 0; return LSM_OK; }else if( fsIsFirst(pFS, iPg) ){ - assert( pPg->flags & PAGE_HASPREV ); - iPg = fsLastPageOnBlock(pFS, lsmGetU32(&pPg->aData[-4])); + iPg = lsmGetU32(&pPg->aData[pFS->nPagesize-4]); }else{ iPg--; } }else{ - if( pRun ){ - if( iPg==pRun->iLastPg ){ - *ppNext = 0; - return LSM_OK; - } - } - - if( fsIsLast(pFS, iPg) ){ - int iBlk = fsRedirectBlock( - pRedir, lsmGetU32(&pPg->aData[pFS->nPagesize-4]) - ); - iPg = fsFirstPageOnBlock(pFS, iBlk); + if( pRun && iPg==pRun->iLastPg ){ + *ppNext = 0; + return LSM_OK; + }else if( fsIsLast(pFS, iPg) ){ + iPg = lsmGetU32(&pPg->aData[pFS->nPagesize-4]); }else{ iPg++; } } - rc = fsPageGet(pFS, pRun, iPg, 0, ppNext, 0); + rc = fsPageGet(pFS, iPg, 0, ppNext, 0); } return rc; } -static Pgno findAppendPoint(FileSystem *pFS, Level *pLvl){ +static Pgno findAppendPoint(FileSystem *pFS){ int i; Pgno *aiAppend = pFS->pDb->pWorker->aiAppend; u32 iRet = 0; for(i=LSM_APPLIST_SZ-1; iRet==0 && i>=0; i--){ - if( (iRet = aiAppend[i]) ){ - if( pLvl ){ - int iBlk = fsPageToBlock(pFS, iRet); - int j; - for(j=0; iRet && jnRight; j++){ - if( fsPageToBlock(pFS, pLvl->aRhs[j].iLastPg)==iBlk ){ - iRet = 0; - } - } - } - if( iRet ) aiAppend[i] = 0; - } + if( (iRet = aiAppend[i]) ) aiAppend[i] = 0; } return iRet; } /* -** Append a page to the left-hand-side of pLvl. Set the ref-count to 1 and -** return a pointer to it. The page is writable until either -** lsmFsPagePersist() is called on it or the ref-count drops to zero. +** Append a page to file iFile. Set the ref-count to 1 and return a pointer +** to it. The page is writable until either lsmFsPagePersist() is called on +** it or the ref-count drops to zero. */ int lsmFsSortedAppend( FileSystem *pFS, Snapshot *pSnapshot, - Level *pLvl, - int bDefer, + Segment *p, Page **ppOut ){ int rc = LSM_OK; Page *pPg = 0; *ppOut = 0; int iApp = 0; int iNext = 0; - Segment *p = &pLvl->lhs; int iPrev = p->iLastPg; - assert( p->pRedirect==0 ); - - if( pFS->pCompress || bDefer ){ + if( pFS->pCompress ){ /* In compressed database mode the page is not assigned a page number ** or location in the database file at this point. This will be done ** by the lsmFsPagePersist() call. */ rc = fsPageBuffer(pFS, 1, &pPg); if( rc==LSM_OK ){ @@ -1784,44 +1487,42 @@ pPg->pSeg = p; pPg->iPg = 0; pPg->flags |= PAGE_DIRTY; pPg->nData = pFS->nPagesize; assert( pPg->aData ); - if( pFS->pCompress==0 ) pPg->nData -= 4; pPg->nRef = 1; pFS->nOut++; } }else{ if( iPrev==0 ){ - iApp = findAppendPoint(pFS, pLvl); + iApp = findAppendPoint(pFS); }else if( fsIsLast(pFS, iPrev) ){ int iNext; - rc = fsBlockNext(pFS, 0, fsPageToBlock(pFS, iPrev), &iNext); + rc = fsBlockNext(pFS, fsPageToBlock(pFS, iPrev), &iNext); if( rc!=LSM_OK ) return rc; iApp = fsFirstPageOnBlock(pFS, iNext); }else{ iApp = iPrev + 1; } /* If this is the first page allocated, or if the page allocated is the - ** last in the block, also allocate the next block here. */ + ** last in the block, allocate a new block here. */ if( iApp==0 || fsIsLast(pFS, iApp) ){ int iNew; /* New block number */ - rc = lsmBlockAllocate(pFS->pDb, 0, &iNew); - if( rc!=LSM_OK ) return rc; + lsmBlockAllocate(pFS->pDb, &iNew); if( iApp==0 ){ iApp = fsFirstPageOnBlock(pFS, iNew); }else{ iNext = fsFirstPageOnBlock(pFS, iNew); } } /* Grab the new page. */ pPg = 0; - rc = fsPageGet(pFS, 0, iApp, 1, &pPg, 0); + rc = fsPageGet(pFS, iApp, 1, &pPg, 0); assert( rc==LSM_OK || pPg==0 ); /* If this is the first or last page of a block, fill in the pointer ** value at the end of the new page. */ if( rc==LSM_OK ){ @@ -1829,14 +1530,15 @@ p->iLastPg = iApp; if( p->iFirst==0 ) p->iFirst = iApp; pPg->flags |= PAGE_DIRTY; if( fsIsLast(pFS, iApp) ){ - lsmPutU32(&pPg->aData[pFS->nPagesize-4], fsPageToBlock(pFS, iNext)); - }else if( fsIsFirst(pFS, iApp) ){ - lsmPutU32(&pPg->aData[-4], fsPageToBlock(pFS, iPrev)); - } + lsmPutU32(&pPg->aData[pFS->nPagesize-4], iNext); + }else + if( fsIsFirst(pFS, iApp) ){ + lsmPutU32(&pPg->aData[pFS->nPagesize-4], iPrev); + } } } *ppOut = pPg; return rc; @@ -1846,20 +1548,21 @@ ** Mark the sorted run passed as the second argument as finished. */ int lsmFsSortedFinish(FileSystem *pFS, Segment *p){ int rc = LSM_OK; if( p && p->iLastPg ){ - assert( p->pRedirect==0 ); + int iBlk; /* Check if the last page of this run happens to be the last of a block. ** If it is, then an extra block has already been allocated for this run. ** Shift this extra block back to the free-block list. ** ** Otherwise, add the first free page in the last block used by the run ** to the lAppend list. */ - if( fsLastPageOnPagesBlock(pFS, p->iLastPg)!=p->iLastPg ){ + iBlk = fsPageToBlock(pFS, p->iLastPg); + if( fsLastPageOnBlock(pFS, fsPageToBlock(pFS, p->iLastPg) )!=p->iLastPg ){ int i; Pgno *aiAppend = pFS->pDb->pWorker->aiAppend; for(i=0; iiLastPg+1; @@ -1866,19 +1569,19 @@ break; } } }else if( pFS->pCompress==0 ){ Page *pLast; - rc = fsPageGet(pFS, 0, p->iLastPg, 0, &pLast, 0); + rc = fsPageGet(pFS, p->iLastPg, 0, &pLast, 0); if( rc==LSM_OK ){ - int iBlk = (int)lsmGetU32(&pLast->aData[pFS->nPagesize-4]); - lsmBlockRefree(pFS->pDb, iBlk); + int iPg = (int)lsmGetU32(&pLast->aData[pFS->nPagesize-4]); + lsmBlockRefree(pFS->pDb, fsPageToBlock(pFS, iPg)); lsmFsPageRelease(pLast); } }else{ int iBlk = 0; - rc = fsBlockNext(pFS, p, fsPageToBlock(pFS, p->iLastPg), &iBlk); + rc = fsBlockNext(pFS, fsPageToBlock(pFS, p->iLastPg), &iBlk); if( rc==LSM_OK ){ lsmBlockRefree(pFS->pDb, iBlk); } } } @@ -1886,13 +1589,13 @@ } /* ** Obtain a reference to page number iPg. */ -int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, Pgno iPg, Page **ppPg){ +int lsmFsDbPageGet(FileSystem *pFS, Pgno iPg, Page **ppPg){ assert( pFS ); - return fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0); + return fsPageGet(pFS, iPg, 0, ppPg, 0); } /* ** Obtain a reference to the last page in the segment passed as the ** second argument. @@ -1903,18 +1606,18 @@ if( pFS->pCompress ){ int nSpace; iPg++; do { nSpace = 0; - rc = fsGetPageBefore(pFS, pSeg, iPg, &iPg); + rc = fsGetPageBefore(pFS, iPg, &iPg); if( rc==LSM_OK ){ - rc = fsPageGet(pFS, pSeg, iPg, 0, ppPg, &nSpace); + rc = fsPageGet(pFS, iPg, 0, ppPg, &nSpace); } }while( rc==LSM_OK && nSpace>0 ); }else{ - rc = fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0); + rc = fsPageGet(pFS, iPg, 0, ppPg, 0); } return rc; } /* @@ -2011,85 +1714,10 @@ */ int lsmFsPageWritable(Page *pPg){ return (pPg->flags & PAGE_DIRTY) ? 1 : 0; } -static void fsMovePage( - FileSystem *pFS, - Segment *pSeg, - int iTo, - int iFrom, - Pgno *piPg -){ - Pgno iPg = *piPg; - if( iFrom==fsPageToBlock(pFS, iPg) ){ - const int nPagePerBlock = ( - pFS->pCompress ? pFS ->nBlocksize : (pFS->nBlocksize / pFS->nPagesize) - ); - *piPg = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock; - } -} - -/* -** Copy the contents of block iFrom to block iTo. -** -** It is safe to assume that there are no outstanding references to pages -** on block iTo. And that block iFrom is not currently being written. In -** other words, the data can be read and written directly. -*/ -int lsmFsMoveBlock(FileSystem *pFS, Segment *pSeg, int iTo, int iFrom){ - Snapshot *p = pFS->pDb->pWorker; - int rc = LSM_OK; - - i64 iFromOff = (i64)(iFrom-1) * pFS->nBlocksize; - i64 iToOff = (i64)(iTo-1) * pFS->nBlocksize; - - assert( iTo!=1 ); - assert( iFrom>iTo ); - - if( pFS->bUseMmap ){ - fsGrowMapping(pFS, (i64)iFrom * pFS->nBlocksize, &rc); - if( rc==LSM_OK ){ - u8 *aMap = (u8 *)(pFS->pMap); - memcpy(&aMap[iToOff], &aMap[iFromOff], pFS->nBlocksize); - } - }else{ - int nSz = pFS->nPagesize; - u8 *aData = (u8 *)lsmMallocRc(pFS->pEnv, nSz, &rc); - if( rc==LSM_OK ){ - const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - int i; - for(i=0; rc==LSM_OK && ipEnv, pFS->fdDb, iOff, aData, nSz); - if( rc==LSM_OK ){ - iOff = iToOff + i*nSz; - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, aData, nSz); - } - } - } - } - - /* Update append-point list if necessary */ - if( rc==LSM_OK ){ - int i; - for(i=0; iaiAppend[i])==iFrom ){ - const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - p->aiAppend[i] -= (i64)(iFrom-iTo) * nPagePerBlock; - } - } - } - - /* Update the Segment structure itself */ - fsMovePage(pFS, pSeg, iTo, iFrom, &pSeg->iFirst); - fsMovePage(pFS, pSeg, iTo, iFrom, &pSeg->iLastPg); - fsMovePage(pFS, pSeg, iTo, iFrom, &pSeg->iRoot); - - return rc; -} - /* ** Append raw data to a segment. This function is only used in compressed ** database mode. */ static Pgno fsAppendData( @@ -2109,21 +1737,21 @@ Pgno iApp = pSeg->iLastPg+1; /* If this is the first data written into the segment, find an append-point ** or allocate a new block. */ if( iApp==1 ){ - pSeg->iFirst = iApp = findAppendPoint(pFS, 0); + pSeg->iFirst = iApp = findAppendPoint(pFS); if( iApp==0 ){ int iBlk; - rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); + rc = lsmBlockAllocate(pFS->pDb, &iBlk); pSeg->iFirst = iApp = fsFirstPageOnBlock(pFS, iBlk); } } iRet = iApp; /* Write as much data as is possible at iApp (usually all of it). */ - iLastOnBlock = fsLastPageOnPagesBlock(pFS, iApp); + iLastOnBlock = fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iApp)); if( rc==LSM_OK ){ int nSpace = iLastOnBlock - iApp + 1; nWrite = LSM_MIN(nData, nSpace); nRem = nData - nWrite; assert( nWrite>=0 ); @@ -2141,11 +1769,11 @@ u8 aPtr[4]; /* Space to serialize a u32 */ int iBlk; /* New block number */ if( nWrite>0 ){ /* Allocate a new block. */ - rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); + rc = lsmBlockAllocate(pFS->pDb, &iBlk); /* Set the "next" pointer on the old block */ if( rc==LSM_OK ){ assert( iApp==(fsPageToBlock(pFS, iApp)*pFS->nBlocksize)-4 ); lsmPutU32(aPtr, iBlk); @@ -2161,13 +1789,12 @@ if( nRem>0 ) iApp = iWrite; } }else{ /* The next block is already allocated. */ assert( nRem>0 ); - assert( pSeg->pRedirect==0 ); - rc = fsBlockNext(pFS, 0, fsPageToBlock(pFS, iApp), &iBlk); - iRet = iApp = fsFirstPageOnBlock(pFS, iBlk); + rc = fsBlockNext(pFS, fsPageToBlock(pFS, iApp), &iBlk); + iApp = fsFirstPageOnBlock(pFS, iBlk); } /* Write the remaining data into the new block */ if( rc==LSM_OK && nRem>0 ){ rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, &aData[nWrite], nRem); @@ -2192,79 +1819,20 @@ ** allocates it. If this fails, LSM_NOMEM is returned. Otherwise, LSM_OK. */ static int fsCompressIntoBuffer(FileSystem *pFS, Page *pPg){ lsm_compress *p = pFS->pCompress; - if( fsAllocateBuffer(pFS, 1) ) return LSM_NOMEM; + if( fsAllocateBuffer(pFS) ) return LSM_NOMEM; assert( pPg->nData==pFS->nPagesize ); pPg->nCompress = pFS->nBuffer; return p->xCompress(p->pCtx, - (char *)pFS->aOBuffer, &pPg->nCompress, + (char *)pFS->aBuffer, &pPg->nCompress, (const char *)pPg->aData, pPg->nData ); } -static int fsAppendPage( - FileSystem *pFS, - Segment *pSeg, - Pgno *piNew, - int *piPrev, - int *piNext -){ - Pgno iPrev = pSeg->iLastPg; - int rc; - assert( iPrev!=0 ); - - *piPrev = 0; - *piNext = 0; - - if( fsIsLast(pFS, iPrev) ){ - /* Grab the first page on the next block (which has already be - ** allocated). In this case set *piPrev to tell the caller to set - ** the "previous block" pointer in the first 4 bytes of the page. - */ - int iNext; - int iBlk = fsPageToBlock(pFS, iPrev); - assert( pSeg->pRedirect==0 ); - rc = fsBlockNext(pFS, 0, iBlk, &iNext); - if( rc!=LSM_OK ) return rc; - *piNew = fsFirstPageOnBlock(pFS, iNext); - *piPrev = iBlk; - }else{ - *piNew = iPrev+1; - if( fsIsLast(pFS, *piNew) ){ - /* Allocate the next block here. */ - int iBlk; - rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); - if( rc!=LSM_OK ) return rc; - *piNext = iBlk; - } - } - - pSeg->nSize++; - pSeg->iLastPg = *piNew; - return LSM_OK; -} - -void lsmFsFlushWaiting(FileSystem *pFS, int *pRc){ - int rc = *pRc; - Page *pPg; - - pPg = pFS->pWaiting; - pFS->pWaiting = 0; - - while( pPg ){ - Page *pNext = pPg->pNextWaiting; - if( rc==LSM_OK ) rc = lsmFsPagePersist(pPg); - assert( pPg->nRef==1 ); - lsmFsPageRelease(pPg); - pPg = pNext; - } - *pRc = rc; -} - /* ** If the page passed as an argument is dirty, update the database file ** (or mapping of the database file) with its current contents and mark ** the page as clean. ** @@ -2287,110 +1855,46 @@ /* Serialize the compressed size into buffer aSz[] */ putRecordSize(aSz, pPg->nCompress, 0); /* Write the serialized page record into the database file. */ pPg->iPg = fsAppendData(pFS, pPg->pSeg, aSz, sizeof(aSz), &rc); - fsAppendData(pFS, pPg->pSeg, pFS->aOBuffer, pPg->nCompress, &rc); + fsAppendData(pFS, pPg->pSeg, pFS->aBuffer, pPg->nCompress, &rc); fsAppendData(pFS, pPg->pSeg, aSz, sizeof(aSz), &rc); /* Now that it has a page number, insert the page into the hash table */ iHash = fsHashKey(pFS->nHash, pPg->iPg); pPg->pHashNext = pFS->apHash[iHash]; pFS->apHash[iHash] = pPg; pPg->pSeg->nSize += (sizeof(aSz) * 2) + pPg->nCompress; - pPg->flags &= ~PAGE_DIRTY; - pFS->nWrite++; - }else{ - - if( pPg->iPg==0 ){ - /* No page number has been assigned yet. This occurs with pages used - ** in the b-tree hierarchy. They were not assigned page numbers when - ** they were created as doing so would cause this call to - ** lsmFsPagePersist() to write an out-of-order page. Instead a page - ** number is assigned here so that the page data will be appended - ** to the current segment. - */ - Page **pp; - int iPrev = 0; - int iNext = 0; - - assert( pPg->pSeg->iFirst ); - assert( pPg->flags & PAGE_FREE ); - assert( (pPg->flags & PAGE_HASPREV)==0 ); - assert( pPg->nData==pFS->nPagesize-4 ); - - rc = fsAppendPage(pFS, pPg->pSeg, &pPg->iPg, &iPrev, &iNext); - if( rc!=LSM_OK ) return rc; - - if( pFS->bUseMmap==0 ){ - int iHash = fsHashKey(pFS->nHash, pPg->iPg); - pPg->pHashNext = pFS->apHash[iHash]; - pFS->apHash[iHash] = pPg; - assert( pPg->pHashNext==0 || pPg->pHashNext->iPg!=pPg->iPg ); - } - - if( iPrev ){ - assert( iNext==0 ); - memmove(&pPg->aData[4], pPg->aData, pPg->nData); - lsmPutU32(pPg->aData, iPrev); - pPg->flags |= PAGE_HASPREV; - pPg->aData += 4; - }else if( iNext ){ - assert( iPrev==0 ); - lsmPutU32(&pPg->aData[pPg->nData], iNext); - }else{ - int nData = pPg->nData; - pPg->nData += 4; - lsmSortedExpandBtreePage(pPg, nData); - } - - pPg->nRef++; - for(pp=&pFS->pWaiting; *pp; pp=&(*pp)->pNextWaiting); - *pp = pPg; - assert( pPg->pNextWaiting==0 ); - - }else{ - i64 iOff; /* Offset to write within database file */ - - iOff = (i64)pFS->nPagesize * (i64)(pPg->iPg-1); - if( pFS->bUseMmap==0 ){ - u8 *aData = pPg->aData - (pPg->flags & PAGE_HASPREV); - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, aData, pFS->nPagesize); - }else if( pPg->flags & PAGE_FREE ){ - fsGrowMapping(pFS, iOff + pFS->nPagesize, &rc); - if( rc==LSM_OK ){ - u8 *aTo = &((u8 *)(pFS->pMap))[iOff]; - u8 *aFrom = pPg->aData - (pPg->flags & PAGE_HASPREV); - memcpy(aTo, aFrom, pFS->nPagesize); - lsmFree(pFS->pEnv, aFrom); - pPg->aData = aTo + (pPg->flags & PAGE_HASPREV); - pPg->flags &= ~PAGE_FREE; - fsPageAddToLru(pFS, pPg); - } - } - - lsmFsFlushWaiting(pFS, &rc); - pPg->flags &= ~PAGE_DIRTY; - pFS->nWrite++; - } - } + }else{ + i64 iOff; /* Offset to write within database file */ + iOff = (i64)pFS->nPagesize * (i64)(pPg->iPg-1); + if( pFS->bUseMmap==0 ){ + rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, pPg->aData,pFS->nPagesize); + }else if( pPg->flags & PAGE_FREE ){ + fsGrowMapping(pFS, iOff + pFS->nPagesize, &rc); + if( rc==LSM_OK ){ + u8 *aTo = &((u8 *)(pFS->pMap))[iOff]; + memcpy(aTo, pPg->aData, pFS->nPagesize); + lsmFree(pFS->pEnv, pPg->aData); + pPg->aData = aTo; + pPg->flags &= ~PAGE_FREE; + fsPageAddToLru(pFS, pPg); + } + } + } + pPg->flags &= ~PAGE_DIRTY; + pFS->nWrite++; } return rc; } /* -** For non-compressed databases, this function is a no-op. For compressed -** databases, it adds a padding record to the segment passed as the third -** argument. -** -** The size of the padding records is selected so that the last byte -** written is the last byte of a disk sector. This means that if a -** snapshot is taken and checkpointed, subsequent worker processes will -** not write to any sector that contains checkpointed data. +** Add a padding record to the segment passed as the third argument. */ int lsmFsSortedPadding( FileSystem *pFS, Snapshot *pSnapshot, Segment *pSeg @@ -2400,37 +1904,46 @@ Pgno iLast2; Pgno iLast = pSeg->iLastPg; /* Current last page of segment */ int nPad; /* Bytes of padding required */ u8 aSz[3]; - iLast2 = (1 + iLast/pFS->szSector) * pFS->szSector - 1; + nPad = pFS->szSector - 1 - (iLast % pFS->szSector); + if( nPad==0 + || (nPad==4 && iLast==fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iLast)) ) + ){ + return LSM_OK; + } + + iLast2 = (1 + (iLast + 6)/pFS->szSector) * pFS->szSector - 1; assert( fsPageToBlock(pFS, iLast)==fsPageToBlock(pFS, iLast2) ); nPad = iLast2 - iLast; - if( iLast2>fsLastPageOnPagesBlock(pFS, iLast) ){ + if( iLast2>fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iLast)) ){ nPad -= 4; - } - assert( nPad>=0 ); - - if( nPad>=6 ){ - pSeg->nSize += nPad; - nPad -= 6; - putRecordSize(aSz, nPad, 1); - fsAppendData(pFS, pSeg, aSz, sizeof(aSz), &rc); - memset(pFS->aOBuffer, 0, nPad); - fsAppendData(pFS, pSeg, pFS->aOBuffer, nPad, &rc); - fsAppendData(pFS, pSeg, aSz, sizeof(aSz), &rc); - }else if( nPad>0 ){ - u8 aBuf[5] = {0,0,0,0,0}; - aBuf[0] = (u8)nPad; - aBuf[nPad-1] = (u8)nPad; - fsAppendData(pFS, pSeg, aBuf, nPad, &rc); - } + if( nPad<6 ){ + nPad += (pFS->szSector - 4); + } + } + assert( nPad>=6 ); + +#if 0 + printf("padding segment with %d bytes at %d...\n", nPad, (int)iLast); +#endif + + pSeg->nSize += nPad; + nPad -= 6; + putRecordSize(aSz, nPad, 1); + + fsAppendData(pFS, pSeg, aSz, sizeof(aSz), &rc); + memset(pFS->aBuffer, 0, nPad); + fsAppendData(pFS, pSeg, pFS->aBuffer, nPad, &rc); + fsAppendData(pFS, pSeg, aSz, sizeof(aSz), &rc); + assert( rc!=LSM_OK - || pSeg->iLastPg==fsLastPageOnPagesBlock(pFS, pSeg->iLastPg) - || ((pSeg->iLastPg + 1) % pFS->szSector)==0 + || pSeg->iLastPg==fsLastPageOnBlock(pFS, fsPageToBlock(pFS, pSeg->iLastPg)) + || ((pSeg->iLastPg + 1) % pFS->szSector)==0 ); } return rc; } @@ -2452,22 +1965,15 @@ int lsmFsPageRelease(Page *pPg){ int rc = LSM_OK; if( pPg ){ assert( pPg->nRef>0 ); pPg->nRef--; - if( pPg->nRef==0 ){ + if( pPg->nRef==0 && pPg->iPg!=0 ){ FileSystem *pFS = pPg->pFS; rc = lsmFsPagePersist(pPg); pFS->nOut--; - assert( pPg->pFS->pCompress - || fsIsFirst(pPg->pFS, pPg->iPg)==0 - || (pPg->flags & PAGE_HASPREV) - ); - pPg->aData -= (pPg->flags & PAGE_HASPREV); - pPg->flags &= ~PAGE_HASPREV; - if( pFS->bUseMmap ){ pPg->pHashNext = pFS->pFree; pFS->pFree = pPg; }else{ assert( pPg->pLruNext==0 ); @@ -2488,10 +1994,17 @@ /* ** Return the total number of pages written to the database file. */ int lsmFsNWrite(FileSystem *pFS){ return pFS->nWrite; } + +/* +** fsync() the database file. +*/ +int lsmFsSyncDb(FileSystem *pFS){ + return lsmEnvSync(pFS->pEnv, pFS->fdDb); +} /* ** Return a copy of the environment pointer used by the file-system object. */ lsm_env *lsmFsEnv(FileSystem *pFS) { @@ -2522,43 +2035,23 @@ */ static Segment *startsWith(Segment *pRun, Pgno iFirst){ return (iFirst==pRun->iFirst) ? pRun : 0; } -static Segment *findSegment(Snapshot *pWorker, Pgno iFirst){ - Level *pLvl; /* Used to iterate through db levels */ - Segment *pSeg = 0; /* Pointer to segment to return */ - - for(pLvl=lsmDbSnapshotLevel(pWorker); pLvl && pSeg==0; pLvl=pLvl->pNext){ - if( 0==(pSeg = startsWith(&pLvl->lhs, iFirst)) ){ - int i; - for(i=0; inRight; i++){ - if( (pSeg = startsWith(&pLvl->aRhs[i], iFirst)) ) break; - } - } - } - - return pSeg; -} - /* ** This function implements the lsm_info(LSM_INFO_ARRAY_STRUCTURE) request. ** If successful, *pzOut is set to point to a nul-terminated string ** containing the array structure and LSM_OK is returned. The caller should ** eventually free the string using lsmFree(). ** ** If an error occurs, *pzOut is set to NULL and an LSM error code returned. */ -int lsmInfoArrayStructure( - lsm_db *pDb, - int bBlock, /* True for block numbers only */ - Pgno iFirst, - char **pzOut -){ +int lsmInfoArrayStructure(lsm_db *pDb, Pgno iFirst, char **pzOut){ int rc = LSM_OK; Snapshot *pWorker; /* Worker snapshot */ Segment *pArray = 0; /* Array to report on */ + Level *pLvl; /* Used to iterate through db levels */ int bUnlock = 0; *pzOut = 0; if( iFirst==0 ) return LSM_ERROR; @@ -2570,14 +2063,22 @@ pWorker = pDb->pWorker; bUnlock = 1; } /* Search for the array that starts on page iFirst */ - pArray = findSegment(pWorker, iFirst); + for(pLvl=lsmDbSnapshotLevel(pWorker); pLvl && pArray==0; pLvl=pLvl->pNext){ + if( 0==(pArray = startsWith(&pLvl->lhs, iFirst)) ){ + int i; + for(i=0; inRight; i++){ + if( (pArray = startsWith(&pLvl->aRhs[i], iFirst)) ) break; + } + } + } if( pArray==0 ){ /* Could not find the requested array. This is an error. */ + *pzOut = 0; rc = LSM_ERROR; }else{ FileSystem *pFS = pDb->pFS; LsmString str; int iBlk; @@ -2585,143 +2086,27 @@ iBlk = fsPageToBlock(pFS, pArray->iFirst); iLastBlk = fsPageToBlock(pFS, pArray->iLastPg); lsmStringInit(&str, pDb->pEnv); - if( bBlock ){ - lsmStringAppendf(&str, "%d", iBlk); - while( iBlk!=iLastBlk ){ - fsBlockNext(pFS, pArray, iBlk, &iBlk); - lsmStringAppendf(&str, " %d", iBlk); - } - }else{ - lsmStringAppendf(&str, "%d", pArray->iFirst); - while( iBlk!=iLastBlk ){ - lsmStringAppendf(&str, " %d", fsLastPageOnBlock(pFS, iBlk)); - fsBlockNext(pFS, pArray, iBlk, &iBlk); - lsmStringAppendf(&str, " %d", fsFirstPageOnBlock(pFS, iBlk)); - } - lsmStringAppendf(&str, " %d", pArray->iLastPg); - } + lsmStringAppendf(&str, "%d", pArray->iFirst); + while( iBlk!=iLastBlk ){ + lsmStringAppendf(&str, " %d", fsLastPageOnBlock(pFS, iBlk)); + fsBlockNext(pFS, iBlk, &iBlk); + lsmStringAppendf(&str, " %d", fsFirstPageOnBlock(pFS, iBlk)); + } + lsmStringAppendf(&str, " %d", pArray->iLastPg); *pzOut = str.z; } if( bUnlock ){ int rcwork = LSM_BUSY; - lsmFinishWork(pDb, 0, &rcwork); - } - return rc; -} - -int lsmFsSegmentContainsPg( - FileSystem *pFS, - Segment *pSeg, - Pgno iPg, - int *pbRes -){ - Redirect *pRedir = pSeg->pRedirect; - int rc = LSM_OK; - int iBlk; - int iLastBlk; - int iPgBlock; /* Block containing page iPg */ - - iPgBlock = fsPageToBlock(pFS, pSeg->iFirst); - iBlk = fsRedirectBlock(pRedir, fsPageToBlock(pFS, pSeg->iFirst)); - iLastBlk = fsRedirectBlock(pRedir, fsPageToBlock(pFS, pSeg->iLastPg)); - - while( iBlk!=iLastBlk && iBlk!=iPgBlock && rc==LSM_OK ){ - rc = fsBlockNext(pFS, pSeg, iBlk, &iBlk); - } - - *pbRes = (iBlk==iPgBlock); - return rc; -} - -/* -** This function implements the lsm_info(LSM_INFO_ARRAY_PAGES) request. -** If successful, *pzOut is set to point to a nul-terminated string -** containing the array structure and LSM_OK is returned. The caller should -** eventually free the string using lsmFree(). -** -** If an error occurs, *pzOut is set to NULL and an LSM error code returned. -*/ -int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut){ - int rc = LSM_OK; - Snapshot *pWorker; /* Worker snapshot */ - Segment *pSeg = 0; /* Array to report on */ - int bUnlock = 0; - - *pzOut = 0; - if( iFirst==0 ) return LSM_ERROR; - - /* Obtain the worker snapshot */ - pWorker = pDb->pWorker; - if( !pWorker ){ - rc = lsmBeginWork(pDb); - if( rc!=LSM_OK ) return rc; - pWorker = pDb->pWorker; - bUnlock = 1; - } - - /* Search for the array that starts on page iFirst */ - pSeg = findSegment(pWorker, iFirst); - - if( pSeg==0 ){ - /* Could not find the requested array. This is an error. */ - rc = LSM_ERROR; - }else{ - Page *pPg = 0; - FileSystem *pFS = pDb->pFS; - LsmString str; - - lsmStringInit(&str, pDb->pEnv); - rc = lsmFsDbPageGet(pFS, pSeg, iFirst, &pPg); - while( rc==LSM_OK && pPg ){ - Page *pNext = 0; - lsmStringAppendf(&str, " %lld", lsmFsPageNumber(pPg)); - rc = lsmFsDbPageNext(pSeg, pPg, 1, &pNext); - lsmFsPageRelease(pPg); - pPg = pNext; - } - - if( rc!=LSM_OK ){ - lsmFree(pDb->pEnv, str.z); - }else{ - *pzOut = str.z; - } - } - - if( bUnlock ){ - int rcwork = LSM_BUSY; - lsmFinishWork(pDb, 0, &rcwork); - } - return rc; -} - -/* -** The following macros are used by the integrity-check code. Associated with -** each block in the database is an 8-bit bit mask (the entry in the aUsed[] -** array). As the integrity-check meanders through the database, it sets the -** following bits to indicate how each block is used. -** -** INTEGRITY_CHECK_FIRST_PG: -** First page of block is in use by sorted run. -** -** INTEGRITY_CHECK_LAST_PG: -** Last page of block is in use by sorted run. -** -** INTEGRITY_CHECK_USED: -** At least one page of the block is in use by a sorted run. -** -** INTEGRITY_CHECK_FREE: -** The free block list contains an entry corresponding to this block. -*/ -#define INTEGRITY_CHECK_FIRST_PG 0x01 -#define INTEGRITY_CHECK_LAST_PG 0x02 -#define INTEGRITY_CHECK_USED 0x04 -#define INTEGRITY_CHECK_FREE 0x08 + lsmFinishWork(pDb, 0, 0, &rcwork); + } + return rc; +} /* ** Helper function for lsmFsIntegrityCheck() */ static void checkBlocks( @@ -2731,84 +2116,33 @@ int nUsed, u8 *aUsed ){ if( pSeg ){ if( pSeg && pSeg->nSize>0 ){ - int rc; - int iBlk; /* Current block (during iteration) */ - int iLastBlk; /* Last block of segment */ - int iFirstBlk; /* First block of segment */ - int bLastIsLastOnBlock; /* True iLast is the last on its block */ - - assert( 0==fsSegmentRedirects(pFS, pSeg) ); - iBlk = iFirstBlk = fsPageToBlock(pFS, pSeg->iFirst); + Pgno iLast = pSeg->iLastPg; + int iBlk; + int iLastBlk; + iBlk = fsPageToBlock(pFS, pSeg->iFirst); iLastBlk = fsPageToBlock(pFS, pSeg->iLastPg); - bLastIsLastOnBlock = (fsLastPageOnBlock(pFS, iLastBlk)==pSeg->iLastPg); - assert( iBlk>0 ); - - do { - /* iBlk is a part of this sorted run. */ - aUsed[iBlk-1] |= INTEGRITY_CHECK_USED; - - /* If the first page of this block is also part of the segment, - ** set the flag to indicate that the first page of iBlk is in use. - */ - if( fsFirstPageOnBlock(pFS, iBlk)==pSeg->iFirst || iBlk!=iFirstBlk ){ - assert( (aUsed[iBlk-1] & INTEGRITY_CHECK_FIRST_PG)==0 ); - aUsed[iBlk-1] |= INTEGRITY_CHECK_FIRST_PG; - } - - /* Unless the sorted run finishes before the last page on this block, - ** the last page of this block is also in use. */ - if( iBlk!=iLastBlk || bLastIsLastOnBlock ){ - assert( (aUsed[iBlk-1] & INTEGRITY_CHECK_LAST_PG)==0 ); - aUsed[iBlk-1] |= INTEGRITY_CHECK_LAST_PG; - } - - /* Special case. The sorted run being scanned is the output run of - ** a level currently undergoing an incremental merge. The sorted - ** run ends on the last page of iBlk, but the next block has already - ** been allocated. So mark it as in use as well. */ - if( iBlk==iLastBlk && bLastIsLastOnBlock && bExtra ){ - int iExtra = 0; - rc = fsBlockNext(pFS, pSeg, iBlk, &iExtra); - assert( rc==LSM_OK ); - - assert( aUsed[iExtra-1]==0 ); - aUsed[iExtra-1] |= INTEGRITY_CHECK_USED; - aUsed[iExtra-1] |= INTEGRITY_CHECK_FIRST_PG; - aUsed[iExtra-1] |= INTEGRITY_CHECK_LAST_PG; - } - - /* Move on to the next block in the sorted run. Or set iBlk to zero - ** in order to break out of the loop if this was the last block in - ** the run. */ - if( iBlk==iLastBlk ){ - iBlk = 0; - }else{ - rc = fsBlockNext(pFS, pSeg, iBlk, &iBlk); - assert( rc==LSM_OK ); - } - }while( iBlk ); - } - } -} - -typedef struct CheckFreelistCtx CheckFreelistCtx; -struct CheckFreelistCtx { - u8 *aUsed; - int nBlock; -}; -static int checkFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ - CheckFreelistCtx *p = (CheckFreelistCtx *)pCtx; - - assert( iBlk>=1 ); - assert( iBlk<=p->nBlock ); - assert( p->aUsed[iBlk-1]==0 ); - p->aUsed[iBlk-1] = INTEGRITY_CHECK_FREE; - return 0; + while( iBlk ){ + assert( iBlk<=nUsed ); + /* assert( aUsed[iBlk-1]==0 ); */ + aUsed[iBlk-1] = 1; + if( iBlk!=iLastBlk ){ + fsBlockNext(pFS, iBlk, &iBlk); + }else{ + iBlk = 0; + } + } + + if( bExtra && iLast==fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iLast)) ){ + fsBlockNext(pFS, iLastBlk, &iBlk); + aUsed[iBlk-1] = 1; + } + } + } } /* ** This function checks that all blocks in the database file are accounted ** for. For each block, exactly one of the following must be true: @@ -2821,32 +2155,25 @@ ** ** If no errors are found, non-zero is returned. If an error is found, an ** assert() fails. */ int lsmFsIntegrityCheck(lsm_db *pDb){ - CheckFreelistCtx ctx; FileSystem *pFS = pDb->pFS; int i; - int rc; + int j; Freelist freelist = {0, 0, 0}; u8 *aUsed; Level *pLevel; Snapshot *pWorker = pDb->pWorker; int nBlock = pWorker->nBlock; -#if 0 - static int nCall = 0; - nCall++; - printf("%d calls\n", nCall); -#endif - aUsed = lsmMallocZero(pDb->pEnv, nBlock); if( aUsed==0 ){ /* Malloc has failed. Since this function is only called within debug - ** builds, this probably means the user is running an OOM injection test. - ** Regardless, it will not be possible to run the integrity-check at this - ** time, so assume the database is Ok and return non-zero. */ + ** builds, this probably means the user is running an OOM injection test. + ** Regardless, it will not be possible to run the integrity-check at this + ** time, so assume the database is Ok and return non-zero. */ return 1; } for(pLevel=pWorker->pLevel; pLevel; pLevel=pLevel->pNext){ int i; @@ -2854,18 +2181,30 @@ for(i=0; inRight; i++){ checkBlocks(pFS, &pLevel->aRhs[i], 0, nBlock, aUsed); } } - /* Mark all blocks in the free-list as used */ - ctx.aUsed = aUsed; - ctx.nBlock = nBlock; - rc = lsmWalkFreelist(pDb, 0, checkFreelistCb, (void *)&ctx); - - if( rc==LSM_OK ){ - for(i=0; inFreelistOvfl ){ + int rc = lsmCheckpointOverflowLoad(pDb, &freelist); + assert( rc==LSM_OK || rc==LSM_NOMEM ); + if( rc!=LSM_OK ) return 1; + } + + for(j=0; j<2; j++){ + Freelist *pFreelist; + if( j==0 ) pFreelist = &pWorker->freelist; + if( j==1 ) pFreelist = &freelist; + + for(i=0; inEntry; i++){ + u32 iBlk = pFreelist->aEntry[i].iBlk; + assert( iBlk<=nBlock ); + assert( aUsed[iBlk-1]==0 ); + aUsed[iBlk-1] = 1; + } + } + + for(i=0; ipEnv, aUsed); lsmFree(pDb->pEnv, freelist.aEntry); return 1; Index: src/lsm_log.c ================================================================== --- src/lsm_log.c +++ src/lsm_log.c @@ -203,13 +203,10 @@ #define LSM_LOG_DELETE_CKSUM 0x09 /* Require a checksum every 32KB. */ #define LSM_CKSUM_MAXDATA (32*1024) -/* Do not wrap a log file smaller than this in bytes. */ -#define LSM_MIN_LOGWRAP (128*1024) - /* ** szSector: ** Commit records must be aligned to end on szSector boundaries. If ** the safety-mode is set to NORMAL or OFF, this value is 1. Otherwise, ** if the safety-mode is set to FULL, it is the size of the file-system @@ -218,15 +215,15 @@ struct LogWriter { u32 cksum0; /* Checksum 0 at offset iOff */ u32 cksum1; /* Checksum 1 at offset iOff */ int iCksumBuf; /* Bytes of buf that have been checksummed */ i64 iOff; /* Offset at start of buffer buf */ + LsmString buf; /* Buffer containing data not yet written */ int szSector; /* Sector size for this transaction */ LogRegion jump; /* Avoid writing to this region */ i64 iRegion1End; /* End of first region written by trans */ i64 iRegion2Start; /* Start of second regions written by trans */ - LsmString buf; /* Buffer containing data not yet written */ }; /* ** Return the result of interpreting the first 4 bytes in buffer aIn as ** a 32-bit unsigned little-endian integer. @@ -303,42 +300,26 @@ ** a snapshot that points to the same data in the database file is synced ** into the db header. */ static int logReclaimSpace(lsm_db *pDb){ int rc = LSM_OK; - int iMeta; - - iMeta = (int)pDb->pShmhdr->iMetaPage; - if( iMeta==1 || iMeta==2 ){ + if( pDb->pShmhdr->iMetaPage ){ DbLog *pLog = &pDb->treehdr.log; - i64 iSyncedId; - - /* Read the snapshot-id of the snapshot stored on meta-page iMeta. Note - ** that in theory, the value read is untrustworthy (due to a race - ** condition - see comments above lsmFsReadSyncedId()). So it is only - ** ever used to conclude that no log space can be reclaimed. If it seems - ** to indicate that it may be possible to reclaim log space, a - ** second call to lsmCheckpointSynced() (which does return trustworthy - ** values) is made below to confirm. */ - rc = lsmFsReadSyncedId(pDb, iMeta, &iSyncedId); - - if( rc==LSM_OK && pLog->iSnapshotId!=iSyncedId ){ - i64 iSnapshotId = 0; - i64 iOff = 0; - rc = lsmCheckpointSynced(pDb, &iSnapshotId, &iOff, 0); - if( rc==LSM_OK && pLog->iSnapshotIdaRegion[iRegion]; - if( iOff>=p->iStart && iOff<=p->iEnd ) break; - p->iStart = 0; - p->iEnd = 0; - } - assert( iRegion<3 ); - pLog->aRegion[iRegion].iStart = iOff; - pLog->iSnapshotId = iSnapshotId; - } + i64 iSnapshotId = 0; + i64 iOff = 0; + rc = lsmCheckpointSynced(pDb, &iSnapshotId, &iOff, 0); + if( rc==LSM_OK && pLog->iSnapshotIdaRegion[iRegion]; + if( iOff>=p->iStart && iOff<=p->iEnd ) break; + p->iStart = 0; + p->iEnd = 0; + } + assert( iRegion<3 ); + pLog->aRegion[iRegion].iStart = iOff; + pLog->iSnapshotId = iSnapshotId; } } return rc; } @@ -355,27 +336,16 @@ int rc = LSM_OK; LogWriter *pNew; LogRegion *aReg; if( pDb->bUseLog==0 ) return LSM_OK; - - /* If the log file has not yet been opened, open it now. Also allocate - ** the LogWriter structure, if it has not already been allocated. */ rc = lsmFsOpenLog(pDb->pFS); - if( pDb->pLogWriter==0 ){ - pNew = lsmMallocZeroRc(pDb->pEnv, sizeof(LogWriter), &rc); - if( pNew ){ - lsmStringInit(&pNew->buf, pDb->pEnv); - rc = lsmStringExtend(&pNew->buf, 2); - } - }else{ - pNew = pDb->pLogWriter; - assert( (u8 *)(&pNew[1])==(u8 *)(&((&pNew->buf)[1])) ); - memset(pNew, 0, ((u8 *)&pNew->buf) - (u8 *)pNew); - pNew->buf.n = 0; - } - + pNew = lsmMallocZeroRc(pDb->pEnv, sizeof(LogWriter), &rc); + if( pNew ){ + lsmStringInit(&pNew->buf, pDb->pEnv); + rc = lsmStringExtend(&pNew->buf, 2); + } if( rc==LSM_OK ){ /* The following call detects whether or not a new snapshot has been ** synced into the database file. If so, it updates the contents of ** the pDb->treehdr.log structure to reclaim any space in the log ** file that is no longer required. @@ -387,11 +357,12 @@ ** the log file. */ rc = logReclaimSpace(pDb); } if( rc!=LSM_OK ){ - lsmLogClose(pDb); + if( pNew ) lsmFree(pDb->pEnv, pNew->buf.z); + lsmFree(pDb->pEnv, pNew); return rc; } /* Set the effective sector-size for this transaction. Sectors are assumed ** to be one byte in size if the safety-mode is OFF or NORMAL, or as @@ -404,12 +375,13 @@ } /* There are now three scenarios: ** ** 1) Regions 0 and 1 are both zero bytes in size and region 2 begins - ** at a file offset greater than LSM_MIN_LOGWRAP. In this case, wrap - ** around to the start and write data into the start of the log file. + ** at a file offset greater than N, where N is the value configured + ** by LSM_CONFIG_LOG_SIZE. In this case, wrap around to the start + ** and write data into the start of the log file. ** ** 2) Region 1 is zero bytes in size and region 2 occurs earlier in the ** file than region 0. In this case, append data to region 2, but ** remember to jump over region 1 if required. ** @@ -421,11 +393,11 @@ assert( aReg[1].iEnd==0 || aReg[1].iEnd>aReg[1].iStart ); pNew->cksum0 = pDb->treehdr.log.cksum0; pNew->cksum1 = pDb->treehdr.log.cksum1; - if( aReg[0].iEnd==0 && aReg[1].iEnd==0 && aReg[2].iStart>=LSM_MIN_LOGWRAP ){ + if( aReg[0].iEnd==0 && aReg[1].iEnd==0 && aReg[2].iStart>=pDb->nLogSz ){ /* Case 1. Wrap around to the start of the file. Write an LSM_LOG_JUMP ** into the log file in this case. Pad it out to 8 bytes using a PAD2 ** record so that the checksums can be updated immediately. */ u8 aJump[] = { LSM_LOG_PAD2, 0x04, 0x00, 0x00, 0x00, 0x00, LSM_LOG_JUMP, 0x00 @@ -492,10 +464,13 @@ pLog->aRegion[1].iStart = pLog->aRegion[2].iStart; pLog->aRegion[1].iEnd = p->iRegion1End; pLog->aRegion[2].iStart = p->iRegion2Start; } } + lsmStringClear(&p->buf); + lsmFree(pDb->pEnv, p); + pDb->pLogWriter = 0; } static int jumpIfRequired( lsm_db *pDb, LogWriter *pLog, @@ -1109,15 +1084,5 @@ lsmStringClear(&buf1); lsmStringClear(&buf2); lsmStringClear(&reader.buf); return rc; } - -void lsmLogClose(lsm_db *db){ - if( db->pLogWriter ){ - lsmFree(db->pEnv, db->pLogWriter->buf.z); - lsmFree(db->pEnv, db->pLogWriter); - db->pLogWriter = 0; - } -} - - Index: src/lsm_main.c ================================================================== --- src/lsm_main.c +++ src/lsm_main.c @@ -39,12 +39,10 @@ ** handle must be holding a pointer to a client snapshot. And the reverse ** - if there are no open cursors and no write transactions then there must ** not be a client snapshot. */ assert( (pDb->pCsr!=0 || pDb->nTransOpen>0)==(pDb->iReader>=0) ); - assert( pDb->iReader<0 || pDb->pClient!=0 ); - assert( pDb->nTransOpen>=0 ); } #else # define assert_db_state(x) #endif @@ -57,17 +55,10 @@ res = memcmp(p1, p2, LSM_MIN(n1, n2)); if( res==0 ) res = (n1-n2); return res; } -static void xLog(void *pCtx, int rc, const char *z){ - (void)(rc); - (void)(pCtx); - fprintf(stderr, "%s\n", z); - fflush(stderr); -} - /* ** Allocate a new db handle. */ int lsm_new(lsm_env *pEnv, lsm_db **ppDb){ lsm_db *pDb; @@ -80,25 +71,24 @@ *ppDb = pDb = (lsm_db *)lsmMallocZero(pEnv, sizeof(lsm_db)); if( pDb==0 ) return LSM_NOMEM_BKPT; /* Initialize the new object */ pDb->pEnv = pEnv; - pDb->nTreeLimit = LSM_DFLT_AUTOFLUSH; + pDb->nTreeLimit = LSM_DFLT_WRITE_BUFFER; pDb->nAutockpt = LSM_DFLT_AUTOCHECKPOINT; - pDb->bAutowork = LSM_DFLT_AUTOWORK; - pDb->eSafety = LSM_DFLT_SAFETY; + pDb->bAutowork = 1; + pDb->eSafety = LSM_SAFETY_NORMAL; pDb->xCmp = xCmp; + pDb->nLogSz = LSM_DFLT_LOG_SIZE; pDb->nDfltPgsz = LSM_DFLT_PAGE_SIZE; pDb->nDfltBlksz = LSM_DFLT_BLOCK_SIZE; - pDb->nMerge = LSM_DFLT_AUTOMERGE; + pDb->nMerge = LSM_DFLT_NMERGE; pDb->nMaxFreelist = LSM_MAX_FREELIST_ENTRIES; - pDb->bUseLog = LSM_DFLT_USE_LOG; + pDb->bUseLog = 1; pDb->iReader = -1; - pDb->bMultiProc = LSM_DFLT_MULTIPLE_PROCESSES; - pDb->bMmap = LSM_DFLT_MMAP; - pDb->xLog = xLog; - pDb->compress.iId = LSM_COMPRESSION_NONE; + pDb->bMultiProc = 1; + pDb->bMmap = LSM_IS_64_BIT; return LSM_OK; } lsm_env *lsm_get_env(lsm_db *pDb){ assert( pDb->pEnv ); @@ -191,19 +181,11 @@ rc = LSM_MISUSE_BKPT; }else{ lsmFreeSnapshot(pDb->pEnv, pDb->pClient); pDb->pClient = 0; lsmDbDatabaseRelease(pDb); - lsmLogClose(pDb); lsmFsClose(pDb->pFS); - - /* Invoke any destructors registered for the compression or - ** compression factory callbacks. */ - if( pDb->factory.xFree ) pDb->factory.xFree(pDb->factory.pCtx); - if( pDb->compress.xFree ) pDb->compress.xFree(pDb->compress.pCtx); - - lsmFree(pDb->pEnv, pDb->rollback.aArray); lsmFree(pDb->pEnv, pDb->aTrans); lsmFree(pDb->pEnv, pDb->apShm); lsmFree(pDb->pEnv, pDb); } } @@ -214,19 +196,16 @@ int rc = LSM_OK; va_list ap; va_start(ap, eParam); switch( eParam ){ - case LSM_CONFIG_AUTOFLUSH: { - /* This parameter is read and written in KB. But all internal - ** processing is done in bytes. */ + case LSM_CONFIG_WRITE_BUFFER: { int *piVal = va_arg(ap, int *); - int iVal = *piVal; - if( iVal>=0 && iVal<=(1024*1024) ){ - pDb->nTreeLimit = iVal*1024; + if( *piVal>=0 ){ + pDb->nTreeLimit = *piVal; } - *piVal = (pDb->nTreeLimit / 1024); + *piVal = pDb->nTreeLimit; break; } case LSM_CONFIG_AUTOWORK: { int *piVal = va_arg(ap, int *); @@ -236,18 +215,24 @@ *piVal = pDb->bAutowork; break; } case LSM_CONFIG_AUTOCHECKPOINT: { - /* This parameter is read and written in KB. But all internal processing - ** (including the lsm_db.nAutockpt variable) is done in bytes. */ int *piVal = va_arg(ap, int *); if( *piVal>=0 ){ - int iVal = *piVal; - pDb->nAutockpt = (i64)iVal * 1024; + pDb->nAutockpt = *piVal; + } + *piVal = pDb->nAutockpt; + break; + } + + case LSM_CONFIG_LOG_SIZE: { + int *piVal = va_arg(ap, int *); + if( *piVal>0 ){ + pDb->nLogSz = *piVal; } - *piVal = (int)(pDb->nAutockpt / 1024); + *piVal = pDb->nLogSz; break; } case LSM_CONFIG_PAGE_SIZE: { int *piVal = va_arg(ap, int *); @@ -265,24 +250,21 @@ } break; } case LSM_CONFIG_BLOCK_SIZE: { - /* This parameter is read and written in KB. But all internal - ** processing is done in bytes. */ int *piVal = va_arg(ap, int *); if( pDb->pDatabase ){ /* If lsm_open() has been called, this is a read-only parameter. - ** Set the output variable to the block-size in KB according to the - ** FileSystem object. */ - *piVal = lsmFsBlockSize(pDb->pFS) / 1024; - }else{ - int iVal = *piVal; - if( iVal>=64 && iVal<=65536 && ((iVal-1) & iVal)==0 ){ - pDb->nDfltBlksz = iVal * 1024; - }else{ - *piVal = pDb->nDfltBlksz / 1024; + ** Set the output variable to the page-size according to the + ** FileSystem object. */ + *piVal = lsmFsBlockSize(pDb->pFS); + }else{ + if( *piVal>=65536 && ((*piVal-1) & *piVal)==0 ){ + pDb->nDfltBlksz = *piVal; + }else{ + *piVal = pDb->nDfltBlksz; } } break; } @@ -295,13 +277,12 @@ break; } case LSM_CONFIG_MMAP: { int *piVal = va_arg(ap, int *); - if( pDb->iReader<0 && *piVal>=0 && *piVal<=1 ){ - pDb->bMmap = *piVal; - rc = lsmFsConfigure(pDb); + if( pDb->pDatabase==0 ){ + pDb->bMmap = (LSM_IS_64_BIT && *piVal); } *piVal = pDb->bMmap; break; } @@ -312,11 +293,11 @@ } *piVal = pDb->bUseLog; break; } - case LSM_CONFIG_AUTOMERGE: { + case LSM_CONFIG_NMERGE: { int *piVal = va_arg(ap, int *); if( *piVal>1 ) pDb->nMerge = *piVal; *piVal = pDb->nMerge; break; } @@ -342,42 +323,22 @@ } break; } case LSM_CONFIG_SET_COMPRESSION: { - lsm_compress *p = va_arg(ap, lsm_compress *); - if( pDb->iReader>=0 && pDb->bInFactory==0 ){ - /* May not change compression schemes with an open transaction */ - rc = LSM_MISUSE_BKPT; - }else{ - if( pDb->compress.xFree ){ - /* Invoke any destructor belonging to the current compression. */ - pDb->compress.xFree(pDb->compress.pCtx); - } - if( p->xBound==0 ){ - memset(&pDb->compress, 0, sizeof(lsm_compress)); - pDb->compress.iId = LSM_COMPRESSION_NONE; - }else{ - memcpy(&pDb->compress, p, sizeof(lsm_compress)); - } - rc = lsmFsConfigure(pDb); - } - break; - } - - case LSM_CONFIG_SET_COMPRESSION_FACTORY: { - lsm_compress_factory *p = va_arg(ap, lsm_compress_factory *); - if( pDb->factory.xFree ){ - /* Invoke any destructor belonging to the current factory. */ - pDb->factory.xFree(pDb->factory.pCtx); - } - memcpy(&pDb->factory, p, sizeof(lsm_compress_factory)); - break; - } - - case LSM_CONFIG_GET_COMPRESSION: { - lsm_compress *p = va_arg(ap, lsm_compress *); + int *p = va_arg(ap, lsm_compress *); + if( pDb->pDatabase ){ + /* If lsm_open() has been called, this call is against the rules. */ + rc = LSM_MISUSE_BKPT; + }else{ + memcpy(&pDb->compress, p, sizeof(lsm_compress)); + } + break; + } + + case LSM_CONFIG_GET_COMPRESSION: { + int *p = va_arg(ap, lsm_compress *); memcpy(p, &pDb->compress, sizeof(lsm_compress)); break; } default: @@ -402,18 +363,18 @@ if( !pDb->pWorker ){ rc = lsmBeginWork(pDb); if( rc!=LSM_OK ) return rc; *pbUnlock = 1; } - if( pp ) *pp = pDb->pWorker; + *pp = pDb->pWorker; return rc; } static void infoFreeWorker(lsm_db *pDb, int bUnlock){ if( bUnlock ){ int rcdummy = LSM_BUSY; - lsmFinishWork(pDb, 0, &rcdummy); + lsmFinishWork(pDb, 0, 0, &rcdummy); } } int lsmStructList( lsm_db *pDb, /* Database handle */ @@ -433,12 +394,12 @@ /* Format the contents of the snapshot as text */ pTopLevel = lsmDbSnapshotLevel(pWorker); lsmStringInit(&s, pDb->pEnv); for(p=pTopLevel; rc==LSM_OK && p; p=p->pNext){ int i; - lsmStringAppendf(&s, "%s{%d", (s.n ? " " : ""), (int)p->iAge); - lsmAppendSegmentList(&s, " ", &p->lhs); + lsmStringAppendf(&s, "%s{", (s.n ? " " : "")); + lsmAppendSegmentList(&s, "", &p->lhs); for(i=0; rc==LSM_OK && inRight; i++){ lsmAppendSegmentList(&s, " ", &p->aRhs[i]); } lsmStringAppend(&s, "}", 1); } @@ -448,76 +409,35 @@ infoFreeWorker(pDb, bUnlock); *pzOut = s.z; return rc; } -static int infoFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ - LsmString *pStr = (LsmString *)pCtx; - lsmStringAppendf(pStr, "%s{%d %lld}", (pStr->n?" ":""), iBlk, iSnapshot); - return 0; -} - int lsmInfoFreelist(lsm_db *pDb, char **pzOut){ Snapshot *pWorker; /* Worker snapshot */ int bUnlock = 0; LsmString s; + int i; int rc; /* Obtain the worker snapshot */ rc = infoGetWorker(pDb, &pWorker, &bUnlock); if( rc!=LSM_OK ) return rc; lsmStringInit(&s, pDb->pEnv); - rc = lsmWalkFreelist(pDb, 0, infoFreelistCb, &s); - if( rc!=LSM_OK ){ - lsmFree(pDb->pEnv, s.z); - }else{ - *pzOut = s.z; + lsmStringAppendf(&s, "%d+%d",pWorker->freelist.nEntry,pWorker->nFreelistOvfl); + for(i=0; ifreelist.nEntry; i++){ + FreelistEntry *p = &pWorker->freelist.aEntry[i]; + lsmStringAppendf(&s, " {%d %d}", p->iBlk, (int)p->iId); } + rc = s.n>=0 ? LSM_OK : LSM_NOMEM; /* Release the snapshot and return */ infoFreeWorker(pDb, bUnlock); + *pzOut = s.z; return rc; } -static int infoTreeSize(lsm_db *db, int *pnOldKB, int *pnNewKB){ - ShmHeader *pShm = db->pShmhdr; - TreeHeader *p = &pShm->hdr1; - - /* The following code suffers from two race conditions, as it accesses and - ** trusts the contents of shared memory without verifying checksums: - ** - ** * The two values read - TreeHeader.root.nByte and oldroot.nByte - are - ** 32-bit fields. It is assumed that reading from one of these - ** is atomic - that it is not possible to read a partially written - ** garbage value. However the two values may be mutually inconsistent. - ** - ** * TreeHeader.iLogOff is a 64-bit value. And lsmCheckpointLogOffset() - ** reads a 64-bit value from a snapshot stored in shared memory. It - ** is assumed that in each case it is possible to read a partially - ** written garbage value. If this occurs, then the value returned - ** for the size of the "old" tree may reflect the size of an "old" - ** tree that was recently flushed to disk. - ** - ** Given the context in which this function is called (as a result of an - ** lsm_info(LSM_INFO_TREE_SIZE) request), neither of these are considered to - ** be problems. - */ - *pnNewKB = ((int)p->root.nByte + 1023) / 1024; - if( p->iOldShmid ){ - if( p->iOldLog==lsmCheckpointLogOffset(pShm->aSnap1) ){ - *pnOldKB = 0; - }else{ - *pnOldKB = ((int)p->oldroot.nByte + 1023) / 1024; - } - }else{ - *pnOldKB = 0; - } - - return LSM_OK; -} - int lsm_info(lsm_db *pDb, int eParam, ...){ int rc = LSM_OK; va_list ap; va_start(ap, eParam); @@ -541,32 +461,19 @@ } case LSM_INFO_ARRAY_STRUCTURE: { Pgno pgno = va_arg(ap, Pgno); char **pzVal = va_arg(ap, char **); - rc = lsmInfoArrayStructure(pDb, 0, pgno, pzVal); - break; - } - - case LSM_INFO_ARRAY_PAGES: { - Pgno pgno = va_arg(ap, Pgno); - char **pzVal = va_arg(ap, char **); - rc = lsmInfoArrayPages(pDb, pgno, pzVal); + rc = lsmInfoArrayStructure(pDb, pgno, pzVal); break; } case LSM_INFO_PAGE_HEX_DUMP: case LSM_INFO_PAGE_ASCII_DUMP: { Pgno pgno = va_arg(ap, Pgno); char **pzVal = va_arg(ap, char **); - int bUnlock = 0; - rc = infoGetWorker(pDb, 0, &bUnlock); - if( rc==LSM_OK ){ - int bHex = (eParam==LSM_INFO_PAGE_HEX_DUMP); - rc = lsmInfoPageDump(pDb, pgno, bHex, pzVal); - } - infoFreeWorker(pDb, bUnlock); + rc = lsmInfoPageDump(pDb, pgno, (eParam==LSM_INFO_PAGE_HEX_DUMP), pzVal); break; } case LSM_INFO_LOG_STRUCTURE: { char **pzVal = va_arg(ap, char **); @@ -575,33 +482,10 @@ } case LSM_INFO_FREELIST: { char **pzVal = va_arg(ap, char **); rc = lsmInfoFreelist(pDb, pzVal); - break; - } - - case LSM_INFO_CHECKPOINT_SIZE: { - int *pnKB = va_arg(ap, int *); - rc = lsmCheckpointSize(pDb, pnKB); - break; - } - - case LSM_INFO_TREE_SIZE: { - int *pnOld = va_arg(ap, int *); - int *pnNew = va_arg(ap, int *); - rc = infoTreeSize(pDb, pnOld, pnNew); - break; - } - - case LSM_INFO_COMPRESSION_ID: { - unsigned int *piOut = va_arg(ap, unsigned int *); - if( pDb->pClient ){ - *piOut = pDb->pClient->iCmpId; - }else{ - rc = lsmInfoCompressionId(pDb, piOut); - } break; } default: rc = LSM_MISUSE; @@ -675,11 +559,11 @@ } /* ** Write a new value into the database. */ -int lsm_insert( +int lsm_write( lsm_db *db, /* Database connection */ const void *pKey, int nKey, /* Key to write or delete */ const void *pVal, int nVal /* Value to write. Or nVal==-1 for a delete */ ){ return doWriteOp(db, 0, pKey, nKey, pVal, nVal); @@ -755,11 +639,11 @@ ** Attempt to seek the cursor to the database entry specified by pKey/nKey. ** If an error occurs (e.g. an OOM or IO error), return an LSM error code. ** Otherwise, return LSM_OK. */ int lsm_csr_seek(lsm_cursor *pCsr, const void *pKey, int nKey, int eSeek){ - return lsmMCursorSeek((MultiCursor *)pCsr, 0, (void *)pKey, nKey, eSeek); + return lsmMCursorSeek((MultiCursor *)pCsr, (void *)pKey, nKey, eSeek); } int lsm_csr_next(lsm_cursor *pCsr){ return lsmMCursorNext((MultiCursor *)pCsr); } @@ -871,10 +755,12 @@ /* A value less than zero means close the innermost nested transaction. */ if( iLevel<0 ) iLevel = LSM_MAX(0, pDb->nTransOpen - 1); if( iLevelnTransOpen ){ if( iLevel==0 ){ + int bAutowork = 0; + /* Commit the transaction to disk. */ if( rc==LSM_OK ) rc = lsmLogCommit(pDb); if( rc==LSM_OK && pDb->eSafety==LSM_SAFETY_FULL ){ rc = lsmFsSyncLog(pDb->pFS); } @@ -907,7 +793,22 @@ dbReleaseClientSnapshot(pDb); } return rc; } + +int lsm_tree_size(lsm_db *db, int *pnOld, int *pnNew){ + ShmHeader *pShm = db->pShmhdr; + + *pnNew = (int)pShm->hdr1.nByte; + *pnOld = 0; + if( pShm->hdr1.iOldShmid ){ + i64 iOff = pShm->hdr1.iOldLog; + if( iOff!=lsmCheckpointLogOffset(pShm->aSnap1) ){ + *pnOld = 1; + } + } + + return LSM_OK; +} Index: src/lsm_mem.c ================================================================== --- src/lsm_mem.c +++ src/lsm_mem.c @@ -12,10 +12,28 @@ ** ** Helper routines for memory allocation. */ #include "lsmInt.h" +/* Default allocation size. */ +#define CHUNKSIZE 16*1024 + +typedef struct Chunk Chunk; + +struct Chunk { + int iOff; /* Offset of free space within pSpace */ + u8 *aData; /* Pointer to space for user allocations */ + int nData; /* Size of buffer aData, in bytes */ + Chunk *pNext; +}; + +struct Mempool { + Chunk *pFirst; /* First in list of chunks */ + Chunk *pLast; /* Last in list of chunks */ + int nUsed; /* Total number of bytes allocated */ +}; + /* ** The following routines are called internally by LSM sub-routines. In ** this case a valid environment pointer must be supplied. */ void *lsmMalloc(lsm_env *pEnv, size_t N){ @@ -100,5 +118,125 @@ if( zRet ){ memcpy(zRet, zIn, nByte+1); } return zRet; } + + +/* +** Allocate a new Chunk structure (using lsmMalloc()). +*/ +static Chunk * poolChunkNew(lsm_env *pEnv, int nMin){ + Chunk *pChunk; + int nAlloc = LSM_MAX(CHUNKSIZE, nMin + sizeof(Chunk)); + + pChunk = (Chunk *)lsmMalloc(pEnv, nAlloc); + if( pChunk ){ + pChunk->pNext = 0; + pChunk->iOff = 0; + pChunk->aData = (u8 *)&pChunk[1]; + pChunk->nData = nAlloc - sizeof(Chunk); + } + + return pChunk; +} + +/* +** Allocate sz bytes from chunk pChunk. +*/ +static u8 *poolChunkAlloc(Chunk *pChunk, int sz){ + u8 *pRet; /* Pointer value to return */ + assert( sz<=(pChunk->nData - pChunk->iOff) ); + pRet = &pChunk->aData[pChunk->iOff]; + pChunk->iOff += sz; + return pRet; +} + + +int lsmPoolNew(lsm_env *pEnv, Mempool **ppPool){ + int rc = LSM_NOMEM; + Mempool *pPool = 0; + Chunk *pChunk; + + pChunk = poolChunkNew(pEnv, sizeof(Mempool)); + if( pChunk ){ + pPool = (Mempool *)poolChunkAlloc(pChunk, sizeof(Mempool)); + pPool->pFirst = pChunk; + pPool->pLast = pChunk; + pPool->nUsed = 0; + rc = LSM_OK; + } + + *ppPool = pPool; + return rc; +} + +void lsmPoolDestroy(lsm_env *pEnv, Mempool *pPool){ + if( pPool ){ + Chunk *pChunk = pPool->pFirst; + while( pChunk ){ + Chunk *pFree = pChunk; + pChunk = pChunk->pNext; + lsmFree(pEnv, pFree); + } + } +} + +void *lsmPoolMalloc(lsm_env *pEnv, Mempool *pPool, int nByte){ + u8 *pRet = 0; + Chunk *pLast = pPool->pLast; + + nByte = ROUND8(nByte); + if( nByte > (pLast->nData - pLast->iOff) ){ + Chunk *pNew = poolChunkNew(pEnv, nByte); + if( !pNew ) return 0; + pLast->pNext = pNew; + pPool->pLast = pNew; + pLast = pNew; + } + + if( pLast ){ + pRet = poolChunkAlloc(pLast, nByte); + pPool->nUsed += nByte; + } + return (void *)pRet; +} + +void *lsmPoolMallocZero(lsm_env *pEnv, Mempool *pPool, int nByte){ + void *pRet = lsmPoolMalloc(pEnv, pPool, nByte); + if( pRet ) memset(pRet, 0, nByte); + return pRet; +} + +/* +** Return the amount of memory currently allocated from this pool. +*/ +int lsmPoolUsed(Mempool *pPool){ + return pPool->nUsed; +} + +void lsmPoolMark(Mempool *pPool, void **ppChunk, int *piOff){ + *ppChunk = (void *)pPool->pLast; + *piOff = pPool->pLast->iOff; +} + +void lsmPoolRollback(lsm_env *pEnv, Mempool *pPool, void *pChunk, int iOff){ + Chunk *pLast = (Chunk *)pChunk; + Chunk *p; + Chunk *pNext; + +#ifdef LSM_EXPENSIVE_DEBUG + /* Check that pLast is actually in the list of chunks */ + for(p=pPool->pFirst; p!=pLast; p=p->pNext); +#endif + + pPool->nUsed -= (pLast->iOff - iOff); + for(p=pLast->pNext; p; p=pNext){ + pPool->nUsed -= p->iOff; + pNext = p->pNext; + lsmFree(pEnv, p); + } + + pLast->pNext = 0; + pLast->iOff = iOff; + pPool->pLast = pLast; +} Index: src/lsm_shared.c ================================================================== --- src/lsm_shared.c +++ src/lsm_shared.c @@ -32,26 +32,19 @@ ** Database structure. There is one such structure for each distinct ** database accessed by this process. They are stored in the singly linked ** list starting at global variable gShared.pDatabase. Database objects are ** reference counted. Once the number of connections to the associated ** database drops to zero, they are removed from the linked list and deleted. -** -** pFile: -** In multi-process mode, this file descriptor is used to obtain locks -** and to access shared-memory. In single process mode, its only job is -** to hold the exclusive lock on the file. -** */ struct Database { /* Protected by the global mutex (enterGlobalMutex/leaveGlobalMutex): */ char *zName; /* Canonical path to database file */ int nName; /* strlen(zName) */ int nDbRef; /* Number of associated lsm_db handles */ Database *pDbNext; /* Next Database structure in global list */ /* Protected by the local mutex (pClientMutex) */ - int bMultiProc; /* True if running in multi-process mode */ lsm_file *pFile; /* Used for locks/shm in multi-proc mode */ LsmFile *pLsmFile; /* List of deferred closes */ lsm_mutex *pClientMutex; /* Protects the apShmChunk[] and pConn */ int nShmChunk; /* Number of entries in apShmChunk[] array */ void **apShmChunk; /* Array of "shared" memory regions */ @@ -89,57 +82,65 @@ #else # define assertNotInFreelist(x,y) #endif /* -** Append an entry to the free-list. If (iId==-1), this is a delete. +** Append an entry to the free-list. */ -int freelistAppend(lsm_db *db, int iBlk, i64 iId){ - lsm_env *pEnv = db->pEnv; - Freelist *p; - int i; +int lsmFreelistAppend(lsm_env *pEnv, Freelist *p, int iBlk, i64 iId){ - assert( iId==-1 || iId>=0 ); - p = db->bUseFreelist ? db->pFreelist : &db->pWorker->freelist; + /* Assert that this is not an attempt to insert a duplicate block number */ +#if 0 + assertNotInFreelist(p, iBlk); +#endif /* Extend the space allocated for the freelist, if required */ assert( p->nAlloc>=p->nEntry ); if( p->nAlloc==p->nEntry ){ int nNew; - int nByte; FreelistEntry *aNew; nNew = (p->nAlloc==0 ? 4 : p->nAlloc*2); - nByte = sizeof(FreelistEntry) * nNew; - aNew = (FreelistEntry *)lsmRealloc(pEnv, p->aEntry, nByte); + aNew = (FreelistEntry *)lsmRealloc(pEnv, p->aEntry, + sizeof(FreelistEntry)*nNew); if( !aNew ) return LSM_NOMEM_BKPT; p->nAlloc = nNew; p->aEntry = aNew; } - for(i=0; inEntry; i++){ - assert( i==0 || p->aEntry[i].iBlk > p->aEntry[i-1].iBlk ); - if( p->aEntry[i].iBlk>=iBlk ) break; - } - - if( inEntry && p->aEntry[i].iBlk==iBlk ){ - /* Clobber an existing entry */ - p->aEntry[i].iId = iId; - }else{ - /* Insert a new entry into the list */ - int nByte = sizeof(FreelistEntry)*(p->nEntry-i); - memmove(&p->aEntry[i+1], &p->aEntry[i], nByte); - p->aEntry[i].iBlk = iBlk; - p->aEntry[i].iId = iId; - p->nEntry++; - } + /* Append the new entry to the freelist */ + p->aEntry[p->nEntry].iBlk = iBlk; + p->aEntry[p->nEntry].iId = iId; + p->nEntry++; return LSM_OK; } + +static int flInsertEntry(lsm_env *pEnv, Freelist *p, int iBlk){ + int rc; + + rc = lsmFreelistAppend(pEnv, p, iBlk, 1); + if( rc==LSM_OK ){ + memmove(&p->aEntry[1], &p->aEntry[0], sizeof(FreelistEntry)*(p->nEntry-1)); + p->aEntry[0].iBlk = iBlk; + p->aEntry[0].iId = 1; + } + return rc; +} + +/* +** Remove the first entry of the free-list. +*/ +static void flRemoveEntry0(Freelist *p){ + int nNew = p->nEntry - 1; + assert( nNew>=0 ); + memmove(&p->aEntry[0], &p->aEntry[1], sizeof(FreelistEntry) * nNew); + p->nEntry = nNew; +} /* -** This function frees all resources held by the Database structure passed +** tHIS Function frees all resources held by the Database structure passed ** as the only argument. */ static void freeDatabase(lsm_env *pEnv, Database *p){ assert( holdingGlobalMutex(pEnv) ); if( p ){ @@ -156,90 +157,10 @@ /* Free the memory allocated for the Database struct itself */ lsmFree(pEnv, p); } } -typedef struct DbTruncateCtx DbTruncateCtx; -struct DbTruncateCtx { - int nBlock; - i64 iInUse; -}; - -static int dbTruncateCb(void *pCtx, int iBlk, i64 iSnapshot){ - DbTruncateCtx *p = (DbTruncateCtx *)pCtx; - if( iBlk!=p->nBlock || (p->iInUse>=0 && iSnapshot>=p->iInUse) ) return 1; - p->nBlock--; - return 0; -} - -static int dbTruncate(lsm_db *pDb, i64 iInUse){ - int rc = LSM_OK; -#if 0 - int i; - DbTruncateCtx ctx; - - assert( pDb->pWorker ); - ctx.nBlock = pDb->pWorker->nBlock; - ctx.iInUse = iInUse; - - rc = lsmWalkFreelist(pDb, 1, dbTruncateCb, (void *)&ctx); - for(i=ctx.nBlock+1; rc==LSM_OK && i<=pDb->pWorker->nBlock; i++){ - rc = freelistAppend(pDb, i, -1); - } - - if( rc==LSM_OK ){ -#ifdef LSM_LOG_FREELIST - if( ctx.nBlock!=pDb->pWorker->nBlock ){ - lsmLogMessage(pDb, 0, - "dbTruncate(): truncated db to %d blocks",ctx.nBlock - ); - } -#endif - pDb->pWorker->nBlock = ctx.nBlock; - } -#endif - return rc; -} - - -/* -** This function is called during database shutdown (when the number of -** connections drops from one to zero). It truncates the database file -** to as small a size as possible without truncating away any blocks that -** contain data. -*/ -static int dbTruncateFile(lsm_db *pDb){ - int rc; - - assert( pDb->pWorker==0 ); - assert( lsmShmAssertLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_EXCL) ); - rc = lsmCheckpointLoadWorker(pDb); - - if( rc==LSM_OK ){ - DbTruncateCtx ctx; - - /* Walk the database free-block-list in reverse order. Set ctx.nBlock - ** to the block number of the last block in the database that actually - ** contains data. */ - ctx.nBlock = pDb->pWorker->nBlock; - ctx.iInUse = -1; - rc = lsmWalkFreelist(pDb, 1, dbTruncateCb, (void *)&ctx); - - /* If the last block that contains data is not already the last block in - ** the database file, truncate the database file so that it is. */ - if( rc==LSM_OK && ctx.nBlock!=pDb->pWorker->nBlock ){ - rc = lsmFsTruncateDb( - pDb->pFS, (i64)ctx.nBlock*lsmFsBlockSize(pDb->pFS) - ); - } - } - - lsmFreeSnapshot(pDb->pEnv, pDb->pWorker); - pDb->pWorker = 0; - return rc; -} - static void doDbDisconnect(lsm_db *pDb){ int rc; /* Block for an exclusive lock on DMS1. This lock serializes all calls ** to doDbConnect() and doDbDisconnect() across all processes. */ @@ -265,20 +186,18 @@ rc = lsmFlushTreeToDisk(pDb); } /* Write a checkpoint to disk. */ if( rc==LSM_OK ){ - rc = lsmCheckpointWrite(pDb, 1, 0); + rc = lsmCheckpointWrite(pDb, 0); } - /* If the checkpoint was written successfully, delete the log file - ** and, if possible, truncate the database file. */ + /* If the checkpoint was written successfully, delete the log file */ if( rc==LSM_OK ){ Database *p = pDb->pDatabase; - dbTruncateFile(pDb); lsmFsCloseAndDeleteLog(pDb->pFS); - if( p->pFile && p->bMultiProc ) lsmEnvShmUnmap(pDb->pEnv, p->pFile, 1); + if( p->pFile ) lsmEnvShmUnmap(pDb->pEnv, p->pFile, 1); } } } lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_UNLOCK, 0); @@ -291,13 +210,12 @@ int nUs = 1000; /* us to wait between DMS1 attempts */ int rc; /* Obtain a pointer to the shared-memory header */ assert( pDb->pShmhdr==0 ); - rc = lsmShmCacheChunks(pDb, 1); + rc = lsmShmChunk(pDb, 0, (void **)&pDb->pShmhdr); if( rc!=LSM_OK ) return rc; - pDb->pShmhdr = (ShmHeader *)pDb->apShm[0]; /* Block for an exclusive lock on DMS1. This lock serializes all calls ** to doDbConnect() and doDbDisconnect() across all processes. */ while( 1 ){ rc = lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL, 1); @@ -323,21 +241,17 @@ } }else if( rc==LSM_BUSY ){ rc = LSM_OK; } - /* Take a shared lock on DMS2. In multi-process mode this lock "cannot" - ** fail, as connections may only hold an exclusive lock on DMS2 if they - ** first hold an exclusive lock on DMS1. And this connection is currently - ** holding the exclusive lock on DSM1. - ** - ** However, if some other connection has the database open in single-process - ** mode, this operation will fail. In this case, return the error to the - ** caller - the attempt to connect to the db has failed. - */ + /* Take a shared lock on DMS2. This lock "cannot" fail, as connections + ** may only hold an exclusive lock on DMS2 if they first hold an exclusive + ** lock on DMS1. And this connection is currently holding the exclusive + ** lock on DSM1. */ if( rc==LSM_OK ){ rc = lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_SHARED, 0); + assert( rc!=LSM_BUSY ); } /* If anything went wrong, unlock DMS2. Unlock DMS1 in any case. */ if( rc!=LSM_OK ){ lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_UNLOCK, 0); @@ -372,11 +286,11 @@ assert( pDb->pDatabase==0 ); rc = enterGlobalMutex(pEnv); if( rc==LSM_OK ){ /* Search the global list for an existing object. TODO: Need something - ** better than the memcmp() below to figure out if a given Database + ** better than the strcmp() below to figure out if a given Database ** object represents the requested file. */ for(p=gShared.pDatabase; p; p=p->pDbNext){ if( nName==p->nName && 0==memcmp(zName, p->zName, nName) ) break; } @@ -385,26 +299,21 @@ p = (Database *)lsmMallocZeroRc(pEnv, sizeof(Database)+nName+1, &rc); /* If the allocation was successful, fill in other fields and ** allocate the client mutex. */ if( rc==LSM_OK ){ - p->bMultiProc = pDb->bMultiProc; p->zName = (char *)&p[1]; p->nName = nName; memcpy((void *)p->zName, zName, nName+1); rc = lsmMutexNew(pEnv, &p->pClientMutex); } - /* If nothing has gone wrong so far, open the shared fd. And if that - ** succeeds and this connection requested single-process mode, - ** attempt to take the exclusive lock on DMS2. */ - if( rc==LSM_OK ){ + /* If running in multi-process mode and nothing has gone wrong so far, + ** open the shared fd */ + if( rc==LSM_OK && pDb->bMultiProc ){ rc = lsmEnvOpen(pDb->pEnv, p->zName, &p->pFile); } - if( rc==LSM_OK && p->bMultiProc==0 ){ - rc = lsmEnvLock(pDb->pEnv, p->pFile, LSM_LOCK_DMS2, LSM_LOCK_EXCL); - } if( rc==LSM_OK ){ p->pDbNext = gShared.pDatabase; gShared.pDatabase = p; }else{ @@ -432,13 +341,10 @@ rc = lsmFsOpen(pDb, zName); } if( rc==LSM_OK ){ rc = doDbConnect(pDb); } - if( rc==LSM_OK ){ - rc = lsmFsConfigure(pDb); - } return rc; } static void dbDeferClose(lsm_db *pDb){ @@ -477,37 +383,38 @@ } lsmMutexEnter(pDb->pEnv, p->pClientMutex); for(ppDb=&p->pConn; *ppDb!=pDb; ppDb=&((*ppDb)->pNext)); *ppDb = pDb->pNext; - dbDeferClose(pDb); + if( lsmDbMultiProc(pDb) ){ + dbDeferClose(pDb); + } lsmMutexLeave(pDb->pEnv, p->pClientMutex); enterGlobalMutex(pDb->pEnv); p->nDbRef--; if( p->nDbRef==0 ){ - LsmFile *pIter; - LsmFile *pNext; Database **pp; /* Remove the Database structure from the linked list. */ for(pp=&gShared.pDatabase; *pp!=p; pp=&((*pp)->pDbNext)); *pp = p->pDbNext; - /* If they were allocated from the heap, free the shared memory chunks */ - if( p->bMultiProc==0 ){ + /* Free the Database object and shared memory buffers. */ + if( p->pFile==0 ){ int i; for(i=0; inShmChunk; i++){ lsmFree(pDb->pEnv, p->apShmChunk[i]); } - } - - /* Close any outstanding file descriptors */ - for(pIter=p->pLsmFile; pIter; pIter=pNext){ - pNext = pIter->pNext; - lsmEnvClose(pDb->pEnv, pIter->pFile); - lsmFree(pDb->pEnv, pIter); + }else{ + LsmFile *pIter; + LsmFile *pNext; + for(pIter=p->pLsmFile; pIter; pIter=pNext){ + pNext = pIter->pNext; + lsmEnvClose(pDb->pEnv, pIter->pFile); + lsmFree(pDb->pEnv, pIter); + } } freeDatabase(pDb->pEnv, p); } leaveGlobalMutex(pDb->pEnv); } @@ -519,245 +426,71 @@ void lsmDbSnapshotSetLevel(Snapshot *pSnap, Level *pLevel){ pSnap->pLevel = pLevel; } -/* TODO: Shuffle things around to get rid of this */ -static int firstSnapshotInUse(lsm_db *, i64 *); - -/* -** Context object used by the lsmWalkFreelist() utility. -*/ -typedef struct WalkFreelistCtx WalkFreelistCtx; -struct WalkFreelistCtx { - lsm_db *pDb; - int bReverse; - Freelist *pFreelist; - int iFree; - int (*xUsr)(void *, int, i64); /* User callback function */ - void *pUsrctx; /* User callback context */ - int bDone; /* Set to true after xUsr() returns true */ -}; - -/* -** Callback used by lsmWalkFreelist(). -*/ -static int walkFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ - WalkFreelistCtx *p = (WalkFreelistCtx *)pCtx; - const int iDir = (p->bReverse ? -1 : 1); - Freelist *pFree = p->pFreelist; - - assert( p->bDone==0 ); - if( pFree ){ - while( (p->iFree < pFree->nEntry) && p->iFree>=0 ){ - FreelistEntry *pEntry = &pFree->aEntry[p->iFree]; - if( (p->bReverse==0 && pEntry->iBlk>iBlk) - || (p->bReverse!=0 && pEntry->iBlkiFree += iDir; - if( pEntry->iId>=0 - && p->xUsr(p->pUsrctx, pEntry->iBlk, pEntry->iId) - ){ - p->bDone = 1; - return 1; - } - if( pEntry->iBlk==iBlk ) return 0; - } - } - } - - if( p->xUsr(p->pUsrctx, iBlk, iSnapshot) ){ - p->bDone = 1; - return 1; - } - return 0; -} - -/* -** The database handle passed as the first argument must be the worker -** connection. This function iterates through the contents of the current -** free block list, invoking the supplied callback once for each list -** element. -** -** The difference between this function and lsmSortedWalkFreelist() is -** that lsmSortedWalkFreelist() only considers those free-list elements -** stored within the LSM. This function also merges in any in-memory -** elements. -*/ -int lsmWalkFreelist( - lsm_db *pDb, /* Database handle (must be worker) */ - int bReverse, /* True to iterate from largest to smallest */ - int (*x)(void *, int, i64), /* Callback function */ - void *pCtx /* First argument to pass to callback */ -){ - const int iDir = (bReverse ? -1 : 1); - int rc; - int iCtx; - - WalkFreelistCtx ctx[2]; - - ctx[0].pDb = pDb; - ctx[0].bReverse = bReverse; - ctx[0].pFreelist = &pDb->pWorker->freelist; - if( ctx[0].pFreelist && bReverse ){ - ctx[0].iFree = ctx[0].pFreelist->nEntry-1; - }else{ - ctx[0].iFree = 0; - } - ctx[0].xUsr = walkFreelistCb; - ctx[0].pUsrctx = (void *)&ctx[1]; - ctx[0].bDone = 0; - - ctx[1].pDb = pDb; - ctx[1].bReverse = bReverse; - ctx[1].pFreelist = pDb->pFreelist; - if( ctx[1].pFreelist && bReverse ){ - ctx[1].iFree = ctx[1].pFreelist->nEntry-1; - }else{ - ctx[1].iFree = 0; - } - ctx[1].xUsr = x; - ctx[1].pUsrctx = pCtx; - ctx[1].bDone = 0; - - rc = lsmSortedWalkFreelist(pDb, bReverse, walkFreelistCb, (void *)&ctx[0]); - - if( ctx[0].bDone==0 ){ - for(iCtx=0; iCtx<2; iCtx++){ - int i; - WalkFreelistCtx *p = &ctx[iCtx]; - for(i=p->iFree; - p->pFreelist && rc==LSM_OK && ipFreelist->nEntry && i>=0; - i += iDir - ){ - FreelistEntry *pEntry = &p->pFreelist->aEntry[i]; - if( pEntry->iId>=0 && p->xUsr(p->pUsrctx, pEntry->iBlk, pEntry->iId) ){ - return LSM_OK; - } - } - } - } - - return rc; -} - - -typedef struct FindFreeblockCtx FindFreeblockCtx; -struct FindFreeblockCtx { - i64 iInUse; - int iRet; - int bNotOne; -}; - -static int findFreeblockCb(void *pCtx, int iBlk, i64 iSnapshot){ - FindFreeblockCtx *p = (FindFreeblockCtx *)pCtx; - if( iSnapshotiInUse && (iBlk!=1 || p->bNotOne==0) ){ - p->iRet = iBlk; - return 1; - } - return 0; -} - -static int findFreeblock(lsm_db *pDb, i64 iInUse, int bNotOne, int *piRet){ - int rc; /* Return code */ - FindFreeblockCtx ctx; /* Context object */ - - ctx.iInUse = iInUse; - ctx.iRet = 0; - ctx.bNotOne = bNotOne; - rc = lsmWalkFreelist(pDb, 0, findFreeblockCb, (void *)&ctx); - *piRet = ctx.iRet; - - return rc; -} /* ** Allocate a new database file block to write data to, either by extending ** the database file or by recycling a free-list entry. The worker snapshot ** must be held in order to call this function. ** ** If successful, *piBlk is set to the block number allocated and LSM_OK is ** returned. Otherwise, *piBlk is zeroed and an lsm error code returned. */ -int lsmBlockAllocate(lsm_db *pDb, int iBefore, int *piBlk){ +int lsmBlockAllocate(lsm_db *pDb, int *piBlk){ Snapshot *p = pDb->pWorker; + Freelist *pFree; /* Database free list */ int iRet = 0; /* Block number of allocated block */ int rc = LSM_OK; - i64 iInUse = 0; /* Snapshot id still in use */ - i64 iSynced = 0; /* Snapshot id synced to disk */ - - assert( p ); - -#ifdef LSM_LOG_FREELIST - { - static int nCall = 0; - char *zFree = 0; - nCall++; - rc = lsmInfoFreelist(pDb, &zFree); - if( rc!=LSM_OK ) return rc; - lsmLogMessage(pDb, 0, "lsmBlockAllocate(): %d freelist: %s", nCall, zFree); - lsmFree(pDb->pEnv, zFree); - } -#endif - - /* Set iInUse to the smallest snapshot id that is either: - ** - ** * Currently in use by a database client, - ** * May be used by a database client in the future, or - ** * Is the most recently checkpointed snapshot (i.e. the one that will - ** be used following recovery if a failure occurs at this point). - */ - rc = lsmCheckpointSynced(pDb, &iSynced, 0, 0); - if( rc==LSM_OK && iSynced==0 ) iSynced = p->iId; - iInUse = iSynced; - if( rc==LSM_OK && pDb->iReader>=0 ){ - assert( pDb->pClient ); - iInUse = LSM_MIN(iInUse, pDb->pClient->iId); - } - if( rc==LSM_OK ) rc = firstSnapshotInUse(pDb, &iInUse); - -#ifdef LSM_LOG_FREELIST - { - lsmLogMessage(pDb, 0, "lsmBlockAllocate(): " - "snapshot-in-use: %lld (iSynced=%lld) (client-id=%lld)", - iInUse, iSynced, (pDb->iReader>=0 ? pDb->pClient->iId : 0) + + assert( pDb->pWorker ); + + pFree = &p->freelist; + if( pFree->nEntry>0 ){ + /* The first block on the free list was freed as part of the work done + ** to create the snapshot with id iFree. So, we can reuse this block if + ** snapshot iFree or later has been checkpointed and all currently + ** active clients are reading from snapshot iFree or later. */ + i64 iFree = pFree->aEntry[0].iId; + int bInUse = 0; + + /* The "is in use" bit */ + rc = lsmLsmInUse(pDb, iFree, &bInUse); + + /* The "has been checkpointed" bit */ + if( rc==LSM_OK && bInUse==0 ){ + i64 iId = 0; + rc = lsmCheckpointSynced(pDb, &iId, 0, 0); + if( rc!=LSM_OK || iIdaEntry[0].iBlk; + flRemoveEntry0(pFree); + assert( iRet!=0 ); + } +#ifdef LSM_LOG_BLOCKS + lsmLogMessage( + pDb, 0, "%s reusing block %d%s", (iRet==0 ? "not " : ""), + pFree->aEntry[0].iBlk, + bInUse==0 ? "" : bInUse==1 ? " (client)" : " (unsynced)" ); - } -#endif - - /* Query the free block list for a suitable block */ - if( rc==LSM_OK ) rc = findFreeblock(pDb, iInUse, (iBefore>0), &iRet); - - if( iBefore>0 && (iRet<=0 || iRet>=iBefore) ){ - iRet = 0; - - }else if( rc==LSM_OK ){ - /* If a block was found in the free block list, use it and remove it from - ** the list. Otherwise, if no suitable block was found, allocate one from - ** the end of the file. */ - if( iRet>0 ){ -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, 0, - "reusing block %d (snapshot-in-use=%lld)", iRet, iInUse); -#endif - rc = freelistAppend(pDb, iRet, -1); - if( rc==LSM_OK ){ - rc = dbTruncate(pDb, iInUse); - } - }else{ - iRet = ++(p->nBlock); -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, 0, "extending file to %d blocks", iRet); -#endif - } - } - - assert( iBefore>0 || iRet>0 || rc!=LSM_OK ); +#endif + } + + /* If no block was allocated from the free-list, allocate one at the + ** end of the file. */ + if( rc==LSM_OK && iRet==0 ){ + iRet = ++pDb->pWorker->nBlock; +#ifdef LSM_LOG_BLOCKS + lsmLogMessage(pDb, 0, "extending file to %d blocks", iRet); +#endif + } + *piBlk = iRet; - return rc; + return LSM_OK; } /* ** Free a database block. The worker snapshot must be held in order to call ** this function. @@ -765,17 +498,18 @@ ** If successful, LSM_OK is returned. Otherwise, an lsm error code (e.g. ** LSM_NOMEM). */ int lsmBlockFree(lsm_db *pDb, int iBlk){ Snapshot *p = pDb->pWorker; + assert( lsmShmAssertWorker(pDb) ); - + /* TODO: Should assert() that lsmCheckpointOverflow() has not been called */ #ifdef LSM_LOG_FREELIST lsmLogMessage(pDb, LSM_OK, "lsmBlockFree(): Free block %d", iBlk); #endif - return freelistAppend(pDb, iBlk, p->iId); + return lsmFreelistAppend(pDb->pEnv, &p->freelist, iBlk, p->iId); } /* ** Refree a database block. The worker snapshot must be held in order to call ** this function. @@ -786,16 +520,18 @@ ** block may be reused immediately. Whereas a freed block can not be reused ** until (at least) after the next checkpoint. */ int lsmBlockRefree(lsm_db *pDb, int iBlk){ int rc = LSM_OK; /* Return code */ + Snapshot *p = pDb->pWorker; -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, LSM_OK, "lsmBlockRefree(): Refree block %d", iBlk); -#endif + if( iBlk==p->nBlock ){ + p->nBlock--; + }else{ + rc = flInsertEntry(pDb->pEnv, &p->freelist, iBlk); + } - rc = freelistAppend(pDb, iBlk, 0); return rc; } /* ** If required, copy a database checkpoint from shared memory into the @@ -804,11 +540,11 @@ ** The WORKER lock must not be held when this is called. This is because ** this function may indirectly call fsync(). And the WORKER lock should ** not be held that long (in case it is required by a client flushing an ** in-memory tree to disk). */ -int lsmCheckpointWrite(lsm_db *pDb, int bTruncate, u32 *pnWrite){ +int lsmCheckpointWrite(lsm_db *pDb, u32 *pnWrite){ int rc; /* Return Code */ u32 nWrite = 0; assert( pDb->pWorker==0 ); assert( 1 || pDb->pClient==0 ); @@ -817,11 +553,10 @@ rc = lsmShmLock(pDb, LSM_LOCK_CHECKPOINTER, LSM_LOCK_EXCL, 0); if( rc!=LSM_OK ) return rc; rc = lsmCheckpointLoad(pDb, 0); if( rc==LSM_OK ){ - int nBlock = lsmCheckpointNBlock(pDb->aSnapshot); ShmHeader *pShm = pDb->pShmhdr; int bDone = 0; /* True if checkpoint is already stored */ /* Check if this checkpoint has already been written to the database ** file. If so, set variable bDone to true. */ @@ -842,31 +577,30 @@ bDone = (iDisk>=iCkpt); } if( rc==LSM_OK && bDone==0 ){ int iMeta = (pShm->iMetaPage % 2) + 1; +#if 0 + lsmLogMessage(pDb, 0, "starting checkpoint"); +#endif if( pDb->eSafety!=LSM_SAFETY_OFF ){ - rc = lsmFsSyncDb(pDb->pFS, nBlock); + rc = lsmFsSyncDb(pDb->pFS); } if( rc==LSM_OK ) rc = lsmCheckpointStore(pDb, iMeta); if( rc==LSM_OK && pDb->eSafety!=LSM_SAFETY_OFF){ - rc = lsmFsSyncDb(pDb->pFS, 0); + rc = lsmFsSyncDb(pDb->pFS); } if( rc==LSM_OK ){ pShm->iMetaPage = iMeta; nWrite = lsmCheckpointNWrite(pDb->aSnapshot, 0) - nWrite; } #ifdef LSM_LOG_WORK - lsmLogMessage(pDb, 0, "finish checkpoint %d", - (int)lsmCheckpointId(pDb->aSnapshot, 0) - ); + lsmLogMessage(pDb, 0, "finish checkpoint %d", + (int)lsmCheckpointId(pDb->aSnapshot, 0) + ); #endif } - - if( rc==LSM_OK && bTruncate ){ - rc = lsmFsTruncateDb(pDb->pFS, (i64)nBlock*lsmFsBlockSize(pDb->pFS)); - } } lsmShmLock(pDb, LSM_LOCK_CHECKPOINTER, LSM_LOCK_UNLOCK, 0); if( pnWrite && rc==LSM_OK ) *pnWrite = nWrite; return rc; @@ -879,19 +613,19 @@ rc = lsmShmLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL, 0); /* Deserialize the current worker snapshot */ if( rc==LSM_OK ){ rc = lsmCheckpointLoadWorker(pDb); + if( pDb->pWorker ) pDb->pWorker->pDatabase = pDb->pDatabase; } return rc; } void lsmFreeSnapshot(lsm_env *pEnv, Snapshot *p){ if( p ){ lsmSortedFreeLevel(pEnv, p->pLevel); lsmFree(pEnv, p->freelist.aEntry); - lsmFree(pEnv, p->redirect.a); lsmFree(pEnv, p); } } /* @@ -898,78 +632,35 @@ ** Argument bFlush is true if the contents of the in-memory tree has just ** been flushed to disk. The significance of this is that once the snapshot ** created to hold the updated state of the database is synced to disk, log ** file space can be recycled. */ -void lsmFinishWork(lsm_db *pDb, int bFlush, int *pRc){ - int rc = *pRc; - assert( rc!=0 || pDb->pWorker ); +void lsmFinishWork(lsm_db *pDb, int bFlush, int nOvfl, int *pRc){ + assert( *pRc!=0 || pDb->pWorker ); if( pDb->pWorker ){ /* If no error has occurred, serialize the worker snapshot and write ** it to shared memory. */ - if( rc==LSM_OK ){ - rc = lsmSaveWorker(pDb, bFlush); - } - - /* Assuming no error has occurred, update a read lock slot with the - ** new snapshot id (see comments above function lsmSetReadLock()). */ - if( rc==LSM_OK ){ - if( pDb->iReader<0 ){ - rc = lsmTreeLoadHeader(pDb, 0); - } - if( rc==LSM_OK ){ - rc = lsmSetReadLock(pDb, pDb->pWorker->iId, pDb->treehdr.iUsedShmid); - } - } - - /* Free the snapshot object. */ + assert( pDb->pWorker->nFreelistOvfl==0 || nOvfl==0 ); + if( *pRc==LSM_OK ){ + *pRc = lsmCheckpointSaveWorker(pDb, bFlush, nOvfl); + } lsmFreeSnapshot(pDb->pEnv, pDb->pWorker); pDb->pWorker = 0; } lsmShmLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_UNLOCK, 0); - *pRc = rc; } + /* ** Called when recovery is finished. */ int lsmFinishRecovery(lsm_db *pDb){ lsmTreeEndTransaction(pDb, 1); return LSM_OK; } -/* -** Check if the currently configured compression functions -** (LSM_CONFIG_SET_COMPRESSION) are compatible with a database that has its -** compression id set to iReq. Compression routines are compatible if iReq -** is zero (indicating the database is empty), or if it is equal to the -** compression id of the configured compression routines. -** -** If the check shows that the current compression are incompatible and there -** is a compression factory registered, give it a chance to install new -** compression routines. -** -** If, after any registered factory is invoked, the compression functions -** are still incompatible, return LSM_MISMATCH. Otherwise, LSM_OK. -*/ -int lsmCheckCompressionId(lsm_db *pDb, u32 iReq){ - if( iReq!=LSM_COMPRESSION_EMPTY && pDb->compress.iId!=iReq ){ - if( pDb->factory.xFactory ){ - pDb->bInFactory = 1; - pDb->factory.xFactory(pDb->factory.pCtx, pDb, iReq); - pDb->bInFactory = 0; - } - if( pDb->compress.iId!=iReq ){ - /* Incompatible */ - return LSM_MISMATCH; - } - } - /* Compatible */ - return LSM_OK; -} - /* ** Begin a read transaction. This function is a no-op if the connection ** passed as the only argument already has an open read transaction. */ int lsmBeginReadTrans(lsm_db *pDb){ @@ -1002,11 +693,11 @@ ** that the shared-memory still contains the same values. If so, proceed. ** Otherwise, relinquish the read-lock and retry the whole procedure ** (starting with loading the in-memory tree header). */ if( rc==LSM_OK ){ u32 iShmMax = pDb->treehdr.iUsedShmid; - u32 iShmMin = pDb->treehdr.iNextShmid+1-LSM_MAX_SHMCHUNKS; + u32 iShmMin = pDb->treehdr.iNextShmid+1-(1<<10); rc = lsmReadlock( pDb, lsmCheckpointId(pDb->aSnapshot, 0), iShmMin, iShmMax ); if( rc==LSM_OK ){ if( lsmTreeLoadHeaderOk(pDb, iTreehdr) @@ -1019,41 +710,31 @@ if( pDb->pClient==0 ){ rc = lsmCheckpointDeserialize(pDb, 0, pDb->aSnapshot,&pDb->pClient); } assert( (rc==LSM_OK)==(pDb->pClient!=0) ); assert( pDb->iReader>=0 ); - - /* Check that the client has the right compression hooks loaded. - ** If not, set rc to LSM_MISMATCH. */ - if( rc==LSM_OK ){ - rc = lsmCheckCompressionId(pDb, pDb->pClient->iCmpId); - } }else{ rc = lsmReleaseReadlock(pDb); } } - if( rc==LSM_BUSY ){ rc = LSM_OK; } } #if 0 if( rc==LSM_OK && pDb->pClient ){ - fprintf(stderr, - "reading %p: snapshot:%d used-shmid:%d trans-id:%d iOldShmid=%d\n", + printf("reading %p: snapshot:%d used-shmid:%d trans-id:%d iOldShmid=%d\n", (void *)pDb, (int)pDb->pClient->iId, (int)pDb->treehdr.iUsedShmid, (int)pDb->treehdr.root.iTransId, (int)pDb->treehdr.iOldShmid ); + fflush(stdout); } #endif } - if( rc==LSM_OK ){ - rc = lsmShmCacheChunks(pDb, pDb->treehdr.nChunk); - } if( rc!=LSM_OK ){ lsmReleaseReadlock(pDb); } if( pDb->pClient==0 && rc==LSM_OK ) rc = LSM_BUSY; return rc; @@ -1075,18 +756,10 @@ if( pClient ){ lsmFreeSnapshot(pDb->pEnv, pDb->pClient); pDb->pClient = 0; } #endif - -#if 0 -if( pDb->pClient && pDb->iReader>=0 ){ - fprintf(stderr, - "finished reading %p: snapshot:%d\n", (void *)pDb, (int)pDb->pClient->iId - ); -} -#endif if( pDb->iReader>=0 ) lsmReleaseReadlock(pDb); } /* ** Open a write transaction. @@ -1094,11 +767,10 @@ int lsmBeginWriteTrans(lsm_db *pDb){ int rc; /* Return code */ ShmHeader *pShm = pDb->pShmhdr; /* Shared memory header */ assert( pDb->nTransOpen==0 ); - assert( pDb->bDiscardOld==0 ); /* If there is no read-transaction open, open one now. */ rc = lsmBeginReadTrans(pDb); /* Attempt to take the WRITER lock */ @@ -1129,11 +801,10 @@ TreeHeader *p = &pDb->treehdr; pShm->bWriter = 1; p->root.iTransId++; if( lsmTreeHasOld(pDb) && p->iOldLog==pDb->pClient->iLogOff ){ lsmTreeDiscardOld(pDb); - pDb->bDiscardOld = 1; } }else{ lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_UNLOCK, 0); if( pDb->pCsr==0 ) lsmFinishReadTrans(pDb); } @@ -1162,20 +833,14 @@ bFlush = 1; lsmTreeMakeOld(pDb); } lsmTreeEndTransaction(pDb, bCommit); - if( rc==LSM_OK ){ - if( bFlush && pDb->bAutowork ){ - rc = lsmSortedAutoWork(pDb, 1); - }else if( bCommit && pDb->bDiscardOld ){ - rc = lsmSetReadLock(pDb, pDb->pClient->iId, pDb->treehdr.iUsedShmid); - } - } - pDb->bDiscardOld = 0; - lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_UNLOCK, 0); - + if( rc==LSM_OK && bFlush && pDb->bAutowork ){ + rc = lsmSortedAutoWork(pDb, 1); + } + lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_UNLOCK, 0); if( bFlush && pDb->bAutowork==0 && pDb->xWork ){ pDb->xWork(pDb, pDb->pWorkCtx); } return rc; } @@ -1196,57 +861,10 @@ && shm_sequence_ge(iShmMax, p->iTreeId) && shm_sequence_ge(p->iTreeId, iShmMin) ); } -/* -** Attempt to populate one of the read-lock slots to contain lock values -** iLsm/iShm. Or, if such a slot exists already, this function is a no-op. -** -** It is not an error if no slot can be populated because the write-lock -** cannot be obtained. If any other error occurs, return an LSM error code. -** Otherwise, LSM_OK. -** -** This function is called at various points to try to ensure that there -** always exists at least one read-lock slot that can be used by a read-only -** client. And so that, in the usual case, there is an "exact match" available -** whenever a read transaction is opened by any client. At present this -** function is called when: -** -** * A write transaction that called lsmTreeDiscardOld() is committed, and -** * Whenever the working snapshot is updated (i.e. lsmFinishWork()). -*/ -int lsmSetReadLock(lsm_db *db, i64 iLsm, u32 iShm){ - int rc = LSM_OK; - ShmHeader *pShm = db->pShmhdr; - int i; - - /* Check if there is already a slot containing the required values. */ - for(i=0; iaReader[i]; - if( p->iLsmId==iLsm && p->iTreeId==iShm ) return LSM_OK; - } - - /* Iterate through all read-lock slots, attempting to take a write-lock - ** on each of them. If a write-lock succeeds, populate the locked slot - ** with the required values and break out of the loop. */ - for(i=0; rc==LSM_OK && iaReader[i]; - p->iLsmId = iLsm; - p->iTreeId = iShm; - lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_UNLOCK, 0); - break; - } - } - - return rc; -} - /* ** Obtain a read-lock on database version identified by the combination ** of snapshot iLsm and tree iTree. Return LSM_OK if successful, or ** an LSM error code otherwise. */ @@ -1343,49 +961,10 @@ } *pbInUse = 0; return rc; } -/* -** This function is called by worker connections to determine the smallest -** snapshot id that is currently in use by a database client. The worker -** connection uses this result to determine whether or not it is safe to -** recycle a database block. -*/ -static int firstSnapshotInUse( - lsm_db *db, /* Database handle */ - i64 *piInUse /* IN/OUT: Smallest snapshot id in use */ -){ - ShmHeader *pShm = db->pShmhdr; - i64 iInUse = *piInUse; - int i; - - assert( iInUse>0 ); - for(i=0; iaReader[i]; - if( p->iLsmId ){ - i64 iThis = p->iLsmId; - if( iThis!=0 && iInUse>iThis ){ - int rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_EXCL, 0); - if( rc==LSM_OK ){ - p->iLsmId = 0; - lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_UNLOCK, 0); - }else if( rc==LSM_BUSY ){ - iInUse = iThis; - }else{ - /* Some error other than LSM_BUSY. Return the error code to - ** the caller in this case. */ - return rc; - } - } - } - } - - *piInUse = iInUse; - return LSM_OK; -} - int lsmTreeInUse(lsm_db *db, u32 iShmid, int *pbInUse){ if( db->treehdr.iUsedShmid==iShmid ){ *pbInUse = 1; return LSM_OK; } @@ -1416,11 +995,11 @@ ** This function may only be called after a successful call to ** lsmDbDatabaseConnect(). It returns true if the connection is in ** multi-process mode, or false otherwise. */ int lsmDbMultiProc(lsm_db *pDb){ - return pDb->pDatabase && pDb->pDatabase->bMultiProc; + return pDb->pDatabase && (pDb->pDatabase->pFile!=0); } /************************************************************************* ************************************************************************** @@ -1428,85 +1007,69 @@ ************************************************************************** ************************************************************************** *************************************************************************/ /* -** Ensure that database connection db has cached pointers to at least the -** first nChunk chunks of shared memory. -*/ -int lsmShmCacheChunks(lsm_db *db, int nChunk){ - int rc = LSM_OK; - if( nChunk>db->nShm ){ - static const int NINCR = 16; - Database *p = db->pDatabase; - lsm_env *pEnv = db->pEnv; - int nAlloc; - int i; - - /* Ensure that the db->apShm[] array is large enough. If an attempt to - ** allocate memory fails, return LSM_NOMEM immediately. The apShm[] array - ** is always extended in multiples of 16 entries - so the actual allocated - ** size can be inferred from nShm. */ - nAlloc = ((db->nShm + NINCR - 1) / NINCR) * NINCR; - while( nChunk>=nAlloc ){ - void **apShm; - nAlloc += NINCR; - apShm = lsmRealloc(pEnv, db->apShm, sizeof(void*)*nAlloc); - if( !apShm ) return LSM_NOMEM_BKPT; - db->apShm = apShm; - } - - /* Enter the client mutex */ - lsmMutexEnter(pEnv, p->pClientMutex); - - /* Extend the Database objects apShmChunk[] array if necessary. Using the - ** same pattern as for the lsm_db.apShm[] array above. */ - nAlloc = ((p->nShmChunk + NINCR - 1) / NINCR) * NINCR; - while( nChunk>=nAlloc ){ - void **apShm; - nAlloc += NINCR; - apShm = lsmRealloc(pEnv, p->apShmChunk, sizeof(void*)*nAlloc); - if( !apShm ){ - rc = LSM_NOMEM_BKPT; - break; - } - p->apShmChunk = apShm; - } - - for(i=db->nShm; rc==LSM_OK && i=p->nShmChunk ){ - void *pChunk = 0; - if( p->bMultiProc==0 ){ - /* Single process mode */ - pChunk = lsmMallocZeroRc(pEnv, LSM_SHM_CHUNK_SIZE, &rc); - }else{ - /* Multi-process mode */ - rc = lsmEnvShmMap(pEnv, p->pFile, i, LSM_SHM_CHUNK_SIZE, &pChunk); - } - if( rc==LSM_OK ){ - p->apShmChunk[i] = pChunk; - p->nShmChunk++; - } - } - if( rc==LSM_OK ){ - db->apShm[i] = p->apShmChunk[i]; - db->nShm++; - } - } - - /* Release the client mutex */ - lsmMutexLeave(pEnv, p->pClientMutex); - } - - return rc; -} - -static int lockSharedFile(lsm_env *pEnv, Database *p, int iLock, int eOp){ - int rc = LSM_OK; - if( p->bMultiProc ){ - rc = lsmEnvLock(pEnv, p->pFile, iLock, eOp); - } +** Retrieve a pointer to shared-memory chunk iChunk. Chunks are numbered +** starting from 0 (i.e. the header chunk is chunk 0). +*/ +int lsmShmChunk(lsm_db *db, int iChunk, void **ppData){ + int rc = LSM_OK; + void *pRet = 0; + Database *p = db->pDatabase; + lsm_env *pEnv = db->pEnv; + + while( iChunk>=db->nShm ){ + void **apShm; + apShm = lsmRealloc(pEnv, db->apShm, sizeof(void*)*(db->nShm+16)); + if( !apShm ) return LSM_NOMEM_BKPT; + memset(&apShm[db->nShm], 0, sizeof(void*)*16); + db->apShm = apShm; + db->nShm += 16; + } + if( db->apShm[iChunk] ){ + *ppData = db->apShm[iChunk]; + return rc; + } + + /* Enter the client mutex */ + assert( iChunk>=0 ); + lsmMutexEnter(pEnv, p->pClientMutex); + + if( iChunk>=p->nShmChunk ){ + int nNew = iChunk+1; + void **apNew; + apNew = (void **)lsmRealloc(pEnv, p->apShmChunk, sizeof(void*) * nNew); + if( apNew==0 ){ + rc = LSM_NOMEM_BKPT; + }else{ + memset(&apNew[p->nShmChunk], 0, sizeof(void*) * (nNew-p->nShmChunk)); + p->apShmChunk = apNew; + p->nShmChunk = nNew; + } + } + + if( rc==LSM_OK && p->apShmChunk[iChunk]==0 ){ + void *pChunk = 0; + if( p->pFile==0 ){ + /* Single process mode */ + pChunk = lsmMallocZeroRc(pEnv, LSM_SHM_CHUNK_SIZE, &rc); + }else{ + /* Multi-process mode */ + rc = lsmEnvShmMap(pEnv, p->pFile, iChunk, LSM_SHM_CHUNK_SIZE, &pChunk); + } + p->apShmChunk[iChunk] = pChunk; + } + + if( rc==LSM_OK ){ + pRet = p->apShmChunk[iChunk]; + } + + /* Release the client mutex */ + lsmMutexLeave(pEnv, p->pClientMutex); + + *ppData = db->apShm[iChunk] = pRet; return rc; } /* ** Attempt to obtain the lock identified by the iLock and bExcl parameters. @@ -1559,21 +1122,21 @@ assert( nExcl==0 || (db->mLock & (me|ms))==0 ); switch( eOp ){ case LSM_LOCK_UNLOCK: if( nShared==0 ){ - lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_UNLOCK); + lsmEnvLock(db->pEnv, p->pFile, iLock, LSM_LOCK_UNLOCK); } db->mLock &= ~(me|ms); break; case LSM_LOCK_SHARED: if( nExcl ){ rc = LSM_BUSY; }else{ if( nShared==0 ){ - rc = lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_SHARED); + rc = lsmEnvLock(db->pEnv, p->pFile, iLock, LSM_LOCK_SHARED); } db->mLock |= ms; db->mLock &= ~me; } break; @@ -1581,11 +1144,11 @@ default: assert( eOp==LSM_LOCK_EXCL ); if( nExcl || nShared ){ rc = LSM_BUSY; }else{ - rc = lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_EXCL); + rc = lsmEnvLock(db->pEnv, p->pFile, iLock, LSM_LOCK_EXCL); db->mLock |= (me|ms); } break; } @@ -1689,26 +1252,26 @@ void lsmShmBarrier(lsm_db *db){ lsmEnvShmBarrier(db->pEnv); } -int lsm_checkpoint(lsm_db *pDb, int *pnKB){ +int lsm_checkpoint(lsm_db *pDb, int *pnByte){ int rc; /* Return code */ u32 nWrite = 0; /* Number of pages checkpointed */ /* Attempt the checkpoint. If successful, nWrite is set to the number of ** pages written between this and the previous checkpoint. */ - rc = lsmCheckpointWrite(pDb, 0, &nWrite); + rc = lsmCheckpointWrite(pDb, &nWrite); - /* If required, calculate the output variable (KB of data checkpointed). + /* If required, calculate the output variable (bytes of data checkpointed). ** Set it to zero if an error occured. */ - if( pnKB ){ - int nKB = 0; + if( pnByte ){ + int nByte = 0; if( rc==LSM_OK && nWrite ){ - nKB = (((i64)nWrite * lsmFsPageSize(pDb->pFS)) + 1023) / 1024; + nByte = (int)nWrite * lsmFsPageSize(pDb->pFS); } - *pnKB = nKB; + *pnByte = nByte; } return rc; } Index: src/lsm_sorted.c ================================================================== --- src/lsm_sorted.c +++ src/lsm_sorted.c @@ -70,13 +70,10 @@ #ifndef _LSM_INT_H # include "lsmInt.h" #endif #include "sqlite4.h" /* only for sqlite4_snprintf() */ -#define LSM_LOG_STRUCTURE 0 -#define LSM_LOG_DATA 0 - /* ** Macros to help decode record types. */ #define rtTopic(eType) ((eType) & LSM_SYSTEMKEY) #define rtIsDelete(eType) (((eType) & 0x0F)==LSM_POINT_DELETE) @@ -185,15 +182,10 @@ ** lsmMCursorFirst() ** lsmMCursorLast() ** lsmMCursorKey() ** lsmMCursorValue() ** lsmMCursorValid() -** -** iFree: -** This variable is only used by cursors providing input data for a -** new top-level segment. Such cursors only ever iterate forwards, not -** backwards. */ struct MultiCursor { lsm_db *pDb; /* Connection that owns this cursor */ MultiCursor *pNext; /* Next cursor owned by connection pDb */ int flags; /* Mask of CURSOR_XXX flags */ @@ -202,42 +194,45 @@ Blob key; /* Cache of current key (or NULL) */ Blob val; /* Cache of current value */ /* All the component cursors: */ TreeCursor *apTreeCsr[2]; /* Up to two tree cursors */ - int iFree; /* Next element of free-list (-ve for eof) */ SegmentPtr *aPtr; /* Array of segment pointers */ int nPtr; /* Size of array aPtr[] */ BtreeCursor *pBtCsr; /* b-tree cursor (db writes only) */ /* Comparison results */ int nTree; /* Size of aTree[] array */ int *aTree; /* Array of comparison results */ /* Used by cursors flushing the in-memory tree only */ + int *pnOvfl; /* Number of free-list entries to store */ void *pSystemVal; /* Pointer to buffer to free */ /* Used by worker cursors only */ Pgno *pPrevMergePtr; }; -/* -** The following constants are used to assign integers to each component -** cursor of a multi-cursor. -*/ -#define CURSOR_DATA_TREE0 0 /* Current tree cursor (apTreeCsr[0]) */ -#define CURSOR_DATA_TREE1 1 /* The "old" tree, if any (apTreeCsr[1]) */ -#define CURSOR_DATA_SYSTEM 2 /* Free-list entries (new-toplevel only) */ -#define CURSOR_DATA_SEGMENT 3 /* First segment pointer (aPtr[0]) */ +#define CURSOR_DATA_TREE0 0 /* Current tree cursor */ +#define CURSOR_DATA_TREE1 1 /* The "old" tree, if any */ +#define CURSOR_DATA_SYSTEM 2 +#define CURSOR_DATA_SEGMENT 3 /* ** CURSOR_IGNORE_DELETE ** If set, this cursor will not visit SORTED_DELETE keys. ** -** CURSOR_FLUSH_FREELIST -** This cursor is being used to create a new toplevel. It should also -** iterate through the contents of the in-memory free block list. +** CURSOR_NEW_SYSTEM +** If set, then after all user data from the in-memory tree and any other +** cursors has been visited, the cursor visits the live (uncommitted) +** versions of the two system keys: FREELIST AND LEVELS. This is used when +** flushing the in-memory tree to disk - the new free-list and levels record +** are flushed along with it. +** +** CURSOR_AT_FREELIST +** This flag is set when sub-cursor CURSOR_DATA_SYSTEM is actually +** pointing at a free list. ** ** CURSOR_IGNORE_SYSTEM ** If set, this cursor ignores system keys. ** ** CURSOR_NEXT_OK @@ -254,11 +249,12 @@ ** Cursor has undergone a successful lsm_csr_seek(LSM_SEEK_EQ) operation. ** The key and value are stored in MultiCursor.key and MultiCursor.val ** respectively. */ #define CURSOR_IGNORE_DELETE 0x00000001 -#define CURSOR_FLUSH_FREELIST 0x00000002 +#define CURSOR_NEW_SYSTEM 0x00000002 +#define CURSOR_AT_FREELIST 0x00000004 #define CURSOR_IGNORE_SYSTEM 0x00000010 #define CURSOR_NEXT_OK 0x00000020 #define CURSOR_PREV_OK 0x00000040 #define CURSOR_READ_SEPARATORS 0x00000080 #define CURSOR_SEEK_EQ 0x00000100 @@ -343,11 +339,11 @@ + ((u32)aOut[1] << 16) + ((u32)aOut[2] << 8) + ((u32)aOut[3]); } -u64 lsmGetU64(u8 *aOut){ +u32 lsmGetU64(u8 *aOut){ return ((u64)aOut[0] << 56) + ((u64)aOut[1] << 48) + ((u64)aOut[2] << 40) + ((u64)aOut[3] << 32) + ((u64)aOut[4] << 24) @@ -369,11 +365,11 @@ static int sortedBlobGrow(lsm_env *pEnv, Blob *pBlob, int nData){ assert( pBlob->pEnv==pEnv || (pBlob->pEnv==0 && pBlob->pData==0) ); if( pBlob->nAllocpData = lsmReallocOrFree(pEnv, pBlob->pData, nData); - if( !pBlob->pData ) return LSM_NOMEM_BKPT; + if( !pBlob->pData ) return LSM_NOMEM; pBlob->nAlloc = nData; pBlob->pEnv = pEnv; } return LSM_OK; } @@ -396,11 +392,10 @@ if( pBlob->pData ) lsmFree(pBlob->pEnv, pBlob->pData); memset(pBlob, 0, sizeof(Blob)); } static int sortedReadData( - Segment *pSeg, Page *pPg, int iOff, int nByte, void **ppData, Blob *pBlob @@ -450,11 +445,11 @@ i -= iEnd; /* Grab the next page in the segment */ do { - rc = lsmFsDbPageNext(pSeg, pPg, 1, &pNext); + rc = lsmFsDbPageNext(0, pPg, 1, &pNext); if( rc==LSM_OK && pNext==0 ){ rc = LSM_CORRUPT_BKPT; } if( rc ) break; lsmFsPageRelease(pPg); @@ -511,11 +506,10 @@ lsmVarintGet64(&aCell[1], &iRet); return iRet; } static u8 *pageGetKey( - Segment *pSeg, /* Segment pPg belongs to */ Page *pPg, /* Page to read from */ int iCell, /* Index of cell on page to read */ int *piTopic, /* OUT: Topic associated with this key */ int *pnKey, /* OUT: Size of key in bytes */ Blob *pBlob /* If required, use this for dynamic memory */ @@ -538,27 +532,26 @@ if( rtIsWrite(eType) ){ pKey += lsmVarintGet32(pKey, &nDummy); } *piTopic = rtTopic(eType); - sortedReadData(pSeg, pPg, pKey-aData, *pnKey, (void **)&pKey, pBlob); + sortedReadData(pPg, pKey-aData, *pnKey, (void **)&pKey, pBlob); return pKey; } static int pageGetKeyCopy( lsm_env *pEnv, /* Environment handle */ - Segment *pSeg, /* Segment pPg belongs to */ Page *pPg, /* Page to read from */ int iCell, /* Index of cell on page to read */ int *piTopic, /* OUT: Topic associated with this key */ Blob *pBlob /* If required, use this for dynamic memory */ ){ int rc = LSM_OK; int nKey; u8 *aKey; - aKey = pageGetKey(pSeg, pPg, iCell, piTopic, &nKey, pBlob); + aKey = pageGetKey(pPg, iCell, piTopic, &nKey, pBlob); assert( (void *)aKey!=pBlob->pData || nKey==pBlob->nData ); if( (void *)aKey!=pBlob->pData ){ rc = sortedBlobSet(pEnv, pBlob, aKey, nKey); } @@ -583,11 +576,10 @@ #define GETVARINT64(a, i) (((i)=((u8*)(a))[0])<=240?1:lsmVarintGet64((a), &(i))) #define GETVARINT32(a, i) (((i)=((u8*)(a))[0])<=240?1:lsmVarintGet32((a), &(i))) static int pageGetBtreeKey( - Segment *pSeg, /* Segment page pPg belongs to */ Page *pPg, int iKey, Pgno *piPtr, int *piTopic, void **ppKey, @@ -610,13 +602,13 @@ if( eType==0 ){ int rc; Pgno iRef; /* Page number of referenced page */ Page *pRef; aCell += GETVARINT64(aCell, iRef); - rc = lsmFsDbPageGet(lsmPageFS(pPg), pSeg, iRef, &pRef); + rc = lsmFsDbPageGet(lsmPageFS(pPg), iRef, &pRef); if( rc!=LSM_OK ) return rc; - pageGetKeyCopy(lsmPageEnv(pPg), pSeg, pRef, 0, &eType, pBlob); + pageGetKeyCopy(lsmPageEnv(pPg), pRef, 0, &eType, pBlob); lsmFsPageRelease(pRef); *ppKey = pBlob->pData; *pnKey = pBlob->nData; }else{ aCell += GETVARINT32(aCell, *pnKey); @@ -641,11 +633,10 @@ iCell = pCsr->aPg[iPg].iCell-1; } if( iPg<0 || iCell<0 ) return LSM_CORRUPT_BKPT; rc = pageGetBtreeKey( - pCsr->pSeg, pCsr->aPg[iPg].pPage, iCell, &dummy, &pCsr->eType, &pCsr->pKey, &pCsr->nKey, &pCsr->blob ); pCsr->eType |= LSM_SEPARATOR; } @@ -703,11 +694,11 @@ iLoad = btreeCursorPtr(aData, nData, pPg->iCell); do { Page *pLoad; pCsr->iPg++; - rc = lsmFsDbPageGet(pCsr->pFS, pCsr->pSeg, iLoad, &pLoad); + rc = lsmFsDbPageGet(pCsr->pFS, iLoad, &pLoad); pCsr->aPg[pCsr->iPg].pPage = pLoad; pCsr->aPg[pCsr->iPg].iCell = 0; if( rc==LSM_OK ){ if( pCsr->iPg==(pCsr->nDepth-1) ) break; aData = fsPageData(pLoad, &nData); @@ -748,11 +739,11 @@ Page *pPg = 0; FileSystem *pFS = pCsr->pFS; int iPg = pCsr->pSeg->iRoot; do { - rc = lsmFsDbPageGet(pFS, pCsr->pSeg, iPg, &pPg); + rc = lsmFsDbPageGet(pFS, iPg, &pPg); assert( (rc==LSM_OK)==(pPg!=0) ); if( rc==LSM_OK ){ u8 *aData; int nData; int flags; @@ -838,11 +829,10 @@ if( p->iPg ){ lsm_env *pEnv = lsmFsEnv(pCsr->pFS); int iCell; /* Current cell number on leaf page */ Pgno iLeaf; /* Page number of current leaf page */ int nDepth; /* Depth of b-tree structure */ - Segment *pSeg = pCsr->pSeg; /* Decode the MergeInput structure */ iLeaf = p->iPg; nDepth = (p->iCell & 0x00FF); iCell = (p->iCell >> 8) - 1; @@ -851,25 +841,24 @@ assert( pCsr->aPg==0 ); pCsr->aPg = (BtreePg *)lsmMallocZeroRc(pEnv, sizeof(BtreePg) * nDepth, &rc); /* Populate the last entry of the aPg[] array */ if( rc==LSM_OK ){ - Page **pp = &pCsr->aPg[nDepth-1].pPage; pCsr->iPg = nDepth-1; pCsr->nDepth = nDepth; pCsr->aPg[pCsr->iPg].iCell = iCell; - rc = lsmFsDbPageGet(pCsr->pFS, pSeg, iLeaf, pp); + rc = lsmFsDbPageGet(pCsr->pFS, iLeaf, &pCsr->aPg[nDepth-1].pPage); } /* Populate any other aPg[] array entries */ if( rc==LSM_OK && nDepth>1 ){ Blob blob = {0,0,0}; void *pSeek; int nSeek; int iTopicSeek; int iPg = 0; - int iLoad = pSeg->iRoot; + int iLoad = pCsr->pSeg->iRoot; Page *pPg = pCsr->aPg[nDepth-1].pPage; if( pageObjGetNRec(pPg)==0 ){ /* This can happen when pPg is the right-most leaf in the b-tree. ** In this case, set the iTopicSeek/pSeek/nSeek key to a value @@ -878,18 +867,18 @@ iTopicSeek = 1000; pSeek = 0; nSeek = 0; }else{ Pgno dummy; - rc = pageGetBtreeKey(pSeg, pPg, + rc = pageGetBtreeKey(pPg, 0, &dummy, &iTopicSeek, &pSeek, &nSeek, &pCsr->blob ); } do { Page *pPg; - rc = lsmFsDbPageGet(pCsr->pFS, pSeg, iLoad, &pPg); + rc = lsmFsDbPageGet(pCsr->pFS, iLoad, &pPg); assert( rc==LSM_OK || pPg==0 ); if( rc==LSM_OK ){ u8 *aData; /* Buffer containing page data */ int nData; /* Size of aData[] in bytes */ int iMin; @@ -909,13 +898,11 @@ void *pKey; int nKey; /* Key for cell iTry */ int iTopic; /* Topic for key pKeyT/nKeyT */ Pgno iPtr; /* Pointer for cell iTry */ int res; /* (pSeek - pKeyT) */ - rc = pageGetBtreeKey( - pSeg, pPg, iTry, &iPtr, &iTopic, &pKey, &nKey, &blob - ); + rc = pageGetBtreeKey(pPg, iTry, &iPtr, &iTopic, &pKey, &nKey,&blob); if( rc!=LSM_OK ) break; res = sortedKeyCompare( xCmp, iTopicSeek, pSeek, nSeek, iTopic, pKey, nKey ); @@ -931,13 +918,11 @@ } pCsr->aPg[iPg].pPage = pPg; pCsr->aPg[iPg].iCell = iCell; iPg++; - assert( iPg!=nDepth-1 - || lsmFsRedirectPage(pCsr->pFS, pSeg->pRedirect, iLoad)==iLeaf - ); + assert( iPg!=nDepth-1 || iLoad==iLeaf ); } }while( rc==LSM_OK && iPg<(nDepth-1) ); sortedBlobFree(&blob); } @@ -955,11 +940,11 @@ int i; for(i=pCsr->iPg-1; i>=0; i--){ if( pCsr->aPg[i].iCell>0 ) break; } assert( i>=0 ); - rc = pageGetBtreeKey(pSeg, + rc = pageGetBtreeKey( pCsr->aPg[i].pPage, pCsr->aPg[i].iCell-1, &dummy, &pCsr->eType, &pCsr->pKey, &pCsr->nKey, &pCsr->blob ); pCsr->eType |= LSM_SEPARATOR; @@ -1012,11 +997,11 @@ int iNew /* Page number of new page */ ){ Page *pPg = 0; /* The new page */ int rc; /* Return Code */ - rc = lsmFsDbPageGet(pFS, pPtr->pSeg, iNew, &pPg); + rc = lsmFsDbPageGet(pFS, iNew, &pPg); assert( rc==LSM_OK || pPg==0 ); segmentPtrSetPage(pPtr, pPg); return rc; } @@ -1026,11 +1011,11 @@ int iOff, int nByte, void **ppData, Blob *pBlob ){ - return sortedReadData(pPtr->pSeg, pPtr->pPg, iOff, nByte, ppData, pBlob); + return sortedReadData(pPtr->pPg, iOff, nByte, ppData, pBlob); } static int segmentPtrNextPage( SegmentPtr *pPtr, /* Load page into this SegmentPtr object */ int eDir /* +1 for next(), -1 for prev() */ @@ -1085,39 +1070,18 @@ } return rc; } - -static Segment *sortedSplitkeySegment(Level *pLevel){ - Merge *pMerge = pLevel->pMerge; - MergeInput *p = &pMerge->splitkey; - Segment *pSeg; - int i; - - for(i=0; inInput; i++){ - if( p->iPg==pMerge->aInput[i].iPg ) break; - } - if( pMerge->nInput==(pLevel->nRight+1) && i>=(pMerge->nInput-1) ){ - pSeg = &pLevel->pNext->lhs; - }else{ - pSeg = &pLevel->aRhs[i]; - } - - return pSeg; -} - -static void sortedSplitkey(lsm_db *pDb, Level *pLevel, int *pRc){ - Segment *pSeg; +void lsmSortedSplitkey(lsm_db *pDb, Level *pLevel, int *pRc){ Page *pPg = 0; lsm_env *pEnv = pDb->pEnv; /* Environment handle */ int rc = *pRc; Merge *pMerge = pLevel->pMerge; - pSeg = sortedSplitkeySegment(pLevel); if( rc==LSM_OK ){ - rc = lsmFsDbPageGet(pDb->pFS, pSeg, pMerge->splitkey.iPg, &pPg); + rc = lsmFsDbPageGet(pDb->pFS, pMerge->splitkey.iPg, &pPg); } if( rc==LSM_OK ){ int iTopic; Blob blob = {0, 0, 0, 0}; u8 *aData; @@ -1126,20 +1090,18 @@ aData = lsmFsPageData(pPg, &nData); if( pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG ){ void *pKey; int nKey; Pgno dummy; - rc = pageGetBtreeKey(pSeg, + rc = pageGetBtreeKey( pPg, pMerge->splitkey.iCell, &dummy, &iTopic, &pKey, &nKey, &blob ); if( rc==LSM_OK && blob.pData!=pKey ){ rc = sortedBlobSet(pEnv, &blob, pKey, nKey); } }else{ - rc = pageGetKeyCopy( - pEnv, pSeg, pPg, pMerge->splitkey.iCell, &iTopic, &blob - ); + rc = pageGetKeyCopy(pEnv, pPg, pMerge->splitkey.iCell, &iTopic, &blob); } pLevel->iSplitTopic = iTopic; pLevel->pSplitKey = blob.pData; pLevel->nSplitKey = blob.nData; @@ -1207,17 +1169,14 @@ rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey ); if( res<0 ) segmentPtrReset(pPtr); } - if( pPtr->pPg==0 && (svFlags & LSM_END_DELETE) ){ - Segment *pSeg = pPtr->pSeg; - rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, pSeg->iFirst, &pPtr->pPg); + rc = lsmFsDbPageGet(pCsr->pDb->pFS, pPtr->pSeg->iFirst, &pPtr->pPg); if( rc!=LSM_OK ) return rc; - pPtr->eType = LSM_START_DELETE | LSM_POINT_DELETE; - pPtr->eType |= (pLvl->iSplitTopic ? LSM_SYSTEMKEY : 0); + pPtr->eType = LSM_START_DELETE | (pLvl->iSplitTopic ? LSM_SYSTEMKEY : 0); pPtr->pKey = pLvl->pSplitKey; pPtr->nKey = pLvl->nSplitKey; } }while( pCsr @@ -1234,16 +1193,15 @@ SegmentPtr *pPtr, int bLast, int *pRc ){ if( *pRc==LSM_OK ){ - Segment *pSeg = pPtr->pSeg; Page *pNew = 0; if( bLast ){ - *pRc = lsmFsDbPageLast(pFS, pSeg, &pNew); + *pRc = lsmFsDbPageLast(pFS, pPtr->pSeg, &pNew); }else{ - *pRc = lsmFsDbPageGet(pFS, pSeg, pSeg->iFirst, &pNew); + *pRc = lsmFsDbPageGet(pFS, pPtr->pSeg->iFirst, &pNew); } segmentPtrSetPage(pPtr, pNew); } } @@ -1269,25 +1227,18 @@ rc = segmentPtrNextPage(pPtr, (bLast ? -1 : 1)); } if( rc==LSM_OK && pPtr->pPg ){ rc = segmentPtrLoadCell(pPtr, bLast ? (pPtr->nCell-1) : 0); - if( rc==LSM_OK && bLast && pPtr->pSeg!=&pLvl->lhs ){ - int res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - if( res<0 ) segmentPtrReset(pPtr); - } } bIgnore = segmentPtrIgnoreSeparators(pCsr, pPtr); if( rc==LSM_OK && pPtr->pPg && bIgnore && rtIsSeparator(pPtr->eType) ){ rc = segmentPtrAdvance(pCsr, pPtr, bLast); } -#if 0 + if( bLast && rc==LSM_OK && pPtr->pPg && pPtr->pSeg==&pLvl->lhs && pLvl->nRight && (pPtr->eType & LSM_START_DELETE) ){ pPtr->iCell++; @@ -1295,11 +1246,10 @@ pPtr->pKey = pLvl->pSplitKey; pPtr->nKey = pLvl->nSplitKey; pPtr->pVal = 0; pPtr->nVal = 0; } -#endif return rc; } static void segmentPtrKey(SegmentPtr *pPtr, void **ppKey, int *pnKey){ @@ -1340,14 +1290,13 @@ for(eDir=-1; eDir<=1; eDir+=2){ Page *pTest = pPtr->pPg; lsmFsPageRef(pTest); while( pTest ){ - Segment *pSeg = pPtr->pSeg; Page *pNext; - int rc = lsmFsDbPageNext(pSeg, pTest, eDir, &pNext); + int rc = lsmFsDbPageNext(pPtr->pSeg, pTest, eDir, &pNext); lsmFsPageRelease(pTest); if( rc ) return 1; pTest = pNext; if( pTest ){ @@ -1361,11 +1310,11 @@ u8 *pPgKey; int res; int iCell; iCell = ((eDir < 0) ? (nCell-1) : 0); - pPgKey = pageGetKey(pSeg, pTest, iCell, &iPgTopic, &nPgKey, &blob); + pPgKey = pageGetKey(pTest, iCell, &iPgTopic, &nPgKey, &blob); res = iTopic - iPgTopic; if( res==0 ) res = pCsr->pDb->xCmp(pKey, nKey, pPgKey, nPgKey); if( (eDir==1 && res>0) || (eDir==-1 && res<0) ){ /* Taking this branch means something has gone wrong. */ char *zMsg = lsmMallocPrintf(pEnv, "Key \"%s\" is not on page %d", @@ -1411,11 +1360,10 @@ #endif static int segmentPtrSearchOversized( MultiCursor *pCsr, /* Cursor context */ SegmentPtr *pPtr, /* Pointer to seek */ - int iTopic, /* Topic of key to search for */ void *pKey, int nKey /* Key to seek to */ ){ int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; int rc = LSM_OK; @@ -1431,19 +1379,19 @@ int iLastTopic; int res; /* Result of comparison */ Page *pNext; /* Load the last key on the current page. */ - pLastKey = pageGetKey(pPtr->pSeg, + pLastKey = pageGetKey( pPtr->pPg, pPtr->nCell-1, &iLastTopic, &nLastKey, &pPtr->blob1 ); /* If the loaded key is >= than (pKey/nKey), break out of the loop. ** If (pKey/nKey) is present in this array, it must be on the current ** page. */ res = sortedKeyCompare( - xCmp, iLastTopic, pLastKey, nLastKey, iTopic, pKey, nKey + xCmp, iLastTopic, pLastKey, nLastKey, 0, pKey, nKey ); if( res>=0 ) break; /* Advance to the next page that contains at least one key. */ pNext = pPtr->pPg; @@ -1619,11 +1567,10 @@ } static int segmentPtrSeek( MultiCursor *pCsr, /* Cursor context */ SegmentPtr *pPtr, /* Pointer to seek */ - int iTopic, /* Key topic to seek to */ void *pKey, int nKey, /* Key to seek to */ int eSeek, /* Search bias - see above */ int *piPtr, /* OUT: FC pointer */ int *pbStop ){ @@ -1631,16 +1578,17 @@ int res; /* Result of comparison operation */ int rc = LSM_OK; int iMin; int iMax; Pgno iPtrOut = 0; + const int iTopic = 0; /* If the current page contains an oversized entry, then there are no ** pointers to one or more of the subsequent pages in the sorted run. ** The following call ensures that the segment-ptr points to the correct ** page in this case. */ - rc = segmentPtrSearchOversized(pCsr, pPtr, iTopic, pKey, nKey); + rc = segmentPtrSearchOversized(pCsr, pPtr, pKey, nKey); iPtrOut = pPtr->iPtr; /* Assert that this page is the right page of this segment for the key ** that we are searching for. Do this by loading page (iPg-1) and testing ** that pKey/nKey is greater than all keys on that page, and then by @@ -1755,30 +1703,30 @@ } static int seekInBtree( MultiCursor *pCsr, /* Multi-cursor object */ Segment *pSeg, /* Seek within this segment */ - int iTopic, void *pKey, int nKey, /* Key to seek to */ Pgno *aPg, /* OUT: Page numbers */ Page **ppPg /* OUT: Leaf (sorted-run) page reference */ ){ int i = 0; int rc; int iPg; Page *pPg = 0; Blob blob = {0, 0, 0}; + int iTopic = 0; /* TODO: Fix me */ iPg = pSeg->iRoot; do { Pgno *piFirst = 0; if( aPg ){ aPg[i++] = iPg; piFirst = &aPg[i]; } - rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, iPg, &pPg); + rc = lsmFsDbPageGet(pCsr->pDb->pFS, iPg, &pPg); assert( rc==LSM_OK || pPg==0 ); if( rc==LSM_OK ){ u8 *aData; /* Buffer containing page data */ int nData; /* Size of aData[] in bytes */ int iMin; @@ -1800,13 +1748,11 @@ void *pKeyT; int nKeyT; /* Key for cell iTry */ int iTopicT; /* Topic for key pKeyT/nKeyT */ Pgno iPtr; /* Pointer associated with cell iTry */ int res; /* (pKey - pKeyT) */ - rc = pageGetBtreeKey( - pSeg, pPg, iTry, &iPtr, &iTopicT, &pKeyT, &nKeyT, &blob - ); + rc = pageGetBtreeKey(pPg, iTry, &iPtr, &iTopicT, &pKeyT, &nKeyT, &blob); if( rc!=LSM_OK ) break; if( piFirst && pKeyT==blob.pData ){ *piFirst = pageGetBtreeRef(pPg, iTry); piFirst = 0; i++; @@ -1838,11 +1784,10 @@ } static int seekInSegment( MultiCursor *pCsr, SegmentPtr *pPtr, - int iTopic, void *pKey, int nKey, int iPg, /* Page to search */ int eSeek, /* Search bias - see above */ int *piPtr, /* OUT: FC pointer */ int *pbStop /* OUT: Stop search flag */ @@ -1851,11 +1796,11 @@ int rc = LSM_OK; if( pPtr->pSeg->iRoot ){ Page *pPg; assert( pPtr->pSeg->iRoot!=0 ); - rc = seekInBtree(pCsr, pPtr->pSeg, iTopic, pKey, nKey, 0, &pPg); + rc = seekInBtree(pCsr, pPtr->pSeg, pKey, nKey, 0, &pPg); if( rc==LSM_OK ) segmentPtrSetPage(pPtr, pPg); }else{ if( iPtr==0 ){ iPtr = pPtr->pSeg->iFirst; } @@ -1863,11 +1808,11 @@ rc = segmentPtrLoadPage(pCsr->pDb->pFS, pPtr, iPtr); } } if( rc==LSM_OK ){ - rc = segmentPtrSeek(pCsr, pPtr, iTopic, pKey, nKey, eSeek, piPtr, pbStop); + rc = segmentPtrSeek(pCsr, pPtr, pKey, nKey, eSeek, piPtr, pbStop); } return rc; } /* @@ -1886,11 +1831,10 @@ */ static int seekInLevel( MultiCursor *pCsr, /* Sorted cursor object to seek */ SegmentPtr *aPtr, /* Pointer to array of (nRhs+1) SPs */ int eSeek, /* Search bias - see above */ - int iTopic, /* Key topic to search for */ void *pKey, int nKey, /* Key to search for */ Pgno *piPgno, /* IN/OUT: fraction cascade pointer (or 0) */ int *pbStop /* OUT: See above */ ){ Level *pLvl = aPtr[0].pLevel; /* Level to seek within */ @@ -1902,11 +1846,11 @@ /* If this is a composite level (one currently undergoing an incremental ** merge), figure out if the search key is larger or smaller than the ** levels split-key. */ if( nRhs ){ - res = sortedKeyCompare(pCsr->pDb->xCmp, iTopic, pKey, nKey, + res = sortedKeyCompare(pCsr->pDb->xCmp, 0, pKey, nKey, pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey ); } /* If (res<0), then key pKey/nKey is smaller than the split-key (or this @@ -1914,44 +1858,26 @@ ** left-hand-side of the level in this case. */ if( res<0 ){ int iPtr = 0; if( nRhs==0 ) iPtr = *piPgno; - rc = seekInSegment( - pCsr, &aPtr[0], iTopic, pKey, nKey, iPtr, eSeek, &iOut, &bStop - ); + rc = seekInSegment(pCsr, &aPtr[0], pKey, nKey, iPtr, eSeek, &iOut, &bStop); if( rc==LSM_OK && nRhs>0 && eSeek==LSM_SEEK_GE && aPtr[0].pPg==0 ){ res = 0; } } if( res>=0 ){ - int bHit = 0; /* True if at least one rhs is not EOF */ int iPtr = *piPgno; int i; for(i=1; rc==LSM_OK && i<=nRhs && bStop==0; i++){ - SegmentPtr *pPtr = &aPtr[i]; iOut = 0; - rc = seekInSegment( - pCsr, pPtr, iTopic, pKey, nKey, iPtr, eSeek, &iOut, &bStop - ); + rc = seekInSegment(pCsr, &aPtr[i], pKey, nKey, iPtr, eSeek, &iOut,&bStop); iPtr = iOut; - - /* If the segment-pointer has settled on a key that is smaller than - ** the splitkey, invalidate the segment-pointer. */ - if( pPtr->pPg ){ - res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - if( res<0 ) segmentPtrReset(pPtr); - } - - if( aPtr[i].pKey ) bHit = 1; - } - - if( rc==LSM_OK && eSeek==LSM_SEEK_LE && bHit==0 ){ + } + + if( rc==LSM_OK && eSeek==LSM_SEEK_LE ){ rc = segmentPtrEnd(pCsr, &aPtr[0], 1); } } assert( eSeek==LSM_SEEK_EQ || bStop==0 ); @@ -1983,52 +1909,17 @@ lsmTreeCursorValue(pTreeCsr, &pVal, &nVal); } break; } - case CURSOR_DATA_SYSTEM: { - Snapshot *pWorker = pCsr->pDb->pWorker; - if( pWorker && (pCsr->flags & CURSOR_FLUSH_FREELIST) ){ - int nEntry = pWorker->freelist.nEntry; - if( pCsr->iFree < (nEntry*2) ){ - FreelistEntry *aEntry = pWorker->freelist.aEntry; - int i = nEntry - 1 - (pCsr->iFree / 2); - u32 iKey = 0; - - if( (pCsr->iFree % 2) ){ - eType = LSM_END_DELETE|LSM_SYSTEMKEY; - iKey = aEntry[i].iBlk-1; - }else if( aEntry[i].iId>=0 ){ - eType = LSM_INSERT|LSM_SYSTEMKEY; - iKey = aEntry[i].iBlk; - - /* If the in-memory entry immediately before this one was a - ** DELETE, and the block number is one greater than the current - ** block number, mark this entry as an "end-delete-range". */ - if( i<(nEntry-1) && aEntry[i+1].iBlk==iKey+1 && aEntry[i+1].iId<0 ){ - eType |= LSM_END_DELETE; - } - - }else{ - eType = LSM_START_DELETE|LSM_SYSTEMKEY; - iKey = aEntry[i].iBlk + 1; - } - - /* If the in-memory entry immediately after this one is a - ** DELETE, and the block number is one less than the current - ** key, mark this entry as an "start-delete-range". */ - if( i>0 && aEntry[i-1].iBlk==iKey-1 && aEntry[i-1].iId<0 ){ - eType |= LSM_START_DELETE; - } - - pKey = pCsr->pSystemVal; - nKey = 4; - lsmPutU32(pKey, ~iKey); - } + case CURSOR_DATA_SYSTEM: + if( pCsr->flags & CURSOR_AT_FREELIST ){ + pKey = (void *)"FREELIST"; + nKey = 8; + eType = LSM_SYSTEMKEY | LSM_INSERT; } break; - } default: { int iPtr = iKey - CURSOR_DATA_SEGMENT; assert( iPtr>=0 ); if( iPtr==pCsr->nPtr ){ @@ -2053,15 +1944,14 @@ if( pnKey ) *pnKey = nKey; if( ppKey ) *ppKey = pKey; } static int sortedDbKeyCompare( - MultiCursor *pCsr, + int (*xCmp)(void *, int, void *, int), int iLhsFlags, void *pLhsKey, int nLhsKey, int iRhsFlags, void *pRhsKey, int nRhsKey ){ - int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; int res; /* Compare the keys, including the system flag. */ res = sortedKeyCompare(xCmp, rtTopic(iLhsFlags), pLhsKey, nLhsKey, @@ -2070,20 +1960,18 @@ /* If a key has the LSM_START_DELETE flag set, but not the LSM_INSERT or ** LSM_POINT_DELETE flags, it is considered a delta larger. This prevents ** the beginning of an open-ended set from masking a database entry or ** delete at a lower level. */ - if( res==0 && (pCsr->flags & CURSOR_IGNORE_DELETE) ){ - const int m = LSM_POINT_DELETE|LSM_INSERT|LSM_END_DELETE |LSM_START_DELETE; + if( res==0 ){ + const int insdel = LSM_POINT_DELETE|LSM_INSERT; int iDel1 = 0; int iDel2 = 0; - - if( LSM_START_DELETE==(iLhsFlags & m) ) iDel1 = +1; - if( LSM_END_DELETE ==(iLhsFlags & m) ) iDel1 = -1; - if( LSM_START_DELETE==(iRhsFlags & m) ) iDel2 = +1; - if( LSM_END_DELETE ==(iRhsFlags & m) ) iDel2 = -1; - + if( LSM_START_DELETE==(iLhsFlags & (LSM_START_DELETE|insdel)) ) iDel1 = +1; + if( LSM_END_DELETE ==(iLhsFlags & (LSM_END_DELETE |insdel)) ) iDel1 = -1; + if( LSM_START_DELETE==(iRhsFlags & (LSM_START_DELETE|insdel)) ) iDel2 = +1; + if( LSM_END_DELETE ==(iRhsFlags & (LSM_END_DELETE |insdel)) ) iDel2 = -1; res = (iDel1 - iDel2); } return res; } @@ -2114,23 +2002,17 @@ iRes = i1; }else{ int res; /* Compare the keys */ - res = sortedDbKeyCompare(pCsr, + res = sortedDbKeyCompare(pCsr->pDb->xCmp, eType1, pKey1, nKey1, eType2, pKey2, nKey2 ); res = res * mul; if( res==0 ){ - /* The two keys are identical. Normally, this means that the key from - ** the newer run clobbers the old. However, if the newer key is a - ** separator key, or a range-delete-boundary only, do not allow it - ** to clobber an older entry. */ - int nc1 = (eType1 & (LSM_INSERT|LSM_POINT_DELETE))==0; - int nc2 = (eType2 & (LSM_INSERT|LSM_POINT_DELETE))==0; - iRes = (nc1 > nc2) ? i2 : i1; + iRes = (rtIsSeparator(eType1) ? i2 : i1); }else if( res<0 ){ iRes = i1; }else{ iRes = i2; } @@ -2157,47 +2039,16 @@ int bReverse ){ int rc; SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; Level *pLvl = pPtr->pLevel; - int bComposite; /* True if pPtr is part of composite level */ + int bComposite; - /* Advance the segment-pointer object. */ rc = segmentPtrAdvance(pCsr, pPtr, bReverse); if( rc!=LSM_OK ) return rc; - bComposite = (pLvl->nRight>0 && pCsr->nPtr>pLvl->nRight); - if( bComposite && pPtr->pPg==0 ){ - int bFix = 0; - if( (bReverse==0)==(pPtr->pSeg==&pLvl->lhs) ){ - int i; - if( bReverse ){ - SegmentPtr *pLhs = &pCsr->aPtr[iPtr - 1 - (pPtr->pSeg - pLvl->aRhs)]; - for(i=0; inRight; i++){ - if( pLhs[i+1].pPg ) break; - } - if( i==pLvl->nRight ){ - bFix = 1; - rc = segmentPtrEnd(pCsr, pLhs, 1); - } - }else{ - bFix = 1; - for(i=0; rc==LSM_OK && inRight; i++){ - rc = sortedRhsFirst(pCsr, pLvl, &pCsr->aPtr[iPtr+1+i]); - } - } - } - - if( bFix ){ - int i; - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, bReverse); - } - } - } - -#if 0 + if( bComposite && pPtr->pSeg==&pLvl->lhs /* lhs of composite level */ && bReverse==0 /* csr advanced forwards */ && pPtr->pPg==0 /* segment at EOF */ ){ int i; @@ -2206,11 +2057,10 @@ } for(i=pCsr->nTree-1; i>0; i--){ multiCursorDoCompare(pCsr, i, 0); } } -#endif return rc; } static void mcursorFreeComponents(MultiCursor *pCsr){ @@ -2314,14 +2164,28 @@ } return LSM_OK; } -static void multiCursorAddOne(MultiCursor *pCsr, Level *pLvl, int *pRc){ - if( *pRc==LSM_OK ){ - int iPtr = pCsr->nPtr; +static int multiCursorAddAll(MultiCursor *pCsr, Snapshot *pSnap){ + Level *pLvl; + int nPtr = 0; + int iPtr = 0; + int rc = LSM_OK; + + for(pLvl=pSnap->pLevel; pLvl; pLvl=pLvl->pNext){ + if( pLvl->iAge<0 ) continue; + nPtr += (1 + pLvl->nRight); + } + + assert( pCsr->aPtr==0 ); + pCsr->aPtr = lsmMallocZeroRc(pCsr->pDb->pEnv, sizeof(SegmentPtr) * nPtr, &rc); + if( rc==LSM_OK ) pCsr->nPtr = nPtr; + + for(pLvl=pSnap->pLevel; pLvl && rc==LSM_OK; pLvl=pLvl->pNext){ int i; + if( pLvl->iAge<0 ) continue; pCsr->aPtr[iPtr].pLevel = pLvl; pCsr->aPtr[iPtr].pSeg = &pLvl->lhs; iPtr++; for(i=0; inRight; i++){ pCsr->aPtr[iPtr].pLevel = pLvl; @@ -2328,38 +2192,11 @@ pCsr->aPtr[iPtr].pSeg = &pLvl->aRhs[i]; iPtr++; } if( pLvl->nRight && pLvl->pSplitKey==0 ){ - sortedSplitkey(pCsr->pDb, pLvl, pRc); - } - pCsr->nPtr = iPtr; - } -} - -static int multiCursorAddAll(MultiCursor *pCsr, Snapshot *pSnap){ - Level *pLvl; - int nPtr = 0; - int iPtr = 0; - int rc = LSM_OK; - - for(pLvl=pSnap->pLevel; pLvl; pLvl=pLvl->pNext){ - /* If the LEVEL_INCOMPLETE flag is set, then this function is being - ** called (indirectly) from within a sortedNewToplevel() call to - ** construct pLvl. In this case ignore pLvl - this cursor is going to - ** be used to retrieve a freelist entry from the LSM, and the partially - ** complete level may confuse it. */ - if( pLvl->flags & LEVEL_INCOMPLETE ) continue; - nPtr += (1 + pLvl->nRight); - } - - assert( pCsr->aPtr==0 ); - pCsr->aPtr = lsmMallocZeroRc(pCsr->pDb->pEnv, sizeof(SegmentPtr) * nPtr, &rc); - - for(pLvl=pSnap->pLevel; pLvl; pLvl=pLvl->pNext){ - if( (pLvl->flags & LEVEL_INCOMPLETE)==0 ){ - multiCursorAddOne(pCsr, pLvl, &rc); + lsmSortedSplitkey(pCsr->pDb, pLvl, &rc); } } return rc; } @@ -2415,15 +2252,14 @@ /* ** If the free-block list is not empty, then have this cursor visit a key ** with (a) the system bit set, and (b) the key "FREELIST" and (c) a value ** blob containing the serialized free-block list. */ -static int multiCursorVisitFreelist(MultiCursor *pCsr){ - int rc = LSM_OK; - pCsr->flags |= CURSOR_FLUSH_FREELIST; - pCsr->pSystemVal = lsmMallocRc(pCsr->pDb->pEnv, 4 + 8, &rc); - return rc; +static void multiCursorVisitFreelist(MultiCursor *pCsr, int *pnOvfl){ + assert( pCsr ); + pCsr->pnOvfl = pnOvfl; + pCsr->flags |= CURSOR_NEW_SYSTEM; } /* ** Allocate and return a new database cursor. */ @@ -2468,24 +2304,18 @@ *pnVal = 0; } break; } - case CURSOR_DATA_SYSTEM: { - Snapshot *pWorker = pCsr->pDb->pWorker; - if( pWorker - && (pCsr->iFree % 2)==0 - && pCsr->iFree < (pWorker->freelist.nEntry*2) - ){ - int iEntry = pWorker->freelist.nEntry - 1 - (pCsr->iFree / 2); - u8 *aVal = &((u8 *)(pCsr->pSystemVal))[4]; - lsmPutU64(aVal, pWorker->freelist.aEntry[iEntry].iId); - *ppVal = aVal; - *pnVal = 8; + case CURSOR_DATA_SYSTEM: + if( pCsr->flags & CURSOR_AT_FREELIST ){ + void *aVal; + rc = lsmCheckpointOverflow(pCsr->pDb, &aVal, pnVal, pCsr->pnOvfl); + assert( pCsr->pSystemVal==0 ); + *ppVal = pCsr->pSystemVal = aVal; } break; - } default: { int iPtr = iVal-CURSOR_DATA_SEGMENT; if( iPtrnPtr ){ SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; @@ -2496,74 +2326,10 @@ } } } assert( rc==LSM_OK || (*ppVal==0 && *pnVal==0) ); - return rc; -} - -static int multiCursorAdvance(MultiCursor *pCsr, int bReverse); - -/* -** This function is called by worker connections to walk the part of the -** free-list stored within the LSM data structure. -*/ -int lsmSortedWalkFreelist( - lsm_db *pDb, /* Database handle */ - int bReverse, /* True to iterate from largest to smallest */ - int (*x)(void *, int, i64), /* Callback function */ - void *pCtx /* First argument to pass to callback */ -){ - MultiCursor *pCsr; /* Cursor used to read db */ - int rc = LSM_OK; /* Return Code */ - Snapshot *pSnap = 0; - - assert( pDb->pWorker ); - if( pDb->bIncrMerge ){ - rc = lsmCheckpointDeserialize(pDb, 0, pDb->pShmhdr->aSnap1, &pSnap); - if( rc!=LSM_OK ) return rc; - }else{ - pSnap = pDb->pWorker; - } - - pCsr = multiCursorNew(pDb, &rc); - if( pCsr ){ - rc = multiCursorAddAll(pCsr, pSnap); - pCsr->flags |= CURSOR_IGNORE_DELETE; - } - - if( rc==LSM_OK ){ - if( bReverse==0 ){ - rc = lsmMCursorLast(pCsr); - }else{ - rc = lsmMCursorSeek(pCsr, 1, "", 0, LSM_SEEK_GE); - } - - while( rc==LSM_OK && lsmMCursorValid(pCsr) && rtIsSystem(pCsr->eType) ){ - void *pKey; int nKey; - void *pVal; int nVal; - - rc = lsmMCursorKey(pCsr, &pKey, &nKey); - if( rc==LSM_OK ) rc = lsmMCursorValue(pCsr, &pVal, &nVal); - if( rc==LSM_OK && (nKey!=4 || nVal!=8) ) rc = LSM_CORRUPT_BKPT; - - if( rc==LSM_OK ){ - int iBlk; - i64 iSnap; - iBlk = (int)(~(lsmGetU32((u8 *)pKey))); - iSnap = (i64)lsmGetU64((u8 *)pVal); - if( x(pCtx, iBlk, iSnap) ) break; - rc = multiCursorAdvance(pCsr, !bReverse); - } - } - } - - lsmMCursorClose(pCsr); - if( pSnap!=pDb->pWorker ){ - lsmFreeSnapshot(pDb->pEnv, pSnap); - } - return rc; } int lsmSortedLoadFreelist( lsm_db *pDb, /* Database handle (must be worker) */ @@ -2631,224 +2397,108 @@ multiCursorGetKey(pCsr, pCsr->aTree[1], &pCsr->eType, &pKey, &nKey); *pRc = sortedBlobSet(pCsr->pDb->pEnv, &pCsr->key, pKey, nKey); } } -#ifndef NDEBUG -static void assertCursorTree(MultiCursor *pCsr){ - int bRev = !!(pCsr->flags & CURSOR_PREV_OK); - int *aSave = pCsr->aTree; - int nSave = pCsr->nTree; - int rc; - - pCsr->aTree = 0; - pCsr->nTree = 0; - rc = multiCursorAllocTree(pCsr); - if( rc==LSM_OK ){ - int i; - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, bRev); - } - - assert( nSave==pCsr->nTree - && 0==memcmp(aSave, pCsr->aTree, sizeof(int)*nSave) - ); - - lsmFree(pCsr->pDb->pEnv, pCsr->aTree); - } - - pCsr->aTree = aSave; - pCsr->nTree = nSave; -} -#else -# define assertCursorTree(x) -#endif - static int mcursorLocationOk(MultiCursor *pCsr, int bDeleteOk){ int eType = pCsr->eType; int iKey; int i; - int rdmask; + int rdmask = 0; assert( pCsr->flags & (CURSOR_NEXT_OK|CURSOR_PREV_OK) ); - assertCursorTree(pCsr); + if( pCsr->flags & CURSOR_NEXT_OK ){ + rdmask = LSM_END_DELETE; + }else{ + rdmask = LSM_START_DELETE; + } - rdmask = (pCsr->flags & CURSOR_NEXT_OK) ? LSM_END_DELETE : LSM_START_DELETE; - - /* If the cursor does not currently point to an actual database key (i.e. - ** it points to a delete key, or the start or end of a range-delete), and - ** the CURSOR_IGNORE_DELETE flag is set, skip past this entry. */ if( (pCsr->flags & CURSOR_IGNORE_DELETE) && bDeleteOk==0 ){ if( (eType & LSM_INSERT)==0 ) return 0; } - - /* If the cursor points to a system key (free-list entry), and the - ** CURSOR_IGNORE_SYSTEM flag is set, skip thie entry. */ if( (pCsr->flags & CURSOR_IGNORE_SYSTEM) && rtTopic(eType)!=0 ){ return 0; } -#ifndef NDEBUG - /* This block fires assert() statements to check one of the assumptions - ** in the comment below - that if the lhs sub-cursor of a level undergoing - ** a merge is valid, then all the rhs sub-cursors must be at EOF. - ** - ** Also assert that all rhs sub-cursors are either at EOF or point to - ** a key that is not less than the level split-key. */ - for(i=0; inPtr; i++){ - SegmentPtr *pPtr = &pCsr->aPtr[i]; - Level *pLvl = pPtr->pLevel; - if( pLvl->nRight && pPtr->pPg ){ - if( pPtr->pSeg==&pLvl->lhs ){ - int j; - for(j=0; jnRight; j++) assert( pPtr[j+1].pPg==0 ); - }else{ - int res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - assert( res>=0 ); - } - } - } -#endif - - /* Now check if this key has already been deleted by a range-delete. If - ** so, skip past it. - ** - ** Assume, for the moment, that the tree contains no levels currently - ** undergoing incremental merge, and that this cursor is iterating forwards - ** through the database keys. The cursor currently points to a key in - ** level L. This key has already been deleted if any of the sub-cursors - ** that point to levels newer than L (or to the in-memory tree) point to - ** a key greater than the current key with the LSM_END_DELETE flag set. - ** - ** Or, if the cursor is iterating backwards through data keys, if any - ** such sub-cursor points to a key smaller than the current key with the - ** LSM_START_DELETE flag set. - ** - ** Why it works with levels undergoing a merge too: - ** - ** When a cursor iterates forwards, the sub-cursors for the rhs of a - ** level are only activated once the lhs reaches EOF. So when iterating - ** forwards, the keys visited are the same as if the level was completely - ** merged. - ** - ** If the cursor is iterating backwards, then the lhs sub-cursor is not - ** initialized until the last of the rhs sub-cursors has reached EOF. - ** Additionally, if the START_DELETE flag is set on the last entry (in - ** reverse order - so the entry with the smallest key) of a rhs sub-cursor, - ** then a pseudo-key equal to the levels split-key with the END_DELETE - ** flag set is visited by the sub-cursor. - */ + /* Check if this key has already been deleted by a range-delete */ iKey = pCsr->aTree[1]; - for(i=0; iflags & CURSOR_IGNORE_DELETE)==0 - ){ - void *pKey; int nKey; - multiCursorGetKey(pCsr, i, 0, &pKey, &nKey); - if( 0==sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(eType), pCsr->key.pData, pCsr->key.nData, - rtTopic(csrflags), pKey, nKey - )){ - continue; - } - } + if( (iKey>0 && (rdmask & lsmTreeCursorFlags(pCsr->apTreeCsr[0]))) + || (iKey>1 && (rdmask & lsmTreeCursorFlags(pCsr->apTreeCsr[1]))) + ){ + return 0; + } + for(i=CURSOR_DATA_SEGMENT; iaPtr[iPtr].pPg && (pCsr->aPtr[iPtr].eType & rdmask) ){ return 0; } } - /* The current cursor position is one this cursor should visit. Return 1. */ return 1; } - -static int multiCursorSetupTree(MultiCursor *pCsr, int bRev){ - int rc; - - rc = multiCursorAllocTree(pCsr); - if( rc==LSM_OK ){ - int i; - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, bRev); - } - } - - assertCursorTree(pCsr); - multiCursorCacheKey(pCsr, &rc); - - if( rc==LSM_OK && mcursorLocationOk(pCsr, 0)==0 ){ - rc = multiCursorAdvance(pCsr, bRev); - } - return rc; -} - static int multiCursorEnd(MultiCursor *pCsr, int bLast){ int rc = LSM_OK; int i; pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK); pCsr->flags |= (bLast ? CURSOR_PREV_OK : CURSOR_NEXT_OK); - pCsr->iFree = 0; - /* Position the two in-memory tree cursors */ - for(i=0; rc==LSM_OK && i<2; i++){ - if( pCsr->apTreeCsr[i] ){ - rc = lsmTreeCursorEnd(pCsr->apTreeCsr[i], bLast); - } + if( pCsr->apTreeCsr[0] ){ + rc = lsmTreeCursorEnd(pCsr->apTreeCsr[0], bLast); + } + if( rc==LSM_OK && pCsr->apTreeCsr[1] ){ + rc = lsmTreeCursorEnd(pCsr->apTreeCsr[1], bLast); + } + + if( pCsr->flags & CURSOR_NEW_SYSTEM ){ + assert( bLast==0 ); + pCsr->flags |= CURSOR_AT_FREELIST; } for(i=0; rc==LSM_OK && inPtr; i++){ SegmentPtr *pPtr = &pCsr->aPtr[i]; Level *pLvl = pPtr->pLevel; - int iRhs; - int bHit = 0; - - if( bLast ){ - for(iRhs=0; iRhsnRight && rc==LSM_OK; iRhs++){ - rc = segmentPtrEnd(pCsr, &pPtr[iRhs+1], 1); - if( pPtr[iRhs+1].pPg ) bHit = 1; - } - if( bHit==0 && rc==LSM_OK ){ - rc = segmentPtrEnd(pCsr, pPtr, 1); - }else{ - segmentPtrReset(pPtr); - } - }else{ - int bLhs = (pPtr->pSeg==&pLvl->lhs); - assert( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[0] ); - - if( bLhs ){ - rc = segmentPtrEnd(pCsr, pPtr, 0); - if( pPtr->pKey ) bHit = 1; - } - for(iRhs=0; iRhsnRight && rc==LSM_OK; iRhs++){ - if( bHit ){ - segmentPtrReset(&pPtr[iRhs+1]); - }else{ - rc = sortedRhsFirst(pCsr, pLvl, &pPtr[iRhs+bLhs]); - } - } - } - i += pLvl->nRight; - } - - /* And the b-tree cursor, if applicable */ + + rc = segmentPtrEnd(pCsr, pPtr, bLast); + if( rc==LSM_OK && bLast==0 && pLvl->nRight && pPtr->pSeg==&pLvl->lhs ){ + int iRhs; + for(iRhs=1+i; rc==LSM_OK && iRhs<1+i+pLvl->nRight; iRhs++){ + SegmentPtr *pRhs = &pCsr->aPtr[iRhs]; + if( pPtr->pPg==0 ){ + rc = sortedRhsFirst(pCsr, pLvl, pRhs); + }else{ + segmentPtrReset(pRhs); + } + } + i += pLvl->nRight; + } + } + if( rc==LSM_OK && pCsr->pBtCsr ){ assert( bLast==0 ); rc = btreeCursorFirst(pCsr->pBtCsr); } if( rc==LSM_OK ){ - rc = multiCursorSetupTree(pCsr, bLast); + rc = multiCursorAllocTree(pCsr); } - + if( rc==LSM_OK ){ + for(i=pCsr->nTree-1; i>0; i--){ + multiCursorDoCompare(pCsr, i, bLast); + } + } + + multiCursorCacheKey(pCsr, &rc); + if( rc==LSM_OK && mcursorLocationOk(pCsr, 0)==0 ){ + if( bLast ){ + rc = lsmMCursorPrev(pCsr); + }else{ + rc = lsmMCursorNext(pCsr); + } + } + return rc; } int mcursorSave(MultiCursor *pCsr){ @@ -2865,13 +2515,11 @@ int mcursorRestore(lsm_db *pDb, MultiCursor *pCsr){ int rc; rc = multiCursorInit(pCsr, pDb->pClient); if( rc==LSM_OK && pCsr->key.pData ){ - rc = lsmMCursorSeek(pCsr, - rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, +1 - ); + rc = lsmMCursorSeek(pCsr, pCsr->key.pData, pCsr->key.nData, +1); } return rc; } int lsmSaveCursors(lsm_db *pDb){ @@ -2966,29 +2614,22 @@ /* ** Seek the cursor. */ -int lsmMCursorSeek( - MultiCursor *pCsr, - int iTopic, - void *pKey, int nKey, - int eSeek -){ +int lsmMCursorSeek(MultiCursor *pCsr, void *pKey, int nKey, int eSeek){ int eESeek = eSeek; /* Effective eSeek parameter */ int bStop = 0; /* Set to true to halt search operation */ int rc = LSM_OK; /* Return code */ int iPtr = 0; /* Used to iterate through pCsr->aPtr[] */ Pgno iPgno = 0; /* FC pointer value */ - assert( pCsr->apTreeCsr[0]==0 || iTopic==0 ); - assert( pCsr->apTreeCsr[1]==0 || iTopic==0 ); - if( eESeek==LSM_SEEK_LEFAST ) eESeek = LSM_SEEK_LE; assert( eESeek==LSM_SEEK_EQ || eESeek==LSM_SEEK_LE || eESeek==LSM_SEEK_GE ); - assert( (pCsr->flags & CURSOR_FLUSH_FREELIST)==0 ); + assert( (pCsr->flags & CURSOR_NEW_SYSTEM)==0 ); + assert( (pCsr->flags & CURSOR_AT_FREELIST)==0 ); assert( pCsr->nPtr==0 || pCsr->aPtr[0].pLevel ); pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK | CURSOR_SEEK_EQ); rc = treeCursorSeek(pCsr, pCsr->apTreeCsr[0], pKey, nKey, eESeek, &bStop); if( rc==LSM_OK && bStop==0 ){ @@ -2997,11 +2638,11 @@ /* Seek all segment pointers. */ for(iPtr=0; iPtrnPtr && rc==LSM_OK && bStop==0; iPtr++){ SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; assert( pPtr->pSeg==&pPtr->pLevel->lhs ); - rc = seekInLevel(pCsr, pPtr, eESeek, iTopic, pKey, nKey, &iPgno, &bStop); + rc = seekInLevel(pCsr, pPtr, eESeek, pKey, nKey, &iPgno, &bStop); iPtr += pPtr->pLevel->nRight; } if( eSeek!=LSM_SEEK_EQ ){ if( rc==LSM_OK ){ @@ -3067,16 +2708,13 @@ ** or less than (if bReverse!=0) the key currently cached in pCsr->key, ** then the cursor has not yet been successfully advanced. */ multiCursorGetKey(pCsr, pCsr->aTree[1], &eNewType, &pNew, &nNew); if( pNew ){ - int typemask = (pCsr->flags & CURSOR_IGNORE_DELETE) ? ~(0) : LSM_SYSTEMKEY; - int res = sortedDbKeyCompare(pCsr, - eNewType & typemask, pNew, nNew, - pCsr->eType & typemask, pCsr->key.pData, pCsr->key.nData + int res = sortedDbKeyCompare(pCsr->pDb->xCmp, + eNewType, pNew, nNew, pCsr->eType, pCsr->key.pData, pCsr->key.nData ); - if( (bReverse==0 && res<=0) || (bReverse!=0 && res>=0) ){ return 0; } multiCursorCacheKey(pCsr, pRc); @@ -3086,51 +2724,22 @@ ** cursor points to a SORTED_DELETE entry, then the cursor has not been ** successfully advanced. ** ** Similarly, if the cursor is configured to skip system keys and the ** current cursor points to a system key, it has not yet been advanced. - */ + */ if( *pRc==LSM_OK && 0==mcursorLocationOk(pCsr, 0) ) return 0; } return 1; } -static void flCsrAdvance(MultiCursor *pCsr){ - assert( pCsr->flags & CURSOR_FLUSH_FREELIST ); - if( pCsr->iFree % 2 ){ - pCsr->iFree++; - }else{ - int nEntry = pCsr->pDb->pWorker->freelist.nEntry; - FreelistEntry *aEntry = pCsr->pDb->pWorker->freelist.aEntry; - - int i = nEntry - 1 - (pCsr->iFree / 2); - - /* If the current entry is a delete and the "end-delete" key will not - ** be attached to the next entry, increment iFree by 1 only. */ - if( aEntry[i].iId<0 ){ - while( 1 ){ - if( i==0 || aEntry[i-1].iBlk!=aEntry[i].iBlk-1 ){ - pCsr->iFree--; - break; - } - if( aEntry[i-1].iId>=0 ) break; - pCsr->iFree += 2; - i--; - } - } - pCsr->iFree += 2; - } -} - static int multiCursorAdvance(MultiCursor *pCsr, int bReverse){ int rc = LSM_OK; /* Return Code */ if( lsmMCursorValid(pCsr) ){ do { int iKey = pCsr->aTree[1]; - assertCursorTree(pCsr); - /* If this multi-cursor is advancing forwards, and the sub-cursor ** being advanced is the one that separator keys may be being read ** from, record the current absolute pointer value. */ if( pCsr->pPrevMergePtr ){ if( iKey==(CURSOR_DATA_SEGMENT+pCsr->nPtr) ){ @@ -3150,13 +2759,14 @@ rc = lsmTreeCursorPrev(pTreeCsr); }else{ rc = lsmTreeCursorNext(pTreeCsr); } }else if( iKey==CURSOR_DATA_SYSTEM ){ - assert( pCsr->flags & CURSOR_FLUSH_FREELIST ); + assert( pCsr->flags & CURSOR_AT_FREELIST ); + assert( pCsr->flags & CURSOR_NEW_SYSTEM ); assert( bReverse==0 ); - flCsrAdvance(pCsr); + pCsr->flags &= ~CURSOR_AT_FREELIST; }else if( iKey==(CURSOR_DATA_SEGMENT+pCsr->nPtr) ){ assert( bReverse==0 && pCsr->pBtCsr ); rc = btreeCursorNext(pCsr->pBtCsr); }else{ rc = segmentCursorAdvance(pCsr, iKey-CURSOR_DATA_SEGMENT, bReverse); @@ -3164,11 +2774,10 @@ if( rc==LSM_OK ){ int i; for(i=(iKey+pCsr->nTree)/2; i>0; i=i/2){ multiCursorDoCompare(pCsr, i, bReverse); } - assertCursorTree(pCsr); } }while( mcursorAdvanceOk(pCsr, bReverse, &rc)==0 ); } return rc; } @@ -3182,11 +2791,11 @@ if( (pCsr->flags & CURSOR_PREV_OK)==0 ) return LSM_MISUSE_BKPT; return multiCursorAdvance(pCsr, 1); } int lsmMCursorKey(MultiCursor *pCsr, void **ppKey, int *pnKey){ - if( (pCsr->flags & CURSOR_SEEK_EQ) || pCsr->aTree==0 ){ + if( pCsr->flags & CURSOR_SEEK_EQ ){ *pnKey = pCsr->key.nData; *ppKey = pCsr->key.pData; }else{ int iKey = pCsr->aTree[1]; @@ -3215,31 +2824,15 @@ } } return LSM_OK; } -/* -** Compare the current key that cursor csr points to with pKey/nKey. Set -** *piRes to the result and return LSM_OK. -*/ -int lsm_csr_cmp(lsm_cursor *csr, const void *pKey, int nKey, int *piRes){ - MultiCursor *pCsr = (MultiCursor *)csr; - void *pCsrkey; int nCsrkey; - int rc; - rc = lsmMCursorKey(pCsr, &pCsrkey, &nCsrkey); - if( rc==LSM_OK ){ - int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; - *piRes = sortedKeyCompare(xCmp, 0, pCsrkey, nCsrkey, 0, (void *)pKey, nKey); - } - return rc; -} - int lsmMCursorValue(MultiCursor *pCsr, void **ppVal, int *pnVal){ void *pVal; int nVal; int rc; - if( (pCsr->flags & CURSOR_SEEK_EQ) || pCsr->aTree==0 ){ + if( pCsr->flags & CURSOR_SEEK_EQ ){ rc = LSM_OK; nVal = pCsr->val.nData; pVal = pCsr->val.pData; }else{ @@ -3311,46 +2904,32 @@ */ static int mergeWorkerMoveHierarchy( MergeWorker *pMW, /* Merge worker */ int bSep /* True for separators run */ ){ + Segment *pSeg; /* Segment being written */ lsm_db *pDb = pMW->pDb; /* Database handle */ int rc = LSM_OK; /* Return code */ int i; Page **apHier = pMW->hier.apHier; int nHier = pMW->hier.nHier; + + pSeg = &pMW->pLevel->lhs; for(i=0; rc==LSM_OK && ipFS, pDb->pWorker, pMW->pLevel, 1, &pNew); + rc = lsmFsSortedAppend(pDb->pFS, pDb->pWorker, pSeg, &pNew); assert( rc==LSM_OK ); if( rc==LSM_OK ){ u8 *a1; int n1; u8 *a2; int n2; a1 = fsPageData(pNew, &n1); a2 = fsPageData(apHier[i], &n2); - - assert( n1==n2 || n1+4==n2 ); - - if( n1==n2 ){ - memcpy(a1, a2, n2); - }else{ - int nEntry = pageGetNRec(a2, n2); - int iEof1 = SEGMENT_EOF(n1, nEntry); - int iEof2 = SEGMENT_EOF(n2, nEntry); - - memcpy(a1, a2, iEof2 - 4); - memcpy(&a1[iEof1], &a2[iEof2], n2 - iEof2); - } - - lsmFsPageRelease(apHier[i]); - apHier[i] = pNew; - -#if 0 assert( n1==n2 || n1+4==n2 || n2+4==n1 ); + if( n1>=n2 ){ /* If n1 (size of the new page) is equal to or greater than n2 (the ** size of the old page), then copy the data into the new page. If ** n1==n2, this could be done with a single memcpy(). However, ** since sometimes n1>n2, the page content and footer must be copied @@ -3367,11 +2946,10 @@ lsmPutU16(&a1[SEGMENT_NRECORD_OFFSET(n1)], 0); lsmPutU64(&a1[SEGMENT_POINTER_OFFSET(n1)], 0); i = i - 1; lsmFsPageRelease(pNew); } -#endif } } #ifdef LSM_DEBUG if( rc==LSM_OK ){ @@ -3404,11 +2982,11 @@ Page *pPg = 0; u8 *aData; int nData; int flags; - rc = lsmFsDbPageGet(pFS, pSeg, iPg, &pPg); + rc = lsmFsDbPageGet(pFS, iPg, &pPg); if( rc!=LSM_OK ) break; aData = fsPageData(pPg, &nData); flags = pageGetFlags(aData, nData); if( flags&SEGMENT_BTREE_FLAG ){ @@ -3497,10 +3075,11 @@ Pgno iPtr, Pgno iKeyPg, void *pKey, int nKey ){ + Segment *pSeg = &pMW->pLevel->lhs; Hierarchy *p = &pMW->hier; lsm_db *pDb = pMW->pDb; /* Database handle */ int rc = LSM_OK; /* Return Code */ int iLevel; /* Level of b-tree hierachy to write to */ int nData; /* Size of aData[] in bytes */ @@ -3552,11 +3131,10 @@ if( nByte<=nFree ) break; /* Otherwise, this page is full. Set the right-hand-child pointer ** to iPtr and release it. */ lsmPutU64(&aData[SEGMENT_POINTER_OFFSET(nData)], iPtr); - assert( lsmFsPageNumber(pOld)==0 ); rc = lsmFsPagePersist(pOld); if( rc==LSM_OK ){ iPtr = lsmFsPageNumber(pOld); lsmFsPageRelease(pOld); } @@ -3564,11 +3142,11 @@ /* Allocate a new page for apHier[iLevel]. */ p->apHier[iLevel] = 0; if( rc==LSM_OK ){ rc = lsmFsSortedAppend( - pDb->pFS, pDb->pWorker, pMW->pLevel, 1, &p->apHier[iLevel] + pDb->pFS, pDb->pWorker, pSeg, &p->apHier[iLevel] ); } if( rc!=LSM_OK ) return rc; aData = fsPageData(p->apHier[iLevel], &nData); @@ -3632,10 +3210,18 @@ u8 *aData; /* Page data for level iLevel */ int iOff; /* Offset on b-tree page to write record to */ int nRec; /* Initial number of records on b-tree page */ Pgno iPtr; /* Pointer value to accompany pKey/nKey */ + Hierarchy *p; + Segment *pSeg; + + /* If there exists a b-tree hierarchy and it is not loaded into + ** memory, load it now. */ + pSeg = &pMW->pLevel->lhs; + p = &pMW->hier; + assert( pMW->aSave[0].bStore==0 ); assert( pMW->aSave[1].bStore==0 ); rc = mergeWorkerBtreeIndirect(pMW); /* Obtain the absolute pointer value to store along with the key in the @@ -3694,29 +3280,10 @@ ){ FileSystem *pFS = pMW->pDb->pFS; return lsmFsSortedPadding(pFS, pMW->pDb->pWorker, &pMW->pLevel->lhs); } -/* -** Release all page references currently held by the merge-worker passed -** as the only argument. Unless an error has occurred, all pages have -** already been released. -*/ -static void mergeWorkerReleaseAll(MergeWorker *pMW){ - int i; - lsmFsPageRelease(pMW->pPage); - pMW->pPage = 0; - - for(i=0; ihier.nHier; i++){ - lsmFsPageRelease(pMW->hier.apHier[i]); - pMW->hier.apHier[i] = 0; - } - lsmFree(pMW->pDb->pEnv, pMW->hier.apHier); - pMW->hier.apHier = 0; - pMW->hier.nHier = 0; -} - static int keyszToSkip(FileSystem *pFS, int nKey){ int nPgsz; /* Nominal database page size */ nPgsz = lsmFsPageSize(pFS); return LSM_MIN(((nKey * 4) / nPgsz), 3); } @@ -3764,12 +3331,13 @@ int rc = LSM_OK; /* Return code */ Page *pNext = 0; /* New page appended to run */ lsm_db *pDb = pMW->pDb; /* Database handle */ Segment *pSeg; /* Run to append to */ - rc = lsmFsSortedAppend(pDb->pFS, pDb->pWorker, pMW->pLevel, 0, &pNext); - assert( rc || pMW->pLevel->lhs.iFirst>0 || pMW->pDb->compress.xCompress ); + pSeg = &pMW->pLevel->lhs; + rc = lsmFsSortedAppend(pDb->pFS, pDb->pWorker, pSeg, &pNext); + assert( rc!=LSM_OK || pSeg->iFirst>0 || pMW->pDb->compress.xCompress ); if( rc==LSM_OK ){ u8 *aData; /* Data buffer belonging to page pNext */ int nData; /* Size of aData[] in bytes */ @@ -3850,11 +3418,11 @@ rc = LSM_OK; iFPtr = pMW->pLevel->pNext->lhs.iFirst; }else if( pCsr->nPtr>0 ){ Segment *pSeg; pSeg = pCsr->aPtr[pCsr->nPtr-1].pSeg; - rc = lsmFsDbPageGet(pMW->pDb->pFS, pSeg, pSeg->iFirst, &pPg); + rc = lsmFsDbPageGet(pMW->pDb->pFS, pSeg->iFirst, &pPg); if( rc==LSM_OK ){ u8 *aData; /* Buffer for page pPg */ int nData; /* Size of aData[] in bytes */ aData = fsPageData(pPg, &nData); iFPtr = pageGetPtr(aData, nData); @@ -3873,11 +3441,11 @@ static int mergeWorkerWrite( MergeWorker *pMW, /* Merge worker object to write into */ int eType, /* One of SORTED_SEPARATOR, WRITE or DELETE */ void *pKey, int nKey, /* Key value */ - void *pVal, int nVal, /* Value value */ + MultiCursor *pCsr, /* Read value (if any) from here */ int iPtr /* Absolute value of page pointer, or 0 */ ){ int rc = LSM_OK; /* Return code */ Merge *pMerge; /* Persistent part of level merge state */ int nHdr; /* Space required for this record header */ @@ -3889,10 +3457,12 @@ int iRPtr = 0; /* Value of pointer written into record */ int iOff; /* Current write offset within page pPg */ Segment *pSeg; /* Segment being written */ int flags = 0; /* If != 0, flags value for page footer */ int bFirst = 0; /* True for first key of output run */ + void *pVal; + int nVal; pMerge = pMW->pLevel->pMerge; pSeg = &pMW->pLevel->lhs; if( pSeg->iFirst==0 && pMW->pPage==0 ){ @@ -3919,19 +3489,20 @@ ** 1) record type - 1 byte. ** 2) Page-pointer-offset - 1 varint ** 3) Key size - 1 varint ** 4) Value size - 1 varint (only if LSM_INSERT flag is set) */ + rc = lsmMCursorValue(pCsr, &pVal, &nVal); if( rc==LSM_OK ){ nHdr = 1 + lsmVarintLen32(iRPtr) + lsmVarintLen32(nKey); if( rtIsWrite(eType) ) nHdr += lsmVarintLen32(nVal); /* If the entire header will not fit on page pPg, or if page pPg is ** marked read-only, advance to the next page of the output run. */ iOff = pMerge->iOutputOff; if( iOff<0 || pPg==0 || iOff+nHdr > SEGMENT_EOF(nData, nRec+1) ){ - iFPtr = *pMW->pCsr->pPrevMergePtr; + iFPtr = *pCsr->pPrevMergePtr; iRPtr = iPtr - iFPtr; iOff = 0; nRec = 0; rc = mergeWorkerNextPage(pMW, iFPtr); pPg = pMW->pPage; @@ -3976,19 +3547,37 @@ /* Write the key and data into the segment. */ assert( iFPtr==pageGetPtr(aData, nData) ); rc = mergeWorkerData(pMW, 0, iFPtr+iRPtr, pKey, nKey); if( rc==LSM_OK && rtIsWrite(eType) ){ + if( rtTopic(eType)==0 ) rc = lsmMCursorValue(pCsr, &pVal, &nVal); if( rc==LSM_OK ){ rc = mergeWorkerData(pMW, 0, iFPtr+iRPtr, pVal, nVal); } } } return rc; } + +static int multiCursorSetupTree(MultiCursor *pCsr, int bRev){ + int rc; + + assert( pCsr->aTree==0 ); + + rc = multiCursorAllocTree(pCsr); + if( rc==LSM_OK ){ + int i; + for(i=pCsr->nTree-1; i>0; i--){ + multiCursorDoCompare(pCsr, i, bRev); + } + } + + multiCursorCacheKey(pCsr, &rc); + return rc; +} /* ** Free all resources allocated by mergeWorkerInit(). */ static void mergeWorkerShutdown(MergeWorker *pMW, int *pRc){ @@ -4040,16 +3629,16 @@ /* Persist and release the output page. */ if( rc==LSM_OK ) rc = mergeWorkerPersistAndRelease(pMW); if( rc==LSM_OK ) rc = mergeWorkerBtreeIndirect(pMW); if( rc==LSM_OK ) rc = mergeWorkerFinishHierarchy(pMW); if( rc==LSM_OK ) rc = mergeWorkerAddPadding(pMW); - lsmFsFlushWaiting(pMW->pDb->pFS, &rc); - mergeWorkerReleaseAll(pMW); lsmFree(pMW->pDb->pEnv, pMW->aGobble); pMW->aGobble = 0; pMW->pCsr = 0; + pMW->pPage = 0; + pMW->pPage = 0; *pRc = rc; } /* @@ -4056,70 +3645,37 @@ ** The cursor passed as the first argument is being used as the input for ** a merge operation. When this function is called, *piFlags contains the ** database entry flags for the current entry. The entry about to be written ** to the output. ** -** Note that this function only has to work for cursors configured to -** iterate forwards (not backwards). */ -static void mergeRangeDeletes(MultiCursor *pCsr, int *piVal, int *piFlags){ +static void mergeRangeDeletes(MultiCursor *pCsr, int *piFlags){ int f = *piFlags; int iKey = pCsr->aTree[1]; int i; - assert( pCsr->flags & CURSOR_NEXT_OK ); if( pCsr->flags & CURSOR_IGNORE_DELETE ){ /* The ignore-delete flag is set when the output of the merge will form ** the oldest level in the database. In this case there is no point in ** retaining any range-delete flags. */ assert( (f & LSM_POINT_DELETE)==0 ); f &= ~(LSM_START_DELETE|LSM_END_DELETE); }else{ - for(i=0; i<(CURSOR_DATA_SEGMENT + pCsr->nPtr); i++){ - if( i!=iKey ){ - int eType; - void *pKey; - int nKey; - int res; - multiCursorGetKey(pCsr, i, &eType, &pKey, &nKey); - - if( pKey ){ - res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, - rtTopic(eType), pKey, nKey - ); - assert( res<=0 ); - if( res==0 ){ - if( (f & (LSM_INSERT|LSM_POINT_DELETE))==0 ){ - if( eType & LSM_INSERT ){ - f |= LSM_INSERT; - *piVal = i; - } - else if( eType & LSM_POINT_DELETE ){ - f |= LSM_POINT_DELETE; - } - } - f |= (eType & (LSM_END_DELETE|LSM_START_DELETE)); - } - - if( i>iKey && (eType & LSM_END_DELETE) && res<0 ){ - if( f & (LSM_INSERT|LSM_POINT_DELETE) ){ - f |= (LSM_END_DELETE|LSM_START_DELETE); - }else{ - f = 0; - } - break; - } - } - } - } - - assert( (f & LSM_INSERT)==0 || (f & LSM_POINT_DELETE)==0 ); - if( (f & LSM_START_DELETE) - && (f & LSM_END_DELETE) - && (f & LSM_POINT_DELETE ) - ){ + if( iKey==0 ){ + int btreeflags = lsmTreeCursorFlags(pCsr->apTreeCsr[1]); + if( btreeflags & LSM_END_DELETE ){ + f |= (LSM_START_DELETE|LSM_END_DELETE); + } + } + + for(i=LSM_MAX(0, iKey+1-CURSOR_DATA_SEGMENT); inPtr; i++){ + if( pCsr->aPtr[i].eType & LSM_END_DELETE ){ + f |= (LSM_START_DELETE|LSM_END_DELETE); + } + } + + if( (f & LSM_START_DELETE) && (f & LSM_END_DELETE) && (f & LSM_INSERT)==0 ){ f = 0; } } *piFlags = f; @@ -4131,24 +3687,18 @@ int rc = LSM_OK; /* Return code */ int eType; /* SORTED_SEPARATOR, WRITE or DELETE */ void *pKey; int nKey; /* Key */ Segment *pSeg; /* Output segment */ Pgno iPtr; - int iVal; pCsr = pMW->pCsr; pSeg = &pMW->pLevel->lhs; /* Pull the next record out of the source cursor. */ lsmMCursorKey(pCsr, &pKey, &nKey); eType = pCsr->eType; - if( eType & LSM_SYSTEMKEY ){ - int i; - i = 1; - } - /* Figure out if the output record may have a different pointer value ** than the previous. This is the case if the current key is identical to ** a key that appears in the lowest level run being merged. If so, set ** iPtr to the absolute pointer value. If not, leave iPtr set to zero, ** indicating that the output pointer value should be a copy of the pointer @@ -4169,17 +3719,16 @@ ){ iPtr = pPtr->iPtr+pPtr->iPgPtr; } } - iVal = pCsr->aTree[1]; - mergeRangeDeletes(pCsr, &iVal, &eType); + mergeRangeDeletes(pCsr, &eType); if( eType!=0 ){ if( pMW->aGobble ){ int iGobble = pCsr->aTree[1] - CURSOR_DATA_SEGMENT; - if( iGobblenPtr && iGobble>=0 ){ + if( iGobblenPtr ){ SegmentPtr *pGobble = &pCsr->aPtr[iGobble]; if( (pGobble->flags & PGFTR_SKIP_THIS_FLAG)==0 ){ pMW->aGobble[iGobble] = lsmFsPageNumber(pGobble->pPg); } } @@ -4186,29 +3735,46 @@ } /* If this is a separator key and we know that the output pointer has not ** changed, there is no point in writing an output record. Otherwise, ** proceed. */ - if( rc==LSM_OK && (rtIsSeparator(eType)==0 || iPtr!=0) ){ + if( rtIsSeparator(eType)==0 || iPtr!=0 ){ /* Write the record into the main run. */ - void *pVal; int nVal; - rc = multiCursorGetVal(pCsr, iVal, &pVal, &nVal); - if( pVal && rc==LSM_OK ){ - assert( nVal>=0 ); - rc = sortedBlobSet(pDb->pEnv, &pCsr->val, pVal, nVal); - pVal = pCsr->val.pData; - } if( rc==LSM_OK ){ - rc = mergeWorkerWrite(pMW, eType, pKey, nKey, pVal, nVal, iPtr); + rc = mergeWorkerWrite(pMW, eType, pKey, nKey, pCsr, iPtr); } } } /* Advance the cursor to the next input record (assuming one exists). */ assert( lsmMCursorValid(pMW->pCsr) ); if( rc==LSM_OK ) rc = lsmMCursorNext(pMW->pCsr); + /* If the cursor is at EOF, the merge is finished. Release all page + ** references currently held by the merge worker and inform the + ** FileSystem object that no further pages will be appended to either + ** the main or separators array. + */ + if( rc==LSM_OK && !lsmMCursorValid(pMW->pCsr) ){ + + mergeWorkerShutdown(pMW, &rc); + if( pSeg->iFirst ){ + rc = lsmFsSortedFinish(pDb->pFS, pSeg); + } + +#ifdef LSM_DEBUG_EXPENSIVE + if( rc==LSM_OK ){ +#if 0 + rc = assertBtreeOk(pDb, pSeg); + if( pMW->pCsr->pBtCsr ){ + Segment *pNext = &pMW->pLevel->pNext->lhs; + rc = assertPointersOk(pDb, pSeg, pNext, 0); + } +#endif + } +#endif + } return rc; } static int mergeWorkerDone(MergeWorker *pMW){ return pMW->pCsr==0 || !lsmMCursorValid(pMW->pCsr); @@ -4230,29 +3796,21 @@ } static int sortedNewToplevel( lsm_db *pDb, /* Connection handle */ int eTree, /* One of the TREE_XXX constants */ + int *pnOvfl, /* OUT: Number of free-list entries stored */ int *pnWrite /* OUT: Number of database pages written */ ){ int rc = LSM_OK; /* Return Code */ MultiCursor *pCsr = 0; Level *pNext = 0; /* The current top level */ Level *pNew; /* The new level itself */ - Segment *pLinked = 0; /* Delete separators from this segment */ - Level *pDel = 0; /* Delete this entire level */ + Segment *pDel = 0; /* Delete separators from this segment */ int nWrite = 0; /* Number of database pages written */ - Freelist freelist; - - if( eTree!=TREE_NONE ){ - rc = lsmShmCacheChunks(pDb, pDb->treehdr.nChunk); - } - - assert( pDb->bUseFreelist==0 ); - pDb->pFreelist = &freelist; - pDb->bUseFreelist = 1; - memset(&freelist, 0, sizeof(freelist)); + + assert( pnOvfl ); /* Allocate the new level structure to write to. */ pNext = lsmDbSnapshotLevel(pDb->pWorker); pNew = (Level *)lsmMallocZeroRc(pDb->pEnv, sizeof(Level), &rc); if( pNew ){ @@ -4264,27 +3822,17 @@ ** segment contains everything in the tree and pointers to the next segment ** in the database (if any). */ pCsr = multiCursorNew(pDb, &rc); if( pCsr ){ pCsr->pDb = pDb; - rc = multiCursorVisitFreelist(pCsr); - if( rc==LSM_OK ){ - rc = multiCursorAddTree(pCsr, pDb->pWorker, eTree); - } - if( rc==LSM_OK && pNext && pNext->pMerge==0 ){ - if( (pNext->flags & LEVEL_FREELIST_ONLY) ){ - pDel = pNext; - pCsr->aPtr = lsmMallocZeroRc(pDb->pEnv, sizeof(SegmentPtr), &rc); - multiCursorAddOne(pCsr, pNext, &rc); - }else if( eTree!=TREE_NONE && pNext->lhs.iRoot ){ - pLinked = &pNext->lhs; - rc = btreeCursorNew(pDb, pLinked, &pCsr->pBtCsr); - } - } - - /* If this will be the only segment in the database, discard any delete - ** markers present in the in-memory tree. */ + multiCursorVisitFreelist(pCsr, pnOvfl); + rc = multiCursorAddTree(pCsr, pDb->pWorker, eTree); + if( rc==LSM_OK && pNext && pNext->pMerge==0 && pNext->lhs.iRoot ){ + pDel = &pNext->lhs; + rc = btreeCursorNew(pDb, pDel, &pCsr->pBtCsr); + } + if( pNext==0 ){ multiCursorIgnoreDelete(pCsr); } } @@ -4297,11 +3845,11 @@ memset(&merge, 0, sizeof(Merge)); memset(&mergeworker, 0, sizeof(MergeWorker)); pNew->pMerge = &merge; - pNew->flags |= LEVEL_INCOMPLETE; + pNew->iAge = -1; mergeworker.pDb = pDb; mergeworker.pLevel = pNew; mergeworker.pCsr = pCsr; pCsr->pPrevMergePtr = &iLeftPtr; @@ -4311,59 +3859,36 @@ /* Do the work to create the new merged segment on disk */ if( rc==LSM_OK ) rc = lsmMCursorFirst(pCsr); while( rc==LSM_OK && mergeWorkerDone(&mergeworker)==0 ){ rc = mergeWorkerStep(&mergeworker); } - mergeWorkerShutdown(&mergeworker, &rc); - assert( rc!=LSM_OK || mergeworker.nWork==0 || pNew->lhs.iFirst ); - if( rc==LSM_OK && pNew->lhs.iFirst ){ - rc = lsmFsSortedFinish(pDb->pFS, &pNew->lhs); - } + nWrite = mergeworker.nWork; - pNew->flags &= ~LEVEL_INCOMPLETE; - if( eTree==TREE_NONE ){ - pNew->flags |= LEVEL_FREELIST_ONLY; - } + mergeWorkerShutdown(&mergeworker, &rc); pNew->pMerge = 0; + pNew->iAge = 0; } - if( rc!=LSM_OK || pNew->lhs.iFirst==0 ){ - assert( rc!=LSM_OK || pDb->pWorker->freelist.nEntry==0 ); + /* Link the new level into the top of the tree. */ + if( rc==LSM_OK ){ + if( pDel ) pDel->iRoot = 0; + }else{ lsmDbSnapshotSetLevel(pDb->pWorker, pNext); sortedFreeLevel(pDb->pEnv, pNew); - }else{ - if( pLinked ){ - pLinked->iRoot = 0; - }else if( pDel ){ - assert( pNew->pNext==pDel ); - pNew->pNext = pDel->pNext; - lsmFsSortedDelete(pDb->pFS, pDb->pWorker, 1, &pDel->lhs); - sortedFreeLevel(pDb->pEnv, pDel); - } - -#if LSM_LOG_STRUCTURE - lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, "new-toplevel"); + } + +#if 0 + lsmSortedDumpStructure(pDb, pDb->pWorker, 1, 0, "new-toplevel"); #endif - if( freelist.nEntry ){ - Freelist *p = &pDb->pWorker->freelist; - lsmFree(pDb->pEnv, p->aEntry); - memcpy(p, &freelist, sizeof(freelist)); - freelist.aEntry = 0; - }else{ - pDb->pWorker->freelist.nEntry = 0; - } - + if( rc==LSM_OK ){ assertBtreeOk(pDb, &pNew->lhs); sortedInvokeWorkHook(pDb); } if( pnWrite ) *pnWrite = nWrite; pDb->pWorker->nWrite += nWrite; - pDb->pFreelist = 0; - pDb->bUseFreelist = 0; - lsmFree(pDb->pEnv, freelist.aEntry); return rc; } /* ** The nMerge levels in the LSM beginning with pLevel consist of a @@ -4399,43 +3924,38 @@ pNew = (Level *)lsmMallocZeroRc(pDb->pEnv, sizeof(Level), &rc); if( pNew ){ pNew->aRhs = (Segment *)lsmMallocZeroRc(pDb->pEnv, nMerge * sizeof(Segment), &rc); } + /* Populate the new Level object */ if( rc==LSM_OK ){ Level *pNext = 0; /* Level following pNew */ int i; - int bFreeOnly = 1; Level *pTopLevel; Level *p = pLevel; Level **pp; pNew->nRight = nMerge; pNew->iAge = pLevel->iAge+1; for(i=0; inRight==0 ); pNext = p->pNext; pNew->aRhs[i] = p->lhs; - if( (p->flags & LEVEL_FREELIST_ONLY)==0 ) bFreeOnly = 0; sortedFreeLevel(pDb->pEnv, p); p = pNext; } - if( bFreeOnly ) pNew->flags |= LEVEL_FREELIST_ONLY; - /* Replace the old levels with the new. */ pTopLevel = lsmDbSnapshotLevel(pDb->pWorker); pNew->pNext = p; for(pp=&pTopLevel; *pp!=pLevel; pp=&((*pp)->pNext)); *pp = pNew; lsmDbSnapshotSetLevel(pDb->pWorker, pTopLevel); /* Determine whether or not the next separators will be linked in */ - if( pNext && pNext->pMerge==0 && pNext->lhs.iRoot && pNext - && (bFreeOnly==0 || (pNext->flags & LEVEL_FREELIST_ONLY)) - ){ + if( pNext && pNext->pMerge==0 && pNext->lhs.iRoot ){ bUseNext = 1; } } /* Allocate the merge object */ @@ -4480,11 +4000,10 @@ ** If the new level is the lowest (oldest) in the db, discard any ** delete keys. Key annihilation. */ pCsr = multiCursorNew(pDb, &rc); if( pCsr ){ - pCsr->flags |= CURSOR_NEXT_OK; rc = multiCursorAddRhs(pCsr, pLevel); } if( rc==LSM_OK && pMerge->nInput > pLevel->nRight ){ rc = btreeCursorNew(pDb, &pNext->lhs, &pCsr->pBtCsr); }else if( pNext ){ @@ -4540,38 +4059,34 @@ } return rc; } +/* TODO: Re-enable this!!! */ static int sortedBtreeGobble( - lsm_db *pDb, /* Worker connection */ - MultiCursor *pCsr, /* Multi-cursor being used for a merge */ - int iGobble /* pCsr->aPtr[] entry to operate on */ + lsm_db *pDb, + MultiCursor *pCsr, + int iGobble ){ int rc = LSM_OK; if( rtTopic(pCsr->eType)==0 ){ Segment *pSeg = pCsr->aPtr[iGobble].pSeg; + Blob *p = &pCsr->key; Pgno *aPg; int nPg; - /* Seek from the root of the b-tree to the segment leaf that may contain - ** a key equal to the one multi-cursor currently points to. Record the - ** page number of each b-tree page and the leaf. The segment may be - ** gobbled up to (but not including) the first of these page numbers. - */ assert( pSeg->iRoot>0 ); aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(Pgno)*32, &rc); if( rc==LSM_OK ){ - rc = seekInBtree(pCsr, pSeg, - rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, aPg, 0 - ); + rc = seekInBtree(pCsr, pSeg, p->pData, p->nData, aPg, 0); } - if( rc==LSM_OK ){ - for(nPg=0; aPg[nPg]; nPg++); - lsmFsGobble(pDb, pSeg, aPg, nPg); - } + for(nPg=0; aPg[nPg]; nPg++); + +#if 0 + lsmFsGobble(pDb, pSeg, aPg, nPg); +#endif lsmFree(pDb->pEnv, aPg); } return rc; } @@ -4588,33 +4103,30 @@ p = p->pNext; }while( p && p->iAge==iAge ); return nRet; } -static int sortedSelectLevel(lsm_db *pDb, int nMerge, Level **ppOut){ +static int sortedSelectLevel(lsm_db *pDb, int bOpt, Level **ppOut){ Level *pTopLevel = lsmDbSnapshotLevel(pDb->pWorker); int rc = LSM_OK; Level *pLevel = 0; /* Output value */ Level *pBest = 0; /* Best level to work on found so far */ - int nBest; /* Number of segments merged at pBest */ + int nBest = pDb->nMerge-1; /* Number of segments merged at pBest */ Level *pThis = 0; /* First in run of levels with age=iAge */ int nThis = 0; /* Number of levels starting at pThis */ - assert( nMerge>=1 ); - nBest = LSM_MAX(1, nMerge-1); - /* Find the longest contiguous run of levels not currently undergoing a ** merge with the same age in the structure. Or the level being merged ** with the largest number of right-hand segments. Work on it. */ for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ if( pLevel->nRight==0 && pThis && pLevel->iAge==pThis->iAge ){ nThis++; }else{ if( nThis>nBest ){ if( (pLevel->iAge!=pThis->iAge+1) - || (pLevel->nRight==0 && sortedCountLevels(pLevel)<=pDb->nMerge) - ){ + || (pLevel->nRight==0 && sortedCountLevels(pLevel)<=pDb->nMerge) + ){ pBest = pThis; nBest = nThis; } } if( pLevel->nRight ){ @@ -4634,25 +4146,13 @@ assert( pThis ); pBest = pThis; nBest = nThis; } - if( pBest==0 && nMerge==1 ){ - int nFree = 0; - int nUsr = 0; - for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ - assert( !pLevel->nRight ); - if( pLevel->flags & LEVEL_FREELIST_ONLY ){ - nFree++; - }else{ - nUsr++; - } - } - if( nUsr>1 ){ - pBest = pTopLevel; - nBest = nFree + nUsr; - } + if( pBest==0 && bOpt && pTopLevel->pNext ){ + pBest = pTopLevel; + nBest = 2; } if( pBest ){ if( pBest->nRight==0 ){ rc = sortedMergeSetup(pDb, pBest, nBest, ppOut); @@ -4674,180 +4174,14 @@ return 1; } return 0; } -typedef struct MoveBlockCtx MoveBlockCtx; -struct MoveBlockCtx { - int iSeen; /* Previous free block on list */ - int iFrom; /* Total number of blocks in file */ -}; - -static int moveBlockCb(void *pCtx, int iBlk, i64 iSnapshot){ - MoveBlockCtx *p = (MoveBlockCtx *)pCtx; - assert( p->iFrom==0 ); - if( iBlk==(p->iSeen-1) ){ - p->iSeen = iBlk; - return 0; - } - p->iFrom = p->iSeen-1; - return 1; -} - -/* -** This function is called to further compact a database for which all -** of the content has already been merged into a single segment. If -** possible, it moves the contents of a single block from the end of the -** file to a free-block that lies closer to the start of the file (allowing -** the file to be eventually truncated). -*/ -static int sortedMoveBlock(lsm_db *pDb, int *pnWrite){ - Snapshot *p = pDb->pWorker; - Level *pLvl = lsmDbSnapshotLevel(p); - int iFrom; /* Block to move */ - int iTo; /* Destination to move block to */ - int rc; /* Return code */ - - MoveBlockCtx sCtx; - - assert( pLvl->pNext==0 && pLvl->nRight==0 ); - assert( p->redirect.n<=LSM_MAX_BLOCK_REDIRECTS ); - - *pnWrite = 0; - - /* Check that the redirect array is not already full. If it is, return - ** without moving any database content. */ - if( p->redirect.n>=LSM_MAX_BLOCK_REDIRECTS ) return LSM_OK; - - /* Find the last block of content in the database file. Do this by - ** traversing the free-list in reverse (descending block number) order. - ** The first block not on the free list is the one that will be moved. - ** Since the db consists of a single segment, there is no ambiguity as - ** to which segment the block belongs to. */ - sCtx.iSeen = p->nBlock+1; - sCtx.iFrom = 0; - rc = lsmWalkFreelist(pDb, 1, moveBlockCb, &sCtx); - if( rc!=LSM_OK || sCtx.iFrom==0 ) return rc; - iFrom = sCtx.iFrom; - - /* Find the first free block in the database, ignoring block 1. Block - ** 1 is tricky as it is smaller than the other blocks. */ - rc = lsmBlockAllocate(pDb, iFrom, &iTo); - if( rc!=LSM_OK || iTo==0 ) return rc; - assert( iTo!=1 && iTopFS, &pLvl->lhs, iTo, iFrom); - if( rc==LSM_OK ){ - if( p->redirect.a==0 ){ - int nByte = sizeof(struct RedirectEntry) * LSM_MAX_BLOCK_REDIRECTS; - p->redirect.a = lsmMallocZeroRc(pDb->pEnv, nByte, &rc); - } - if( rc==LSM_OK ){ - - /* Check if the block just moved was already redirected. */ - int i; - for(i=0; iredirect.n; i++){ - if( p->redirect.a[i].iTo==iFrom ) break; - } - - if( i==p->redirect.n ){ - /* Block iFrom was not already redirected. Add a new array entry. */ - memmove(&p->redirect.a[1], &p->redirect.a[0], - sizeof(struct RedirectEntry) * p->redirect.n - ); - p->redirect.a[0].iFrom = iFrom; - p->redirect.a[0].iTo = iTo; - p->redirect.n++; - }else{ - /* Block iFrom was already redirected. Overwrite existing entry. */ - p->redirect.a[i].iTo = iTo; - } - - rc = lsmBlockFree(pDb, iFrom); - - *pnWrite = lsmFsBlockSize(pDb->pFS) / lsmFsPageSize(pDb->pFS); - pLvl->lhs.pRedirect = &p->redirect; - } - } - -#if LSM_LOG_STRUCTURE - if( rc==LSM_OK ){ - char aBuf[64]; - sprintf(aBuf, "move-block %d/%d", p->redirect.n-1, LSM_MAX_BLOCK_REDIRECTS); - lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, aBuf); - } -#endif - return rc; -} - -/* -*/ -static int mergeInsertFreelistSegments( - lsm_db *pDb, - int nFree, - MergeWorker *pMW -){ - int rc = LSM_OK; - if( nFree>0 ){ - MultiCursor *pCsr = pMW->pCsr; - Level *pLvl = pMW->pLevel; - SegmentPtr *aNew1; - Segment *aNew2; - - Level *pIter; - Level *pNext; - int i = 0; - - aNew1 = (SegmentPtr *)lsmMallocZeroRc( - pDb->pEnv, sizeof(SegmentPtr) * (pCsr->nPtr+nFree), &rc - ); - if( rc ) return rc; - memcpy(&aNew1[nFree], pCsr->aPtr, sizeof(SegmentPtr)*pCsr->nPtr); - pCsr->nPtr += nFree; - lsmFree(pDb->pEnv, pCsr->aTree); - lsmFree(pDb->pEnv, pCsr->aPtr); - pCsr->aTree = 0; - pCsr->aPtr = aNew1; - - aNew2 = (Segment *)lsmMallocZeroRc( - pDb->pEnv, sizeof(Segment) * (pLvl->nRight+nFree), &rc - ); - if( rc ) return rc; - memcpy(&aNew2[nFree], pLvl->aRhs, sizeof(Segment)*pLvl->nRight); - pLvl->nRight += nFree; - lsmFree(pDb->pEnv, pLvl->aRhs); - pLvl->aRhs = aNew2; - - for(pIter=pDb->pWorker->pLevel; rc==LSM_OK && pIter!=pLvl; pIter=pNext){ - Segment *pSeg = &pLvl->aRhs[i]; - memcpy(pSeg, &pIter->lhs, sizeof(Segment)); - - pCsr->aPtr[i].pSeg = pSeg; - pCsr->aPtr[i].pLevel = pLvl; - rc = segmentPtrEnd(pCsr, &pCsr->aPtr[i], 0); - - pDb->pWorker->pLevel = pNext = pIter->pNext; - sortedFreeLevel(pDb->pEnv, pIter); - i++; - } - assert( i==nFree ); - assert( rc!=LSM_OK || pDb->pWorker->pLevel==pLvl ); - - for(i=nFree; inPtr; i++){ - pCsr->aPtr[i].pSeg = &pLvl->aRhs[i]; - } - - lsmFree(pDb->pEnv, pMW->aGobble); - pMW->aGobble = 0; - } - return rc; -} - static int sortedWork( lsm_db *pDb, /* Database handle. Must be worker. */ int nWork, /* Number of pages of work to do */ - int nMerge, /* Try to merge this many levels at once */ + int bOptimize, /* True to merge less than nMerge levels */ int bFlush, /* Set if call is to make room for a flush */ int *pnWrite /* OUT: Actual number of pages written */ ){ int rc = LSM_OK; /* Return Code */ int nRemaining = nWork; /* Units of work to do before returning */ @@ -4858,85 +4192,37 @@ while( nRemaining>0 ){ Level *pLevel = 0; /* Find a level to work on. */ - rc = sortedSelectLevel(pDb, nMerge, &pLevel); + rc = sortedSelectLevel(pDb, bOptimize, &pLevel); assert( rc==LSM_OK || pLevel==0 ); if( pLevel==0 ){ - int nDone = 0; - Level *pTopLevel = lsmDbSnapshotLevel(pDb->pWorker); - if( bFlush==0 && nMerge==1 && pTopLevel && pTopLevel->pNext==0 ){ - rc = sortedMoveBlock(pDb, &nDone); - } - nRemaining -= nDone; - /* Could not find any work to do. Finished. */ - if( nDone==0 ) break; + break; }else{ - int bSave = 0; - Freelist freelist = {0, 0, 0}; MergeWorker mergeworker; /* State used to work on the level merge */ - assert( pDb->bIncrMerge==0 ); - assert( pDb->pFreelist==0 && pDb->bUseFreelist==0 ); - - pDb->bIncrMerge = 1; rc = mergeWorkerInit(pDb, pLevel, &mergeworker); assert( mergeworker.nWork==0 ); - while( rc==LSM_OK && 0==mergeWorkerDone(&mergeworker) - && (mergeworker.nWorkbUseFreelist) - ){ - int eType = rtTopic(mergeworker.pCsr->eType); - rc = mergeWorkerStep(&mergeworker); - - /* If the cursor now points at the first entry past the end of the - ** user data (i.e. either to EOF or to the first free-list entry - ** that will be added to the run), then check if it is possible to - ** merge in any free-list entries that are either in-memory or in - ** free-list-only blocks. */ - if( rc==LSM_OK && nMerge==1 && eType==0 - && (rtTopic(mergeworker.pCsr->eType) || mergeWorkerDone(&mergeworker)) - ){ - int nFree = 0; /* Number of free-list-only levels to merge */ - Level *pLvl; - assert( pDb->pFreelist==0 && pDb->bUseFreelist==0 ); - - /* Now check if all levels containing data newer than this one - ** are single-segment free-list only levels. If so, they will be - ** merged in now. */ - for(pLvl=pDb->pWorker->pLevel; - pLvl!=mergeworker.pLevel && (pLvl->flags & LEVEL_FREELIST_ONLY); - pLvl=pLvl->pNext - ){ - assert( pLvl->nRight==0 ); - nFree++; - } - if( pLvl==mergeworker.pLevel ){ - - rc = mergeInsertFreelistSegments(pDb, nFree, &mergeworker); - if( rc==LSM_OK ){ - rc = multiCursorVisitFreelist(mergeworker.pCsr); - } - if( rc==LSM_OK ){ - rc = multiCursorSetupTree(mergeworker.pCsr, 0); - pDb->pFreelist = &freelist; - pDb->bUseFreelist = 1; - } - } - } + && mergeworker.nWorknRight; i++){ SegmentPtr *pGobble = &mergeworker.pCsr->aPtr[i]; if( pGobble->pSeg->iRoot ){ @@ -4943,86 +4229,61 @@ rc = sortedBtreeGobble(pDb, mergeworker.pCsr, i); }else if( mergeworker.aGobble[i] ){ lsmFsGobble(pDb, pGobble->pSeg, &mergeworker.aGobble[i], 1); } } - }else{ - int i; - int bEmpty; - mergeWorkerShutdown(&mergeworker, &rc); - bEmpty = (pLevel->lhs.iFirst==0); - - if( bEmpty==0 && rc==LSM_OK ){ - rc = lsmFsSortedFinish(pDb->pFS, &pLevel->lhs); - } - - if( pDb->bUseFreelist ){ - Freelist *p = &pDb->pWorker->freelist; - lsmFree(pDb->pEnv, p->aEntry); - memcpy(p, &freelist, sizeof(freelist)); - pDb->bUseFreelist = 0; - pDb->pFreelist = 0; - bSave = 1; - } - + }else if( pLevel->lhs.iFirst==0 ){ + /* If the new level is completely empty, remove it from the + ** database snapshot. This can only happen if all input keys were + ** annihilated. Since keys are only annihilated if the new level + ** is the last in the linked list (contains the most ancient of + ** database content), this guarantees that pLevel->pNext==0. */ + + Level *pTop; /* Top level of worker snapshot */ + Level **pp; /* Read/write iterator for Level.pNext list */ + assert( pLevel->pNext==0 ); + + /* Remove the level from the worker snapshot. */ + pTop = lsmDbSnapshotLevel(pWorker); + for(pp=&pTop; *pp!=pLevel; pp=&((*pp)->pNext)); + *pp = pLevel->pNext; + lsmDbSnapshotSetLevel(pWorker, pTop); + + /* Free the Level structure. */ + lsmFsSortedDelete(pDb->pFS, pWorker, 1, &pLevel->lhs); + sortedFreeLevel(pDb->pEnv, pLevel); + }else{ + int i; + + /* Free the separators of the next level, if required. */ + if( pLevel->pMerge->nInput > pLevel->nRight ){ + assert( pLevel->pNext->lhs.iRoot ); + pLevel->pNext->lhs.iRoot = 0; + } + + /* Free the right-hand-side of pLevel */ for(i=0; inRight; i++){ lsmFsSortedDelete(pDb->pFS, pWorker, 1, &pLevel->aRhs[i]); } - - if( bEmpty ){ - /* If the new level is completely empty, remove it from the - ** database snapshot. This can only happen if all input keys were - ** annihilated. Since keys are only annihilated if the new level - ** is the last in the linked list (contains the most ancient of - ** database content), this guarantees that pLevel->pNext==0. */ - Level *pTop; /* Top level of worker snapshot */ - Level **pp; /* Read/write iterator for Level.pNext list */ - - assert( pLevel->pNext==0 ); - - /* Remove the level from the worker snapshot. */ - pTop = lsmDbSnapshotLevel(pWorker); - for(pp=&pTop; *pp!=pLevel; pp=&((*pp)->pNext)); - *pp = pLevel->pNext; - lsmDbSnapshotSetLevel(pWorker, pTop); - - /* Free the Level structure. */ - sortedFreeLevel(pDb->pEnv, pLevel); - }else{ - - /* Free the separators of the next level, if required. */ - if( pLevel->pMerge->nInput > pLevel->nRight ){ - assert( pLevel->pNext->lhs.iRoot ); - pLevel->pNext->lhs.iRoot = 0; - } - - /* Zero the right-hand-side of pLevel */ - lsmFree(pDb->pEnv, pLevel->aRhs); - pLevel->nRight = 0; - pLevel->aRhs = 0; - - /* Free the Merge object */ - lsmFree(pDb->pEnv, pLevel->pMerge); - pLevel->pMerge = 0; - } - - if( bSave && rc==LSM_OK ){ - pDb->bIncrMerge = 0; - rc = lsmSaveWorker(pDb, 0); - } + lsmFree(pDb->pEnv, pLevel->aRhs); + pLevel->nRight = 0; + pLevel->aRhs = 0; + + /* Free the Merge object */ + lsmFree(pDb->pEnv, pLevel->pMerge); + pLevel->pMerge = 0; } } /* Clean up the MergeWorker object initialized above. If no error ** has occurred, invoke the work-hook to inform the application that ** the database structure has changed. */ mergeWorkerShutdown(&mergeworker, &rc); - pDb->bIncrMerge = 0; if( rc==LSM_OK ) sortedInvokeWorkHook(pDb); -#if LSM_LOG_STRUCTURE - lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, "work"); +#if 0 + lsmSortedDumpStructure(pDb, pDb->pWorker, 1, 0, "work"); #endif assertBtreeOk(pDb, &pLevel->lhs); assertRunInOrder(pDb, &pLevel->lhs); /* If bFlush is true and the database is no longer considered "full", @@ -5053,10 +4314,13 @@ int rc = LSM_OK; int bRet = 0; assert( pDb->pWorker ); if( *pRc==LSM_OK ){ + if( pDb->nTransOpen==0 ){ + rc = lsmTreeLoadHeader(pDb, 0); + } if( rc==LSM_OK && pDb->treehdr.iOldShmid && pDb->treehdr.iOldLog!=pDb->pWorker->iLogOff ){ bRet = 1; @@ -5067,168 +4331,130 @@ } assert( *pRc==LSM_OK || bRet==0 ); return bRet; } -/* -** Create a new free-list only top-level segment. Return LSM_OK if successful -** or an LSM error code if some error occurs. -*/ -static int sortedNewFreelistOnly(lsm_db *pDb){ - return sortedNewToplevel(pDb, TREE_NONE, 0); -} - -int lsmSaveWorker(lsm_db *pDb, int bFlush){ - Snapshot *p = pDb->pWorker; - if( p->freelist.nEntry>pDb->nMaxFreelist ){ - int rc = sortedNewFreelistOnly(pDb); - if( rc!=LSM_OK ) return rc; - } - return lsmCheckpointSaveWorker(pDb, bFlush); -} - static int doLsmSingleWork( lsm_db *pDb, int bShutdown, - int nMerge, /* Minimum segments to merge together */ + int flags, int nPage, /* Number of pages to write to disk */ int *pnWrite, /* OUT: Pages actually written to disk */ int *pbCkpt /* OUT: True if an auto-checkpoint is req. */ ){ - Snapshot *pWorker; /* Worker snapshot */ int rc = LSM_OK; /* Return code */ - int bDirty = 0; + int nOvfl = 0; + int bFlush = 0; int nMax = nPage; /* Maximum pages to write to disk */ int nRem = nPage; int bCkpt = 0; - - assert( nPage>0 ); + int bToplevel = 0; /* Open the worker 'transaction'. It will be closed before this function ** returns. */ assert( pDb->pWorker==0 ); rc = lsmBeginWork(pDb); + assert( rc!=8 ); if( rc!=LSM_OK ) return rc; - pWorker = pDb->pWorker; /* If this connection is doing auto-checkpoints, set nMax (and nRem) so - ** that this call stops writing when the auto-checkpoint is due. The - ** caller will do the checkpoint, then possibly call this function again. */ + ** that this call stops writing when the auto-checkpoint is due. */ if( bShutdown==0 && pDb->nAutockpt ){ u32 nSync; u32 nUnsync; int nPgsz; + int nMax; lsmCheckpointSynced(pDb, 0, 0, &nSync); nUnsync = lsmCheckpointNWrite(pDb->pShmhdr->aSnap1, 0); nPgsz = lsmCheckpointPgsz(pDb->pShmhdr->aSnap1); - nMax = LSM_MIN(nMax, (pDb->nAutockpt/nPgsz) - (int)(nUnsync-nSync)); + nMax = (pDb->nAutockpt/nPgsz) - (nUnsync-nSync); if( nMaxnTransOpen==0 ){ - rc = lsmTreeLoadHeader(pDb, 0); - } if( sortedTreeHasOld(pDb, &rc) ){ - /* sortedDbIsFull() returns non-zero if either (a) there are too many - ** levels in total in the db, or (b) there are too many levels with the - ** the same age in the db. Either way, call sortedWork() to merge - ** existing segments together until this condition is cleared. */ if( sortedDbIsFull(pDb) ){ int nPg = 0; - rc = sortedWork(pDb, nRem, nMerge, 1, &nPg); + rc = sortedWork(pDb, nRem, 0, 1, &nPg); nRem -= nPg; assert( rc!=LSM_OK || nRem<=0 || !sortedDbIsFull(pDb) ); - bDirty = 1; + bToplevel = 1; } - if( rc==LSM_OK && nRem>0 ){ int nPg = 0; - rc = sortedNewToplevel(pDb, TREE_OLD, &nPg); + rc = sortedNewToplevel(pDb, TREE_OLD, &nOvfl, &nPg); nRem -= nPg; - if( rc==LSM_OK ){ - if( pDb->nTransOpen>0 ){ - lsmTreeDiscardOld(pDb); - } - rc = lsmSaveWorker(pDb, 1); - bDirty = 0; + if( rc==LSM_OK && pDb->nTransOpen>0 ){ + lsmTreeDiscardOld(pDb); } + bFlush = 1; + bToplevel = 0; } } /* If nPage is still greater than zero, do some merging. */ if( rc==LSM_OK && nRem>0 && bShutdown==0 ){ int nPg = 0; - rc = sortedWork(pDb, nRem, nMerge, 0, &nPg); - nRem -= nPg; - if( nPg ) bDirty = 1; - } - - /* If the in-memory part of the free-list is too large, write a new - ** top-level containing just the in-memory free-list entries to disk. */ - if( rc==LSM_OK && pDb->pWorker->freelist.nEntry > pDb->nMaxFreelist ){ - int nPg = 0; - while( rc==LSM_OK && lsmDatabaseFull(pDb) ){ - rc = sortedWork(pDb, 16, nMerge, 1, &nPg); - nRem -= nPg; - } - if( rc==LSM_OK ){ - rc = sortedNewFreelistOnly(pDb); - } - nRem -= nPg; - if( nPg ) bDirty = 1; - } - - if( rc==LSM_OK ){ - *pnWrite = (nMax - nRem); - *pbCkpt = (bCkpt && nRem<=0); - if( nMerge==1 && pDb->nAutockpt>0 && *pnWrite>0 - && pWorker->pLevel - && pWorker->pLevel->nRight==0 - && pWorker->pLevel->pNext==0 - ){ - *pbCkpt = 1; - } - } - - if( rc==LSM_OK && bDirty ){ - lsmFinishWork(pDb, 0, &rc); - }else{ - int rcdummy = LSM_BUSY; - lsmFinishWork(pDb, 0, &rcdummy); - *pnWrite = 0; - } - assert( pDb->pWorker==0 ); + int bOptimize = ((flags & LSM_WORK_OPTIMIZE) ? 1 : 0); + rc = sortedWork(pDb, nRem, bOptimize, 0, &nPg); + nRem -= nPg; + if( nPg ){ + bToplevel = 1; + nOvfl = 0; + } + } + + if( rc==LSM_OK && bToplevel && lsmCheckpointOverflowRequired(pDb) ){ + while( rc==LSM_OK && sortedDbIsFull(pDb) ){ + int nPg = 0; + rc = sortedWork(pDb, 16, 0, 1, &nPg); + } + if( rc==LSM_OK && lsmCheckpointOverflowRequired(pDb) ){ + rc = sortedNewToplevel(pDb, TREE_NONE, &nOvfl, 0); + } + } + + if( rc==LSM_OK && (nRem!=nMax) ){ + lsmFinishWork(pDb, bFlush, nOvfl, &rc); + }else{ + int rcdummy = LSM_BUSY; + assert( rc!=LSM_OK || bFlush==0 ); + lsmFinishWork(pDb, 0, 0, &rcdummy); + } + assert( pDb->pWorker==0 ); + + if( rc==LSM_OK ){ + if( pnWrite ) *pnWrite = (nMax - nRem); + if( pbCkpt ) *pbCkpt = (bCkpt && nRem<=0); + }else{ + if( pnWrite ) *pnWrite = 0; + if( pbCkpt ) *pbCkpt = 0; + } + return rc; } -static int doLsmWork(lsm_db *pDb, int nMerge, int nPage, int *pnWrite){ - int rc = LSM_OK; /* Return code */ - int nWrite = 0; /* Number of pages written */ - - assert( nMerge>=1 ); - - if( nPage!=0 ){ - int bCkpt = 0; - do { - int nThis = 0; - int nReq = (nPage>=0) ? (nPage-nWrite) : ((int)0x7FFFFFFF); - - bCkpt = 0; - rc = doLsmSingleWork(pDb, 0, nMerge, nReq, &nThis, &bCkpt); - nWrite += nThis; - if( rc==LSM_OK && bCkpt ){ - rc = lsm_checkpoint(pDb, 0); - } - }while( rc==LSM_OK && bCkpt && (nWritenTransOpen || pDb->pCsr ) return LSM_MISUSE_BKPT; - if( nMerge<=0 ) nMerge = pDb->nMerge; - - /* Convert from KB to pages */ - nPgsz = lsmFsPageSize(pDb->pFS); - if( nKB>=0 ){ - nPage = ((i64)nKB * 1024 + nPgsz - 1) / nPgsz; - }else{ - nPage = -1; - } - - rc = doLsmWork(pDb, nMerge, nPage, &nWrite); - - if( pnWrite ){ - /* Convert back from pages to KB */ - *pnWrite = (int)(((i64)nWrite * 1024 + nPgsz - 1) / nPgsz); - } - return rc; -} - -int lsm_flush(lsm_db *db){ - int rc; - - if( db->nTransOpen>0 || db->pCsr ){ - rc = LSM_MISUSE_BKPT; - }else{ - rc = lsmBeginWriteTrans(db); - if( rc==LSM_OK ){ - lsmFlushTreeToDisk(db); - lsmTreeDiscardOld(db); - lsmTreeMakeOld(db); - lsmTreeDiscardOld(db); - } - - if( rc==LSM_OK ){ - rc = lsmFinishWriteTrans(db, 1); - }else{ - lsmFinishWriteTrans(db, 0); - } - lsmFinishReadTrans(db); - } - - return rc; + + return doLsmWork(pDb, flags, nPage, pnWrite); } /* ** This function is called in auto-work mode to perform merging work on ** the data structure. It performs enough merging work to prevent the @@ -5332,12 +4514,11 @@ nRemaining = nUnit * nDepth; #ifdef LSM_LOG_WORK lsmLogMessage(pDb, rc, "lsmSortedAutoWork(): %d*%d = %d pages", nUnit, nDepth, nRemaining); #endif - assert( nRemaining>=0 ); - rc = doLsmWork(pDb, pDb->nMerge, nRemaining, 0); + rc = doLsmWork(pDb, LSM_WORK_FLUSH, nRemaining, 0); if( rc==LSM_BUSY ) rc = LSM_OK; if( bRestore && pDb->pCsr ){ lsmFreeSnapshot(pDb->pEnv, pDb->pClient); pDb->pClient = 0; @@ -5358,21 +4539,21 @@ ** This function is only called during system shutdown. The contents of ** any in-memory trees present (old or current) are written out to disk. */ int lsmFlushTreeToDisk(lsm_db *pDb){ int rc; + int nOvfl = 0; rc = lsmBeginWork(pDb); while( rc==LSM_OK && sortedDbIsFull(pDb) ){ - rc = sortedWork(pDb, 256, pDb->nMerge, 1, 0); + rc = sortedWork(pDb, 256, 0, 1, 0); } if( rc==LSM_OK ){ - rc = sortedNewToplevel(pDb, TREE_BOTH, 0); + rc = sortedNewToplevel(pDb, TREE_BOTH, &nOvfl, 0); } - - lsmFinishWork(pDb, 1, &rc); + lsmFinishWork(pDb, 1, nOvfl, &rc); return rc; } /* ** Return a string representation of the segment passed as the only argument. @@ -5410,32 +4591,22 @@ return z; } static int fileToString( - lsm_db *pDb, /* For xMalloc() */ + lsm_env *pEnv, /* For xMalloc() */ char *aBuf, int nBuf, int nMin, Segment *pSeg ){ int i = 0; - if( pSeg ){ - char *zSeg; - - zSeg = segToString(pDb->pEnv, pSeg, nMin); - i += sqlite4_snprintf(&aBuf[i], nBuf-i, "%s", zSeg); - lsmFree(pDb->pEnv, zSeg); - -#ifdef LSM_LOG_FREELIST - lsmInfoArrayStructure(pDb, 1, pSeg->iFirst, &zSeg); - i += sqlite4_snprintf(&aBuf[i], nBuf-i, " (%s)", zSeg); - lsmFree(pDb->pEnv, zSeg); -#endif - }else{ - aBuf[0] = '\0'; - } + char *zSeg; + + zSeg = segToString(pEnv, pSeg, nMin); + i += sqlite4_snprintf(&aBuf[i], nBuf-i, "%s", zSeg); + lsmFree(pEnv, zSeg); return i; } void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){ @@ -5475,16 +4646,16 @@ aCell += lsmVarintGet32(aCell, &iPgPtr); if( eType==0 ){ Pgno iRef; /* Page number of referenced page */ aCell += lsmVarintGet64(aCell, &iRef); - lsmFsDbPageGet(pDb->pFS, pRun, iRef, &pRef); - aKey = pageGetKey(pRun, pRef, 0, &iTopic, &nKey, &blob); + lsmFsDbPageGet(pDb->pFS, iRef, &pRef); + aKey = pageGetKey(pRef, 0, &iTopic, &nKey, &blob); }else{ aCell += lsmVarintGet32(aCell, &nKey); if( rtIsWrite(eType) ) aCell += lsmVarintGet32(aCell, &nVal); - sortedReadData(0, pPg, (aCell-aData), nKey+nVal, (void **)&aKey, &blob); + sortedReadData(pPg, (aCell-aData), nKey+nVal, (void **)&aKey, &blob); aVal = &aKey[nKey]; iTopic = eType; } lsmStringAppendf(&s, "%s%2X:", (i==0?"":" "), iTopic); @@ -5508,12 +4679,11 @@ sortedBlobFree(&blob); } static void infoCellDump( - lsm_db *pDb, /* Database handle */ - Segment *pSeg, /* Segment page belongs to */ + lsm_db *pDb, int bIndirect, /* True to follow indirect refs */ Page *pPg, int iCell, int *peType, int *piPgPtr, @@ -5538,12 +4708,12 @@ if( eType==0 ){ int dummy; Pgno iRef; /* Page number of referenced page */ aCell += lsmVarintGet64(aCell, &iRef); if( bIndirect ){ - lsmFsDbPageGet(pDb->pFS, pSeg, iRef, &pRef); - pageGetKeyCopy(pDb->pEnv, pSeg, pRef, 0, &dummy, pBlob); + lsmFsDbPageGet(pDb->pFS, iRef, &pRef); + pageGetKeyCopy(pDb->pEnv, pRef, 0, &dummy, pBlob); aKey = (u8 *)pBlob->pData; nKey = pBlob->nData; lsmFsPageRelease(pRef); }else{ aKey = (u8 *)""; @@ -5550,11 +4720,11 @@ nKey = 11; } }else{ aCell += lsmVarintGet32(aCell, &nKey); if( rtIsWrite(eType) ) aCell += lsmVarintGet32(aCell, &nVal); - sortedReadData(pSeg, pPg, (aCell-aData), nKey+nVal, (void **)&aKey, pBlob); + sortedReadData(pPg, (aCell-aData), nKey+nVal, (void **)&aKey, pBlob); aVal = &aKey[nKey]; } if( peType ) *peType = eType; if( piPgPtr ) *piPgPtr = iPgPtr; @@ -5589,42 +4759,20 @@ ){ int rc = LSM_OK; /* Return code */ Page *pPg = 0; /* Handle for page iPg */ int i, j; /* Loop counters */ const int perLine = 16; /* Bytes per line in the raw hex dump */ - Segment *pSeg = 0; - Snapshot *pSnap; int bValues = (flags & INFO_PAGE_DUMP_VALUES); int bHex = (flags & INFO_PAGE_DUMP_HEX); int bData = (flags & INFO_PAGE_DUMP_DATA); int bIndirect = (flags & INFO_PAGE_DUMP_INDIRECT); *pzOut = 0; if( iPg==0 ) return LSM_ERROR; - assert( pDb->pClient || pDb->pWorker ); - pSnap = pDb->pClient; - if( pSnap==0 ) pSnap = pDb->pWorker; - if( pSnap->redirect.n>0 ){ - Level *pLvl; - int bUse = 0; - for(pLvl=pSnap->pLevel; pLvl->pNext; pLvl=pLvl->pNext); - pSeg = (pLvl->nRight==0 ? &pLvl->lhs : &pLvl->aRhs[pLvl->nRight-1]); - rc = lsmFsSegmentContainsPg(pDb->pFS, pSeg, iPg, &bUse); - if( bUse==0 ){ - pSeg = 0; - } - } - - /* iPg is a real page number (not subject to redirection). So it is safe - ** to pass a NULL in place of the segment pointer as the second argument - ** to lsmFsDbPageGet() here. */ - if( rc==LSM_OK ){ - rc = lsmFsDbPageGet(pDb->pFS, 0, iPg, &pPg); - } - + rc = lsmFsDbPageGet(pDb->pFS, iPg, &pPg); if( rc==LSM_OK ){ Blob blob = {0, 0, 0, 0}; int nKeyWidth = 0; LsmString str; int nRec; @@ -5637,34 +4785,33 @@ nRec = pageGetNRec(aData, nData); iPtr = pageGetPtr(aData, nData); flags = pageGetFlags(aData, nData); lsmStringInit(&str, pDb->pEnv); - lsmStringAppendf(&str, "Page : %lld (%d bytes)\n", iPg, nData); + lsmStringAppendf(&str, "Page : %d (%d bytes)\n", iPg, nData); lsmStringAppendf(&str, "nRec : %d\n", nRec); lsmStringAppendf(&str, "iPtr : %d\n", iPtr); lsmStringAppendf(&str, "flags: %04x\n", flags); lsmStringAppendf(&str, "\n"); for(iCell=0; iCellnKeyWidth ) nKeyWidth = nKey; } if( bHex ) nKeyWidth = nKeyWidth * 2; for(iCell=0; iCell0 && bValues ){ lsmStringAppendf(&str, "%*s", nKeyWidth - (nKey*(1+bHex)), ""); lsmStringAppendf(&str, " "); infoAppendBlob(&str, bHex, aVal, nVal); } - if( rtTopic(eType) ){ - int iBlk = (int)~lsmGetU32(aKey); - lsmStringAppendf(&str, " (block=%d", iBlk); - if( nVal>0 ){ - i64 iSnap = lsmGetU64(aVal); - lsmStringAppendf(&str, " snapshot=%lld", iSnap); - } - lsmStringAppendf(&str, ")"); - } lsmStringAppendf(&str, "\n"); } if( bData ){ lsmStringAppendf(&str, "\n-------------------" @@ -5743,11 +4881,11 @@ zSeg = segToString(pDb->pEnv, pRun, 0); lsmLogMessage(pDb, LSM_OK, "Segment: %s", zSeg); lsmFree(pDb->pEnv, zSeg); - lsmFsDbPageGet(pDb->pFS, pRun, pRun->iFirst, &pPg); + lsmFsDbPageGet(pDb->pFS, pRun->iFirst, &pPg); while( pPg ){ Page *pNext; char *z = 0; infoPageDump(pDb, lsmFsPageNumber(pPg), flags, &z); lsmLogMessage(pDb, LSM_OK, "%s", z); @@ -5773,26 +4911,18 @@ int bVals, /* Output the values from each segment */ const char *zWhy /* Caption to print near top of dump */ ){ Snapshot *pDump = pSnap; Level *pTopLevel; - char *zFree = 0; - assert( pSnap ); pTopLevel = lsmDbSnapshotLevel(pDump); if( pDb->xLog && pTopLevel ){ - static int nCall = 0; Level *pLevel; int iLevel = 0; - nCall++; - lsmLogMessage(pDb, LSM_OK, "Database structure %d (%s)", nCall, zWhy); - -#if 0 - if( nCall==1031 || nCall==1032 ) bKeys=1; -#endif + lsmLogMessage(pDb, LSM_OK, "Database structure (%s)", zWhy); for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ char zLeft[1024]; char zRight[1024]; int i = 0; @@ -5808,44 +4938,36 @@ for(i=0; inRight; i++){ aRight[nRight++] = &pLevel->aRhs[i]; } -#ifdef LSM_LOG_FREELIST - if( nRight ){ - memmove(&aRight[1], aRight, sizeof(aRight[0])*nRight); - aRight[0] = 0; - nRight++; - } -#endif - for(i=0; ipEnv, zLeft, sizeof(zLeft), 28, aLeft[i]); } if( ipEnv, zRight, sizeof(zRight), 28, aRight[i]); } if( i==0 ){ - sqlite4_snprintf(zLevel, sizeof(zLevel), "L%d: (age=%d) (flags=%.4x)", - iLevel, (int)pLevel->iAge, (int)pLevel->flags + sqlite4_snprintf(zLevel, sizeof(zLevel), "L%d: (age=%d)", + iLevel, pLevel->iAge ); }else{ zLevel[0] = '\0'; } if( nRight==0 ){ - iPad = 10; + iPad = 28 - (strlen(zLeft)/2) ; } - lsmLogMessage(pDb, LSM_OK, "% 25s % *s% -35s %s", + lsmLogMessage(pDb, LSM_OK, "% 7s % *s% -35s %s", zLevel, iPad, "", zLeft, zRight ); } iLevel++; @@ -5859,16 +4981,10 @@ sortedDumpSegment(pDb, &pLevel->aRhs[i], bVals); } } } } - - lsmInfoFreelist(pDb, &zFree); - lsmLogMessage(pDb, LSM_OK, "Freelist: %s", zFree); - lsmFree(pDb->pEnv, zFree); - - assert( lsmFsIntegrityCheck(pDb) ); } void lsmSortedFreeLevel(lsm_env *pEnv, Level *pLevel){ Level *pNext; Level *p; @@ -5885,29 +5001,17 @@ lsmTreeCursorSave(pCsr->apTreeCsr[0]); lsmTreeCursorSave(pCsr->apTreeCsr[1]); } } -void lsmSortedExpandBtreePage(Page *pPg, int nOrig){ - u8 *aData; - int nData; - int nEntry; - int iHdr; - - aData = lsmFsPageData(pPg, &nData); - nEntry = pageGetNRec(aData, nOrig); - iHdr = SEGMENT_EOF(nOrig, nEntry); - memmove(&aData[iHdr + (nData-nOrig)], &aData[iHdr], nOrig-iHdr); -} - #ifdef LSM_DEBUG_EXPENSIVE static void assertRunInOrder(lsm_db *pDb, Segment *pSeg){ Page *pPg = 0; Blob blob1 = {0, 0, 0, 0}; Blob blob2 = {0, 0, 0, 0}; - lsmFsDbPageGet(pDb->pFS, pSeg, pSeg->iFirst, &pPg); + lsmFsDbPageGet(pDb->pFS, pSeg->iFirst, &pPg); while( pPg ){ u8 *aData; int nData; Page *pNext; aData = lsmFsPageData(pPg, &nData); @@ -5914,21 +5018,21 @@ if( 0==(pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG) ){ int i; int nRec = pageGetNRec(aData, nData); for(i=0; ipEnv, pSeg, pPg, i, &iTopic1, &blob1); + pageGetKeyCopy(pDb->pEnv, pPg, i, &iTopic1, &blob1); if( i==0 && blob2.nData ){ assert( sortedKeyCompare( pDb->xCmp, iTopic2, blob2.pData, blob2.nData, iTopic1, blob1.pData, blob1.nData )<0 ); } if( i<(nRec-1) ){ - pageGetKeyCopy(pDb->pEnv, pSeg, pPg, i+1, &iTopic2, &blob2); + pageGetKeyCopy(pDb->pEnv, pPg, i+1, &iTopic2, &blob2); assert( sortedKeyCompare( pDb->xCmp, iTopic1, blob1.pData, blob1.nData, iTopic2, blob2.pData, blob2.nData )<0 ); } @@ -6058,11 +5162,11 @@ rc = btreeCursorNew(pDb, pSeg, &pCsr); if( rc==LSM_OK ){ rc = btreeCursorFirst(pCsr); } if( rc==LSM_OK ){ - rc = lsmFsDbPageGet(pFS, pSeg, pSeg->iFirst, &pPg); + rc = lsmFsDbPageGet(pFS, pSeg->iFirst, &pPg); } while( rc==LSM_OK ){ Page *pNext; u8 *aData; @@ -6080,11 +5184,11 @@ && 0!=pageGetNRec(aData, nData) ){ u8 *pKey; int nKey; int iTopic; - pKey = pageGetKey(pSeg, pPg, 0, &iTopic, &nKey, &blob); + pKey = pageGetKey(pPg, 0, &iTopic, &nKey, &blob); assert( nKey==pCsr->nKey && 0==memcmp(pKey, pCsr->pKey, nKey) ); assert( lsmFsPageNumber(pPg)==pCsr->iPtr ); rc = btreeCursorNext(pCsr); } } Index: src/lsm_tree.c ================================================================== --- src/lsm_tree.c +++ src/lsm_tree.c @@ -141,12 +141,12 @@ int nKey; /* Size of pKey in bytes */ int nValue; /* Size of pValue. Or negative. */ u8 flags; /* Various LSM_XXX flags */ }; -#define TKV_KEY(p) ((void *)&(p)[1]) -#define TKV_VAL(p) ((void *)(((u8 *)&(p)[1]) + (p)->nKey)) +#define TK_KEY(p) ((void *)&(p)[1]) +#define TK_VAL(p) ((void *)(((u8 *)&(p)[1]) + (p)->nKey)) /* ** A single tree node. A node structure may contain up to 3 key/value ** pairs. Internal (non-leaf) nodes have up to 4 children. ** @@ -239,11 +239,12 @@ /* ** Zero the IntArray object. */ static void intArrayFree(lsm_env *pEnv, IntArray *p){ - p->nArray = 0; + lsmFree(pEnv, p->aArray); + memset(p, 0, sizeof(IntArray)); } /* ** Return the number of entries currently in the int-array object. */ @@ -289,28 +290,32 @@ /* ** Return a pointer to the mapped memory location associated with *-shm ** file offset iPtr. */ -static void *treeShmptr(lsm_db *pDb, u32 iPtr){ +static void *treeShmptr(lsm_db *pDb, u32 iPtr, int *pRc){ + /* TODO: This will likely be way too slow. If it is, chunks should be + ** cached as part of the db handle. */ + void *pChunk; + int iChunk = (iPtr>>15); + assert( LSM_SHM_CHUNK_SIZE==(1<<15) ); - assert( (iPtr>>15)nShm ); - assert( pDb->apShm[iPtr>>15] ); + if( (pDb->nShm<=iChunk || 0==(pChunk = pDb->apShm[iChunk])) ){ + *pRc = lsmShmChunk(pDb, iChunk, &pChunk); + } - return iPtr?(&((u8*)(pDb->apShm[iPtr>>15]))[iPtr & (LSM_SHM_CHUNK_SIZE-1)]):0; + if( iPtr==0 || *pRc ) return 0; + return &((u8 *)pChunk)[iPtr & (LSM_SHM_CHUNK_SIZE-1)]; } static ShmChunk * treeShmChunk(lsm_db *pDb, int iChunk){ - return (ShmChunk *)(pDb->apShm[iChunk]); + int rcdummy = LSM_OK; + return (ShmChunk *)treeShmptr(pDb, iChunk*LSM_SHM_CHUNK_SIZE, &rcdummy); } static ShmChunk * treeShmChunkRc(lsm_db *pDb, int iChunk, int *pRc){ - assert( *pRc==LSM_OK ); - if( iChunknShm || LSM_OK==(*pRc = lsmShmCacheChunks(pDb, iChunk+1)) ){ - return (ShmChunk *)(pDb->apShm[iChunk]); - } - return 0; + return (ShmChunk *)treeShmptr(pDb, iChunk*LSM_SHM_CHUNK_SIZE, pRc); } #ifndef NDEBUG static void assertIsWorkingChild( @@ -320,20 +325,20 @@ int iCell ){ TreeNode *p; int rc = LSM_OK; u32 iPtr = getChildPtr(pParent, WORKING_VERSION, iCell); - p = treeShmptr(db, iPtr); - assert( p==pNode ); + p = treeShmptr(db, iPtr, &rc); + assert( p==pNode || rc!=LSM_OK ); } #else # define assertIsWorkingChild(w,x,y,z) #endif /* Values for the third argument to treeShmkey(). */ -#define TKV_LOADKEY 1 -#define TKV_LOADVAL 2 +#define TK_LOADKEY 1 +#define TK_LOADVAL 2 static TreeKey *treeShmkey( lsm_db *pDb, /* Database handle */ u32 iPtr, /* Shmptr to TreeKey struct */ int eLoad, /* Either zero or a TREEKEY_LOADXXX value */ @@ -340,18 +345,18 @@ TreeBlob *pBlob, /* Used if dynamic memory is required */ int *pRc /* IN/OUT: Error code */ ){ TreeKey *pRet; - assert( eLoad==TKV_LOADKEY || eLoad==TKV_LOADVAL ); - pRet = (TreeKey *)treeShmptr(pDb, iPtr); + assert( eLoad==TK_LOADKEY || eLoad==TK_LOADVAL ); + pRet = (TreeKey *)treeShmptr(pDb, iPtr, pRc); if( pRet ){ int nReq; /* Bytes of space required at pRet */ int nAvail; /* Bytes of space available at pRet */ nReq = sizeof(TreeKey) + pRet->nKey; - if( eLoad==TKV_LOADVAL && pRet->nValue>0 ){ + if( eLoad==TK_LOADVAL && pRet->nValue>0 ){ nReq += pRet->nValue; } assert( LSM_SHM_CHUNK_SIZE==(1<<15) ); nAvail = LSM_SHM_CHUNK_SIZE - (iPtr & (LSM_SHM_CHUNK_SIZE-1)); @@ -358,11 +363,11 @@ if( nAvaila[nLoad], p, n); nLoad += n; if( nLoad==nReq ) break; @@ -485,23 +490,23 @@ int rc = LSM_OK; LsmString s; TreeNode *pNode; TreeBlob b = {0, 0}; - pNode = (TreeNode *)treeShmptr(pDb, iNode); + pNode = (TreeNode *)treeShmptr(pDb, iNode, &rc); if( nHeight==0 ){ /* Append the nIndent bytes of space to string s. */ lsmStringInit(&s, pDb->pEnv); /* Append each key to string s. */ for(i=0; i<3; i++){ u32 iPtr = pNode->aiKeyPtr[i]; if( iPtr ){ - TreeKey *pKey = treeShmkey(pDb, pNode->aiKeyPtr[i],TKV_LOADKEY, &b,&rc); + TreeKey *pKey = treeShmkey(pDb, pNode->aiKeyPtr[i], TK_LOADKEY, &b,&rc); strAppendFlags(&s, pKey->flags); - lsmAppendStrBlob(&s, TKV_KEY(pKey), pKey->nKey); + lsmAppendStrBlob(&s, TK_KEY(pKey), pKey->nKey); lsmStringAppend(&s, " ", -1); } } printf("% 6d %.*sleaf%.*s: %s\n", @@ -516,14 +521,14 @@ if( iPtr ){ dump_node_contents(pDb, iPtr, zPath, nPath+2, nHeight-1); } if( i!=3 && pNode->aiKeyPtr[i] ){ - TreeKey *pKey = treeShmkey(pDb, pNode->aiKeyPtr[i], TKV_LOADKEY,&b,&rc); + TreeKey *pKey = treeShmkey(pDb, pNode->aiKeyPtr[i], TK_LOADKEY, &b,&rc); lsmStringInit(&s, pDb->pEnv); strAppendFlags(&s, pKey->flags); - lsmAppendStrBlob(&s, TKV_KEY(pKey), pKey->nKey); + lsmAppendStrBlob(&s, TK_KEY(pKey), pKey->nKey); printf("% 6d %.*s%.*s: %s\n", iNode, nPath+1, zPath, 20-nPath-1, zSpace, s.z); lsmStringClear(&s); } } @@ -565,11 +570,11 @@ ** is pointing to. */ static TreeKey *csrGetKey(TreeCursor *pCsr, TreeBlob *pBlob, int *pRc){ TreeKey *pRet = (TreeKey *)treeShmkey(pCsr->pDb, pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[pCsr->aiCell[pCsr->iNode]], - TKV_LOADVAL, pBlob, pRc + TK_LOADVAL, pBlob, pRc ); assert( pRet==0 || assertFlagsOk(pRet->flags) ); return pRet; } @@ -595,11 +600,11 @@ int rc = LSM_OK; if( pCsr->pSave ){ TreeKey *pKey = pCsr->pSave; pCsr->pSave = 0; if( pRes ){ - rc = lsmTreeCursorSeek(pCsr, TKV_KEY(pKey), pKey->nKey, pRes); + rc = lsmTreeCursorSeek(pCsr, TK_KEY(pKey), pKey->nKey, pRes); } } return rc; } @@ -671,21 +676,21 @@ *pRc = rc; return 0; } /* Set the header values for the chunk just finished */ - pHdr = (ShmChunk *)treeShmptr(pDb, iChunk*CHUNK_SIZE); + pHdr = (ShmChunk *)treeShmptr(pDb, iChunk*CHUNK_SIZE, pRc); pHdr->iNext = iNext; /* Advance to the next chunk */ iWrite = iNext * CHUNK_SIZE + CHUNK_HDR; } /* Allocate space at iWrite. */ iRet = iWrite; pDb->treehdr.iWrite = iWrite + nByte; - pDb->treehdr.root.nByte += nByte; + pDb->treehdr.nByte += nByte; } return iRet; } /* @@ -693,11 +698,11 @@ */ static void *treeShmallocZero(lsm_db *pDb, int nByte, u32 *piPtr, int *pRc){ u32 iPtr; void *p; iPtr = treeShmalloc(pDb, 1, nByte, pRc); - p = treeShmptr(pDb, iPtr); + p = treeShmptr(pDb, iPtr, pRc); if( p ){ assert( *pRc==LSM_OK ); memset(p, 0, nByte); *piPtr = iPtr; } @@ -725,11 +730,11 @@ u8 *a; int n; /* Allocate space for the TreeKey structure itself */ *piPtr = iPtr = treeShmalloc(pDb, 1, sizeof(TreeKey), pRc); - p = treeShmptr(pDb, iPtr); + p = treeShmptr(pDb, iPtr, pRc); if( *pRc ) return 0; p->nKey = nKey; p->nValue = nVal; /* Allocate and populate the space required for the key and value. */ @@ -743,11 +748,11 @@ iWrite = (pDb->treehdr.iWrite & (LSM_SHM_CHUNK_SIZE-1)); iWrite = LSM_MAX(iWrite, LSM_SHM_CHUNK_HDR); nAlloc = LSM_MIN((LSM_SHM_CHUNK_SIZE-iWrite), nRem); - aAlloc = treeShmptr(pDb, treeShmalloc(pDb, 0, nAlloc, pRc)); + aAlloc = treeShmptr(pDb, treeShmalloc(pDb, 0, nAlloc, pRc), pRc); if( aAlloc==0 ) break; memcpy(aAlloc, &a[n-nRem], nAlloc); nRem -= nAlloc; } a = pVal; @@ -1043,10 +1048,21 @@ } } return rc; } + +/* +** Empty the contents of the in-memory tree. +*/ +void lsmTreeClear(lsm_db *pDb){ + pDb->treehdr.root.iTransId = 1; + pDb->treehdr.root.iRoot = 0; + pDb->treehdr.root.nHeight = 0; + pDb->treehdr.nByte = 0; + pDb->treehdr.iUsedShmid = pDb->treehdr.iNextShmid-1; +} void lsmTreeMakeOld(lsm_db *pDb){ if( pDb->treehdr.iOldShmid==0 ){ pDb->treehdr.iOldLog = pDb->treehdr.log.aRegion[2].iEnd; pDb->treehdr.oldcksum0 = pDb->treehdr.log.cksum0; @@ -1055,11 +1071,11 @@ memcpy(&pDb->treehdr.oldroot, &pDb->treehdr.root, sizeof(TreeRoot)); pDb->treehdr.root.iTransId = 1; pDb->treehdr.root.iRoot = 0; pDb->treehdr.root.nHeight = 0; - pDb->treehdr.root.nByte = 0; + pDb->treehdr.nByte = 0; } } void lsmTreeDiscardOld(lsm_db *pDb){ assert( lsmShmAssertLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_EXCL) @@ -1372,11 +1388,11 @@ while( iNode>=0 ){ TreeNode *pNode = pCsr->apTreeNode[iNode]; if( iCell<3 && pNode->aiKeyPtr[iCell] ){ int rc = LSM_OK; - TreeKey *pKey = treeShmptr(db, pNode->aiKeyPtr[iCell]); + TreeKey *pKey = treeShmptr(db, pNode->aiKeyPtr[iCell], &rc); assert( rc==LSM_OK ); return ((pKey->flags & LSM_END_DELETE) ? 1 : 0); } iNode--; iCell = pCsr->aiCell[iNode]; @@ -1395,11 +1411,11 @@ while( iNode>=0 ){ TreeNode *pNode = pCsr->apTreeNode[iNode]; int iCell = pCsr->aiCell[iNode]-1; if( iCell>=0 && pNode->aiKeyPtr[iCell] ){ int rc = LSM_OK; - TreeKey *pKey = treeShmptr(db, pNode->aiKeyPtr[iCell]); + TreeKey *pKey = treeShmptr(db, pNode->aiKeyPtr[iCell], &rc); assert( rc==LSM_OK ); return ((pKey->flags & LSM_START_DELETE) ? 1 : 0); } iNode--; } @@ -1462,11 +1478,11 @@ } if( res==0 && (flags & (LSM_END_DELETE|LSM_START_DELETE)) ){ if( pRes->flags & LSM_INSERT ){ nVal = pRes->nValue; - pVal = TKV_VAL(pRes); + pVal = TK_VAL(pRes); } flags = flags | pRes->flags; } if( flags & (LSM_INSERT|LSM_POINT_DELETE) ){ @@ -1636,11 +1652,11 @@ iDir = -1; }else{ iDir = +1; } iPeer = getChildPtr(pParent, WORKING_VERSION, iPSlot+iDir); - pPeer = (TreeNode *)treeShmptr(db, iPeer); + pPeer = (TreeNode *)treeShmptr(db, iPeer, &rc); assertIsWorkingChild(db, pNode, pParent, iPSlot); /* Allocate the first new leaf node. This is always required. */ if( bLeaf ){ pNew1 = (TreeNode *)newTreeLeaf(db, &iNew1, &rc); @@ -1826,13 +1842,13 @@ iKey = csr.apTreeNode[csr.iNode]->aiKeyPtr[csr.aiCell[csr.iNode]]; lsmTreeCursorPrev(&csr); treeOverwriteKey(db, &csr, iKey, &rc); - pKey = treeShmkey(db, iKey, TKV_LOADKEY, &blob, &rc); + pKey = treeShmkey(db, iKey, TK_LOADKEY, &blob, &rc); if( pKey ){ - rc = lsmTreeCursorSeek(&csr, TKV_KEY(pKey), pKey->nKey, &res); + rc = lsmTreeCursorSeek(&csr, TK_KEY(pKey), pKey->nKey, &res); } if( rc==LSM_OK ){ assert( res==0 && csr.iNode==iNode ); rc = lsmTreeCursorNext(&csr); if( rc==LSM_OK ){ @@ -1877,11 +1893,11 @@ /* ** Return, in bytes, the amount of memory currently used by the tree ** structure. */ int lsmTreeSize(lsm_db *pDb){ - return pDb->treehdr.root.nByte; + return pDb->treehdr.nByte; } /* ** Open a cursor on the in-memory tree pTree. */ @@ -1918,11 +1934,11 @@ int cmp = 0; int rc = LSM_OK; assert( pCsr->iNode>=0 ); p = csrGetKey(pCsr, &pCsr->blob, &rc); if( p ){ - cmp = pCsr->pDb->xCmp(TKV_KEY(p), p->nKey, pKey, nKey); + cmp = pCsr->pDb->xCmp(TK_KEY(p), p->nKey, pKey, nKey); } return cmp; } #endif @@ -1965,18 +1981,18 @@ while( iNodePtr ){ TreeNode *pNode; /* Node at location iNodePtr */ int iTest; /* Index of second key to test (0 or 2) */ TreeKey *pTreeKey; /* Key to compare against */ - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); + pNode = (TreeNode *)treeShmptr(pDb, iNodePtr, &rc); iNode++; pCsr->apTreeNode[iNode] = pNode; /* Compare (pKey/nKey) with the key in the middle slot of B-tree node ** pNode. The middle slot is never empty. If the comparison is a match, ** then the search is finished. Break out of the loop. */ - pTreeKey = treeShmkey(pDb, pNode->aiKeyPtr[1], TKV_LOADKEY, &b, &rc); + pTreeKey = treeShmkey(pDb, pNode->aiKeyPtr[1], TK_LOADKEY, &b, &rc); if( rc!=LSM_OK ) break; res = xCmp((void *)&pTreeKey[1], pTreeKey->nKey, pKey, nKey); if( res==0 ){ pCsr->aiCell[iNode] = 1; break; @@ -1984,11 +2000,11 @@ /* Based on the results of the previous comparison, compare (pKey/nKey) ** to either the left or right key of the B-tree node, if such a key ** exists. */ iTest = (res>0 ? 0 : 2); - pTreeKey = treeShmkey(pDb, pNode->aiKeyPtr[iTest], TKV_LOADKEY, &b, &rc); + pTreeKey = treeShmkey(pDb, pNode->aiKeyPtr[iTest], TK_LOADKEY, &b, &rc); if( rc ) break; if( pTreeKey==0 ){ iTest = 1; }else{ res = xCmp((void *)&pTreeKey[1], pTreeKey->nKey, pKey, nKey); @@ -2059,11 +2075,11 @@ if( pCsr->iNodeiTransId, iCell) ){ do { u32 iNodePtr; pCsr->iNode++; iNodePtr = getChildPtr(pNode, pRoot->iTransId, iCell); - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); + pNode = (TreeNode *)treeShmptr(pDb, iNodePtr, &rc); pCsr->apTreeNode[pCsr->iNode] = pNode; iCell = pCsr->aiCell[pCsr->iNode] = (pNode->aiKeyPtr[0]==0); }while( pCsr->iNode < iLeaf ); } @@ -2078,11 +2094,11 @@ } #ifndef NDEBUG if( pCsr->iNode>=0 ){ TreeKey *pK2 = csrGetKey(pCsr, &pCsr->blob, &rc); - assert( rc || pDb->xCmp(TKV_KEY(pK2),pK2->nKey,TKV_KEY(pK1),pK1->nKey)>=0 ); + assert( rc || pDb->xCmp(TK_KEY(pK2),pK2->nKey,TK_KEY(pK1),pK1->nKey)>=0 ); } tblobFree(pDb, &key1); #endif return rc; @@ -2124,11 +2140,11 @@ if( pCsr->iNodeiTransId, iCell) ){ do { u32 iNodePtr; pCsr->iNode++; iNodePtr = getChildPtr(pNode, pRoot->iTransId, iCell); - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); + pNode = (TreeNode *)treeShmptr(pDb, iNodePtr, &rc); if( rc!=LSM_OK ) break; pCsr->apTreeNode[pCsr->iNode] = pNode; iCell = 1 + (pNode->aiKeyPtr[2]!=0) + (pCsr->iNode < iLeaf); pCsr->aiCell[pCsr->iNode] = iCell; }while( pCsr->iNode < iLeaf ); @@ -2146,11 +2162,11 @@ } #ifndef NDEBUG if( pCsr->iNode>=0 ){ TreeKey *pK2 = csrGetKey(pCsr, &pCsr->blob, &rc); - assert( rc || pDb->xCmp(TKV_KEY(pK2), pK2->nKey, TKV_KEY(pK1), pK1->nKey)<0 ); + assert( rc || pDb->xCmp(TK_KEY(pK2), pK2->nKey, TK_KEY(pK1), pK1->nKey)<0 ); } tblobFree(pDb, &key1); #endif return rc; @@ -2174,11 +2190,11 @@ iNodePtr = pRoot->iRoot; while( iNodePtr ){ int iCell; TreeNode *pNode; - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); + pNode = (TreeNode *)treeShmptr(pDb, iNodePtr, &rc); if( rc ) break; if( bLast ){ iCell = ((pNode->aiKeyPtr[2]==0) ? 2 : 3); }else{ @@ -2201,11 +2217,11 @@ int lsmTreeCursorFlags(TreeCursor *pCsr){ int flags = 0; if( pCsr && pCsr->iNode>=0 ){ int rc = LSM_OK; TreeKey *pKey = (TreeKey *)treeShmptr(pCsr->pDb, - pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[pCsr->aiCell[pCsr->iNode]] + pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[pCsr->aiCell[pCsr->iNode]], &rc ); assert( rc==LSM_OK ); flags = pKey->flags; } return flags; @@ -2238,11 +2254,11 @@ if( res==0 ){ TreeKey *pTreeKey = csrGetKey(pCsr, &pCsr->blob, &rc); if( rc==LSM_OK ){ if( pTreeKey->flags & LSM_INSERT ){ *pnVal = pTreeKey->nValue; - *ppVal = TKV_VAL(pTreeKey); + *ppVal = TK_VAL(pTreeKey); }else{ *ppVal = 0; *pnVal = -1; } } @@ -2278,10 +2294,11 @@ /* ** Roll back to mark pMark. Structure *pMark should have been previously ** populated by a call to lsmTreeMark(). */ void lsmTreeRollback(lsm_db *pDb, TreeMark *pMark){ + int rcdummy = LSM_OK; int iIdx; int nIdx; u32 iNext; ShmChunk *pChunk; u32 iChunk; @@ -2289,12 +2306,12 @@ /* Revert all required v2 pointers. */ nIdx = intArraySize(&pDb->rollback); for(iIdx = pMark->iRollback; iIdxrollback, iIdx)); - assert( pNode ); + pNode = treeShmptr(pDb, intArrayEntry(&pDb->rollback, iIdx), &rcdummy); + assert( pNode && rcdummy==LSM_OK ); pNode->iV2 = 0; pNode->iV2Child = 0; pNode->iV2Ptr = 0; } intArrayTruncate(&pDb->rollback, pMark->iRollback); @@ -2371,14 +2388,16 @@ ** is true, the transaction is committed. Otherwise it is rolled back. */ int lsmTreeEndTransaction(lsm_db *pDb, int bCommit){ ShmHeader *pShm = pDb->pShmhdr; - treeHeaderChecksum(&pDb->treehdr, pDb->treehdr.aCksum); - memcpy(&pShm->hdr2, &pDb->treehdr, sizeof(TreeHeader)); - lsmShmBarrier(pDb); - memcpy(&pShm->hdr1, &pDb->treehdr, sizeof(TreeHeader)); + if( bCommit ){ + treeHeaderChecksum(&pDb->treehdr, pDb->treehdr.aCksum); + memcpy(&pShm->hdr2, &pDb->treehdr, sizeof(TreeHeader)); + lsmShmBarrier(pDb); + memcpy(&pShm->hdr1, &pDb->treehdr, sizeof(TreeHeader)); + } pShm->bWriter = 0; intArrayFree(pDb->pEnv, &pDb->rollback); return LSM_OK; } @@ -2423,5 +2442,7 @@ tblobFree(csr.pDb, &csr.blob); return nEntry; } #endif + + Index: src/lsm_unix.c ================================================================== --- src/lsm_unix.c +++ src/lsm_unix.c @@ -36,15 +36,10 @@ #include #include #include "lsmInt.h" -/* There is no fdatasync() call on Android */ -#ifdef __ANDROID__ -# define fdatasync(x) fsync(x) -#endif - /* ** An open file is an instance of the following object */ typedef struct PosixFile PosixFile; struct PosixFile { @@ -121,19 +116,15 @@ static int lsmPosixOsTruncate( lsm_file *pFile, /* File to write to */ lsm_i64 nSize /* Size to truncate file to */ ){ + int rc = LSM_OK; + int prc; /* Posix Return Code */ PosixFile *p = (PosixFile *)pFile; - int rc = LSM_OK; /* Return code */ - int prc; /* Posix Return Code */ - struct stat sStat; /* Result of fstat() invocation */ - - prc = fstat(p->fd, &sStat); - if( prc==0 && sStat.st_size>nSize ){ - prc = ftruncate(p->fd, (off_t)nSize); - } + + prc = ftruncate(p->fd, (off_t)nSize); if( prc<0 ) rc = lsm_ioerr(); return rc; } @@ -197,28 +188,26 @@ PosixFile *p = (PosixFile *)pFile; struct stat buf; if( p->pMap ){ munmap(p->pMap, p->nMap); - *ppOut = p->pMap = 0; - *pnOut = p->nMap = 0; - } - - if( iMin>=0 ){ - memset(&buf, 0, sizeof(buf)); - prc = fstat(p->fd, &buf); - if( prc!=0 ) return LSM_IOERR_BKPT; - iSz = buf.st_size; - if( iSzfd, iSz); - if( prc!=0 ) return LSM_IOERR_BKPT; - } - - p->pMap = mmap(0, iSz, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd, 0); - p->nMap = iSz; - } + p->pMap = 0; + p->nMap = 0; + } + + memset(&buf, 0, sizeof(buf)); + prc = fstat(p->fd, &buf); + if( prc!=0 ) return LSM_IOERR_BKPT; + iSz = buf.st_size; + if( iSzfd, iSz); + if( prc!=0 ) return LSM_IOERR_BKPT; + } + + p->pMap = mmap(0, iSz, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd, 0); + p->nMap = iSz; *ppOut = p->pMap; *pnOut = p->nMap; return LSM_OK; } @@ -414,21 +403,20 @@ lsm_free(p->pEnv, p); return LSM_OK; } static int lsmPosixOsSleep(lsm_env *pEnv, int us){ -#if 0 - /* Apparently on Android usleep() returns void */ - if( usleep(us) ) return LSM_IOERR; -#endif - usleep(us); + if( usleep(us) ){ + return LSM_IOERR; + } return LSM_OK; } /**************************************************************************** ** Memory allocation routines. */ +#define ROUND8(x) (((x)+7)&~7) #define BLOCK_HDR_SIZE ROUND8( sizeof(sqlite4_size_t) ) static void *lsmPosixOsMalloc(lsm_env *pEnv, int N){ unsigned char * m; N += BLOCK_HDR_SIZE; @@ -472,10 +460,11 @@ static sqlite4_size_t lsmPosixOsMSize(lsm_env *pEnv, void *p){ unsigned char * m = (unsigned char *)p; return *((sqlite4_size_t*)(m-BLOCK_HDR_SIZE)); } +#undef ROUND8 #undef BLOCK_HDR_SIZE #ifdef LSM_MUTEX_PTHREADS /************************************************************************* Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -27,11 +27,11 @@ #endif /* ** Dummy function used as a unique symbol for SQLITE4_DYNAMIC */ -void sqlite4_dynamic(void *pArg,void *p){ (void)pArg; (void)p; } +void sqlite4_dynamic(void *p){ (void)p; } #ifndef SQLITE4_AMALGAMATION /* IMPLEMENTATION-OF: R-46656-45156 The sqlite4_version[] string constant ** contains the text of SQLITE4_VERSION macro. */ @@ -366,26 +366,27 @@ pEnv->pLogArg = va_arg(ap, void*); break; } /* - ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_KVSTORE_PUSH, zName,xFactory); + ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_KVSTORE_PUSH, zName, xFactory); ** ** Push a new KVStore factory onto the factory stack. The new factory ** takes priority over prior factories. */ case SQLITE4_ENVCONFIG_KVSTORE_PUSH: { + typedef int(*PFactory)(sqlite4_env*, KVStore **, const char *, unsigned int); const char *zName = va_arg(ap, const char*); int nName = sqlite4Strlen30(zName); KVFactory *pMkr = sqlite4_malloc(pEnv, sizeof(*pMkr)+nName+1); char *z; if( pMkr==0 ) return SQLITE4_NOMEM; z = (char*)&pMkr[1]; memcpy(z, zName, nName+1); memset(pMkr, 0, sizeof(*pMkr)); pMkr->zName = z; - pMkr->xFactory = va_arg(ap, sqlite4_kvfactory); + pMkr->xFactory = va_arg(ap, PFactory); sqlite4_mutex_enter(pEnv->pFactoryMutex); pMkr->pNext = pEnv->pFactory; pEnv->pFactory = pMkr; sqlite4_mutex_leave(pEnv->pFactoryMutex); break; @@ -639,11 +640,11 @@ static int collNocaseCmp( void *NotUsed, int nKey1, const void *pKey1, int nKey2, const void *pKey2 ){ - int r = sqlite4_strnicmp( + int r = sqlite4StrNICmp( (const char *)pKey1, (const char *)pKey2, (nKey1=nIn ){ int i; u8 *aIn = (u8 *)pKey1; u8 *aOut = (u8 *)pKey2; for(i=0; inDb<=2 ); assert( db->aDb==db->aDbStatic ); { FuncDef *pNext, *pSame, *p; @@ -1054,34 +1052,10 @@ rc = sqlite4ApiExit(db, rc); sqlite4_mutex_leave(db->mutex); return rc; } -int sqlite4_create_mi_function( - sqlite4 *db, - const char *zFunc, - int nArg, - int enc, - void *p, - void (*xFunc)(sqlite4_context*,int,sqlite4_value **), - void (*xDestroy)(void *) -){ - int rc; - int n; - - n = nArg + (nArg>=0); - sqlite4_mutex_enter(db->mutex); - rc = sqlite4_create_function_v2(db, zFunc, n, enc, p, xFunc, 0,0,xDestroy); - if( rc==SQLITE4_OK ){ - FuncDef *p = sqlite4FindFunction(db, zFunc, -1, n, enc, 0); - p->bMatchinfo = 1; - } - rc = sqlite4ApiExit(db, rc); - sqlite4_mutex_leave(db->mutex); - return rc; -} - #ifndef SQLITE4_OMIT_UTF16 int sqlite4_create_function16( sqlite4 *db, const void *zFunctionName, int nArg, @@ -1265,11 +1239,11 @@ z = (void *)outOfMem; }else{ z = sqlite4_value_text16(db->pErr); if( z==0 ){ sqlite4ValueSetStr(db->pErr, -1, sqlite4ErrStr(db->errCode), - SQLITE4_UTF8, SQLITE4_STATIC, 0); + SQLITE4_UTF8, SQLITE4_STATIC); z = sqlite4_value_text16(db->pErr); } /* A malloc() may have failed within the call to sqlite4_value_text16() ** above. If this is the case, then the db->mallocFailed flag needs to ** be cleared before returning. Do this directly, instead of via @@ -1730,13 +1704,13 @@ db->flags |= SQLITE4_AutoIndex | SQLITE4_EnableTrigger | SQLITE4_ForeignKeys ; - sqlite4HashInit(pEnv, &db->aCollSeq, 0); + sqlite4HashInit(pEnv, &db->aCollSeq); #ifndef SQLITE4_OMIT_VIRTUALTABLE - sqlite4HashInit(pEnv, &db->aModule, 0); + sqlite4HashInit(pEnv, &db->aModule); #endif /* Add the default collation sequence BINARY. BINARY works for both UTF-8 ** and UTF-16, so add a version for each to avoid any unnecessary ** conversions. The only error that can occur here is a malloc() failure. @@ -1808,13 +1782,15 @@ if( rc!=SQLITE4_OK ){ goto opendb_out; } } +#ifdef SQLITE4_ENABLE_FTS3 if( !db->mallocFailed && rc==SQLITE4_OK ){ - rc = sqlite4InitFts5(db); + rc = sqlite4Fts3Init(db); } +#endif #ifdef SQLITE4_ENABLE_ICU if( !db->mallocFailed && rc==SQLITE4_OK ){ rc = sqlite4IcuInit(db); } @@ -2012,11 +1988,11 @@ sqlite4_mutex_enter(db->mutex); /* Find the named key-value store */ for(i=0; inDb; i++){ Db *pDb = &db->aDb[i]; - if( pDb->pKV && (0==zDbName || 0==sqlite4_stricmp(zDbName, pDb->zName)) ){ + if( pDb->pKV && (0==zDbName || 0==sqlite4StrICmp(zDbName, pDb->zName)) ){ pKV = pDb->pKV; break; } } Index: src/math.c ================================================================== --- src/math.c +++ src/math.c @@ -214,24 +214,10 @@ r.m = A.m/B.m; r.e = A.e - B.e; return r; } -/* -** Test if A is infinite. -*/ -int sqlite4_num_isinf(sqlite4_num A){ - return A.e>SQLITE4_MX_EXP && A.m!=0; -} - -/* -** Test if A is NaN. -*/ -int sqlite4_num_isnan(sqlite4_num A){ - return A.e>SQLITE4_MX_EXP && A.m==0; -} - /* ** Compare numbers A and B. Return: ** ** 1 if ASQLITE4_MX_EXP ){ if( B.m==0 ) return 0; return B.sign ? 3 : 1; } if( A.sign!=B.sign ){ - if ( A.m==0 && B.m==0 ) return 2; return A.sign ? 1 : 3; } adjustExponent(&A, &B); if( A.sign ){ sqlite4_num t = A; @@ -336,11 +321,11 @@ i = incr; }else{ i = 0; } if( nIn<=0 ) goto not_a_valid_number; - if( nIn>=incr*3 + if( nIn>=incr*2 && ((c=zIn[i])=='i' || c=='I') && ((c=zIn[i+incr])=='n' || c=='N') && ((c=zIn[i+incr*2])=='f' || c=='F') ){ r.e = SQLITE4_MX_EXP+1; @@ -348,11 +333,11 @@ return r; } while( i='0' && c<='9' ){ - if( c=='0' && nDigit==0 ){ + if( c==0 && nDigit==0 ){ if( seenRadix && r.e > -(SQLITE4_MX_EXP+1000) ) r.e--; continue; } nDigit++; if( nDigit<=18 ){ @@ -362,62 +347,45 @@ if( c!='0' ) r.approx = 1; if( !seenRadix ) r.e++; } }else if( c=='.' ){ seenRadix = 1; - }else if( c=='e' || c=='E' ){ - int exp = 0; - int expsign = 0; - int nEDigit = 0; - if( zIn[i]=='-' ){ - expsign = 1; - i += incr; - }else if( zIn[i]=='+' ){ - i += incr; - } - if( i>=nIn ) goto not_a_valid_number; - while( i'9' ) goto not_a_valid_number; - if( c=='0' && nEDigit==0 ) continue; - nEDigit++; - if( nEDigit>3 ) goto not_a_valid_number; - exp = exp*10 + c - '0'; - } - if( expsign ) exp = -exp; - r.e += exp; - break; - }else{ - goto not_a_valid_number; - } - } + }else{ + break; + } + } + if( c=='e' || c=='E' ){ + int exp = 0; + int expsign = 0; + int nEDigit = 0; + if( zIn[i]=='-' ){ + expsign = 1; + i += incr; + }else if( zIn[i]=='+' ){ + i += incr; + } + if( i>=nIn ) goto not_a_valid_number; + while( i'9' ) break; + if( c=='0' && nEDigit==0 ) continue; + nEDigit++; + if( nEDigit>3 ) goto not_a_valid_number; + exp = exp*10 + c - '0'; + } + if( expsign ) exp = -exp; + r.e += exp; + } + if( c!=0 ) goto not_a_valid_number; return r; not_a_valid_number: r.e = SQLITE4_MX_EXP+1; r.m = 0; return r; } -/* -** Convert an sqlite4_int64 to a number and return that number. -*/ -sqlite4_num sqlite4_num_from_int64(sqlite4_int64 n){ - sqlite4_num r; - r.approx = 0; - r.e = 0; - r.sign = n < 0; - if( n>=0 ){ - r.m = n; - }else if( n!=SMALLEST_INT64 ){ - r.m = -n; - }else{ - r.m = 1+(u64)LARGEST_INT64; - } - return r; -} - /* ** Convert an integer into text in the buffer supplied. The ** text is zero-terminated and right-justified in the buffer. ** A pointer to the first character of text is returned. ** @@ -499,14 +467,12 @@ removeTrailingZeros(zNum, &n); if( n>0 ){ zOut[0] = '.'; memcpy(zOut+1, zNum, n); nOut += n; - zOut[n+1] = 0; - }else{ - zOut[0] = 0; } + zOut[n+1] = 0; return nOut; } if( x.e<0 && x.e >= -n-5 ){ /* Values less than 1 and with no more than 5 subsequent zeros prior ** to the first significant digit. Ex: 0.0000012345 */ DELETED src/mem.c Index: src/mem.c ================================================================== --- src/mem.c +++ /dev/null @@ -1,434 +0,0 @@ -/* -** 2013-01-01 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file contains the implementation of the "sqlite4_mm" memory -** allocator object. -*/ -#include "sqlite4.h" -#include "mem.h" -#include - -/************************************************************************* -** The SQLITE4_MM_SYSTEM memory allocator. This allocator uses the -** malloc/realloc/free from the system library. It also tries to use -** the memory allocation sizer from the system library if such a routine -** exists. If there is no msize in the system library, then each allocation -** is increased in size by 8 bytes and the size of the allocation is stored -** in those initial 8 bytes. -** -** C-preprocessor macro summary: -** -** HAVE_MALLOC_USABLE_SIZE The configure script sets this symbol if -** the malloc_usable_size() interface exists -** on the target platform. Or, this symbol -** can be set manually, if desired. -** If an equivalent interface exists by -** a different name, using a separate -D -** option to rename it. This symbol will -** be enabled automatically on windows -** systems, and malloc_usable_size() will -** be redefined to _msize(), unless the -** SQLITE4_WITHOUT_MSIZE macro is defined. -** -** SQLITE4_WITHOUT_ZONEMALLOC Some older macs lack support for the zone -** memory allocator. Set this symbol to enable -** building on older macs. -** -** SQLITE4_WITHOUT_MSIZE Set this symbol to disable the use of -** _msize() on windows systems. This might -** be necessary when compiling for Delphi, -** for example. -*/ - -/* -** Windows systems have malloc_usable_size() but it is called _msize(). -** The use of _msize() is automatic, but can be disabled by compiling -** with -DSQLITE4_WITHOUT_MSIZE -*/ -#if !defined(HAVE_MALLOC_USABLE_SIZE) && SQLITE4_OS_WIN \ - && !defined(SQLITE4_WITHOUT_MSIZE) -# define HAVE_MALLOC_USABLE_SIZE 1 -# define SQLITE4_MALLOCSIZE _msize -#endif - -#if defined(__APPLE__) && !defined(SQLITE4_WITHOUT_ZONEMALLOC) - -/* -** Use the zone allocator available on apple products unless the -** SQLITE4_WITHOUT_ZONEMALLOC symbol is defined. -*/ -#include -#include -#include -static malloc_zone_t* _sqliteZone_; -#define SQLITE4_MALLOC(x) malloc_zone_malloc(_sqliteZone_, (x)) -#define SQLITE4_FREE(x) malloc_zone_free(_sqliteZone_, (x)); -#define SQLITE4_REALLOC(x,y) malloc_zone_realloc(_sqliteZone_, (x), (y)) -#define SQLITE4_MALLOCSIZE(x) \ - (_sqliteZone_ ? _sqliteZone_->size(_sqliteZone_,x) : malloc_size(x)) - -#else /* if not __APPLE__ */ - -/* -** Use standard C library malloc and free on non-Apple systems. -** Also used by Apple systems if SQLITE4_WITHOUT_ZONEMALLOC is defined. -*/ -#define SQLITE4_MALLOC(x) malloc(x) -#define SQLITE4_FREE(x) free(x) -#define SQLITE4_REALLOC(x,y) realloc((x),(y)) - -#ifdef HAVE_MALLOC_USABLE_SIZE -# ifndef SQLITE4_MALLOCSIZE -# include -# define SQLITE4_MALLOCSIZE(x) malloc_usable_size(x) -# endif -#else -# undef SQLITE4_MALLOCSIZE -#endif - -#endif /* __APPLE__ or not __APPLE__ */ - -/* -** Implementations of core routines -*/ -static void *mmSysMalloc(sqlite4_mm *pMM, sqlite4_int64 iSize){ -#ifdef SQLITE4_MALLOCSIZE - return SQLITE4_MALLOC(iSize); -#else - unsigned char *pRes = SQLITE4_MALLOC(iSize+8); - if( pRes ){ - *(sqlite4_int64*)pRes = iSize; - pRes += 8; - } - return pRes; -#endif -} -static void *mmSysRealloc(sqlite4_mm *pMM, void *pOld, sqlite4_int64 iSz){ -#ifdef SQLITE4_MALLOCSIZE - return SQLITE4_REALLOC(pOld, iSz); -#else - unsigned char *pRes; - if( pOld==0 ) return mmSysMalloc(pMM, iSz); - pRes = (unsigned char*)pOld; - pRes -= 8; - pRes = SQLITE4_REALLOC(pRes, iSz+8); - if( pRes ){ - *(sqlite4_int64*)pRes = iSz; - pRes += 8; - } - return pRes; -#endif -} -static void mmSysFree(sqlite4_mm *pNotUsed, void *pOld){ -#ifdef SQLITE4_MALLOCSIZE - SQLITE4_FREE(pOld); -#else - unsigned char *pRes; - if( pOld==0 ) return; - pRes = (unsigned char *)pOld; - pRes -= 8; - SQLITE4_FREE(pRes); -#endif -} -static sqlite4_int64 mmSysMsize(sqlite4_mm *pNotUsed, void *pOld){ -#ifdef SQLITE4_MALLOCSIZE - return SQLITE4_MALLOCSIZE(pOld); -#else - unsigned char *pX; - if( pOld==0 ) return 0; - pX = (unsigned char *)pOld; - pX -= 8; - return *(sqlite4_int64*)pX; -#endif -} - -static const sqlite4_mm_methods mmSysMethods = { - /* iVersion */ 1, - /* xMalloc */ mmSysMalloc, - /* xRealloc */ mmSysRealloc, - /* xFree */ mmSysFree, - /* xMsize */ mmSysMsize, - /* xMember */ 0, - /* xBenign */ 0, - /* xFinal */ 0 -}; -static sqlite4_mm mmSystem = { - /* pMethods */ &mmSysMethods -}; - -/************************************************************************* -** The SQLITE4_MM_OVERFLOW memory allocator. -** -** This memory allocator has two child memory allocators, A and B. Always -** try to fulfill the request using A first, then overflow to B if the request -** on A fails. The A allocator must support the xMember method. -*/ -struct mmOvfl { - sqlite4_mm base; /* Base class - must be first */ - int (*xMemberOfA)(sqlite4_mm*, const void*); - sqlite4_mm *pA; /* Primary memory allocator */ - sqlite4_mm *pB; /* Backup memory allocator in case pA fails */ -}; - -static void *mmOvflMalloc(sqlite4_mm *pMM, sqlite4_int64 iSz){ - const struct mmOvfl *pOvfl = (const struct mmOvfl*)pMM; - void *pRes; - pRes = pOvfl->pA->pMethods->xMalloc(pOvfl->pA, iSz); - if( pRes==0 ){ - pRes = pOvfl->pB->pMethods->xMalloc(pOvfl->pB, iSz); - } - return pRes; -} -static void *mmOvflRealloc(sqlite4_mm *pMM, void *pOld, sqlite4_int64 iSz){ - const struct mmOvfl *pOvfl; - void *pRes; - if( pOld==0 ) return mmOvflMalloc(pMM, iSz); - pOvfl = (const struct mmOvfl*)pMM; - if( pOvfl->xMemberOfA(pOvfl->pA, pOld) ){ - pRes = pOvfl->pA->pMethods->xRealloc(pOvfl->pA, pOld, iSz); - }else{ - pRes = pOvfl->pB->pMethods->xRealloc(pOvfl->pB, pOld, iSz); - } - return pRes; -} -static void mmOvflFree(sqlite4_mm *pMM, void *pOld){ - const struct mmOvfl *pOvfl; - if( pOld==0 ) return; - pOvfl = (const struct mmOvfl*)pMM; - if( pOvfl->xMemberOfA(pOvfl->pA, pOld) ){ - pOvfl->pA->pMethods->xFree(pOvfl->pA, pOld); - }else{ - pOvfl->pB->pMethods->xFree(pOvfl->pB, pOld); - } -} -static sqlite4_int64 mmOvflMsize(sqlite4_mm *pMM, void *pOld){ - const struct mmOvfl *pOvfl; - sqlite4_int64 iSz; - if( pOld==0 ) return 0; - pOvfl = (const struct mmOvfl*)pMM; - if( pOvfl->xMemberOfA(pOvfl->pA, pOld) ){ - iSz = sqlite4_mm_msize(pOvfl->pA, pOld); - }else{ - iSz = sqlite4_mm_msize(pOvfl->pB, pOld); - } - return iSz; -} -static int mmOvflMember(sqlite4_mm *pMM, const void *pOld){ - const struct mmOvfl *pOvfl; - int iRes; - if( pOld==0 ) return 0; - pOvfl = (const struct mmOvfl*)pMM; - if( pOvfl->xMemberOfA(pOvfl->pA, pOld) ){ - iRes = 1; - }else{ - iRes = sqlite4_mm_member(pOvfl->pB, pOld); - } - return iRes; -} -static void mmOvflBenign(sqlite4_mm *pMM, int bEnable){ - struct mmOvfl *pOvfl = (struct mmOvfl*)pMM; - sqlite4_mm_benign_failures(pOvfl->pA, bEnable); - sqlite4_mm_benign_failures(pOvfl->pB, bEnable); -} -static void mmOvflFinal(sqlite4_mm *pMM){ - struct mmOvfl *pOvfl = (struct mmOvfl*)pMM; - sqlite4_mm *pA = pOvfl->pA; - sqlite4_mm *pB = pOvfl->pB; - mmOvflFree(pMM, pMM); - sqlite4_mm_destroy(pA); - sqlite4_mm_destroy(pB); -} -static const sqlite4_mm_methods mmOvflMethods = { - /* iVersion */ 1, - /* xMalloc */ mmOvflMalloc, - /* xRealloc */ mmOvflRealloc, - /* xFree */ mmOvflFree, - /* xMsize */ mmOvflMsize, - /* xMember */ mmOvflMember, - /* xBenign */ mmOvflBenign, - /* xFinal */ mmOvflFinal -}; -static sqlite4_mm *mmOvflNew(sqlite4_mm *pA, sqlite4_mm *pB){ - struct mmOvfl *pOvfl; - if( pA->pMethods->xMember==0 ) return 0; - pOvfl = sqlite4_mm_malloc(pA, sizeof(*pOvfl)); - if( pOvfl==0 ){ - pOvfl = sqlite4_mm_malloc(pB, sizeof(*pOvfl)); - } - if( pOvfl ){ - pOvfl->base.pMethods = &mmOvflMethods; - pOvfl->xMemberOfA = pA->pMethods->xMember; - pOvfl->pA = pA; - pOvfl->pB = pB; - } - return &pOvfl->base; -} - - -/************************************************************************* -** The SQLITE4_MM_ONESIZE memory allocator. -** -** All memory allocations are rounded up to a single size, "sz". A request -** for an allocation larger than sz bytes fails. All allocations come out -** of a single initial buffer with "cnt" chunks of "sz" bytes each. -** -** Space to hold the sqlite4_mm object comes from the first block in the -** allocation space. -*/ -struct mmOnesz { - sqlite4_mm base; /* Base class. Must be first. */ - const void *pSpace; /* Space to allocate */ - const void *pLast; /* Last possible allocation */ - struct mmOneszBlock *pFree; /* List of free blocks */ - int sz; /* Size of each allocation */ -}; - -/* A free block in the buffer */ -struct mmOneszBlock { - struct mmOneszBlock *pNext; /* Next on the freelist */ -}; - -static void *mmOneszMalloc(sqlite4_mm *pMM, sqlite4_int64 iSz){ - struct mmOnesz *pOnesz = (struct mmOnesz*)pMM; - void *pRes; - if( iSz>pOnesz->sz ) return 0; - if( pOnesz->pFree==0 ) return 0; - pRes = pOnesz->pFree; - pOnesz->pFree = pOnesz->pFree->pNext; - return pRes; -} -static void mmOneszFree(sqlite4_mm *pMM, void *pOld){ - struct mmOnesz *pOnesz = (struct mmOnesz*)pMM; - if( pOld ){ - struct mmOneszBlock *pBlock = (struct mmOneszBlock*)pOld; - pBlock->pNext = pOnesz->pFree; - pOnesz->pFree = pBlock; - } -} -static void *mmOneszRealloc(sqlite4_mm *pMM, void *pOld, sqlite4_int64 iSz){ - struct mmOnesz *pOnesz = (struct mmOnesz*)pMM; - if( pOld==0 ) return mmOneszMalloc(pMM, iSz); - if( iSz<=0 ){ - mmOneszFree(pMM, pOld); - return 0; - } - if( iSz>pOnesz->sz ) return 0; - return pOld; -} -static sqlite4_int64 mmOneszMsize(sqlite4_mm *pMM, void *pOld){ - struct mmOnesz *pOnesz = (struct mmOnesz*)pMM; - return pOld ? pOnesz->sz : 0; -} -static int mmOneszMember(sqlite4_mm *pMM, const void *pOld){ - struct mmOnesz *pOnesz = (struct mmOnesz*)pMM; - return pOld && pOld>=pOnesz->pSpace && pOld<=pOnesz->pLast; -} -static const sqlite4_mm_methods mmOneszMethods = { - /* iVersion */ 1, - /* xMalloc */ mmOneszMalloc, - /* xRealloc */ mmOneszRealloc, - /* xFree */ mmOneszFree, - /* xMsize */ mmOneszMsize, - /* xMember */ mmOneszMember, - /* xBenign */ 0, - /* xFinal */ 0 -}; -static sqlite4_mm *mmOneszNew(void *pSpace, int sz, int cnt){ - struct mmOnesz *pOnesz; - unsigned char *pMem; - if( szbase.pMethods = &mmOneszMethods; - pOnesz->pSpace = (const void*)pMem; - pOnesz->sz = sz; - pOnesz->pLast = (const void*)(pMem + sz*(cnt-2)); - pOnesz->pFree = 0; - while( cnt ){ - struct mmOneszBlock *pBlock = (struct mmOneszBlock*)pMem; - pBlock->pNext = pOnesz->pFree; - pOnesz->pFree = pBlock; - cnt--; - pMem += sz; - } - return &pOnesz->base; -} - -/************************************************************************* -** Main interfaces. -*/ -void *sqlite4_mm_malloc(sqlite4_mm *pMM, sqlite4_int64 iSize){ - if( pMM==0 ) pMM = &mmSystem; - return pMM->pMethods->xMalloc(pMM,iSize); -} -void *sqlite4_mm_realloc(sqlite4_mm *pMM, void *pOld, sqlite4_int64 iSize){ - if( pMM==0 ) pMM = &mmSystem; - return pMM->pMethods->xRealloc(pMM,pOld,iSize); -} -void sqlite4_mm_free(sqlite4_mm *pMM, void *pOld){ - if( pMM==0 ) pMM = &mmSystem; - pMM->pMethods->xFree(pMM,pOld); -} -sqlite4_int64 sqlite4_mm_msize(sqlite4_mm *pMM, void *pOld){ - if( pMM==0 ) pMM = &mmSystem; - return pMM->pMethods->xMsize(pMM,pOld); -} -int sqlite4_mm_member(sqlite4_mm *pMM, const void *pOld){ - return (pMM && pMM->pMethods->xMember!=0) ? - pMM->pMethods->xMember(pMM,pOld) : -1; -} -void sqlite4_mm_benign_failure(sqlite4_mm *pMM, int bEnable){ - if( pMM && pMM->pMethods->xBenign ){ - pMM->pMethods->xBenign(pMM, bEnable); - } -} -void sqlite4_mm_destroy(sqlite4_mm *pMM){ - if( pMM && pMM->pMethods->xFinal ) pMM->pMethods->xFinal(pMM); -} - -/* -** Create a new memory allocation object. eType determines the type of -** memory allocator and the arguments. -*/ -sqlite4_mm *sqlite4_mm_new(sqlite4_mm_type eType, ...){ - va_list ap; - sqlite4_mm *pMM; - - va_start(ap, eType); - switch( eType ){ - case SQLITE4_MM_SYSTEM: { - pMM = &mmSystem; - break; - } - case SQLITE4_MM_OVERFLOW: { - sqlite4_mm *pA = va_arg(ap, sqlite4_mm*); - sqlite4_mm *pB = va_arg(ap, sqlite4_mm*); - pMM = mmOvflNew(pA, pB); - break; - } - case SQLITE4_MM_ONESIZE: { - void *pSpace = va_arg(ap, void*); - int sz = va_arg(ap, int); - int cnt = va_arg(ap, int); - pMM = mmOneszNew(pSpace, sz, cnt); - break; - } - default: { - pMM = 0; - break; - } - } - va_end(ap); - return pMM; -} DELETED src/mem.h Index: src/mem.h ================================================================== --- src/mem.h +++ /dev/null @@ -1,109 +0,0 @@ -/* -** 2013-01-01 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This header file defines the interface that the SQLite4 memory -** management logic. -** -** Some of this will eventually fold into sqliteInt.h. -*/ - - -/* -** object declarations -*/ -typedef struct sqlite4_mm sqlite4_mm; -typedef struct sqlite4_mm_methods sqlite4_mm_methods; - -/* -** Base class. Each implementation extends this with additional -** fields specific to its own needs. This needs to be public so that -** applications can supply their on customized memory allocators. -*/ -struct sqlite4_mm { - const sqlite4_mm_methods *pMethods; -}; - -/* -** An instance of the following object defines a BESPOKE memory alloator -*/ -struct sqlite4_mm_methods { - int iVersion; - void *(*xMalloc)(sqlite4_mm*, sqlite4_int64); - void *(*xRealloc)(sqlite4_mm*, void*, sqlite4_int64); - void (*xFree)(sqlite4_mm*, void*); - sqlite4_int64 (*xMsize)(sqlite4_mm*, void*); - int (*xMember)(sqlite4_mm*, const void*); - void (*xBenign)(sqlite4_mm*, int); - void (*xFinal)(sqlite4_mm*); -}; - -/* -** Available memory management types: -*/ -typedef enum { - SQLITE4_MM_SYSTEM, /* Use the system malloc() */ - SQLITE4_MM_ONESIZE, /* All allocations map to a fixed size */ - SQLITE4_MM_OVERFLOW, /* Two allocators. Use A first; failover to B */ - SQLITE4_MM_COMPACT, /* Like memsys3 from SQLite3 */ - SQLITE4_MM_ROBSON, /* Like memsys5 from SQLite3 */ - SQLITE4_MM_LINEAR, /* Allocate from a fixed buffer w/o free */ - SQLITE4_MM_DEBUG, /* Debugging memory allocator */ - SQLITE4_MM_STATS, /* Keep memory statistics */ - SQLITE4_MM_BESPOKE /* Caller-defined implementation */ -} sqlite4_mm_type; - -/* -** Allocate a new memory manager. Return NULL if unable. -*/ -sqlite4_mm *sqlite4_mm_new(sqlite4_mm_type, ...); - -/* -** Free the sqlite4_mm object. -** -** All outstanding memory for the allocator must be freed prior to -** invoking this interface, or else the behavior is undefined. -*/ -void sqlite4_mm_destroy(sqlite4_mm*); - -/* -** Core memory allocation routines: -*/ -void *sqlite4_mm_malloc(sqlite4_mm*, sqlite4_int64); -void *sqlite4_mm_realloc(sqlite4_mm*, void*, sqlite4_int64); -void sqlite4_mm_free(sqlite4_mm*, void*); - -/* -** Return the size of a memory allocation. -** -** All memory allocators in SQLite4 must be able to report their size. -** When using system malloc() on system that lack the malloc_usable_size() -** routine or its equivalent, then the sqlite4_mm object allocates 8 extra -** bytes for each memory allocation and stores the allocation size in those -** initial 8 bytes. -*/ -sqlite4_int64 sqlite4_mm_msize(sqlite4_mm*, void*); - -/* -** Check to see if pOld is a memory allocation from pMM. If it is, return -** 1. If not, return 0. If we cannot determine an answer, return -1. -** -** If pOld is not a valid memory allocation or is a memory allocation that -** has previously been freed, then the result of this routine is undefined. -*/ -int sqlite4_mm_member(sqlite4_mm *pMM, const void *pOld); - -/* -** Enable or disable benign failure mode. Benign failure mode can be -** nested. In benign failure mode, OOM errors do not necessarily propagate -** back out to the application but can be dealt with internally. Memory -** allocations that occur in benign failure mode are considered "optional". -*/ -void sqlite4_mm_benign_failures(sqlite4_mm*, int bEnable); Index: src/mutex_noop.c ================================================================== --- src/mutex_noop.c +++ src/mutex_noop.c @@ -35,12 +35,12 @@ ** ** This routines provide no mutual exclusion or error checking. */ static int noopMutexInit(void *p){ UNUSED_PARAMETER(p); return SQLITE4_OK; } static int noopMutexEnd(void *p){ UNUSED_PARAMETER(p); return SQLITE4_OK; } -static sqlite4_mutex *noopMutexAlloc(void *p, int id){ - UNUSED_PARAMETER(p); +static sqlite4_mutex *noopMutexAlloc(sqlite4_env *pEnv, int id){ + UNUSED_PARAMETER(pEnv); UNUSED_PARAMETER(id); return (sqlite4_mutex*)8; } static void noopMutexFree(sqlite4_mutex *p){ UNUSED_PARAMETER(p); return; } static void noopMutexEnter(sqlite4_mutex *p){ UNUSED_PARAMETER(p); return; } Index: src/parse.y ================================================================== --- src/parse.y +++ src/parse.y @@ -92,18 +92,10 @@ /* ** An instance of this structure holds the ATTACH key and the key type. */ struct AttachKey { int type; Token key; }; -/* -** One or more VALUES claues -*/ -struct ValueList { - ExprList *pList; - Select *pSelect; -}; - } // end %include // Input is a single SQL command input ::= cmdlist. cmdlist ::= cmdlist ecmd. @@ -685,56 +677,30 @@ sqlite4ExprListSetName(pParse, A, &X, 1); } ////////////////////////// The INSERT command ///////////////////////////////// // -cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) valuelist(Y). - {sqlite4Insert(pParse, X, Y.pList, Y.pSelect, F, R);} +cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) + VALUES LP itemlist(Y) RP. + {sqlite4Insert(pParse, X, Y, 0, F, R);} cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). {sqlite4Insert(pParse, X, 0, S, F, R);} cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES. {sqlite4Insert(pParse, X, 0, 0, F, R);} %type insert_cmd {u8} insert_cmd(A) ::= INSERT orconf(R). {A = R;} insert_cmd(A) ::= REPLACE. {A = OE_Replace;} -// A ValueList is either a single VALUES clause or a comma-separated list -// of VALUES clauses. If it is a single VALUES clause then the -// ValueList.pList field points to the expression list of that clause. -// If it is a list of VALUES clauses, then those clauses are transformed -// into a set of SELECT statements without FROM clauses and connected by -// UNION ALL and the ValueList.pSelect points to the right-most SELECT in -// that compound. -%type valuelist {struct ValueList} -%destructor valuelist { - sqlite4ExprListDelete(pParse->db, $$.pList); - sqlite4SelectDelete(pParse->db, $$.pSelect); -} -valuelist(A) ::= VALUES LP nexprlist(X) RP. { - A.pList = X; - A.pSelect = 0; -} -valuelist(A) ::= valuelist(X) COMMA LP exprlist(Y) RP. { - Select *pRight = sqlite4SelectNew(pParse, Y, 0, 0, 0, 0, 0, 0, 0, 0); - if( X.pList ){ - X.pSelect = sqlite4SelectNew(pParse, X.pList, 0, 0, 0, 0, 0, 0, 0, 0); - X.pList = 0; - } - A.pList = 0; - if( X.pSelect==0 || pRight==0 ){ - sqlite4SelectDelete(pParse->db, pRight); - sqlite4SelectDelete(pParse->db, X.pSelect); - A.pSelect = 0; - }else{ - pRight->op = TK_ALL; - pRight->pPrior = X.pSelect; - pRight->selFlags |= SF_Values; - pRight->pPrior->selFlags |= SF_Values; - A.pSelect = pRight; - } -} + +%type itemlist {ExprList*} +%destructor itemlist {sqlite4ExprListDelete(pParse->db, $$);} + +itemlist(A) ::= itemlist(X) COMMA expr(Y). + {A = sqlite4ExprListAppend(pParse,X,Y.pExpr);} +itemlist(A) ::= expr(X). + {A = sqlite4ExprListAppend(pParse,0,X.pExpr);} %type inscollist_opt {IdList*} %destructor inscollist_opt {sqlite4IdListDelete(pParse->db, $$);} %type inscollist {IdList*} %destructor inscollist {sqlite4IdListDelete(pParse->db, $$);} @@ -879,11 +845,11 @@ {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} expr(A) ::= expr(X) CONCAT(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} %type likeop {struct LikeOp} likeop(A) ::= LIKE_KW(X). {A.eOperator = X; A.not = 0;} likeop(A) ::= NOT LIKE_KW(X). {A.eOperator = X; A.not = 1;} -/* likeop(A) ::= MATCH(X). {A.eOperator = X; A.not = 0;} */ +likeop(A) ::= MATCH(X). {A.eOperator = X; A.not = 0;} likeop(A) ::= NOT MATCH(X). {A.eOperator = X; A.not = 1;} expr(A) ::= expr(X) likeop(OP) expr(Y). [LIKE_KW] { ExprList *pList; pList = sqlite4ExprListAppend(pParse,0, Y.pExpr); pList = sqlite4ExprListAppend(pParse,pList, X.pExpr); @@ -903,14 +869,10 @@ A.zStart = X.zStart; A.zEnd = E.zEnd; if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc; } -expr(A) ::= expr(L) MATCH expr(R). { - spanBinaryExpr(&A, pParse, TK_MATCH, &L, &R); -} - %include { /* Construct an expression node for a unary postfix operator */ static void spanUnaryPostfix( ExprSpan *pOut, /* Write the new expression node here */ @@ -1123,26 +1085,15 @@ {A = sqlite4ExprListAppend(pParse,0,Y.pExpr);} ///////////////////////////// The CREATE INDEX command /////////////////////// // -%type createindex {CreateIndex} -%destructor createindex {sqlite4SrcListDelete(pParse->db, $$.pTblName);} - -createindex(C) ::= createkw(S) uniqueflag(U) INDEX ifnotexists(NE) - nm(X) dbnm(D) ON nm(Y). { - C.bUnique = U; - C.bIfnotexist = NE; - C.tCreate = S; - C.tName1 = X; - C.tName2 = D; - C.pTblName = sqlite4SrcListAppend(pParse->db, 0, &Y, 0); -} - -cmd ::= createindex(C) LP idxlist(Z) RP(E). { - sqlite4CreateIndex(pParse, &C.tName1, &C.tName2, C.pTblName, Z, - C.bUnique, &C.tCreate, &E, SQLITE4_SO_ASC, C.bIfnotexist, 0); +cmd ::= createkw(S) uniqueflag(U) INDEX ifnotexists(NE) nm(X) dbnm(D) + ON nm(Y) LP idxlist(Z) RP(E). { + sqlite4CreateIndex(pParse, &X, &D, + sqlite4SrcListAppend(pParse->db,0,&Y,0), Z, U, + &S, &E, SQLITE4_SO_ASC, NE, 0); } %type uniqueflag {int} uniqueflag(A) ::= UNIQUE. {A = OE_Abort;} uniqueflag(A) ::= . {A = OE_None;} @@ -1179,37 +1130,10 @@ %type collate {Token} collate(C) ::= . {C.z = 0; C.n = 0;} collate(C) ::= COLLATE ids(X). {C = X;} -%type uidxlist_opt {ExprList*} -%destructor uidxlist_opt {sqlite4ExprListDelete(pParse->db, $$);} -%type uidxlist {ExprList*} -%destructor uidxlist {sqlite4ExprListDelete(pParse->db, $$);} - -cmd ::= createindex(C) USING nm(F) LP uidxlist_opt(U) RP(E). { - sqlite4CreateUsingIndex(pParse, &C, U, &F, &E); -} - -uidxlist_opt(A) ::= uidxlist(B). { A = B; } -uidxlist_opt(A) ::= . { A = 0; } - -uidxlist(A) ::= uidxlist(Z) COMMA ids(X) EQ ids(Y). { - A = sqlite4ExprListAppend(pParse, Z, sqlite4PExpr(pParse, @Y, 0, 0, &Y)); - sqlite4ExprListSetName(pParse, A, &X, 1); -} -uidxlist(A) ::= uidxlist(Z) COMMA ids(Y). { - A = sqlite4ExprListAppend(pParse, Z, sqlite4PExpr(pParse, @Y, 0, 0, &Y)); -} -uidxlist(A) ::= ids(X) EQ ids(Y). { - A = sqlite4ExprListAppend(pParse, 0, sqlite4PExpr(pParse, @Y, 0, 0, &Y)); - sqlite4ExprListSetName(pParse, A, &X, 1); -} -uidxlist(A) ::= ids(Y). { - A = sqlite4ExprListAppend(pParse, 0, sqlite4PExpr(pParse, @Y, 0, 0, &Y)); -} - ///////////////////////////// The DROP INDEX command ///////////////////////// // cmd ::= DROP INDEX ifexists(E) fullname(X). {sqlite4DropIndex(pParse, X, E);} @@ -1326,12 +1250,12 @@ UPDATE orconf(R) trnm(X) tridxby SET setlist(Y) where_opt(Z). { A = sqlite4TriggerUpdateStep(pParse->db, &X, Y, Z, R); } // INSERT trigger_cmd(A) ::= - insert_cmd(R) INTO trnm(X) inscollist_opt(F) valuelist(Y). - {A = sqlite4TriggerInsertStep(pParse->db, &X, F, Y.pList, Y.pSelect, R);} + insert_cmd(R) INTO trnm(X) inscollist_opt(F) VALUES LP itemlist(Y) RP. + {A = sqlite4TriggerInsertStep(pParse->db, &X, F, Y, 0, R);} trigger_cmd(A) ::= insert_cmd(R) INTO trnm(X) inscollist_opt(F) select(S). {A = sqlite4TriggerInsertStep(pParse->db, &X, F, 0, S, R);} // DELETE Index: src/pragma.c ================================================================== --- src/pragma.c +++ src/pragma.c @@ -26,11 +26,11 @@ if( sqlite4Isdigit(*z) ){ return (u8)sqlite4Atoi(z); } n = sqlite4Strlen30(z); for(i=0; izName)==0 ){ + if( sqlite4StrICmp(zLeft, p->zName)==0 ){ sqlite4 *db = pParse->db; Vdbe *v; v = sqlite4GetVdbe(pParse); assert( v!=0 ); /* Already allocated by sqlite4Pragma() */ if( ALWAYS(v) ){ @@ -222,87 +222,24 @@ /* The flagPragma() subroutine also generates any necessary code ** there is nothing more to do here */ }else #endif /* SQLITE4_OMIT_FLAG_PRAGMAS */ - if( sqlite4_stricmp(zLeft, "lsm_flush")==0 ){ + if( sqlite4StrICmp(zLeft, "lsm_flush")==0 ){ sqlite4_kvstore_control(db, zDb, SQLITE4_KVCTRL_LSM_FLUSH, 0); }else - if( sqlite4_stricmp(zLeft, "lsm_checkpoint")==0 ){ + if( sqlite4StrICmp(zLeft, "lsm_checkpoint")==0 ){ sqlite4_kvstore_control(db, zDb, SQLITE4_KVCTRL_LSM_CHECKPOINT, 0); }else - if( sqlite4_stricmp(zLeft, "lsm_merge")==0 ){ + if( sqlite4StrICmp(zLeft, "lsm_merge")==0 ){ int nPage = zRight ? sqlite4Atoi(zRight) : 1000; sqlite4_kvstore_control(db, zDb, SQLITE4_KVCTRL_LSM_MERGE, &nPage); returnSingleInt(pParse, "nWrite", (sqlite4_int64)nPage); }else - /* - ** PRAGMA fts_check() - */ - if( sqlite4_stricmp(zLeft, "fts_check")==0 && zRight ){ - int iCksum1; - int iCksum2; - Index *pIdx; - Table *pTab; - Vdbe *v = sqlite4GetVdbe(pParse); - if( v==0 || sqlite4ReadSchema(pParse) ) goto pragma_out; - - iCksum1 = ++pParse->nMem; - iCksum2 = ++pParse->nMem; - sqlite4VdbeAddOp2(v, OP_Integer, 0, iCksum1); - sqlite4VdbeAddOp2(v, OP_Integer, 0, iCksum2); - - pIdx = sqlite4FindIndex(db, zRight, zDb); - if( pIdx && pIdx->eIndexType==SQLITE4_INDEX_FTS5 ){ - int iTab = pParse->nTab++; - int iAddr; - int iReg; - int i; - - pTab = pIdx->pTable; - sqlite4OpenPrimaryKey(pParse, iTab, iDb, pTab, OP_OpenRead); - iAddr = sqlite4VdbeAddOp2(v, OP_Rewind, iTab, 0); - - iReg = pParse->nMem+1; - pParse->nMem += (1 + pTab->nCol); - - sqlite4VdbeAddOp2(v, OP_RowKey, iTab, iReg); - for(i=0; inCol; i++){ - sqlite4VdbeAddOp3(v, OP_Column, iTab, i, iReg+1+i); - } - sqlite4Fts5CodeCksum(pParse, pIdx, iCksum1, iReg, 0); - - sqlite4VdbeAddOp2(v, OP_Next, iTab, iAddr+1); - sqlite4VdbeJumpHere(v, iAddr); - sqlite4VdbeAddOp1(v, OP_Close, iTab); - - sqlite4VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); - iAddr = sqlite4VdbeAddOp2(v, OP_Rewind, iTab, 0); - - iReg = pParse->nMem+1; - pParse->nMem += 2; - sqlite4VdbeAddOp2(v, OP_RowKey, iTab, iReg); - sqlite4VdbeAddOp2(v, OP_RowData, iTab, iReg+1); - sqlite4Fts5CodeCksum(pParse, pIdx, iCksum2, iReg, 1); - - sqlite4VdbeAddOp2(v, OP_Next, iTab, iAddr+1); - sqlite4VdbeJumpHere(v, iAddr); - sqlite4VdbeAddOp1(v, OP_Close, iTab); - - iReg = ++pParse->nMem; - sqlite4VdbeAddOp4(v, OP_String8, 0, iReg, 0, "ok", 0); - iAddr = sqlite4VdbeAddOp3(v, OP_Eq, iCksum1, 0, iCksum2); - sqlite4VdbeAddOp4(v, OP_String8, 0, iReg, 0, "error - cksum mismatch", 0); - sqlite4VdbeJumpHere(v, iAddr); - sqlite4VdbeAddOp2(v, OP_ResultRow, iReg, 1); - - sqlite4VdbeSetNumCols(v, 1); - } - } #ifndef SQLITE4_OMIT_SCHEMA_PRAGMAS /* @@ -315,11 +252,11 @@ ** name: Column name ** type: Column declaration type. ** notnull: True if 'NOT NULL' is part of column declaration ** dflt_value: The default value for the column, if any. */ - if( sqlite4_stricmp(zLeft, "table_info")==0 && zRight ){ + if( sqlite4StrICmp(zLeft, "table_info")==0 && zRight ){ Table *pTab; if( sqlite4ReadSchema(pParse) ) goto pragma_out; pTab = sqlite4FindTable(db, zRight, zDb); if( pTab ){ int i; @@ -353,11 +290,11 @@ sqlite4VdbeAddOp2(v, OP_ResultRow, 1, 6); } } }else - if( sqlite4_stricmp(zLeft, "index_info")==0 && zRight ){ + if( sqlite4StrICmp(zLeft, "index_info")==0 && zRight ){ Index *pIdx; Table *pTab; if( sqlite4ReadSchema(pParse) ) goto pragma_out; pIdx = sqlite4FindIndex(db, zRight, zDb); if( pIdx ){ @@ -377,11 +314,11 @@ sqlite4VdbeAddOp2(v, OP_ResultRow, 1, 3); } } }else - if( sqlite4_stricmp(zLeft, "index_list")==0 && zRight ){ + if( sqlite4StrICmp(zLeft, "index_list")==0 && zRight ){ Index *pIdx; Table *pTab; if( sqlite4ReadSchema(pParse) ) goto pragma_out; pTab = sqlite4FindTable(db, zRight, zDb); if( pTab ){ @@ -404,11 +341,11 @@ } } } }else - if( sqlite4_stricmp(zLeft, "database_list")==0 ){ + if( sqlite4StrICmp(zLeft, "database_list")==0 ){ int i; if( sqlite4ReadSchema(pParse) ) goto pragma_out; sqlite4VdbeSetNumCols(v, 3); pParse->nMem = 3; sqlite4VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE4_STATIC); @@ -423,11 +360,11 @@ "filename", 0); sqlite4VdbeAddOp2(v, OP_ResultRow, 1, 3); } }else - if( sqlite4_stricmp(zLeft, "collation_list")==0 ){ + if( sqlite4StrICmp(zLeft, "collation_list")==0 ){ int i = 0; HashElem *p; sqlite4VdbeSetNumCols(v, 2); pParse->nMem = 2; sqlite4VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE4_STATIC); @@ -440,11 +377,11 @@ } }else #endif /* SQLITE4_OMIT_SCHEMA_PRAGMAS */ #ifndef SQLITE4_OMIT_FOREIGN_KEY - if( sqlite4_stricmp(zLeft, "foreign_key_list")==0 && zRight ){ + if( sqlite4StrICmp(zLeft, "foreign_key_list")==0 && zRight ){ FKey *pFK; Table *pTab; if( sqlite4ReadSchema(pParse) ) goto pragma_out; pTab = sqlite4FindTable(db, zRight, zDb); if( pTab ){ @@ -486,11 +423,11 @@ } }else #endif /* !defined(SQLITE4_OMIT_FOREIGN_KEY) */ #ifndef NDEBUG - if( sqlite4_stricmp(zLeft, "parser_trace")==0 ){ + if( sqlite4StrICmp(zLeft, "parser_trace")==0 ){ if( zRight ){ if( sqlite4GetBoolean(zRight) ){ sqlite4ParserTrace(stderr, "parser: "); }else{ sqlite4ParserTrace(0, 0); @@ -500,11 +437,11 @@ #endif /* Reinstall the LIKE and GLOB functions. The variant of LIKE ** used will be case sensitive or not depending on the RHS. */ - if( sqlite4_stricmp(zLeft, "case_sensitive_like")==0 ){ + if( sqlite4StrICmp(zLeft, "case_sensitive_like")==0 ){ if( zRight ){ sqlite4RegisterLikeFunctions(db, sqlite4GetBoolean(zRight)); } }else @@ -534,11 +471,11 @@ ** ** In the second form this pragma sets the text encoding to be used in ** new database files created using this database handle. It is only ** useful if invoked immediately after the main database i */ - if( sqlite4_stricmp(zLeft, "encoding")==0 ){ + if( sqlite4StrICmp(zLeft, "encoding")==0 ){ static const struct EncName { char *zName; u8 enc; } encnames[] = { { "UTF8", SQLITE4_UTF8 }, @@ -571,11 +508,11 @@ if( !(DbHasProperty(db, 0, DB_SchemaLoaded)) || DbHasProperty(db, 0, DB_Empty) ){ for(pEnc=&encnames[0]; pEnc->zName; pEnc++){ - if( 0==sqlite4_stricmp(zRight, pEnc->zName) ){ + if( 0==sqlite4StrICmp(zRight, pEnc->zName) ){ ENC(pParse->db) = pEnc->enc ? pEnc->enc : SQLITE4_UTF16NATIVE; break; } } if( !pEnc->zName ){ @@ -592,11 +529,11 @@ ** PRAGMA compile_options ** ** Return the names of all compile-time options used in this build, ** one option per row. */ - if( sqlite4_stricmp(zLeft, "compile_options")==0 ){ + if( sqlite4StrICmp(zLeft, "compile_options")==0 ){ int i = 0; const char *zOpt; sqlite4VdbeSetNumCols(v, 1); pParse->nMem = 1; sqlite4VdbeSetColName(v, 0, COLNAME_NAME, "compile_option", SQLITE4_STATIC); @@ -611,21 +548,21 @@ /* ** PRAGMA kvdump ** ** Print an ascii rendering of the complete content of the database file. */ - if( sqlite4_stricmp(zLeft, "kvdump")==0 ){ + if( sqlite4StrICmp(zLeft, "kvdump")==0 ){ sqlite4KVStoreDump(db->aDb[0].pKV); }else #endif /* SQLITE4_OMIT_COMPILEOPTION_DIAGS */ /* ** PRAGMA integrity_check ** ** Check that for each table, the content of any auxilliary indexes are ** consistent with the primary key index. */ - if( sqlite4_stricmp(zLeft, "integrity_check")==0 ){ + if( sqlite4StrICmp(zLeft, "integrity_check")==0 ){ const int baseCsr = 1; /* Base cursor for OpenAllIndexes() call */ const int regErrcnt = 1; /* Register containing error count */ const int regErrstr = 2; /* Register containing error string */ const int regTmp = 3; /* Register for tmp use */ @@ -783,11 +720,11 @@ ** PRAGMA shrink_memory ** ** This pragma attempts to free as much memory as possible from the ** current database connection. */ - if( sqlite4_stricmp(zLeft, "shrink_memory")==0 ){ + if( sqlite4StrICmp(zLeft, "shrink_memory")==0 ){ sqlite4_db_release_memory(db); }else {/* Empty ELSE clause */} Index: src/resolve.c ================================================================== --- src/resolve.c +++ src/resolve.c @@ -107,11 +107,11 @@ */ static int nameInUsingClause(IdList *pUsing, const char *zCol){ if( pUsing ){ int k; for(k=0; knId; k++){ - if( sqlite4_stricmp(pUsing->a[k].zName, zCol)==0 ) return 1; + if( sqlite4StrICmp(pUsing->a[k].zName, zCol)==0 ) return 1; } } return 0; } @@ -119,11 +119,11 @@ ** Return true if table pTab has an implicit primary key, and zCol points ** to a column name that resolves to the implicit primary key (i.e. "rowid"). */ int isRowidReference(Table *pTab, const char *zCol){ int ret = 0; - if( 0==sqlite4_stricmp(zCol, "ROWID") ){ + if( 0==sqlite4StrICmp(zCol, "ROWID") ){ /* If the call to FindPrimaryKey() returns NULL, then pTab must be a ** sub-select or a view. Neither of these have an IPK. */ Index *pPk = sqlite4FindPrimaryKey(pTab, 0); if( pPk && pPk->aiColumn[0]==-1 ) ret = 1; } @@ -200,17 +200,17 @@ iDb = sqlite4SchemaToIndex(db, pTab->pSchema); assert( pTab->nCol>0 ); if( zTab ){ if( pItem->zAlias ){ char *zTabName = pItem->zAlias; - if( sqlite4_stricmp(zTabName, zTab)!=0 ) continue; + if( sqlite4StrICmp(zTabName, zTab)!=0 ) continue; }else{ char *zTabName = pTab->zName; - if( NEVER(zTabName==0) || sqlite4_stricmp(zTabName, zTab)!=0 ){ + if( NEVER(zTabName==0) || sqlite4StrICmp(zTabName, zTab)!=0 ){ continue; } - if( zDb!=0 && sqlite4_stricmp(db->aDb[iDb].zName, zDb)!=0 ){ + if( zDb!=0 && sqlite4StrICmp(db->aDb[iDb].zName, zDb)!=0 ){ continue; } } } if( 0==(cntTab++) ){ @@ -218,11 +218,11 @@ pExpr->pTab = pTab; pSchema = pTab->pSchema; pMatch = pItem; } for(j=0, pCol=pTab->aCol; jnCol; j++, pCol++){ - if( sqlite4_stricmp(pCol->zName, zCol)==0 ){ + if( sqlite4StrICmp(pCol->zName, zCol)==0 ){ /* If there has been exactly one prior match and this match ** is for the right-hand table of a NATURAL JOIN or is in a ** USING clause, then skip this match. */ if( cnt==1 ){ @@ -247,14 +247,14 @@ */ if( zDb==0 && zTab!=0 && cnt==0 && pParse->pTriggerTab!=0 ){ int op = pParse->eTriggerOp; Table *pTab = 0; assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT ); - if( op!=TK_DELETE && sqlite4_stricmp("new",zTab) == 0 ){ + if( op!=TK_DELETE && sqlite4StrICmp("new",zTab) == 0 ){ pExpr->iTable = 1; pTab = pParse->pTriggerTab; - }else if( op!=TK_INSERT && sqlite4_stricmp("old",zTab)==0 ){ + }else if( op!=TK_INSERT && sqlite4StrICmp("old",zTab)==0 ){ pExpr->iTable = 0; pTab = pParse->pTriggerTab; } if( pTab ){ @@ -261,11 +261,11 @@ int iCol; pSchema = pTab->pSchema; cntTab++; for(iCol=0; iColnCol; iCol++){ Column *pCol = &pTab->aCol[iCol]; - if( sqlite4_stricmp(pCol->zName, zCol)==0 ){ + if( sqlite4StrICmp(pCol->zName, zCol)==0 ){ break; } } if( iCol>=pTab->nCol && isRowidReference(pTab, zCol) ){ iCol = -1; /* IMP: R-44911-55124 */ @@ -313,11 +313,11 @@ ** resolved by the time the WHERE clause is resolved. */ if( cnt==0 && (pEList = pNC->pEList)!=0 && zTab==0 ){ for(j=0; jnExpr; j++){ char *zAs = pEList->a[j].zName; - if( zAs!=0 && sqlite4_stricmp(zAs, zCol)==0 ){ + if( zAs!=0 && sqlite4StrICmp(zAs, zCol)==0 ){ Expr *pOrig; assert( pExpr->pLeft==0 && pExpr->pRight==0 ); assert( pExpr->x.pList==0 ); assert( pExpr->x.pSelect==0 ); pOrig = pEList->a[j].pExpr; @@ -339,10 +339,26 @@ */ if( cnt==0 ){ pNC = pNC->pNext; } } + + /* + ** If X and Y are NULL (in other words if only the column name Z is + ** supplied) and the value of Z is enclosed in double-quotes, then + ** Z is a string literal if it doesn't match any column names. In that + ** case, we need to return right away and not make any changes to + ** pExpr. + ** + ** Because no reference was made to outer contexts, the pNC->nRef + ** fields are not changed in any context. + */ + if( cnt==0 && zTab==0 && ExprHasProperty(pExpr,EP_DblQuoted) ){ + pExpr->op = TK_STRING; + pExpr->pTab = 0; + return WRC_Prune; + } /* ** cnt==0 means there was not match. cnt>1 means there were two or ** more matches. Either way, we have an error. */ @@ -418,76 +434,10 @@ ExprSetProperty(p, EP_Resolved); } return p; } -static void resolveMatchArg(Parse *pParse, NameContext *pNC, Expr *pExpr){ - SrcList *pSrc = pNC->pSrcList; - SrcListItem *pItem; - char *zLhs; - int i; - Index *pIdx; - - if( pExpr->op!=TK_ID || pSrc==0 || pExpr==0 ){ - sqlite4ErrorMsg(pParse, "first argument xxx must be a table name"); - return; - } - zLhs = pExpr->u.zToken; - - for(i=0; inSrc; i++){ - pItem = &pSrc->a[i]; - if( pItem->zAlias && sqlite4_stricmp(zLhs, pItem->zAlias)==0 ) break; - if( pItem->zAlias==0 && sqlite4_stricmp(zLhs, pItem->zName)==0 ) break; - } - if( i==pSrc->nSrc ){ - sqlite4ErrorMsg(pParse, "no such table: %s", zLhs); - return; - } - - pExpr->op = TK_NULL; - pExpr->iTable = pItem->iCursor; - ExprSetProperty(pExpr, EP_Resolved); -} - -static void resolveMatch(Parse *pParse, NameContext *pNC, Expr *pExpr){ - Expr *pLeft = pExpr->pLeft; - SrcList *pSrc = pNC->pSrcList; - SrcListItem *pItem; - char *zLhs; - int i; - Index *pIdx; - - if( pLeft->op!=TK_ID || pSrc==0 ){ - sqlite4ErrorMsg(pParse, "lhs of MATCH operator must be a table name"); - return; - } - zLhs = pLeft->u.zToken; - - for(i=0; inSrc; i++){ - pItem = &pSrc->a[i]; - if( pItem->zAlias && sqlite4_stricmp(zLhs, pItem->zAlias)==0 ) break; - if( pItem->zAlias==0 && sqlite4_stricmp(zLhs, pItem->zName)==0 ) break; - } - - if( i==pSrc->nSrc ){ - sqlite4ErrorMsg(pParse, "no such table: %s", zLhs); - return; - } - - for(pIdx=pItem->pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->eIndexType==SQLITE4_INDEX_FTS5 ) break; - } - if( !pIdx ){ - sqlite4ErrorMsg(pParse, "no index to process MATCH operator"); - return; - } - - pExpr->pLeft = 0; - pExpr->pIdx = pIdx; - sqlite4ExprDelete(pParse->db, pLeft); -} - /* ** This routine is callback for sqlite4WalkExpr(). ** ** Resolve symbolic names into TK_COLUMN operators for the current ** node in the expression tree. Return 0 to continue the search down @@ -626,20 +576,13 @@ } if( is_agg ){ pExpr->op = TK_AGG_FUNCTION; pNC->hasAgg = 1; } - - if( pParse->nErr==0 ){ - if( pDef->bMatchinfo ){ - resolveMatchArg(pParse, pNC, n>0 ? pList->a[0].pExpr : 0); - } - if( is_agg ) pNC->allowAgg = 0; - sqlite4WalkExprList(pWalker, pList); - if( is_agg ) pNC->allowAgg = 1; - } - + if( is_agg ) pNC->allowAgg = 0; + sqlite4WalkExprList(pWalker, pList); + if( is_agg ) pNC->allowAgg = 1; /* FIX ME: Compute pExpr->affinity based on the expected return ** type of the function */ return WRC_Prune; } @@ -670,14 +613,10 @@ sqlite4ErrorMsg(pParse,"parameters prohibited in CHECK constraints"); } break; } #endif - case TK_MATCH: { - resolveMatch(pParse, pNC, pExpr); - break; - } } return (pParse->nErr || pParse->db->mallocFailed) ? WRC_Abort : WRC_Continue; } /* @@ -703,11 +642,11 @@ if( pE->op==TK_ID ){ char *zCol = pE->u.zToken; for(i=0; inExpr; i++){ char *zAs = pEList->a[i].zName; - if( zAs!=0 && sqlite4_stricmp(zAs, zCol)==0 ){ + if( zAs!=0 && sqlite4StrICmp(zAs, zCol)==0 ){ return i+1; } } } return 0; Index: src/rowset.c ================================================================== --- src/rowset.c +++ src/rowset.c @@ -155,19 +155,16 @@ p->isSorted = 1; } static u8 *rowsetAllocateChunk(RowSet *p, int nByte){ - int rowChunkSize = ROUND8(sizeof(RowSetChunk)); - RowSetChunk *pNew; /* New RowSetChunk */ + enum { rowChunkSize = ROUND8(sizeof(RowSetChunk)) }; + u8 *pNew; /* New RowSetChunk */ int nAlloc; /* Bytes to request from malloc() */ nAlloc = rowChunkSize + nByte; - pNew = (RowSetChunk *)sqlite4DbMallocRaw(p->db, nAlloc); - if( !pNew ) return 0; - pNew->pNextChunk = p->pChunk; - p->pChunk = pNew; - return (u8 *)(&pNew[1]); + pNew = (u8 *)sqlite4DbMallocRaw(p->db, nAlloc); + return (pNew ? (pNew + rowChunkSize) : 0); } static int rowsetEntryKeyCmp(RowSetEntry *pLeft, const u8 *aKey, int nKey){ int nCmp = SQLITE4_MIN(pLeft->nKey, nKey); int res; Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -71,11 +71,10 @@ } if( pEList==0 ){ pEList = sqlite4ExprListAppend(pParse, 0, sqlite4Expr(db,TK_ALL,0)); } pNew->pEList = pEList; - if( pSrc==0 ) pSrc = sqlite4DbMallocZero(db, sizeof(*pSrc)); pNew->pSrc = pSrc; pNew->pWhere = pWhere; pNew->pGroupBy = pGroupBy; pNew->pHaving = pHaving; pNew->pOrderBy = pOrderBy; @@ -150,11 +149,11 @@ apAll[2] = pC; for(i=0; i<3 && apAll[i]; i++){ p = apAll[i]; for(j=0; jn==aKeyword[j].nChar - && sqlite4_strnicmp((char*)p->z, &zKeyText[aKeyword[j].i], p->n)==0 ){ + && sqlite4StrNICmp((char*)p->z, &zKeyText[aKeyword[j].i], p->n)==0 ){ jointype |= aKeyword[j].code; break; } } testcase( j==0 || j==1 || j==2 || j==3 || j==4 || j==5 || j==6 ); @@ -187,11 +186,11 @@ ** is not contained in the table. */ static int columnIndex(Table *pTab, const char *zCol){ int i; for(i=0; inCol; i++){ - if( sqlite4_stricmp(pTab->aCol[i].zName, zCol)==0 ) return i; + if( sqlite4StrICmp(pTab->aCol[i].zName, zCol)==0 ) return i; } return -1; } /* @@ -1280,11 +1279,11 @@ int nCol; /* Number of columns in the result set */ Expr *p; /* Expression for a single result column */ char *zName; /* Column name */ int nName; /* Size of name in zName[] */ - *pnCol = nCol = pEList ? pEList->nExpr : 0; + *pnCol = nCol = pEList->nExpr; aCol = *paCol = sqlite4DbMallocZero(db, sizeof(aCol[0])*nCol); if( aCol==0 ) return SQLITE4_NOMEM; for(i=0, pCol=aCol; ipEList && pPrior->pEList ); if( p->pEList->nExpr!=pPrior->pEList->nExpr ){ - if( p->selFlags & SF_Values ){ - sqlite4ErrorMsg(pParse, "all VALUES must have the same number of terms"); - }else{ - sqlite4ErrorMsg(pParse, "SELECTs to the left and right of %s" - " do not have the same number of result columns", selectOpName(p->op)); - } + sqlite4ErrorMsg(pParse, "SELECTs to the left and right of %s" + " do not have the same number of result columns", selectOpName(p->op)); rc = 1; goto multi_select_end; } /* Compound SELECTs that have an ORDER BY clause are handled separately. @@ -3121,13 +3116,13 @@ if( NEVER(ExprHasProperty(pExpr, EP_xIsSelect)) ) return 0; pEList = pExpr->x.pList; if( pEList==0 || pEList->nExpr!=1 ) return 0; if( pEList->a[0].pExpr->op!=TK_AGG_COLUMN ) return WHERE_ORDERBY_NORMAL; assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( sqlite4_stricmp(pExpr->u.zToken,"min")==0 ){ + if( sqlite4StrICmp(pExpr->u.zToken,"min")==0 ){ return WHERE_ORDERBY_MIN; - }else if( sqlite4_stricmp(pExpr->u.zToken,"max")==0 ){ + }else if( sqlite4StrICmp(pExpr->u.zToken,"max")==0 ){ return WHERE_ORDERBY_MAX; } return WHERE_ORDERBY_NORMAL; } @@ -3142,11 +3137,11 @@ if( pFrom->pTab && pFrom->zIndex ){ Table *pTab = pFrom->pTab; char *zIndex = pFrom->zIndex; Index *pIdx; for(pIdx=pTab->pIndex; - pIdx && sqlite4_stricmp(pIdx->zName, zIndex); + pIdx && sqlite4StrICmp(pIdx->zName, zIndex); pIdx=pIdx->pNext ); if( !pIdx ){ sqlite4ErrorMsg(pParse, "no such index: %s", zIndex, 0); pParse->checkSchema = 1; @@ -3319,11 +3314,11 @@ char *zTabName = pFrom->zAlias; if( zTabName==0 ){ zTabName = pTab->zName; } if( db->mallocFailed ) break; - if( zTName && sqlite4_stricmp(zTName, zTabName)!=0 ){ + if( zTName && sqlite4StrICmp(zTName, zTabName)!=0 ){ continue; } tableSeen = 1; for(j=0; jnCol; j++){ Expr *pExpr, *pRight; @@ -4172,11 +4167,11 @@ ** to sort the rows, not to eliminate duplicates). */ sqlite4ExprCacheClear(pParse); regBase = sqlite4GetTempRange(pParse, nGroup); sqlite4ExprCodeExprList(pParse, pGroupBy, regBase, 0); sqlite4VdbeAddOp3(v, OP_MakeIdxKey, sAggInfo.sortingIdx,regBase,regKey); - sqlite4VdbeChangeP5(v, OPFLAG_SEQCOUNT); + sqlite4VdbeChangeP5(v, 1); sqlite4ReleaseTempRange(pParse, regBase, nGroup); /* Encode the record for the sorting index. The record contains all ** required column values from the elements of the FROM clause. ** If no column values are required, insert a NULL into the sorting Index: src/shell.c ================================================================== --- src/shell.c +++ src/shell.c @@ -34,18 +34,22 @@ #include #include "sqlite4.h" #include #include -#if !defined(_WIN32) && !defined(WIN32) +#if !defined(_WIN32) && !defined(WIN32) && !defined(__OS2__) # include # if !defined(__RTP__) && !defined(_WRS_KERNEL) # include # endif # include # include #endif + +#ifdef __OS2__ +# include +#endif #ifdef HAVE_EDITLINE # include #endif #if defined(HAVE_READLINE) && HAVE_READLINE==1 @@ -62,14 +66,10 @@ #if defined(_WIN32) || defined(WIN32) # include #define isatty(h) _isatty(h) #define access(f,m) _access((f),(m)) -#undef popen -#define popen(a,b) _popen((a),(b)) -#undef pclose -#define pclose(x) _pclose(x) #else /* Make sure isatty() has a prototype. */ extern int isatty(int); #endif @@ -88,11 +88,11 @@ /* ctype macros that work with signed characters */ #define IsSpace(X) isspace((unsigned char)X) #define IsDigit(X) isdigit((unsigned char)X) #define ToLower(X) (char)tolower((unsigned char)X) -#if !defined(_WIN32) && !defined(WIN32) && !defined(_WRS_KERNEL) +#if !defined(_WIN32) && !defined(WIN32) && !defined(__OS2__) && !defined(__RTP__) && !defined(_WRS_KERNEL) #include #include /* Saved resource information for the beginning of an operation */ static struct rusage sBegin; @@ -177,12 +177,12 @@ } } /* Return the difference of two FILETIME structs in seconds */ static double timeDiff(FILETIME *pStart, FILETIME *pEnd){ - sqlite4_int64 i64Start = *((sqlite4_int64 *) pStart); - sqlite4_int64 i64End = *((sqlite4_int64 *) pEnd); + sqlite_int64 i64Start = *((sqlite_int64 *) pStart); + sqlite_int64 i64End = *((sqlite_int64 *) pEnd); return (double) ((i64End - i64Start) / 10000000.0); } /* ** Print the timing results. @@ -319,11 +319,11 @@ ){ assert( 0==argc ); assert( zShellStatic ); UNUSED_PARAMETER(argc); UNUSED_PARAMETER(argv); - sqlite4_result_text(context, zShellStatic, -1, SQLITE4_STATIC, 0); + sqlite4_result_text(context, zShellStatic, -1, SQLITE4_STATIC); } /* ** This routine reads a line of text from FILE in, stores @@ -417,11 +417,10 @@ sqlite4 *db; /* The database */ int echoOn; /* True to echo input commands */ int statsOn; /* True to display memory stats before each finalize */ int cnt; /* Number of records displayed so far */ FILE *out; /* Write results here */ - FILE *traceOut; /* Output for sqlite4_trace() */ int nErr; /* Number of errors seen */ int mode; /* An output mode setting */ int writableSchema; /* True if PRAGMA writable_schema=ON */ int showHeader; /* True to show column names in List or Column mode */ char *zDestTable; /* Name of destination table when MODE_Insert */ @@ -495,11 +494,11 @@ */ static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ int i; char *zBlob = (char *)pBlob; fprintf(out,"X'"); - for(i=0; icolWidth) ){ w = p->colWidth[i]; }else{ w = 0; } - if( w==0 ){ + if( w<=0 ){ w = strlen30(azCol[i] ? azCol[i] : ""); if( w<10 ) w = 10; n = strlen30(azArg && azArg[i] ? azArg[i] : p->nullvalue); if( wactualWidth) ){ p->actualWidth[i] = w; } if( p->showHeader ){ - if( w<0 ){ - fprintf(p->out,"%*.*s%s",-w,-w,azCol[i], i==nArg-1 ? "\n": " "); - }else{ - fprintf(p->out,"%-*.*s%s",w,w,azCol[i], i==nArg-1 ? "\n": " "); - } + fprintf(p->out,"%-*.*s%s",w,w,azCol[i], i==nArg-1 ? "\n": " "); } } if( p->showHeader ){ for(i=0; iactualWidth) ){ w = p->actualWidth[i]; - if( w<0 ) w = -w; }else{ w = 10; } fprintf(p->out,"%-*.*s%s",w,w,"-----------------------------------" "----------------------------------------------------------", @@ -741,17 +732,12 @@ } if( p->mode==MODE_Explain && azArg[i] && strlen30(azArg[i])>w ){ w = strlen30(azArg[i]); } - if( w<0 ){ - fprintf(p->out,"%*.*s%s",-w,-w, - azArg[i] ? azArg[i] : p->nullvalue, i==nArg-1 ? "\n": " "); - }else{ - fprintf(p->out,"%-*.*s%s",w,w, - azArg[i] ? azArg[i] : p->nullvalue, i==nArg-1 ? "\n": " "); - } + fprintf(p->out,"%-*.*s%s",w,w, + azArg[i] ? azArg[i] : p->nullvalue, i==nArg-1 ? "\n": " "); } break; } case MODE_Semi: case MODE_List: { @@ -797,18 +783,18 @@ } case MODE_Tcl: { if( p->cnt++==0 && p->showHeader ){ for(i=0; iout,azCol[i] ? azCol[i] : ""); - if(iout, "%s", p->separator); + fprintf(p->out, "%s", p->separator); } fprintf(p->out,"\n"); } if( azArg==0 ) break; for(i=0; iout, azArg[i] ? azArg[i] : p->nullvalue); - if(iout, "%s", p->separator); + fprintf(p->out, "%s", p->separator); } fprintf(p->out,"\n"); break; } case MODE_Csv: { @@ -949,54 +935,36 @@ return zIn; } /* -** Execute a query statement that will generate SQL output. Print -** the result columns, comma-separated, on a line and then add a -** semicolon terminator to the end of that line. +** Execute a query statement that has a single result column. Print +** that result column on a line by itself with a semicolon terminator. ** -** If the number of columns is 1 and that column contains text "--" -** then write the semicolon on a separate line. That way, if a -** "--" comment occurs at the end of the statement, the comment -** won't consume the semicolon terminator. +** This is used, for example, to show the schema of the database by +** querying the SQLITE4_MASTER table. */ static int run_table_dump_query( struct callback_data *p, /* Query context */ const char *zSelect, /* SELECT statement to extract content */ const char *zFirstRow /* Print before first row, if not NULL */ ){ sqlite4_stmt *pSelect; int rc; - int nResult; - int i; - const char *z; rc = sqlite4_prepare(p->db, zSelect, -1, &pSelect, 0); if( rc!=SQLITE4_OK || !pSelect ){ fprintf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, sqlite4_errmsg(p->db)); p->nErr++; return rc; } rc = sqlite4_step(pSelect); - nResult = sqlite4_column_count(pSelect); while( rc==SQLITE4_ROW ){ if( zFirstRow ){ fprintf(p->out, "%s", zFirstRow); zFirstRow = 0; } - z = (char const *)sqlite4_column_text(pSelect, 0); - fprintf(p->out, "%s", z); - for(i=1; iout, ",%s", sqlite4_column_text(pSelect, i)); - } - if( z==0 ) z = ""; - while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; - if( z[0] ){ - fprintf(p->out, "\n;\n"); - }else{ - fprintf(p->out, ";\n"); - } + fprintf(p->out, "%s;\n", sqlite4_column_text(pSelect, 0)); rc = sqlite4_step(pSelect); } rc = sqlite4_finalize(pSelect); if( rc!=SQLITE4_OK ){ fprintf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, sqlite4_errmsg(p->db)); @@ -1261,26 +1229,23 @@ if( rc!=SQLITE4_OK || !pTableInfo ){ return 1; } zSelect = appendText(zSelect, "SELECT 'INSERT INTO ' || ", 0); - /* Always quote the table name, even if it appears to be pure ascii, - ** in case it is a keyword. Ex: INSERT INTO "table" ... */ zTmp = appendText(zTmp, zTable, '"'); if( zTmp ){ zSelect = appendText(zSelect, zTmp, '\''); - free(zTmp); } zSelect = appendText(zSelect, " || ' VALUES(' || ", 0); rc = sqlite4_step(pTableInfo); while( rc==SQLITE4_ROW ){ const char *zText = (const char *)sqlite4_column_text(pTableInfo, 1); zSelect = appendText(zSelect, "quote(", 0); zSelect = appendText(zSelect, zText, '"'); rc = sqlite4_step(pTableInfo); if( rc==SQLITE4_ROW ){ - zSelect = appendText(zSelect, "), ", 0); + zSelect = appendText(zSelect, ") || ',' || ", 0); }else{ zSelect = appendText(zSelect, ") ", 0); } nRow++; } @@ -1295,11 +1260,11 @@ rc = run_table_dump_query(p, zSelect, zPrepStmt); if( rc==SQLITE4_CORRUPT ){ zSelect = appendText(zSelect, " ORDER BY rowid DESC", 0); run_table_dump_query(p, zSelect, 0); } - free(zSelect); + if( zSelect ) free(zSelect); } return 0; } /* @@ -1373,14 +1338,13 @@ " insert SQL insert statements for TABLE\n" " line One value per line\n" " list Values delimited by .separator string\n" " tabs Tab-separated values\n" " tcl TCL list elements\n" - ".nullvalue STRING Use STRING in place of NULL values\n" + ".nullvalue STRING Print STRING in place of NULL values\n" ".output FILENAME Send output to FILENAME\n" ".output stdout Send output to the screen\n" - ".print STRING... Print literal STRING\n" ".prompt MAIN CONTINUE Replace the standard prompts\n" ".quit Exit this program\n" ".read FILENAME Execute SQL in FILENAME\n" ".schema ?TABLE? Show the CREATE statements\n" " If TABLE specified, only show tables matching\n" @@ -1389,11 +1353,10 @@ ".show Show the current values for various settings\n" ".stats ON|OFF Turn stats on or off\n" ".tables ?TABLE? List names of tables\n" " If TABLE specified, only list tables matching\n" " LIKE pattern TABLE.\n" - ".trace FILE|off Output each SQL statement as it is run\n" ".width NUM1 NUM2 ... Set column widths for \"column\" mode\n" ; static char zTimerHelp[] = ".timer ON|OFF Turn the CPU timer measurement on or off\n" @@ -1478,56 +1441,10 @@ val = 1; } return val; } -/* -** Close an output file, assuming it is not stderr or stdout -*/ -static void output_file_close(FILE *f){ - if( f && f!=stdout && f!=stderr ) fclose(f); -} - -/* -** Try to open an output file. The names "stdout" and "stderr" are -** recognized and do the right thing. NULL is returned if the output -** filename is "off". -*/ -static FILE *output_file_open(const char *zFile){ - FILE *f; - if( strcmp(zFile,"stdout")==0 ){ - f = stdout; - }else if( strcmp(zFile, "stderr")==0 ){ - f = stderr; - }else if( strcmp(zFile, "off")==0 ){ - f = 0; - }else{ - f = fopen(zFile, "wb"); - if( f==0 ){ - fprintf(stderr, "Error: cannot open \"%s\"\n", zFile); - } - } - return f; -} - -/* -** A routine for handling output from sqlite4_trace(). -*/ -static void sql_trace_callback(void *pArg, const char *z){ - FILE *f = (FILE*)pArg; - if( f ) fprintf(f, "%s\n", z); -} - -/* -** A no-op routine that runs with the ".breakpoint" doc-command. This is -** a useful spot to set a debugger breakpoint. -*/ -static void test_breakpoint(void){ - static int nCall = 0; - nCall++; -} - /* ** If an input line begins with "." then invoke this routine to ** process that line. ** ** Return 1 on error, 2 to exit, and 0 otherwise. @@ -1557,26 +1474,19 @@ while( zLine[i] && !IsSpace(zLine[i]) ){ i++; } if( zLine[i] ) zLine[i++] = 0; resolve_backslashes(azArg[nArg-1]); } } + if( nArg==0 ) return 0; /* no tokens, no error */ n = strlen30(azArg[0]); c = azArg[0][0]; if( c=='b' && n>=3 && strncmp(azArg[0], "bail", n)==0 && nArg>1 && nArg<3 ){ bail_on_error = booleanValue(azArg[1]); }else - - /* The undocumented ".breakpoint" command causes a call to the no-op - ** routine named test_breakpoint(). - */ - if( c=='b' && n>=3 && strncmp(azArg[0], "breakpoint", n)==0 ){ - test_breakpoint(); - }else - if( c=='d' && n>1 && strncmp(azArg[0], "databases", n)==0 && nArg==1 ){ struct callback_data data; char *zErrMsg = 0; open_db(p); memcpy(&data, p, sizeof(data)); @@ -1805,11 +1715,11 @@ if( z[j]=='"' ){ j++; if( z[j]==0 ) break; } z[k++] = z[j]; } z[k] = 0; } - sqlite4_bind_text(pStmt, i+1, azCol[i], -1, SQLITE4_STATIC, 0); + sqlite4_bind_text(pStmt, i+1, azCol[i], -1, SQLITE4_STATIC); } sqlite4_step(pStmt); rc = sqlite4_reset(pStmt); free(zLine); if( rc!=SQLITE4_OK ){ @@ -1904,12 +1814,26 @@ }else #endif if( c=='l' && strncmp(azArg[0], "log", n)==0 && nArg>=2 ){ const char *zFile = azArg[1]; - output_file_close(p->pLog); - p->pLog = output_file_open(zFile); + if( p->pLog && p->pLog!=stdout && p->pLog!=stderr ){ + fclose(p->pLog); + p->pLog = 0; + } + if( strcmp(zFile,"stdout")==0 ){ + p->pLog = stdout; + }else if( strcmp(zFile, "stderr")==0 ){ + p->pLog = stderr; + }else if( strcmp(zFile, "off")==0 ){ + p->pLog = 0; + }else{ + p->pLog = fopen(zFile, "w"); + if( p->pLog==0 ){ + fprintf(stderr, "Error: cannot open \"%s\"\n", zFile); + } + } }else if( c=='m' && strncmp(azArg[0], "mode", n)==0 && nArg==2 ){ int n2 = strlen30(azArg[1]); if( (n2==4 && strncmp(azArg[1],"line",n2)==0) @@ -1924,11 +1848,10 @@ p->mode = MODE_List; }else if( n2==4 && strncmp(azArg[1],"html",n2)==0 ){ p->mode = MODE_Html; }else if( n2==3 && strncmp(azArg[1],"tcl",n2)==0 ){ p->mode = MODE_Tcl; - sqlite4_snprintf(p->separator, sizeof(p->separator), " "); }else if( n2==3 && strncmp(azArg[1],"csv",n2)==0 ){ p->mode = MODE_Csv; sqlite4_snprintf(p->separator, sizeof(p->separator), ","); }else if( n2==4 && strncmp(azArg[1],"tabs",n2)==0 ){ p->mode = MODE_List; @@ -1959,48 +1882,28 @@ sqlite4_snprintf(p->nullvalue, sizeof(p->nullvalue), "%.*s", (int)ArraySize(p->nullvalue)-1, azArg[1]); }else if( c=='o' && strncmp(azArg[0], "output", n)==0 && nArg==2 ){ - if( p->outfile[0]=='|' ){ - pclose(p->out); - }else{ - output_file_close(p->out); - } - p->outfile[0] = 0; - if( azArg[1][0]=='|' ){ - p->out = popen(&azArg[1][1], "w"); - if( p->out==0 ){ - fprintf(stderr,"Error: cannot open pipe \"%s\"\n", &azArg[1][1]); - p->out = stdout; - rc = 1; - }else{ - sqlite4_snprintf(p->outfile, sizeof(p->outfile), "%s", azArg[1]); - } - }else{ - p->out = output_file_open(azArg[1]); - if( p->out==0 ){ - if( strcmp(azArg[1],"off")!=0 ){ - fprintf(stderr,"Error: cannot write to \"%s\"\n", azArg[1]); - } + if( p->out!=stdout ){ + fclose(p->out); + } + if( strcmp(azArg[1],"stdout")==0 ){ + p->out = stdout; + sqlite4_snprintf(p->outfile, sizeof(p->outfile), "stdout"); + }else{ + p->out = fopen(azArg[1], "wb"); + if( p->out==0 ){ + fprintf(stderr,"Error: cannot write to \"%s\"\n", azArg[1]); p->out = stdout; rc = 1; } else { - sqlite4_snprintf(p->outfile, sizeof(p->outfile), "%s", azArg[1]); + sqlite4_snprintf(p->outfile, sizeof(p->outfile), "%s", azArg[1]); } } }else - if( c=='p' && n>=3 && strncmp(azArg[0], "print", n)==0 ){ - int i; - for(i=1; i1 ) fprintf(p->out, " "); - fprintf(p->out, "%s", azArg[i]); - } - fprintf(p->out, "\n"); - }else - if( c=='p' && strncmp(azArg[0], "prompt", n)==0 && (nArg==2 || nArg==3)){ if( nArg >= 2) { strncpy(mainPrompt,azArg[1],(int)ArraySize(mainPrompt)-1); } if( nArg >= 3) { @@ -2064,29 +1967,27 @@ rc = SQLITE4_OK; }else{ zShellStatic = azArg[1]; rc = sqlite4_exec(p->db, "SELECT sql FROM " - " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" + " (SELECT sql sql, type type, tbl_name tbl_name, name name" " FROM sqlite_master UNION ALL" - " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " + " SELECT sql, type, tbl_name, name FROM sqlite_temp_master) " "WHERE lower(tbl_name) LIKE shellstatic()" " AND type!='meta' AND sql NOTNULL " - "ORDER BY substr(type,2,1)," - " CASE type WHEN 'view' THEN x ELSE name END", + "ORDER BY substr(type,2,1), name", callback, &data, &zErrMsg); zShellStatic = 0; } }else{ rc = sqlite4_exec(p->db, "SELECT sql FROM " - " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" + " (SELECT sql sql, type type, tbl_name tbl_name, name name" " FROM sqlite_master UNION ALL" - " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " + " SELECT sql, type, tbl_name, name FROM sqlite_temp_master) " "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%'" - "ORDER BY substr(type,2,1)," - " CASE type WHEN 'view' THEN x ELSE name END", + "ORDER BY substr(type,2,1), name", callback, &data, &zErrMsg ); } if( zErrMsg ){ fprintf(stderr,"Error: %s\n", zErrMsg); @@ -2170,13 +2071,13 @@ sqlite4_free(0, zSql); if( rc ) return rc; nRow = nAlloc = 0; azResult = 0; if( nArg>1 ){ - sqlite4_bind_text(pStmt, 1, azArg[1], -1, SQLITE4_TRANSIENT, 0); + sqlite4_bind_text(pStmt, 1, azArg[1], -1, SQLITE4_TRANSIENT); }else{ - sqlite4_bind_text(pStmt, 1, "%", -1, SQLITE4_STATIC, 0); + sqlite4_bind_text(pStmt, 1, "%", -1, SQLITE4_STATIC); } while( sqlite4_step(pStmt)==SQLITE4_ROW ){ if( nRow>=nAlloc ){ char **azNew; int n = nAlloc*2 + 10; @@ -2304,35 +2205,15 @@ && nArg==2 ){ enableTimer = booleanValue(azArg[1]); }else - if( c=='t' && strncmp(azArg[0], "trace", n)==0 && nArg>1 ){ - open_db(p); - output_file_close(p->traceOut); - p->traceOut = output_file_open(azArg[1]); -#if !defined(SQLITE4_OMIT_TRACE) && !defined(SQLITE4_OMIT_FLOATING_POINT) - if( p->traceOut==0 ){ - sqlite4_trace(p->db, 0, 0); - }else{ - sqlite4_trace(p->db, sql_trace_callback, p->traceOut); - } -#endif - }else - if( c=='v' && strncmp(azArg[0], "version", n)==0 ){ printf("SQLite %s %s\n" /*extra-version-info*/, sqlite4_libversion(), sqlite4_sourceid()); }else -#if defined(SQLITE4_DEBUG) && defined(SQLITE4_ENABLE_WHERETRACE) - if( c=='w' && strncmp(azArg[0], "wheretrace", n)==0 ){ - extern int sqlite4WhereTrace; - sqlite4WhereTrace = atoi(azArg[1]); - }else -#endif - if( c=='w' && strncmp(azArg[0], "width", n)==0 && nArg>1 ){ int j; assert( nArg<=ArraySize(azArg) ); for(j=1; jcolWidth); j++){ p->colWidth[j-1] = atoi(azArg[j]); @@ -2436,13 +2317,11 @@ while( errCnt==0 || !bail_on_error || (in==0 && stdin_is_interactive) ){ fflush(p->out); free(zLine); zLine = one_input_line(zSql, in); if( zLine==0 ){ - /* End of input */ - if( stdin_is_interactive ) printf("\n"); - break; + break; /* We have reached EOF */ } if( seenInterrupt ){ if( in!=0 ) break; seenInterrupt = 0; } @@ -2525,43 +2404,42 @@ return errCnt; } /* ** Return a pathname which is the user's home directory. A -** 0 return indicates an error of some kind. +** 0 return indicates an error of some kind. Space to hold the +** resulting string is obtained from malloc(). The calling +** function should free the result. */ static char *find_home_dir(void){ - static char *home_dir = NULL; - if( home_dir ) return home_dir; - -#if !defined(_WIN32) && !defined(WIN32) && !defined(__RTP__) && !defined(_WRS_KERNEL) - { - struct passwd *pwent; - uid_t uid = getuid(); - if( (pwent=getpwuid(uid)) != NULL) { - home_dir = pwent->pw_dir; - } + char *home_dir = NULL; + +#if !defined(_WIN32) && !defined(WIN32) && !defined(__OS2__) && !defined(_WIN32_WCE) && !defined(__RTP__) && !defined(_WRS_KERNEL) + struct passwd *pwent; + uid_t uid = getuid(); + if( (pwent=getpwuid(uid)) != NULL) { + home_dir = pwent->pw_dir; } #endif #if defined(_WIN32_WCE) /* Windows CE (arm-wince-mingw32ce-gcc) does not provide getenv() */ - home_dir = "/"; + home_dir = strdup("/"); #else -#if defined(_WIN32) || defined(WIN32) +#if defined(_WIN32) || defined(WIN32) || defined(__OS2__) if (!home_dir) { home_dir = getenv("USERPROFILE"); } #endif if (!home_dir) { home_dir = getenv("HOME"); } -#if defined(_WIN32) || defined(WIN32) +#if defined(_WIN32) || defined(WIN32) || defined(__OS2__) if (!home_dir) { char *zDrive, *zPath; int n; zDrive = getenv("HOMEDRIVE"); zPath = getenv("HOMEPATH"); @@ -2637,30 +2515,29 @@ /* ** Show available command line options */ static const char zOptions[] = + " -help show this message\n" + " -init filename read/process named file\n" + " -echo print commands before execution\n" + " -[no]header turn headers on or off\n" " -bail stop after hitting an error\n" + " -interactive force interactive I/O\n" " -batch force batch I/O\n" " -column set output mode to 'column'\n" - " -cmd COMMAND run \"COMMAND\" before reading stdin\n" " -csv set output mode to 'csv'\n" - " -echo print commands before execution\n" - " -init FILENAME read/process named file\n" - " -[no]header turn headers on or off\n" -#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) - " -heap SIZE Size of heap for memsys3 or memsys5\n" -#endif - " -help show this message\n" " -html set output mode to HTML\n" - " -interactive force interactive I/O\n" " -line set output mode to 'line'\n" " -list set output mode to 'list'\n" - " -nullvalue TEXT set text string for NULL values. Default ''\n" - " -separator SEP set output field separator. Default: '|'\n" + " -separator 'x' set output field separator (|)\n" " -stats print memory stats before each finalize\n" + " -nullvalue 'text' set text string for NULL values\n" " -version show SQLite version\n" +#ifdef SQLITE4_ENABLE_MULTIPLEX + " -multiplex enable the multiplexor VFS\n" +#endif ; static void usage(int showDetail){ fprintf(stderr, "Usage: %s [OPTIONS] FILENAME [SQL]\n" "FILENAME is the name of an SQLite database. A new database is created\n" @@ -2685,23 +2562,10 @@ sqlite4_snprintf(mainPrompt,sizeof(mainPrompt), "sqlite> "); sqlite4_snprintf(continuePrompt,sizeof(continuePrompt)," ...> "); sqlite4_env_config(0, SQLITE4_ENVCONFIG_SINGLETHREAD); } -/* -** Get the argument to an --option. Throw an error and die if no argument -** is available. -*/ -static char *cmdline_option_value(int argc, char **argv, int i){ - if( i==argc ){ - fprintf(stderr, "%s: Error: missing argument to %s\n", - argv[0], argv[argc-1]); - exit(1); - } - return argv[i]; -} - int main(int argc, char **argv){ char *zErrMsg = 0; struct callback_data data; const char *zInitFile = 0; char *zFirstCmd = 0; @@ -2727,67 +2591,78 @@ /* Do an initial pass through the command-line argument to locate ** the name of the database file, the name of the initialization file, ** the size of the alternative malloc heap, ** and the first command to execute. */ - for(i=1; i0x7fff0000 ) szHeap = 0x7fff0000; sqlite4_config(0,SQLITE4_CONFIG_HEAP, malloc((int)szHeap),(int)szHeap,64); #endif +#ifdef SQLITE4_ENABLE_MULTIPLEX + }else if( strcmp(argv[i],"-multiplex")==0 ){ + extern int sqlite4_multiple_initialize(const char*,int); + sqlite4_multiplex_initialize(0, 1); +#endif } } - if( data.zDbFilename==0 ){ -#ifndef SQLITE_OMIT_MEMORYDB + if( i=argc){ + fprintf(stderr,"%s: Error: missing argument for option: %s\n", Argv0, z); + fprintf(stderr,"Use -help for a list of options.\n"); + return 1; + } sqlite4_snprintf(data.separator, sizeof(data.separator), - "%s",cmdline_option_value(argc,argv,++i)); + "%.*s",(int)sizeof(data.separator)-1,argv[i]); }else if( strcmp(z,"-nullvalue")==0 ){ + i++; + if(i>=argc){ + fprintf(stderr,"%s: Error: missing argument for option: %s\n", Argv0, z); + fprintf(stderr,"Use -help for a list of options.\n"); + return 1; + } sqlite4_snprintf(data.nullvalue, sizeof(data.nullvalue), - "%s",cmdline_option_value(argc,argv,++i)); + "%.*s",(int)sizeof(data.nullvalue)-1,argv[i]); }else if( strcmp(z,"-header")==0 ){ data.showHeader = 1; }else if( strcmp(z,"-noheader")==0 ){ data.showHeader = 0; }else if( strcmp(z,"-echo")==0 ){ @@ -2856,27 +2742,10 @@ }else if( strcmp(z,"-multiplex")==0 ){ i++; #endif }else if( strcmp(z,"-help")==0 || strcmp(z, "--help")==0 ){ usage(1); - }else if( strcmp(z,"-cmd")==0 ){ - if( i==argc-1 ) break; - z = cmdline_option_value(argc,argv,++i); - if( z[0]=='.' ){ - rc = do_meta_command(z, &data); - if( rc && bail_on_error ) return rc; - }else{ - open_db(&data); - rc = shell_exec(data.db, z, shell_callback, &data, &zErrMsg); - if( zErrMsg!=0 ){ - fprintf(stderr,"Error: %s\n", zErrMsg); - if( bail_on_error ) return rc!=0 ? rc : 1; - }else if( rc!=0 ){ - fprintf(stderr,"Error: unable to process SQL \"%s\"\n", z); - if( bail_on_error ) return rc; - } - } }else{ fprintf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); fprintf(stderr,"Use -help for a list of options.\n"); return 1; } @@ -2925,10 +2794,11 @@ if( zHistory ){ stifle_history(100); write_history(zHistory); free(zHistory); } + free(zHome); }else{ rc = process_input(&data, stdin); } } set_table_name(&data, 0); Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -122,10 +122,11 @@ #define SQLITE4_ENVCONFIG_LOOKASIDE 10 /* size, count */ #define SQLITE4_ENVCONFIG_LOG 11 /* xLog, pArg */ #define SQLITE4_ENVCONFIG_KVSTORE_PUSH 12 /* name, factory */ #define SQLITE4_ENVCONFIG_KVSTORE_POP 13 /* name */ #define SQLITE4_ENVCONFIG_KVSTORE_GET 14 /* name, *factor */ + /* ** CAPIREF: Compile-Time Library Version Numbers ** ** ^(The [SQLITE4_VERSION] C preprocessor macro in the sqlite4.h header @@ -228,12 +229,11 @@ ** object is used by two or more threads at the same time. This ** corresponds to [SQLITE4_ENVCONFIG_MULTITHREAD]. ** ** ^The sqlite4_threadsafe(E) function returns two if the same ** [database connection] can be used at the same time from two or more -** separate threads. This setting corresponds to -** [SQLITE4_ENVCONFIG_SERIALIZED]. +** separate threads. This setting corresponds to [SQLITE4_ENVCONFIG_SERIALIZED]. ** ** Note that SQLite4 is always threadsafe in this sense: Two or more ** objects each associated with different [sqlite4_env] objects can ** always be used at the same time in separate threads. */ @@ -254,14 +254,18 @@ */ typedef struct sqlite4 sqlite4; /* ** CAPIREF: 64-Bit Integer Types -** KEYWORDS: sqlite4_int64 sqlite4_uint64 +** KEYWORDS: sqlite_int64 sqlite_uint64 ** ** Because there is no cross-platform way to specify 64-bit integer types ** SQLite includes typedefs for 64-bit signed and unsigned integers. +** +** The sqlite4_int64 and sqlite4_uint64 are the preferred type definitions. +** The sqlite_int64 and sqlite_uint64 types are supported for backwards +** compatibility only. ** ** ^The sqlite4_int64 and sqlite_int64 types can store integer values ** between -9223372036854775808 and +9223372036854775807 inclusive. ^The ** sqlite4_uint64 and sqlite_uint64 types can store integer values ** between 0 and +18446744073709551615 inclusive. @@ -408,19 +412,20 @@ ** ** See also: [SQLITE4_IOERR_READ | extended result codes], ** [sqlite4_vtab_on_conflict()] [SQLITE4_ROLLBACK | result codes]. */ #define SQLITE4_OK 0 /* Successful result */ +/* beginning-of-error-codes */ #define SQLITE4_ERROR 1 /* SQL error or missing database */ #define SQLITE4_INTERNAL 2 /* Internal logic error in SQLite */ #define SQLITE4_PERM 3 /* Access permission denied */ #define SQLITE4_ABORT 4 /* Callback routine requested an abort */ #define SQLITE4_BUSY 5 /* The database file is locked */ #define SQLITE4_LOCKED 6 /* A table in the database is locked */ #define SQLITE4_NOMEM 7 /* A malloc() failed */ #define SQLITE4_READONLY 8 /* Attempt to write a readonly database */ -#define SQLITE4_INTERRUPT 9 /* Stopped terminated by sqlite4_interrupt()*/ +#define SQLITE4_INTERRUPT 9 /* Operation terminated by sqlite4_interrupt()*/ #define SQLITE4_IOERR 10 /* Some kind of disk I/O error occurred */ #define SQLITE4_CORRUPT 11 /* The database disk image is malformed */ #define SQLITE4_NOTFOUND 12 /* Unknown opcode in sqlite4_file_control() */ #define SQLITE4_FULL 13 /* Insertion failed because database is full */ #define SQLITE4_CANTOPEN 14 /* Unable to open the database file */ @@ -432,15 +437,16 @@ #define SQLITE4_MISMATCH 20 /* Data type mismatch */ #define SQLITE4_MISUSE 21 /* Library used incorrectly */ #define SQLITE4_NOLFS 22 /* Uses OS features not supported on host */ #define SQLITE4_AUTH 23 /* Authorization denied */ #define SQLITE4_FORMAT 24 /* Auxiliary database format error */ -#define SQLITE4_RANGE 25 /* 2nd param to sqlite4_bind out of range */ +#define SQLITE4_RANGE 25 /* 2nd parameter to sqlite4_bind out of range */ #define SQLITE4_NOTADB 26 /* File opened that is not a database file */ #define SQLITE4_ROW 100 /* sqlite4_step() has another row ready */ #define SQLITE4_DONE 101 /* sqlite4_step() has finished executing */ #define SQLITE4_INEXACT 102 /* xSeek method of storage finds nearby ans */ +/* end-of-error-codes */ /* ** CAPIREF: Extended Result Codes ** KEYWORDS: {extended error code} {extended error codes} ** KEYWORDS: {extended result code} {extended result codes} @@ -521,11 +527,11 @@ struct sqlite4_mutex_methods *pMutexMethods; /* Subclasses will typically add additional fields */ }; /* -** CAPIREF: Initialize An SQLite Environment +** CAPIREF: Initialize The SQLite Library ** ** ^The sqlite4_initialize(A) routine initializes an sqlite4_env object A. ** ^The sqlite4_shutdown(A) routine ** deallocates any resources that were allocated by sqlite4_initialize(A). ** A parameter value of NULL means to use the default environment. @@ -742,17 +748,23 @@ /* ** CAPIREF: Last Insert Rowid ** -** ^The sqlite4_last_insert_rowid(D) routine returns the primary key value -** of the the most recent successful [INSERT] from [database connection] D -** into a table where the primary key of the inserted row is a single -** integer. -** ^If there have been no successful [INSERT]s of rows with a single integer -** PRIMARY KEY value on database connection D, then this routine returns -** zero. +** ^Each entry in an SQLite table has a unique 64-bit signed +** integer key called the [ROWID | "rowid"]. ^The rowid is always available +** as an undeclared column named ROWID, OID, or _ROWID_ as long as those +** names are not also used by explicitly declared columns. ^If +** the table has a column of type [INTEGER PRIMARY KEY] then that column +** is another alias for the rowid. +** +** ^This routine returns the [rowid] of the most recent +** successful [INSERT] into the database from the [database connection] +** in the first argument. ^As of SQLite version 3.7.7, this routines +** records the last insert rowid of both ordinary tables and [virtual tables]. +** ^If no successful [INSERT]s +** have ever occurred on that database connection, zero is returned. ** ** ^(If an [INSERT] occurs within a trigger or within a [virtual table] ** method, then this routine will return the [rowid] of the inserted ** row as long as the trigger or virtual table method is running. ** But once the trigger or virtual table method ends, the value returned @@ -1831,20 +1843,17 @@ ** index is out of range. ^[SQLITE4_NOMEM] is returned if malloc() fails. ** ** See also: [sqlite4_bind_parameter_count()], ** [sqlite4_bind_parameter_name()], and [sqlite4_bind_parameter_index()]. */ -int sqlite4_bind_blob(sqlite4_stmt*, int, const void*, int n, - void(*)(void*,void*),void*); +int sqlite4_bind_blob(sqlite4_stmt*, int, const void*, int n, void(*)(void*)); int sqlite4_bind_double(sqlite4_stmt*, int, double); int sqlite4_bind_int(sqlite4_stmt*, int, int); int sqlite4_bind_int64(sqlite4_stmt*, int, sqlite4_int64); int sqlite4_bind_null(sqlite4_stmt*, int); -int sqlite4_bind_text(sqlite4_stmt*, int, const char*, int n, - void(*)(void*,void*),void*); -int sqlite4_bind_text16(sqlite4_stmt*, int, const void*, int, - void(*)(void*,void*),void*); +int sqlite4_bind_text(sqlite4_stmt*, int, const char*, int n, void(*)(void*)); +int sqlite4_bind_text16(sqlite4_stmt*, int, const void*, int, void(*)(void*)); int sqlite4_bind_value(sqlite4_stmt*, int, const sqlite4_value*); int sqlite4_bind_zeroblob(sqlite4_stmt*, int, int n); /* ** CAPIREF: Number Of SQL Parameters @@ -2665,12 +2674,11 @@ ** ** These routines must be called from the same thread in which ** the SQL function is running. */ void *sqlite4_get_auxdata(sqlite4_context*, int N); -void sqlite4_set_auxdata(sqlite4_context*, int N, void*, - void (*)(void*,void*),void*); +void sqlite4_set_auxdata(sqlite4_context*, int N, void*, void (*)(void*)); /* ** CAPIREF: Constants Defining Special Destructor Behavior ** @@ -2683,12 +2691,12 @@ ** the content before returning. ** ** The typedef is necessary to work around problems in certain ** C++ compilers. See ticket #2191. */ -typedef void (*sqlite4_destructor_type)(void*,void*); -void sqlite4_dynamic(void*,void*); +typedef void (*sqlite4_destructor_type)(void*); +void sqlite4_dynamic(void*); #define SQLITE4_STATIC ((sqlite4_destructor_type)0) #define SQLITE4_TRANSIENT ((sqlite4_destructor_type)-1) #define SQLITE4_DYNAMIC (sqlite4_dynamic) @@ -2800,29 +2808,24 @@ ** ** If these routines are called from within the different thread ** than the one containing the application-defined function that received ** the [sqlite4_context] pointer, the results are undefined. */ -void sqlite4_result_blob(sqlite4_context*, const void*, int, - void(*)(void*,void*),void*); +void sqlite4_result_blob(sqlite4_context*, const void*, int, void(*)(void*)); void sqlite4_result_double(sqlite4_context*, double); void sqlite4_result_error(sqlite4_context*, const char*, int); void sqlite4_result_error16(sqlite4_context*, const void*, int); void sqlite4_result_error_toobig(sqlite4_context*); void sqlite4_result_error_nomem(sqlite4_context*); void sqlite4_result_error_code(sqlite4_context*, int); void sqlite4_result_int(sqlite4_context*, int); void sqlite4_result_int64(sqlite4_context*, sqlite4_int64); void sqlite4_result_null(sqlite4_context*); -void sqlite4_result_text(sqlite4_context*, const char*, int, - void(*)(void*,void*),void*); -void sqlite4_result_text16(sqlite4_context*, const void*, int, - void(*)(void*,void*),void*); -void sqlite4_result_text16le(sqlite4_context*, const void*, int, - void(*)(void*,void*),void*); -void sqlite4_result_text16be(sqlite4_context*, const void*, int, - void(*)(void*,void*),void*); +void sqlite4_result_text(sqlite4_context*, const char*, int, void(*)(void*)); +void sqlite4_result_text16(sqlite4_context*, const void*, int, void(*)(void*)); +void sqlite4_result_text16le(sqlite4_context*, const void*, int,void(*)(void*)); +void sqlite4_result_text16be(sqlite4_context*, const void*, int,void(*)(void*)); void sqlite4_result_value(sqlite4_context*, sqlite4_value*); void sqlite4_result_zeroblob(sqlite4_context*, int n); /* ** CAPIREF: Define New Collating Sequences @@ -4125,11 +4128,10 @@ ** ^The [sqlite4_strnicmp()] API allows applications and extensions to ** compare the contents of two buffers containing UTF-8 strings in a ** case-independent fashion, using the same definition of case independence ** that SQLite uses internally when comparing identifiers. */ -int sqlite4_stricmp(const char*, const char*); int sqlite4_strnicmp(const char *, const char *, int); /* ** CAPIREF: Error Logging Interface ** @@ -4318,35 +4320,21 @@ typedef struct sqlite4_kv_methods sqlite4_kv_methods; /* ** CAPI4REF: Key-value storage engine open flags ** -** Allowed values to the flags parameter of an [sqlite4_kvfactory] object. +** Allowed values to the flags parameter of an sqlite4_kvstore object +** factory. ** -** The flags parameter to the sqlite4_kvfactory fuction (the fourth parameter) +** The flags parameter to the sqlite4_kvstore factory (the fourth parameter) ** is an OR-ed combination of these values and the ** [SQLITE4_OPEN_READONLY | SQLITE4_OPEN_xxxxx] flags that appear as ** arguments to [sqlite4_open()]. */ #define SQLITE4_KVOPEN_TEMPORARY 0x00010000 /* A temporary database */ #define SQLITE4_KVOPEN_NO_TRANSACTIONS 0x00020000 /* No transactions needed */ -/* -** CAPIREF: Key-value storage object factory -** -** New key/value storage engines can be added to SQLite4 at run-time. -** In order to create a new KV storage engine, the application must -** supply a "factory" function that creates an instance of the -** sqlite4_kvstore object. This is typedef defines the signature -** of that factory function. -*/ -typedef int (*sqlite4_kvfactory)( - sqlite4_env *pEnv, /* The environment to use */ - sqlite4_kvstore **ppKVStore, /* OUT: New KV store returned here */ - const char *zFilename, /* Name of database file to open */ - unsigned flags /* Bit flags */ -); /* ** CAPI4REF: Representation Of Numbers ** ** Every number in SQLite is represented in memory by an instance of @@ -4354,11 +4342,11 @@ */ typedef struct sqlite4_num sqlite4_num; struct sqlite4_num { unsigned char sign; /* Sign of the overall value */ unsigned char approx; /* True if the value is approximate */ - short e; /* The exponent. */ + unsigned short e; /* The exponent. */ sqlite4_uint64 m; /* The significant */ }; /* ** CAPI4REF: Operations On SQLite Number Objects @@ -4382,156 +4370,10 @@ /* ** CAPI4REF: Flags For Text-To-Numeric Conversion */ #define SQLITE4_PREFIX_ONLY 0x10 #define SQLITE4_IGNORE_WHITESPACE 0x20 - -typedef struct sqlite4_tokenizer sqlite4_tokenizer; - -/* -** CAPI4REF: Register an FTS tokenizer implementation -** -** xTokenize: -** This function does the actual tokenization of an input string. For -** each token in the input, the callback function is invoked once. The -** arguments passed to the callback by the tokenizer must be as follows: -** -** xCallback(ctx, iWeight, zToken, nToken, iSrc, nSrc); -** -** ctx - Copy of the first argument passed to xTokenize. -** iOff - Offset of token in document. -** iWeight - Weight assigned to the token by the tokenizer. Larger values -** indicate more important tokens. -** zToken - Pointer to buffer containing token text. -** nToken - Number of bytes in zToken. -** iSrc - Byte offset in source document of the start of this token. -** nSrc - Bytes of text in source document tokenized to this token. -*/ -int sqlite4_create_tokenizer( - sqlite4 *db, - const char *zName, - void *pCtx, - int (*xCreate)(void*, const char**, int, sqlite4_tokenizer**), - int (*xTokenize)(void*, sqlite4_tokenizer*, - const char*, int, - int(*x)(void *ctx, int iWeight, int iOff, - const char *zToken, int nToken, int iSrc, int nSrc) - ), - int (*xDestroy)(sqlite4_tokenizer *) -); - -/* -** CAPI4REF: Register a matchinfo function. -*/ -int sqlite4_create_mi_function( - sqlite4 *db, - const char *zFunc, - int nArg, - int enc, - void *p, - void (*xFunc)(sqlite4_context*,int,sqlite4_value **), - void (*xDestroy)(void *) -); - -/* -** CAPIREF: Matchinfo APIs. -** -** Special functions that may be called from within matchinfo UDFs. All -** return an SQLite error code - SQLITE4_OK if successful, or some other -** error code otherwise. -** -** sqlite4_mi_column_count(): -** Set *pn to the number of columns in the queried table. -** -** sqlite4_mi_phrase_count(): -** Set *pn to the number of phrases in the query. -** -** sqlite4_mi_stream_count(): -** Set *pn to the number of streams in the FTS index. -** -** sqlite4_mi_phrase_token_count(): -** Set *pn to the number of tokens in phrase iP of the query. -** -** sqlite4_mi_size(): -** Set *pn to the number of tokens belonging to stream iS in the value -** stored in column iC of the current row. -** -** Either or both of iS and iC may be negative. If iC is negative, then the -** output value is the total number of tokens for the specified stream (or -** streams) across all table columns. Similarly, if iS is negative, the -** output value is the total number of tokens in the specified column or -** columns, regardless of stream. -** -** sqlite4_mi_total_size(): -** Similar to sqlite4_mi_size(), except the output parameter is set to -** the total number of tokens belonging to the specified column(s) -** and stream(s) in all rows of the table, not just the current row. -** -** sqlite4_mi_total_rows(): -** Set *pn to the total number of rows in the indexed table. -** -** sqlite4_mi_row_count(): -** Set the output parameter to the total number of rows in the table that -** contain at least one instance of the phrase identified by parameter -** iP in the column(s) and stream(s) identified by parameters iC and iS. -** -** sqlite4_mi_match_count(): -** Set the output parameter to the total number of occurences of phrase -** iP in the current row that belong to the column(s) and stream(s) -** identified by parameters iC and iS. -** -** Parameter iP may also be negative. In this case, the output value is -** set to the total number of occurrences of all query phrases in the -** current row, subject to the constraints imposed by iC and iS. -** -** sqlite4_mi_match_detail(): -** This function is used to access the details of the iMatch'th match -** (of any phrase) in the current row. Matches are sorted in order of -** occurrence. If parameter iMatch is equal to or greater than the number -** of matches in the current row, SQLITE_NOTFOUND is returned. Otherwise, -** unless an error occurs, SQLITE4_OK is returned and the *piOff, *piC, *piS, -** and *piP output parameters are set to the token offset, column number, -** stream number and phrase number respectively. -** -** It is anticipated that this function be used to iterate through matches -** in order of occurrence. It is optimized so that it is fastest when -** called with the iMatch parameter set to 0, P or P+1, where P is the -** iMatch value passed to the previous call. -** -** sqlite4_mi_column_value(): -** Set *ppVal to point to an sqlite4_value object containing the value -** read from column iCol of the current row. This object is valid until -** the function callback returns. -** -** sqlite4_mi_tokenize(): -** Use the tokenizer associated with the FTS index to tokenize the -** document in buffer zDoc (size nDoc bytes). Invoke callback function -** x() once for each token, in the same way as the xTokenizer() method -** of a tokenizer implementation. -*/ -int sqlite4_mi_column_count(sqlite4_context *, int *pn); -int sqlite4_mi_phrase_count(sqlite4_context *, int *pn); -int sqlite4_mi_stream_count(sqlite4_context *, int *pn); -int sqlite4_mi_phrase_token_count(sqlite4_context *, int iP, int *pn); - -int sqlite4_mi_total_size(sqlite4_context *, int iC, int iS, int *pn); -int sqlite4_mi_total_rows(sqlite4_context *, int *pn); - -int sqlite4_mi_row_count(sqlite4_context *, int iC, int iS, int iP, int *pn); - -int sqlite4_mi_size(sqlite4_context *, int iC, int iS, int *pn); -int sqlite4_mi_match_count(sqlite4_context *, int iC, int iS, int iP, int *pn); -int sqlite4_mi_match_detail( - sqlite4_context *, int iMatch, int *piOff, int *piC, int *piS, int *piP -); -int sqlite4_mi_column_value(sqlite4_context *, int iC, sqlite4_value **ppVal); - -int sqlite4_mi_tokenize(sqlite4_context *, const char *zDoc, int nDoc, - void *p, int(*x)(void *, int, int, const char *, int, int, int) -); - - /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. */ Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -17,11 +17,11 @@ #define SQLITE4_OMIT_ANALYZE 1 #define SQLITE4_OMIT_PROGRESS_CALLBACK 1 #define SQLITE4_OMIT_VIRTUALTABLE 1 #define SQLITE4_OMIT_XFER_OPT 1 -#define SQLITE4_OMIT_LOCALTIME 1 +/* #define SQLITE4_OMIT_AUTOMATIC_INDEX 1 */ /* ** These #defines should enable >2GB file support on POSIX if the ** underlying operating system supports it. If the OS lacks ** large file support, or if the OS is windows, these should be no-ops. @@ -289,10 +289,31 @@ #else # define ALWAYS(X) (X) # define NEVER(X) (X) #endif +/* +** Return true (non-zero) if the input is a integer that is too large +** to fit in 32-bits. This macro is used inside of various testcase() +** macros to verify that we have tested SQLite for large-file support. +*/ +#define IS_BIG_INT(X) (((X)&~(i64)0xffffffff)!=0) + +/* +** The macro unlikely() is a hint that surrounds a boolean +** expression that is usually false. Macro likely() surrounds +** a boolean expression that is usually true. GCC is able to +** use these hints to generate better code, sometimes. +*/ +#if defined(__GNUC__) && 0 +# define likely(X) __builtin_expect((X),1) +# define unlikely(X) __builtin_expect((X),0) +#else +# define likely(X) !!(X) +# define unlikely(X) !!(X) +#endif + #include "sqlite4.h" #include "hash.h" #include "parse.h" #include #include @@ -303,13 +324,13 @@ /* ** If compiling for a processor that lacks floating point support, ** substitute integer for floating-point */ #ifdef SQLITE4_OMIT_FLOATING_POINT -# define double sqlite4_int64 -# define float sqlite4_int64 -# define LONGDOUBLE_TYPE sqlite4_int64 +# define double sqlite_int64 +# define float sqlite_int64 +# define LONGDOUBLE_TYPE sqlite_int64 # ifndef SQLITE4_BIG_DBL # define SQLITE4_BIG_DBL (((sqlite4_int64)1)<<50) # endif # define SQLITE4_OMIT_DATETIME_FUNCS 1 # define SQLITE4_OMIT_TRACE 1 @@ -329,10 +350,29 @@ #define OMIT_TEMPDB 1 #else #define OMIT_TEMPDB 0 #endif +/* +** The "file format" number is an integer that is incremented whenever +** the VDBE-level file format changes. The following macros define the +** the default file format for new databases and the maximum file format +** that the library can read. +*/ +#define SQLITE4_MAX_FILE_FORMAT 4 +#ifndef SQLITE4_DEFAULT_FILE_FORMAT +# define SQLITE4_DEFAULT_FILE_FORMAT 4 +#endif + +/* +** Determine whether triggers are recursive by default. This can be +** changed at run-time using a pragma. +*/ +#ifndef SQLITE4_DEFAULT_RECURSIVE_TRIGGERS +# define SQLITE4_DEFAULT_RECURSIVE_TRIGGERS 0 +#endif + /* ** Provide a default value for SQLITE4_TEMP_STORE in case it is not specified ** on the command-line */ #ifndef SQLITE4_TEMP_STORE @@ -462,29 +502,26 @@ ** These macros are designed to work correctly on both 32-bit and 64-bit ** compilers. */ #define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) #define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) -#define LARGEST_UINT64 (0xffffffff|(((u64)0xffffffff)<<32)) +#define LARGEST_UINT64 (0xffffffff|(((i64)0xffffffff)<<32)) /* ** Round up a number to the next larger multiple of 8. This is used ** to force 8-byte alignment on 64-bit architectures. */ #define ROUND8(x) (((x)+7)&~7) -/* -** Round down to the nearest multiple of 8 -*/ -#define ROUNDDOWN8(x) ((x)&~7) - -/* -** Min and max macros. -*/ #define SQLITE4_MIN(a,b) (((a)<(b)) ? (a) : (b)) #define SQLITE4_MAX(a,b) (((a)>(b)) ? (a) : (b)) +/* +** Round down to the nearest multiple of 8 +*/ +#define ROUNDDOWN8(x) ((x)&~7) + /* ** Assert that the pointer X is aligned to an 8-byte boundary. This ** macro is used only within assert() to verify that the code gets ** all alignment restrictions correct. ** @@ -550,11 +587,10 @@ typedef struct AggInfoFunc AggInfoFunc; typedef struct AuthContext AuthContext; typedef struct AutoincInfo AutoincInfo; typedef struct CollSeq CollSeq; typedef struct Column Column; -typedef struct CreateIndex CreateIndex; typedef struct Db Db; typedef struct Schema Schema; typedef struct Expr Expr; typedef struct ExprList ExprList; typedef struct ExprListItem ExprListItem; @@ -561,14 +597,10 @@ typedef struct ExprSpan ExprSpan; typedef struct FKey FKey; typedef struct FuncDestructor FuncDestructor; typedef struct FuncDef FuncDef; typedef struct FuncDefTable FuncDefTable; -typedef struct Fts5Tokenizer Fts5Tokenizer; -typedef struct Fts5Index Fts5Index; -typedef struct Fts5Info Fts5Info; -typedef struct Fts5Cursor Fts5Cursor; typedef struct IdList IdList; typedef struct IdListItem IdListItem; typedef struct Index Index; typedef struct IndexSample IndexSample; typedef struct KeyClass KeyClass; @@ -599,11 +631,11 @@ typedef struct WhereInfo WhereInfo; typedef struct WhereLevel WhereLevel; #include "vdbe.h" -#include "kv.h" +#include "storage.h" #include "os.h" #include "mutex.h" @@ -638,11 +670,10 @@ void (*xStep)(sqlite4_context*,int,sqlite4_value**); /* Aggregate step */ void (*xFinalize)(sqlite4_context*); /* Aggregate finalizer */ char *zName; /* SQL name of the function. */ FuncDef *pNextName; /* Next function with a different name */ FuncDestructor *pDestructor; /* Reference counted destructor function */ - u8 bMatchinfo; /* True for matchinfo function */ }; /* ** A table of SQL functions. ** @@ -876,14 +907,10 @@ sqlite4 *pUnlockConnection; /* Connection to watch for unlock */ void *pUnlockArg; /* Argument to xUnlockNotify */ void (*xUnlockNotify)(void **, int); /* Unlock notify callback */ sqlite4 *pNextBlocked; /* Next in list of all blocked connections */ #endif - -#ifndef SQLITE_OMIT_FTS5 - Fts5Tokenizer *pTokenizer; /* First in list of tokenizers */ -#endif }; /* ** A macro to discover the encoding of a database. */ @@ -1183,10 +1210,36 @@ }; /* ** Each SQL table is represented in memory by an instance of the ** following structure. +** +** Table.zName is the name of the table. The case of the original +** CREATE TABLE statement is stored, but case is not significant for +** comparisons. +** +** Table.nCol is the number of columns in this table. Table.aCol is a +** pointer to an array of Column structures, one for each column. +** +** If the table has an INTEGER PRIMARY KEY, then Table.iPKey is the index of +** the column that is that key. Otherwise Table.iPKey is negative. Note +** that the datatype of the PRIMARY KEY must be INTEGER for this field to +** be set. An INTEGER PRIMARY KEY is used as the rowid for each row of +** the table. If a table has no INTEGER PRIMARY KEY, then a random rowid +** is generated for each row of the table. TF_HasPrimaryKey is set if +** the table has any PRIMARY KEY, INTEGER or otherwise. +** +** Table.tnum is the page number for the root BTree page of the table in the +** database file. If Table.iDb is the index of the database table backend +** in sqlite.aDb[]. 0 is for the main database and 1 is for the file that +** holds temporary tables and indices. If TF_Ephemeral is set +** then the table is stored in a file that is automatically deleted +** when the VDBE cursor to the table is closed. In this case Table.tnum +** refers VDBE cursor number that holds the table open, not to the root +** page number. Transient tables are used to hold the results of a +** sub-query that appears instead of a real table name in the FROM clause +** of a SELECT statement. */ struct Table { char *zName; /* Name of the table or view */ int nCol; /* Number of columns in this table */ Column *aCol; /* Information about each column */ @@ -1398,11 +1451,11 @@ tRowcnt *aiRowEst; /* Result of ANALYZE: Est. rows selected by each column */ Table *pTable; /* The SQL table being indexed */ int tnum; /* Page containing root of this index in database file */ u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ u8 eIndexType; /* SQLITE4_INDEX_USER, UNIQUE or PRIMARYKEY */ - u8 fIndex; /* One or more of the IDX_* flags below */ + u8 bUnordered; /* Use this index for == or IN queries only */ char *zColAff; /* String defining the affinity of each column */ Index *pNext; /* The next index associated with the same table */ Schema *pSchema; /* Schema containing this index */ u8 *aSortOrder; /* Array of size Index.nColumn. True==DESC, False==ASC */ char **azColl; /* Array of collation sequence names for index */ @@ -1409,23 +1462,17 @@ #ifdef SQLITE4_ENABLE_STAT3 int nSample; /* Number of elements in aSample[] */ tRowcnt avgEq; /* Average nEq value for key values not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ #endif - Fts5Index *pFts; /* Fts5 data (or NULL if this is not an fts index) */ }; /* Index.eIndexType must be set to one of the following. */ #define SQLITE4_INDEX_USER 0 /* Index created by CREATE INDEX statement */ #define SQLITE4_INDEX_UNIQUE 1 /* Index created by UNIQUE constraint */ #define SQLITE4_INDEX_PRIMARYKEY 2 /* Index is the tables PRIMARY KEY */ -#define SQLITE4_INDEX_FTS5 3 /* Index is an FTS5 index */ -#define SQLITE4_INDEX_TEMP 4 /* Index is an automatic index */ - -/* Allowed values for Index.fIndex */ -#define IDX_IntPK 0x01 /* An INTEGER PRIMARY KEY index */ -#define IDX_Unordered 0x02 /* Implemented as a hashing index */ +#define SQLITE4_INDEX_TEMP 3 /* Index is an automatic index */ /* ** Each sample stored in the sqlite_stat3 table is represented in memory ** using a structure of this type. See documentation at the top of the ** analyze.c source file for additional information. @@ -1454,24 +1501,10 @@ struct Token { const char *z; /* Text of the token. Not NULL-terminated! */ unsigned int n; /* Number of characters in this token */ }; -/* -** An instance of this structure holds the results of parsing the first -** part of a CREATE INDEX statement. Instances exist only transiently -** during parsing. -*/ -struct CreateIndex { - int bUnique; /* True if the UNIQUE keyword was present */ - int bIfnotexist; /* True if IF NOT EXISTS was present */ - Token tCreate; /* CREATE token */ - Token tName1; /* First part of two part name */ - Token tName2; /* Second part of two part name */ - SrcList *pTblName; /* Table index is created on */ -}; - /* ** One for each column used in source tables. */ struct AggInfoCol { Table *pTab; /* Source table */ @@ -1639,11 +1672,10 @@ i16 iRightJoinTable; /* If EP_FromJoin, the right table of the join */ u8 flags2; /* Second set of flags. EP2_... */ u8 op2; /* If a TK_REGISTER, the original value of Expr.op */ AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ Table *pTab; /* Table for TK_COLUMN expressions. */ - Index *pIdx; /* Fts index used by MATCH expressions */ #if SQLITE4_MAX_EXPR_DEPTH>0 int nHeight; /* Height of the tree headed by this node */ #endif }; @@ -1654,10 +1686,11 @@ #define EP_Agg 0x0002 /* Contains one or more aggregate functions */ #define EP_Resolved 0x0004 /* IDs have been resolved to COLUMNs */ #define EP_Error 0x0008 /* Expression contains one or more errors */ #define EP_Distinct 0x0010 /* Aggregate function with DISTINCT keyword */ #define EP_VarSelect 0x0020 /* pSelect is correlated, not constant */ +#define EP_DblQuoted 0x0040 /* token.z was originally in "..." */ #define EP_InfixFunc 0x0080 /* True for an infix function: LIKE, GLOB, etc */ #define EP_ExpCollate 0x0100 /* Collating sequence specified explicitly */ #define EP_FixedDest 0x0200 /* Result needed in a specific register */ #define EP_IntValue 0x0400 /* Integer value contained in u.iValue */ #define EP_xIsSelect 0x0800 /* x.pSelect is valid (otherwise x.pList is) */ @@ -2044,11 +2077,10 @@ #define SF_Aggregate 0x04 /* Contains aggregate functions */ #define SF_UsesEphemeral 0x08 /* Uses the OpenEphemeral opcode */ #define SF_Expanded 0x10 /* sqlite4SelectExpand() called on this */ #define SF_HasTypeInfo 0x20 /* FROM subqueries have Table metadata */ #define SF_UseSorter 0x40 /* Sort using a sorter */ -#define SF_Values 0x80 /* Synthesized from VALUES clause */ /* ** The results of a select can be distributed in several ways. The ** "SRT" prefix means "SELECT Result Type". @@ -2260,11 +2292,11 @@ */ #define OPFLAG_NCHANGE 0x01 /* Set to update db->nChange */ #define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ #define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */ #define OPFLAG_APPEND 0x08 /* This is likely to be an append */ -#define OPFLAG_SEQCOUNT 0x10 /* Append sequence number to key */ +#define OPFLAG_USESEEKRESULT 0x10 /* Try to avoid a seek on insert */ #define OPFLAG_CLEARCACHE 0x20 /* Clear pseudo-table cache in OP_Column */ #define OPFLAG_APPENDBIAS 0x40 /* Bias inserts for appending */ /* * Each trigger present in the database schema is stored as an instance of @@ -2395,18 +2427,23 @@ int iDb; /* 0 for main database. 1 for TEMP, 2.. for ATTACHed */ char **pzErrMsg; /* Error message stored here */ int rc; /* Result code stored here */ } InitData; +/* +** FIXME: missing docs +*/ +typedef int (*KVFactory_factory_f)(sqlite4_env*,sqlite4_kvstore**,const char*,unsigned); + /* ** A pluggable storage engine */ typedef struct KVFactory { - struct KVFactory *pNext; /* Next in list of all storage engines */ - const char *zName; /* Name of this factory */ - sqlite4_kvfactory xFactory; /* Function to make an sqlite4_kvstore obj */ - int isPerm; /* True if a built-in. Cannot be popped */ + struct KVFactory *pNext; /* Next in list of them all */ + const char *zName; /* Name of this factory */ + KVFactory_factory_f xFactory; + int isPerm; /* True if a built-in. Cannot be popped */ } KVFactory; /* ** An instance of this structure defines the run-time environment. */ @@ -2454,24 +2491,11 @@ NameContext *pNC; /* Naming context */ int i; /* Integer value */ } u; }; -/* -** An instance of this structure is used as the p4 argument to some fts5 -** related vdbe opcodes. -*/ -struct Fts5Info { - int iDb; /* Database containing this index */ - int iRoot; /* Root page number of index */ - int iTbl; /* Root page number of indexed table */ - int nCol; /* Number of columns in indexed table */ - char **azCol; /* Column names for table */ - Fts5Tokenizer *pTokenizer; /* Tokenizer module */ - sqlite4_tokenizer *p; /* Tokenizer instance */ -}; - +/* Forward declarations */ int sqlite4WalkExpr(Walker*, Expr*); int sqlite4WalkExprList(Walker*, ExprList*); int sqlite4WalkSelect(Walker*, Select*); int sqlite4WalkSelectExpr(Walker*, Select*); int sqlite4WalkSelectFrom(Walker*, Select*); @@ -2551,11 +2575,13 @@ #endif /* ** Internal function prototypes */ +int sqlite4StrICmp(const char *, const char *); int sqlite4Strlen30(const char*); +#define sqlite4StrNICmp sqlite4_strnicmp int sqlite4MallocInit(sqlite4_env*); void sqlite4MallocEnd(sqlite4_env*); void *sqlite4Malloc(sqlite4_env*, int); void *sqlite4MallocZero(sqlite4_env*, int); @@ -2964,11 +2990,11 @@ u8 sqlite4GetBoolean(const char *z); const void *sqlite4ValueText(sqlite4_value*, u8); int sqlite4ValueBytes(sqlite4_value*, u8); void sqlite4ValueSetStr(sqlite4_value*, int, const void *,u8, - void(*)(void*,void*),void*); + void(*)(void*)); void sqlite4ValueFree(sqlite4_value*); sqlite4_value *sqlite4ValueNew(sqlite4 *); char *sqlite4Utf16to8(sqlite4 *, const void*, int, u8); #ifdef SQLITE4_ENABLE_STAT3 char *sqlite4Utf8to16(sqlite4 *, u8, char *, int, int *); @@ -3234,30 +3260,6 @@ #define MEMTYPE_HEAP 0x01 /* General heap allocations */ #define MEMTYPE_LOOKASIDE 0x02 /* Might have been lookaside memory */ #define MEMTYPE_SCRATCH 0x04 /* Scratch allocations */ #define MEMTYPE_DB 0x10 /* Uses sqlite4DbMalloc, not sqlite_malloc */ -int sqlite4InitFts5(sqlite4 *db); -int sqlite4InitFts5Func(sqlite4 *db); -void sqlite4ShutdownFts5(sqlite4 *db); -void sqlite4CreateUsingIndex(Parse*, CreateIndex*, ExprList*, Token*, Token*); - -int sqlite4Fts5IndexSz(void); -void sqlite4Fts5IndexInit(Parse *, Index *, ExprList *); -void sqlite4Fts5IndexFree(sqlite4 *, Index *); - -int sqlite4Fts5Update(sqlite4 *, Fts5Info *, int, Mem *, Mem *, int, char **); -void sqlite4Fts5FreeInfo(sqlite4 *db, Fts5Info *); -void sqlite4Fts5CodeUpdate(Parse *, Index *pIdx, int, int, int, int); -void sqlite4Fts5CodeCksum(Parse *, Index *, int, int, int); -void sqlite4Fts5CodeQuery(Parse *, Index *, int, int, int); - -int sqlite4Fts5Pk(Fts5Cursor *, int, KVByteArray **, KVSize *); -int sqlite4Fts5Next(Fts5Cursor *pCsr); - -int sqlite4Fts5EntryCksum(sqlite4 *, Fts5Info *, Mem *, Mem *, i64 *); -int sqlite4Fts5RowCksum(sqlite4 *, Fts5Info *, Mem *, Mem *, i64 *); -int sqlite4Fts5Open(sqlite4*, Fts5Info*, const char*, int, Fts5Cursor**,char**); -int sqlite4Fts5Valid(Fts5Cursor *); -void sqlite4Fts5Close(Fts5Cursor *); - #endif /* _SQLITEINT_H_ */ ADDED src/storage.c Index: src/storage.c ================================================================== --- /dev/null +++ src/storage.c @@ -0,0 +1,496 @@ +/* +** 2012 January 21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** General wrapper functions around the various KV storage engine +** implementations. It also implements tracing of calls to the KV +** engine and some higher-level ensembles of the low-level storage +** calls. +*/ +#include "sqliteInt.h" + +/* +** Names of error codes used for tracing. +*/ +static const char *kvErrName(int e){ + const char *zName; + switch( e ){ + case SQLITE4_OK: zName = "OK"; break; + case SQLITE4_ERROR: zName = "ERROR"; break; + case SQLITE4_INTERNAL: zName = "INTERNAL"; break; + case SQLITE4_PERM: zName = "PERM"; break; + case SQLITE4_ABORT: zName = "ABORT"; break; + case SQLITE4_BUSY: zName = "BUSY"; break; + case SQLITE4_LOCKED: zName = "LOCKED"; break; + case SQLITE4_NOMEM: zName = "NOMEM"; break; + case SQLITE4_READONLY: zName = "READONLY"; break; + case SQLITE4_INTERRUPT: zName = "INTERRUPT"; break; + case SQLITE4_IOERR: zName = "IOERR"; break; + case SQLITE4_CORRUPT: zName = "CORRUPT"; break; + case SQLITE4_NOTFOUND: zName = "NOTFOUND"; break; + case SQLITE4_FULL: zName = "FULL"; break; + case SQLITE4_CANTOPEN: zName = "CANTOPEN"; break; + case SQLITE4_PROTOCOL: zName = "PROTOCOL"; break; + case SQLITE4_EMPTY: zName = "EMPTY"; break; + case SQLITE4_SCHEMA: zName = "SCHEMA"; break; + case SQLITE4_TOOBIG: zName = "TOOBIG"; break; + case SQLITE4_CONSTRAINT: zName = "CONSTRAINT"; break; + case SQLITE4_MISMATCH: zName = "MISMATCH"; break; + case SQLITE4_MISUSE: zName = "MISUSE"; break; + case SQLITE4_NOLFS: zName = "NOLFS"; break; + case SQLITE4_AUTH: zName = "AUTH"; break; + case SQLITE4_FORMAT: zName = "FORMAT"; break; + case SQLITE4_RANGE: zName = "RANGE"; break; + case SQLITE4_NOTADB: zName = "NOTADB"; break; + case SQLITE4_ROW: zName = "ROW"; break; + case SQLITE4_DONE: zName = "DONE"; break; + case SQLITE4_INEXACT: zName = "INEXACT"; break; + default: zName = "???"; break; + } + return zName; +} + +/* +** Do any requested tracing +*/ +static void kvTrace(KVStore *p, const char *zFormat, ...){ + if( p->fTrace ){ + va_list ap; + char *z; + + va_start(ap, zFormat); + z = sqlite4_vmprintf(p->pEnv, zFormat, ap); + va_end(ap); + printf("%s.%s\n", p->zKVName, z); + fflush(stdout); + sqlite4_free(p->pEnv, z); + } +} + +/* +** Open a storage engine via URI +*/ +int sqlite4KVStoreOpen( + sqlite4 *db, /* The database connection doing the open */ + const char *zName, /* Symbolic name for this database */ + const char *zUri, /* URI for this database */ + KVStore **ppKVStore, /* Write the new KVStore object here */ + unsigned flags /* Option flags */ +){ + KVStore *pNew = 0; + int rc; + sqlite4_env *pEnv = &sqlite4DefaultEnv; /* OR db->pEnv */ + const char *zStorageName; + KVFactory *pMkr; + int (*xFactory)(sqlite4_env*,sqlite4_kvstore**,const char*,unsigned); + + if( (flags & SQLITE4_KVOPEN_TEMPORARY)!=0 || zUri==0 || zUri[0]==0 ){ + zStorageName = "temp"; + }else{ + zStorageName = sqlite4_uri_parameter(zName, "kv"); + if( zStorageName==0 ){ + if( memcmp(":memory:", zUri, 8)==0 ){ + zStorageName = "temp"; + }else{ + zStorageName = "main"; + } + } + } + *ppKVStore = 0; + sqlite4_mutex_enter(pEnv->pFactoryMutex); + for(pMkr=pEnv->pFactory; pMkr && strcmp(zStorageName,pMkr->zName); + pMkr=pMkr->pNext){} + xFactory = pMkr ? pMkr->xFactory : 0; + sqlite4_mutex_leave(pEnv->pFactoryMutex); + if( xFactory==0 ){ + return SQLITE4_ERROR; + } + rc = xFactory(pEnv, &pNew, zUri, flags); + *ppKVStore = pNew; + if( pNew ){ + sqlite4_randomness(pEnv, sizeof(pNew->kvId), &pNew->kvId); + sqlite4_snprintf(pNew->zKVName, sizeof(pNew->zKVName), + "%s", zName); + pNew->fTrace = (db->flags & SQLITE4_KvTrace)!=0; + kvTrace(pNew, "open(%s,%d,0x%04x)", zUri, pNew->kvId, flags); + } + return rc; +} + +/* Convert binary data to hex for display in trace messages */ +static void binToHex(char *zOut, int mxOut, const KVByteArray *a, KVSize n){ + int i; + if( n>mxOut/2-1 ) n = mxOut/2-1; + for(i=0; i>4)&0xf]; + zOut[i*2+1] = "0123456789abcdef"[a[i]&0xf]; + } + zOut[i*2] = 0; +} + +/* +** The following wrapper functions invoke the underlying methods of +** the storage object and add optional tracing. +*/ +int sqlite4KVStoreReplace( + KVStore *p, + const KVByteArray *pKey, KVSize nKey, + const KVByteArray *pData, KVSize nData +){ + if( p->fTrace ){ + char zKey[52], zData[52]; + binToHex(zKey, sizeof(zKey), pKey, nKey); + binToHex(zData, sizeof(zData), pData, nData); + kvTrace(p, "xReplace(%d,%s,%d,%s,%d)", + p->kvId, zKey, (int)nKey, zData, (int)nData); + } + return p->pStoreVfunc->xReplace(p,pKey,nKey,pData,nData); +} +int sqlite4KVStoreOpenCursor(KVStore *p, KVCursor **ppKVCursor){ + KVCursor *pCur; + int rc; + + rc = p->pStoreVfunc->xOpenCursor(p, &pCur); + *ppKVCursor = pCur; + if( pCur ){ + sqlite4_randomness(pCur->pEnv, sizeof(pCur->curId), &pCur->curId); + pCur->fTrace = p->fTrace; + pCur->pStore = p; + } + kvTrace(p, "xOpenCursor(%d,%d) -> %s", + p->kvId, pCur?pCur->curId:-1, kvErrName(rc)); + return rc; +} +int sqlite4KVCursorSeek( + KVCursor *p, + const KVByteArray *pKey, KVSize nKey, + int dir +){ + int rc; + assert( dir==0 || dir==(+1) || dir==(-1) || dir==(-2) ); + rc = p->pStoreVfunc->xSeek(p,pKey,nKey,dir); + if( p->fTrace ){ + char zKey[52]; + binToHex(zKey, sizeof(zKey), pKey, nKey); + kvTrace(p->pStore, "xSeek(%d,%s,%d,%d) -> %s", + p->curId, zKey, (int)nKey, dir, kvErrName(rc)); + } + return rc; +} +int sqlite4KVCursorNext(KVCursor *p){ + int rc; + rc = p->pStoreVfunc->xNext(p); + kvTrace(p->pStore, "xNext(%d) -> %s", p->curId, kvErrName(rc)); + return rc; +} +int sqlite4KVCursorPrev(KVCursor *p){ + int rc; + rc = p->pStoreVfunc->xPrev(p); + kvTrace(p->pStore, "xPrev(%d) -> %s", p->curId, kvErrName(rc)); + return rc; +} +int sqlite4KVCursorDelete(KVCursor *p){ + int rc; + rc = p->pStoreVfunc->xDelete(p); + kvTrace(p->pStore, "xDelete(%d) -> %s", p->curId, kvErrName(rc)); + return rc; +} +int sqlite4KVCursorReset(KVCursor *p){ + int rc; + rc = p->pStoreVfunc->xReset(p); + kvTrace(p->pStore, "xReset(%d) -> %s", p->curId, kvErrName(rc)); + return rc; +} +int sqlite4KVCursorKey(KVCursor *p, const KVByteArray **ppKey, KVSize *pnKey){ + int rc; + rc = p->pStoreVfunc->xKey(p, ppKey, pnKey); + if( p->fTrace ){ + if( rc==SQLITE4_OK ){ + char zKey[52]; + binToHex(zKey, sizeof(zKey), *ppKey, *pnKey); + kvTrace(p->pStore, "xKey(%d,%s,%d)", p->curId, zKey, (int)*pnKey); + }else{ + kvTrace(p->pStore, "xKey(%d,)", p->curId, rc); + } + } + return rc; +} +int sqlite4KVCursorData( + KVCursor *p, + KVSize ofst, + KVSize n, + const KVByteArray **ppData, + KVSize *pnData +){ + int rc; + rc = p->pStoreVfunc->xData(p, ofst, n, ppData, pnData); + if( p->fTrace ){ + if( rc==SQLITE4_OK ){ + char zData[52]; + binToHex(zData, sizeof(zData), *ppData, *pnData); + kvTrace(p->pStore, "xData(%d,%d,%d,%s,%d)", + p->curId, (int)ofst, (int)n, zData, (int)*pnData); + }else{ + kvTrace(p->pStore, "xData(%d,%d,%d,)", + p->curId, (int)ofst, (int)n, rc); + } + } + return rc; +} +int sqlite4KVCursorClose(KVCursor *p){ + int rc = SQLITE4_OK; + if( p ){ + KVStore *pStore = p->pStore; + int curId = p->curId; + rc = p->pStoreVfunc->xCloseCursor(p); + kvTrace(pStore, "xCloseCursor(%d) -> %s", curId, kvErrName(rc)); + } + return rc; +} +int sqlite4KVStoreBegin(KVStore *p, int iLevel){ + int rc; + rc = p->pStoreVfunc->xBegin(p, iLevel); + kvTrace(p, "xBegin(%d,%d) -> %s", p->kvId, iLevel, kvErrName(rc)); + assert( p->iTransLevel==iLevel || rc!=SQLITE4_OK ); + return rc; +} +int sqlite4KVStoreCommitPhaseOne(KVStore *p, int iLevel){ + int rc; + assert( iLevel>=0 ); + assert( iLevel<=p->iTransLevel ); + if( p->iTransLevel==iLevel ) return SQLITE4_OK; + if( p->pStoreVfunc->xCommitPhaseOne ){ + rc = p->pStoreVfunc->xCommitPhaseOne(p, iLevel); + }else{ + rc = SQLITE4_OK; + } + kvTrace(p, "xCommitPhaseOne(%d,%d) -> %s", p->kvId, iLevel, kvErrName(rc)); + assert( p->iTransLevel>iLevel ); + return rc; +} +int sqlite4KVStoreCommitPhaseTwo(KVStore *p, int iLevel){ + int rc; + assert( iLevel>=0 ); + assert( iLevel<=p->iTransLevel ); + if( p->iTransLevel==iLevel ) return SQLITE4_OK; + rc = p->pStoreVfunc->xCommitPhaseTwo(p, iLevel); + kvTrace(p, "xCommitPhaseTwo(%d,%d) -> %s", p->kvId, iLevel, kvErrName(rc)); + assert( p->iTransLevel==iLevel || rc!=SQLITE4_OK ); + return rc; +} +int sqlite4KVStoreCommit(KVStore *p, int iLevel){ + int rc; + rc = sqlite4KVStoreCommitPhaseOne(p, iLevel); + if( rc==SQLITE4_OK ) rc = sqlite4KVStoreCommitPhaseTwo(p, iLevel); + return rc; +} +int sqlite4KVStoreRollback(KVStore *p, int iLevel){ + int rc; + assert( iLevel>=0 ); + assert( iLevel<=p->iTransLevel ); + rc = p->pStoreVfunc->xRollback(p, iLevel); + kvTrace(p, "xRollback(%d,%d) -> %s", p->kvId, iLevel, kvErrName(rc)); + assert( p->iTransLevel==iLevel || rc!=SQLITE4_OK ); + return rc; +} +int sqlite4KVStoreRevert(KVStore *p, int iLevel){ + int rc; + assert( iLevel>0 ); + assert( iLevel<=p->iTransLevel ); + if( p->pStoreVfunc->xRevert ){ + rc = p->pStoreVfunc->xRevert(p, iLevel); + kvTrace(p, "xRevert(%d,%d) -> %s", p->kvId, iLevel, kvErrName(rc)); + }else{ + rc = sqlite4KVStoreRollback(p, iLevel-1); + if( rc==SQLITE4_OK ){ + rc = sqlite4KVStoreBegin(p, iLevel); + } + } + assert( p->iTransLevel==iLevel || rc!=SQLITE4_OK ); + return rc; +} +int sqlite4KVStoreClose(KVStore *p){ + int rc; + if( p ){ + kvTrace(p, "xClose(%d)", p->kvId); + rc = p->pStoreVfunc->xClose(p); + } + return rc; +} + +/* +** Key for the meta-data +*/ +static const KVByteArray metadataKey[] = { 0x00, 0x00 }; + +static void writeMetaArray(KVByteArray *aMeta, int iElem, u32 iVal){ + int i = sizeof(u32) * iElem; + aMeta[i+0] = (iVal>>24)&0xff; + aMeta[i+1] = (iVal>>16)&0xff; + aMeta[i+2] = (iVal>>8) &0xff; + aMeta[i+3] = (iVal>>0) &0xff; +} + +/* +** Read nMeta unsigned 32-bit integers of metadata beginning at iStart. +*/ +int sqlite4KVStoreGetMeta(KVStore *p, int iStart, int nMeta, unsigned int *a){ + KVCursor *pCur; + int rc; + int i, j; + KVSize nData; + const KVByteArray *aData; + + rc = sqlite4KVStoreOpenCursor(p, &pCur); + if( rc==SQLITE4_OK ){ + rc = sqlite4KVCursorSeek(pCur, metadataKey, sizeof(metadataKey), 0); + if( rc==SQLITE4_NOTFOUND ){ + rc = SQLITE4_OK; + nData = 0; + }else if( rc==SQLITE4_OK ){ + rc = sqlite4KVCursorData(pCur, 0, -1, &aData, &nData); + } + if( rc==SQLITE4_OK ){ + i = 0; + j = iStart*4; + while( i=0 ){ + for(i=0; i<16 && i>4]; + zOut[i*3+1] = base16[v&0xf]; + zOut[16*3+3+i] = (v>=0x20 && v<=0x7e) ? v : '.'; + } + while( i<16 ){ + zOut[i*3] = ' '; + zOut[i*3+1] = ' '; + zOut[16*3+3+i] = ' '; + i++; + } + sqlite4DebugPrintf("%.3s %s\n", zPrefix, zOut); + n -= 16; + if( n<=0 ) break; + a += 16; + zPrefix = " "; + } +} + +/* +** Dump the entire content of a key-value database +*/ +void sqlite4KVStoreDump(KVStore *pStore){ + int rc; + int nRow = 0; + KVCursor *pCur; + KVSize nKey, nData; + KVByteArray const *aKey; + KVByteArray const *aData; + static const KVByteArray aProbe[] = { 0x00 }; + + rc = sqlite4KVStoreOpenCursor(pStore, &pCur); + if( rc==SQLITE4_OK ){ + rc = sqlite4KVCursorSeek(pCur, aProbe, 1, +1); + while( rc!=SQLITE4_NOTFOUND ){ + rc = sqlite4KVCursorKey(pCur, &aKey, &nKey); + if( rc ) break; + if( nRow>0 ) sqlite4DebugPrintf("\n"); + nRow++; + outputBinary(aKey, nKey, "K: "); + rc = sqlite4KVCursorData(pCur, 0, -1, &aData, &nData); + if( rc ) break; + outputBinary(aData, nData, "V: "); + rc = sqlite4KVCursorNext(pCur); + } + sqlite4KVCursorClose(pCur); + } +} +#endif /* SQLITE4_DEBUG */ ADDED src/storage.h Index: src/storage.h ================================================================== --- /dev/null +++ src/storage.h @@ -0,0 +1,179 @@ +/* +** 2012 January 20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This header file defines the interface to the KV storage engine(s). +** +** Notes on the storage subsystem interface: +** +** The storage subsystem is a key/value database. All keys and values are +** binary with arbitrary content. Keys are unique. Keys compare in +** memcmp() order. Shorter keys appear first. +** +** The xBegin, xCommit, and xRollback methods change the transaction level +** of the store. The transaction level is a non-negative integer that is +** initialized to zero. The transaction level must be at least 1 in order +** for content to be read. The transaction level must be at least 2 for +** content to be modified. +** +** The xBegin method increases transaction level. The increase may be no +** more than 1 unless the transaction level is initially 0 in which case +** it can be increased immediately to 2. Increasing the transaction level +** to 1 or more makes a "snapshot" of the database file such that changes +** made by other connections are not visible. An xBegin call may fail +** with SQLITE4_BUSY if the initial transaction level is 0 or 1. +** +** A read-only database will fail an attempt to increase xBegin above 1. An +** implementation that does not support nested transactions will fail any +** attempt to increase the transaction level above 2. +** +** The xCommitPhaseOne and xCommitPhaseTwo methods implement a 2-phase +** commit that lowers the transaction level to the value given in the +** second argument, making all the changes made at higher transaction levels +** permanent. A rollback is still possible following phase one. If +** possible, errors should be reported during phase one so that a +** multiple-database transaction can still be rolled back if the +** phase one fails on a different database. Implementations that do not +** support two-phase commit can implement xCommitPhaseOne as a no-op function +** returning SQLITE4_OK. +** +** The xRollback method lowers the transaction level to the value given in +** its argument and reverts or undoes all changes made at higher transaction +** levels. An xRollback to level N causes the database to revert to the state +** it was in on the most recent xBegin to level N+1. +** +** The xRevert(N) method causes the state of the database file to go back +** to what it was immediately after the most recent xCommit(N). Higher-level +** subtransactions are cancelled. This call is equivalent to xRollback(N-1) +** followed by xBegin(N) but is atomic and might be more efficient. +** +** The xReplace method replaces the value for an existing entry with the +** given key, or creates a new entry with the given key and value if no +** prior entry exists with the given key. The key and value pointers passed +** into xReplace belong to the caller will likely be destroyed when the +** call to xReplace returns so the xReplace routine must make its own +** copy of that information. +** +** A cursor is at all times pointing to ether an entry in the database or +** to EOF. EOF means "no entry". Cursor operations other than xCloseCursor +** will fail if the transaction level is less than 1. +** +** The xSeek method moves a cursor to an entry in the database that matches +** the supplied key as closely as possible. If the dir argument is 0, then +** the match must be exact or else the seek fails and the cursor is left +** pointing to EOF. If dir is negative, then an exact match is +** found if it is available, otherwise the cursor is positioned at the largest +** entry that is less than the search key or to EOF if the store contains no +** entry less than the search key. If dir is positive, then an exist match +** is found if it is available, otherwise the cursor is left pointing the +** the smallest entry that is larger than the search key, or to EOF if there +** are no entries larger than the search key. +** +** The return code from xSeek might be one of the following: +** +** SQLITE4_OK The cursor is left pointing to any entry that +** exactly matchings the probe key. +** +** SQLITE4_INEXACT The cursor is left pointing to the nearest entry +** to the probe it could find, either before or after +** the probe, according to the dir argument. +** +** SQLITE4_NOTFOUND No suitable entry could be found. Either dir==0 and +** there was no exact match, or dir<0 and the probe is +** smaller than every entry in the database, or dir>0 and +** the probe is larger than every entry in the database. +** +** xSeek might also return some error code like SQLITE4_IOERR or +** SQLITE4_NOMEM. +** +** The xNext method will only be called following an xSeek with a positive dir, +** or another xNext. The xPrev method will only be called following an xSeek +** with a negative dir or another xPrev. Both xNext and xPrev will return +** SQLITE4_OK on success and SQLITE4_NOTFOUND if they run off the end of the +** database. Both routines might also return error codes such as +** SQLITE4_IOERR, SQLITE4_CORRUPT, or SQLITE4_NOMEM. +** +** Values returned by xKey and xData are guaranteed to remain stable until +** the next xSeek, xNext, xPrev, xReset, xDelete, or xCloseCursor on the same +** cursor. This is true even if the transaction level is reduced to zero, +** or if the content of the entry is changed by xInsert, xDelete on a different +** cursor, or xRollback. The content returned by repeated calls to xKey and +** xData is allowed (but is not required) to change if xInsert, xDelete, or +** xRollback are invoked in between the calls, but the content returned by +** every call must be stable until the cursor moves, or is reset or closed. +** The cursor owns the values returned by xKey and xData and will take +** responsiblity for freeing memory used to hold those values when appropriate. +** +** The xDelete method deletes the entry that the cursor is currently +** pointing at. However, subsequent xNext or xPrev calls behave as if the +** entries is not actually deleted until the cursor moves. In other words +** it is acceptable to xDelete an entry out from under a cursor. Subsequent +** xNext or xPrev calls on that cursor will work the same as if the entry +** had not been deleted. Two cursors can be pointing to the same entry and +** one cursor can xDelete and the other cursor is expected to continue +** functioning normally, including responding correctly to subsequent +** xNext and xPrev calls. +*/ + +/* Typedefs of datatypes */ +typedef struct sqlite4_kvstore KVStore; +typedef struct sqlite4_kv_methods KVStoreMethods; +typedef struct sqlite4_kvcursor KVCursor; +typedef unsigned char KVByteArray; +typedef sqlite4_kvsize KVSize; + + +int sqlite4KVStoreOpenMem(sqlite4_env*, KVStore**, const char *, unsigned); +int sqlite4KVStoreOpenLsm(sqlite4_env*, KVStore**, const char *, unsigned); +int sqlite4KVStoreOpen( + sqlite4*, + const char *zLabel, + const char *zUri, + KVStore**, + unsigned flags +); +int sqlite4KVStoreReplace( + KVStore*, + const KVByteArray *pKey, KVSize nKey, + const KVByteArray *pData, KVSize nData +); +int sqlite4KVStoreOpenCursor(KVStore *p, KVCursor **ppKVCursor); +int sqlite4KVCursorSeek( + KVCursor *p, + const KVByteArray *pKey, KVSize nKey, + int dir +); +int sqlite4KVCursorNext(KVCursor *p); +int sqlite4KVCursorPrev(KVCursor *p); +int sqlite4KVCursorDelete(KVCursor *p); +int sqlite4KVCursorReset(KVCursor *p); +int sqlite4KVCursorKey(KVCursor *p, const KVByteArray **ppKey, KVSize *pnKey); +int sqlite4KVCursorData( + KVCursor *p, + KVSize ofst, + KVSize n, + const KVByteArray **ppData, + KVSize *pnData +); +int sqlite4KVCursorClose(KVCursor *p); +int sqlite4KVStoreBegin(KVStore *p, int iLevel); +int sqlite4KVStoreCommitPhaseOne(KVStore *p, int iLevel); +int sqlite4KVStoreCommitPhaseTwo(KVStore *p, int iLevel); +int sqlite4KVStoreCommit(KVStore *p, int iLevel); +int sqlite4KVStoreRollback(KVStore *p, int iLevel); +int sqlite4KVStoreRevert(KVStore *p, int iLevel); +int sqlite4KVStoreClose(KVStore *p); + +int sqlite4KVStoreGetMeta(KVStore *p, int, int, unsigned int*); +int sqlite4KVStorePutMeta(sqlite4*, KVStore *p, int, int, unsigned int*); +#ifdef SQLITE4_DEBUG + void sqlite4KVStoreDump(KVStore *p); +#endif Index: src/tclsqlite.c ================================================================== --- src/tclsqlite.c +++ src/tclsqlite.c @@ -504,11 +504,11 @@ char c = zType[0]; if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){ /* Only return a BLOB type if the Tcl variable is a bytearray and ** has no string representation. */ data = Tcl_GetByteArrayFromObj(pVar, &n); - sqlite4_result_blob(context, data, n, SQLITE4_TRANSIENT, 0); + sqlite4_result_blob(context, data, n, SQLITE4_TRANSIENT); }else if( c=='b' && strcmp(zType,"boolean")==0 ){ Tcl_GetIntFromObj(0, pVar, &n); sqlite4_result_int(context, n); }else if( c=='d' && strcmp(zType,"double")==0 ){ double r; @@ -519,11 +519,11 @@ Tcl_WideInt v; Tcl_GetWideIntFromObj(0, pVar, &v); sqlite4_result_int64(context, v); }else{ data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n); - sqlite4_result_text(context, (char *)data, n, SQLITE4_TRANSIENT, 0); + sqlite4_result_text(context, (char *)data, n, SQLITE4_TRANSIENT); } } } #ifndef SQLITE4_OMIT_AUTHORIZATION @@ -846,11 +846,11 @@ (c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0) ){ /* Load a BLOB type if the Tcl variable is a bytearray and ** it has no string representation or the host ** parameter name begins with "@". */ data = Tcl_GetByteArrayFromObj(pVar, &n); - sqlite4_bind_blob(pStmt, i, data, n, SQLITE4_STATIC, 0); + sqlite4_bind_blob(pStmt, i, data, n, SQLITE4_STATIC); Tcl_IncrRefCount(pVar); pPreStmt->apParm[iParm++] = pVar; }else if( c=='b' && strcmp(zType,"boolean")==0 ){ Tcl_GetIntFromObj(interp, pVar, &n); sqlite4_bind_int(pStmt, i, n); @@ -863,11 +863,11 @@ Tcl_WideInt v; Tcl_GetWideIntFromObj(interp, pVar, &v); sqlite4_bind_int64(pStmt, i, v); }else{ data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n); - sqlite4_bind_text(pStmt, i, (char *)data, n, SQLITE4_STATIC, 0); + sqlite4_bind_text(pStmt, i, (char *)data, n, SQLITE4_STATIC); Tcl_IncrRefCount(pVar); pPreStmt->apParm[iParm++] = pVar; } }else{ sqlite4_bind_null(pStmt, i); @@ -1668,11 +1668,11 @@ if( (nNull>0 && strcmp(azCol[i], zNull)==0) || strlen30(azCol[i])==0 ){ sqlite4_bind_null(pStmt, i+1); }else{ - sqlite4_bind_text(pStmt, i+1, azCol[i], -1, SQLITE4_STATIC, 0); + sqlite4_bind_text(pStmt, i+1, azCol[i], -1, SQLITE4_STATIC); } } sqlite4_step(pStmt); rc = sqlite4_reset(pStmt); free(zLine); @@ -2801,11 +2801,11 @@ unsigned char digest[16]; char zBuf[33]; p = sqlite4_aggregate_context(context, sizeof(*p)); MD5Final(digest,p); MD5DigestToBase16(digest, zBuf); - sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT, 0); + sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT); } int Md5_Register(sqlite4 *db){ int rc = sqlite4_create_function(db, "md5sum", -1, SQLITE4_UTF8, 0, 0, md5step, md5finalize); sqlite4_overload_function(db, "md5sum", -1); /* To exercise this API */ Index: src/tokenize.c ================================================================== --- src/tokenize.c +++ src/tokenize.c @@ -225,10 +225,11 @@ } case '~': { *tokenType = TK_BITNOT; return 1; } + case '`': case '\'': case '"': { int delim = z[0]; testcase( delim=='`' ); testcase( delim=='\'' ); Index: src/trigger.c ================================================================== --- src/trigger.c +++ src/trigger.c @@ -55,11 +55,11 @@ if( pTmpSchema!=pTab->pSchema ){ HashElem *p; for(p=sqliteHashFirst(&pTmpSchema->trigHash); p; p=sqliteHashNext(p)){ Trigger *pTrig = (Trigger *)sqliteHashData(p); if( pTrig->pTabSchema==pTab->pSchema - && 0==sqlite4_stricmp(pTrig->table, pTab->zName) + && 0==sqlite4StrICmp(pTrig->table, pTab->zName) ){ pTrig->pNext = (pList ? pList : pTab->pTrigger); pList = pTrig; } } @@ -188,11 +188,11 @@ } goto trigger_cleanup; } /* Do not create a trigger on a system table */ - if( sqlite4_strnicmp(pTab->zName, "sqlite_", 7)==0 ){ + if( sqlite4StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ sqlite4ErrorMsg(pParse, "cannot create trigger on system table"); pParse->nErr++; goto trigger_cleanup; } @@ -500,11 +500,11 @@ zDb = pName->a[0].zDatabase; zName = pName->a[0].zName; nName = sqlite4Strlen30(zName); for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ - if( zDb && sqlite4_stricmp(db->aDb[j].zName, zDb) ) continue; + if( zDb && sqlite4StrICmp(db->aDb[j].zName, zDb) ) continue; pTrigger = sqlite4HashFind(&(db->aDb[j].pSchema->trigHash), zName, nName); if( pTrigger ) break; } if( !pTrigger ){ if( !noErr ){ Index: src/update.c ================================================================== --- src/update.c +++ src/update.c @@ -210,11 +210,11 @@ goto update_cleanup; } /* Resolve the column name on the left of the assignment */ for(j=0; jnCol; j++){ - if( sqlite4_stricmp(pTab->aCol[j].zName, pChanges->a[i].zName)==0 ) break; + if( sqlite4StrICmp(pTab->aCol[j].zName, pChanges->a[i].zName)==0 ) break; } if( j==pTab->nCol ){ sqlite4ErrorMsg(pParse, "no such column: %s", pChanges->a[i].zName); pParse->checkSchema = 1; goto update_cleanup; @@ -506,11 +506,11 @@ j1 = sqlite4VdbeAddOp4(v, OP_NotFound, iCur+iPk, 0, regOldKey, 0, P4_INT32); sqlite4GenerateRowIndexDelete(pParse, pTab, 0, iCur, aRegIdx); /* Delete the old record */ if( hasFK || bChngPk ){ - sqlite4VdbeAddOp2(v, OP_Delete, iCur+iPk, 0); + sqlite4VdbeAddOp2(v, OP_Delete, iCur, 0); } sqlite4VdbeJumpHere(v, j1); if( hasFK ){ sqlite4FkCheck(pParse, pTab, 0, regNew); Index: src/utf.c ================================================================== --- src/utf.c +++ src/utf.c @@ -185,44 +185,11 @@ *pzNext = zIn; return c; } -/* -** Versions of stricmp and strnicmp that work with (simple) unicode -** case mapping. -*/ -int sqlite4_stricmp(const char *zLeft, const char *zRight){ - unsigned char *a, *b; - unsigned int ac, bc; - int x; - a = (unsigned char *)zLeft; - b = (unsigned char *)zRight; - do{ - READ_UTF8(a, 0, ac); - ac = sqlite4_tolower(ac); - READ_UTF8(b, 0, bc); - bc = sqlite4_tolower(bc); - }while( ac==bc && ac!=0 ); - return ac - bc; -} -int sqlite4_strnicmp(const char *zLeft, const char *zRight, int N){ - unsigned char *a, *b, *aTerm, *bTerm; - unsigned int ac, bc; - int x; - a = (unsigned char *)zLeft; - b = (unsigned char *)zRight; - aTerm = a + N; - bTerm = b + N; - do{ - READ_UTF8(a, aTerm, ac); - ac = sqlite4_tolower(ac); - READ_UTF8(b, bTerm, bc); - bc = sqlite4_tolower(bc); - }while( ac==bc && ac!=0 && amallocFailed ){ sqlite4VdbeMemRelease(&m); m.z = 0; } @@ -500,11 +467,11 @@ #ifdef SQLITE4_ENABLE_STAT3 char *sqlite4Utf8to16(sqlite4 *db, u8 enc, char *z, int n, int *pnOut){ Mem m; memset(&m, 0, sizeof(m)); m.db = db; - sqlite4VdbeMemSetStr(&m, z, n, SQLITE4_UTF8, SQLITE4_STATIC, 0); + sqlite4VdbeMemSetStr(&m, z, n, SQLITE4_UTF8, SQLITE4_STATIC); if( sqlite4VdbeMemTranslate(&m, enc) ){ assert( db->mallocFailed ); return 0; } assert( m.z==m.zMalloc ); @@ -589,278 +556,5 @@ assert( (z-zBuf)==n ); } } #endif /* SQLITE4_TEST */ #endif /* SQLITE4_OMIT_UTF16 */ - - -/* -** Return true if the argument corresponds to a unicode codepoint -** classified as either a letter or a number. Otherwise false. -** -** The results are undefined if the value passed to this function -** is less than zero. -*/ -int sqlite4_isalnum(int c){ - /* Each unsigned integer in the following array corresponds to a contiguous - ** range of unicode codepoints that are not either letters or numbers (i.e. - ** codepoints for which this function should return 0). - ** - ** The most significant 22 bits in each 32-bit value contain the first - ** codepoint in the range. The least significant 10 bits are used to store - ** the size of the range (always at least 1). In other words, the value - ** ((C<<22) + N) represents a range of N codepoints starting with codepoint - ** C. It is not possible to represent a range larger than 1023 codepoints - ** using this format. - */ - const static unsigned int aEntry[] = { - 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, - 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, - 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, - 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01, - 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01, - 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802, - 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F, - 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401, - 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804, - 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403, - 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812, - 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001, - 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802, - 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805, - 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401, - 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03, - 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807, - 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001, - 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01, - 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804, - 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001, - 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802, - 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01, - 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06, - 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007, - 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006, - 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417, - 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14, - 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07, - 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01, - 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001, - 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802, - 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F, - 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002, - 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802, - 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006, - 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D, - 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802, - 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027, - 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403, - 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805, - 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04, - 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401, - 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005, - 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B, - 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A, - 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001, - 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59, - 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807, - 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01, - 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E, - 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100, - 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10, - 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402, - 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804, - 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012, - 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004, - 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002, - 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, - 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, - 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, - 0x037FFC02, 0x03E3FC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, - 0x03F4F802, 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, - 0x03F95013, 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, - 0x03FCEC06, 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, - 0x04040003, 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, - 0x040E7C01, 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, - 0x04280403, 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, - 0x04294009, 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, - 0x04420003, 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, - 0x04460003, 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, - 0x05BD442E, 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, - 0x07480046, 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, - 0x075C5401, 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, - 0x075EA401, 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, - 0x07C2800F, 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, - 0x07C4C03C, 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, - 0x07C94002, 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, - 0x07CE8025, 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, - 0x07D108B6, 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, - 0x07D7EC46, 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, - 0x38008060, 0x380400F0, 0x3C000001, 0x3FFFF401, 0x40000001, - 0x43FFF401, - }; - static const unsigned int aAscii[4] = { - 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, - }; - - if( c<128 ){ - return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); - }else if( c<(1<<22) ){ - unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; - int iRes; - int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; - int iLo = 0; - while( iHi>=iLo ){ - int iTest = (iHi + iLo) / 2; - if( key >= aEntry[iTest] ){ - iRes = iTest; - iLo = iTest+1; - }else{ - iHi = iTest-1; - } - } - assert( aEntry[0]=aEntry[iRes] ); - return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); - } - return 1; -} - -/* -** Interpret the argument as a unicode codepoint. If the codepoint -** is an upper case character that has a lower case equivalent, -** return the codepoint corresponding to the lower case version. -** Otherwise, return a copy of the argument. -** -** The results are undefined if the value passed to this function -** is less than zero. -*/ -int sqlite4_tolower(int c){ - /* Each entry in the following array defines a rule for folding a range - ** of codepoints to lower case. The rule applies to a range of nRange - ** codepoints starting at codepoint iCode. - ** - ** If the least significant bit in flags is clear, then the rule applies - ** to all nRange codepoints (i.e. all nRange codepoints are upper case and - ** need to be folded). Or, if it is set, then the rule only applies to - ** every second codepoint in the range, starting with codepoint C. - ** - ** The 7 most significant bits in flags are an index into the aiOff[] - ** array. If a specific codepoint C does require folding, then its lower - ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF). - ** - ** The contents of this array are generated by parsing the CaseFolding.txt - ** file distributed as part of the "Unicode Character Database". See - ** http://www.unicode.org for details. - */ - static const struct TableEntry { - unsigned short iCode; - unsigned char flags; - unsigned char nRange; - } aEntry[] = { - {65, 14, 26}, {181, 64, 1}, {192, 14, 23}, - {216, 14, 7}, {256, 1, 48}, {306, 1, 6}, - {313, 1, 16}, {330, 1, 46}, {376, 116, 1}, - {377, 1, 6}, {383, 104, 1}, {385, 50, 1}, - {386, 1, 4}, {390, 44, 1}, {391, 0, 1}, - {393, 42, 2}, {395, 0, 1}, {398, 32, 1}, - {399, 38, 1}, {400, 40, 1}, {401, 0, 1}, - {403, 42, 1}, {404, 46, 1}, {406, 52, 1}, - {407, 48, 1}, {408, 0, 1}, {412, 52, 1}, - {413, 54, 1}, {415, 56, 1}, {416, 1, 6}, - {422, 60, 1}, {423, 0, 1}, {425, 60, 1}, - {428, 0, 1}, {430, 60, 1}, {431, 0, 1}, - {433, 58, 2}, {435, 1, 4}, {439, 62, 1}, - {440, 0, 1}, {444, 0, 1}, {452, 2, 1}, - {453, 0, 1}, {455, 2, 1}, {456, 0, 1}, - {458, 2, 1}, {459, 1, 18}, {478, 1, 18}, - {497, 2, 1}, {498, 1, 4}, {502, 122, 1}, - {503, 134, 1}, {504, 1, 40}, {544, 110, 1}, - {546, 1, 18}, {570, 70, 1}, {571, 0, 1}, - {573, 108, 1}, {574, 68, 1}, {577, 0, 1}, - {579, 106, 1}, {580, 28, 1}, {581, 30, 1}, - {582, 1, 10}, {837, 36, 1}, {880, 1, 4}, - {886, 0, 1}, {902, 18, 1}, {904, 16, 3}, - {908, 26, 1}, {910, 24, 2}, {913, 14, 17}, - {931, 14, 9}, {962, 0, 1}, {975, 4, 1}, - {976, 140, 1}, {977, 142, 1}, {981, 146, 1}, - {982, 144, 1}, {984, 1, 24}, {1008, 136, 1}, - {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1}, - {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1}, - {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32}, - {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1}, - {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38}, - {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1}, - {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1}, - {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6}, - {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6}, - {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8}, - {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2}, - {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1}, - {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2}, - {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2}, - {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2}, - {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1}, - {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16}, - {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47}, - {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1}, - {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1}, - {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1}, - {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2}, - {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1}, - {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14}, - {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, - {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, - {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, - {65313, 14, 26}, - }; - static const unsigned short aiOff[] = { - 1, 2, 8, 15, 16, 26, 28, 32, - 37, 38, 40, 48, 63, 64, 69, 71, - 79, 80, 116, 202, 203, 205, 206, 207, - 209, 210, 211, 213, 214, 217, 218, 219, - 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, - 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, - 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, - 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, - 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, - 65514, 65521, 65527, 65528, 65529, - }; - - int ret = c; - - assert( c>=0 ); - assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); - - if( c<128 ){ - if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); - }else if( c<65536 ){ - int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; - int iLo = 0; - int iRes = -1; - - while( iHi>=iLo ){ - int iTest = (iHi + iLo) / 2; - int cmp = (c - aEntry[iTest].iCode); - if( cmp>=0 ){ - iRes = iTest; - iLo = iTest+1; - }else{ - iHi = iTest-1; - } - } - assert( iRes<0 || c>=aEntry[iRes].iCode ); - - if( iRes>=0 ){ - const struct TableEntry *p = &aEntry[iRes]; - if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ - ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; - assert( ret>0 ); - } - } - } - - else if( c>=66560 && c<66600 ){ - ret = c + 40; - } - - return ret; -} Index: src/util.c ================================================================== --- src/util.c +++ src/util.c @@ -130,13 +130,13 @@ char *z; va_list ap; va_start(ap, zFormat); z = sqlite4VMPrintf(db, zFormat, ap); va_end(ap); - sqlite4ValueSetStr(db->pErr, -1, z, SQLITE4_UTF8, SQLITE4_DYNAMIC, 0); + sqlite4ValueSetStr(db->pErr, -1, z, SQLITE4_UTF8, SQLITE4_DYNAMIC); }else{ - sqlite4ValueSetStr(db->pErr, 0, 0, SQLITE4_UTF8, SQLITE4_STATIC, 0); + sqlite4ValueSetStr(db->pErr, 0, 0, SQLITE4_UTF8, SQLITE4_STATIC); } } } /* @@ -215,10 +215,38 @@ } } z[j] = 0; return j; } + +/* Convenient short-hand */ +#define UpperToLower sqlite4UpperToLower + +/* +** Some systems have stricmp(). Others have strcasecmp(). Because +** there is no consistency, we will define our own. +** +** IMPLEMENTATION-OF: R-20522-24639 The sqlite4_strnicmp() API allows +** applications and extensions to compare the contents of two buffers +** containing UTF-8 strings in a case-independent fashion, using the same +** definition of case independence that SQLite uses internally when +** comparing identifiers. +*/ +int sqlite4StrICmp(const char *zLeft, const char *zRight){ + register unsigned char *a, *b; + a = (unsigned char *)zLeft; + b = (unsigned char *)zRight; + while( *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } + return UpperToLower[*a] - UpperToLower[*b]; +} +int sqlite4_strnicmp(const char *zLeft, const char *zRight, int N){ + register unsigned char *a, *b; + a = (unsigned char *)zLeft; + b = (unsigned char *)zRight; + while( N-- > 0 && *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } + return N<0 ? 0 : UpperToLower[*a] - UpperToLower[*b]; +} /* ** The string z[] is an text representation of a real number. ** Convert this string to a double and write it into *pResult. ** Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -462,22 +462,10 @@ p->zErrMsg = sqlite4DbStrDup(db, pVtab->zErrMsg); sqlite4_free(db->pEnv, pVtab->zErrMsg); pVtab->zErrMsg = 0; } -/* -** Return a pointer to a register in the root frame. -*/ -static Mem *sqlite4RegisterInRootFrame(Vdbe *p, int i){ - if( p->pFrame ){ - VdbeFrame *pFrame; - for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); - return &pFrame->aMem[i]; - }else{ - return &p->aMem[i]; - } -} /* ** Execute as much of a VDBE program as we can then return. ** ** sqlite4VdbeMakeReady() must be called before this routine in order to @@ -882,12 +870,11 @@ pOp->opcode = OP_String; pOp->p1 = sqlite4Strlen30(pOp->p4.z); #ifndef SQLITE4_OMIT_UTF16 if( encoding!=SQLITE4_UTF8 ){ - rc = sqlite4VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE4_UTF8, - SQLITE4_STATIC, 0); + rc = sqlite4VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE4_UTF8, SQLITE4_STATIC); if( rc==SQLITE4_TOOBIG ) goto too_big; if( SQLITE4_OK!=sqlite4VdbeChangeEncoding(pOut, encoding) ) goto no_mem; assert( pOut->zMalloc==pOut->z ); assert( pOut->flags & MEM_Dyn ); pOut->zMalloc = 0; @@ -949,11 +936,11 @@ ** P4 points to a blob of data P1 bytes long. Store this ** blob in register P2. */ case OP_Blob: { /* out2-prerelease */ assert( pOp->p1 <= SQLITE4_MAX_LENGTH ); - sqlite4VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0, 0); + sqlite4VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0); pOut->enc = encoding; UPDATE_MAX_BLOBSIZE(pOut); break; } @@ -1301,18 +1288,10 @@ case OP_CollSeq: { assert( pOp->p4type==P4_COLLSEQ ); break; } -/* Opcode: Mifunction P1 -*/ -case OP_Mifunction: { - pc++; - pOp++; - /* fall through to OP_Function */ -}; - /* Opcode: Function P1 P2 P3 P4 P5 ** ** Invoke a user function (P4 is a pointer to a Function structure that ** defines the function) with P5 arguments taken from register P2 and ** successors. The result of the function is stored in register P3. @@ -1363,17 +1342,10 @@ ctx.s.flags = MEM_Null; ctx.s.db = db; ctx.s.xDel = 0; ctx.s.zMalloc = 0; - if( pOp[-1].opcode==OP_Mifunction ){ - ctx.pFts = p->apCsr[pOp[-1].p1]->pFts; - apVal++; - n--; - }else{ - ctx.pFts = 0; - } /* The output cell may already have a buffer allocated. Move ** the pointer to ctx.s so in case the user-function can use ** the already allocated buffer instead of allocating a new one. */ @@ -2196,17 +2168,13 @@ ** ** This instruction encodes the N values into a database key and writes ** the result to register P3. No affinity transformations are applied to ** the input values before they are encoded. ** -** If the OPFLAG_SEQCOUNT bit of P5 is set, then a sequence number -** (unique within the cursor) is appended to the record. The sole purpose -** of this is to ensure that the key blob is unique within the cursors table. -** -** If the OPFLAG_LASTROWID bit of P5 is set and the value of the first and -** only field of the key is an integer, then set the lastRowid field to the -** value of that integer. +** If P5 is non-zero, then a sequence number (unique within the cursor) +** is appended to the record. The sole purpose of this is to ensure that +** the key blob is unique within the cursors table. */ case OP_MakeIdxKey: { VdbeCursor *pC; KeyInfo *pKeyInfo; Mem *pData0; /* First in array of input registers */ @@ -2226,22 +2194,19 @@ /* If pOp->p5 is non-zero, encode the sequence number blob to append to ** the end of the key. Variable nSeq is set to the number of bytes in ** the encoded key. */ nSeq = 0; - if( pOp->p5 & OPFLAG_SEQCOUNT ){ + if( pOp->p5 ){ iSeq = pC->seqCount++; do { nSeq++; aSeq[sizeof(aSeq)-nSeq] = (u8)(iSeq & 0x007F); iSeq = iSeq >> 7; }while( iSeq ); aSeq[sizeof(aSeq)-nSeq] |= 0x80; } - if( (pOp->p5 & OPFLAG_LASTROWID)!=0 && (pData0->flags & MEM_Int)!=0 ){ - lastRowid = pData0->u.i; - } memAboutToChange(p, pOut); nField = pKeyInfo->nField; if( pOp->p4type==P4_INT32 && pOp->p4.i ){ @@ -2256,12 +2221,11 @@ sqlite4DbFree(db, aRec); }else{ if( nSeq ){ memcpy(&aRec[nRec], &aSeq[sizeof(aSeq)-nSeq], nSeq); } - rc = sqlite4VdbeMemSetStr(pOut, (char *)aRec, nRec+nSeq, 0, - SQLITE4_DYNAMIC, 0); + rc = sqlite4VdbeMemSetStr(pOut, (char *)aRec, nRec+nSeq, 0, SQLITE4_DYNAMIC); REGISTER_TRACE(pOp->p3, pOut); UPDATE_MAX_BLOBSIZE(pOut); } break; @@ -2347,12 +2311,11 @@ pData0, pC->pKeyInfo->nField, pC->iRoot, pC->pKeyInfo, &aRec, &nRec, 0 ); if( rc ){ sqlite4DbFree(db, aRec); }else{ - rc = sqlite4VdbeMemSetStr(pKeyOut, (char *)aRec, nRec, 0, - SQLITE4_DYNAMIC, 0); + rc = sqlite4VdbeMemSetStr(pKeyOut, (char *)aRec, nRec, 0, SQLITE4_DYNAMIC); REGISTER_TRACE(keyReg, pKeyOut); UPDATE_MAX_BLOBSIZE(pKeyOut); } } @@ -2364,11 +2327,11 @@ aRec = 0; rc = sqlite4VdbeEncodeData(db, pData0, nField, &aRec, &nRec); if( rc ){ sqlite4DbFree(db, aRec); }else{ - rc = sqlite4VdbeMemSetStr(pOut, (char *)aRec, nRec, 0, SQLITE4_DYNAMIC,0); + rc = sqlite4VdbeMemSetStr(pOut, (char *)aRec, nRec, 0, SQLITE4_DYNAMIC); REGISTER_TRACE(pOp->p3, pOut); UPDATE_MAX_BLOBSIZE(pOut); } } break; @@ -2451,11 +2414,11 @@ ** number. For example, to commit or rollback the top level transaction ** iSave==2. */ iSave = db->nSavepoint+1; for(pSave=db->pSavepoint; pSave; pSave=pSave->pNext){ if( zSave ){ - if( pSave->zName && 0==sqlite4_stricmp(zSave, pSave->zName) ) break; + if( pSave->zName && 0==sqlite4StrICmp(zSave, pSave->zName) ) break; }else{ if( pSave->pNext==0 ) break; } iSave--; } @@ -2583,11 +2546,11 @@ break; } /* Opcode: VerifyCookie P1 P2 P3 * * ** -** cHECK THe value of global database parameter number 0 (the +** Check the value of global database parameter number 0 (the ** schema version) and make sure it is equal to P2 and that the ** generation counter on the local schema parse equals P3. ** ** P1 is the database number which is 0 for the main database file ** and 1 for the file holding temporary tables and some higher number @@ -2881,40 +2844,31 @@ KVByteArray *aPkKey; KVSize nPkKey; pPk = p->apCsr[pOp->p1]; pIdx = p->apCsr[pOp->p3]; - - if( pIdx->pFts ){ - rc = sqlite4Fts5Pk(pIdx->pFts, pPk->iRoot, &aPkKey, &nPkKey); - if( rc==SQLITE4_OK ){ - rc = sqlite4KVCursorSeek(pPk->pKVCur, aPkKey, nPkKey, 0); - if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_CORRUPT_BKPT; - pPk->nullRow = 0; - } - }else{ - assert( pIdx->pKeyInfo->nPK>0 ); - assert( pPk->pKeyInfo->nPK==0 ); - rc = sqlite4KVCursorKey(pIdx->pKVCur, &aKey, &nKey); - if( rc==SQLITE4_OK ){ - nShort = sqlite4VdbeShortKey(aKey, nKey, - pIdx->pKeyInfo->nField - pIdx->pKeyInfo->nPK - ); - - nPkKey = sqlite4VarintLen(pPk->iRoot) + nKey - nShort; - aPkKey = sqlite4DbMallocRaw(db, nPkKey); - - if( aPkKey ){ - putVarint32(aPkKey, pPk->iRoot); - memcpy(&aPkKey[nPkKey - (nKey-nShort)], &aKey[nShort], nKey-nShort); - rc = sqlite4KVCursorSeek(pPk->pKVCur, aPkKey, nPkKey, 0); - if( rc==SQLITE4_NOTFOUND ){ - rc = SQLITE4_CORRUPT_BKPT; - } - pPk->nullRow = 0; - sqlite4DbFree(db, aPkKey); - } + assert( pIdx->pKeyInfo->nPK>0 ); + assert( pPk->pKeyInfo->nPK==0 ); + + rc = sqlite4KVCursorKey(pIdx->pKVCur, &aKey, &nKey); + if( rc==SQLITE4_OK ){ + nShort = sqlite4VdbeShortKey(aKey, nKey, + pIdx->pKeyInfo->nField - pIdx->pKeyInfo->nPK + ); + + nPkKey = sqlite4VarintLen(pPk->iRoot) + nKey - nShort; + aPkKey = sqlite4DbMallocRaw(db, nPkKey); + + if( aPkKey ){ + putVarint32(aPkKey, pPk->iRoot); + memcpy(&aPkKey[nPkKey - (nKey-nShort)], &aKey[nShort], nKey-nShort); + rc = sqlite4KVCursorSeek(pPk->pKVCur, aPkKey, nPkKey, 0); + if( rc==SQLITE4_NOTFOUND ){ + rc = SQLITE4_CORRUPT_BKPT; + } + pPk->nullRow = 0; + sqlite4DbFree(db, aPkKey); } } break; } @@ -3252,18 +3206,16 @@ pOut->u.i = p->apCsr[pOp->p1]->seqCount++; break; } -/* Opcode: NewRowid P1 P2 P3 * * -** -** Get a new integer primary key (a.k.a "rowid") for table P1. The integer -** should not be currently in use as a primary key on that table. -** -** If P3 is not zero, then it is the number of a register in the top-level -** frame that holds a lower bound for the new rowid. In other words, the -** new rowid must be no less than reg[P3]+1. +/* Opcode: NewRowid P1 P2 * * * +** +** Get a new integer record number (a.k.a "rowid") used as the key to a table. +** The record number is not previously used as a key in the database +** table that cursor P1 points to. The new record number is written +** to register P2. */ case OP_NewRowid: { /* out2-prerelease */ i64 v; /* The new rowid */ VdbeCursor *pC; /* Cursor of table to get the new rowid */ const KVByteArray *aKey; /* Key of an existing row */ @@ -3306,28 +3258,15 @@ if( n==0 ) rc = SQLITE4_CORRUPT_BKPT; if( v!=pC->iRoot ) rc = SQLITE4_CORRUPT_BKPT; } if( rc==SQLITE4_OK ){ n = sqlite4VdbeDecodeIntKey(&aKey[n], nKey-n, &v); - if( n==0 || v==LARGEST_INT64 ) rc = SQLITE4_FULL; + if( n==0 ) rc = SQLITE4_CORRUPT_BKPT; } }else{ break; } -#ifndef SQLITE_OMIT_AUTOINCREMENT - if( pOp->p3 && rc==SQLITE4_OK ){ - pIn3 = sqlite4RegisterInRootFrame(p, pOp->p3); - assert( memIsValid(pIn3) ); - REGISTER_TRACE(pOp->p3, pIn3); - sqlite4VdbeMemIntegerify(pIn3); - assert( (pIn3->flags & MEM_Int)!=0 ); /* mem(P3) holds an integer */ - if( pIn3->u.i==MAX_ROWID ){ - rc = SQLITE4_FULL; - } - if( vu.i ) v = pIn3->u.i; - } -#endif pOut->flags = MEM_Int; pOut->u.i = v+1; break; } @@ -3387,10 +3326,18 @@ ** ** If the OPFLAG_NCHANGE flag of P5 is set, then the row change count is ** incremented (otherwise not). If the OPFLAG_LASTROWID flag of P5 is set, ** then rowid is stored for subsequent return by the ** sqlite4_last_insert_rowid() function (otherwise it is unmodified). +** +** If the OPFLAG_USESEEKRESULT flag of P5 is set and if the result of +** the last seek operation (OP_NotExists) was a success, then this +** operation will not attempt to find the appropriate row before doing +** the insert but will instead overwrite the row that the cursor is +** currently pointing to. Presumably, the prior OP_NotExists opcode +** has already positioned the cursor correctly. This is an optimization +** that boosts performance by avoiding redundant seeks. ** ** If the OPFLAG_ISUPDATE flag is set, then this opcode is part of an ** UPDATE operation. Otherwise (if the flag is clear) then this opcode ** is part of an INSERT operation. The difference is only important to ** the update hook. @@ -3522,11 +3469,11 @@ if( (pIn3->flags & MEM_Blob) && pIn3->n==nKey && 0==memcmp(pIn3->z, aKey, nKey) ){ pc = pOp->p2-1; }else{ - sqlite4VdbeMemSetStr(pIn3, (const char*)aKey, nKey, 0, SQLITE4_TRANSIENT,0); + sqlite4VdbeMemSetStr(pIn3, (const char*)aKey, nKey, 0, SQLITE4_TRANSIENT); } break; }; @@ -3589,11 +3536,11 @@ rc = sqlite4KVCursorData(pCrsr, 0, -1, &pData, &nData); } if( rc==SQLITE4_OK && nData>db->aLimit[SQLITE4_LIMIT_LENGTH] ){ goto too_big; } - sqlite4VdbeMemSetStr(pOut, (const char*)pData, nData, 0, SQLITE4_TRANSIENT,0); + sqlite4VdbeMemSetStr(pOut, (const char*)pData, nData, 0, SQLITE4_TRANSIENT); pOut->enc = SQLITE4_UTF8; /* In case the blob is ever cast to text */ UPDATE_MAX_BLOBSIZE(pOut); break; } @@ -4110,12 +4057,11 @@ pIn1 = &aMem[pOp->p1]; pOut = &aMem[pOp->p3]; if( (pIn1->flags & MEM_RowSet) && (aKey = sqlite4RowSetRead(pIn1->u.pRowSet, &nKey)) ){ - rc = sqlite4VdbeMemSetStr(pOut, (char const *)aKey, nKey, 0, - SQLITE4_TRANSIENT, 0); + rc = sqlite4VdbeMemSetStr(pOut, (char const *)aKey, nKey, 0, SQLITE4_TRANSIENT); sqlite4RowSetNext(pIn1->u.pRowSet); }else{ /* The RowSet is empty */ sqlite4VdbeMemSetNull(pIn1); pc = pOp->p2 - 1; @@ -4301,20 +4247,23 @@ ** an integer. */ case OP_MemMax: { /* in2 */ Mem *pIn1; VdbeFrame *pFrame; - pIn1 = sqlite4RegisterInRootFrame(p, pOp->p1); + if( p->pFrame ){ + for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); + pIn1 = &pFrame->aMem[pOp->p1]; + }else{ + pIn1 = &aMem[pOp->p1]; + } assert( memIsValid(pIn1) ); sqlite4VdbeMemIntegerify(pIn1); pIn2 = &aMem[pOp->p2]; - REGISTER_TRACE(pOp->p1, pIn1); sqlite4VdbeMemIntegerify(pIn2); if( pIn1->u.iu.i){ pIn1->u.i = pIn2->u.i; } - REGISTER_TRACE(pOp->p1, pIn1); break; } #endif /* SQLITE4_OMIT_AUTOINCREMENT */ /* Opcode: IfPos P1 P2 * * * @@ -4806,11 +4755,11 @@ case OP_VUpdate: { sqlite4_vtab *pVtab; sqlite4_module *pModule; int nArg; int i; - sqlite4_int64 rowid; + sqlite_int64 rowid; Mem **apArg; Mem *pX; assert( pOp->p2==1 || pOp->p5==OE_Fail || pOp->p5==OE_Rollback || pOp->p5==OE_Abort || pOp->p5==OE_Ignore || pOp->p5==OE_Replace @@ -4876,143 +4825,10 @@ #endif /* SQLITE4_DEBUG */ break; } #endif -/* Opcode: FtsUpdate P1 P2 P3 P4 P5 -** -** This opcode is used to write to an FTS index. P4 points to an Fts5Info -** object describing the index. -** -** If argument P5 is non-zero, then entries are removed from the FTS index. -** If it is zero, then entries are inserted. In other words, when a row -** is deleted from a table with an FTS index, this opcode is invoked with -** P5==1. When a row is inserted, it is invoked with P5==0. If an existing -** row is updated, this opcode is invoked twice - once with P5==1 and then -** again with P5==0. -** -** Register P1 contains the PK (a blob in key format) of the affected row. -** P3 is the first in an array of N registers, where N is the number of -** columns in the indexed table. Each register contains the value for the -** corresponding table column. -** -** If P2 is non-zero, then it is a register containing the root page number -** of the fts index to update. If it is zero, then the root page of the -** index is available as part of the Fts5Info structure. -*/ -case OP_FtsUpdate: { - Fts5Info *pInfo; /* Description of fts5 index to update */ - Mem *pKey; /* Primary key of indexed row */ - Mem *aArg; /* Pointer to array of N arguments */ - Mem *pRoot; /* Root page number */ - int iRoot; - - assert( pOp->p4type==P4_FTS5INFO ); - pInfo = pOp->p4.pFtsInfo; - aArg = &aMem[pOp->p3]; - pKey = &aMem[pOp->p1]; - - if( pOp->p2 ){ - iRoot = aMem[pOp->p2].u.i; - }else{ - iRoot = 0; - } - - rc = sqlite4Fts5Update(db, pInfo, iRoot, pKey, aArg, pOp->p5, &p->zErrMsg); - break; -} - -/* -** Opcode: FtsCksum P1 * P3 P4 P5 -** -** This opcode is used by the integrity-check procedure that verifies that -** the contents of an fts5 index and its corresponding table match. -*/ -case OP_FtsCksum: { - Fts5Info *pInfo; /* Description of fts5 index to update */ - Mem *pKey; /* Primary key of row */ - Mem *aArg; /* Pointer to array of N values */ - i64 cksum; /* Checksum for this row or index entry */ - - assert( pOp->p4type==P4_FTS5INFO ); - pInfo = pOp->p4.pFtsInfo; - - pOut = &aMem[pOp->p1]; - pKey = &aMem[pOp->p3]; - aArg = &aMem[pOp->p3+1]; - cksum = 0; - - if( pOp->p5 ){ - sqlite4Fts5EntryCksum(db, pInfo, pKey, aArg, &cksum); - pOut->u.i = pOut->u.i ^ cksum; - }else{ - sqlite4Fts5RowCksum(db, pInfo, pKey, aArg, &cksum); - pOut->u.i = pOut->u.i ^ cksum; - } - break; -} - -/* Opcode: FtsOpen P1 P2 P3 P4 P5 -** -** Open an FTS cursor named P1. P4 points to an Fts5Info object. -** -** Register P3 contains the MATCH expression that this cursor will iterate -** through the matches for. P5 is set to 0 to iterate through the results -** in ascending PK order, or 1 for descending PK order. -** -** If the expression matches zero rows, jump to instruction P2. Otherwise, -** leave the cursor pointing at the first match and fall through to the -** next instruction. -*/ -case OP_FtsOpen: { /* jump */ - Fts5Info *pInfo; /* Description of fts5 index to update */ - char *zErr; - VdbeCursor *pCur; - char *zMatch; - Mem *pMatch; - - pMatch = &aMem[pOp->p3]; - Stringify(pMatch, encoding); - zMatch = pMatch->z; - - assert( pOp->p4type==P4_FTS5INFO ); - pInfo = pOp->p4.pFtsInfo; - pCur = allocateCursor(p, pOp->p1, 0, pInfo->iDb, 0); - if( pCur ){ - rc = sqlite4Fts5Open(db, pInfo, zMatch, pOp->p5, &pCur->pFts, &p->zErrMsg); - } - if( rc==SQLITE4_OK && 0==sqlite4Fts5Valid(pCur->pFts) ){ - pc = pOp->p2-1; - } - break; -} - -/* Opcode: FtsNext P1 P2 * * * -** -** Advance FTS cursor P1 to the next entry and jump to instruction P2. Or, -** if there is no next entry, set the cursor to point to EOF and fall through -** to the next instruction. -*/ -case OP_FtsNext: { - VdbeCursor *pCsr; - - pCsr = p->apCsr[pOp->p1]; - rc = sqlite4Fts5Next(pCsr->pFts); - if( rc==SQLITE4_OK && sqlite4Fts5Valid(pCsr->pFts) ) pc = pOp->p2-1; - - break; -} - -/* Opcode: FtsPk P1 P2 * * * -** -** P1 is an FTS cursor that points to a valid entry (not EOF). Copy the PK -** blob for the current entry to register P2. -*/ -case OP_FtsPk: { - assert( 0 ); - break; -} /* Opcode: Noop * * * * * ** ** Do nothing. This instruction is often useful as a jump ** destination. @@ -5019,11 +4835,11 @@ */ /* ** The magic Explain opcode are only inserted when explain==2 (which ** is to say when the EXPLAIN QUERY PLAN syntax is used.) ** This opcode records information from the optimizer. It is the -** the same as a no-op. This opcode never appears in a real VM program. +** the same as a no-op. This opcodesnever appears in a real VM program. */ default: { /* This is really OP_Noop and OP_Explain */ assert( pOp->opcode==OP_Noop || pOp->opcode==OP_Explain ); break; } Index: src/vdbe.h ================================================================== --- src/vdbe.h +++ src/vdbe.h @@ -60,11 +60,10 @@ Mem *pMem; /* Used when p4type is P4_MEM */ VTable *pVtab; /* Used when p4type is P4_VTAB */ KeyInfo *pKeyInfo; /* Used when p4type is P4_KEYINFO */ int *ai; /* Used when p4type is P4_INTARRAY */ SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ - Fts5Info *pFtsInfo; /* Used when p4type is P4_FTS5INDEXINFO */ int (*xAdvance)(VdbeCursor*); } p4; #ifdef SQLITE4_DEBUG char *zComment; /* Comment to improve readability */ #endif @@ -119,11 +118,10 @@ #define P4_INT64 (-13) /* P4 is a 64-bit signed integer */ #define P4_INT32 (-14) /* P4 is a 32-bit signed integer */ #define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ #define P4_SUBPROGRAM (-18) /* P4 is a pointer to a SubProgram structure */ #define P4_ADVANCE (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */ -#define P4_FTS5INFO (-20) /* P4 points to an Fts5Info structure */ /* When adding a P4 argument using P4_KEYINFO, a copy of the KeyInfo structure ** is made. That copy is freed when the Vdbe is finalized. But if the ** argument is P4_KEYINFO_HANDOFF, the passed in pointer is used. It still ** gets freed when the Vdbe is finalized so it still should be obtained @@ -202,11 +200,11 @@ #endif void sqlite4VdbeResetStepResult(Vdbe*); void sqlite4VdbeRewind(Vdbe*); int sqlite4VdbeReset(Vdbe*); void sqlite4VdbeSetNumCols(Vdbe*,int); -int sqlite4VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*,void*)); +int sqlite4VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*)); void sqlite4VdbeCountChanges(Vdbe*); sqlite4 *sqlite4VdbeDb(Vdbe*); void sqlite4VdbeSetSql(Vdbe*, const char *z, int n); void sqlite4VdbeSwap(Vdbe*,Vdbe*); VdbeOp *sqlite4VdbeTakeOpArray(Vdbe*, int*, int*); Index: src/vdbeInt.h ================================================================== --- src/vdbeInt.h +++ src/vdbeInt.h @@ -68,11 +68,10 @@ const sqlite4_module *pModule; /* Module for cursor pVtabCursor */ i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred move-to */ i64 lastRowid; /* Last rowid from a Next or NextIdx operation */ VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ - Fts5Cursor *pFts; /* Fts5 cursor object (or NULL) */ /* Result of last sqlite4-Moveto() done by an OP_NotExists or ** OP_IsUnique opcode on this cursor. */ int seekResult; }; @@ -147,12 +146,11 @@ u8 enc; /* SQLITE4_UTF8, SQLITE4_UTF16BE, SQLITE4_UTF16LE */ #ifdef SQLITE4_DEBUG Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ void *pFiller; /* So that sizeof(Mem) is a multiple of 8 */ #endif - void (*xDel)(void*,void*); /* Function to delete Mem.z */ - void *pDelArg; /* First argument to xDel() */ + void (*xDel)(void *); /* If not null, call this function to delete Mem.z */ char *zMalloc; /* Dynamic buffer allocated by sqlite4_malloc() */ }; /* One or more of the following flags are set to indicate the validOK ** representations of the value stored in the Mem struct. @@ -216,12 +214,11 @@ struct VdbeFunc { FuncDef *pFunc; /* The definition of the function */ int nAux; /* Number of entries allocated for apAux[] */ struct AuxData { void *pAux; /* Aux data for the i-th argument */ - void (*xDelete)(void*,void*); /* Destructor for the aux data */ - void *pDeleteArg; /* First argument to xDelete */ + void (*xDelete)(void *); /* Destructor for the aux data */ } apAux[1]; /* One slot for each function argument */ }; /* ** The "context" argument for a installable function. A pointer to an @@ -241,11 +238,10 @@ VdbeFunc *pVdbeFunc; /* Auxilary data, if created. */ Mem s; /* The return value is stored here */ Mem *pMem; /* Memory cell used to store aggregate context */ int isError; /* Error code returned by the function. */ CollSeq *pColl; /* Collating sequence */ - Fts5Cursor *pFts; /* fts5 cursor for matchinfo functions */ }; /* ** An Explain object accumulates indented output which is helpful ** in describing recursive data structures. @@ -398,12 +394,11 @@ int sqlite4VdbeMemTooBig(Mem*); int sqlite4VdbeMemCopy(Mem*, const Mem*); void sqlite4VdbeMemShallowCopy(Mem*, const Mem*, int); void sqlite4VdbeMemMove(Mem*, Mem*); int sqlite4VdbeMemNulTerminate(Mem*); -int sqlite4VdbeMemSetStr(Mem*, const char*, int, u8, - void(*)(void*,void*),void*); +int sqlite4VdbeMemSetStr(Mem*, const char*, int, u8, void(*)(void*)); void sqlite4VdbeMemSetInt64(Mem*, i64); #ifdef SQLITE4_OMIT_FLOATING_POINT # define sqlite4VdbeMemSetDouble sqlite4VdbeMemSetInt64 #else void sqlite4VdbeMemSetDouble(Mem*, double); Index: src/vdbeapi.c ================================================================== --- src/vdbeapi.c +++ src/vdbeapi.c @@ -187,51 +187,49 @@ ** The setStrOrError() funtion calls sqlite4VdbeMemSetStr() to store the ** result as a string or blob but if the string or blob is too large, it ** then sets the error code to SQLITE4_TOOBIG */ static void setResultStrOrError( - sqlite4_context *pCtx, /* Function context */ - const char *z, /* String pointer */ - int n, /* Bytes in string, or negative */ - u8 enc, /* Encoding of z. 0 for BLOBs */ - void (*xDel)(void*,void*), /* Destructor function */ - void *pDelArg /* First argument to xDel() */ + sqlite4_context *pCtx, /* Function context */ + const char *z, /* String pointer */ + int n, /* Bytes in string, or negative */ + u8 enc, /* Encoding of z. 0 for BLOBs */ + void (*xDel)(void*) /* Destructor function */ ){ if( xDel==SQLITE4_DYNAMIC ){ assert( sqlite4MemdebugHasType(z, MEMTYPE_HEAP) ); assert( sqlite4MemdebugNoType(z, ~MEMTYPE_HEAP) ); sqlite4MemdebugSetType((char*)z, MEMTYPE_DB | MEMTYPE_HEAP); } - if( sqlite4VdbeMemSetStr(&pCtx->s, z, n, enc, xDel,pDelArg)==SQLITE4_TOOBIG ){ + if( sqlite4VdbeMemSetStr(&pCtx->s, z, n, enc, xDel)==SQLITE4_TOOBIG ){ sqlite4_result_error_toobig(pCtx); } } void sqlite4_result_blob( sqlite4_context *pCtx, const void *z, int n, - void (*xDel)(void*,void*), - void *pDelArg + void (*xDel)(void *) ){ assert( n>=0 ); assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, 0, xDel, pDelArg); + setResultStrOrError(pCtx, z, n, 0, xDel); } void sqlite4_result_double(sqlite4_context *pCtx, double rVal){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); sqlite4VdbeMemSetDouble(&pCtx->s, rVal); } void sqlite4_result_error(sqlite4_context *pCtx, const char *z, int n){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); pCtx->isError = SQLITE4_ERROR; - sqlite4VdbeMemSetStr(&pCtx->s, z, n, SQLITE4_UTF8, SQLITE4_TRANSIENT, 0); + sqlite4VdbeMemSetStr(&pCtx->s, z, n, SQLITE4_UTF8, SQLITE4_TRANSIENT); } #ifndef SQLITE4_OMIT_UTF16 void sqlite4_result_error16(sqlite4_context *pCtx, const void *z, int n){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); pCtx->isError = SQLITE4_ERROR; - sqlite4VdbeMemSetStr(&pCtx->s, z, n, SQLITE4_UTF16NATIVE,SQLITE4_TRANSIENT,0); + sqlite4VdbeMemSetStr(&pCtx->s, z, n, SQLITE4_UTF16NATIVE, SQLITE4_TRANSIENT); } #endif void sqlite4_result_int(sqlite4_context *pCtx, int iVal){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); sqlite4VdbeMemSetInt64(&pCtx->s, (i64)iVal); @@ -246,46 +244,42 @@ } void sqlite4_result_text( sqlite4_context *pCtx, const char *z, int n, - void (*xDel)(void*,void*), - void *pDelArg + void (*xDel)(void *) ){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE4_UTF8, xDel, pDelArg); + setResultStrOrError(pCtx, z, n, SQLITE4_UTF8, xDel); } #ifndef SQLITE4_OMIT_UTF16 void sqlite4_result_text16( sqlite4_context *pCtx, const void *z, int n, - void (*xDel)(void*,void*), - void *pDelArg + void (*xDel)(void *) ){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE4_UTF16NATIVE, xDel, pDelArg); + setResultStrOrError(pCtx, z, n, SQLITE4_UTF16NATIVE, xDel); } void sqlite4_result_text16be( sqlite4_context *pCtx, const void *z, int n, - void (*xDel)(void*,void*), - void *pDelArg + void (*xDel)(void *) ){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE4_UTF16BE, xDel, pDelArg); + setResultStrOrError(pCtx, z, n, SQLITE4_UTF16BE, xDel); } void sqlite4_result_text16le( sqlite4_context *pCtx, const void *z, int n, - void (*xDel)(void*,void*), - void *pDelArg + void (*xDel)(void *) ){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE4_UTF16LE, xDel, pDelArg); + setResultStrOrError(pCtx, z, n, SQLITE4_UTF16LE, xDel); } #endif /* SQLITE4_OMIT_UTF16 */ void sqlite4_result_value(sqlite4_context *pCtx, sqlite4_value *pValue){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); sqlite4VdbeMemCopy(&pCtx->s, pValue); @@ -296,20 +290,20 @@ } void sqlite4_result_error_code(sqlite4_context *pCtx, int errCode){ pCtx->isError = errCode; if( pCtx->s.flags & MEM_Null ){ sqlite4VdbeMemSetStr(&pCtx->s, sqlite4ErrStr(errCode), -1, - SQLITE4_UTF8, SQLITE4_STATIC, 0); + SQLITE4_UTF8, SQLITE4_STATIC); } } /* Force an SQLITE4_TOOBIG error. */ void sqlite4_result_error_toobig(sqlite4_context *pCtx){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); pCtx->isError = SQLITE4_TOOBIG; sqlite4VdbeMemSetStr(&pCtx->s, "string or blob too big", -1, - SQLITE4_UTF8, SQLITE4_STATIC, 0); + SQLITE4_UTF8, SQLITE4_STATIC); } /* An SQLITE4_NOMEM error. */ void sqlite4_result_error_nomem(sqlite4_context *pCtx){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); @@ -595,12 +589,11 @@ */ void sqlite4_set_auxdata( sqlite4_context *pCtx, int iArg, void *pAux, - void (*xDelete)(void*,void*), - void *pDeleteArg + void (*xDelete)(void*) ){ struct AuxData *pAuxData; VdbeFunc *pVdbeFunc; if( iArg<0 ) goto failed; @@ -619,20 +612,19 @@ pVdbeFunc->pFunc = pCtx->pFunc; } pAuxData = &pVdbeFunc->apAux[iArg]; if( pAuxData->pAux && pAuxData->xDelete ){ - pAuxData->xDelete(pAuxData->pDeleteArg, pAuxData->pAux); + pAuxData->xDelete(pAuxData->pAux); } pAuxData->pAux = pAux; pAuxData->xDelete = xDelete; - pAuxData->pDeleteArg = pDeleteArg; return; failed: if( xDelete ){ - xDelete(pDeleteArg, pAux); + xDelete(pAux); } } #ifndef SQLITE4_OMIT_DEPRECATED /* @@ -1014,36 +1006,35 @@ /* ** Bind a text or BLOB value. */ static int bindText( - sqlite4_stmt *pStmt, /* The statement to bind against */ - int i, /* Index of the parameter to bind */ - const void *zData, /* Pointer to the data to be bound */ - int nData, /* Number of bytes of data to be bound */ - void (*xDel)(void*,void*), /* Destructor for the data */ - void *pDelArg, /* First argument to xDel() */ - u8 encoding /* Encoding for the data */ + sqlite4_stmt *pStmt, /* The statement to bind against */ + int i, /* Index of the parameter to bind */ + const void *zData, /* Pointer to the data to be bound */ + int nData, /* Number of bytes of data to be bound */ + void (*xDel)(void*), /* Destructor for the data */ + u8 encoding /* Encoding for the data */ ){ Vdbe *p = (Vdbe *)pStmt; Mem *pVar; int rc; rc = vdbeUnbind(p, i); if( rc==SQLITE4_OK ){ if( zData!=0 ){ pVar = &p->aVar[i-1]; - rc = sqlite4VdbeMemSetStr(pVar, zData, nData, encoding, xDel, pDelArg); + rc = sqlite4VdbeMemSetStr(pVar, zData, nData, encoding, xDel); if( rc==SQLITE4_OK && encoding!=0 ){ rc = sqlite4VdbeChangeEncoding(pVar, ENC(p->db)); } sqlite4Error(p->db, rc, 0); rc = sqlite4ApiExit(p->db, rc); } sqlite4_mutex_leave(p->db->mutex); }else if( xDel!=SQLITE4_STATIC && xDel!=SQLITE4_TRANSIENT ){ - xDel(pDelArg, (void*)zData); + xDel((void*)zData); } return rc; } @@ -1053,14 +1044,13 @@ int sqlite4_bind_blob( sqlite4_stmt *pStmt, int i, const void *zData, int nData, - void (*xDel)(void*,void*), - void *pDelArg + void (*xDel)(void*) ){ - return bindText(pStmt, i, zData, nData, xDel, pDelArg, 0); + return bindText(pStmt, i, zData, nData, xDel, 0); } int sqlite4_bind_double(sqlite4_stmt *pStmt, int i, double rValue){ int rc; Vdbe *p = (Vdbe *)pStmt; rc = vdbeUnbind(p, i); @@ -1095,25 +1085,23 @@ int sqlite4_bind_text( sqlite4_stmt *pStmt, int i, const char *zData, int nData, - void (*xDel)(void*,void*), - void *pDelArg + void (*xDel)(void*) ){ - return bindText(pStmt, i, zData, nData, xDel, pDelArg, SQLITE4_UTF8); + return bindText(pStmt, i, zData, nData, xDel, SQLITE4_UTF8); } #ifndef SQLITE4_OMIT_UTF16 int sqlite4_bind_text16( sqlite4_stmt *pStmt, int i, const void *zData, int nData, - void (*xDel)(void*,void*), - void *pDelArg + void (*xDel)(void*) ){ - return bindText(pStmt, i, zData, nData, xDel, pDelArg, SQLITE4_UTF16NATIVE); + return bindText(pStmt, i, zData, nData, xDel, SQLITE4_UTF16NATIVE); } #endif /* SQLITE4_OMIT_UTF16 */ int sqlite4_bind_value(sqlite4_stmt *pStmt, int i, const sqlite4_value *pValue){ int rc; switch( pValue->type ){ @@ -1127,17 +1115,16 @@ } case SQLITE4_BLOB: { if( pValue->flags & MEM_Zero ){ rc = sqlite4_bind_zeroblob(pStmt, i, pValue->u.nZero); }else{ - rc = sqlite4_bind_blob(pStmt, i, pValue->z, pValue->n, - SQLITE4_TRANSIENT, 0); + rc = sqlite4_bind_blob(pStmt, i, pValue->z, pValue->n,SQLITE4_TRANSIENT); } break; } case SQLITE4_TEXT: { - rc = bindText(pStmt,i, pValue->z, pValue->n, SQLITE4_TRANSIENT, 0, + rc = bindText(pStmt,i, pValue->z, pValue->n, SQLITE4_TRANSIENT, pValue->enc); break; } default: { rc = sqlite4_bind_null(pStmt, i); Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -626,14 +626,10 @@ } case P4_VTAB : { if( db->pnBytesFreed==0 ) sqlite4VtabUnlock((VTable *)p4); break; } - case P4_FTS5INFO : { - sqlite4Fts5FreeInfo(db, (Fts5Info *)p4); - break; - } } } } /* @@ -1193,11 +1189,11 @@ return SQLITE4_ERROR; } pMem->flags = MEM_Dyn|MEM_Str|MEM_Term; z = displayP4(pOp, pMem->z, 32); if( z!=pMem->z ){ - sqlite4VdbeMemSetStr(pMem, z, -1, SQLITE4_UTF8, 0, 0); + sqlite4VdbeMemSetStr(pMem, z, -1, SQLITE4_UTF8, 0); }else{ assert( pMem->z!=0 ); pMem->n = sqlite4Strlen30(pMem->z); pMem->enc = SQLITE4_UTF8; } @@ -1498,11 +1494,10 @@ */ void sqlite4VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ if( pCx==0 ){ return; } - sqlite4Fts5Close(pCx->pFts); if( pCx->pKVCur ){ sqlite4KVCursorClose(pCx->pKVCur); } if( pCx->pTmpKV ){ sqlite4KVStoreClose(pCx->pTmpKV); @@ -1636,25 +1631,23 @@ int sqlite4VdbeSetColName( Vdbe *p, /* Vdbe being configured */ int idx, /* Index of column zName applies to */ int var, /* One of the COLNAME_* constants */ const char *zName, /* Pointer to buffer containing name */ - void (*xDel)(void*,void*) /* Memory management strategy for zName */ + void (*xDel)(void*) /* Memory management strategy for zName */ ){ int rc; Mem *pColName; assert( idxnResColumn ); assert( vardb->mallocFailed ){ assert( !zName || xDel!=SQLITE4_DYNAMIC ); return SQLITE4_NOMEM; } assert( p->aColName!=0 ); pColName = &(p->aColName[idx+var*p->nResColumn]); - rc = sqlite4VdbeMemSetStr(pColName, zName, -1, SQLITE4_UTF8, xDel, 0); + rc = sqlite4VdbeMemSetStr(pColName, zName, -1, SQLITE4_UTF8, xDel); assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 ); return rc; } /* @@ -1976,12 +1969,11 @@ sqlite4 *db = p->db; int rc = p->rc; if( p->zErrMsg ){ u8 mallocFailed = db->mallocFailed; sqlite4BeginBenignMalloc(db->pEnv); - sqlite4ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE4_UTF8, - SQLITE4_TRANSIENT, 0); + sqlite4ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE4_UTF8, SQLITE4_TRANSIENT); sqlite4EndBenignMalloc(db->pEnv); db->mallocFailed = mallocFailed; db->errCode = rc; }else{ sqlite4Error(db, rc, 0); @@ -2024,12 +2016,11 @@ /* The expired flag was set on the VDBE before the first call ** to sqlite4_step(). For consistency (since sqlite4_step() was ** called), set the database error in this case as well. */ sqlite4Error(db, p->rc, 0); - sqlite4ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE4_UTF8, - SQLITE4_TRANSIENT, 0); + sqlite4ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE4_UTF8, SQLITE4_TRANSIENT); sqlite4DbFree(db, p->zErrMsg); p->zErrMsg = 0; } /* Reclaim all memory used by the VDBE @@ -2087,11 +2078,11 @@ int i; for(i=0; inAux; i++){ struct AuxData *pAux = &pVdbeFunc->apAux[i]; if( (i>31 || !(mask&(((u32)1)<pAux ){ if( pAux->xDelete ){ - pAux->xDelete(pAux->pDeleteArg, pAux->pAux); + pAux->xDelete(pAux->pAux); } pAux->pAux = 0; } } } Index: src/vdbecodec.c ================================================================== --- src/vdbecodec.c +++ src/vdbecodec.c @@ -129,41 +129,34 @@ if( n!=size ) return SQLITE4_CORRUPT; r = (double)x; if( e&1 ) r = -r; if( e&2 ){ e = -(e>>2); - if( e==0 ){ - r *= 1e+300*1e+300; - }else{ - while( e<=-10 ){ r /= 1.0e10; e += 10; } - while( e<0 ){ r /= 10.0; e++; } - } + while( e<=-10 ){ r /= 1.0e10; e += 10; } + while( e<0 ){ r /= 10.0; e++; } }else{ e = e>>2; while( e>=10 ){ r *= 1.0e10; e -= 10; } while( e>0 ){ r *= 10.0; e--; } } sqlite4VdbeMemSetDouble(pOut, r); }else if( cclass==0 ){ if( size==0 ){ - sqlite4VdbeMemSetStr(pOut, "", 0, SQLITE4_UTF8, SQLITE4_TRANSIENT, 0); + sqlite4VdbeMemSetStr(pOut, "", 0, SQLITE4_UTF8, SQLITE4_TRANSIENT); }else if( p->a[ofst]>0x02 ){ sqlite4VdbeMemSetStr(pOut, (char*)(p->a+ofst), size, - SQLITE4_UTF8, SQLITE4_TRANSIENT, 0); + SQLITE4_UTF8, SQLITE4_TRANSIENT); }else{ - static const u8 enc[] = {SQLITE4_UTF8,SQLITE4_UTF16LE,SQLITE4_UTF16BE }; + static const u8 enc[] = { SQLITE4_UTF8, SQLITE4_UTF16LE, SQLITE4_UTF16BE }; sqlite4VdbeMemSetStr(pOut, (char*)(p->a+ofst+1), size-1, - enc[p->a[ofst]], SQLITE4_TRANSIENT, 0); + enc[p->a[ofst]], SQLITE4_TRANSIENT); } }else{ - sqlite4VdbeMemSetStr(pOut, (char*)(p->a+ofst), size, 0, - SQLITE4_TRANSIENT, 0); + sqlite4VdbeMemSetStr(pOut, (char*)(p->a+ofst), size, 0, SQLITE4_TRANSIENT); } } - testcase( i==iVal ); - testcase( i==iVal+1 ); - if( i<=iVal ){ + if( inOut<=p->nAlloc ); if( p->nOut+needed>p->nAlloc ){ u8 *aNew; p->nAlloc = p->nAlloc + needed + 10; aNew = sqlite4DbRealloc(p->db, p->aOut, p->nAlloc); if( aNew==0 ){ @@ -602,12 +594,12 @@ memcpy(p->aOut+p->nOut, pEnc->z, pEnc->n); p->nOut += pEnc->n; }else{ int nSpc = p->nAlloc-p->nOut; n = pColl->xMkKey(pColl->pUser, pEnc->n, pEnc->z, nSpc, p->aOut+p->nOut); - if( n+1>nSpc ){ - if( enlargeEncoderAllocation(p, n+1) ) return SQLITE4_NOMEM; + if( n>nSpc ){ + if( enlargeEncoderAllocation(p, n) ) return SQLITE4_NOMEM; n = pColl->xMkKey(pColl->pUser, pEnc->n, pEnc->z, n, p->aOut+p->nOut); } p->nOut += n; } p->aOut[p->nOut++] = 0x00; @@ -642,11 +634,10 @@ p->aOut[p->nOut++] = 0x00; } if( sortOrder==SQLITE4_SO_DESC ){ for(i=iStart; inOut; i++) p->aOut[i] ^= 0xff; } - assert( p->nOut<=p->nAlloc ); return SQLITE4_OK; } /* ** Variables aKey/nKey contain an encoded index key. This function returns Index: src/vdbemem.c ================================================================== --- src/vdbemem.c +++ src/vdbemem.c @@ -91,11 +91,11 @@ if( pMem->z && preserve && pMem->zMalloc && pMem->z!=pMem->zMalloc ){ memcpy(pMem->zMalloc, pMem->z, pMem->n); } if( pMem->flags&MEM_Dyn && pMem->xDel ){ assert( pMem->xDel!=SQLITE4_DYNAMIC ); - pMem->xDel(pMem->pDelArg, (void *)(pMem->z)); + pMem->xDel((void *)(pMem->z)); } pMem->z = pMem->zMalloc; if( pMem->z==0 ){ pMem->flags = MEM_Null; @@ -242,11 +242,11 @@ assert( (p->flags & MEM_Agg)==0 ); sqlite4VdbeMemRelease(p); }else if( p->flags&MEM_Dyn && p->xDel ){ assert( (p->flags&MEM_RowSet)==0 ); assert( p->xDel!=SQLITE4_DYNAMIC ); - p->xDel(p->pDelArg, (void *)p->z); + p->xDel((void *)p->z); p->xDel = 0; }else if( p->flags&MEM_RowSet ){ sqlite4RowSetClear(p->u.pRowSet); }else if( p->flags&MEM_Frame ){ sqlite4VdbeMemSetNull(p); @@ -634,16 +634,15 @@ ** stored without allocating memory, then it is. If a memory allocation ** is required to store the string, then value of pMem is unchanged. In ** either case, SQLITE4_TOOBIG is returned. */ int sqlite4VdbeMemSetStr( - Mem *pMem, /* Memory cell to set to string value */ - const char *z, /* String pointer */ - int n, /* Bytes in string, or negative */ - u8 enc, /* Encoding of z. 0 for BLOBs */ - void (*xDel)(void*,void*),/* Destructor function */ - void *pDelArg /* First argument to xDel() */ + Mem *pMem, /* Memory cell to set to string value */ + const char *z, /* String pointer */ + int n, /* Bytes in string, or negative */ + u8 enc, /* Encoding of z. 0 for BLOBs */ + void (*xDel)(void*) /* Destructor function */ ){ int nByte = n; /* New value for pMem->n */ int iLimit; /* Maximum allowed string or blob size */ u16 flags = 0; /* New value for pMem->flags */ @@ -694,11 +693,10 @@ pMem->xDel = 0; }else{ sqlite4VdbeMemRelease(pMem); pMem->z = (char *)z; pMem->xDel = xDel; - pMem->pDelArg = pDelArg; flags |= ((xDel==SQLITE4_STATIC)?MEM_Static:MEM_Dyn); } pMem->n = nByte; pMem->flags = flags; @@ -951,11 +949,11 @@ if( ExprHasProperty(pExpr, EP_IntValue) ){ sqlite4VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); }else{ zVal = sqlite4MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); if( zVal==0 ) goto no_mem; - sqlite4ValueSetStr(pVal, -1, zVal, SQLITE4_UTF8, SQLITE4_DYNAMIC, 0); + sqlite4ValueSetStr(pVal, -1, zVal, SQLITE4_UTF8, SQLITE4_DYNAMIC); if( op==TK_FLOAT ) pVal->type = SQLITE4_FLOAT; } if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE4_AFF_NONE ){ sqlite4ValueApplyAffinity(pVal, SQLITE4_AFF_NUMERIC, SQLITE4_UTF8); }else{ @@ -992,11 +990,11 @@ if( !pVal ) goto no_mem; zVal = &pExpr->u.zToken[2]; nVal = sqlite4Strlen30(zVal)-1; assert( zVal[nVal]=='\'' ); sqlite4VdbeMemSetStr(pVal, sqlite4HexToBlob(db, zVal, nVal), nVal/2, - 0, SQLITE4_DYNAMIC, 0); + 0, SQLITE4_DYNAMIC); } #endif if( pVal ){ sqlite4VdbeMemStoreType(pVal); @@ -1014,18 +1012,17 @@ /* ** Change the string value of an sqlite4_value object */ void sqlite4ValueSetStr( - sqlite4_value *v, /* Value to be set */ - int n, /* Length of string z */ - const void *z, /* Text of the new string */ - u8 enc, /* Encoding to use */ - void (*xDel)(void*,void*), /* Destructor for the string */ - void *pDelArg /* First argument to xDel() */ + sqlite4_value *v, /* Value to be set */ + int n, /* Length of string z */ + const void *z, /* Text of the new string */ + u8 enc, /* Encoding to use */ + void (*xDel)(void*) /* Destructor for the string */ ){ - if( v ) sqlite4VdbeMemSetStr((Mem *)v, z, n, enc, xDel, pDelArg); + if( v ) sqlite4VdbeMemSetStr((Mem *)v, z, n, enc, xDel); } /* ** Free an sqlite4_value object */ Index: src/vdbetrace.c ================================================================== --- src/vdbetrace.c +++ src/vdbetrace.c @@ -128,11 +128,11 @@ u8 enc = ENC(db); if( enc!=SQLITE4_UTF8 ){ Mem utf8; memset(&utf8, 0, sizeof(utf8)); utf8.db = db; - sqlite4VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE4_STATIC, 0); + sqlite4VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE4_STATIC); sqlite4VdbeChangeEncoding(&utf8, SQLITE4_UTF8); sqlite4XPrintf(&out, "'%.*q'", utf8.n, utf8.z); sqlite4VdbeMemRelease(&utf8); }else #endif Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -770,11 +770,11 @@ ExprList *pList; if( pExpr->op!=TK_FUNCTION ){ return 0; } - if( sqlite4_stricmp(pExpr->u.zToken,"match")!=0 ){ + if( sqlite4StrICmp(pExpr->u.zToken,"match")!=0 ){ return 0; } pList = pExpr->x.pList; if( pList->nExpr!=2 ){ return 0; @@ -1456,11 +1456,11 @@ && p->iColumn==pIdx->aiColumn[iCol] && p->iTable==iBase ){ CollSeq *pColl = sqlite4ExprCollSeq(pParse, p); assert( pColl || p->iColumn==-1 ); - if( 0==pColl || 0==sqlite4_stricmp(pColl->zName, zColl) ){ + if( 0==pColl || 0==sqlite4StrICmp(pColl->zName, zColl) ){ return i; } } } @@ -1660,11 +1660,11 @@ Index *pPk; Table *pTab; if( !pOrderBy ) return 0; if( wsFlags & WHERE_COLUMN_IN ) return 0; - if( pIdx->fIndex & IDX_Unordered ) return 0; + if( pIdx->bUnordered ) return 0; pTab = pIdx->pTable; pPk = sqlite4FindPrimaryKey(pTab, 0); nTerm = pOrderBy->nExpr; nIdxCol = pIdx->nColumn + (pIdx==pPk ? 0 : pPk->nColumn); @@ -2862,58 +2862,10 @@ } return rc; } #endif /* defined(SQLITE4_ENABLE_STAT3) */ -/* -** Try to find a MATCH expression that constrains the pTabItem table in the -** WHERE clause. If one exists, set *piTerm to the index in the pWC->a[] array -** and return non-zero. If no such expression exists, return 0. -*/ -static int findMatchExpr( - Parse *pParse, - WhereClause *pWC, - SrcListItem *pTabItem, - int *piTerm -){ - int i; - int iCsr = pTabItem->iCursor; - - for(i=0; inTerm; i++){ - Expr *pMatch = pWC->a[i].pExpr; - if( pMatch->iTable==iCsr && pMatch->op==TK_MATCH ) break; - } - if( i==pWC->nTerm ) return 0; - - *piTerm = i; - return 1; -} - -static int bestMatchIdx( - Parse *pParse, - WhereClause *pWC, - SrcListItem *pTabItem, - Bitmask notReady, - WhereCost *pCost -){ - int iTerm; - - if( 0==findMatchExpr(pParse, pWC, pTabItem, &iTerm) ) return 0; - - /* Check that the MATCH expression is not composed using values from any - ** tables that are not ready. If it does, return 0. */ - if( notReady & pWC->a[iTerm].prereqAll ) return 0; - - pCost->used = pWC->a[iTerm].prereqAll; - pCost->rCost = 1.0; - pCost->plan.wsFlags = WHERE_INDEXED; - pCost->plan.nEq = 0; - pCost->plan.nRow = 10; - pCost->plan.u.pIdx = pWC->a[iTerm].pExpr->pIdx; - return 1; -} - /* ** Find the best query plan for accessing a particular table. Write the ** best query plan and its cost into the WhereCost object supplied as the ** last parameter. ** @@ -3071,12 +3023,10 @@ #ifdef SQLITE4_ENABLE_STAT3 WhereTerm *pFirstTerm = 0; /* First term matching the index */ #endif int nCol = pProbe->nColumn; /* Total columns in index record */ - if( pProbe->eIndexType==SQLITE4_INDEX_FTS5 ) continue; - /* Unless pProbe is the primary key index, then the encoded PK column ** values are at the end of each record. Set variable nCol to the total ** number of columns encoded into each index record, including the PK ** columns. */ if( pProbe!=pPk ) nCol += pPk->nColumn; @@ -3122,11 +3072,11 @@ testcase( wsFlags & WHERE_COLUMN_IN ); testcase( wsFlags & WHERE_COLUMN_NULL ); if( (wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){ wsFlags |= WHERE_UNIQUE; } - }else if( (pProbe->fIndex & IDX_Unordered)==0 ){ + }else if( pProbe->bUnordered==0 ){ int j = idxColumnNumber(pProbe, pPk, nEq); if( findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE|WO_GT|WO_GE, pProbe) ){ WhereTerm *pTop = findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE, pProbe); WhereTerm *pBtm = findTerm(pWC, iCur, j, notReady, WO_GT|WO_GE, pProbe); whereRangeScanEst(pParse, pProbe, nEq, pBtm, pTop, &rangeDiv); @@ -3926,31 +3876,10 @@ pLevel->iLeftJoin = ++pParse->nMem; sqlite4VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); VdbeComment((v, "init LEFT JOIN no-match flag")); } - if( (pLevel->plan.wsFlags & WHERE_INDEXED) - && (pLevel->plan.u.pIdx->eIndexType==SQLITE4_INDEX_FTS5) - ){ - /* Case -1: An FTS query */ - int iTerm; - int rMatch; - int rFree; - findMatchExpr(pParse, pWC, pTabItem, &iTerm); - - rMatch = sqlite4ExprCodeTemp(pParse, pWC->a[iTerm].pExpr->pRight, &rFree); - pWC->a[iTerm].wtFlags |= TERM_CODED; - sqlite4Fts5CodeQuery(pParse, - pLevel->plan.u.pIdx, pLevel->iIdxCur, addrBrk, rMatch - ); - sqlite4ReleaseTempReg(pParse, rFree); - - pLevel->p2 = sqlite4VdbeCurrentAddr(v); - sqlite4VdbeAddOp3(v, OP_SeekPk, iCur, 0, pLevel->iIdxCur); - pLevel->op = OP_FtsNext; - pLevel->p1 = pLevel->iIdxCur; - }else #ifndef SQLITE4_OMIT_VIRTUALTABLE if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){ /* Case 0: The table is a virtual-table. Use the VFilter and VNext ** to access the data. */ @@ -4235,11 +4164,11 @@ ** If it is, jump to the next iteration of the loop. */ r1 = sqlite4GetTempReg(pParse); testcase( pLevel->plan.wsFlags & WHERE_BTM_LIMIT ); testcase( pLevel->plan.wsFlags & WHERE_TOP_LIMIT ); if( (pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){ - sqlite4ExprCodeGetColumnOfTable(v, pIdx->pTable, iCur, iIneq, r1); + sqlite4VdbeAddOp3(v, OP_Column, iCur, iIneq, r1); sqlite4VdbeAddOp2(v, OP_IsNull, r1, addrCont); } sqlite4ReleaseTempReg(pParse, r1); /* Record the instruction used to terminate the loop. Disable @@ -4854,14 +4783,10 @@ if( pTabItem->pIndex==0 ) nUnconstrained++; WHERETRACE(("=== trying table %d with isOptimal=%d ===\n", j, isOptimal)); assert( pTabItem->pTab ); - - if( bestMatchIdx(pParse, pWC, pTabItem, notReady, &sCost) ){ - /* no-op */ - }else #ifndef SQLITE4_OMIT_VIRTUALTABLE if( IsVirtual(pTabItem->pTab) ){ sqlite4_index_info **pp = &pWInfo->a[j].pIdxInfo; bestVirtualIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, &sCost, pp); @@ -4869,11 +4794,10 @@ #endif { bestKVIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, pDist, &sCost); } - assert( isOptimal || (sCost.used¬Ready)==0 ); /* If an INDEXED BY clause is present, then the plan must use that ** index if it uses any index at all */ assert( pTabItem->pIndex==0 @@ -5031,11 +4955,11 @@ #endif if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ Index *pIx = pLevel->plan.u.pIdx; if( pIx->eIndexType==SQLITE4_INDEX_PRIMARYKEY ){ pLevel->iIdxCur = pTabItem->iCursor; - }else if( pIx->eIndexType!=SQLITE4_INDEX_FTS5 ){ + }else{ KeyInfo *pKey = sqlite4IndexKeyinfo(pParse, pIx); int iIdxCur = pLevel->iIdxCur; assert( pIx->pSchema==pTab->pSchema ); assert( iIdxCur>=0 ); sqlite4VdbeAddOp4(v, OP_OpenRead, iIdxCur, pIx->tnum, iDb, @@ -5224,28 +5148,10 @@ while( pOpp1==pLevel->iTabCur && pOp->opcode==OP_Column ){ pOp->p1 = pLevel->iIdxCur; } - pOp++; - } - } - - if( (pLevel->plan.wsFlags & WHERE_INDEXED) - && (pLevel->plan.u.pIdx->eIndexType==SQLITE4_INDEX_FTS5) - ){ - VdbeOp *pOp; - VdbeOp *pEnd; - - assert( pLevel->iTabCur!=pLevel->iIdxCur ); - pOp = sqlite4VdbeGetOp(v, pWInfo->iTop); - pEnd = &pOp[sqlite4VdbeCurrentAddr(v) - pWInfo->iTop]; - - while( pOpp1==pLevel->iTabCur && pOp->opcode==OP_Mifunction ){ - pOp->p1 = pLevel->iIdxCur; - } pOp++; } } } Index: test/badutf.test ================================================================== --- test/badutf.test +++ test/badutf.test @@ -11,10 +11,11 @@ # This file implements regression tests for SQLite library. # # This file checks to make sure SQLite is able to gracefully # handle malformed UTF-8. # +# $Id: badutf.test,v 1.2 2007/09/12 17:01:45 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl do_test badutf-1.1 { @@ -24,23 +25,23 @@ do_test badutf-1.2 { sqlite4_exec db {SELECT hex('%81') AS x} } {0 {x 81}} do_test badutf-1.3 { sqlite4_exec db {SELECT hex('%bf') AS x} -} {0 {x bf}} +} {0 {x BF}} do_test badutf-1.4 { sqlite4_exec db {SELECT hex('%c0') AS x} -} {0 {x c0}} +} {0 {x C0}} do_test badutf-1.5 { sqlite4_exec db {SELECT hex('%e0') AS x} -} {0 {x e0}} +} {0 {x E0}} do_test badutf-1.6 { sqlite4_exec db {SELECT hex('%f0') AS x} -} {0 {x f0}} +} {0 {x F0}} do_test badutf-1.7 { sqlite4_exec db {SELECT hex('%ff') AS x} -} {0 {x ff}} +} {0 {x FF}} sqlite4 db2 {} ifcapable utf16 { do_test badutf-1.10 { db2 eval {PRAGMA encoding=UTF16be} @@ -49,35 +50,35 @@ do_test badutf-1.11 { sqlite4_exec db2 {SELECT hex('%81') AS x} } {0 {x 0081}} do_test badutf-1.12 { sqlite4_exec db2 {SELECT hex('%bf') AS x} - } {0 {x 00bf}} + } {0 {x 00BF}} do_test badutf-1.13 { sqlite4_exec db2 {SELECT hex('%c0') AS x} - } {0 {x fffd}} + } {0 {x FFFD}} do_test badutf-1.14 { sqlite4_exec db2 {SELECT hex('%c1') AS x} - } {0 {x fffd}} + } {0 {x FFFD}} do_test badutf-1.15 { sqlite4_exec db2 {SELECT hex('%c0%bf') AS x} - } {0 {x fffd}} + } {0 {x FFFD}} do_test badutf-1.16 { sqlite4_exec db2 {SELECT hex('%c1%bf') AS x} - } {0 {x fffd}} + } {0 {x FFFD}} do_test badutf-1.17 { sqlite4_exec db2 {SELECT hex('%c3%bf') AS x} - } {0 {x 00ff}} + } {0 {x 00FF}} do_test badutf-1.18 { sqlite4_exec db2 {SELECT hex('%e0') AS x} - } {0 {x fffd}} + } {0 {x FFFD}} do_test badutf-1.19 { sqlite4_exec db2 {SELECT hex('%f0') AS x} - } {0 {x fffd}} + } {0 {x FFFD}} do_test badutf-1.20 { sqlite4_exec db2 {SELECT hex('%ff') AS x} - } {0 {x fffd}} + } {0 {x FFFD}} } ifcapable bloblit { do_test badutf-2.1 { @@ -116,27 +117,27 @@ sqlite4_exec db {SELECT length('%80%80%80%80%80%f0%80%80%80%ff') AS x} } {0 {x 7}} do_test badutf-4.1 { sqlite4_exec db {SELECT hex(trim('%80%80%80%f0%80%80%80%ff','%80%ff')) AS x} -} {0 {x f0}} +} {0 {x F0}} do_test badutf-4.2 { sqlite4_exec db {SELECT hex(ltrim('%80%80%80%f0%80%80%80%ff','%80%ff')) AS x} -} {0 {x f0808080ff}} +} {0 {x F0808080FF}} do_test badutf-4.3 { sqlite4_exec db {SELECT hex(rtrim('%80%80%80%f0%80%80%80%ff','%80%ff')) AS x} -} {0 {x 808080f0}} +} {0 {x 808080F0}} do_test badutf-4.4 { sqlite4_exec db {SELECT hex(trim('%80%80%80%f0%80%80%80%ff','%ff%80')) AS x} -} {0 {x 808080f0808080ff}} +} {0 {x 808080F0808080FF}} do_test badutf-4.5 { sqlite4_exec db {SELECT hex(trim('%ff%80%80%f0%80%80%80%ff','%ff%80')) AS x} -} {0 {x 80f0808080ff}} +} {0 {x 80F0808080FF}} do_test badutf-4.6 { sqlite4_exec db {SELECT hex(trim('%ff%80%f0%80%80%80%ff','%ff%80')) AS x} -} {0 {x f0808080ff}} +} {0 {x F0808080FF}} do_test badutf-4.7 { sqlite4_exec db {SELECT hex(trim('%ff%80%f0%80%80%80%ff','%ff%80%80')) AS x} -} {0 {x ff80f0808080ff}} +} {0 {x FF80F0808080FF}} db2 close finish_test Index: test/bind.test ================================================================== --- test/bind.test +++ test/bind.test @@ -301,19 +301,19 @@ } {hello hello hello} set enc [db eval {PRAGMA encoding}] if {$enc=="UTF-8" || $enc==""} { do_test bind-6.5 { execsql {SELECT hex(a), hex(b), hex(c) FROM t1} - } {68656c6c6f00746865726500 68656c6c6f007468657265 68656c6c6f} + } {68656C6C6F00746865726500 68656C6C6F007468657265 68656C6C6F} } elseif {$enc=="UTF-16le"} { do_test bind-6.5 { execsql {SELECT hex(a), hex(b), hex(c) FROM t1} - } {680065006c006c006f000000740068006500720065000000 680065006c006c006f00000074006800650072006500 680065006c006c006f00} + } {680065006C006C006F000000740068006500720065000000 680065006C006C006F00000074006800650072006500 680065006C006C006F00} } elseif {$enc=="UTF-16be"} { do_test bind-6.5 { execsql {SELECT hex(a), hex(b), hex(c) FROM t1} - } {00680065006c006c006f0000007400680065007200650000 00680065006c006c006f000000740068006500720065 00680065006c006c006f} + } {00680065006C006C006F0000007400680065007200650000 00680065006C006C006F000000740068006500720065 00680065006C006C006F} } else { do_test bind-6.5 { set "Unknown database encoding: $::enc" } {} } @@ -350,19 +350,19 @@ execsql {SELECT * FROM t1} } {hi hi hi} if {$enc=="UTF-8"} { do_test bind-7.4 { execsql {SELECT hex(a), hex(b), hex(c) FROM t1} - } {68690079616c6c00 68690079616c6c 6869} + } {68690079616C6C00 68690079616C6C 6869} } elseif {$enc=="UTF-16le"} { do_test bind-7.4 { execsql {SELECT hex(a), hex(b), hex(c) FROM t1} - } {680069000000790061006c006c000000 680069000000790061006c006c00 68006900} + } {680069000000790061006C006C000000 680069000000790061006C006C00 68006900} } elseif {$enc=="UTF-16be"} { do_test bind-7.4 { execsql {SELECT hex(a), hex(b), hex(c) FROM t1} - } {00680069000000790061006c006c0000 00680069000000790061006c006c 00680069} + } {00680069000000790061006C006C0000 00680069000000790061006C006C 00680069} } do_test bind-7.5 { execsql {SELECT typeof(a), typeof(b), typeof(c) FROM t1} } {text text text} } @@ -624,17 +624,17 @@ sqlite4_finalize $VM execsql { SELECT typeof(x), length(x), quote(x), length(cast(x AS BLOB)), quote(cast(x AS BLOB)) FROM t3 } - } {text 3 'abc' 10 x'6162630078797a007071'} + } {text 3 'abc' 10 X'6162630078797A007071'} do_test bind-12.2 { sqlite4_create_function $DB execsql { SELECT quote(cast(x_coalesce(x) AS blob)) FROM t3 } - } {x'6162630078797a007071'} + } {X'6162630078797A007071'} } # Test the operation of sqlite4_clear_bindings # do_test bind-13.1 { Index: test/ckpt1.test ================================================================== --- test/ckpt1.test +++ test/ckpt1.test @@ -74,23 +74,21 @@ INSERT INTO t1 SELECT randstr(100,100), randstr(100,100) FROM t1; -- 16K INSERT INTO t1 SELECT randstr(100,100), randstr(100,100) FROM t1; -- 32K INSERT INTO t1 SELECT randstr(100,100), randstr(100,100) FROM t1; -- 64K } do_test 3.2 { - sqlite4_lsm_flush db main - sqlite4_lsm_work db main -nmerge 1 -npage 1000000 + sqlite4_lsm_work db main -optimize 1000000 execsql { SELECT count(*) FROM t1 } } {65536} do_test 3.3 { db close sqlite4 db test.db execsql { SELECT count(*) FROM t1 } } {65536} do_test 3.4 { execsql { INSERT INTO t1 VALUES(randstr(100,100), randstr(100,100)) } - sqlite4_lsm_flush db main - sqlite4_lsm_work db main -nmerge 1 -npage 1000000 + sqlite4_lsm_work db main -optimize 1000000 execsql { SELECT count(*) FROM t1 } } {65537} finish_test Index: test/csr1.test ================================================================== --- test/csr1.test +++ test/csr1.test @@ -40,14 +40,14 @@ do_execsql_test 1.1 { SELECT * FROM t1 } { 0 0 1 1 2 4 3 9 4 16 5 25 6 36 7 49 8 64 9 81 } db eval { SELECT a, b FROM t1 } { do_test 1.2.1 { - list [catch { sqlite4_lsm_work db main -nmerge 2 -npage 10 } msg] $msg + list [catch { sqlite4_lsm_work db main -flush 0 } msg] $msg } {1 SQLITE4_MISUSE} do_test 1.2.2 { - list [catch { sqlite4_lsm_work db main -nmerge 2 -npage 10 } msg] $msg + list [catch { sqlite4_lsm_work db main 1 } msg] $msg } {1 SQLITE4_MISUSE} break } @@ -62,11 +62,11 @@ sqlite4 db2 ./test.db list [catch { db2 eval { BEGIN ; INSERT INTO t1 VALUES(1, 2) } } msg] $msg } {1 {database is locked}} do_execsql_test 2.3 { COMMIT } -do_test 2.4 { sqlite4_lsm_work db2 main -npage 0 } {0} +do_test 2.4 { sqlite4_lsm_work db2 main -flush 0 } {0} db2 close #------------------------------------------------------------------------- # Check that if a transaction is committed and this causes the in-memory @@ -75,12 +75,12 @@ populate_db_2 do_execsql_test 3.1 { BEGIN; INSERT INTO t1 VALUES(10, randstr(910, 910)); } -do_test 3.2 { sqlite4_lsm_config db main autoflush } [expr 1*1024*1024] -do_test 3.3 { sqlite4_lsm_config db main autoflush 4096 } 4096 +do_test 3.2 { sqlite4_lsm_config db main write-buffer } [expr 2*1024*1024] +do_test 3.3 { sqlite4_lsm_config db main write-buffer 4096 } 4096 do_test 3.4 { set res [list] db eval { SELECT a, length(b) AS l FROM t1 } { lappend res $a $l Index: test/ctime.test ================================================================== --- test/ctime.test +++ test/ctime.test @@ -71,10 +71,15 @@ do_test ctime-1.4.2 { catchsql { SELECT sqlite_compileoption_used('THREADSAFE'); } } {0 1} +do_test ctime-1.4.3 { + catchsql { + SELECT sqlite_compileoption_used("THREADSAFE"); + } +} {0 1} do_test ctime-1.5 { set ans1 [ catchsql { SELECT sqlite_compileoption_used('THREADSAFE=0'); } ] @@ -115,10 +120,15 @@ do_test ctime-2.1.2 { catchsql { SELECT sqlite_compileoption_used(NULL); } } {0 {{}}} +do_test ctime-2.1.3 { + catchsql { + SELECT sqlite_compileoption_used(""); + } +} {0 0} do_test ctime-2.1.4 { catchsql { SELECT sqlite_compileoption_used(''); } } {0 0} DELETED test/fts5create.test Index: test/fts5create.test ================================================================== --- test/fts5create.test +++ /dev/null @@ -1,99 +0,0 @@ -# 2012 December 18 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#************************************************************************* -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix fts5create - -do_execsql_test 1.1 { - CREATE TABLE t1(a, b, c); -} - -do_execsql_test 1.2 { - CREATE INDEX ft1 ON t1 USING fts5(); - SELECT * FROM sqlite_master; -} { - table t1 t1 2 "CREATE TABLE t1(a, b, c)" - index ft1 t1 3 "CREATE INDEX ft1 ON t1 USING fts5()" -} - -do_execsql_test 1.3 { - DROP INDEX ft1; - SELECT * FROM sqlite_master; -} { - table t1 t1 2 "CREATE TABLE t1(a, b, c)" -} - -do_execsql_test 1.4 { - CREATE INDEX ft1 ON t1 USING fts5(); - SELECT * FROM sqlite_master; -} { - table t1 t1 2 "CREATE TABLE t1(a, b, c)" - index ft1 t1 3 "CREATE INDEX ft1 ON t1 USING fts5()" -} - -do_execsql_test 1.5 { - DROP INDEX ft1; - SELECT * FROM sqlite_master; -} { - table t1 t1 2 "CREATE TABLE t1(a, b, c)" -} - -do_catchsql_test 1.6 { - CREATE UNIQUE INDEX ft2 ON t1 USING fts5(); -} {1 {USING fts5 index may not be UNIQUE}} - -#------------------------------------------------------------------------- -# -reset_db - -do_execsql_test 2.1 { - CREATE TABLE t2(x, y); - CREATE INDEX fulltext ON t2 USING fts5(tokenizer=simple); - SELECT * FROM sqlite_master; -} { - table t2 t2 2 {CREATE TABLE t2(x, y)} - index fulltext t2 3 {CREATE INDEX fulltext ON t2 USING fts5(tokenizer=simple)} -} - -do_catchsql_test 2.2 { - DROP INDEX fulltext; - CREATE INDEX ft ON t2 USING fts5(tukenizer=simple); -} {1 {unrecognized argument: "tukenizer"}} - -do_catchsql_test 2.3 { - CREATE INDEX ft ON t2 USING fts5("a b c"); -} {1 {unrecognized argument: "a b c"}} - -do_catchsql_test 2.4 { - CREATE INDEX ft ON t2 USING fts5(tokenizer="nosuch"); -} {1 {no such tokenizer: "nosuch"}} - -#------------------------------------------------------------------------- -# -reset_db - -do_execsql_test 3.1 { - CREATE TABLE t1(a, b, c, PRIMARY KEY(a)); - INSERT INTO t1 VALUES(1, 'a b c d', 'e f g h'); - INSERT INTO t1 VALUES(2, 'e f g h', 'a b c d'); -} - -do_execsql_test 3.2 { - CREATE INDEX ft ON t1 USING fts5(); - PRAGMA fts_check(ft); -} {ok} - - -finish_test - - DELETED test/fts5expr1.test Index: test/fts5expr1.test ================================================================== --- test/fts5expr1.test +++ /dev/null @@ -1,74 +0,0 @@ -# 2012 December 17 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#************************************************************************* -# -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix fts5expr1 - -foreach {tn expr res} { - 1 { abc } {"abc"} - 2 { abc AND def } {"abc" AND "def"} - 3 { (abc) } {"abc"} - 4 { (((abc))) } {"abc"} - 5 { one OR two AND three } {"one" OR ("two" AND "three")} - 6 { one AND two OR three } {("one" AND "two") OR "three"} - - 7 { "abc def" } {"abc"+"def"} - 8 { abc+def } {"abc"+"def"} - 9 { "abc def" + ghi } {"abc"+"def"+"ghi"} - - 10 { one AND two OR three } {("one" AND "two") OR "three"} - 11 { one AND (two OR three) } {"one" AND ("two" OR "three")} - - 12 { "abc""def" } {"abc"+"def"} - 13 { "abc" "def" } {"abc" AND "def"} - 14 { "abc" + "def" } {"abc"+"def"} - - 15 { a NOT b AND c OR d } {(("a" NOT "b") AND "c") OR "d"} - 16 { a OR b AND c NOT d } {"a" OR ("b" AND ("c" NOT "d"))} - 17 { (a OR b) AND c NOT d } {("a" OR "b") AND ("c" NOT "d")} - - 18 { a NEAR/10 b } {"a" NEAR/10 "b"} - 19 { a NEAR b } {"a" NEAR/10 "b"} - 20 { a AND b OR c + d NEAR/52 e } {("a" AND "b") OR ("c"+"d" NEAR/52 "e")} - -} { - do_execsql_test 1.$tn { - SELECT fts5_parse_expr('simple', $expr, '') - } [list [string trim $res]] -} - -do_execsql_test 2.0 { - CREATE TABLE t1(a, b, c); -} - -foreach {tn expr res} { - 1 { a : abc } {"a":"abc"} - 2 { b : abc + def} {"b":"abc"+"def"} -} { - do_execsql_test 2.$tn { - SELECT fts5_parse_expr('simple', $expr, 't1') - } [list [string trim $res]] -} - -breakpoint -foreach {tn expr res} { - 1 { abc* } {"abc"*} -} { - do_execsql_test 3.$tn { - SELECT fts5_parse_expr('simple', $expr, 't1') - } [list [string trim $res]] -} - - -finish_test DELETED test/fts5query1.test Index: test/fts5query1.test ================================================================== --- test/fts5query1.test +++ /dev/null @@ -1,184 +0,0 @@ -# 2012 December 17 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#************************************************************************* -# -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix fts5query1 - -do_execsql_test 1.0 { - CREATE TABLE t1(a PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1 USING fts5(); -} - -foreach {tn stmt} { - 1 "INSERT INTO t1 VALUES(1, 'a b c', 'd e f')" - 2 "INSERT INTO t1 VALUES(2, 'b c e', 'A A a')" - 3 "INSERT INTO t1 VALUES(3, 'd A A', 'e c a')" - 4 "DELETE FROM t1 WHERE a=1" - 5 "DELETE FROM t1" - 6 "INSERT INTO t1 VALUES(1, 'May you do', 'good and not evil')" - 7 "INSERT INTO t1 VALUES(2, 'May you find', 'forgiveness for yourself')" - 8 "UPDATE t1 SET b = 'and forgive others' WHERE a = 2" - 9 "UPDATE t1 SET a = 4, c = 'a b c d' WHERE a = 2" -} { - do_execsql_test 1.$tn.1 $stmt - do_execsql_test 1.$tn.2 {PRAGMA fts_check(i1)} ok -} - -do_execsql_test 2.0 { - DROP TABLE t1; - CREATE TABLE t1(x PRIMARY KEY, y); - CREATE INDEX i1 ON t1 USING fts5(); - - INSERT INTO t1 VALUES(1, 'o n e'); - INSERT INTO t1 VALUES(2, 't w o'); - INSERT INTO t1 VALUES(3, 't h r e e'); - INSERT INTO t1 VALUES(4, 'f o u r'); -} - -foreach {tn stmt res} { - 1 {SELECT x FROM t1 WHERE t1 MATCH 't'} {2 3} - 2 {SELECT x FROM t1 WHERE t1 MATCH 'abc'} {} - 3 {SELECT x FROM t1 WHERE t1 MATCH 't+h'} {3} - 4 {SELECT x FROM t1 WHERE t1 MATCH 't+o'} {} -} { - do_execsql_test 2.$tn $stmt $res -} - -do_execsql_test 3.0 { - DROP TABLE t1; - CREATE TABLE t1(x PRIMARY KEY, y); - CREATE INDEX i1 ON t1 USING fts5(); - - INSERT INTO t1 VALUES(1, 'a b c d e f g h i j k l m n o p q r s t u'); - INSERT INTO t1 VALUES(2, 'a e i o u b c d f g h j k l m n p q r s t'); - INSERT INTO t1 VALUES(3, 'b c d f g h j k l m n p q r s t v w x y z'); - INSERT INTO t1 VALUES(4, 'a e i o u'); -} - -foreach {tn stmt res} { - 1 {SELECT x FROM t1 WHERE t1 MATCH 'a NEAR/5 i'} {2 4} - 2 {SELECT x FROM t1 WHERE t1 MATCH 'a NEAR/3 b'} {1} - 3 {SELECT x FROM t1 WHERE t1 MATCH 'a NEAR/2 d'} {1} - 4 {SELECT x FROM t1 WHERE t1 MATCH 'a NEAR/2 e'} {2 4} - 5 {SELECT x FROM t1 WHERE t1 MATCH 'a NEAR/3 e'} {1 2 4} - 6 {SELECT x FROM t1 WHERE t1 MATCH 'b+c NEAR/2 g+h'} {2 3} - 7 {SELECT x FROM t1 WHERE t1 MATCH 'b+c NEAR/3 g+h'} {1 2 3} - 8 {SELECT x FROM t1 WHERE t1 MATCH 'b+c NEAR/2 g+h+j'} {2 3} - 9 {SELECT x FROM t1 WHERE t1 MATCH 'b+c+d NEAR/1 g+h'} {2 3} - 10 {SELECT x FROM t1 WHERE t1 MATCH 'a AND d'} {1 2} - 11 {SELECT x FROM t1 WHERE t1 MATCH 'a OR d'} {1 2 3 4} - 12 {SELECT x FROM t1 WHERE t1 MATCH 'a NOT d'} {4} -} { - do_execsql_test 3.$tn $stmt $res -} - -do_execsql_test 4.0 { - CREATE TABLE t2(docid PRIMARY KEY, a, b, c); - CREATE INDEX i2 ON t2 USING fts5(); - INSERT INTO t2 VALUES(136895, 'qkfl my qkfl krag gw', NULL, NULL); -} - -do_execsql_test 4.1 { - SELECT docid FROM t2 WHERE t2 MATCH 'qkfl NEAR/2 gw'; -} {136895} - -do_execsql_test 6.0 { - CREATE TABLE t3(docid PRIMARY KEY, a, b, c); - CREATE INDEX i3 ON t3 USING fts5(); - INSERT INTO t3 VALUES(123, 'fix the hash table', NULL, NULL); -} -do_execsql_test 5.1 { - SELECT docid FROM t3 WHERE t3 MATCH 'h*'; -} {123} - -do_execsql_test 6.0 { - BEGIN TRANSACTION; - CREATE TABLE t4(docid PRIMARY KEY, a); - CREATE INDEX i4 ON t4 USING fts5(); - INSERT INTO "t4" VALUES(34, 'abc mnm xyz'); - INSERT INTO "t4" VALUES(50, 'abc mnm xyz'); - COMMIT; -} -do_execsql_test 6.1 { - SELECT docid FROM t4 WHERE t4 MATCH 'm*' -} {34 50} - -#------------------------------------------------------------------------- -# -do_execsql_test 7.0 { - BEGIN TRANSACTION; - CREATE TABLE t7(docid PRIMARY KEY, a, b, c); - CREATE INDEX i7 ON t7 USING fts5(); - INSERT INTO t7 VALUES(1, 'a b c', 'd e f', 'a b c'); - INSERT INTO t7 VALUES(2, 'x y z', 'a b c', 'a b c'); - COMMIT; -} - -foreach {tn expr res} { - 1 {a} {1 2} - 2 {a:a} {1} - 3 {b:a} {2} - 4 {c:a} {1 2} - 5 {a:a*} {1} -} { - do_execsql_test 7.$tn {SELECT docid FROM t7 WHERE t7 MATCH $expr} $res -} - -#------------------------------------------------------------------------- -# -do_execsql_test 8.0 { - CREATE TABLE t8(a PRIMARY KEY, b, c); - CREATE INDEX i8 ON t8 USING fts5(); - INSERT INTO t8 VALUES('one', 'a b c', 'a a a'); - INSERT INTO t8 VALUES('two', 'd e f', 'b b b'); -} - -#do_execsql_test 8.1 { -# SELECT rank(t8) FROM t8 WHERE t8 MATCH 'b a' -#} - -do_execsql_test 9.0 { - CREATE TABLE t9(a PRIMARY KEY, b); - CREATE INDEX i9 ON t9 USING fts5(); - INSERT INTO t9 VALUES('one', - 'a b c d e f g h i j k l m n o p q r s t u v w x y z ' || - 'a b c d e f g h i j k l m n o p q r s t u v w x y z' - ); -} - -#do_execsql_test 9.1 { -# SELECT snippet(t9) FROM t9 WHERE t9 MATCH 'b' -#} - -do_execsql_test 10.1 { - CREATE TABLE ft(content); - CREATE INDEX fti ON ft USING fts5(); -} -do_execsql_test 10.2 { - INSERT INTO ft VALUES('a b c d e'); - INSERT INTO ft VALUES('f g h i j'); -} -do_execsql_test 10.3 { SELECT rowid FROM ft WHERE ft MATCH 'c' } {1} -do_execsql_test 10.4 { SELECT rowid FROM ft WHERE ft MATCH 'f' } {2} - -do_execsql_test 10.5 { - DELETE FROM ft; - CREATE TABLE ft2(a, b, c); - CREATE INDEX fti2 ON ft2 USING fts5(); - INSERT INTO ft2 VALUES('1 2 3 4 5', '6 7 8 9 10', '11 12 13 14 15'); - SELECT snippet(ft2, '[', ']', '...', -1, 3) FROM ft2 WHERE ft2 MATCH '5'; -} {{...3 4 [5]}} - -finish_test - DELETED test/fts5rnd1.test Index: test/fts5rnd1.test ================================================================== --- test/fts5rnd1.test +++ /dev/null @@ -1,452 +0,0 @@ -# 2009 December 03 -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Brute force (random data) tests for fts5. -# - -#------------------------------------------------------------------------- -# -# The FTS3 tests implemented in this file focus on testing that FTS3 -# returns the correct set of documents for various types of full-text -# query. This is done using pseudo-randomly generated data and queries. -# The expected result of each query is calculated using Tcl code. -# -# 1. The database is initialized to contain a single table with three -# columns. 100 rows are inserted into the table. Each of the three -# values in each row is a document consisting of between 0 and 100 -# terms. Terms are selected from a vocabulary of $G(nVocab) terms. -# -# 2. The following is performed 100 times: -# -# a. A row is inserted into the database. The row contents are -# generated as in step 1. The docid is a pseudo-randomly selected -# value between 0 and 1000000. -# -# b. A psuedo-randomly selected row is updated. One of its columns is -# set to contain a new document generated in the same way as the -# documents in step 1. -# -# c. A psuedo-randomly selected row is deleted. -# -# d. For each of several types of fts3 queries, 10 SELECT queries -# of the form: -# -# SELECT docid FROM WHERE MATCH '' -# -# are evaluated. The results are compared to those calculated by -# Tcl code in this file. The patterns used for the different query -# types are: -# -# 1. query = -# 2. query = -# 3. query = " " -# 4. query = " " -# 5. query = " " -# 6. query = NEAR -# 7. query = NEAR/11 NEAR/11 -# 8. query = OR -# 9. query = NOT -# 10. query = AND -# 11. query = NEAR OR NEAR -# 12. query = NEAR NOT NEAR -# 13. query = NEAR AND NEAR -# -# where is a term psuedo-randomly selected from the vocabulary -# and prefix is the first 2 characters of such a term followed by -# a "*" character. -# -# Every second iteration, steps (a) through (d) above are performed -# within a single transaction. This forces the queries in (d) to -# read data from both the database and the in-memory hash table -# that caches the full-text index entries created by steps (a), (b) -# and (c) until the transaction is committed. -# -# The procedure above is run 5 times, using advisory fts3 node sizes of 50, -# 500, 1000 and 2000 bytes. -# -# After the test using an advisory node-size of 50, an OOM test is run using -# the database. This test is similar to step (d) above, except that it tests -# the effects of transient and persistent OOM conditions encountered while -# executing each query. -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl - -# If this build does not include FTS3, skip the tests in this file. -# -#ifcapable !fts3 { finish_test ; return } -source $testdir/fts3_common.tcl -source $testdir/malloc_common.tcl - -set G(nVocab) 100 - -set nVocab 100 -set lVocab [list] - -expr srand(0) - -# Generate a vocabulary of nVocab words. Each word is between 2 and 4 -# characters long. -# -set lChar {a b c d e f g h i j k l m n o p q r s t u v w x y z} -for {set i 0} {$i < $nVocab} {incr i} { - set len [expr int(rand()*3)+2] - set word [lindex $lChar [expr int(rand()*26)]] - append word [lindex $lChar [expr int(rand()*26)]] - if {$len>2} { append word [lindex $lChar [expr int(rand()*26)]] } - if {$len>3} { append word [lindex $lChar [expr int(rand()*26)]] } - lappend lVocab $word -} - -proc random_term {} { - lindex $::lVocab [expr {int(rand()*$::nVocab)}] -} - -# Return a document consisting of $nWord arbitrarily selected terms -# from the $::lVocab list. -# -proc generate_doc {nWord} { - set doc [list] - for {set i 0} {$i < $nWord} {incr i} { - lappend doc [random_term] - } - return $doc -} - - - -# Primitives to update the table. -# -unset -nocomplain t1 -proc insert_row {docid} { - set a [generate_doc [expr int((rand()*100))]] - set b [generate_doc [expr int((rand()*100))]] - set c [generate_doc [expr int((rand()*100))]] - execsql { INSERT INTO t1(docid, a, b, c) VALUES($docid, $a, $b, $c) } - set ::t1($docid) [list $a $b $c] -} -proc delete_row {docid} { - execsql { DELETE FROM t1 WHERE docid = $docid } - catch {unset ::t1($docid)} -} -proc update_row {docid} { - set cols {a b c} - set iCol [expr int(rand()*3)] - set doc [generate_doc [expr int((rand()*100))]] - lset ::t1($docid) $iCol $doc - execsql "UPDATE t1 SET [lindex $cols $iCol] = \$doc WHERE docid = \$docid" -} - -proc simple_phrase {zPrefix} { - set ret [list] - - set reg [string map {* {[^ ]*}} $zPrefix] - set reg " $reg " - - foreach key [lsort -integer [array names ::t1]] { - set value $::t1($key) - set cnt [list] - foreach col $value { - if {[regexp $reg " $col "]} { lappend ret $key ; break } - } - } - - #lsort -uniq -integer $ret - set ret -} - -# This [proc] is used to test the FTS3 matchinfo() function. -# -proc simple_token_matchinfo {zToken bDesc} { - - set nDoc(0) 0 - set nDoc(1) 0 - set nDoc(2) 0 - set nHit(0) 0 - set nHit(1) 0 - set nHit(2) 0 - - set dir -inc - if {$bDesc} { set dir -dec } - - foreach key [array names ::t1] { - set value $::t1($key) - set a($key) [list] - foreach i {0 1 2} col $value { - set hit [llength [lsearch -all $col $zToken]] - lappend a($key) $hit - incr nHit($i) $hit - if {$hit>0} { incr nDoc($i) } - } - } - - set ret [list] - foreach docid [lsort -integer $dir [array names a]] { - if { [lindex [lsort -integer $a($docid)] end] } { - set matchinfo [list 1 3] - foreach i {0 1 2} hit $a($docid) { - lappend matchinfo $hit $nHit($i) $nDoc($i) - } - lappend ret $docid $matchinfo - } - } - - set ret -} - -proc simple_near {termlist nNear} { - set ret [list] - - foreach {key value} [array get ::t1] { - foreach v $value { - - set l [lsearch -exact -all $v [lindex $termlist 0]] - foreach T [lrange $termlist 1 end] { - set l2 [list] - foreach i $l { - set iStart [expr $i - $nNear - 1] - set iEnd [expr $i + $nNear + 1] - if {$iStart < 0} {set iStart 0} - foreach i2 [lsearch -exact -all [lrange $v $iStart $iEnd] $T] { - incr i2 $iStart - if {$i2 != $i} { lappend l2 $i2 } - } - } - set l [lsort -uniq -integer $l2] - } - - if {[llength $l]} { -#puts "MATCH($key): $v" - lappend ret $key - } - } - } - - lsort -unique -integer $ret -} - -# The following three procs: -# -# setup_not A B -# setup_or A B -# setup_and A B -# -# each take two arguments. Both arguments must be lists of integer values -# sorted by value. The return value is the list produced by evaluating -# the equivalent of "A op B", where op is the FTS3 operator NOT, OR or -# AND. -# -proc setop_not {A B} { - foreach b $B { set n($b) {} } - set ret [list] - foreach a $A { if {![info exists n($a)]} {lappend ret $a} } - return $ret -} -proc setop_or {A B} { - lsort -integer -uniq [concat $A $B] -} -proc setop_and {A B} { - foreach b $B { set n($b) {} } - set ret [list] - foreach a $A { if {[info exists n($a)]} {lappend ret $a} } - return $ret -} - -proc mit {blob} { - set scan(littleEndian) i* - set scan(bigEndian) I* - binary scan $blob $scan($::tcl_platform(byteOrder)) r - return $r -} -db func mit mit -set sqlite_fts3_enable_parentheses 1 - -proc do_orderbydocid_test {tn sql res} { - uplevel [list do_select_test $tn.asc "$sql ORDER BY docid ASC" $res] - uplevel [list do_select_test $tn.desc "$sql ORDER BY docid DESC" \ - [lsort -int -dec $res] - ] -} - -#------------------------------------------------------------------------- -# -set NUM_TRIALS 10 -catch { array unset ::t1 } - -# Create the fts5 table. Populate it (and the Tcl array) with 100 rows. -# -db transaction { - execsql { - DROP TABLE IF EXISTS t1; - CREATE TABLE t1(docid PRIMARY KEY, a, b, c); - CREATE INDEX i1 ON t1 USING fts5(); - } - for {set i 0} {$i < 100} {incr i} { insert_row $i } -} - -for {set iTest 1} {$iTest <= $NUM_TRIALS} {incr iTest} { - catchsql COMMIT - - set DO_MALLOC_TEST 0 - set nRep 10 - # if {$iTest==100 && $nodesize==50} { - # set DO_MALLOC_TEST 1 - # set nRep 2 - # } - - set ::testprefix fts5rnd-1.$iTest - - # Delete one row, update one row and insert one row. - # - set rows [array names ::t1] - set nRow [llength $rows] - set iUpdate [lindex $rows [expr {int(rand()*$nRow)}]] - set iDelete $iUpdate - while {$iDelete == $iUpdate} { - set iDelete [lindex $rows [expr {int(rand()*$nRow)}]] - } - set iInsert $iUpdate - while {[info exists ::t1($iInsert)]} { - set iInsert [expr {int(rand()*1000000)}] - } - execsql BEGIN - insert_row $iInsert - update_row $iUpdate - delete_row $iDelete - if {0==($iTest%2)} { execsql COMMIT } - - if {0==($iTest%2)} { - #do_test 0 { fts3_integrity_check t1 } ok - } - - # Pick 10 terms from the vocabulary. Check that the results of querying - # the database for the set of documents containing each of these terms - # is the same as the result obtained by scanning the contents of the Tcl - # array for each term. - # - for {set i 0} {$i < 10} {incr i} { - set term [random_term] - do_orderbydocid_test 1.$i.asc { - SELECT docid FROM t1 WHERE t1 MATCH $term - } [simple_phrase $term] - do_orderbydocid_test 1.$i.desc { - SELECT docid FROM t1 WHERE t1 MATCH $term - } [simple_phrase $term] - } - - # This time, use the first two characters of each term as a term prefix - # to query for. Test that querying the Tcl array produces the same results - # as querying the FTS3 table for the prefix. - # - for {set i 0} {$i < $nRep} {incr i} { - set prefix [string range [random_term] 0 end-1] - set match "${prefix}*" - do_orderbydocid_test 2.$i { - SELECT docid FROM t1 WHERE t1 MATCH $match - } [simple_phrase $match] - } - - # Similar to the above, except for phrase queries. - # - for {set i 0} {$i < $nRep} {incr i} { - set term [list [random_term] [random_term]] - set match "\"$term\"" - do_orderbydocid_test 3.$i { - SELECT docid FROM t1 WHERE t1 MATCH $match - } [simple_phrase $term] - } - - # Three word phrases. - # - for {set i 0} {$i < $nRep} {incr i} { - set term [list [random_term] [random_term] [random_term]] - set match "\"$term\"" - do_orderbydocid_test 4.$i { - SELECT docid FROM t1 WHERE t1 MATCH $match - } [simple_phrase $term] - } - - # Three word phrases made up of term-prefixes. - # - for {set i 0} {$i < $nRep} {incr i} { - set query "[string range [random_term] 0 end-1]* + " - append query "[string range [random_term] 0 end-1]* + " - append query "[string range [random_term] 0 end-1]*" - - do_orderbydocid_test 5.$i { - SELECT docid FROM t1 WHERE t1 MATCH $match - } [simple_phrase $query] - } - - # A NEAR query with terms as the arguments: - # - # ... MATCH '$term1 NEAR $term2' ... - # - for {set i 0} {$i < $nRep} {incr i} { - set terms [list [random_term] [random_term]] - set match [join $terms " NEAR "] - do_orderbydocid_test 6.$i { - SELECT docid FROM t1 WHERE t1 MATCH $match - } [simple_near $terms 10] - } - - # A 3-way NEAR query with terms as the arguments. - # - for {set i 0} {$i < $nRep} {incr i} { - set terms [list [random_term] [random_term] [random_term]] - set nNear 11 - set match [join $terms " NEAR/$nNear "] - do_orderbydocid_test 7.$i { - SELECT docid FROM t1 WHERE t1 MATCH $match - } [simple_near $terms $nNear] - } - - # Set operations on simple term queries. - # - foreach {tn op proc} { - 8 OR setop_or - 9 NOT setop_not - 10 AND setop_and - } { - for {set i 0} {$i < $nRep} {incr i} { - set term1 [random_term] - set term2 [random_term] - set match "$term1 $op $term2" - do_orderbydocid_test $tn.$i { - SELECT docid FROM t1 WHERE t1 MATCH $match - } [$proc [simple_phrase $term1] [simple_phrase $term2]] - } - } - - # Set operations on NEAR queries. - # - foreach {tn op proc} { - 11 OR setop_or - 12 NOT setop_not - 13 AND setop_and - } { - for {set i 0} {$i < $nRep} {incr i} { - set term1 [random_term] - set term2 [random_term] - set term3 [random_term] - set term4 [random_term] - set match "$term1 NEAR $term2 $op $term3 NEAR $term4" - do_orderbydocid_test $tn.$i { - SELECT docid FROM t1 WHERE t1 MATCH $match - } [$proc \ - [simple_near [list $term1 $term2] 10] \ - [simple_near [list $term3 $term4] 10] - ] - } - } - - catchsql COMMIT -} - -finish_test DELETED test/fts5snippet.test Index: test/fts5snippet.test ================================================================== --- test/fts5snippet.test +++ /dev/null @@ -1,292 +0,0 @@ -# 2013 January 10 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#************************************************************************* -# -# The tests in this file test the FTS5 snippet() function. -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl - -# If SQLITE4_ENABLE_FTS3 is not defined, omit this file. -source $testdir/fts3_common.tcl - -set DO_MALLOC_TEST 0 - -# Transform the list $L to its "normal" form. So that it can be compared to -# another list with the same set of elements using [string compare]. -# -proc normalize {L} { - set ret [list] - foreach l $L {lappend ret $l} - return $ret -} - -# Document text used by a few tests. Contains the English names of all -# integers between 1 and 300. -# -set numbers [normalize { - one two three four five six seven eight nine ten eleven twelve thirteen - fourteen fifteen sixteen seventeen eighteen nineteen twenty twentyone - twentytwo twentythree twentyfour twentyfive twentysix twentyseven - twentyeight twentynine thirty thirtyone thirtytwo thirtythree thirtyfour - thirtyfive thirtysix thirtyseven thirtyeight thirtynine forty fortyone - fortytwo fortythree fortyfour fortyfive fortysix fortyseven fortyeight - fortynine fifty fiftyone fiftytwo fiftythree fiftyfour fiftyfive fiftysix - fiftyseven fiftyeight fiftynine sixty sixtyone sixtytwo sixtythree sixtyfour - sixtyfive sixtysix sixtyseven sixtyeight sixtynine seventy seventyone - seventytwo seventythree seventyfour seventyfive seventysix seventyseven - seventyeight seventynine eighty eightyone eightytwo eightythree eightyfour - eightyfive eightysix eightyseven eightyeight eightynine ninety ninetyone - ninetytwo ninetythree ninetyfour ninetyfive ninetysix ninetyseven - ninetyeight ninetynine onehundred onehundredone onehundredtwo - onehundredthree onehundredfour onehundredfive onehundredsix onehundredseven - onehundredeight onehundrednine onehundredten onehundredeleven - onehundredtwelve onehundredthirteen onehundredfourteen onehundredfifteen - onehundredsixteen onehundredseventeen onehundredeighteen onehundrednineteen - onehundredtwenty onehundredtwentyone onehundredtwentytwo - onehundredtwentythree onehundredtwentyfour onehundredtwentyfive - onehundredtwentysix onehundredtwentyseven onehundredtwentyeight - onehundredtwentynine onehundredthirty onehundredthirtyone - onehundredthirtytwo onehundredthirtythree onehundredthirtyfour - onehundredthirtyfive onehundredthirtysix onehundredthirtyseven - onehundredthirtyeight onehundredthirtynine onehundredforty - onehundredfortyone onehundredfortytwo onehundredfortythree - onehundredfortyfour onehundredfortyfive onehundredfortysix - onehundredfortyseven onehundredfortyeight onehundredfortynine - onehundredfifty onehundredfiftyone onehundredfiftytwo onehundredfiftythree - onehundredfiftyfour onehundredfiftyfive onehundredfiftysix - onehundredfiftyseven onehundredfiftyeight onehundredfiftynine - onehundredsixty onehundredsixtyone onehundredsixtytwo onehundredsixtythree - onehundredsixtyfour onehundredsixtyfive onehundredsixtysix - onehundredsixtyseven onehundredsixtyeight onehundredsixtynine - onehundredseventy onehundredseventyone onehundredseventytwo - onehundredseventythree onehundredseventyfour onehundredseventyfive - onehundredseventysix onehundredseventyseven onehundredseventyeight - onehundredseventynine onehundredeighty onehundredeightyone - onehundredeightytwo onehundredeightythree onehundredeightyfour - onehundredeightyfive onehundredeightysix onehundredeightyseven - onehundredeightyeight onehundredeightynine onehundredninety - onehundredninetyone onehundredninetytwo onehundredninetythree - onehundredninetyfour onehundredninetyfive onehundredninetysix - onehundredninetyseven onehundredninetyeight onehundredninetynine twohundred - twohundredone twohundredtwo twohundredthree twohundredfour twohundredfive - twohundredsix twohundredseven twohundredeight twohundrednine twohundredten - twohundredeleven twohundredtwelve twohundredthirteen twohundredfourteen - twohundredfifteen twohundredsixteen twohundredseventeen twohundredeighteen - twohundrednineteen twohundredtwenty twohundredtwentyone twohundredtwentytwo - twohundredtwentythree twohundredtwentyfour twohundredtwentyfive - twohundredtwentysix twohundredtwentyseven twohundredtwentyeight - twohundredtwentynine twohundredthirty twohundredthirtyone - twohundredthirtytwo twohundredthirtythree twohundredthirtyfour - twohundredthirtyfive twohundredthirtysix twohundredthirtyseven - twohundredthirtyeight twohundredthirtynine twohundredforty - twohundredfortyone twohundredfortytwo twohundredfortythree - twohundredfortyfour twohundredfortyfive twohundredfortysix - twohundredfortyseven twohundredfortyeight twohundredfortynine - twohundredfifty twohundredfiftyone twohundredfiftytwo twohundredfiftythree - twohundredfiftyfour twohundredfiftyfive twohundredfiftysix - twohundredfiftyseven twohundredfiftyeight twohundredfiftynine - twohundredsixty twohundredsixtyone twohundredsixtytwo twohundredsixtythree - twohundredsixtyfour twohundredsixtyfive twohundredsixtysix - twohundredsixtyseven twohundredsixtyeight twohundredsixtynine - twohundredseventy twohundredseventyone twohundredseventytwo - twohundredseventythree twohundredseventyfour twohundredseventyfive - twohundredseventysix twohundredseventyseven twohundredseventyeight - twohundredseventynine twohundredeighty twohundredeightyone - twohundredeightytwo twohundredeightythree twohundredeightyfour - twohundredeightyfive twohundredeightysix twohundredeightyseven - twohundredeightyeight twohundredeightynine twohundredninety - twohundredninetyone twohundredninetytwo twohundredninetythree - twohundredninetyfour twohundredninetyfive twohundredninetysix - twohundredninetyseven twohundredninetyeight twohundredninetynine - threehundred -}] - -foreach {DO_MALLOC_TEST enc} { - 0 utf8 - 1 utf8 - 1 utf16 -} { -if {$DO_MALLOC_TEST || $enc=="utf16"} continue - - db close - forcedelete test.db - sqlite4 db test.db - sqlite4_db_config_lookaside db 0 0 0 - db eval "PRAGMA encoding = \"$enc\"" - - # Set variable $T to the test name prefix for this iteration of the loop. - # - set T "fts5snippet-$enc" - - ########################################################################## - # Test the snippet function. - # - proc do_snippet_test {name expr iCol nTok args} { - set res [list] - foreach a $args { lappend res [string trim $a] } - do_select_test $name { - SELECT snippet(ft,'{','}','...',$iCol,$nTok) FROM ft WHERE ft MATCH $expr - } $res - } - do_test $T.3.1 { - execsql { - DROP TABLE IF EXISTS ft; - CREATE TABLE ft(content); - CREATE INDEX fti ON ft USING fts5(); - - INSERT INTO ft VALUES('one two three four five six seven eight nine ten'); - } - } {} - do_snippet_test $T.3.2 one 0 5 "{one} two three four five..." - do_snippet_test $T.3.3 two 0 5 "one {two} three four five..." - do_snippet_test $T.3.4 three 0 5 "one two {three} four five..." - do_snippet_test $T.3.5 four 0 5 "...two three {four} five six..." - do_snippet_test $T.3.6 five 0 5 "...three four {five} six seven..." - do_snippet_test $T.3.7 six 0 5 "...four five {six} seven eight..." - do_snippet_test $T.3.8 seven 0 5 "...five six {seven} eight nine..." - do_snippet_test $T.3.9 eight 0 5 "...six seven {eight} nine ten" - do_snippet_test $T.3.10 nine 0 5 "...six seven eight {nine} ten" - do_snippet_test $T.3.11 ten 0 5 "...six seven eight nine {ten}" - - do_test $T.4.1 { - execsql { - INSERT INTO ft VALUES( - 'one two three four five ' - || 'six seven eight nine ten ' - || 'eleven twelve thirteen fourteen fifteen ' - || 'sixteen seventeen eighteen nineteen twenty ' - || 'one two three four five ' - || 'six seven eight nine ten ' - || 'eleven twelve thirteen fourteen fifteen ' - || 'sixteen seventeen eighteen nineteen twenty' - ); - } - } {} - - do_snippet_test $T.4.2 {one nine} 0 5 { - {one} two three...eight {nine} ten - } { - {one} two three...eight {nine} ten... - } - - do_snippet_test $T.4.3 {one nine} 0 -5 { - {one} two three four five...six seven eight {nine} ten - } { - {one} two three four five...seven eight {nine} ten eleven... - } - do_snippet_test $T.4.3 {one nineteen} 0 -5 { - ...eighteen {nineteen} twenty {one} two... - } - do_snippet_test $T.4.4 {two nineteen} 0 -5 { - ...eighteen {nineteen} twenty one {two}... - } - do_snippet_test $T.4.5 {three nineteen} 0 -5 { - ...{nineteen} twenty one two {three}... - } - - do_snippet_test $T.4.6 {four nineteen} 0 -5 { - ...two three {four} five six...seventeen eighteen {nineteen} twenty one... - } - do_snippet_test $T.4.7 {four NEAR nineteen} 0 -5 { - ...seventeen eighteen {nineteen} twenty one...two three {four} five six... - } - - do_snippet_test $T.4.8 {four nineteen} 0 5 { - ...three {four} five...eighteen {nineteen} twenty... - } - do_snippet_test $T.4.9 {four NEAR nineteen} 0 5 { - ...eighteen {nineteen} twenty...three {four} five... - } - do_snippet_test $T.4.10 {four NEAR nineteen} 0 -5 { - ...seventeen eighteen {nineteen} twenty one...two three {four} five six... - } - do_snippet_test $T.4.11 {four NOT (nineteen+twentyone)} 0 5 { - ...two three {four} five six... - } { - ...two three {four} five six... - } - do_snippet_test $T.4.12 {four OR nineteen NEAR twentyone} 0 5 { - ...two three {four} five six... - } { - ...two three {four} five six... - } - - do_test $T.5.1 { - execsql { - DROP TABLE IF EXISTS ft; - CREATE TABLE ft(a, b, c); - CREATE INDEX fti ON ft USING fts5(); - INSERT INTO ft VALUES( - 'one two three four five', - 'four five six seven eight', - 'seven eight nine ten eleven' - ); - } - } {} - - do_snippet_test $T.5.2 {five} -1 3 {...three four {five}} - do_snippet_test $T.5.3 {five} 0 3 {...three four {five}} - do_snippet_test $T.5.4 {five} 1 3 {four {five} six...} - do_snippet_test $T.5.5 {five} 2 3 {seven eight nine...} - - do_test $T.5.6 { - execsql { UPDATE ft SET b = NULL } - } {} - - do_snippet_test $T.5.7 {five} -1 3 {...three four {five}} - do_snippet_test $T.5.8 {five} 0 3 {...three four {five}} - do_snippet_test $T.5.9 {five} 1 3 {} - do_snippet_test $T.5.10 {five} 2 3 {seven eight nine...} - - do_snippet_test $T.5.11 {one "seven eight nine"} -1 -3 { - {one} two three...{seven} {eight} {nine}... - } - - do_test $T.6.1 { - execsql { - DROP TABLE IF EXISTS ft; - CREATE TABLE ft(x); - CREATE INDEX fti ON ft USING fts5(); - INSERT INTO ft VALUES($numbers); - } - } {} - do_snippet_test $T.6.2 { - one fifty onehundred onehundredfifty twohundredfifty threehundred - } -1 4 { - {one}...{fifty}...{onehundred}...{onehundredfifty}... - } - do_snippet_test $T.6.3 { - one fifty onehundred onehundredfifty twohundredfifty threehundred - } -1 -4 { - {one} two three four...fortyeight fortynine {fifty} fiftyone...ninetyeight ninetynine {onehundred} onehundredone...onehundredfortyeight onehundredfortynine {onehundredfifty} onehundredfiftyone... - } - - do_test $T.7.1 { - execsql { - BEGIN; - DROP TABLE IF EXISTS ft; - CREATE TABLE ft(x); - CREATE INDEX fti ON ft USING fts5(); - } - set testresults [list] - for {set i 1} {$i < 150} {incr i} { - set commas [string repeat , $i] - execsql {INSERT INTO ft VALUES('one' || $commas || 'two')} - lappend testresults "{one}$commas{two}" - } - execsql COMMIT - } {} - eval [list do_snippet_test $T.7.2 {one two} -1 3] $testresults - -} - -finish_test Index: test/in.test ================================================================== --- test/in.test +++ test/in.test @@ -271,10 +271,15 @@ do_test in-8.1 { execsql { SELECT b FROM t1 WHERE a IN ('hello','there') } +} {world} +do_test in-8.2 { + execsql { + SELECT b FROM t1 WHERE a IN ("hello",'there') + } } {world} # Test constructs of the form: expr IN tablename # do_test in-9.1 { Index: test/insert.test ================================================================== --- test/insert.test +++ test/insert.test @@ -360,23 +360,8 @@ INSERT INTO t3 SELECT * FROM (SELECT * FROM t3 UNION ALL SELECT 1,2,3) } } {} } -# Multiple VALUES clauses -# -do_test insert-10.1 { - execsql { - CREATE TABLE t10(a,b,c); - INSERT INTO t10 VALUES(1,2,3), (4,5,6), (7,8,9); - SELECT * FROM t10; - } -} {1 2 3 4 5 6 7 8 9} -do_test insert-10.2 { - catchsql { - INSERT INTO t10 VALUES(11,12,13), (14,15); - } -} {1 {all VALUES must have the same number of terms}} - integrity_check insert-99.0 finish_test Index: test/join.test ================================================================== --- test/join.test +++ test/join.test @@ -429,25 +429,25 @@ # A test for ticket #247. # do_test join-7.1 { execsql { CREATE TABLE t7 (x, y); - INSERT INTO t7 VALUES ('pa1', 1); - INSERT INTO t7 VALUES ('pa2', NULL); - INSERT INTO t7 VALUES ('pa3', NULL); - INSERT INTO t7 VALUES ('pa4', 2); - INSERT INTO t7 VALUES ('pa30', 131); - INSERT INTO t7 VALUES ('pa31', 130); - INSERT INTO t7 VALUES ('pa28', NULL); + INSERT INTO t7 VALUES ("pa1", 1); + INSERT INTO t7 VALUES ("pa2", NULL); + INSERT INTO t7 VALUES ("pa3", NULL); + INSERT INTO t7 VALUES ("pa4", 2); + INSERT INTO t7 VALUES ("pa30", 131); + INSERT INTO t7 VALUES ("pa31", 130); + INSERT INTO t7 VALUES ("pa28", NULL); CREATE TABLE t8 (a integer primary key, b); - INSERT INTO t8 VALUES (1, 'pa1'); - INSERT INTO t8 VALUES (2, 'pa4'); + INSERT INTO t8 VALUES (1, "pa1"); + INSERT INTO t8 VALUES (2, "pa4"); INSERT INTO t8 VALUES (3, NULL); INSERT INTO t8 VALUES (4, NULL); - INSERT INTO t8 VALUES (130, 'pa31'); - INSERT INTO t8 VALUES (131, 'pa30'); + INSERT INTO t8 VALUES (130, "pa31"); + INSERT INTO t8 VALUES (131, "pa30"); SELECT coalesce(t8.a,999) from t7 LEFT JOIN t8 on y=a; } } {1 999 999 2 131 130 999} DELETED test/lsm1.test Index: test/lsm1.test ================================================================== --- test/lsm1.test +++ /dev/null @@ -1,205 +0,0 @@ -# 2012 November 02 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix lsm1 -db close - - -proc reopen {{bClear 0}} { - catch {db close} - if {$bClear} { forcedelete test.db } - lsm_open db test.db {mmap 0 automerge 2 autowork 0} -} - -proc contents {} { - db csr_open csr - set res [list] - for {csr first} {[csr valid]} {csr next} { - lappend res [list [csr key] [csr value]] - } - csr close - set res -} - -proc fetch {key} { - db csr_open csr - csr seek $key eq - set val [csr value] - csr close - set val -} - -proc dbwrite {list} { - foreach {k v} $list { - db write $k $v - } -} - -proc do_contents_test {tn res} { - set con [contents] - set res2 [list] - foreach r $res {lappend res2 $r} - uplevel do_test $tn [list [list set {} $con]] [list $res2] -} - -do_test 1.1 { - reopen - db write abc def - db close -} {} - -do_test 1.2 { - reopen - db csr_open csr - csr seek abc eq -} {} - -do_test 1.3 { - list [csr valid] [csr key] [csr value] -} {1 abc def} - -do_test 1.4 { - db delete abc - csr seek abc eq - csr valid -} {0} - -do_test 1.5 { csr close } {} -do_test 1.6 { db close } {} - - -do_test 2.1 { - forcedelete test.db - reopen - db write aaa one - db write bbb two - db write ccc three - db write ddd four - db write eee five - db write fff six - reopen - db delete_range a bbb - reopen -breakpoint - db work 10 -} {1} - -do_contents_test 2.2 { {bbb two} {ccc three} {ddd four} {eee five} {fff six} } - - -#------------------------------------------------------------------------- - -# The following populates the db with a single age=1 segment, containing -# the six keys inserted below. -do_test 3.1 { - reopen 1 - db write aaa one - db write ddd four - db write fff six - reopen - db write bbb two - db write ccc three - db write eee five - reopen - db work 10 -} {1} - -do_test 3.2 { - db write bx seven - reopen - db delete_range aaa bx - reopen - db work 10 -} {2} - -do_contents_test 3.3 { - {aaa one} {bx seven} {ccc three} {ddd four} {eee five} {fff six} -} - -do_test 3.4 { fetch ddd } four - -#------------------------------------------------------------------------- -# -do_test 4.1 { - reopen 1 - dbwrite { 222 helloworld } - db flush - db delete_range 111 222 - db delete_range 222 333 - db flush - contents -} {{222 helloworld}} - -do_test 4.2 { fetch 222 } helloworld - -#------------------------------------------------------------------------- -# -do_test 5.1 { - reopen 1 - - dbwrite { 10 ten } ; db flush - dbwrite { 20 twenty } ; db flush - db work 10 - - dbwrite { 30 thirty } ; db flush - dbwrite { 40 forty } ; db flush - db work 10 - - db delete_range 11 29 ; db flush - db delete_range 20 39 ; db flush - db work 10 - - contents -} {{10 ten} {40 forty}} - -do_test 5.2 { - reopen 1 - db config {automerge 4} - - dbwrite { 10 ten } ; db flush - dbwrite { 20 twenty } ; db flush - dbwrite { 30 thirty } ; db flush - dbwrite { 40 forty } ; db flush - db work 10 - - db delete_range 10 17 ; db flush - dbwrite {17 seventeen} ; db flush - db delete_range 10 17 ; db flush - - db config {automerge 3} - db work 10 - - contents -} {{10 ten} {17 seventeen} {20 twenty} {30 thirty} {40 forty}} - -do_test 5.3 { - reopen 1 - db config {automerge 4} - - dbwrite { 10 ten } ; db flush - dbwrite { 20 twenty } ; db flush - dbwrite { 30 thirty } ; db flush - dbwrite { 40 forty } ; db flush - db work 10 - - db delete_range 10 17 ; db flush - db delete_range 12 19 ; db flush - dbwrite {17 seventeen} ; db flush - - db config {automerge 3} - db work 10 - - contents -} {{10 ten} {17 seventeen} {20 twenty} {30 thirty} {40 forty}} - -finish_test DELETED test/lsm2.test Index: test/lsm2.test ================================================================== --- test/lsm2.test +++ /dev/null @@ -1,45 +0,0 @@ -# 2012 November 02 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix lsm2 -db close - -expr srand(0) - -proc insert_data {nRow} { - for {set i 0} {$i < $nRow} {incr i} { - set key [string range [expr rand()] 0 10] - set val [string repeat [string range [expr rand()] 0 10] 100] - db write $key $val - } -} - -do_test 1.1 { - forcedelete test.db - lsm_open db test.db [list mmap 0 block_size [expr 256*1024]] - insert_data 10000 -} {} - -do_test 1.2 { - db flush - db delete_range 0 9 - db flush - db work 1 1000000 - db checkpoint - - db begin 1 - insert_data 10000 - db commit 0 -} {} - -finish_test DELETED test/lsm3.test Index: test/lsm3.test ================================================================== --- test/lsm3.test +++ /dev/null @@ -1,89 +0,0 @@ -# 2012 November 02 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# This file tests that the LSM_CONFIG_MULTIPLE_PROCESSES parameter seems -# to work as documented. -# -set testdir [file dirname $argv0] -source $testdir/tester.tcl -source $testdir/lock_common.tcl -set testprefix lsm3 -db close - -do_multiclient_test tn { - - # The [do_multiclient_test] command automatically opens a connection - # in each process (or three connections in this process). We don't want - # them in this case. - code1 { db close } - code2 { db2 close } - code3 { db3 close } - - if { $tn==1 } { - set locked {1 {database is locked}} - } else { - set locked {0 {}} - } - - # Open a single-process connection to the database from an external - # process (if $tn==1, otherwise open it from within the current - # process). - code2 { sqlite4 db2 file:test.db?lsm_multiple_processes=0 } - - # Try to open some other connections to the database file, both in multi - # and single process mode. If ($tn==1), then all such attempts fail. Or, - # if ($tn==2), they all succeed. - do_test $tn.1 { - catch { db close } - list [catch {sqlite4 db file:test.db?lsm_multiple_processes=0} msg] $msg - } $locked - do_test $tn.2 { - catch { db close } - list [catch {sqlite4 db file:test.db?lsm_multiple_processes=0} msg] $msg - } $locked - do_test $tn.3 { - catch { db close } - list [catch {sqlite4 db file:test.db?lsm_multiple_processes=1} msg] $msg - } $locked - do_test $tn.4 { - catch { db close } - list [catch {sqlite4 db file:test.db?lsm_multiple_processes=1} msg] $msg - } $locked - - # Now open a connection from an external process in multi-proc mode. - # Observe that further connections are allowed if they are from within - # the same process or if the LSM_CONFIG_MULTIPLE_PROCESSES parameter - # is set to true. - code2 { - db2 close - sqlite4 db2 file:test.db - } - - do_test $tn.5 { - catch { db close } - list [catch {sqlite4 db file:test.db?lsm_multiple_processes=0} msg] $msg - } $locked - do_test $tn.6 { - catch { db close } - list [catch {sqlite4 db file:test.db?lsm_multiple_processes=0} msg] $msg - } $locked - do_test $tn.7 { - catch { db close } - list [catch {sqlite4 db file:test.db?lsm_multiple_processes=1} msg] $msg - } {0 {}} - do_test $tn.8 { - catch { db close } - list [catch {sqlite4 db file:test.db?lsm_multiple_processes=1} msg] $msg - } {0 {}} -} - - -finish_test DELETED test/lsm4.test Index: test/lsm4.test ================================================================== --- test/lsm4.test +++ /dev/null @@ -1,122 +0,0 @@ -# 2013 February 06 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# The focus of this file is testing the LSM library. More specifically, -# it focuses on testing the compression, compression-id and -# compression-factory functionality. -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix lsm4 -db close - -# Compression scheme ids (defined in test_lsm.c): -# -set compression_id(encrypt) 43 -set compression_id(rle) 44 -set compression_id(noop) 45 - -proc db_fetch {db key} { - db csr_open csr - csr seek $key eq - set ret [csr value] - csr close - set ret -} - -do_test 1.1 { - lsm_open db test.db - db config {set_compression noop} - db write 1 abc - db write 2 def - db close -} {} - -do_test 1.2 { - lsm_open db test.db - db config {set_compression noop} - list [db_fetch db 1] [db_fetch db 2] -} {abc def} - -do_test 1.3 { - db close - lsm_open db test.db - db config {set_compression rle} - list [catch {db_fetch db 1} msg] $msg -} {1 {error in lsm_csr_open() - 50}} - -do_test 1.4 { - db close - lsm_open db test.db - list [catch {db_fetch db 1} msg] $msg -} {1 {error in lsm_csr_open() - 50}} - -do_test 1.5 { - db config {set_compression_factory true} - list [db_fetch db 1] [db_fetch db 2] -} {abc def} - -do_test 1.6 { db info compression_id } $compression_id(noop) -db close - -#------------------------------------------------------------------------- -# -forcedelete test.db - -do_test 2.1 { - lsm_open db test.db - db info compression_id -} {0} - -do_test 2.2 { - db write 1 abc - db write 2 abc - db info compression_id -} {0} - -do_test 2.3 { - lsm_open db2 test.db - db2 info compression_id -} {0} - -do_test 2.4 { - db close - db2 info compression_id -} {0} - -do_test 2.5 { - db2 close - lsm_open db test.db - db info compression_id -} {1} - -db close -forcedelete test.db - -do_test 2.6 { - lsm_open db test.db - db config {set_compression rle} - db write 3 three - db write 4 four - db close - - lsm_open db test.db - db info compression_id -} $compression_id(rle) - -do_test 2.7 { - db config {set_compression rle} - list [db_fetch db 3] [db_fetch db 4] -} {three four} - -finish_test - DELETED test/lsm5.test Index: test/lsm5.test ================================================================== --- test/lsm5.test +++ /dev/null @@ -1,42 +0,0 @@ -# 2013 February 08 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# The focus of this file is testing the LSM library. -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix lsm5 -db close - -#------------------------------------------------------------------------- -# When the database system is shut down (i.e. when the last connection -# disconnects), an attempt is made to truncate the database file to the -# minimum number of blocks required. -# -# This test case checks that this process does not actually cause the -# database to grow. -# -do_test 1.1 { - lsm_open db test.db - db config {mmap 0} -} {0} -do_test 1.2 { - db write 1 one - db write 2 two - db close -} {} -do_test 1.3 { - expr [file size test.db] < (64*1024) -} 1 - -finish_test - DELETED test/num.test Index: test/num.test ================================================================== --- test/num.test +++ /dev/null @@ -1,91 +0,0 @@ -# 2001 September 15 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# This file implements regression tests for SQLite library. The -# focus of this file is testing the sqlite_*_printf() interface. -# -# $Id: printf.test,v 1.31 2009/02/01 00:21:10 drh Exp $ - -set testdir [file dirname $argv0] -source $testdir/tester.tcl - -do_test num-1.1.1 { - sqlite4_num_compare 20 20 -} {equal} -do_test num-1.1.2 { - sqlite4_num_compare 20 2e1 -} {equal} -do_test num-1.1.3 { - sqlite4_num_compare -00034 -3.4e1 -} {equal} -do_test num-1.1.4 { - sqlite4_num_compare -inf +inf -} {lesser} -do_test num-1.1.5 { - sqlite4_num_compare -inf 0 -} {lesser} -do_test num-1.1.6 { - sqlite4_num_compare inf 4 -} {greater} -do_test num-1.1.7 { - sqlite4_num_compare nan 7 -} {incomparable} -# Is +0 > -0? -#do_test num-equal-1.1.4 { -# sqlite4_num_compare +0 -0 -#} {equal} - -do_test num-2.1.1 { - sqlite4_num_to_text [sqlite4_num_from_text 37] -} {37} -do_test num-2.1.2 { - sqlite4_num_to_text [sqlite4_num_from_text 37 2] -} {37} -do_test num-2.1.4 { - sqlite4_num_compare [sqlite4_num_from_text 2.9e2X 5] 290 -} {equal} -do_test num-2.1.5 { - sqlite4_num_isnan [sqlite4_num_from_text inf 2] -} {true} -do_test num-2.1.6 { - sqlite4_num_isinf [sqlite4_num_from_text inf 3] -} {true} - -do_test num-3.1.1 { - sqlite4_num_to_text [sqlite4_num_add 5 7] -} {12} - -do_test num-4.1.1 { - sqlite4_num_to_text [sqlite4_num_sub 9 3] -} {6} -do_test num-4.1.2 { - sqlite4_num_to_text [sqlite4_num_sub 5 12] -} {-7} -do_test num-4.2.1 { - sqlite4_num_compare [sqlite4_num_sub 1 1] [sqlite4_num_sub -1 -1] -} {equal} - -do_test num-5.1.1 { - sqlite4_num_to_text [sqlite4_num_mul 9 8] -} {72} - -do_test num-6.1.1 { - sqlite4_num_to_text [sqlite4_num_div 6 5] -} {1.2} -do_test num-6.1.2 { - sqlite4_num_compare 2 [sqlite4_num_div 2 1] -} {equal} -do_test num-6.1.3 { - sqlite4_num_to_text [sqlite4_num_div 2 1] -} {2} -do_test num-6.1.4 { - sqlite4_num_to_text [sqlite4_num_div 22 10] -} {2.2} -finish_test Index: test/permutations.test ================================================================== --- test/permutations.test +++ test/permutations.test @@ -133,16 +133,13 @@ test_suite "src4" -prefix "" -description { } -files { simple.test simple2.test log3.test - lsm1.test lsm2.test csr1.test ckpt1.test mc1.test - fts5expr1.test fts5query1.test fts5rnd1.test fts5create.test - fts5snippet.test aggerror.test attach.test autoindex1.test badutf.test Index: test/quote.test ================================================================== --- test/quote.test +++ test/quote.test @@ -32,10 +32,13 @@ catchsql {SELECT * FROM '@abc'} } {0 {5 hello}} do_test quote-1.2.2 { catchsql {SELECT * FROM [@abc]} ;# SqlServer compatibility } {0 {5 hello}} +do_test quote-1.2.3 { + catchsql {SELECT * FROM `@abc`} ;# MySQL compatibility +} {0 {5 hello}} do_test quote-1.3 { catchsql { SELECT '@abc'.'!pqr', '@abc'.'#xyz'+5 FROM '@abc' } } {0 {hello 10}} @@ -46,10 +49,15 @@ } {0 {!pqr 5}} do_test quote-1.3.2 { catchsql { SELECT "!pqr", "#xyz"+5 FROM '@abc' } +} {0 {hello 10}} +do_test quote-1.3.3 { + catchsql { + SELECT [!pqr], `#xyz`+5 FROM '@abc' + } } {0 {hello 10}} do_test quote-1.3.4 { set r [catch { execsql {SELECT '@abc'.'!pqr', '@abc'.'#xyz'+5 FROM '@abc'} } msg ] Index: test/select6.test ================================================================== --- test/select6.test +++ test/select6.test @@ -163,15 +163,15 @@ } } {x 3 y 2} do_test select6-3.2 { execsql { SELECT * FROM - (SELECT a.q AS x, a.p, b.r + (SELECT a.q, a.p, b.r FROM (SELECT count(*) as p , b as q FROM t2 GROUP BY q) AS a, (SELECT max(a) as r, b as s FROM t2 GROUP BY s) as b WHERE a.q=b.s ORDER BY a.q) - ORDER BY x + ORDER BY "a.q" } } {1 1 1 2 2 3 3 4 7 4 8 15 5 5 20} do_test select6-3.3 { execsql { SELECT a,b,a+b FROM (SELECT avg(x) as 'a', avg(y) as 'b' FROM t1) DELETED test/shell1.test Index: test/shell1.test ================================================================== --- test/shell1.test +++ /dev/null @@ -1,763 +0,0 @@ -# 2009 Nov 11 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# The focus of this file is testing the CLI shell tool. -# -# - -# Test plan: -# -# shell1-1.*: Basic command line option handling. -# shell1-2.*: Basic "dot" command token parsing. -# shell1-3.*: Basic test that "dot" command can be called. -# -set testdir [file dirname $argv0] -source $testdir/tester.tcl -if {$tcl_platform(platform)=="windows"} { - set CLI "sqlite4.exe" -} else { - set CLI "./sqlite4" -} -if {![file executable $CLI]} { - finish_test - return -} -db close -forcedelete test.db test.db-journal test.db-wal -sqlite4 db test.db - -#---------------------------------------------------------------------------- -# Test cases shell1-1.*: Basic command line option handling. -# - -# invalid option -do_test shell1-1.1.1 { - set res [catchcmd "-bad test.db" ""] - set rc [lindex $res 0] - list $rc \ - [regexp {Error: unknown option: -bad} $res] -} {1 1} -# error on extra options -do_test shell1-1.1.2 { - set res [catchcmd "-bad test.db \"select 3\" \"select 4\"" ""] - set rc [lindex $res 0] - list $rc \ - [regexp {Error: too many options: "select 4"} $res] -} {1 1} -# error on extra options -do_test shell1-1.1.3 { - set res [catchcmd "-bad FOO test.db BAD" ".quit"] - set rc [lindex $res 0] - list $rc \ - [regexp {Error: too many options: "BAD"} $res] -} {1 1} - -# -help -do_test shell1-1.2.1 { - set res [catchcmd "-help test.db" ""] - set rc [lindex $res 0] - list $rc \ - [regexp {Usage} $res] \ - [regexp {\-init} $res] \ - [regexp {\-version} $res] -} {1 1 1 1} - -# -init filename read/process named file -do_test shell1-1.3.1 { - catchcmd "-init FOO test.db" "" -} {0 {}} -do_test shell1-1.3.2 { - set res [catchcmd "-init FOO test.db .quit BAD" ""] - set rc [lindex $res 0] - list $rc \ - [regexp {Error: too many options: "BAD"} $res] -} {1 1} - -# -echo print commands before execution -do_test shell1-1.4.1 { - catchcmd "-echo test.db" "" -} {0 {}} - -# -[no]header turn headers on or off -do_test shell1-1.5.1 { - catchcmd "-header test.db" "" -} {0 {}} -do_test shell1-1.5.2 { - catchcmd "-noheader test.db" "" -} {0 {}} - -# -bail stop after hitting an error -do_test shell1-1.6.1 { - catchcmd "-bail test.db" "" -} {0 {}} - -# -interactive force interactive I/O -do_test shell1-1.7.1 { - set res [catchcmd "-interactive test.db" ".quit"] - set rc [lindex $res 0] - list $rc \ - [regexp {SQLite version} $res] \ - [regexp {Enter SQL statements} $res] -} {0 1 1} - -# -batch force batch I/O -do_test shell1-1.8.1 { - catchcmd "-batch test.db" "" -} {0 {}} - -# -column set output mode to 'column' -do_test shell1-1.9.1 { - catchcmd "-column test.db" "" -} {0 {}} - -# -csv set output mode to 'csv' -do_test shell1-1.10.1 { - catchcmd "-csv test.db" "" -} {0 {}} - -# -html set output mode to HTML -do_test shell1-1.11.1 { - catchcmd "-html test.db" "" -} {0 {}} - -# -line set output mode to 'line' -do_test shell1-1.12.1 { - catchcmd "-line test.db" "" -} {0 {}} - -# -list set output mode to 'list' -do_test shell1-1.13.1 { - catchcmd "-list test.db" "" -} {0 {}} - -# -separator 'x' set output field separator (|) -do_test shell1-1.14.1 { - catchcmd "-separator 'x' test.db" "" -} {0 {}} -do_test shell1-1.14.2 { - catchcmd "-separator x test.db" "" -} {0 {}} -do_test shell1-1.14.3 { - set res [catchcmd "-separator" ""] - set rc [lindex $res 0] - list $rc \ - [regexp {Error: missing argument to -separator} $res] -} {1 1} - -# -stats print memory stats before each finalize -do_test shell1-1.14b.1 { - catchcmd "-stats test.db" "" -} {0 {}} - -# -nullvalue 'text' set text string for NULL values -do_test shell1-1.15.1 { - catchcmd "-nullvalue 'x' test.db" "" -} {0 {}} -do_test shell1-1.15.2 { - catchcmd "-nullvalue x test.db" "" -} {0 {}} -do_test shell1-1.15.3 { - set res [catchcmd "-nullvalue" ""] - set rc [lindex $res 0] - list $rc \ - [regexp {Error: missing argument to -nullvalue} $res] -} {1 1} - -# -version show SQLite version -do_test shell1-1.16.1 { - set x [catchcmd "-version test.db" ""] -} {/4.[0-9.]+ 20\d\d-[01]\d-\d\d \d\d:\d\d:\d\d [0-9a-f]+/} - -#---------------------------------------------------------------------------- -# Test cases shell1-2.*: Basic "dot" command token parsing. -# - -# check first token handling -do_test shell1-2.1.1 { - catchcmd "test.db" ".foo" -} {1 {Error: unknown command or invalid arguments: "foo". Enter ".help" for help}} -do_test shell1-2.1.2 { - catchcmd "test.db" ".\"foo OFF\"" -} {1 {Error: unknown command or invalid arguments: "foo OFF". Enter ".help" for help}} -do_test shell1-2.1.3 { - catchcmd "test.db" ".\'foo OFF\'" -} {1 {Error: unknown command or invalid arguments: "foo OFF". Enter ".help" for help}} - -# unbalanced quotes -do_test shell1-2.2.1 { - catchcmd "test.db" ".\"foo OFF" -} {1 {Error: unknown command or invalid arguments: "foo OFF". Enter ".help" for help}} -do_test shell1-2.2.2 { - catchcmd "test.db" ".\'foo OFF" -} {1 {Error: unknown command or invalid arguments: "foo OFF". Enter ".help" for help}} -do_test shell1-2.2.3 { - catchcmd "test.db" ".explain \"OFF" -} {0 {}} -do_test shell1-2.2.4 { - catchcmd "test.db" ".explain \'OFF" -} {0 {}} -do_test shell1-2.2.5 { - catchcmd "test.db" ".mode \"insert FOO" -} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} -do_test shell1-2.2.6 { - catchcmd "test.db" ".mode \'insert FOO" -} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} - -# check multiple tokens, and quoted tokens -do_test shell1-2.3.1 { - catchcmd "test.db" ".explain 1" -} {0 {}} -do_test shell1-2.3.2 { - catchcmd "test.db" ".explain on" -} {0 {}} -do_test shell1-2.3.3 { - catchcmd "test.db" ".explain \"1 2 3\"" -} {0 {}} -do_test shell1-2.3.4 { - catchcmd "test.db" ".explain \"OFF\"" -} {0 {}} -do_test shell1-2.3.5 { - catchcmd "test.db" ".\'explain\' \'OFF\'" -} {0 {}} -do_test shell1-2.3.6 { - catchcmd "test.db" ".explain \'OFF\'" -} {0 {}} -do_test shell1-2.3.7 { - catchcmd "test.db" ".\'explain\' \'OFF\'" -} {0 {}} - -# check quoted args are unquoted -do_test shell1-2.4.1 { - catchcmd "test.db" ".mode FOO" -} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} -do_test shell1-2.4.2 { - catchcmd "test.db" ".mode csv" -} {0 {}} -do_test shell1-2.4.2 { - catchcmd "test.db" ".mode \"csv\"" -} {0 {}} - - -#---------------------------------------------------------------------------- -# Test cases shell1-3.*: Basic test that "dot" command can be called. -# - - -# .bail ON|OFF Stop after hitting an error. Default OFF -do_test shell1-3.2.1 { - catchcmd "test.db" ".bail" -} {1 {Error: unknown command or invalid arguments: "bail". Enter ".help" for help}} -do_test shell1-3.2.2 { - catchcmd "test.db" ".bail ON" -} {0 {}} -do_test shell1-3.2.3 { - catchcmd "test.db" ".bail OFF" -} {0 {}} -do_test shell1-3.2.4 { - # too many arguments - catchcmd "test.db" ".bail OFF BAD" -} {1 {Error: unknown command or invalid arguments: "bail". Enter ".help" for help}} - -# .databases List names and files of attached databases -do_test shell1-3.3.1 { - catchcmd "-csv test.db" ".databases" -} "/0 +.*main +[string map {/ .} [string range [get_pwd] 0 10]].*/" -do_test shell1-3.3.2 { - # too many arguments - catchcmd "test.db" ".databases BAD" -} {1 {Error: unknown command or invalid arguments: "databases". Enter ".help" for help}} - -# .dump ?TABLE? ... Dump the database in an SQL text format -# If TABLE specified, only dump tables matching -# LIKE pattern TABLE. -do_test shell1-3.4.1 { - set res [catchcmd "test.db" ".dump"] - list [regexp {BEGIN TRANSACTION;} $res] \ - [regexp {COMMIT;} $res] -} {1 1} -do_test shell1-3.4.2 { - set res [catchcmd "test.db" ".dump FOO"] - list [regexp {BEGIN TRANSACTION;} $res] \ - [regexp {COMMIT;} $res] -} {1 1} -do_test shell1-3.4.3 { - # too many arguments - catchcmd "test.db" ".dump FOO BAD" -} {1 {Error: unknown command or invalid arguments: "dump". Enter ".help" for help}} - -# .echo ON|OFF Turn command echo on or off -do_test shell1-3.5.1 { - catchcmd "test.db" ".echo" -} {1 {Error: unknown command or invalid arguments: "echo". Enter ".help" for help}} -do_test shell1-3.5.2 { - catchcmd "test.db" ".echo ON" -} {0 {}} -do_test shell1-3.5.3 { - catchcmd "test.db" ".echo OFF" -} {0 {}} -do_test shell1-3.5.4 { - # too many arguments - catchcmd "test.db" ".echo OFF BAD" -} {1 {Error: unknown command or invalid arguments: "echo". Enter ".help" for help}} - -# .exit Exit this program -do_test shell1-3.6.1 { - catchcmd "test.db" ".exit" -} {0 {}} -do_test shell1-3.6.2 { - # too many arguments - catchcmd "test.db" ".exit BAD" -} {1 {Error: unknown command or invalid arguments: "exit". Enter ".help" for help}} - -# .explain ON|OFF Turn output mode suitable for EXPLAIN on or off. -do_test shell1-3.7.1 { - catchcmd "test.db" ".explain" - # explain is the exception to the booleans. without an option, it turns it on. -} {0 {}} -do_test shell1-3.7.2 { - catchcmd "test.db" ".explain ON" -} {0 {}} -do_test shell1-3.7.3 { - catchcmd "test.db" ".explain OFF" -} {0 {}} -do_test shell1-3.7.4 { - # too many arguments - catchcmd "test.db" ".explain OFF BAD" -} {1 {Error: unknown command or invalid arguments: "explain". Enter ".help" for help}} - - -# .header(s) ON|OFF Turn display of headers on or off -do_test shell1-3.9.1 { - catchcmd "test.db" ".header" -} {1 {Error: unknown command or invalid arguments: "header". Enter ".help" for help}} -do_test shell1-3.9.2 { - catchcmd "test.db" ".header ON" -} {0 {}} -do_test shell1-3.9.3 { - catchcmd "test.db" ".header OFF" -} {0 {}} -do_test shell1-3.9.4 { - # too many arguments - catchcmd "test.db" ".header OFF BAD" -} {1 {Error: unknown command or invalid arguments: "header". Enter ".help" for help}} - -do_test shell1-3.9.5 { - catchcmd "test.db" ".headers" -} {1 {Error: unknown command or invalid arguments: "headers". Enter ".help" for help}} -do_test shell1-3.9.6 { - catchcmd "test.db" ".headers ON" -} {0 {}} -do_test shell1-3.9.7 { - catchcmd "test.db" ".headers OFF" -} {0 {}} -do_test shell1-3.9.8 { - # too many arguments - catchcmd "test.db" ".headers OFF BAD" -} {1 {Error: unknown command or invalid arguments: "headers". Enter ".help" for help}} - -# .help Show this message -do_test shell1-3.10.1 { - set res [catchcmd "test.db" ".help"] - # look for a few of the possible help commands - list [regexp {.help} $res] \ - [regexp {.quit} $res] \ - [regexp {.show} $res] -} {1 1 1} -do_test shell1-3.10.2 { - # we allow .help to take extra args (it is help after all) - set res [catchcmd "test.db" ".help BAD"] - # look for a few of the possible help commands - list [regexp {.help} $res] \ - [regexp {.quit} $res] \ - [regexp {.show} $res] -} {1 1 1} - -# .import FILE TABLE Import data from FILE into TABLE -do_test shell1-3.11.1 { - catchcmd "test.db" ".import" -} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} -do_test shell1-3.11.2 { - catchcmd "test.db" ".import FOO" -} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} -do_test shell1-3.11.2 { - catchcmd "test.db" ".import FOO BAR" -} {1 {Error: no such table: BAR}} -do_test shell1-3.11.3 { - # too many arguments - catchcmd "test.db" ".import FOO BAR BAD" -} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} - -# .indices ?TABLE? Show names of all indices -# If TABLE specified, only show indices for tables -# matching LIKE pattern TABLE. -do_test shell1-3.12.1 { - catchcmd "test.db" ".indices" -} {0 {}} -do_test shell1-3.12.2 { - catchcmd "test.db" ".indices FOO" -} {0 {}} -do_test shell1-3.12.3 { - # too many arguments - catchcmd "test.db" ".indices FOO BAD" -} {1 {Error: unknown command or invalid arguments: "indices". Enter ".help" for help}} - -# .mode MODE ?TABLE? Set output mode where MODE is one of: -# csv Comma-separated values -# column Left-aligned columns. (See .width) -# html HTML code -# insert SQL insert statements for TABLE -# line One value per line -# list Values delimited by .separator string -# tabs Tab-separated values -# tcl TCL list elements -do_test shell1-3.13.1 { - catchcmd "test.db" ".mode" -} {1 {Error: unknown command or invalid arguments: "mode". Enter ".help" for help}} -do_test shell1-3.13.2 { - catchcmd "test.db" ".mode FOO" -} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} -do_test shell1-3.13.3 { - catchcmd "test.db" ".mode csv" -} {0 {}} -do_test shell1-3.13.4 { - catchcmd "test.db" ".mode column" -} {0 {}} -do_test shell1-3.13.5 { - catchcmd "test.db" ".mode html" -} {0 {}} -do_test shell1-3.13.6 { - catchcmd "test.db" ".mode insert" -} {0 {}} -do_test shell1-3.13.7 { - catchcmd "test.db" ".mode line" -} {0 {}} -do_test shell1-3.13.8 { - catchcmd "test.db" ".mode list" -} {0 {}} -do_test shell1-3.13.9 { - catchcmd "test.db" ".mode tabs" -} {0 {}} -do_test shell1-3.13.10 { - catchcmd "test.db" ".mode tcl" -} {0 {}} -do_test shell1-3.13.11 { - # too many arguments - catchcmd "test.db" ".mode tcl BAD" -} {1 {Error: invalid arguments: "BAD". Enter ".help" for help}} - -# don't allow partial mode type matches -do_test shell1-3.13.12 { - catchcmd "test.db" ".mode l" -} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} -do_test shell1-3.13.13 { - catchcmd "test.db" ".mode li" -} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} -do_test shell1-3.13.14 { - catchcmd "test.db" ".mode lin" -} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} - -# .nullvalue STRING Print STRING in place of NULL values -do_test shell1-3.14.1 { - catchcmd "test.db" ".nullvalue" -} {1 {Error: unknown command or invalid arguments: "nullvalue". Enter ".help" for help}} -do_test shell1-3.14.2 { - catchcmd "test.db" ".nullvalue FOO" -} {0 {}} -do_test shell1-3.14.3 { - # too many arguments - catchcmd "test.db" ".nullvalue FOO BAD" -} {1 {Error: unknown command or invalid arguments: "nullvalue". Enter ".help" for help}} - -# .output FILENAME Send output to FILENAME -do_test shell1-3.15.1 { - catchcmd "test.db" ".output" -} {1 {Error: unknown command or invalid arguments: "output". Enter ".help" for help}} -do_test shell1-3.15.2 { - catchcmd "test.db" ".output FOO" -} {0 {}} -do_test shell1-3.15.3 { - # too many arguments - catchcmd "test.db" ".output FOO BAD" -} {1 {Error: unknown command or invalid arguments: "output". Enter ".help" for help}} - -# .output stdout Send output to the screen -do_test shell1-3.16.1 { - catchcmd "test.db" ".output stdout" -} {0 {}} -do_test shell1-3.16.2 { - # too many arguments - catchcmd "test.db" ".output stdout BAD" -} {1 {Error: unknown command or invalid arguments: "output". Enter ".help" for help}} - -# .prompt MAIN CONTINUE Replace the standard prompts -do_test shell1-3.17.1 { - catchcmd "test.db" ".prompt" -} {1 {Error: unknown command or invalid arguments: "prompt". Enter ".help" for help}} -do_test shell1-3.17.2 { - catchcmd "test.db" ".prompt FOO" -} {0 {}} -do_test shell1-3.17.3 { - catchcmd "test.db" ".prompt FOO BAR" -} {0 {}} -do_test shell1-3.17.4 { - # too many arguments - catchcmd "test.db" ".prompt FOO BAR BAD" -} {1 {Error: unknown command or invalid arguments: "prompt". Enter ".help" for help}} - -# .quit Exit this program -do_test shell1-3.18.1 { - catchcmd "test.db" ".quit" -} {0 {}} -do_test shell1-3.18.2 { - # too many arguments - catchcmd "test.db" ".quit BAD" -} {1 {Error: unknown command or invalid arguments: "quit". Enter ".help" for help}} - -# .read FILENAME Execute SQL in FILENAME -do_test shell1-3.19.1 { - catchcmd "test.db" ".read" -} {1 {Error: unknown command or invalid arguments: "read". Enter ".help" for help}} -do_test shell1-3.19.2 { - file delete -force FOO - catchcmd "test.db" ".read FOO" -} {1 {Error: cannot open "FOO"}} -do_test shell1-3.19.3 { - # too many arguments - catchcmd "test.db" ".read FOO BAD" -} {1 {Error: unknown command or invalid arguments: "read". Enter ".help" for help}} - -# .schema ?TABLE? Show the CREATE statements -# If TABLE specified, only show tables matching -# LIKE pattern TABLE. -do_test shell1-3.21.1 { - catchcmd "test.db" ".schema" -} {0 {}} -do_test shell1-3.21.2 { - catchcmd "test.db" ".schema FOO" -} {0 {}} -do_test shell1-3.21.3 { - # too many arguments - catchcmd "test.db" ".schema FOO BAD" -} {1 {Error: unknown command or invalid arguments: "schema". Enter ".help" for help}} - -do_test shell1-3.21.4 { - catchcmd "test.db" { - CREATE TABLE t1(x); - CREATE VIEW v2 AS SELECT x+1 AS y FROM t1; - CREATE VIEW v1 AS SELECT y+1 FROM v2; - } - catchcmd "test.db" ".schema" -} {0 {CREATE TABLE t1(x); -CREATE VIEW v2 AS SELECT x+1 AS y FROM t1; -CREATE VIEW v1 AS SELECT y+1 FROM v2;}} -db eval {DROP VIEW v1; DROP VIEW v2; DROP TABLE t1;} - -# .separator STRING Change separator used by output mode and .import -do_test shell1-3.22.1 { - catchcmd "test.db" ".separator" -} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} -do_test shell1-3.22.2 { - catchcmd "test.db" ".separator FOO" -} {0 {}} -do_test shell1-3.22.3 { - # too many arguments - catchcmd "test.db" ".separator FOO BAD" -} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} - -# .show Show the current values for various settings -do_test shell1-3.23.1 { - set res [catchcmd "test.db" ".show"] - list [regexp {echo:} $res] \ - [regexp {explain:} $res] \ - [regexp {headers:} $res] \ - [regexp {mode:} $res] \ - [regexp {nullvalue:} $res] \ - [regexp {output:} $res] \ - [regexp {separator:} $res] \ - [regexp {stats:} $res] \ - [regexp {width:} $res] -} {1 1 1 1 1 1 1 1 1} -do_test shell1-3.23.2 { - # too many arguments - catchcmd "test.db" ".show BAD" -} {1 {Error: unknown command or invalid arguments: "show". Enter ".help" for help}} - -# .stats ON|OFF Turn stats on or off -do_test shell1-3.23b.1 { - catchcmd "test.db" ".stats" -} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} -do_test shell1-3.23b.2 { - catchcmd "test.db" ".stats ON" -} {0 {}} -do_test shell1-3.23b.3 { - catchcmd "test.db" ".stats OFF" -} {0 {}} -do_test shell1-3.23b.4 { - # too many arguments - catchcmd "test.db" ".stats OFF BAD" -} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} - -# .tables ?TABLE? List names of tables -# If TABLE specified, only list tables matching -# LIKE pattern TABLE. -do_test shell1-3.24.1 { - catchcmd "test.db" ".tables" -} {0 {}} -do_test shell1-3.24.2 { - catchcmd "test.db" ".tables FOO" -} {0 {}} -do_test shell1-3.24.3 { - # too many arguments - catchcmd "test.db" ".tables FOO BAD" -} {1 {Error: unknown command or invalid arguments: "tables". Enter ".help" for help}} - -# .width NUM NUM ... Set column widths for "column" mode -do_test shell1-3.26.1 { - catchcmd "test.db" ".width" -} {1 {Error: unknown command or invalid arguments: "width". Enter ".help" for help}} -do_test shell1-3.26.2 { - catchcmd "test.db" ".width xxx" - # this should be treated the same as a '0' width for col 1 -} {0 {}} -do_test shell1-3.26.3 { - catchcmd "test.db" ".width xxx yyy" - # this should be treated the same as a '0' width for col 1 and 2 -} {0 {}} -do_test shell1-3.26.4 { - catchcmd "test.db" ".width 1 1" - # this should be treated the same as a '1' width for col 1 and 2 -} {0 {}} -do_test shell1-3.26.5 { - catchcmd "test.db" ".mode column\n.width 10 -10\nSELECT 'abcdefg', 123456;" - # this should be treated the same as a '1' width for col 1 and 2 -} {0 {abcdefg 123456}} -do_test shell1-3.26.6 { - catchcmd "test.db" ".mode column\n.width -10 10\nSELECT 'abcdefg', 123456;" - # this should be treated the same as a '1' width for col 1 and 2 -} {0 { abcdefg 123456 }} - - -# .timer ON|OFF Turn the CPU timer measurement on or off -do_test shell1-3.27.1 { - catchcmd "test.db" ".timer" -} {1 {Error: unknown command or invalid arguments: "timer". Enter ".help" for help}} -do_test shell1-3.27.2 { - catchcmd "test.db" ".timer ON" -} {0 {}} -do_test shell1-3.27.3 { - catchcmd "test.db" ".timer OFF" -} {0 {}} -do_test shell1-3.27.4 { - # too many arguments - catchcmd "test.db" ".timer OFF BAD" -} {1 {Error: unknown command or invalid arguments: "timer". Enter ".help" for help}} - -do_test shell1-3-28.1 { - catchcmd test.db \ - ".log stdout\nSELECT coalesce(sqlite_log(123,'hello'),'456');" -} "0 {(123) hello\n456}" - -do_test shell1-3-29.1 { - catchcmd "test.db" ".print this is a test" -} {0 {this is a test}} - -# Test the output of the ".dump" command -# -do_test shell1-4.1 { - db eval { - CREATE TABLE t1(x); - INSERT INTO t1 VALUES(null); - INSERT INTO t1 VALUES(''); - INSERT INTO t1 VALUES(1); - INSERT INTO t1 VALUES(2.25); - INSERT INTO t1 VALUES('hello'); - INSERT INTO t1 VALUES(x'807f'); - } - catchcmd test.db {.dump} -} {0 {PRAGMA foreign_keys=OFF; -BEGIN TRANSACTION; -CREATE TABLE t1(x); -INSERT INTO "t1" VALUES(NULL); -INSERT INTO "t1" VALUES(''); -INSERT INTO "t1" VALUES(1); -INSERT INTO "t1" VALUES(2.25); -INSERT INTO "t1" VALUES('hello'); -INSERT INTO "t1" VALUES(X'807F'); -COMMIT;}} - -# Test the output of ".mode insert" -# -do_test shell1-4.2 { - catchcmd test.db ".mode insert t1\nselect * from t1;" -} {0 {INSERT INTO t1 VALUES(NULL); -INSERT INTO t1 VALUES(''); -INSERT INTO t1 VALUES(1); -INSERT INTO t1 VALUES(2.25); -INSERT INTO t1 VALUES('hello'); -INSERT INTO t1 VALUES(X'807f');}} - -# Test the output of ".mode tcl" -# -do_test shell1-4.3 { - catchcmd test.db ".mode tcl\nselect * from t1;" -} {0 {"" -"" -"1" -"2.25" -"hello" -"\200\177"}} - -# Test the output of ".mode tcl" with multiple columns -# -do_test shell1-4.4 { - db eval { - CREATE TABLE t2(x,y); - INSERT INTO t2 VALUES(null, ''); - INSERT INTO t2 VALUES(1, 2.25); - INSERT INTO t2 VALUES('hello', x'807f'); - } - catchcmd test.db ".mode tcl\nselect * from t2;" -} {0 {"" "" -"1" "2.25" -"hello" "\200\177"}} - -# Test the output of ".mode tcl" with ".nullvalue" -# -do_test shell1-4.5 { - catchcmd test.db ".mode tcl\n.nullvalue NULL\nselect * from t2;" -} {0 {"NULL" "" -"1" "2.25" -"hello" "\200\177"}} - -# Test the output of ".mode tcl" with Tcl reserved characters -# -do_test shell1-4.6 { - db eval { - CREATE TABLE tcl1(x); - INSERT INTO tcl1 VALUES('"'); - INSERT INTO tcl1 VALUES('['); - INSERT INTO tcl1 VALUES(']'); - INSERT INTO tcl1 VALUES('\{'); - INSERT INTO tcl1 VALUES('\}'); - INSERT INTO tcl1 VALUES(';'); - INSERT INTO tcl1 VALUES('$'); - } - foreach {x y} [catchcmd test.db ".mode tcl\nselect * from tcl1;"] break - list $x $y [llength $y] -} {0 {"\"" -"[" -"]" -"\\{" -"\\}" -";" -"$"} 7} - -finish_test DELETED test/shell2.test Index: test/shell2.test ================================================================== --- test/shell2.test +++ /dev/null @@ -1,197 +0,0 @@ -# 2009 Nov 11 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# The focus of this file is testing the CLI shell tool. -# -# $Id: shell2.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ -# - -# Test plan: -# -# shell2-1.*: Misc. test of various tickets and reported errors. -# -set testdir [file dirname $argv0] -source $testdir/tester.tcl -if {$tcl_platform(platform)=="windows"} { - set CLI "sqlite4.exe" -} else { - set CLI "./sqlite4" -} -if {![file executable $CLI]} { - finish_test - return -} -db close -forcedelete test.db test.db-journal test.db-wal -sqlite4 db test.db - - -#---------------------------------------------------------------------------- -# shell2-1.*: Misc. test of various tickets and reported errors. -# - -# Batch mode not creating databases. -# Reported on mailing list by Ken Zalewski. -# Ticket [aeff892c57]. -do_test shell2-1.1.1 { - file delete -force foo.db - set rc [ catchcmd "-batch foo.db" "CREATE TABLE t1(a);" ] - set fexist [file exist foo.db] - list $rc $fexist -} {{0 {}} 1} - -# Shell silently ignores extra parameters. -# Ticket [f5cb008a65]. -do_test shell2-1.2.1 { - set rc [catch { eval exec $CLI \":memory:\" \"select 3\" \"select 4\" } msg] - list $rc \ - [regexp {Error: too many options: "select 4"} $msg] -} {1 1} - -# Test a problem reported on the mailing list. The shell was at one point -# returning the generic SQLITE_ERROR message ("SQL error or missing database") -# instead of the "too many levels..." message in the test below. -# -do_test shell2-1.3 { - catchcmd "-batch test.db" { - PRAGMA recursive_triggers = ON; - CREATE TABLE t5(a PRIMARY KEY, b, c); - INSERT INTO t5 VALUES(1, 2, 3); - CREATE TRIGGER au_tble AFTER UPDATE ON t5 BEGIN - UPDATE OR IGNORE t5 SET a = new.a, c = 10; - END; - - UPDATE OR REPLACE t5 SET a = 4 WHERE a = 1; - } -} {1 {Error: near line 9: too many levels of trigger recursion}} - - - -# Shell not echoing all commands with echo on. -# Ticket [eb620916be]. - -# Test with echo off -# NB. whitespace is important -do_test shell2-1.4.1 { - file delete -force foo.db - catchcmd "foo.db" {CREATE TABLE foo(a); -INSERT INTO foo(a) VALUES(1); -SELECT * FROM foo;} -} {0 1} - -# Test with echo on using command line option -# NB. whitespace is important -do_test shell2-1.4.2 { - file delete -force foo.db - catchcmd "-echo foo.db" {CREATE TABLE foo(a); -INSERT INTO foo(a) VALUES(1); -SELECT * FROM foo;} -} {0 {CREATE TABLE foo(a); -INSERT INTO foo(a) VALUES(1); -SELECT * FROM foo; -1}} - -# Test with echo on using dot command -# NB. whitespace is important -do_test shell2-1.4.3 { - file delete -force foo.db - catchcmd "foo.db" {.echo ON -CREATE TABLE foo(a); -INSERT INTO foo(a) VALUES(1); -SELECT * FROM foo;} -} {0 {CREATE TABLE foo(a); -INSERT INTO foo(a) VALUES(1); -SELECT * FROM foo; -1}} - -# Test with echo on using dot command and -# turning off mid- processing. -# NB. whitespace is important -do_test shell2-1.4.4 { - file delete -force foo.db - catchcmd "foo.db" {.echo ON -CREATE TABLE foo(a); -.echo OFF -INSERT INTO foo(a) VALUES(1); -SELECT * FROM foo;} -} {0 {CREATE TABLE foo(a); -.echo OFF -1}} - -# Test with echo on using dot command and -# multiple commands per line. -# NB. whitespace is important -do_test shell2-1.4.5 { - file delete -force foo.db - catchcmd "foo.db" {.echo ON -CREATE TABLE foo1(a); -INSERT INTO foo1(a) VALUES(1); -CREATE TABLE foo2(b); -INSERT INTO foo2(b) VALUES(1); -SELECT * FROM foo1; SELECT * FROM foo2; -INSERT INTO foo1(a) VALUES(2); INSERT INTO foo2(b) VALUES(2); -SELECT * FROM foo1; SELECT * FROM foo2; -} -} {0 {CREATE TABLE foo1(a); -INSERT INTO foo1(a) VALUES(1); -CREATE TABLE foo2(b); -INSERT INTO foo2(b) VALUES(1); -SELECT * FROM foo1; -1 -SELECT * FROM foo2; -1 -INSERT INTO foo1(a) VALUES(2); -INSERT INTO foo2(b) VALUES(2); -SELECT * FROM foo1; -1 -2 -SELECT * FROM foo2; -1 -2}} - -# Test with echo on and headers on using dot command and -# multiple commands per line. -# NB. whitespace is important -do_test shell2-1.4.6 { - file delete -force foo.db - catchcmd "foo.db" {.echo ON -.headers ON -CREATE TABLE foo1(a); -INSERT INTO foo1(a) VALUES(1); -CREATE TABLE foo2(b); -INSERT INTO foo2(b) VALUES(1); -SELECT * FROM foo1; SELECT * FROM foo2; -INSERT INTO foo1(a) VALUES(2); INSERT INTO foo2(b) VALUES(2); -SELECT * FROM foo1; SELECT * FROM foo2; -} -} {0 {.headers ON -CREATE TABLE foo1(a); -INSERT INTO foo1(a) VALUES(1); -CREATE TABLE foo2(b); -INSERT INTO foo2(b) VALUES(1); -SELECT * FROM foo1; -a -1 -SELECT * FROM foo2; -b -1 -INSERT INTO foo1(a) VALUES(2); -INSERT INTO foo2(b) VALUES(2); -SELECT * FROM foo1; -a -1 -2 -SELECT * FROM foo2; -b -1 -2}} - -finish_test DELETED test/shell3.test Index: test/shell3.test ================================================================== --- test/shell3.test +++ /dev/null @@ -1,97 +0,0 @@ -# 2009 Dec 16 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# The focus of this file is testing the CLI shell tool. -# -# $Id: shell2.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ -# - -# Test plan: -# -# shell3-1.*: Basic tests for running SQL statments from command line. -# shell3-2.*: Basic tests for running SQL file from command line. -# -set testdir [file dirname $argv0] -source $testdir/tester.tcl -if {$tcl_platform(platform)=="windows"} { - set CLI "sqlite4.exe" -} else { - set CLI "./sqlite4" -} -if {![file executable $CLI]} { - finish_test - return -} -db close -forcedelete test.db test.db-journal test.db-wal -sqlite4 db test.db - -#---------------------------------------------------------------------------- -# shell3-1.*: Basic tests for running SQL statments from command line. -# - -# Run SQL statement from command line -do_test shell3-1.1 { - file delete -force foo.db - set rc [ catchcmd "foo.db \"CREATE TABLE t1(a);\"" ] - set fexist [file exist foo.db] - list $rc $fexist -} {{0 {}} 1} -do_test shell3-1.2 { - catchcmd "foo.db" ".tables" -} {0 t1} -do_test shell3-1.3 { - catchcmd "foo.db \"DROP TABLE t1;\"" -} {0 {}} -do_test shell3-1.4 { - catchcmd "foo.db" ".tables" -} {0 {}} -do_test shell3-1.5 { - catchcmd "foo.db \"CREATE TABLE t1(a); DROP TABLE t1;\"" -} {0 {}} -do_test shell3-1.6 { - catchcmd "foo.db" ".tables" -} {0 {}} -do_test shell3-1.7 { - catchcmd "foo.db \"CREATE TABLE\"" -} {1 {Error: near "TABLE": syntax error}} - -#---------------------------------------------------------------------------- -# shell3-2.*: Basic tests for running SQL file from command line. -# - -# Run SQL file from command line -do_test shell3-2.1 { - file delete -force foo.db - set rc [ catchcmd "foo.db" "CREATE TABLE t1(a);" ] - set fexist [file exist foo.db] - list $rc $fexist -} {{0 {}} 1} -do_test shell3-2.2 { - catchcmd "foo.db" ".tables" -} {0 t1} -do_test shell3-2.3 { - catchcmd "foo.db" "DROP TABLE t1;" -} {0 {}} -do_test shell3-2.4 { - catchcmd "foo.db" ".tables" -} {0 {}} -do_test shell3-2.5 { - catchcmd "foo.db" "CREATE TABLE t1(a); DROP TABLE t1;" -} {0 {}} -do_test shell3-2.6 { - catchcmd "foo.db" ".tables" -} {0 {}} -do_test shell3-2.7 { - catchcmd "foo.db" "CREATE TABLE" -} {1 {Error: incomplete SQL: CREATE TABLE}} - -finish_test DELETED test/shell4.test Index: test/shell4.test ================================================================== --- test/shell4.test +++ /dev/null @@ -1,116 +0,0 @@ -# 2010 July 28 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# The focus of this file is testing the CLI shell tool. -# These tests are specific to the .stats command. -# -# $Id: shell4.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ -# - -# Test plan: -# -# shell4-1.*: Basic tests specific to the "stats" command. -# -set testdir [file dirname $argv0] -source $testdir/tester.tcl -if {$tcl_platform(platform)=="windows"} { - set CLI "sqlite4.exe" -} else { - set CLI "./sqlite4" -} -if {![file executable $CLI]} { - finish_test - return -} -db close -forcedelete test.db test.db-journal test.db-wal -sqlite4 db test.db - -#---------------------------------------------------------------------------- -# Test cases shell4-1.*: Tests specific to the "stats" command. -# - -# should default to off -do_test shell4-1.1.1 { - set res [catchcmd "test.db" ".show"] - list [regexp {stats: off} $res] -} {1} - -do_test shell4-1.1.2 { - set res [catchcmd "test.db" ".show"] - list [regexp {stats: on} $res] -} {0} - -# -stats should turn it on -do_test shell4-1.2.1 { - set res [catchcmd "-stats test.db" ".show"] - list [regexp {stats: on} $res] -} {1} - -do_test shell4-1.2.2 { - set res [catchcmd "-stats test.db" ".show"] - list [regexp {stats: off} $res] -} {0} - -# .stats ON|OFF Turn stats on or off -do_test shell4-1.3.1 { - catchcmd "test.db" ".stats" -} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} -do_test shell4-1.3.2 { - catchcmd "test.db" ".stats ON" -} {0 {}} -do_test shell4-1.3.3 { - catchcmd "test.db" ".stats OFF" -} {0 {}} -do_test shell4-1.3.4 { - # too many arguments - catchcmd "test.db" ".stats OFF BAD" -} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} - -# NB. whitespace is important -do_test shell4-1.4.1 { - set res [catchcmd "test.db" {.show}] - list [regexp {stats: off} $res] -} {1} - -do_test shell4-1.4.2 { - set res [catchcmd "test.db" {.stats ON -.show -}] - list [regexp {stats: on} $res] -} {1} - -do_test shell4-1.4.3 { - set res [catchcmd "test.db" {.stats OFF -.show -}] - list [regexp {stats: off} $res] -} {1} - -# make sure stats not present when off -do_test shell4-1.5.1 { - set res [catchcmd "test.db" {SELECT 1;}] - list [regexp {Memory Used} $res] \ - [regexp {Heap Usage} $res] \ - [regexp {Autoindex Inserts} $res] -} {0 0 0} - -# make sure stats are present when on -do_test shell4-1.5.2 { - set res [catchcmd "test.db" {.stats ON -SELECT 1; -}] - list [regexp {Memory Used} $res] \ - [regexp {Heap Usage} $res] \ - [regexp {Autoindex Inserts} $res] -} {1 1 1} - -finish_test DELETED test/shell5.test Index: test/shell5.test ================================================================== --- test/shell5.test +++ /dev/null @@ -1,229 +0,0 @@ -# 2010 August 4 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# The focus of this file is testing the CLI shell tool. -# These tests are specific to the .import command. -# -# $Id: shell5.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ -# - -# Test plan: -# -# shell5-1.*: Basic tests specific to the ".import" command. -# -set testdir [file dirname $argv0] -source $testdir/tester.tcl -if {$tcl_platform(platform)=="windows"} { - set CLI "sqlite4.exe" -} else { - set CLI "./sqlite4" -} -if {![file executable $CLI]} { - finish_test - return -} -db close -forcedelete test.db test.db-journal test.db-wal -sqlite4 db test.db - -#---------------------------------------------------------------------------- -# Test cases shell5-1.*: Basic handling of the .import and .separator commands. -# - -# .import FILE TABLE Import data from FILE into TABLE -do_test shell5-1.1.1 { - catchcmd "test.db" ".import" -} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} -do_test shell5-1.1.2 { - catchcmd "test.db" ".import FOO" -} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} -do_test shell5-1.1.2 { - catchcmd "test.db" ".import FOO BAR" -} {1 {Error: no such table: BAR}} -do_test shell5-1.1.3 { - # too many arguments - catchcmd "test.db" ".import FOO BAR BAD" -} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} - -# .separator STRING Change separator used by output mode and .import -do_test shell1-1.2.1 { - catchcmd "test.db" ".separator" -} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} -do_test shell1-1.2.2 { - catchcmd "test.db" ".separator FOO" -} {0 {}} -do_test shell1-1.2.3 { - # too many arguments - catchcmd "test.db" ".separator FOO BAD" -} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} - -# separator should default to "|" -do_test shell5-1.3.1 { - set res [catchcmd "test.db" ".show"] - list [regexp {separator: \"\|\"} $res] -} {1} - -# set separator to different value. -# check that .show reports new value -do_test shell5-1.3.2 { - set res [catchcmd "test.db" {.separator , -.show}] - list [regexp {separator: \",\"} $res] -} {1} - -# import file doesn't exist -do_test shell5-1.4.1 { - file delete -force FOO - set res [catchcmd "test.db" {CREATE TABLE t1(a, b); -.import FOO t1}] -} {1 {Error: cannot open "FOO"}} - -# empty import file -do_test shell5-1.4.2 { - file delete -force shell5.csv - set in [open shell5.csv w] - close $in - set res [catchcmd "test.db" {.import shell5.csv t1 -SELECT COUNT(*) FROM t1;}] -} {0 0} - -# import file with 1 row, 1 column (expecting 2 cols) -do_test shell5-1.4.3 { - set in [open shell5.csv w] - puts $in "1" - close $in - set res [catchcmd "test.db" {.import shell5.csv t1}] -} {1 {Error: shell5.csv line 1: expected 2 columns of data but found 1}} - -# import file with 1 row, 3 columns (expecting 2 cols) -do_test shell5-1.4.4 { - set in [open shell5.csv w] - puts $in "1|2|3" - close $in - set res [catchcmd "test.db" {.import shell5.csv t1}] -} {1 {Error: shell5.csv line 1: expected 2 columns of data but found 3}} - -# import file with 1 row, 2 columns -do_test shell5-1.4.5 { - set in [open shell5.csv w] - puts $in "1|2" - close $in - set res [catchcmd "test.db" {.import shell5.csv t1 -SELECT COUNT(*) FROM t1;}] -} {0 1} - -# import file with 2 rows, 2 columns -# note we end up with 3 rows because of the 1 row -# imported above. -do_test shell5-1.4.6 { - set in [open shell5.csv w] - puts $in "2|3" - puts $in "3|4" - close $in - set res [catchcmd "test.db" {.import shell5.csv t1 -SELECT COUNT(*) FROM t1;}] -} {0 3} - -# import file with 1 row, 2 columns, using a comma -do_test shell5-1.4.7 { - set in [open shell5.csv w] - puts $in "4,5" - close $in - set res [catchcmd "test.db" {.separator , -.import shell5.csv t1 -SELECT COUNT(*) FROM t1;}] -} {0 4} - -# import file with 1 row, 2 columns, text data -do_test shell5-1.4.8.1 { - set in [open shell5.csv w] - puts $in "5|Now is the time for all good men to come to the aid of their country." - close $in - set res [catchcmd "test.db" {.import shell5.csv t1 -SELECT COUNT(*) FROM t1;}] -} {0 5} - -do_test shell5-1.4.8.2 { - catchcmd "test.db" {SELECT b FROM t1 WHERE a='5';} -} {0 {Now is the time for all good men to come to the aid of their country.}} - -# import file with 1 row, 2 columns, quoted text data -# note that currently sqlite doesn't support quoted fields, and -# imports the entire field, quotes and all. -do_test shell5-1.4.9.1 { - set in [open shell5.csv w] - puts $in "6|'Now is the time for all good men to come to the aid of their country.'" - close $in - set res [catchcmd "test.db" {.import shell5.csv t1 -SELECT COUNT(*) FROM t1;}] -} {0 6} - -do_test shell5-1.4.9.2 { - catchcmd "test.db" {SELECT b FROM t1 WHERE a='6';} -} {0 {'Now is the time for all good men to come to the aid of their country.'}} - -# import file with 1 row, 2 columns, quoted text data -do_test shell5-1.4.10.1 { - set in [open shell5.csv w] - puts $in "7|\"Now is the time for all good men to come to the aid of their country.\"" - close $in - set res [catchcmd "test.db" {.import shell5.csv t1 -SELECT COUNT(*) FROM t1;}] -} {0 7} - -do_test shell5-1.4.10.2 { - catchcmd "test.db" {SELECT b FROM t1 WHERE a='7';} -} {0 {Now is the time for all good men to come to the aid of their country.}} - -# check importing very long field -do_test shell5-1.5.1 { - set str [string repeat X 999] - set in [open shell5.csv w] - puts $in "8|$str" - close $in - set res [catchcmd "test.db" {.import shell5.csv t1 -SELECT length(b) FROM t1 WHERE a='8';}] -} {0 999} - -# try importing into a table with a large number of columns. -# This is limited by SQLITE_MAX_VARIABLE_NUMBER, which defaults to 999. -set cols 999 -do_test shell5-1.6.1 { - set sql {CREATE TABLE t2(} - set data {} - for {set i 1} {$i<$cols} {incr i} { - append sql "c$i," - append data "$i|" - } - append sql "c$cols);" - append data "$cols" - catchcmd "test.db" $sql - set in [open shell5.csv w] - puts $in $data - close $in - set res [catchcmd "test.db" {.import shell5.csv t2 -SELECT COUNT(*) FROM t2;}] -} {0 1} - -# try importing a large number of rows -set rows 999999 -do_test shell5-1.7.1 { - set in [open shell5.csv w] - for {set i 1} {$i<=$rows} {incr i} { - puts $in $i - } - close $in - set res [catchcmd "test.db" {CREATE TABLE t3(a); -.import shell5.csv t3 -SELECT COUNT(*) FROM t3;}] -} [list 0 $rows] - -finish_test Index: test/simple.test ================================================================== --- test/simple.test +++ test/simple.test @@ -98,11 +98,10 @@ #------------------------------------------------------------------------- reset_db -breakpoint do_execsql_test 4.1 { CREATE TABLE t1(k PRIMARY KEY, v) } do_execsql_test 4.2 { CREATE INDEX i1 ON t1(v) } do_execsql_test 4.3 { SELECT * FROM sqlite_master Index: test/subquery.test ================================================================== --- test/subquery.test +++ test/subquery.test @@ -387,11 +387,11 @@ INSERT INTO t5 VALUES(1,11); INSERT INTO t5 VALUES(2,22); INSERT INTO t5 VALUES(3,33); INSERT INTO t5 VALUES(4,44); SELECT b FROM t5 WHERE a IN - (SELECT callcnt(y)+0 FROM t4 WHERE x='two') + (SELECT callcnt(y)+0 FROM t4 WHERE x="two") } } {22} do_test subquery-5.2 { # This is the key test. The subquery should have only run once. If # The double-quoted identifier "two" were causing the subquery to be Index: test/substr.test ================================================================== --- test/substr.test +++ test/substr.test @@ -126,15 +126,15 @@ # If these blobs were strings, then they would contain multi-byte # characters. But since they are blobs, the substr indices refer # to bytes. # subblob-test 4.1 61E188B462E28D8563E3919663 1 1 61 -subblob-test 4.2 61E188B462E28D8563E3919663 2 1 e1 -subblob-test 4.3 61E188B462E28D8563E3919663 1 2 61e1 +subblob-test 4.2 61E188B462E28D8563E3919663 2 1 E1 +subblob-test 4.3 61E188B462E28D8563E3919663 1 2 61E1 subblob-test 4.4 61E188B462E28D8563E3919663 -2 1 96 -subblob-test 4.5 61E188B462E28D8563E3919663 -5 4 63e39196 -subblob-test 4.6 61E188B462E28D8563E3919663 -100 98 61e188b462e28d8563e391 +subblob-test 4.5 61E188B462E28D8563E3919663 -5 4 63E39196 +subblob-test 4.6 61E188B462E28D8563E3919663 -100 98 61E188B462E28D8563E391 # Two-argument SUBSTR # proc substr-2-test {id string idx result} { db eval { Index: test/test_config.c ================================================================== --- test/test_config.c +++ test/test_config.c @@ -322,10 +322,16 @@ #ifdef SQLITE4_OMIT_INTEGRITY_CHECK Tcl_SetVar2(interp, "sqlite_options", "integrityck", "0", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "integrityck", "1", TCL_GLOBAL_ONLY); #endif + +#if defined(SQLITE4_DEFAULT_FILE_FORMAT) && SQLITE4_DEFAULT_FILE_FORMAT==1 + Tcl_SetVar2(interp, "sqlite_options", "legacyformat", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "legacyformat", "0", TCL_GLOBAL_ONLY); +#endif #ifdef SQLITE4_OMIT_LIKE_OPTIMIZATION Tcl_SetVar2(interp, "sqlite_options", "like_opt", "0", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "like_opt", "1", TCL_GLOBAL_ONLY); @@ -562,10 +568,11 @@ LINKVAR( MAX_LIKE_PATTERN_LENGTH ); LINKVAR( MAX_TRIGGER_DEPTH ); LINKVAR( DEFAULT_TEMP_CACHE_SIZE ); LINKVAR( DEFAULT_CACHE_SIZE ); LINKVAR( DEFAULT_PAGE_SIZE ); + LINKVAR( DEFAULT_FILE_FORMAT ); LINKVAR( MAX_ATTACHED ); LINKVAR( MAX_DEFAULT_PAGE_SIZE ); { static const int cv_TEMP_STORE = SQLITE4_TEMP_STORE; Index: test/test_func.c ================================================================== --- test/test_func.c +++ test/test_func.c @@ -67,11 +67,11 @@ sqlite4_randomness(pEnv, n, zBuf); for(i=0; ipEnv, p); } /* @@ -263,11 +258,11 @@ sqlite4_result_error_nomem(pCtx); return; } pCounter->cnt = sqlite4_value_int(argv[0]); pCounter->pEnv = sqlite4_context_env(pCtx); - sqlite4_set_auxdata(pCtx, 0, pCounter, counterFree, 0); + sqlite4_set_auxdata(pCtx, 0, pCounter, counterFree); }else{ pCounter->cnt++; } sqlite4_result_int(pCtx, pCounter->cnt); } @@ -326,11 +321,11 @@ char *zErr; sqlite4_env *pEnv = sqlite4_context_env(pCtx); assert( pStmt==0 ); zErr = sqlite4_mprintf(pEnv, "sqlite4_prepare() error: %s", sqlite4_errmsg(db)); - sqlite4_result_text(pCtx, zErr, -1, SQLITE4_DYNAMIC, 0); + sqlite4_result_text(pCtx, zErr, -1, SQLITE4_DYNAMIC); sqlite4_result_error_code(pCtx, rc); } } @@ -379,11 +374,11 @@ zOut = sqlite4_malloc(sqlite4_context_env(pCtx), n/2 ); if( zOut==0 ){ sqlite4_result_error_nomem(pCtx); }else{ testHexToBin(zIn, zOut); - sqlite4_result_text16be(pCtx, zOut, n/2, SQLITE4_DYNAMIC, 0); + sqlite4_result_text16be(pCtx, zOut, n/2, SQLITE4_DYNAMIC); } } #endif /* @@ -406,11 +401,11 @@ zOut = sqlite4_malloc(sqlite4_context_env(pCtx), n/2 ); if( zOut==0 ){ sqlite4_result_error_nomem(pCtx); }else{ testHexToBin(zIn, zOut); - sqlite4_result_text(pCtx, zOut, n/2, SQLITE4_DYNAMIC, 0); + sqlite4_result_text(pCtx, zOut, n/2, SQLITE4_DYNAMIC); } } /* ** hex_to_utf16le(HEX) @@ -433,11 +428,11 @@ zOut = sqlite4_malloc(sqlite4_context_env(pCtx), n/2 ); if( zOut==0 ){ sqlite4_result_error_nomem(pCtx); }else{ testHexToBin(zIn, zOut); - sqlite4_result_text16le(pCtx, zOut, n/2, SQLITE4_DYNAMIC, 0); + sqlite4_result_text16le(pCtx, zOut, n/2, SQLITE4_DYNAMIC); } } #endif static int registerTestFunctions(sqlite4 *db){ DELETED test/test_kv.c Index: test/test_kv.c ================================================================== --- test/test_kv.c +++ /dev/null @@ -1,495 +0,0 @@ -/* -** 2011 January 21 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Code for testing the storage subsystem using the Storage interface. -*/ -#include "sqliteInt.h" - -/* Defined in test1.c */ -extern void *sqlite4TestTextToPtr(const char*); - -/* Defined in test_hexio.c */ -extern void sqlite4TestBinToHex(unsigned char*,int); -extern int sqlite4TestHexToBin(const unsigned char *in,int,unsigned char *out); - -/* Set the TCL result to an integer. -*/ -static void storageSetTclErrorName(Tcl_Interp *interp, int rc){ - extern const char *sqlite4TestErrorName(int); - Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite4TestErrorName(rc), -1)); -} - -/* -** TCLCMD: storage_open URI ?FLAGS? -** -** Return a string that identifies the new storage object. -*/ -static int test_storage_open( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - KVStore *pNew = 0; - int rc; - int flags = 0; - sqlite4 db; - char zRes[50]; - if( objc!=2 && objc!=3 ){ - Tcl_WrongNumArgs(interp, 1, objv, "URI ?FLAGS?"); - return TCL_ERROR; - } - if( objc==3 && Tcl_GetIntFromObj(interp, objv[2], &flags) ){ - return TCL_ERROR; - } - memset(&db, 0, sizeof(db)); - rc = sqlite4KVStoreOpen(&db, "test", Tcl_GetString(objv[1]), &pNew, flags); - if( rc ){ - sqlite4KVStoreClose(pNew); - storageSetTclErrorName(interp, rc); - return TCL_ERROR; - } - sqlite4_snprintf(zRes,sizeof(zRes), "%p", pNew); - Tcl_SetObjResult(interp, Tcl_NewStringObj(zRes,-1)); - return TCL_OK; -} - -/* -** TCLCMD: storage_close STORAGE -** -** Close a storage object. -*/ -static int test_storage_close( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - KVStore *pOld = 0; - int rc; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 2, objv, "STORAGE"); - return TCL_ERROR; - } - pOld = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); - rc = sqlite4KVStoreClose(pOld); - if( rc ){ - storageSetTclErrorName(interp, rc); - return TCL_ERROR; - } - return TCL_OK; -} - -/* -** TCLCMD: storage_open_cursor STORAGE -** -** Return a string that identifies the new storage cursor -*/ -static int test_storage_open_cursor( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - KVCursor *pNew = 0; - KVStore *pStore = 0; - int rc; - char zRes[50]; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 2, objv, "STORAGE"); - return TCL_ERROR; - } - pStore = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); - rc = sqlite4KVStoreOpenCursor(pStore, &pNew); - if( rc ){ - sqlite4KVCursorClose(pNew); - storageSetTclErrorName(interp, rc); - return TCL_ERROR; - } - sqlite4_snprintf(zRes,sizeof(zRes), "%p", pNew); - Tcl_SetObjResult(interp, Tcl_NewStringObj(zRes,-1)); - return TCL_OK; -} - -/* -** TCLCMD: storage_close_cursor CURSOR -** -** Close a cursor object. -*/ -static int test_storage_close_cursor( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - KVCursor *pOld = 0; - int rc; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 2, objv, "CURSOR"); - return TCL_ERROR; - } - pOld = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); - rc = sqlite4KVCursorClose(pOld); - if( rc ){ - storageSetTclErrorName(interp, rc); - return TCL_ERROR; - } - return TCL_OK; -} - -/* Decode hex into a binary key */ -static void sqlite4DecodeHex(Tcl_Obj *pObj, unsigned char *a, int *pN){ - const unsigned char *pIn; - int nIn; - pIn = (const unsigned char*)Tcl_GetStringFromObj(pObj, &nIn); - *pN = sqlite4TestHexToBin(pIn, nIn, a); -} - -/* -** TCLCMD: storage_replace STORAGE KEY VALUE -** -** Insert content into a KV storage object. KEY and VALUE are hex. -*/ -static int test_storage_replace( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - KVStore *p = 0; - int rc; - int nKey, nData; - unsigned char zKey[200]; - unsigned char zData[200]; - if( objc!=4 ){ - Tcl_WrongNumArgs(interp, 2, objv, "STORAGE KEY VALUE"); - return TCL_ERROR; - } - p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); - sqlite4DecodeHex(objv[2], zKey, &nKey); - sqlite4DecodeHex(objv[3], zData, &nData); - rc = sqlite4KVStoreReplace(p, zKey, nKey, zData, nData); - if( rc ){ - storageSetTclErrorName(interp, rc); - return TCL_ERROR; - } - return TCL_OK; -} - -/* -** TCLCMD: storage_begin STORAGE LEVEL -** -** Increase the transaction level -*/ -static int test_storage_begin( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - KVStore *p = 0; - int rc; - int iLevel; - if( objc!=3 ){ - Tcl_WrongNumArgs(interp, 2, objv, "STORAGE LEVEL"); - return TCL_ERROR; - } - p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); - if( Tcl_GetIntFromObj(interp, objv[2], &iLevel) ) return TCL_ERROR; - rc = sqlite4KVStoreBegin(p, iLevel); - if( rc ){ - storageSetTclErrorName(interp, rc); - return TCL_ERROR; - } - return TCL_OK; -} - -/* -** TCLCMD: storage_commit STORAGE LEVEL -** -** Increase the transaction level -*/ -static int test_storage_commit( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - KVStore *p = 0; - int rc; - int iLevel; - if( objc!=3 ){ - Tcl_WrongNumArgs(interp, 2, objv, "STORAGE LEVEL"); - return TCL_ERROR; - } - p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); - if( Tcl_GetIntFromObj(interp, objv[2], &iLevel) ) return TCL_ERROR; - rc = sqlite4KVStoreCommitPhaseOne(p, iLevel); - if( rc ){ - storageSetTclErrorName(interp, rc); - return TCL_ERROR; - } - rc = sqlite4KVStoreCommitPhaseTwo(p, iLevel); - if( rc ){ - storageSetTclErrorName(interp, rc); - return TCL_ERROR; - } - return TCL_OK; -} - -/* -** TCLCMD: storage_rollback STORAGE LEVEL -** -** Increase the transaction level -*/ -static int test_storage_rollback( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - KVStore *p = 0; - int rc; - int iLevel; - if( objc!=3 ){ - Tcl_WrongNumArgs(interp, 2, objv, "STORAGE LEVEL"); - return TCL_ERROR; - } - p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); - if( Tcl_GetIntFromObj(interp, objv[2], &iLevel) ) return TCL_ERROR; - rc = sqlite4KVStoreRollback(p, iLevel); - if( rc ){ - storageSetTclErrorName(interp, rc); - return TCL_ERROR; - } - return TCL_OK; -} - -/* -** TCLCMD: storage_seek CURSOR KEY DIRECTION -** -** Move a cursor object -*/ -static int test_storage_seek( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - KVCursor *p = 0; - int rc; - int nKey, dir; - unsigned char aKey[100]; - if( objc!=4 ){ - Tcl_WrongNumArgs(interp, 4, objv, "CURSOR KEY DIRECTION"); - return TCL_ERROR; - } - p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); - sqlite4DecodeHex(objv[2], aKey, &nKey); - if( Tcl_GetIntFromObj(interp, objv[3], &dir) ) return TCL_ERROR; - rc = sqlite4KVCursorSeek(p, aKey, nKey, dir); - storageSetTclErrorName(interp, rc); - return TCL_OK; -} - -/* -** TCLCMD: storage_next CURSOR -** -** Move the cursor to the next entry -*/ -static int test_storage_next( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - KVCursor *p = 0; - int rc; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 2, objv, "CURSOR"); - return TCL_ERROR; - } - p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); - rc = sqlite4KVCursorNext(p); - storageSetTclErrorName(interp, rc); - return TCL_OK; -} - -/* -** TCLCMD: storage_prev CURSOR -** -** Move the cursor to the previous entry -*/ -static int test_storage_prev( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - KVCursor *p = 0; - int rc; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 2, objv, "CURSOR"); - return TCL_ERROR; - } - p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); - rc = sqlite4KVCursorPrev(p); - storageSetTclErrorName(interp, rc); - return TCL_OK; -} - -/* -** TCLCMD: storage_delete CURSOR -** -** delete the entry under the cursor -*/ -static int test_storage_delete( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - KVCursor *p = 0; - int rc; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 2, objv, "CURSOR"); - return TCL_ERROR; - } - p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); - rc = sqlite4KVCursorDelete(p); - storageSetTclErrorName(interp, rc); - return TCL_OK; -} - -/* -** TCLCMD: storage_reset CURSOR -** -** Reset the cursor -*/ -static int test_storage_reset( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - KVCursor *p = 0; - int rc; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 2, objv, "CURSOR"); - return TCL_ERROR; - } - p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); - rc = sqlite4KVCursorReset(p); - storageSetTclErrorName(interp, rc); - return TCL_OK; -} - -/* -** TCLCMD: storage_key CURSOR -** -** Return the complete key of the cursor -*/ -static int test_storage_key( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - KVCursor *p = 0; - int rc; - const unsigned char *aKey; - int nKey; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 2, objv, "CURSOR"); - return TCL_ERROR; - } - p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); - rc = sqlite4KVCursorKey(p, &aKey, &nKey); - if( rc ){ - storageSetTclErrorName(interp, rc); - }else{ - unsigned char zBuf[500]; - memcpy(zBuf, aKey, nKey); - sqlite4TestBinToHex(zBuf, nKey); - Tcl_SetObjResult(interp, Tcl_NewStringObj((char*)zBuf, -1)); - } - return TCL_OK; -} - -/* -** TCLCMD: storage_data CURSOR -** -** Return the complete data of the cursor -*/ -static int test_storage_data( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - KVCursor *p = 0; - int rc; - const unsigned char *aData; - int nData; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 2, objv, "CURSOR"); - return TCL_ERROR; - } - p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); - rc = sqlite4KVCursorData(p, 0, 0, &aData, &nData); - if( rc ){ - storageSetTclErrorName(interp, rc); - }else{ - unsigned char zBuf[500]; - memcpy(zBuf, aData, nData); - sqlite4TestBinToHex(zBuf, nData); - Tcl_SetObjResult(interp, Tcl_NewStringObj((char*)zBuf, -1)); - } - return TCL_OK; -} - - - -/* -** Register the TCL commands defined above with the TCL interpreter. -** -** This routine should be the only externally visible symbol in this -** source code file. -*/ -int Sqliteteststorage_Init(Tcl_Interp *interp){ - struct SyscallCmd { - const char *zName; - Tcl_ObjCmdProc *xCmd; - } aCmd[] = { - { "storage_open", test_storage_open }, - { "storage_close", test_storage_close }, - { "storage_open_cursor", test_storage_open_cursor }, - { "storage_close_cursor", test_storage_close_cursor }, - { "storage_replace", test_storage_replace }, - { "storage_begin", test_storage_begin }, - { "storage_commit", test_storage_commit }, - { "storage_rollback", test_storage_rollback }, - { "storage_seek", test_storage_seek }, - { "storage_next", test_storage_next }, - { "storage_prev", test_storage_prev }, - { "storage_delete", test_storage_delete }, - { "storage_reset", test_storage_reset }, - { "storage_key", test_storage_key }, - { "storage_data", test_storage_data }, - }; - int i; - - for(i=0; ipReal->pStoreVfunc->xBegin(p->pReal, iLevel); - p->base.iTransLevel = p->pReal->iTransLevel; - return rc; -} - -static int kvwrapCommitPhaseOne(KVStore *pKVStore, int iLevel){ - int rc; - KVWrap *p = (KVWrap *)pKVStore; - rc = p->pReal->pStoreVfunc->xCommitPhaseOne(p->pReal, iLevel); - p->base.iTransLevel = p->pReal->iTransLevel; - return rc; -} - -static int kvwrapCommitPhaseTwo(KVStore *pKVStore, int iLevel){ - int rc; - KVWrap *p = (KVWrap *)pKVStore; - rc = p->pReal->pStoreVfunc->xCommitPhaseTwo(p->pReal, iLevel); - p->base.iTransLevel = p->pReal->iTransLevel; - return rc; -} - -static int kvwrapRollback(KVStore *pKVStore, int iLevel){ - int rc; - KVWrap *p = (KVWrap *)pKVStore; - rc = p->pReal->pStoreVfunc->xRollback(p->pReal, iLevel); - p->base.iTransLevel = p->pReal->iTransLevel; - return rc; -} - -static int kvwrapRevert(KVStore *pKVStore, int iLevel){ - int rc; - KVWrap *p = (KVWrap *)pKVStore; - rc = p->pReal->pStoreVfunc->xRevert(p->pReal, iLevel); - p->base.iTransLevel = p->pReal->iTransLevel; - return rc; -} - -static int kvwrapReplace( - KVStore *pKVStore, - const KVByteArray *aKey, KVSize nKey, - const KVByteArray *aData, KVSize nData -){ - KVWrap *p = (KVWrap *)pKVStore; - return p->pReal->pStoreVfunc->xReplace(p->pReal, aKey, nKey, aData, nData); -} - -/* -** Create a new cursor object. -*/ -static int kvwrapOpenCursor(KVStore *pKVStore, KVCursor **ppKVCursor){ - int rc = SQLITE4_OK; - KVWrap *p = (KVWrap *)pKVStore; - KVWrapCsr *pCsr; - - pCsr = (KVWrapCsr *)sqlite4_malloc(0, sizeof(KVWrapCsr)); - if( pCsr==0 ){ - rc = SQLITE4_NOMEM; - }else{ - memset(pCsr, 0, sizeof(KVWrapCsr)); - rc = p->pReal->pStoreVfunc->xOpenCursor(p->pReal, &pCsr->pReal); - if( rc!=SQLITE4_OK ){ - sqlite4_free(0, pCsr); - pCsr = 0; - }else{ - pCsr->base.pStore = pKVStore; - pCsr->base.pStoreVfunc = pKVStore->pStoreVfunc; - } - } - - *ppKVCursor = (KVCursor*)pCsr; - return rc; -} - -/* -** Reset a cursor -*/ -static int kvwrapReset(KVCursor *pKVCursor){ - KVWrap *p = (KVWrap *)(pKVCursor->pStore); - KVWrapCsr *pCsr = (KVWrapCsr *)pKVCursor; - return p->pReal->pStoreVfunc->xReset(pCsr->pReal); -} - -/* -** Destroy a cursor object -*/ -static int kvwrapCloseCursor(KVCursor *pKVCursor){ - int rc; - KVWrap *p = (KVWrap *)(pKVCursor->pStore); - KVWrapCsr *pCsr = (KVWrapCsr *)pKVCursor; - rc = p->pReal->pStoreVfunc->xCloseCursor(pCsr->pReal); - sqlite4_free(0, pCsr); - return rc; -} - -/* -** Move a cursor to the next non-deleted node. -*/ -static int kvwrapNextEntry(KVCursor *pKVCursor){ - int rc; - KVWrap *p = (KVWrap *)(pKVCursor->pStore); - KVWrapCsr *pCsr = (KVWrapCsr *)pKVCursor; - rc = p->pReal->pStoreVfunc->xNext(pCsr->pReal); - kvwg.nStep++; - return rc; -} - -/* -** Move a cursor to the previous non-deleted node. -*/ -static int kvwrapPrevEntry(KVCursor *pKVCursor){ - int rc; - KVWrap *p = (KVWrap *)(pKVCursor->pStore); - KVWrapCsr *pCsr = (KVWrapCsr *)pKVCursor; - rc = p->pReal->pStoreVfunc->xPrev(pCsr->pReal); - kvwg.nStep++; - return rc; -} - -/* -** Seek a cursor. -*/ -static int kvwrapSeek( - KVCursor *pKVCursor, - const KVByteArray *aKey, - KVSize nKey, - int dir -){ - KVWrap *p = (KVWrap *)(pKVCursor->pStore); - KVWrapCsr *pCsr = (KVWrapCsr *)pKVCursor; - - /* If aKey[0]==0, this is a seek to retrieve meta-data. Don't count this. */ - if( aKey[0] ) kvwg.nSeek++; - - return p->pReal->pStoreVfunc->xSeek(pCsr->pReal, aKey, nKey, dir); -} - -/* -** Delete the entry that the cursor is pointing to. -** -** Though the entry is "deleted", it still continues to exist as a -** phantom. Subsequent xNext or xPrev calls will work, as will -** calls to xKey and xData, thought the result from xKey and xData -** are undefined. -*/ -static int kvwrapDelete(KVCursor *pKVCursor){ - KVWrap *p = (KVWrap *)(pKVCursor->pStore); - KVWrapCsr *pCsr = (KVWrapCsr *)pKVCursor; - return p->pReal->pStoreVfunc->xDelete(pCsr->pReal); -} - -/* -** Return the key of the node the cursor is pointing to. -*/ -static int kvwrapKey( - KVCursor *pKVCursor, /* The cursor whose key is desired */ - const KVByteArray **paKey, /* Make this point to the key */ - KVSize *pN /* Make this point to the size of the key */ -){ - KVWrap *p = (KVWrap *)(pKVCursor->pStore); - KVWrapCsr *pCsr = (KVWrapCsr *)pKVCursor; - return p->pReal->pStoreVfunc->xKey(pCsr->pReal, paKey, pN); -} - -/* -** Return the data of the node the cursor is pointing to. -*/ -static int kvwrapData( - KVCursor *pKVCursor, /* The cursor from which to take the data */ - KVSize ofst, /* Offset into the data to begin reading */ - KVSize n, /* Number of bytes requested */ - const KVByteArray **paData, /* Pointer to the data written here */ - KVSize *pNData /* Number of bytes delivered */ -){ - KVWrap *p = (KVWrap *)(pKVCursor->pStore); - KVWrapCsr *pCsr = (KVWrapCsr *)pKVCursor; - return p->pReal->pStoreVfunc->xData(pCsr->pReal, ofst, n, paData, pNData); -} - -/* -** Destructor for the entire in-memory storage tree. -*/ -static int kvwrapClose(KVStore *pKVStore){ - int rc; - KVWrap *p = (KVWrap *)pKVStore; - rc = p->pReal->pStoreVfunc->xClose(p->pReal); - sqlite4_free(0, p); - return rc; -} - -/* -** Invoke the xControl() method of the underlying KVStore object. -*/ -static int kvwrapControl(KVStore *pKVStore, int op, void *pArg){ - KVWrap *p = (KVWrap *)pKVStore; - return p->pReal->pStoreVfunc->xControl(p->pReal, op, pArg); -} - -static int newFileStorage( - sqlite4_env *pEnv, - KVStore **ppKVStore, - const char *zName, - unsigned openFlags -){ - - /* Virtual methods for an LSM data store */ - static const KVStoreMethods kvwrapMethods = { - 1, - sizeof(KVStoreMethods), - kvwrapReplace, - kvwrapOpenCursor, - kvwrapSeek, - kvwrapNextEntry, - kvwrapPrevEntry, - kvwrapDelete, - kvwrapKey, - kvwrapData, - kvwrapReset, - kvwrapCloseCursor, - kvwrapBegin, - kvwrapCommitPhaseOne, - kvwrapCommitPhaseTwo, - kvwrapRollback, - kvwrapRevert, - kvwrapClose, - kvwrapControl - }; - - KVWrap *pNew; - int rc = SQLITE4_OK; - - pNew = (KVWrap *)sqlite4_malloc(0, sizeof(KVWrap)); - if( pNew==0 ){ - rc = SQLITE4_NOMEM; - }else{ - memset(pNew, 0, sizeof(KVWrap)); - pNew->base.pStoreVfunc = &kvwrapMethods; - rc = kvwg.xFactory(pEnv, &pNew->pReal, zName, openFlags); - if( rc!=SQLITE4_OK ){ - sqlite4_free(0, pNew); - pNew = 0; - } - } - - *ppKVStore = (KVStore*)pNew; - return rc; -} - -static int kvwrap_install_cmd(Tcl_Interp *interp, int objc, Tcl_Obj **objv){ - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 2, objv, ""); - return TCL_ERROR; - } - - if( kvwg.xFactory==0 ){ - sqlite4_env_config(0, SQLITE4_ENVCONFIG_KVSTORE_GET,"main", &kvwg.xFactory); - sqlite4_env_config(0, SQLITE4_ENVCONFIG_KVSTORE_PUSH,"main",newFileStorage); - } - return TCL_OK; -} - -static int kvwrap_uninstall_cmd(Tcl_Interp *interp, int objc, Tcl_Obj **objv){ - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 2, objv, ""); - return TCL_ERROR; - } - - if( kvwg.xFactory ){ - sqlite4_env_config(0, SQLITE4_ENVCONFIG_KVSTORE_POP,"main", &kvwg.xFactory); - kvwg.xFactory = 0; - } - return TCL_OK; -} - -static int kvwrap_seek_cmd(Tcl_Interp *interp, int objc, Tcl_Obj **objv){ - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 2, objv, ""); - return TCL_ERROR; - } - - Tcl_SetObjResult(interp, Tcl_NewIntObj(kvwg.nSeek)); - return TCL_OK; -} - -static int kvwrap_step_cmd(Tcl_Interp *interp, int objc, Tcl_Obj **objv){ - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 2, objv, ""); - return TCL_ERROR; - } - - Tcl_SetObjResult(interp, Tcl_NewIntObj(kvwg.nStep)); - return TCL_OK; -} - -static int kvwrap_reset_cmd(Tcl_Interp *interp, int objc, Tcl_Obj **objv){ - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 2, objv, ""); - return TCL_ERROR; - } - - kvwg.nStep = 0; - kvwg.nSeek = 0; - - Tcl_ResetResult(interp); - return TCL_OK; -} - - -/* -** TCLCMD: kvwrap SUB-COMMAND -*/ -static int kvwrap_command( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - struct Subcmd { - const char *zCmd; - int (*xCmd)(Tcl_Interp *, int, Tcl_Obj **); - } aSub[] = { - { "install", kvwrap_install_cmd }, - { "step", kvwrap_step_cmd }, - { "seek", kvwrap_seek_cmd }, - { "reset", kvwrap_reset_cmd }, - { "uninstall", kvwrap_uninstall_cmd }, - }; - int iSub; - int rc; - - rc = Tcl_GetIndexFromObjStruct( - interp, objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub - ); - if( rc==TCL_OK ){ - rc = aSub[iSub].xCmd(interp, objc, (Tcl_Obj **)objv); - } - - return rc; -} - -/* -** Register the TCL commands defined above with the TCL interpreter. -** -** This routine should be the only externally visible symbol in this -** source code file. -*/ -int Sqliteteststorage2_Init(Tcl_Interp *interp){ - Tcl_CreateObjCommand(interp, "kvwrap", kvwrap_command, 0, 0); - return TCL_OK; -} Index: test/test_lsm.c ================================================================== --- test/test_lsm.c +++ test/test_lsm.c @@ -7,163 +7,36 @@ ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* +** */ #include #include "lsm.h" #include "sqlite4.h" -#include -#include extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite4 **ppDb); extern const char *sqlite4TestErrorName(int); -/************************************************************************* -*/ -#define ENCRYPTION_XOR_MASK 0x23b2bbb6 -static int testCompressEncBound(void *pCtx, int nSrc){ - return nSrc; -} -static int testCompressEncCompress( - void *pCtx, - char *pOut, int *pnOut, - const char *pIn, int nIn -){ - int i; - unsigned int *aIn = (unsigned int *)pOut; - unsigned int *aOut = (unsigned int *)pIn; - - assert( (nIn%4)==0 ); - for(i=0; i<(nIn/4); i++){ - aOut[i] = (aIn[i] ^ ENCRYPTION_XOR_MASK); - } - *pnOut = nIn; - - return LSM_OK; -} -static int testCompressEncUncompress( - void *pCtx, - char *pOut, int *pnOut, - const char *pIn, int nIn -){ - return testCompressEncCompress(pCtx, pOut, pnOut, pIn, nIn); -} -static void testCompressEncFree(void *pCtx){ - /* no-op */ -} -/* -** End of compression routines "encrypt". -*************************************************************************/ - -/************************************************************************* -*/ -static int testCompressRleBound(void *pCtx, int nSrc){ - return nSrc*2; -} -static int testCompressRleCompress( - void *pCtx, - char *pOut, int *pnOut, - const char *pIn, int nIn -){ - int iOut = 0; - int i; - char c; - int n; - - c = pIn[0]; - n = 1; - for(i=1; icsr); - ckfree((char *)pCsr); - } -} - -static void test_lsm_del(void *ctx){ - TclLsm *p = (TclLsm *)ctx; - if( p ){ - lsm_close(p->db); - ckfree((char *)p); - } -} - -static int testInfoLsm(Tcl_Interp *interp, lsm_db *db, Tcl_Obj *pObj){ - struct Lsminfo { - const char *zOpt; - int eOpt; - } aInfo[] = { - { "compression_id", LSM_INFO_COMPRESSION_ID }, - { 0, 0 } - }; - int rc; - int iOpt; - - rc = Tcl_GetIndexFromObjStruct( - interp, pObj, aInfo, sizeof(aInfo[0]), "option", 0, &iOpt - ); - if( rc==LSM_OK ){ - switch( aInfo[iOpt].eOpt ){ - case LSM_INFO_COMPRESSION_ID: { - unsigned int iCmpId = 0; - rc = lsm_info(db, LSM_INFO_COMPRESSION_ID, &iCmpId); - if( rc==LSM_OK ){ - Tcl_SetObjResult(interp, Tcl_NewWideIntObj((Tcl_WideInt)iCmpId)); - }else{ - test_lsm_error(interp, "lsm_info", rc); - } - break; - } - } - } - - return rc; -} - -/* -** Usage: CSR sub-command ... -*/ -static int test_lsm_cursor_cmd( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - struct Subcmd { - const char *zCmd; - int nArg; - const char *zUsage; - } aCmd[] = { - /* 0 */ {"close", 0, ""}, - /* 1 */ {"seek", 2, "KEY SEEK-TYPE"}, - /* 2 */ {"first", 0, ""}, - /* 3 */ {"last", 0, ""}, - /* 4 */ {"next", 0, ""}, - /* 5 */ {"prev", 0, ""}, - /* 6 */ {"key", 0, ""}, - /* 7 */ {"value", 0, ""}, - /* 8 */ {"valid", 0, ""}, - {0, 0, 0} - }; - int iCmd; - int rc; - TclLsmCursor *pCsr = (TclLsmCursor *)clientData; - - rc = Tcl_GetIndexFromObjStruct( - interp, objv[1], aCmd, sizeof(aCmd[0]), "sub-command", 0, &iCmd - ); - if( rc!=TCL_OK ) return rc; - if( aCmd[iCmd].nArg>=0 && objc!=(2 + aCmd[iCmd].nArg) ){ - Tcl_WrongNumArgs(interp, 2, objv, aCmd[iCmd].zUsage); - return TCL_ERROR; - } - - switch( iCmd ){ - - case 0: assert( 0==strcmp(aCmd[0].zCmd, "close") ); { - Tcl_DeleteCommand(interp, Tcl_GetStringFromObj(objv[0], 0)); - return TCL_OK; - } - - case 1: assert( 0==strcmp(aCmd[1].zCmd, "seek") ); { - struct Seekbias { - const char *zBias; - int eBias; - } aBias[] = { - {"eq", LSM_SEEK_EQ}, - {"le", LSM_SEEK_LE}, - {"lefast", LSM_SEEK_LEFAST}, - {"ge", LSM_SEEK_GE}, - {0, 0} - }; - int iBias; - const char *zKey; int nKey; - zKey = Tcl_GetStringFromObj(objv[2], &nKey); - - rc = Tcl_GetIndexFromObjStruct( - interp, objv[3], aBias, sizeof(aBias[0]), "bias", 0, &iBias - ); - if( rc!=TCL_OK ) return rc; - - rc = lsm_csr_seek(pCsr->csr, zKey, nKey, aBias[iBias].eBias); - return test_lsm_error(interp, "lsm_seek", rc); - } - - case 2: - case 3: - case 4: - case 5: { - const char *zApi; - - assert( 0==strcmp(aCmd[2].zCmd, "first") ); - assert( 0==strcmp(aCmd[3].zCmd, "last") ); - assert( 0==strcmp(aCmd[4].zCmd, "next") ); - assert( 0==strcmp(aCmd[5].zCmd, "prev") ); - - switch( iCmd ){ - case 2: rc = lsm_csr_first(pCsr->csr); zApi = "lsm_csr_first"; break; - case 3: rc = lsm_csr_last(pCsr->csr); zApi = "lsm_csr_last"; break; - case 4: rc = lsm_csr_next(pCsr->csr); zApi = "lsm_csr_next"; break; - case 5: rc = lsm_csr_prev(pCsr->csr); zApi = "lsm_csr_prev"; break; - } - - return test_lsm_error(interp, zApi, rc); - } - - case 6: assert( 0==strcmp(aCmd[6].zCmd, "key") ); { - const void *pKey; int nKey; - rc = lsm_csr_key(pCsr->csr, &pKey, &nKey); - if( rc!=LSM_OK ) test_lsm_error(interp, "lsm_csr_key", rc); - - Tcl_SetObjResult(interp, Tcl_NewStringObj((const char *)pKey, nKey)); - return TCL_OK; - } - - case 7: assert( 0==strcmp(aCmd[7].zCmd, "value") ); { - const void *pVal; int nVal; - rc = lsm_csr_value(pCsr->csr, &pVal, &nVal); - if( rc!=LSM_OK ) test_lsm_error(interp, "lsm_csr_value", rc); - - Tcl_SetObjResult(interp, Tcl_NewStringObj((const char *)pVal, nVal)); - return TCL_OK; - } - - case 8: assert( 0==strcmp(aCmd[8].zCmd, "valid") ); { - int bValid = lsm_csr_valid(pCsr->csr); - Tcl_SetObjResult(interp, Tcl_NewBooleanObj(bValid)); - return TCL_OK; - } - } - - Tcl_AppendResult(interp, "internal error", 0); - return TCL_ERROR; -} - -/* -** Usage: DB sub-command ... -*/ -static int test_lsm_cmd( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - struct Subcmd { - const char *zCmd; - int nArg; - const char *zUsage; - } aCmd[] = { - /* 0 */ {"close", 0, ""}, - /* 1 */ {"write", 2, "KEY VALUE"}, - /* 2 */ {"delete", 1, "KEY"}, - /* 3 */ {"delete_range", 2, "START-KEY END-KEY"}, - /* 4 */ {"begin", 1, "LEVEL"}, - /* 5 */ {"commit", 1, "LEVEL"}, - /* 6 */ {"rollback", 1, "LEVEL"}, - /* 7 */ {"csr_open", 1, "CSR"}, - /* 8 */ {"work", -1, "?NMERGE? NPAGE"}, - /* 9 */ {"flush", 0, ""}, - /* 10 */ {"config", 1, "LIST"}, - /* 11 */ {"checkpoint", 0, ""}, - /* 12 */ {"info", 1, "OPTION"}, - {0, 0, 0} - }; - int iCmd; - int rc; - TclLsm *p = (TclLsm *)clientData; - - if( objc<2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND ..."); - return TCL_ERROR; - } - - rc = Tcl_GetIndexFromObjStruct( - interp, objv[1], aCmd, sizeof(aCmd[0]), "sub-command", 0, &iCmd - ); - if( rc!=TCL_OK ) return rc; - if( aCmd[iCmd].nArg>=0 && objc!=(2 + aCmd[iCmd].nArg) ){ - Tcl_WrongNumArgs(interp, 2, objv, aCmd[iCmd].zUsage); - return TCL_ERROR; - } - - switch( iCmd ){ - - case 0: assert( 0==strcmp(aCmd[0].zCmd, "close") ); { - Tcl_DeleteCommand(interp, Tcl_GetStringFromObj(objv[0], 0)); - return TCL_OK; - } - - case 1: assert( 0==strcmp(aCmd[1].zCmd, "write") ); { - const char *zKey; int nKey; - const char *zVal; int nVal; - - zKey = Tcl_GetStringFromObj(objv[2], &nKey); - zVal = Tcl_GetStringFromObj(objv[3], &nVal); - - rc = lsm_insert(p->db, zKey, nKey, zVal, nVal); - return test_lsm_error(interp, "lsm_insert", rc); - } - - case 2: assert( 0==strcmp(aCmd[2].zCmd, "delete") ); { - const char *zKey; int nKey; - - zKey = Tcl_GetStringFromObj(objv[2], &nKey); - - rc = lsm_delete(p->db, zKey, nKey); - return test_lsm_error(interp, "lsm_delete", rc); - } - - case 3: assert( 0==strcmp(aCmd[3].zCmd, "delete_range") ); { - const char *zKey1; int nKey1; - const char *zKey2; int nKey2; - - zKey1 = Tcl_GetStringFromObj(objv[2], &nKey1); - zKey2 = Tcl_GetStringFromObj(objv[3], &nKey2); - - rc = lsm_delete_range(p->db, zKey1, nKey1, zKey2, nKey2); - return test_lsm_error(interp, "lsm_delete_range", rc); - } - - case 4: - case 5: - case 6: { - const char *zApi; - int iLevel; - - rc = Tcl_GetIntFromObj(interp, objv[2], &iLevel); - if( rc!=TCL_OK ) return rc; - - assert( 0==strcmp(aCmd[4].zCmd, "begin") ); - assert( 0==strcmp(aCmd[5].zCmd, "commit") ); - assert( 0==strcmp(aCmd[6].zCmd, "rollback") ); - switch( iCmd ){ - case 4: rc = lsm_begin(p->db, iLevel); zApi = "lsm_begin"; break; - case 5: rc = lsm_commit(p->db, iLevel); zApi = "lsm_commit"; break; - case 6: rc = lsm_rollback(p->db, iLevel); zApi = "lsm_rollback"; break; - } - - return test_lsm_error(interp, zApi, rc); - } - - case 7: assert( 0==strcmp(aCmd[7].zCmd, "csr_open") ); { - const char *zCsr = Tcl_GetString(objv[2]); - TclLsmCursor *pCsr; - - pCsr = (TclLsmCursor *)ckalloc(sizeof(TclLsmCursor)); - rc = lsm_csr_open(p->db, &pCsr->csr); - if( rc!=LSM_OK ){ - test_lsm_cursor_del(pCsr); - return test_lsm_error(interp, "lsm_csr_open", rc); - } - - Tcl_CreateObjCommand( - interp, zCsr, test_lsm_cursor_cmd, - (ClientData)pCsr, test_lsm_cursor_del - ); - Tcl_SetObjResult(interp, objv[2]); - return TCL_OK; - } - - case 8: assert( 0==strcmp(aCmd[8].zCmd, "work") ); { - int nWork = 0; - int nMerge = 1; - int nWrite = 0; - - if( objc==3 ){ - rc = Tcl_GetIntFromObj(interp, objv[2], &nWork); - }else if( objc==4 ){ - rc = Tcl_GetIntFromObj(interp, objv[2], &nMerge); - if( rc!=TCL_OK ) return rc; - rc = Tcl_GetIntFromObj(interp, objv[3], &nWork); - }else{ - Tcl_WrongNumArgs(interp, 2, objv, "?NMERGE? NWRITE"); - return TCL_ERROR; - } - if( rc!=TCL_OK ) return rc; - - rc = lsm_work(p->db, nMerge, nWork, &nWrite); - if( rc!=LSM_OK ) return test_lsm_error(interp, "lsm_work", rc); - Tcl_SetObjResult(interp, Tcl_NewIntObj(nWrite)); - return TCL_OK; - } - - case 9: assert( 0==strcmp(aCmd[9].zCmd, "flush") ); { - rc = lsm_flush(p->db); - return test_lsm_error(interp, "lsm_flush", rc); - } - - case 10: assert( 0==strcmp(aCmd[10].zCmd, "config") ); { - return testConfigureLsm(interp, p->db, objv[2]); - } - - case 11: assert( 0==strcmp(aCmd[11].zCmd, "checkpoint") ); { - rc = lsm_checkpoint(p->db, 0); - return test_lsm_error(interp, "lsm_checkpoint", rc); - } - - case 12: assert( 0==strcmp(aCmd[12].zCmd, "info") ); { - return testInfoLsm(interp, p->db, objv[2]); - } - - default: - assert( 0 ); - } - - Tcl_AppendResult(interp, "internal error", 0); - return TCL_ERROR; -} - -static void xLog(void *pCtx, int rc, const char *z){ - (void)(rc); - (void)(pCtx); - fprintf(stderr, "%s\n", z); - fflush(stderr); -} - -/* -** Usage: lsm_open DB filename ?config? -*/ -static int test_lsm_open( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - TclLsm *p; - int rc; - const char *zDb = 0; - const char *zFile = 0; - - if( objc!=3 && objc!=4 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB FILENAME ?CONFIG?"); - return TCL_ERROR; - } - - zDb = Tcl_GetString(objv[1]); - zFile = Tcl_GetString(objv[2]); - - p = (TclLsm *)ckalloc(sizeof(TclLsm)); - rc = lsm_new(0, &p->db); - if( rc!=LSM_OK ){ - test_lsm_del((void *)p); - test_lsm_error(interp, "lsm_new", rc); - return TCL_ERROR; - } - - if( objc==4 ){ - rc = testConfigureLsm(interp, p->db, objv[3]); - if( rc!=TCL_OK ){ - test_lsm_del((void *)p); - return rc; - } - } - - lsm_config_log(p->db, xLog, 0); - - rc = lsm_open(p->db, zFile); - if( rc!=LSM_OK ){ - test_lsm_del((void *)p); - test_lsm_error(interp, "lsm_open", rc); - return TCL_ERROR; - } - - Tcl_CreateObjCommand(interp, zDb, test_lsm_cmd, (ClientData)p, test_lsm_del); - Tcl_SetObjResult(interp, objv[1]); - return TCL_OK; -} - int SqlitetestLsm_Init(Tcl_Interp *interp){ struct SyscallCmd { const char *zName; Tcl_ObjCmdProc *xCmd; } aCmd[] = { - { "sqlite4_lsm_work", test_sqlite4_lsm_work }, - { "sqlite4_lsm_checkpoint", test_sqlite4_lsm_checkpoint }, - { "sqlite4_lsm_flush", test_sqlite4_lsm_flush }, - { "sqlite4_lsm_info", test_sqlite4_lsm_info }, - { "sqlite4_lsm_config", test_sqlite4_lsm_config }, - { "lsm_open", test_lsm_open }, + { "sqlite4_lsm_work", test_lsm_work }, + { "sqlite4_lsm_checkpoint", test_lsm_checkpoint }, + { "sqlite4_lsm_flush", test_lsm_flush }, + { "sqlite4_lsm_info", test_lsm_info }, + { "sqlite4_lsm_config", test_lsm_config }, }; int i; for(i=0; imutex); pVal = sqlite4ValueNew(db); - sqlite4ValueSetStr(pVal, -1, "x_sqlite_exec", SQLITE4_UTF8, - SQLITE4_STATIC, 0); + sqlite4ValueSetStr(pVal, -1, "x_sqlite_exec", SQLITE4_UTF8, SQLITE4_STATIC); zUtf16 = sqlite4ValueText(pVal, SQLITE4_UTF16NATIVE); if( db->mallocFailed ){ rc = SQLITE4_NOMEM; }else{ rc = sqlite4_create_function16(db, zUtf16, @@ -1707,22 +1706,22 @@ */ static void testFunc(sqlite4_context *context, int argc, sqlite4_value **argv){ while( argc>=2 ){ const char *zArg0 = (char*)sqlite4_value_text(argv[0]); if( zArg0 ){ - if( 0==sqlite4_stricmp(zArg0, "int") ){ + if( 0==sqlite4StrICmp(zArg0, "int") ){ sqlite4_result_int(context, sqlite4_value_int(argv[1])); - }else if( sqlite4_stricmp(zArg0,"int64")==0 ){ + }else if( sqlite4StrICmp(zArg0,"int64")==0 ){ sqlite4_result_int64(context, sqlite4_value_int64(argv[1])); - }else if( sqlite4_stricmp(zArg0,"string")==0 ){ + }else if( sqlite4StrICmp(zArg0,"string")==0 ){ sqlite4_result_text(context, (char*)sqlite4_value_text(argv[1]), -1, - SQLITE4_TRANSIENT, 0); - }else if( sqlite4_stricmp(zArg0,"double")==0 ){ + SQLITE4_TRANSIENT); + }else if( sqlite4StrICmp(zArg0,"double")==0 ){ sqlite4_result_double(context, sqlite4_value_double(argv[1])); - }else if( sqlite4_stricmp(zArg0,"null")==0 ){ + }else if( sqlite4StrICmp(zArg0,"null")==0 ){ sqlite4_result_null(context); - }else if( sqlite4_stricmp(zArg0,"value")==0 ){ + }else if( sqlite4StrICmp(zArg0,"value")==0 ){ sqlite4_result_value(context, argv[sqlite4_value_int(argv[1])]); }else{ goto error_out; } }else{ @@ -2099,18 +2098,18 @@ if( getStmtPointer(interp, argv[1], &pStmt) ) return TCL_ERROR; if( Tcl_GetInt(interp, argv[2], &idx) ) return TCL_ERROR; if( strcmp(argv[4],"null")==0 ){ rc = sqlite4_bind_null(pStmt, idx); }else if( strcmp(argv[4],"static")==0 ){ - rc = sqlite4_bind_text(pStmt, idx, sqlite_static_bind_value, -1, 0, 0); + rc = sqlite4_bind_text(pStmt, idx, sqlite_static_bind_value, -1, 0); }else if( strcmp(argv[4],"static-nbytes")==0 ){ rc = sqlite4_bind_text(pStmt, idx, sqlite_static_bind_value, - sqlite_static_bind_nbyte, 0, 0); + sqlite_static_bind_nbyte, 0); }else if( strcmp(argv[4],"normal")==0 ){ - rc = sqlite4_bind_text(pStmt, idx, argv[3], -1, SQLITE4_TRANSIENT, 0); + rc = sqlite4_bind_text(pStmt, idx, argv[3], -1, SQLITE4_TRANSIENT); }else if( strcmp(argv[4],"blob10")==0 ){ - rc = sqlite4_bind_text(pStmt, idx, "abc\000xyz\000pq", 10,SQLITE4_STATIC,0); + rc = sqlite4_bind_text(pStmt, idx, "abc\000xyz\000pq", 10, SQLITE4_STATIC); }else{ Tcl_AppendResult(interp, "4th argument should be " "\"null\" or \"static\" or \"normal\"", 0); return TCL_ERROR; } @@ -2186,15 +2185,15 @@ } sqlite4BeginBenignMalloc(pEnv); pVal = sqlite4ValueNew(0); if( pVal ){ - sqlite4ValueSetStr(pVal, nA, zA, encin, SQLITE4_STATIC, 0); + sqlite4ValueSetStr(pVal, nA, zA, encin, SQLITE4_STATIC); n = sqlite4_value_bytes(pVal); Tcl_ListObjAppendElement(i,pX, Tcl_NewStringObj((char*)sqlite4_value_text(pVal),n)); - sqlite4ValueSetStr(pVal, nB, zB, encin, SQLITE4_STATIC, 0); + sqlite4ValueSetStr(pVal, nB, zB, encin, SQLITE4_STATIC); n = sqlite4_value_bytes(pVal); Tcl_ListObjAppendElement(i,pX, Tcl_NewStringObj((char*)sqlite4_value_text(pVal),n)); sqlite4ValueFree(pVal); } @@ -2388,17 +2387,16 @@ Tcl_ListObjAppendElement(interp, pX, Tcl_NewStringObj("UTF-8", -1)); Tcl_ListObjAppendElement(interp, pX, Tcl_NewStringObj((char*)sqlite4_value_text(argv[0]), -1)); Tcl_EvalObjEx(interp, pX, 0); Tcl_DecrRefCount(pX); - sqlite4_result_text(pCtx, Tcl_GetStringResult(interp), -1, - SQLITE4_TRANSIENT, 0); + sqlite4_result_text(pCtx, Tcl_GetStringResult(interp), -1, SQLITE4_TRANSIENT); pVal = sqlite4ValueNew(0); sqlite4ValueSetStr(pVal, -1, Tcl_GetStringResult(interp), - SQLITE4_UTF8, SQLITE4_STATIC, 0); + SQLITE4_UTF8, SQLITE4_STATIC); sqlite4_result_text16be(pCtx, sqlite4_value_text16be(pVal), - -1, SQLITE4_TRANSIENT, 0); + -1, SQLITE4_TRANSIENT); sqlite4ValueFree(pVal); } static void test_function_utf16le( sqlite4_context *pCtx, int nArg, @@ -2415,13 +2413,12 @@ Tcl_NewStringObj((char*)sqlite4_value_text(argv[0]), -1)); Tcl_EvalObjEx(interp, pX, 0); Tcl_DecrRefCount(pX); pVal = sqlite4ValueNew(0); sqlite4ValueSetStr(pVal, -1, Tcl_GetStringResult(interp), - SQLITE4_UTF8, SQLITE4_STATIC, 0); - sqlite4_result_text(pCtx, (char*)sqlite4_value_text(pVal), -1, - SQLITE4_TRANSIENT, 0); + SQLITE4_UTF8, SQLITE4_STATIC); + sqlite4_result_text(pCtx,(char*)sqlite4_value_text(pVal),-1,SQLITE4_TRANSIENT); sqlite4ValueFree(pVal); } static void test_function_utf16be( sqlite4_context *pCtx, int nArg, @@ -2438,17 +2435,17 @@ Tcl_NewStringObj((char*)sqlite4_value_text(argv[0]), -1)); Tcl_EvalObjEx(interp, pX, 0); Tcl_DecrRefCount(pX); pVal = sqlite4ValueNew(0); sqlite4ValueSetStr(pVal, -1, Tcl_GetStringResult(interp), - SQLITE4_UTF8, SQLITE4_STATIC, 0); + SQLITE4_UTF8, SQLITE4_STATIC); sqlite4_result_text16(pCtx, sqlite4_value_text16le(pVal), - -1, SQLITE4_TRANSIENT, 0); + -1, SQLITE4_TRANSIENT); sqlite4_result_text16be(pCtx, sqlite4_value_text16le(pVal), - -1, SQLITE4_TRANSIENT, 0); + -1, SQLITE4_TRANSIENT); sqlite4_result_text16le(pCtx, sqlite4_value_text16le(pVal), - -1, SQLITE4_TRANSIENT, 0); + -1, SQLITE4_TRANSIENT); sqlite4ValueFree(pVal); } #endif /* SQLITE4_OMIT_UTF16 */ static int test_function( void * clientData, @@ -2789,11 +2786,11 @@ if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; if( Tcl_GetIntFromObj(interp, objv[2], &idx) ) return TCL_ERROR; value = (char*)Tcl_GetByteArrayFromObj(objv[3], &bytes); if( Tcl_GetIntFromObj(interp, objv[4], &bytes) ) return TCL_ERROR; - rc = sqlite4_bind_text(pStmt, idx, value, bytes, SQLITE4_TRANSIENT, 0); + rc = sqlite4_bind_text(pStmt, idx, value, bytes, SQLITE4_TRANSIENT); if( sqlite4TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; if( rc!=SQLITE4_OK ){ Tcl_AppendResult(interp, sqlite4TestErrorName(rc), 0); return TCL_ERROR; } @@ -2837,11 +2834,11 @@ if( getStmtPointer(interp, Tcl_GetString(oStmt), &pStmt) ) return TCL_ERROR; if( Tcl_GetIntFromObj(interp, oN, &idx) ) return TCL_ERROR; value = (char*)Tcl_GetByteArrayFromObj(oString, 0); if( Tcl_GetIntFromObj(interp, oBytes, &bytes) ) return TCL_ERROR; - rc = sqlite4_bind_text16(pStmt, idx, (void *)value, bytes, xDel, 0); + rc = sqlite4_bind_text16(pStmt, idx, (void *)value, bytes, xDel); if( sqlite4TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; if( rc!=SQLITE4_OK ){ Tcl_AppendResult(interp, sqlite4TestErrorName(rc), 0); return TCL_ERROR; } @@ -2884,11 +2881,11 @@ if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; if( Tcl_GetIntFromObj(interp, objv[2], &idx) ) return TCL_ERROR; value = Tcl_GetString(objv[3]); if( Tcl_GetIntFromObj(interp, objv[4], &bytes) ) return TCL_ERROR; - rc = sqlite4_bind_blob(pStmt, idx, value, bytes, xDestructor, 0); + rc = sqlite4_bind_blob(pStmt, idx, value, bytes, xDestructor); if( sqlite4TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; if( rc!=SQLITE4_OK ){ return TCL_ERROR; } @@ -4351,209 +4348,10 @@ } sqlite4_test_control(SQLITE4_TESTCTRL_OPTIMIZATIONS, db, mask); return TCL_OK; } -#define NUM_FORMAT "sign:%d approx:%d e:%d m:%lld" - -/* Append a return value representing a sqlite4_num. -*/ -static void append_num_result( Tcl_Interp *interp, sqlite4_num A ){ - char buf[100]; - sprintf( buf, NUM_FORMAT, A.sign, A.approx, A.e, A.m ); - Tcl_AppendResult(interp, buf, 0); -} - -/* Convert a string either representing a sqlite4_num (listing its fields as -** returned by append_num_result) or that can be parsed as one. Invalid -** strings become NaN. -*/ -static sqlite4_num test_parse_num( char *arg ){ - sqlite4_num A; - int sign, approx, e; - if( sscanf( arg, NUM_FORMAT, &sign, &approx, &e, &A.m)==4 ){ - A.sign = sign; - A.approx = approx; - A.e = e; - return A; - } else { - return sqlite4_num_from_text(arg, -1, 0); - } -} - -/* Convert return values of sqlite4_num to strings that will be readable in -** the tests. -*/ -static char *describe_num_comparison( int code ){ - switch( code ){ - case 0: return "incomparable"; - case 1: return "lesser"; - case 2: return "equal"; - case 3: return "greater"; - default: return "error"; - } -} - -/* Compare two numbers A and B. Returns "incomparable", "lesser", "equal", -** "greater", or "error". -*/ -static int test_num_compare( - void *NotUsed, - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int argc, /* Number of arguments */ - char **argv /* Text of each argument */ -){ - sqlite4_num A, B; - int cmp; - if( argc!=3 ){ - Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " NUM NUM\"", 0); - return TCL_ERROR; - } - - A = test_parse_num( argv[1] ); - B = test_parse_num( argv[2] ); - cmp = sqlite4_num_compare(A, B); - Tcl_AppendResult( interp, describe_num_comparison( cmp ), 0); - return TCL_OK; -} - -/* Create a sqlite4_num from a string. The optional second argument specifies -** how many bytes may be read. -*/ -static int test_num_from_text( - void *NotUsed, - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int argc, /* Number of arguments */ - char **argv /* Text of each argument */ -){ - sqlite4_num A; - int len; - if( argc!=2 && argc!=3 ){ - Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " STRING\" or \"", argv[0], " STRING INTEGER\"", 0); - return TCL_ERROR; - } - - if( argc==3 ){ - if ( Tcl_GetInt(interp, argv[2], &len) ) return TCL_ERROR; - }else{ - len = -1; - } - - A = sqlite4_num_from_text( argv[1], len, 0 ); - append_num_result(interp, A); - return TCL_OK; -} - -static int test_num_to_text( - void *NotUsed, - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int argc, /* Number of arguments */ - char **argv /* Text of each argument */ -){ - char text[30]; - if( argc!=2 ){ - Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " NUM\"", 0); - return TCL_ERROR; - } - sqlite4_num_to_text( test_parse_num( argv[1] ), text ); - Tcl_AppendResult( interp, text, 0 ); - return TCL_OK; -} - -static int test_num_binary_op( - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int argc, /* Number of arguments */ - char **argv, /* Text of each argument */ - sqlite4_num (*op) (sqlite4_num, sqlite4_num) -){ - sqlite4_num A, B, R; - if( argc!=3 ){ - Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " NUM NUM\"", 0); - return TCL_ERROR; - } - A = test_parse_num(argv[1]); - B = test_parse_num(argv[2]); - R = op(A, B); - append_num_result(interp, R); - return TCL_OK; -} - -static int test_num_add( - void *NotUsed, - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int argc, /* Number of arguments */ - char **argv /* Text of each argument */ -){ - return test_num_binary_op( interp, argc, argv, sqlite4_num_add ); -} - -static int test_num_sub( - void *NotUsed, - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int argc, /* Number of arguments */ - char **argv /* Text of each argument */ -){ - return test_num_binary_op( interp, argc, argv, sqlite4_num_sub ); -} - -static int test_num_mul( - void *NotUsed, - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int argc, /* Number of arguments */ - char **argv /* Text of each argument */ -){ - return test_num_binary_op( interp, argc, argv, sqlite4_num_mul ); -} - -static int test_num_div( - void *NotUsed, - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int argc, /* Number of arguments */ - char **argv /* Text of each argument */ -){ - return test_num_binary_op( interp, argc, argv, sqlite4_num_div ); -} - -static int test_num_predicate( - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int argc, /* Number of arguments */ - char **argv, /* Text of each argument */ - int (*pred) (sqlite4_num) -){ - sqlite4_num A; - if( argc!=2 ){ - Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " NUM\"", 0); - return TCL_ERROR; - } - A = test_parse_num(argv[1]); - Tcl_AppendResult(interp, pred(A) ? "true" : "false", 0); - return TCL_OK; -} - -static int test_num_isinf( - void *NotUsed, - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int argc, /* Number of arguments */ - char **argv /* Text of each argument */ -){ - return test_num_predicate( interp, argc, argv, sqlite4_num_isinf ); -} - -static int test_num_isnan( - void *NotUsed, - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int argc, /* Number of arguments */ - char **argv /* Text of each argument */ -){ - return test_num_predicate( interp, argc, argv, sqlite4_num_isnan ); -} - /* ** Register commands with the TCL interpreter. */ int Sqlitetest1_Init(Tcl_Interp *interp){ extern int sqlite4_search_count; @@ -4602,20 +4400,11 @@ { "sqlite_delete_function", (Tcl_CmdProc*)delete_function }, { "sqlite_delete_collation", (Tcl_CmdProc*)delete_collation }, { "sqlite4_get_autocommit", (Tcl_CmdProc*)get_autocommit }, { "sqlite4_stack_used", (Tcl_CmdProc*)test_stack_used }, { "printf", (Tcl_CmdProc*)test_printf }, - { "sqlite4IoTrace", (Tcl_CmdProc*)test_io_trace }, - { "sqlite4_num_compare", (Tcl_CmdProc*)test_num_compare }, - { "sqlite4_num_from_text", (Tcl_CmdProc*)test_num_from_text }, - { "sqlite4_num_to_text", (Tcl_CmdProc*)test_num_to_text }, - { "sqlite4_num_add", (Tcl_CmdProc*)test_num_add }, - { "sqlite4_num_sub", (Tcl_CmdProc*)test_num_sub }, - { "sqlite4_num_mul", (Tcl_CmdProc*)test_num_mul }, - { "sqlite4_num_div", (Tcl_CmdProc*)test_num_div }, - { "sqlite4_num_isinf", (Tcl_CmdProc*)test_num_isinf }, - { "sqlite4_num_isnan", (Tcl_CmdProc*)test_num_isnan }, + { "sqlite4IoTrace", (Tcl_CmdProc*)test_io_trace }, }; static struct { char *zName; Tcl_ObjCmdProc *xProc; void *clientData; Index: test/test_malloc.c ================================================================== --- test/test_malloc.c +++ test/test_malloc.c @@ -46,11 +46,11 @@ /* ** Check to see if a fault should be simulated. Return true to simulate ** the fault. Return false if the fault should not be simulated. */ static int faultsimStep(void){ - if( !memfault.enable ){ + if( likely(!memfault.enable) ){ return 0; } if( memfault.iCountdown>0 ){ memfault.iCountdown--; return 0; ADDED test/test_storage.c Index: test/test_storage.c ================================================================== --- /dev/null +++ test/test_storage.c @@ -0,0 +1,495 @@ +/* +** 2011 January 21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Code for testing the storage subsystem using the Storage interface. +*/ +#include "sqliteInt.h" + +/* Defined in test1.c */ +extern void *sqlite4TestTextToPtr(const char*); + +/* Defined in test_hexio.c */ +extern void sqlite4TestBinToHex(unsigned char*,int); +extern int sqlite4TestHexToBin(const unsigned char *in,int,unsigned char *out); + +/* Set the TCL result to an integer. +*/ +static void storageSetTclErrorName(Tcl_Interp *interp, int rc){ + extern const char *sqlite4TestErrorName(int); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite4TestErrorName(rc), -1)); +} + +/* +** TCLCMD: storage_open URI ?FLAGS? +** +** Return a string that identifies the new storage object. +*/ +static int test_storage_open( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + KVStore *pNew = 0; + int rc; + int flags = 0; + sqlite4 db; + char zRes[50]; + if( objc!=2 && objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "URI ?FLAGS?"); + return TCL_ERROR; + } + if( objc==3 && Tcl_GetIntFromObj(interp, objv[2], &flags) ){ + return TCL_ERROR; + } + memset(&db, 0, sizeof(db)); + rc = sqlite4KVStoreOpen(&db, "test", Tcl_GetString(objv[1]), &pNew, flags); + if( rc ){ + sqlite4KVStoreClose(pNew); + storageSetTclErrorName(interp, rc); + return TCL_ERROR; + } + sqlite4_snprintf(zRes,sizeof(zRes), "%p", pNew); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zRes,-1)); + return TCL_OK; +} + +/* +** TCLCMD: storage_close STORAGE +** +** Close a storage object. +*/ +static int test_storage_close( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + KVStore *pOld = 0; + int rc; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 2, objv, "STORAGE"); + return TCL_ERROR; + } + pOld = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); + rc = sqlite4KVStoreClose(pOld); + if( rc ){ + storageSetTclErrorName(interp, rc); + return TCL_ERROR; + } + return TCL_OK; +} + +/* +** TCLCMD: storage_open_cursor STORAGE +** +** Return a string that identifies the new storage cursor +*/ +static int test_storage_open_cursor( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + KVCursor *pNew = 0; + KVStore *pStore = 0; + int rc; + char zRes[50]; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 2, objv, "STORAGE"); + return TCL_ERROR; + } + pStore = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); + rc = sqlite4KVStoreOpenCursor(pStore, &pNew); + if( rc ){ + sqlite4KVCursorClose(pNew); + storageSetTclErrorName(interp, rc); + return TCL_ERROR; + } + sqlite4_snprintf(zRes,sizeof(zRes), "%p", pNew); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zRes,-1)); + return TCL_OK; +} + +/* +** TCLCMD: storage_close_cursor CURSOR +** +** Close a cursor object. +*/ +static int test_storage_close_cursor( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + KVCursor *pOld = 0; + int rc; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 2, objv, "CURSOR"); + return TCL_ERROR; + } + pOld = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); + rc = sqlite4KVCursorClose(pOld); + if( rc ){ + storageSetTclErrorName(interp, rc); + return TCL_ERROR; + } + return TCL_OK; +} + +/* Decode hex into a binary key */ +static void sqlite4DecodeHex(Tcl_Obj *pObj, unsigned char *a, int *pN){ + const unsigned char *pIn; + int nIn; + pIn = (const unsigned char*)Tcl_GetStringFromObj(pObj, &nIn); + *pN = sqlite4TestHexToBin(pIn, nIn, a); +} + +/* +** TCLCMD: storage_replace STORAGE KEY VALUE +** +** Insert content into a KV storage object. KEY and VALUE are hex. +*/ +static int test_storage_replace( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + KVStore *p = 0; + int rc; + int nKey, nData; + unsigned char zKey[200]; + unsigned char zData[200]; + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 2, objv, "STORAGE KEY VALUE"); + return TCL_ERROR; + } + p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); + sqlite4DecodeHex(objv[2], zKey, &nKey); + sqlite4DecodeHex(objv[3], zData, &nData); + rc = sqlite4KVStoreReplace(p, zKey, nKey, zData, nData); + if( rc ){ + storageSetTclErrorName(interp, rc); + return TCL_ERROR; + } + return TCL_OK; +} + +/* +** TCLCMD: storage_begin STORAGE LEVEL +** +** Increase the transaction level +*/ +static int test_storage_begin( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + KVStore *p = 0; + int rc; + int iLevel; + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 2, objv, "STORAGE LEVEL"); + return TCL_ERROR; + } + p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); + if( Tcl_GetIntFromObj(interp, objv[2], &iLevel) ) return TCL_ERROR; + rc = sqlite4KVStoreBegin(p, iLevel); + if( rc ){ + storageSetTclErrorName(interp, rc); + return TCL_ERROR; + } + return TCL_OK; +} + +/* +** TCLCMD: storage_commit STORAGE LEVEL +** +** Increase the transaction level +*/ +static int test_storage_commit( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + KVStore *p = 0; + int rc; + int iLevel; + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 2, objv, "STORAGE LEVEL"); + return TCL_ERROR; + } + p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); + if( Tcl_GetIntFromObj(interp, objv[2], &iLevel) ) return TCL_ERROR; + rc = sqlite4KVStoreCommitPhaseOne(p, iLevel); + if( rc ){ + storageSetTclErrorName(interp, rc); + return TCL_ERROR; + } + rc = sqlite4KVStoreCommitPhaseTwo(p, iLevel); + if( rc ){ + storageSetTclErrorName(interp, rc); + return TCL_ERROR; + } + return TCL_OK; +} + +/* +** TCLCMD: storage_rollback STORAGE LEVEL +** +** Increase the transaction level +*/ +static int test_storage_rollback( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + KVStore *p = 0; + int rc; + int iLevel; + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 2, objv, "STORAGE LEVEL"); + return TCL_ERROR; + } + p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); + if( Tcl_GetIntFromObj(interp, objv[2], &iLevel) ) return TCL_ERROR; + rc = sqlite4KVStoreRollback(p, iLevel); + if( rc ){ + storageSetTclErrorName(interp, rc); + return TCL_ERROR; + } + return TCL_OK; +} + +/* +** TCLCMD: storage_seek CURSOR KEY DIRECTION +** +** Move a cursor object +*/ +static int test_storage_seek( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + KVCursor *p = 0; + int rc; + int nKey, dir; + unsigned char aKey[100]; + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 4, objv, "CURSOR KEY DIRECTION"); + return TCL_ERROR; + } + p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); + sqlite4DecodeHex(objv[2], aKey, &nKey); + if( Tcl_GetIntFromObj(interp, objv[3], &dir) ) return TCL_ERROR; + rc = sqlite4KVCursorSeek(p, aKey, nKey, dir); + storageSetTclErrorName(interp, rc); + return TCL_OK; +} + +/* +** TCLCMD: storage_next CURSOR +** +** Move the cursor to the next entry +*/ +static int test_storage_next( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + KVCursor *p = 0; + int rc; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 2, objv, "CURSOR"); + return TCL_ERROR; + } + p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); + rc = sqlite4KVCursorNext(p); + storageSetTclErrorName(interp, rc); + return TCL_OK; +} + +/* +** TCLCMD: storage_prev CURSOR +** +** Move the cursor to the previous entry +*/ +static int test_storage_prev( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + KVCursor *p = 0; + int rc; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 2, objv, "CURSOR"); + return TCL_ERROR; + } + p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); + rc = sqlite4KVCursorPrev(p); + storageSetTclErrorName(interp, rc); + return TCL_OK; +} + +/* +** TCLCMD: storage_delete CURSOR +** +** delete the entry under the cursor +*/ +static int test_storage_delete( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + KVCursor *p = 0; + int rc; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 2, objv, "CURSOR"); + return TCL_ERROR; + } + p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); + rc = sqlite4KVCursorDelete(p); + storageSetTclErrorName(interp, rc); + return TCL_OK; +} + +/* +** TCLCMD: storage_reset CURSOR +** +** Reset the cursor +*/ +static int test_storage_reset( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + KVCursor *p = 0; + int rc; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 2, objv, "CURSOR"); + return TCL_ERROR; + } + p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); + rc = sqlite4KVCursorReset(p); + storageSetTclErrorName(interp, rc); + return TCL_OK; +} + +/* +** TCLCMD: storage_key CURSOR +** +** Return the complete key of the cursor +*/ +static int test_storage_key( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + KVCursor *p = 0; + int rc; + const unsigned char *aKey; + int nKey; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 2, objv, "CURSOR"); + return TCL_ERROR; + } + p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); + rc = sqlite4KVCursorKey(p, &aKey, &nKey); + if( rc ){ + storageSetTclErrorName(interp, rc); + }else{ + unsigned char zBuf[500]; + memcpy(zBuf, aKey, nKey); + sqlite4TestBinToHex(zBuf, nKey); + Tcl_SetObjResult(interp, Tcl_NewStringObj((char*)zBuf, -1)); + } + return TCL_OK; +} + +/* +** TCLCMD: storage_data CURSOR +** +** Return the complete data of the cursor +*/ +static int test_storage_data( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + KVCursor *p = 0; + int rc; + const unsigned char *aData; + int nData; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 2, objv, "CURSOR"); + return TCL_ERROR; + } + p = sqlite4TestTextToPtr(Tcl_GetString(objv[1])); + rc = sqlite4KVCursorData(p, 0, 0, &aData, &nData); + if( rc ){ + storageSetTclErrorName(interp, rc); + }else{ + unsigned char zBuf[500]; + memcpy(zBuf, aData, nData); + sqlite4TestBinToHex(zBuf, nData); + Tcl_SetObjResult(interp, Tcl_NewStringObj((char*)zBuf, -1)); + } + return TCL_OK; +} + + + +/* +** Register the TCL commands defined above with the TCL interpreter. +** +** This routine should be the only externally visible symbol in this +** source code file. +*/ +int Sqliteteststorage_Init(Tcl_Interp *interp){ + struct SyscallCmd { + const char *zName; + Tcl_ObjCmdProc *xCmd; + } aCmd[] = { + { "storage_open", test_storage_open }, + { "storage_close", test_storage_close }, + { "storage_open_cursor", test_storage_open_cursor }, + { "storage_close_cursor", test_storage_close_cursor }, + { "storage_replace", test_storage_replace }, + { "storage_begin", test_storage_begin }, + { "storage_commit", test_storage_commit }, + { "storage_rollback", test_storage_rollback }, + { "storage_seek", test_storage_seek }, + { "storage_next", test_storage_next }, + { "storage_prev", test_storage_prev }, + { "storage_delete", test_storage_delete }, + { "storage_reset", test_storage_reset }, + { "storage_key", test_storage_key }, + { "storage_data", test_storage_data }, + }; + int i; + + for(i=0; ipReal->pStoreVfunc->xBegin(p->pReal, iLevel); + p->base.iTransLevel = p->pReal->iTransLevel; + return rc; +} + +static int kvwrapCommitPhaseOne(KVStore *pKVStore, int iLevel){ + int rc; + KVWrap *p = (KVWrap *)pKVStore; + rc = p->pReal->pStoreVfunc->xCommitPhaseOne(p->pReal, iLevel); + p->base.iTransLevel = p->pReal->iTransLevel; + return rc; +} + +static int kvwrapCommitPhaseTwo(KVStore *pKVStore, int iLevel){ + int rc; + KVWrap *p = (KVWrap *)pKVStore; + rc = p->pReal->pStoreVfunc->xCommitPhaseTwo(p->pReal, iLevel); + p->base.iTransLevel = p->pReal->iTransLevel; + return rc; +} + +static int kvwrapRollback(KVStore *pKVStore, int iLevel){ + int rc; + KVWrap *p = (KVWrap *)pKVStore; + rc = p->pReal->pStoreVfunc->xRollback(p->pReal, iLevel); + p->base.iTransLevel = p->pReal->iTransLevel; + return rc; +} + +static int kvwrapRevert(KVStore *pKVStore, int iLevel){ + int rc; + KVWrap *p = (KVWrap *)pKVStore; + rc = p->pReal->pStoreVfunc->xRevert(p->pReal, iLevel); + p->base.iTransLevel = p->pReal->iTransLevel; + return rc; +} + +static int kvwrapReplace( + KVStore *pKVStore, + const KVByteArray *aKey, KVSize nKey, + const KVByteArray *aData, KVSize nData +){ + KVWrap *p = (KVWrap *)pKVStore; + return p->pReal->pStoreVfunc->xReplace(p->pReal, aKey, nKey, aData, nData); +} + +/* +** Create a new cursor object. +*/ +static int kvwrapOpenCursor(KVStore *pKVStore, KVCursor **ppKVCursor){ + int rc = SQLITE4_OK; + KVWrap *p = (KVWrap *)pKVStore; + KVWrapCsr *pCsr; + + pCsr = (KVWrapCsr *)sqlite4_malloc(0, sizeof(KVWrapCsr)); + if( pCsr==0 ){ + rc = SQLITE4_NOMEM; + }else{ + memset(pCsr, 0, sizeof(KVWrapCsr)); + rc = p->pReal->pStoreVfunc->xOpenCursor(p->pReal, &pCsr->pReal); + if( rc!=SQLITE4_OK ){ + sqlite4_free(0, pCsr); + pCsr = 0; + }else{ + pCsr->base.pStore = pKVStore; + pCsr->base.pStoreVfunc = pKVStore->pStoreVfunc; + } + } + + *ppKVCursor = (KVCursor*)pCsr; + return rc; +} + +/* +** Reset a cursor +*/ +static int kvwrapReset(KVCursor *pKVCursor){ + KVWrap *p = (KVWrap *)(pKVCursor->pStore); + KVWrapCsr *pCsr = (KVWrapCsr *)pKVCursor; + return p->pReal->pStoreVfunc->xReset(pCsr->pReal); +} + +/* +** Destroy a cursor object +*/ +static int kvwrapCloseCursor(KVCursor *pKVCursor){ + int rc; + KVWrap *p = (KVWrap *)(pKVCursor->pStore); + KVWrapCsr *pCsr = (KVWrapCsr *)pKVCursor; + rc = p->pReal->pStoreVfunc->xCloseCursor(pCsr->pReal); + sqlite4_free(0, pCsr); + return rc; +} + +/* +** Move a cursor to the next non-deleted node. +*/ +static int kvwrapNextEntry(KVCursor *pKVCursor){ + int rc; + KVWrap *p = (KVWrap *)(pKVCursor->pStore); + KVWrapCsr *pCsr = (KVWrapCsr *)pKVCursor; + rc = p->pReal->pStoreVfunc->xNext(pCsr->pReal); + kvwg.nStep++; + return rc; +} + +/* +** Move a cursor to the previous non-deleted node. +*/ +static int kvwrapPrevEntry(KVCursor *pKVCursor){ + int rc; + KVWrap *p = (KVWrap *)(pKVCursor->pStore); + KVWrapCsr *pCsr = (KVWrapCsr *)pKVCursor; + rc = p->pReal->pStoreVfunc->xPrev(pCsr->pReal); + kvwg.nStep++; + return rc; +} + +/* +** Seek a cursor. +*/ +static int kvwrapSeek( + KVCursor *pKVCursor, + const KVByteArray *aKey, + KVSize nKey, + int dir +){ + KVWrap *p = (KVWrap *)(pKVCursor->pStore); + KVWrapCsr *pCsr = (KVWrapCsr *)pKVCursor; + + /* If aKey[0]==0, this is a seek to retrieve meta-data. Don't count this. */ + if( aKey[0] ) kvwg.nSeek++; + + return p->pReal->pStoreVfunc->xSeek(pCsr->pReal, aKey, nKey, dir); +} + +/* +** Delete the entry that the cursor is pointing to. +** +** Though the entry is "deleted", it still continues to exist as a +** phantom. Subsequent xNext or xPrev calls will work, as will +** calls to xKey and xData, thought the result from xKey and xData +** are undefined. +*/ +static int kvwrapDelete(KVCursor *pKVCursor){ + KVWrap *p = (KVWrap *)(pKVCursor->pStore); + KVWrapCsr *pCsr = (KVWrapCsr *)pKVCursor; + return p->pReal->pStoreVfunc->xDelete(pCsr->pReal); +} + +/* +** Return the key of the node the cursor is pointing to. +*/ +static int kvwrapKey( + KVCursor *pKVCursor, /* The cursor whose key is desired */ + const KVByteArray **paKey, /* Make this point to the key */ + KVSize *pN /* Make this point to the size of the key */ +){ + KVWrap *p = (KVWrap *)(pKVCursor->pStore); + KVWrapCsr *pCsr = (KVWrapCsr *)pKVCursor; + return p->pReal->pStoreVfunc->xKey(pCsr->pReal, paKey, pN); +} + +/* +** Return the data of the node the cursor is pointing to. +*/ +static int kvwrapData( + KVCursor *pKVCursor, /* The cursor from which to take the data */ + KVSize ofst, /* Offset into the data to begin reading */ + KVSize n, /* Number of bytes requested */ + const KVByteArray **paData, /* Pointer to the data written here */ + KVSize *pNData /* Number of bytes delivered */ +){ + KVWrap *p = (KVWrap *)(pKVCursor->pStore); + KVWrapCsr *pCsr = (KVWrapCsr *)pKVCursor; + return p->pReal->pStoreVfunc->xData(pCsr->pReal, ofst, n, paData, pNData); +} + +/* +** Destructor for the entire in-memory storage tree. +*/ +static int kvwrapClose(KVStore *pKVStore){ + int rc; + KVWrap *p = (KVWrap *)pKVStore; + rc = p->pReal->pStoreVfunc->xClose(p->pReal); + sqlite4_free(0, p); + return rc; +} + +/* +** Invoke the xControl() method of the underlying KVStore object. +*/ +static int kvwrapControl(KVStore *pKVStore, int op, void *pArg){ + KVWrap *p = (KVWrap *)pKVStore; + return p->pReal->pStoreVfunc->xControl(p->pReal, op, pArg); +} + +static int newFileStorage( + sqlite4_env *pEnv, + KVStore **ppKVStore, + const char *zName, + unsigned openFlags +){ + + /* Virtual methods for an LSM data store */ + static const KVStoreMethods kvwrapMethods = { + 1, + sizeof(KVStoreMethods), + kvwrapReplace, + kvwrapOpenCursor, + kvwrapSeek, + kvwrapNextEntry, + kvwrapPrevEntry, + kvwrapDelete, + kvwrapKey, + kvwrapData, + kvwrapReset, + kvwrapCloseCursor, + kvwrapBegin, + kvwrapCommitPhaseOne, + kvwrapCommitPhaseTwo, + kvwrapRollback, + kvwrapRevert, + kvwrapClose, + kvwrapControl + }; + + KVWrap *pNew; + int rc = SQLITE4_OK; + + pNew = (KVWrap *)sqlite4_malloc(0, sizeof(KVWrap)); + if( pNew==0 ){ + rc = SQLITE4_NOMEM; + }else{ + memset(pNew, 0, sizeof(KVWrap)); + pNew->base.pStoreVfunc = &kvwrapMethods; + rc = kvwg.xFactory(pEnv, &pNew->pReal, zName, openFlags); + if( rc!=SQLITE4_OK ){ + sqlite4_free(0, pNew); + pNew = 0; + } + } + + *ppKVStore = (KVStore*)pNew; + return rc; +} + +static int kvwrap_install_cmd(Tcl_Interp *interp, int objc, Tcl_Obj **objv){ + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 2, objv, ""); + return TCL_ERROR; + } + + if( kvwg.xFactory==0 ){ + sqlite4_env_config(0, SQLITE4_ENVCONFIG_KVSTORE_GET, "main", &kvwg.xFactory); + sqlite4_env_config(0, SQLITE4_ENVCONFIG_KVSTORE_PUSH, "main",newFileStorage); + } + return TCL_OK; +} + +static int kvwrap_seek_cmd(Tcl_Interp *interp, int objc, Tcl_Obj **objv){ + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 2, objv, ""); + return TCL_ERROR; + } + + Tcl_SetObjResult(interp, Tcl_NewIntObj(kvwg.nSeek)); + return TCL_OK; +} + +static int kvwrap_step_cmd(Tcl_Interp *interp, int objc, Tcl_Obj **objv){ + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 2, objv, ""); + return TCL_ERROR; + } + + Tcl_SetObjResult(interp, Tcl_NewIntObj(kvwg.nStep)); + return TCL_OK; +} + +static int kvwrap_reset_cmd(Tcl_Interp *interp, int objc, Tcl_Obj **objv){ + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 2, objv, ""); + return TCL_ERROR; + } + + kvwg.nStep = 0; + kvwg.nSeek = 0; + + Tcl_ResetResult(interp); + return TCL_OK; +} + + +/* +** TCLCMD: kvwrap SUB-COMMAND +*/ +static int kvwrap_command( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + struct Subcmd { + const char *zCmd; + int (*xCmd)(Tcl_Interp *, int, Tcl_Obj **); + } aSub[] = { + { "install", kvwrap_install_cmd }, + { "step", kvwrap_step_cmd }, + { "seek", kvwrap_seek_cmd }, + { "reset", kvwrap_reset_cmd }, + }; + int iSub; + int rc; + + rc = Tcl_GetIndexFromObjStruct( + interp, objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub + ); + if( rc==TCL_OK ){ + rc = aSub[iSub].xCmd(interp, objc, (Tcl_Obj **)objv); + } + + return rc; +} + +/* +** Register the TCL commands defined above with the TCL interpreter. +** +** This routine should be the only externally visible symbol in this +** source code file. +*/ +int Sqliteteststorage2_Init(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "kvwrap", kvwrap_command, 0, 0); + return TCL_OK; +} Index: test/test_utf.c ================================================================== --- test/test_utf.c +++ test/test_utf.c @@ -101,11 +101,11 @@ { 0, 0 } }; struct EncName *pEnc; char *z = Tcl_GetString(pObj); for(pEnc=&encnames[0]; pEnc->zName; pEnc++){ - if( 0==sqlite4_stricmp(z, pEnc->zName) ){ + if( 0==sqlite4StrICmp(z, pEnc->zName) ){ break; } } if( !pEnc->enc ){ Tcl_AppendResult(interp, "No such encoding: ", z, 0); @@ -114,13 +114,11 @@ return SQLITE4_UTF16NATIVE; } return pEnc->enc; } -static void freeStr(void *pEnv, void *pStr){ - sqlite4_free((sqlite4_env*)pEnv, pStr); -} +static void freeStr(void *pStr){ sqlite4_free(0, pStr); } /* ** Usage: test_translate ?? ** */ @@ -134,11 +132,11 @@ u8 enc_to; sqlite4_value *pVal; char *z; int len; - void (*xDel)(void*,void*) = SQLITE4_STATIC; + void (*xDel)(void *p) = SQLITE4_STATIC; if( objc!=4 && objc!=5 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", Tcl_GetStringFromObj(objv[0], 0), " ", 0 @@ -159,19 +157,19 @@ if( enc_from==SQLITE4_UTF8 ){ z = Tcl_GetString(objv[1]); if( objc==5 ){ z = sqlite4_mprintf(0, "%s", z); } - sqlite4ValueSetStr(pVal, -1, z, enc_from, xDel, 0); + sqlite4ValueSetStr(pVal, -1, z, enc_from, xDel); }else{ z = (char*)Tcl_GetByteArrayFromObj(objv[1], &len); if( objc==5 ){ char *zTmp = z; z = sqlite4_malloc(0, len); memcpy(z, zTmp, len); } - sqlite4ValueSetStr(pVal, -1, z, enc_from, xDel, 0); + sqlite4ValueSetStr(pVal, -1, z, enc_from, xDel); } z = (char *)sqlite4ValueText(pVal, enc_to); len = sqlite4ValueBytes(pVal, enc_to) + (enc_to==SQLITE4_UTF8?1:2); Tcl_SetObjResult(interp, Tcl_NewByteArrayObj((u8*)z, len)); Index: test/tester.tcl ================================================================== --- test/tester.tcl +++ test/tester.tcl @@ -147,28 +147,10 @@ return 100; # TODO: Good default? } return $::G(file-retry-delay) } -# Return the string representing the name of the current directory. On -# Windows, the result is "normalized" to whatever our parent command shell -# is using to prevent case-mismatch issues. -# -proc get_pwd {} { - if {$::tcl_platform(platform) eq "windows"} { - # - # NOTE: Cannot use [file normalize] here because it would alter the - # case of the result to what Tcl considers canonical, which would - # defeat the purpose of this procedure. - # - return [string map [list \\ /] \ - [string trim [exec -- $::env(ComSpec) /c echo %CD%]]] - } else { - return [pwd] - } -} - # Copy file $from into $to. This is used because some versions of # TCL for windows (notably the 8.4.1 binary package shipped with the # current mingw release) have a broken "file copy" command. # proc copy_file {from to} { @@ -492,49 +474,23 @@ if {![info exists ::G(match)] || [string match $::G(match) $name]} { if {[catch {uplevel #0 "$cmd;\n"} result]} { puts "\nError: $result" fail_test $name - } else { - if {[regexp {^~?/.*/$} $expected]} { - if {[string index $expected 0]=="~"} { - set re [string map {# {[-0-9.]+}} [string range $expected 2 end-1]] - set ok [expr {![regexp $re $result]}] - } else { - set re [string map {# {[-0-9.]+}} [string range $expected 1 end-1]] - set ok [regexp $re $result] - } - } else { - set ok [expr {[string compare $result $expected]==0}] - } - if {!$ok} { - # if {![info exists ::testprefix] || $::testprefix eq ""} { - # error "no test prefix" - # } - puts "\nExpected: \[$expected\]\n Got: \[$result\]" - fail_test $name - } else { - puts " Ok" - } + } elseif {[string compare $result $expected]} { + puts "\nExpected: \[$expected\]\n Got: \[$result\]" + fail_test $name + } else { + puts " Ok" } } else { puts " Omitted" omit_test $name "pattern mismatch" 0 } flush stdout } -proc catchcmd {db {cmd ""}} { - global CLI - set out [open cmds.txt w] - puts $out $cmd - close $out - set line "exec $CLI $db < cmds.txt" - set rc [catch { eval $line } msg] - list $rc $msg -} - proc filepath_normalize {p} { # test cases should be written to assume "unix"-like file paths if {$::tcl_platform(platform)!="unix"} { # lreverse*2 as a hack to remove any unneeded {} after the string map lreverse [lreverse [string map {\\ /} [regsub -nocase -all {[a-z]:[/\\]+} $p {/}]]] @@ -760,11 +716,10 @@ puts "******************************************************************" } if {$::cmdlinearg(binarylog)} { vfslog finalize binarylog } - kvwrap uninstall if {[lindex [sqlite4_env_status SQLITE4_ENVSTATUS_MALLOC_COUNT 0] 1]>0 || [sqlite4_memory_used]>0} { puts "Unfreed memory: [sqlite4_memory_used] bytes in\ [lindex [sqlite4_env_status SQLITE4_ENVSTATUS_MALLOC_COUNT 0] 1] allocations" incr nErr @@ -1565,11 +1520,11 @@ # by SQLite 3 tests. # proc optimize_db {} { #catch { sqlite4_lsm_flush db main - sqlite4_lsm_work db main -nmerge 1 -npage 100000 + sqlite4_lsm_work db main -opt -flush 100000 sqlite4_lsm_checkpoint db main #} return "" } Index: test/tkt-3a77c9714e.test ================================================================== --- test/tkt-3a77c9714e.test +++ test/tkt-3a77c9714e.test @@ -37,11 +37,11 @@ ) } {ABCDEF ABCDEF} do_execsql_test 2.1 { CREATE TABLE [Beginnings] ( - [Id] INTEGER PRIMARY KEY /*AUTOINCREMENT*/,[Title] TEXT, [EndingId] INTEGER + [Id] INTEGER PRIMARY KEY AUTOINCREMENT,[Title] TEXT, [EndingId] INTEGER ); CREATE TABLE [Endings] (Id INT,Title TEXT,EndingId INT); INSERT INTO Beginnings (Id, Title, EndingId) VALUES (1, 'FACTOR', 18); INSERT INTO Beginnings (Id, Title, EndingId) VALUES (2, 'SWIMM', 18); INSERT INTO Endings (Id, Title, EndingId) VALUES (1, 'ING', 18); Index: test/tkt-3fe897352e.test ================================================================== --- test/tkt-3fe897352e.test +++ test/tkt-3fe897352e.test @@ -31,30 +31,30 @@ PRAGMA encoding=UTF8; CREATE TABLE t1(x); INSERT INTO t1 VALUES(hex_to_utf16be('D800')); SELECT hex(x) FROM t1; } -} {eda080} +} {EDA080} do_test tkt-3fe89-1.2 { db eval { DELETE FROM t1; INSERT INTO t1 VALUES(hex_to_utf16le('00D8')); SELECT hex(x) FROM t1; } -} {eda080} +} {EDA080} do_test tkt-3fe89-1.3 { db eval { DELETE FROM t1; INSERT INTO t1 VALUES(hex_to_utf16be('DFFF')); SELECT hex(x) FROM t1; } -} {edbfbf} +} {EDBFBF} do_test tkt-3fe89-1.4 { db eval { DELETE FROM t1; INSERT INTO t1 VALUES(hex_to_utf16le('FFDF')); SELECT hex(x) FROM t1; } -} {edbfbf} +} {EDBFBF} finish_test Index: test/tkt3442.test ================================================================== --- test/tkt3442.test +++ test/tkt3442.test @@ -48,10 +48,13 @@ # ifcapable explain { do_test tkt3442-1.2 { EQP { SELECT node FROM listhash WHERE id='5000' LIMIT 1; } } {0 0 0 {SEARCH TABLE listhash USING INDEX ididx (id=?) (~1 rows)}} + do_test tkt3442-1.3 { + EQP { SELECT node FROM listhash WHERE id="5000" LIMIT 1; } + } {0 0 0 {SEARCH TABLE listhash USING INDEX ididx (id=?) (~1 rows)}} } # Some extra tests testing other permutations of 5000. # Index: test/tkt3493.test ================================================================== --- test/tkt3493.test +++ test/tkt3493.test @@ -18,14 +18,14 @@ source $testdir/tester.tcl do_test tkt3493-1.1 { execsql { BEGIN; - CREATE TABLE A (id INTEGER PRIMARY KEY /*AUTOINCREMENT*/, val TEXT); + CREATE TABLE A (id INTEGER PRIMARY KEY AUTOINCREMENT, val TEXT); INSERT INTO A VALUES(1,'123'); INSERT INTO A VALUES(2,'456'); - CREATE TABLE B (id INTEGER PRIMARY KEY /*AUTOINCREMENT*/, val TEXT); + CREATE TABLE B (id INTEGER PRIMARY KEY AUTOINCREMENT, val TEXT); INSERT INTO B VALUES(1,1); INSERT INTO B VALUES(2,2); CREATE TABLE A_B (B_id INTEGER NOT NULL, A_id INTEGER); INSERT INTO A_B VALUES(1,1); INSERT INTO A_B VALUES(2,2); Index: test/tkt3841.test ================================================================== --- test/tkt3841.test +++ test/tkt3841.test @@ -25,18 +25,18 @@ do_test tkt3841.1 { execsql { CREATE TABLE table2 (key TEXT, x TEXT); CREATE TABLE list (key TEXT, value TEXT); - INSERT INTO table2 VALUES ('a', 'alist'); - INSERT INTO table2 VALUES ('b', 'blist'); - INSERT INTO list VALUES ('a', 1); - INSERT INTO list VALUES ('a', 2); - INSERT INTO list VALUES ('a', 3); - INSERT INTO list VALUES ('b', 4); - INSERT INTO list VALUES ('b', 5); - INSERT INTO list VALUES ('b', 6); + INSERT INTO table2 VALUES ("a", "alist"); + INSERT INTO table2 VALUES ("b", "blist"); + INSERT INTO list VALUES ("a", 1); + INSERT INTO list VALUES ("a", 2); + INSERT INTO list VALUES ("a", 3); + INSERT INTO list VALUES ("b", 4); + INSERT INTO list VALUES ("b", 5); + INSERT INTO list VALUES ("b", 6); SELECT table2.x, (SELECT group_concat(list.value) FROM list Index: test/trigger1.test ================================================================== --- test/trigger1.test +++ test/trigger1.test @@ -288,25 +288,13 @@ CREATE TEMP TABLE t2(x,y); INSERT INTO t1 VALUES(1,2); SELECT * FROM t2; } } {1 {no such table: main.t2}} - do_test trigger-3.6.1 { - catchsql { - DROP TRIGGER r1; - CREATE TEMP TRIGGER r1 AFTER INSERT ON t1 BEGIN - INSERT INTO t2 VALUES(NEW.a,NEW.b), (NEW.b*100, NEW.a*100); - END; - INSERT INTO t1 VALUES(1,2); - SELECT * FROM t2; - } - } {0 {1 2 200 100}} - do_test trigger-3.6.2 { - catchsql { - DROP TRIGGER r1; - DELETE FROM t1; - DELETE FROM t2; + do_test trigger-3.6 { + catchsql { + DROP TRIGGER r1; CREATE TEMP TRIGGER r1 AFTER INSERT ON t1 BEGIN INSERT INTO t2 VALUES(NEW.a,NEW.b); END; INSERT INTO t1 VALUES(1,2); SELECT * FROM t2; DELETED tool/fixnumbers.tcl Index: tool/fixnumbers.tcl ================================================================== --- tool/fixnumbers.tcl +++ /dev/null @@ -1,74 +0,0 @@ - -if {[llength $argv]!=1} { - puts stderr "Usage: $argv0 FILENAME" - exit -1 -} -set zFile [lindex $argv 0] -if {![file exists $zFile]} { - puts stderr "No such file: $zFile" - exit -1 -} -set fd [open $zFile] -set out "" - -set idx(1) 0 -set idx(2) 0 -set idx(3) 0 - -set iSeenToc 0 -set before_toc "" -set after_toc "" -set out "" - -while {![eof $fd]} { - set line [gets $fd] - - if {[string trim $line]=="
"} { - if {$iSeenToc!=0} {error "start_of_toc mismatch"} - set iSeenToc 1 - set before_toc $out - set out "" - continue - } - if {[string trim $line]=="
"} { - if {$iSeenToc!=1} {error "end_of_toc mismatch"} - set iSeenToc 2 - continue - } - if {$iSeenToc==1} continue - - if {[regexp {} $line -> iLevel]} { - if {$iLevel==1 || $idx(1)>0} { - incr idx($iLevel) - for {set i [expr $iLevel+1]} {$i < 4} {incr i} { set idx($i) 0 } - set num "" - for {set i 1} {$i <= $iLevel} {incr i} { append num "$idx($i)." } - - regsub -all {<[^>]*>} $line "" tag - regsub -all {[0123456789.]} $tag "" tag - set tag [string map {" " _} [string trim [string tolower $tag]]] - - set pattern [subst -nocom {]*>[0123456789. ]*}] - regsub $pattern $line "$num " line - - regsub -all {<[^>]*>} $line "" text - set margin [string repeat { } [expr $iLevel * 6]] - append toc "${margin}$text
\n" - } - } - append out "$line\n" -} -if {$iSeenToc!=2} {error "No toc divs..."} -set after_toc $out - - - -puts $before_toc -puts "
" -puts $toc -puts "
" -puts $after_toc - - - - Index: tool/lsmperf.tcl ================================================================== --- tool/lsmperf.tcl +++ tool/lsmperf.tcl @@ -1,14 +1,10 @@ #!/bin/sh # \ exec tclsh "$0" "$@" -package require sqlite3 -########################################################################## -# Procedures used when running tests (collecting data). -# proc exec_lsmtest_speed {nSec spec} { set fd [open [list |./lsmtest speed2 {*}$spec]] set res [list] puts -nonewline "./lsmtest speed2" @@ -34,368 +30,181 @@ } catch { close $fd } set res } -proc run_speed_test {zDb nTimeout nWrite nFetch nPause nRepeat zSystem zName} { - - set spec [list -w $nWrite -f $nFetch -r $nRepeat -p $nPause -system $zSystem] - set res [exec_lsmtest_speed $nTimeout $spec] - set cmd "lsmtest speed2 -w $nWrite -f $nFetch -r $nRepeat -p $nPause" - append cmd " -system \"$zSystem\"" - - sqlite3 db $zDb - - db eval { - PRAGMA synchronous = OFF; - CREATE TABLE IF NOT EXISTS run( - runid INTEGER PRIMARY KEY, nWrite INT, nFetch INT, nPause INT, - cmd TEXT, name TEXT - ); - CREATE TABLE IF NOT EXISTS sample( - runid INT, sample INT, writems INT, fetchms INT, - PRIMARY KEY(runid, sample) - ); - INSERT INTO run VALUES(NULL, $nWrite, $nFetch, $nPause, $cmd, $zName); - } - - set id [db last_insert_rowid] - foreach sample $res { - foreach {a b c} $sample {} - db eval { INSERT INTO sample VALUES($id, $a, $b, $c) } - } - - db close -} -# -# End of procs used while gathering data. -########################################################################## - -########################################################################## - -proc chart_y_to_canvas {y} { - foreach v [uplevel {info vars c_*}] { upvar $v $v } - set ytotal [expr ($c_height - $c_top_margin - $c_bottom_margin)] - expr $c_height - $c_bottom_margin - int($ytotal * $y / $c_ymax) -} -proc chart_sample_to_canvas {nSample x} { - foreach v [uplevel {info vars c_*}] { upvar $v $v } - set xtotal [expr ($c_width - $c_left_margin - $c_right_margin)] - expr {$c_left_margin + ($x*$xtotal/$nSample)} -} - -proc draw_line_series {nSample lSeries tag} { - foreach v [uplevel {info vars c_*}] { upvar $v $v } - set xtotal [expr ($c_width - $c_left_margin - $c_right_margin)] - set ytotal [expr ($c_height - $c_top_margin - $c_bottom_margin)] - - set x 0 - for {set i 1} {$i < $nSample} {incr i} { - set x2 [expr $x + int( double($xtotal - $x) / ($nSample-$i) )] - if {[lindex $lSeries $i-1] < $c_ymax && [lindex $lSeries $i] < $c_ymax} { - - set y1 [chart_y_to_canvas [lindex $lSeries $i-1]] - set y2 [chart_y_to_canvas [lindex $lSeries $i]] - .c create line \ - [expr ($c_left_margin + $x)] $y1 \ - [expr ($c_left_margin + $x2)] $y2 \ - -tag $tag - } - - set x $x2 - } -} - -proc draw_points_series {nSample lSeries tag} { - foreach v [uplevel {info vars c_*}] { upvar $v $v } - set xtotal [expr ($c_width - $c_left_margin - $c_right_margin)] - set ytotal [expr ($c_height - $c_top_margin - $c_bottom_margin)] - - for {set i 0} {$i < $nSample} {incr i} { - set x [chart_sample_to_canvas $nSample $i] - set y [chart_y_to_canvas [lindex $lSeries $i]] - .c create rectangle $x $y [expr $x+2] [expr $y+2] -tag $tag - } -} - -proc draw_bars_series {nSample nShift lSeries tag} { - foreach v [uplevel {info vars c_*}] { upvar $v $v } - set xtotal [expr ($c_width - $c_left_margin - $c_right_margin)] - set ytotal [expr ($c_height - $c_top_margin - $c_bottom_margin)] - - set b [expr $c_height - $c_bottom_margin] - for {set i 0} {$i < $nSample} {incr i} { - set x [expr [chart_sample_to_canvas $nSample $i] + $nShift] - set y [chart_y_to_canvas [lindex $lSeries $i]] - .c create rectangle $x $y [expr $x+$c_bar_width-1] $b -tag $tag - } -} - -proc draw_text {iRun iRun2} { - foreach v [uplevel {info vars c_*}] { upvar $v $v } - - set y $c_height - if {$iRun2!=""} { - array set metrics [font metrics {-size 8}] - set cmd [db one {SELECT cmd FROM run WHERE runid=$iRun2}] - .c create text 10 $y -anchor sw -text $cmd -fill grey70 -font {-size 8} - set y [expr $y-$metrics(-ascent)] - } - set cmd [db one {SELECT cmd FROM run WHERE runid=$iRun}] - .c create text 10 $y -anchor sw -text $cmd -fill grey70 -font {-size 8} -} - -proc format_integer {n} { - if { ($n % 1000000)==0 } { return "[expr $n/1000000]M" } - if { $n>1000000 && ($n % 100000)==0 } { - return "[expr double($n)/1000000]M" - } - if { ($n % 1000)==0 } { return "[expr $n/1000]K" } - return $n -} - -proc populate_chart {nSample nShift iRun colors} { - foreach v [uplevel {info vars c_*}] { upvar $v $v } - upvar nWrite nWrite - upvar nFetch nFetch - set name [db one "SELECT name FROM run WHERE runid=$iRun"] - - set lWrite [list] - set lFetch [list] - for {set i 0} {$i < $nSample} {incr i} { - set q "SELECT writems, fetchms FROM sample WHERE runid=$iRun AND sample=$i" - db eval $q break - lappend lWrite [expr {(1000.0*$nWrite) / $writems}] - lappend lFetch [expr {(1000.0*$nFetch) / $fetchms}] - } - - draw_bars_series $nSample $nShift $lWrite writes_p_$iRun - draw_points_series $nSample $lFetch fetches_p_$iRun - - set lWrite [list] - set lFetch [list] - for {set i 0} {$i < $nSample} {incr i} { - set q "SELECT 1000.0 * ($i+1) * $nWrite / sum(writems) AS r1, " - append q " 1000.0 * ($i+1) * $nFetch / sum(fetchms) AS r2 " - append q "FROM sample WHERE runid=$iRun AND sample<=$i" - - db eval $q break - lappend lWrite $r1 - lappend lFetch $r2 - } - - draw_line_series $nSample $lWrite writes_$iRun - draw_line_series $nSample $lFetch fetches_$iRun - - # Create the legend for the data drawn above. - # - array set metrics [font metrics default] - set y $c_legend_ypadding - set x [expr $c_width - $c_legend_padding] - set xsym [expr { $c_width - - [font measure default "$name cumulative fetches/sec"] - }] - - foreach {t g} { - "$name fetches/sec" { - rectangle 0 0 -2 2 -tag fetches_p_$iRun - } - "$name writes/sec" { - rectangle 0 0 $c_bar_width $metrics(-ascent) -tag writes_p_$iRun - } - "$name cumulative fetches/sec" { - line 0 0 -30 0 -tag fetches_$iRun - } - "$name cumulative writes/sec" { - line 0 0 -30 0 -tag writes_$iRun - } - } { - .c create text $x $y -tag legend_$iRun -text [subst $t] -anchor e - - set id [eval [concat {.c create} [subst $g]]] - .c addtag legend_$iRun withtag $id - - set box [.c bbox $id] - set xmove [expr {$xsym - ([lindex $box 0]/2 + [lindex $box 2]/2)}] - set ymove [expr $y - ([lindex $box 1]/2 + [lindex $box 3]/2)] - - .c move $id $xmove $ymove - incr y $metrics(-linespace) - } - - .c itemconfigure writes_$iRun -fill [lindex $colors 0] -width 2 - .c itemconfigure fetches_$iRun -fill [lindex $colors 2] - .c itemconfigure writes_p_$iRun -fill [lindex $colors 1] - .c itemconfigure fetches_p_$iRun -fill [lindex $colors 3] - catch { .c itemconfigure fetches_p_$iRun -outline [lindex $colors 3] } - catch { .c itemconfigure writes_p_$iRun -outline [lindex $colors 1] } -} - -proc generate_chart {png db iRun {iRun2 {}}} { - sqlite3 db $db - db eval { SELECT nWrite, nFetch FROM run WHERE runid=$iRun } {} - if {0==[info exists nWrite]} { - error "No such run in db $db: $iRun" - } - - set c_left_margin 50 - set c_bottom_margin 60 - set c_top_margin 20 - set c_right_margin 400 - set c_width 1250 - set c_height 350 - - set c_ymax 300000 - set c_ytick 50000 - set c_ticksize 5 - set c_nxtick 10 - set c_dbsize_padding 20 - set c_legend_padding 20 - set c_legend_ypadding 100 - set c_bar_width 2 - - package require Tk - canvas .c - .c configure -width $c_width -height $c_height - pack .c -fill both -expand 1 - - # Make the background white - .c configure -background white - draw_text $iRun $iRun2 - - # Draw the box for the chart - # - set y [expr $c_height - $c_bottom_margin] - set x [expr $c_width - $c_right_margin] - .c create rectangle $c_left_margin $c_top_margin $x $y - - # Draw the vertical ticks - # - set ytotal [expr ($c_height - $c_top_margin - $c_bottom_margin)] - for {set y $c_ytick} {$y <= $c_ymax} {incr y $c_ytick} { - - # Calculate the canvas y coord at which to draw the tick - set ypix [expr {$c_height - $c_bottom_margin - ($y * $ytotal) / $c_ymax}] - - set left_tick_x $c_left_margin - set right_tick_x [expr $c_width - $c_right_margin - $c_ticksize] - foreach x [list $left_tick_x $right_tick_x] { - .c create line $x $ypix [expr $x+$c_ticksize] $ypix - } - - .c create text $c_left_margin $ypix -anchor e -text "[format_integer $y] " - set x [expr $c_width - $c_right_margin] - .c create text $x $ypix -anchor w -text " [format_integer $y]" - } - - # Figure out the total number of samples for this chart - # - set nSample [db one {SELECT count(*) FROM sample where runid=$iRun}] - if {$nSample==0} { error "No such run: $iRun" } - #set nSample 100 - - # Draw the horizontal ticks - # - set w [expr {$c_width - $c_left_margin - $c_right_margin}] - set xincr [expr ($nSample * $nWrite) / $c_nxtick] - for {set i 1} {$i <= $c_nxtick} {incr i} { - set x [expr $i * $xincr] - set xpix [expr {$c_left_margin + ($w / $c_nxtick) * $i}] - - set b [expr $c_height-$c_bottom_margin] - .c create line $xpix $b $xpix [expr $b-$c_ticksize] - .c create text $xpix $b -anchor n -text [format_integer $x] - } - - set x [expr (($c_width-$c_right_margin-$c_left_margin) / 2) + $c_left_margin] - set y [expr $c_height - $c_bottom_margin + $c_dbsize_padding] - .c create text $x $y -anchor n -text "Database Size (number of entries)" - - populate_chart $nSample 0 $iRun {black grey55 black black} - if {$iRun2 != ""} { - #populate_chart $nSample 3 $iRun2 {royalblue skyblue royalblue royalblue} - - set s $c_bar_width - populate_chart $nSample $s $iRun2 {royalblue lightsteelblue royalblue royalblue} - set box [.c bbox legend_$iRun] - set shift [expr $c_legend_padding + [lindex $box 3]-[lindex $box 1]] - .c move legend_$iRun2 0 $shift - } - - .c lower fetches_p_$iRun - .c lower fetches_p_$iRun2 - .c lower writes_p_$iRun - .c lower writes_p_$iRun2 - - bind .c exit - bind . exit - db close -} - -proc capture_photo {z} { - package require Img - set img [image create photo -format window -data .c] - $img write $z -format GIF -} - -# "autoflush=1M multi_proc=0" -# "autoflush=1M multi_proc=0 mt_mode=4 mt_min_ckpt=2M mt_max_ckpt=3M" -# "autoflush=4M multi_proc=0 autocheckpoint=8M" -# "autoflush=4M multi_proc=0 mt_mode=4" -proc run_all_tests {} { - set nInsert 50000 - set nSelect 50000 - set nSleep 20000 - set nPause 2500 - - #set nInsert 5000 - #set nSelect 5000 - #set nSleep 2000 - #set nPause 250 - - set bFirst 0 - - foreach {name config} { - single-threaded "autoflush=1M multi_proc=0" - multi-threaded - "autoflush=1M multi_proc=0 mt_mode=4 mt_min_ckpt=2M mt_max_ckpt=3M" - single-threaded "autoflush=4M multi_proc=0 autocheckpoint=8M" - multi-threaded "autoflush=4M multi_proc=0 mt_mode=4" - } { - if {$bFirst!=0} { - puts "sleeping 20 seconds..." ; after $nSleep - } - run_speed_test res.db 900 $nInsert $nSelect 0 200 $config $name - set bFirst 1 - } - - # Tests with a 2.5 second delay. - # - foreach {name config} { - single-threaded "autoflush=4M multi_proc=0 autocheckpoint=8M" - multi-threaded "autoflush=4M multi_proc=0 mt_mode=4" - } { - puts "sleeping 20 seconds..." ; after $nSleep - run_speed_test res.db 900 $nInsert $nSelect $nPause 200 $config $name - } -} - -#run_all_tests - -generate_chart png res.db 1 2 -update -capture_photo lsmperf1.gif -destroy .c - -generate_chart png res.db 3 4 -update -capture_photo lsmperf2.gif -destroy .c - -generate_chart png res.db 5 6 -update -capture_photo lsmperf3.gif -destroy .c - -exit +proc write_to_file {zFile zScript} { + set fd [open $zFile w] + puts $fd $zScript + close $fd +} + +proc exec_gnuplot_script {script png} { + write_to_file out " + $script + pause -1 + " + + set script " + set terminal pngcairo size 1200,400 + $script + " + exec gnuplot << $script > $png 2>/dev/null +} + +proc make_totalset {res nOp bFetch} { + set ret "" + set nMs 0 + set nIns 0 + foreach row $res { + foreach {i msInsert msFetch} $row {} + incr nIns $nOp + if {$bFetch==0} { + incr nMs $msInsert + } else { + incr nMs $msFetch + } + append ret "$nIns [expr $nIns*1000.0/$nMs]\n" + } + append ret "end\n" + set ret +} + +proc make_dataset {res iRes nWrite nShift nOp} { + set ret "" + foreach row $res { + set i [lindex $row 0] + set j [lindex $row [expr $iRes+1]] + set x [expr $i*$nWrite + $nShift] + append ret "$x [expr int($nOp * 1000.0 / $j)]\n" + } + append ret "end\n" + set ret +} + +proc do_write_test {zPng nSec nWrite nFetch nRepeat lSys} { + + if {[llength $lSys]!=2 && [llength $lSys]!=4} { + error "lSys must be a list of 2 or 4 elements" + } + + set lRes [list] + foreach {name sys} $lSys { + set wt [list -w $nWrite -r $nRepeat -f $nFetch -system $sys] + lappend lRes [exec_lsmtest_speed $nSec $wt] + if {$sys != [lindex $lSys end]} { + puts "Sleeping 20 seconds..." + #after 20000 + } + } + + # Set up the header part of the gnuplot script. + # + set xmax 0 + foreach res $lRes { + set xthis [expr [lindex $res end 0]*$nWrite + 5*$nWrite/4] + if {$xthis>$xmax} {set xmax $xthis} + } + + append labeltext "Test parameters:\\n" + append labeltext " $nWrite writes per iteration\\n" + append labeltext " $nFetch fetches per iteration\\n" + append labeltext " key size is 12 bytes\\n" + append labeltext " value size is 100 bytes\\n" + set labelx [expr int($xmax * 1.2)] + + set nWrite2 [expr $nWrite/2] + set y2setup "" + if {$nFetch>0} { + set y2setup { + set ytics nomirror + set y2tics nomirror + set y2range [0:*] + } + } + set script [subst -nocommands { + set boxwidth $nWrite2 + set xlabel "Database Size" + set y2label "Queries per second" + set ylabel "Writes per second" + set yrange [0:*] + set xrange [0:$xmax] + set key outside bottom + $y2setup + set label 1 "$labeltext" at screen 0.95,graph 1.0 right + }] + + + set cols [list {#B0C4DE #00008B #000000} {#F08080 #8B0000 #FFA500}] + set cols [lrange $cols 0 [expr ([llength $lSys]/2)-1]] + + set nShift [expr ($nWrite/2)] + set plot1 "" + set plot2 "" + set plot3 "" + set plot4 "" + set data1 "" + set data2 "" + set data3 "" + set data4 "" + + foreach {name sys} $lSys res $lRes col $cols { + foreach {c1 c2 c3} $col {} + + # First plot. Writes per second (bar chart). + # + if {$plot1 != ""} { set plot1 ", $plot1" } + set plot1 "\"-\" ti \"$name writes/sec\" with boxes fs solid lc rgb \"$c1\"$plot1" + set data1 "[make_dataset $res 0 $nWrite $nShift $nWrite] $data1" + + # Third plot. Cumulative writes per second (line chart). + # + set plot3 ",\"-\" ti \"$name cumulative writes/sec\" with lines lc rgb \"$c2\" lw 2 $plot3" + set data3 "[make_totalset $res $nWrite 0] $data3" + + if {$nFetch>0} { + set new ", \"-\" ti \"$name fetches/sec\" axis x1y2 " + append new "with points lw 1 lc rgb \"$c2\"" + set plot2 "$new $plot2" + set data2 "[make_dataset $res 1 $nWrite $nWrite $nFetch] $data2" + + set new ",\"-\" ti \"$name cumulative fetches/sec\" " + append new "with lines lc rgb \"$c2\" lw 1 " + set plot4 "$new $plot4" + set data4 "[make_totalset $res $nFetch 1] $data4" + } + + incr nShift [expr $nWrite/4] + } + append script "plot " + append script $plot1 + append script $plot2 + append script $plot3 + append script $plot4 + append script "\n" + append script $data1 + append script $data2 + append script $data3 + append script $data4 + + append script "pause -1\n" + exec_gnuplot_script $script $zPng +} + +do_write_test x.png 600 50000 50000 20 { + lsm-st "mmap=1 multi_proc=0 safety=1 threads=1 autowork=1" + lsm-st2 "page_size=1024 mmap=1 multi_proc=0 safety=1 threads=1 autowork=1" +} + +# lsm-mt "mmap=1 multi_proc=0 threads=2 autowork=0 autocheckpoint=8192000" +# lsm-mt "mmap=1 multi_proc=0 safety=1 threads=3 autowork=0" +# lsm-st "mmap=1 multi_proc=0 safety=1 threads=1 autowork=1" +# lsm-mt "mmap=1 multi_proc=0 safety=1 threads=3 autowork=0" +# lsm-mt "mmap=1 multi_proc=0 safety=1 threads=3 autowork=0" +# LevelDB leveldb +# lsm-st "mmap=1 multi_proc=0 safety=1 threads=1 autowork=1" +# LevelDB leveldb +# SQLite sqlite3 + + Index: tool/lsmview.tcl ================================================================== --- tool/lsmview.tcl +++ tool/lsmview.tcl @@ -41,11 +41,11 @@ } namespace import ::autoscroll::* ############################################################################# proc exec_lsmtest_show {args} { - set fd [open [list |lsmtest show {*}$args 2>/dev/null]] + set fd [open [list |lsmtest show {*}$args]] set res "" while {![eof $fd]} { set line [gets $fd] if {[regexp {^\#.*} $line]} continue if {[regexp {^Leaked*} $line]} continue @@ -166,20 +166,20 @@ eval $myScript [list $segment] } proc draw_level {C level tags} { - set lhs [lindex $level 1] + set lhs [lindex $level 0] set l [lindex $tags end] draw_segment $C $lhs [concat $tags "$l.s0"] foreach {x1 y1 x2 y2} [$C bbox "$l.s0"] {} set i 0 set y 0 set x [expr $x2+10] - foreach seg [lrange $level 2 end] { + foreach seg [lrange $level 1 end] { set tag "$l.s[incr i]" draw_segment $C $seg [concat $tags $tag] $C move $tag $x $y foreach {x1 y1 x2 y2} [$C bbox $tag] {} @@ -196,15 +196,15 @@ # Figure out the scale to use. # set nMaxWidth 0.0 foreach level $structure { set nRight 0 - foreach seg [lrange $level 2 end] { + foreach seg [lrange $level 1 end] { set sz [lindex $seg 3] if {$sz > $nRight} { set nRight $sz } } - set nLeft [lindex $level 1 3] + set nLeft [lindex $level 0 3] set nTotal [log2 [expr max($nLeft, 2)]] if {$nRight} {set nTotal [expr $nTotal + [log2 [expr max($nRight, 2)]]]} if {$nTotal > $nMaxWidth} { set nMaxWidth $nTotal } } @@ -219,13 +219,12 @@ draw_level $C $level $tag foreach {x1 y1 x2 y2} [$C bbox $tag] {} $C move $tag [expr ($W-$x2)/2] $y incr y [expr $y2 + $::G(vertical_padding)] - set age [lindex $level 0] foreach {x1 y1 x2 y2} [$C bbox $tag.text] {} - $C create text 5 $y1 -anchor nw -text [string toupper "${tag} ($age):"] + $C create text 10 $y1 -anchor nw -text [string toupper "${tag}:"] } if {[info exists myVertShift]} { set H [winfo height $C] set region [$C bbox all] @@ -247,15 +246,12 @@ set line [$C create line [expr ($xf1+$xf2)/2] $yf2 [expr ($xt1+$xt2)/2] $yt1] $C itemconfigure $line -arrow last } proc draw_internal_pointers {C iLevel level} { - set iAge [lindex $level 0] ;# Age of this level - set lSeg [lrange $level 1 end] ;# List of segments that make up this level - - for {set j 2} {$j < [llength $lSeg]} {incr j} { - if {[lindex $lSeg $j 2]==0} { + for {set j 2} {$j < [llength $level]} {incr j} { + if {[lindex $level $j 2]==0} { draw_one_pointer $C "l$iLevel.s[expr $j-1]" "l$iLevel.s$j" } } } @@ -268,32 +264,28 @@ set i2 [expr $i+1] set l1 [lindex $structure $i] set l2 [lindex $structure $i2] - # Set to true if levels $i and $i2 are currently undergoing a merge - # (have one or more rhs segments), respectively. - # - set bMerge1 [expr [llength $l1]>2] - set bMerge2 [expr [llength $l2]>2] + set bMerge1 [expr [llength $l1]>1] + set bMerge2 [expr [llength $l2]>1] if {$bMerge2} { - if {[lindex $l2 2 2]==0} { + if {[lindex $l2 1 2]==0} { draw_one_pointer $C "l$i.s0" "l$i2.s1" if {$bMerge1} { - draw_one_pointer $C "l$i.s[expr [llength $l1]-2]" "l$i2.s1" + draw_one_pointer $C "l$i.s[expr [llength $l1]-1]" "l$i2.s1" } } } else { - - set bBtree [expr [lindex $l2 1 2]!=0] + set bBtree [expr [lindex $l2 0 2]!=0] if {$bBtree==0 || $bMerge1} { draw_one_pointer $C "l$i.s0" "l$i2.s0" } if {$bBtree==0} { - draw_one_pointer $C "l$i.s[expr [llength $l1]-2]" "l$i2.s0" + draw_one_pointer $C "l$i.s[expr [llength $l1]-1]" "l$i2.s0" } } } } @@ -372,70 +364,29 @@ link_varset $C myText myDb myData $C delete all draw_canvas_content $C $myData [list static_select_callback $C] } -# The first parameter is the number of pages stored on each block of the -# database file (i.e. if the page size is 4K and the block size 1M, 256). -# The second parameter is a list of pages in a segment. -# -# The return value is a list of blocks occupied by the segment. -# -proc pagelist_to_blocklist {nBlkPg pglist} { - set blklist [list] - foreach pg $pglist { - set blk [expr 1 + (($pg-1) / $nBlkPg)] - if {[lindex $blklist end]!=$blk} { - lappend blklist $blk - } - } - set blklist -} - proc static_select_callback {C segment} { - link_varset $C myText myDb myData myTree myCfg + link_varset $C myText myDb myData myTree foreach {iFirst iLast iRoot nSize $segment} $segment {} - set data [string trim [exec_lsmtest_show -c $myCfg $myDb array-pages $iFirst]] + set data [string trim [exec_lsmtest_show $myDb array $iFirst]] $myText delete 0.0 end - # Delete the existing tree entries. $myTree delete [$myTree children {}] - set nBlksz [expr [exec_lsmtest_show -c $myCfg $myDb blocksize] * 1024] - set nPgsz [exec_lsmtest_show -c $myCfg $myDb pagesize] - - if {[regexp {c=1} $myCfg] || [regexp {co=1} $myCfg] - || [regexp {com=1} $myCfg] || [regexp {comp=1} $myCfg] - || [regexp {compr=1} $myCfg] || [regexp {compres=1} $myCfg] - || [regexp {compress=1} $myCfg] || [regexp {compressi=1} $myCfg] - || [regexp {compressio=1} $myCfg] || [regexp {compression=1} $myCfg] - } { - set nBlkPg $nBlksz - } else { - set nBlkPg [expr ($nBlksz / $nPgsz)] - } - - foreach pg $data { - set blk [expr 1 + (($pg-1) / $nBlkPg)] - if {[info exists block($blk)]} { - incr block($blk) 1 - } else { - set block($blk) 1 - } - } - foreach blk [pagelist_to_blocklist $nBlkPg $data] { - set nPg $block($blk) - set blkid($blk) [$myTree insert {} end -text "block $blk ($nPg pages)"] - } - foreach pg $data { - set blk [expr 1 + (($pg-1) / $nBlkPg)] - set id $blkid($blk) - set item [$myTree insert $id end -text $pg] - if {$pg == $iRoot} { - set rootitem $item - set rootparent $id + foreach {first last} $data { + set nPg [expr {$last - $first + 1}] + set caption "${first}..${last} ($nPg pages)" + set id [$myTree insert {} end -text $caption] + for {set i $first} {$i <= $last} {incr i} { + set item [$myTree insert $id end -text $i] + if {$i == $iRoot} { + set rootitem $item + set rootparent $id + } } } if {[info exists rootitem]} { $myTree item $rootparent -open 1 @@ -445,13 +396,13 @@ $myTree selection set [lindex [$myTree children $id] 0] } } proc static_page_callback {C pgno} { - link_varset $C myText myDb myData myPgno myMode myCfg + link_varset $C myText myDb myData myPgno myMode set myPgno $pgno - set data [string trim [exec_lsmtest_show -c $myCfg $myDb $myMode $myPgno]] + set data [string trim [exec_lsmtest_show $myDb $myMode $myPgno]] $myText delete 0.0 end $myText insert 0.0 $data } proc static_treeview_select {C} { @@ -495,40 +446,39 @@ close $fd exec $::env(VISUAL) $fn & after 1000 [subst {catch {file delete -force $fn}}] } -proc static_setup {zConfig zDb} { +proc static_setup {zDb} { panedwindow .pan -orient horizontal frame .pan.c set C [scrollable canvas .pan.c.c -background white -width 400 -height 600] label .pan.c.info -border 2 -relief sunken -height 2 pack .pan.c.info -side bottom -fill x pack .pan.c.c -side top -fill both -expand 1 - link_varset $C myText myDb myData myTree mySelected myMode myModeButton myCfg + link_varset $C myText myDb myData myTree mySelected myMode myModeButton set myDb $zDb frame .pan.t frame .pan.t.f set myModeButton [button .pan.t.f.pagemode] $myModeButton configure -command [list static_toggle_pagemode $C] $myModeButton configure -width 20 $myModeButton configure -text "Switch to HEX" set myMode page-ascii - set myCfg $zConfig set myText [scrollable text .pan.t.text -background white -width 80] button .pan.t.f.view -command [list show_text_in_editor $myText] \ -text Export pack .pan.t.f.pagemode .pan.t.f.view -side right pack .pan.t.f -fill x pack .pan.t.text -expand 1 -fill both set myTree [scrollable ::ttk::treeview .pan.tree] - set myData [string trim [exec_lsmtest_show -c $myCfg $zDb]] + set myData [string trim [exec_lsmtest_show $zDb]] $myText configure -wrap none bind $C exit focus $C @@ -538,21 +488,18 @@ set mySelected "l0.s0.main" static_redraw $C bind $C [list static_redraw $C] bind $myTree <> [list static_treeview_select $C] - static_select_callback $C [lindex $myData 0 1] + static_select_callback $C [lindex $myData 0 0] } -if {[llength $argv] > 2} { - puts stderr "Usage: $argv0 ?CONFIG? ?DATABASE?" +if {[llength $argv] > 1} { + puts stderr "Usage: $argv0 ?DATABASE?" exit -1 } -if {[llength $argv]>0} { - set zConfig "" - set zDb [lindex $argv end] - if {[llength $argv]>1} {set zConfig [lindex $argv 0]} - static_setup $zConfig $zDb +if {[llength $argv]==1} { + static_setup [lindex $argv 0] } else { dynamic_setup fileevent stdin readable [list dynamic_input $C $S] } Index: tool/mkkeywordhash.c ================================================================== --- tool/mkkeywordhash.c +++ tool/mkkeywordhash.c @@ -574,11 +574,11 @@ printf(" h = ((charMap(z[0])*4) ^\n" " (charMap(z[n-1])*3) ^\n" " n) %% %d;\n", bestSize); printf(" for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){\n"); printf(" if( aLen[i]==n &&" - " sqlite4_strnicmp(&zText[aOffset[i]],z,n)==0 ){\n"); + " sqlite4StrNICmp(&zText[aOffset[i]],z,n)==0 ){\n"); for(i=0; i www/lsmapi.wiki -# -# before committing changes to API comments in file lsm.h. -# - -set document_preamble { -LSM API Reference - -
- -

-This page contains the LSM API Reference Manual. It is intended to complement -the LSM User Manual. -} - -set document_text "" -set document_toc "" - -set document_defines [list] -set document_functions [list] -set document_types [list] - -proc put_section {} { - global S - - if {$::state != "off"} { - - set style {style=text-decoration:none} - - regexp {^([^( ]*)} $S(heading) -> id - set id [string tolower [string map {- _} $id]] - if {$id == "file"} {set id filespace} - - set defs [list] - foreach line [split $S(code) "\n"] { - if {[regexp {^#define (LSM_[^ ]*)} $line -> sym]} { - lappend defs $sym - lappend ::document_defines $sym - } - - if {[regexp {struct (lsm_[a-z]*)} $line -> sym]} { - lappend defs $sym - lappend ::document_types $sym - } - - if {[regexp { (lsm_[a-z_]*)\(} $line -> sym]} { - lappend defs $sym - lappend ::document_functions $sym - } - - } - - append ::document_text "

$S(heading)" - foreach sym $defs { append ::document_text "" } - append ::document_text "

\n" - - append ::document_text "$S(code)\n" - append ::document_text "$S(text)" - - append ::document_toc "
  • $S(heading)\n" - } - - set S(heading) "" - set S(code) "" - set S(text) "" -} - -# Process the command line arguments. -# -if {[llength $argv]!=1} { - puts stderr "Usage: $argv0 " - exit -1 -} -set zipvfsh [lindex $argv 0] -set fd [open $zipvfsh] - -# The $state variable may be set to one of: -# -# off -# on -# table -# deflist -# -set state off - -while {![eof $fd]} { - set line [gets $fd] - - switch -regexp -- $line { - {^/\* *$} { # no-op } - {^\*/ *$} { # no-op } - {^ *$} { # no-op } - - {^\*\* CAPI: .*$} { - put_section - regexp {.*CAPI: (.*)} $line -> S(heading) - set state on - } - {.*ENDOFAPI.*} { - put_section - set state off - } - - {^\*\* *[^ ]*: *$} { - regexp {( *)([^ ]*):} $line -> ws title - if {$state == "on"} { - if {[string length $ws] > 1} { - append S(text)
    • - set state table - } else { - append S(text)
    - set state deflist - } - } - if {$state == "table"} { - append S(text) "
    $title" - } - if {$state == "deflist"} { - append S(text)
    $title
    - } - } - - {^\*\* *$} { - if {$state == "on" || $state=="deflist"} { - append S(text)

    - } - } - - {^\*\*.*$} { - if {$state != "off"} { - regexp {^\*\*( *)(.*)} $line -> ws text - - if {$state == "table" && [string length $ws]<=1} { - append S(text)

    - set state on - } - if {$state == "deflist" && [string length $ws]<=1} { - append S(text)

    - set state on - } - - if {$state == "on" && [string length $ws]>=3} { - append S(text) 

    $text
    - } else { - append S(text) $text - } - - append S(text) "\n" - } - } - - default { - if {$state != "off"} { - if {$state == "table"} { append S(text) } - if {$state == "deflist"} { append S(text) } - set state on - } - if {$state == "on"} { - append S(code) $line - append S(code) "\n" - } - } - } -} -put_section - -close $fd - -puts $document_preamble -puts "

    LSM API Topics

    " -puts
      -puts $document_toc -puts
    -set s "display:block;float:left;width:35ex" -puts "

    All LSM API Functions

    " -foreach sym [lsort $document_functions] { - puts "$sym" -} -puts "
    " -puts "

    All LSM API Types

    " -foreach sym [lsort $document_types] { - puts "$sym" -} -puts "
    " -puts "

    All LSM API Constants

    " -foreach sym [lsort $document_defines] { - puts "$sym" -} -puts "
    " -puts $document_text Index: tool/mksqlite4c.tcl ================================================================== --- tool/mksqlite4c.tcl +++ tool/mksqlite4c.tcl @@ -92,20 +92,20 @@ # foreach hdr { hash.h hwtime.h keywordhash.h - kv.h lsm.h lsmInt.h mutex.h opcodes.h os.h parse.h sqlite4.h sqliteInt.h sqliteLimit.h + storage.h vdbe.h vdbeInt.h } { set available_hdr($hdr) 1 } @@ -158,11 +158,10 @@ } } elseif {![info exists seen_hdr($hdr)]} { set seen_hdr($hdr) 1 puts $out $line } else { - regsub {\*/} $line {} line puts $out "/* $line */" } } elseif {[regexp {^#ifdef __cplusplus} $line]} { puts $out "#if 0" } elseif {!$linemacros && [regexp {^#line} $line]} { @@ -250,11 +249,11 @@ lsm_str.c lsm_tree.c lsm_unix.c lsm_varint.c - kv.c + storage.c kvmem.c kvlsm.c rowset.c vdbemem.c @@ -284,13 +283,10 @@ select.c trigger.c update.c where.c - fts5.c - fts5func.c - parse.c tokenize.c complete.c ADDED tool/shell1.test Index: tool/shell1.test ================================================================== --- /dev/null +++ tool/shell1.test @@ -0,0 +1,720 @@ +# 2009 Nov 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the CLI shell tool. +# +# $Id: shell1.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ +# + +# Test plan: +# +# shell1-1.*: Basic command line option handling. +# shell1-2.*: Basic "dot" command token parsing. +# shell1-3.*: Basic test that "dot" command can be called. +# + +package require sqlite4 + +set CLI "./sqlite4" + +proc do_test {name cmd expected} { + puts -nonewline "$name ..." + set res [uplevel $cmd] + if {$res eq $expected} { + puts Ok + } else { + puts Error + puts " Got: $res" + puts " Expected: $expected" + exit + } +} + +proc execsql {sql} { + uplevel [list db eval $sql] +} + +proc catchsql {sql} { + set rc [catch {uplevel [list db eval $sql]} msg] + list $rc $msg +} + +proc catchcmd {db {cmd ""}} { + global CLI + set out [open cmds.txt w] + puts $out $cmd + close $out + set line "exec $CLI $db < cmds.txt" + set rc [catch { eval $line } msg] + list $rc $msg +} + +file delete -force test.db test.db.journal +sqlite4 db test.db + +#---------------------------------------------------------------------------- +# Test cases shell1-1.*: Basic command line option handling. +# + +# invalid option +do_test shell1-1.1.1 { + set res [catchcmd "-bad test.db" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: unknown option: -bad} $res] +} {1 1} +# error on extra options +do_test shell1-1.1.2 { + set res [catchcmd "-bad test.db \"select 3\" \"select 4\"" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: too many options: "select 4"} $res] +} {1 1} +# error on extra options +do_test shell1-1.1.3 { + set res [catchcmd "-bad FOO test.db BAD" ".quit"] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: too many options: "BAD"} $res] +} {1 1} + +# -help +do_test shell1-1.2.1 { + set res [catchcmd "-help test.db" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Usage} $res] \ + [regexp {\-init} $res] \ + [regexp {\-version} $res] +} {1 1 1 1} + +# -init filename read/process named file +do_test shell1-1.3.1 { + catchcmd "-init FOO test.db" "" +} {0 {}} +do_test shell1-1.3.2 { + set res [catchcmd "-init FOO test.db .quit BAD" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: too many options: "BAD"} $res] +} {1 1} + +# -echo print commands before execution +do_test shell1-1.4.1 { + catchcmd "-echo test.db" "" +} {0 {}} + +# -[no]header turn headers on or off +do_test shell1-1.5.1 { + catchcmd "-header test.db" "" +} {0 {}} +do_test shell1-1.5.2 { + catchcmd "-noheader test.db" "" +} {0 {}} + +# -bail stop after hitting an error +do_test shell1-1.6.1 { + catchcmd "-bail test.db" "" +} {0 {}} + +# -interactive force interactive I/O +do_test shell1-1.7.1 { + set res [catchcmd "-interactive test.db" ".quit"] + set rc [lindex $res 0] + list $rc \ + [regexp {SQLite version} $res] \ + [regexp {Enter SQL statements} $res] +} {0 1 1} + +# -batch force batch I/O +do_test shell1-1.8.1 { + catchcmd "-batch test.db" "" +} {0 {}} + +# -column set output mode to 'column' +do_test shell1-1.9.1 { + catchcmd "-column test.db" "" +} {0 {}} + +# -csv set output mode to 'csv' +do_test shell1-1.10.1 { + catchcmd "-csv test.db" "" +} {0 {}} + +# -html set output mode to HTML +do_test shell1-1.11.1 { + catchcmd "-html test.db" "" +} {0 {}} + +# -line set output mode to 'line' +do_test shell1-1.12.1 { + catchcmd "-line test.db" "" +} {0 {}} + +# -list set output mode to 'list' +do_test shell1-1.13.1 { + catchcmd "-list test.db" "" +} {0 {}} + +# -separator 'x' set output field separator (|) +do_test shell1-1.14.1 { + catchcmd "-separator 'x' test.db" "" +} {0 {}} +do_test shell1-1.14.2 { + catchcmd "-separator x test.db" "" +} {0 {}} +do_test shell1-1.14.3 { + set res [catchcmd "-separator" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: missing argument for option: -separator} $res] +} {1 1} + +# -stats print memory stats before each finalize +do_test shell1-1.14b.1 { + catchcmd "-stats test.db" "" +} {0 {}} + +# -nullvalue 'text' set text string for NULL values +do_test shell1-1.15.1 { + catchcmd "-nullvalue 'x' test.db" "" +} {0 {}} +do_test shell1-1.15.2 { + catchcmd "-nullvalue x test.db" "" +} {0 {}} +do_test shell1-1.15.3 { + set res [catchcmd "-nullvalue" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: missing argument for option: -nullvalue} $res] +} {1 1} + +# -version show SQLite version +do_test shell1-1.16.1 { + set x [catchcmd "-version test.db" ""] + regexp {0 \{3.\d.\d+ 20\d\d-[01]\d-\d\d \d\d:\d\d:\d\d [0-9a-f]+\}} $x +} 1 + +#---------------------------------------------------------------------------- +# Test cases shell1-2.*: Basic "dot" command token parsing. +# + +# check first token handling +do_test shell1-2.1.1 { + catchcmd "test.db" ".foo" +} {1 {Error: unknown command or invalid arguments: "foo". Enter ".help" for help}} +do_test shell1-2.1.2 { + catchcmd "test.db" ".\"foo OFF\"" +} {1 {Error: unknown command or invalid arguments: "foo OFF". Enter ".help" for help}} +do_test shell1-2.1.3 { + catchcmd "test.db" ".\'foo OFF\'" +} {1 {Error: unknown command or invalid arguments: "foo OFF". Enter ".help" for help}} + +# unbalanced quotes +do_test shell1-2.2.1 { + catchcmd "test.db" ".\"foo OFF" +} {1 {Error: unknown command or invalid arguments: "foo OFF". Enter ".help" for help}} +do_test shell1-2.2.2 { + catchcmd "test.db" ".\'foo OFF" +} {1 {Error: unknown command or invalid arguments: "foo OFF". Enter ".help" for help}} +do_test shell1-2.2.3 { + catchcmd "test.db" ".explain \"OFF" +} {0 {}} +do_test shell1-2.2.4 { + catchcmd "test.db" ".explain \'OFF" +} {0 {}} +do_test shell1-2.2.5 { + catchcmd "test.db" ".mode \"insert FOO" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} +do_test shell1-2.2.6 { + catchcmd "test.db" ".mode \'insert FOO" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} + +# check multiple tokens, and quoted tokens +do_test shell1-2.3.1 { + catchcmd "test.db" ".explain 1" +} {0 {}} +do_test shell1-2.3.2 { + catchcmd "test.db" ".explain on" +} {0 {}} +do_test shell1-2.3.3 { + catchcmd "test.db" ".explain \"1 2 3\"" +} {0 {}} +do_test shell1-2.3.4 { + catchcmd "test.db" ".explain \"OFF\"" +} {0 {}} +do_test shell1-2.3.5 { + catchcmd "test.db" ".\'explain\' \'OFF\'" +} {0 {}} +do_test shell1-2.3.6 { + catchcmd "test.db" ".explain \'OFF\'" +} {0 {}} +do_test shell1-2.3.7 { + catchcmd "test.db" ".\'explain\' \'OFF\'" +} {0 {}} + +# check quoted args are unquoted +do_test shell1-2.4.1 { + catchcmd "test.db" ".mode FOO" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} +do_test shell1-2.4.2 { + catchcmd "test.db" ".mode csv" +} {0 {}} +do_test shell1-2.4.2 { + catchcmd "test.db" ".mode \"csv\"" +} {0 {}} + + +#---------------------------------------------------------------------------- +# Test cases shell1-3.*: Basic test that "dot" command can be called. +# + +# .backup ?DB? FILE Backup DB (default "main") to FILE +do_test shell1-3.1.1 { + catchcmd "test.db" ".backup" +} {1 {Error: unknown command or invalid arguments: "backup". Enter ".help" for help}} +do_test shell1-3.1.2 { + catchcmd "test.db" ".backup FOO" +} {0 {}} +do_test shell1-3.1.3 { + catchcmd "test.db" ".backup FOO BAR" +} {1 {Error: unknown database FOO}} +do_test shell1-3.1.4 { + # too many arguments + catchcmd "test.db" ".backup FOO BAR BAD" +} {1 {Error: unknown command or invalid arguments: "backup". Enter ".help" for help}} + +# .bail ON|OFF Stop after hitting an error. Default OFF +do_test shell1-3.2.1 { + catchcmd "test.db" ".bail" +} {1 {Error: unknown command or invalid arguments: "bail". Enter ".help" for help}} +do_test shell1-3.2.2 { + catchcmd "test.db" ".bail ON" +} {0 {}} +do_test shell1-3.2.3 { + catchcmd "test.db" ".bail OFF" +} {0 {}} +do_test shell1-3.2.4 { + # too many arguments + catchcmd "test.db" ".bail OFF BAD" +} {1 {Error: unknown command or invalid arguments: "bail". Enter ".help" for help}} + +# .databases List names and files of attached databases +do_test shell1-3.3.1 { + set res [catchcmd "test.db" ".databases"] + regexp {0.*main.*test\.db} $res +} {1} +do_test shell1-3.3.2 { + # too many arguments + catchcmd "test.db" ".databases BAD" +} {1 {Error: unknown command or invalid arguments: "databases". Enter ".help" for help}} + +# .dump ?TABLE? ... Dump the database in an SQL text format +# If TABLE specified, only dump tables matching +# LIKE pattern TABLE. +do_test shell1-3.4.1 { + set res [catchcmd "test.db" ".dump"] + list [regexp {BEGIN TRANSACTION;} $res] \ + [regexp {COMMIT;} $res] +} {1 1} +do_test shell1-3.4.2 { + set res [catchcmd "test.db" ".dump FOO"] + list [regexp {BEGIN TRANSACTION;} $res] \ + [regexp {COMMIT;} $res] +} {1 1} +do_test shell1-3.4.3 { + # too many arguments + catchcmd "test.db" ".dump FOO BAD" +} {1 {Error: unknown command or invalid arguments: "dump". Enter ".help" for help}} + +# .echo ON|OFF Turn command echo on or off +do_test shell1-3.5.1 { + catchcmd "test.db" ".echo" +} {1 {Error: unknown command or invalid arguments: "echo". Enter ".help" for help}} +do_test shell1-3.5.2 { + catchcmd "test.db" ".echo ON" +} {0 {}} +do_test shell1-3.5.3 { + catchcmd "test.db" ".echo OFF" +} {0 {}} +do_test shell1-3.5.4 { + # too many arguments + catchcmd "test.db" ".echo OFF BAD" +} {1 {Error: unknown command or invalid arguments: "echo". Enter ".help" for help}} + +# .exit Exit this program +do_test shell1-3.6.1 { + catchcmd "test.db" ".exit" +} {0 {}} +do_test shell1-3.6.2 { + # too many arguments + catchcmd "test.db" ".exit BAD" +} {1 {Error: unknown command or invalid arguments: "exit". Enter ".help" for help}} + +# .explain ON|OFF Turn output mode suitable for EXPLAIN on or off. +do_test shell1-3.7.1 { + catchcmd "test.db" ".explain" + # explain is the exception to the booleans. without an option, it turns it on. +} {0 {}} +do_test shell1-3.7.2 { + catchcmd "test.db" ".explain ON" +} {0 {}} +do_test shell1-3.7.3 { + catchcmd "test.db" ".explain OFF" +} {0 {}} +do_test shell1-3.7.4 { + # too many arguments + catchcmd "test.db" ".explain OFF BAD" +} {1 {Error: unknown command or invalid arguments: "explain". Enter ".help" for help}} + + +# .header(s) ON|OFF Turn display of headers on or off +do_test shell1-3.9.1 { + catchcmd "test.db" ".header" +} {1 {Error: unknown command or invalid arguments: "header". Enter ".help" for help}} +do_test shell1-3.9.2 { + catchcmd "test.db" ".header ON" +} {0 {}} +do_test shell1-3.9.3 { + catchcmd "test.db" ".header OFF" +} {0 {}} +do_test shell1-3.9.4 { + # too many arguments + catchcmd "test.db" ".header OFF BAD" +} {1 {Error: unknown command or invalid arguments: "header". Enter ".help" for help}} + +do_test shell1-3.9.5 { + catchcmd "test.db" ".headers" +} {1 {Error: unknown command or invalid arguments: "headers". Enter ".help" for help}} +do_test shell1-3.9.6 { + catchcmd "test.db" ".headers ON" +} {0 {}} +do_test shell1-3.9.7 { + catchcmd "test.db" ".headers OFF" +} {0 {}} +do_test shell1-3.9.8 { + # too many arguments + catchcmd "test.db" ".headers OFF BAD" +} {1 {Error: unknown command or invalid arguments: "headers". Enter ".help" for help}} + +# .help Show this message +do_test shell1-3.10.1 { + set res [catchcmd "test.db" ".help"] + # look for a few of the possible help commands + list [regexp {.help} $res] \ + [regexp {.quit} $res] \ + [regexp {.show} $res] +} {1 1 1} +do_test shell1-3.10.2 { + # we allow .help to take extra args (it is help after all) + set res [catchcmd "test.db" ".help BAD"] + # look for a few of the possible help commands + list [regexp {.help} $res] \ + [regexp {.quit} $res] \ + [regexp {.show} $res] +} {1 1 1} + +# .import FILE TABLE Import data from FILE into TABLE +do_test shell1-3.11.1 { + catchcmd "test.db" ".import" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} +do_test shell1-3.11.2 { + catchcmd "test.db" ".import FOO" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} +do_test shell1-3.11.2 { + catchcmd "test.db" ".import FOO BAR" +} {1 {Error: no such table: BAR}} +do_test shell1-3.11.3 { + # too many arguments + catchcmd "test.db" ".import FOO BAR BAD" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} + +# .indices ?TABLE? Show names of all indices +# If TABLE specified, only show indices for tables +# matching LIKE pattern TABLE. +do_test shell1-3.12.1 { + catchcmd "test.db" ".indices" +} {0 {}} +do_test shell1-3.12.2 { + catchcmd "test.db" ".indices FOO" +} {0 {}} +do_test shell1-3.12.3 { + # too many arguments + catchcmd "test.db" ".indices FOO BAD" +} {1 {Error: unknown command or invalid arguments: "indices". Enter ".help" for help}} + +# .mode MODE ?TABLE? Set output mode where MODE is one of: +# csv Comma-separated values +# column Left-aligned columns. (See .width) +# html HTML code +# insert SQL insert statements for TABLE +# line One value per line +# list Values delimited by .separator string +# tabs Tab-separated values +# tcl TCL list elements +do_test shell1-3.13.1 { + catchcmd "test.db" ".mode" +} {1 {Error: unknown command or invalid arguments: "mode". Enter ".help" for help}} +do_test shell1-3.13.2 { + catchcmd "test.db" ".mode FOO" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} +do_test shell1-3.13.3 { + catchcmd "test.db" ".mode csv" +} {0 {}} +do_test shell1-3.13.4 { + catchcmd "test.db" ".mode column" +} {0 {}} +do_test shell1-3.13.5 { + catchcmd "test.db" ".mode html" +} {0 {}} +do_test shell1-3.13.6 { + catchcmd "test.db" ".mode insert" +} {0 {}} +do_test shell1-3.13.7 { + catchcmd "test.db" ".mode line" +} {0 {}} +do_test shell1-3.13.8 { + catchcmd "test.db" ".mode list" +} {0 {}} +do_test shell1-3.13.9 { + catchcmd "test.db" ".mode tabs" +} {0 {}} +do_test shell1-3.13.10 { + catchcmd "test.db" ".mode tcl" +} {0 {}} +do_test shell1-3.13.11 { + # too many arguments + catchcmd "test.db" ".mode tcl BAD" +} {1 {Error: invalid arguments: "BAD". Enter ".help" for help}} + +# don't allow partial mode type matches +do_test shell1-3.13.12 { + catchcmd "test.db" ".mode l" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} +do_test shell1-3.13.13 { + catchcmd "test.db" ".mode li" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} +do_test shell1-3.13.14 { + catchcmd "test.db" ".mode lin" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} + +# .nullvalue STRING Print STRING in place of NULL values +do_test shell1-3.14.1 { + catchcmd "test.db" ".nullvalue" +} {1 {Error: unknown command or invalid arguments: "nullvalue". Enter ".help" for help}} +do_test shell1-3.14.2 { + catchcmd "test.db" ".nullvalue FOO" +} {0 {}} +do_test shell1-3.14.3 { + # too many arguments + catchcmd "test.db" ".nullvalue FOO BAD" +} {1 {Error: unknown command or invalid arguments: "nullvalue". Enter ".help" for help}} + +# .output FILENAME Send output to FILENAME +do_test shell1-3.15.1 { + catchcmd "test.db" ".output" +} {1 {Error: unknown command or invalid arguments: "output". Enter ".help" for help}} +do_test shell1-3.15.2 { + catchcmd "test.db" ".output FOO" +} {0 {}} +do_test shell1-3.15.3 { + # too many arguments + catchcmd "test.db" ".output FOO BAD" +} {1 {Error: unknown command or invalid arguments: "output". Enter ".help" for help}} + +# .output stdout Send output to the screen +do_test shell1-3.16.1 { + catchcmd "test.db" ".output stdout" +} {0 {}} +do_test shell1-3.16.2 { + # too many arguments + catchcmd "test.db" ".output stdout BAD" +} {1 {Error: unknown command or invalid arguments: "output". Enter ".help" for help}} + +# .prompt MAIN CONTINUE Replace the standard prompts +do_test shell1-3.17.1 { + catchcmd "test.db" ".prompt" +} {1 {Error: unknown command or invalid arguments: "prompt". Enter ".help" for help}} +do_test shell1-3.17.2 { + catchcmd "test.db" ".prompt FOO" +} {0 {}} +do_test shell1-3.17.3 { + catchcmd "test.db" ".prompt FOO BAR" +} {0 {}} +do_test shell1-3.17.4 { + # too many arguments + catchcmd "test.db" ".prompt FOO BAR BAD" +} {1 {Error: unknown command or invalid arguments: "prompt". Enter ".help" for help}} + +# .quit Exit this program +do_test shell1-3.18.1 { + catchcmd "test.db" ".quit" +} {0 {}} +do_test shell1-3.18.2 { + # too many arguments + catchcmd "test.db" ".quit BAD" +} {1 {Error: unknown command or invalid arguments: "quit". Enter ".help" for help}} + +# .read FILENAME Execute SQL in FILENAME +do_test shell1-3.19.1 { + catchcmd "test.db" ".read" +} {1 {Error: unknown command or invalid arguments: "read". Enter ".help" for help}} +do_test shell1-3.19.2 { + file delete -force FOO + catchcmd "test.db" ".read FOO" +} {1 {Error: cannot open "FOO"}} +do_test shell1-3.19.3 { + # too many arguments + catchcmd "test.db" ".read FOO BAD" +} {1 {Error: unknown command or invalid arguments: "read". Enter ".help" for help}} + +# .restore ?DB? FILE Restore content of DB (default "main") from FILE +do_test shell1-3.20.1 { + catchcmd "test.db" ".restore" +} {1 {Error: unknown command or invalid arguments: "restore". Enter ".help" for help}} +do_test shell1-3.20.2 { + catchcmd "test.db" ".restore FOO" +} {0 {}} +do_test shell1-3.20.3 { + catchcmd "test.db" ".restore FOO BAR" +} {1 {Error: unknown database FOO}} +do_test shell1-3.20.4 { + # too many arguments + catchcmd "test.db" ".restore FOO BAR BAD" +} {1 {Error: unknown command or invalid arguments: "restore". Enter ".help" for help}} + +# .schema ?TABLE? Show the CREATE statements +# If TABLE specified, only show tables matching +# LIKE pattern TABLE. +do_test shell1-3.21.1 { + catchcmd "test.db" ".schema" +} {0 {}} +do_test shell1-3.21.2 { + catchcmd "test.db" ".schema FOO" +} {0 {}} +do_test shell1-3.21.3 { + # too many arguments + catchcmd "test.db" ".schema FOO BAD" +} {1 {Error: unknown command or invalid arguments: "schema". Enter ".help" for help}} + +# .separator STRING Change separator used by output mode and .import +do_test shell1-3.22.1 { + catchcmd "test.db" ".separator" +} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} +do_test shell1-3.22.2 { + catchcmd "test.db" ".separator FOO" +} {0 {}} +do_test shell1-3.22.3 { + # too many arguments + catchcmd "test.db" ".separator FOO BAD" +} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} + +# .show Show the current values for various settings +do_test shell1-3.23.1 { + set res [catchcmd "test.db" ".show"] + list [regexp {echo:} $res] \ + [regexp {explain:} $res] \ + [regexp {headers:} $res] \ + [regexp {mode:} $res] \ + [regexp {nullvalue:} $res] \ + [regexp {output:} $res] \ + [regexp {separator:} $res] \ + [regexp {stats:} $res] \ + [regexp {width:} $res] +} {1 1 1 1 1 1 1 1 1} +do_test shell1-3.23.2 { + # too many arguments + catchcmd "test.db" ".show BAD" +} {1 {Error: unknown command or invalid arguments: "show". Enter ".help" for help}} + +# .stats ON|OFF Turn stats on or off +do_test shell1-3.23b.1 { + catchcmd "test.db" ".stats" +} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} +do_test shell1-3.23b.2 { + catchcmd "test.db" ".stats ON" +} {0 {}} +do_test shell1-3.23b.3 { + catchcmd "test.db" ".stats OFF" +} {0 {}} +do_test shell1-3.23b.4 { + # too many arguments + catchcmd "test.db" ".stats OFF BAD" +} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} + +# .tables ?TABLE? List names of tables +# If TABLE specified, only list tables matching +# LIKE pattern TABLE. +do_test shell1-3.24.1 { + catchcmd "test.db" ".tables" +} {0 {}} +do_test shell1-3.24.2 { + catchcmd "test.db" ".tables FOO" +} {0 {}} +do_test shell1-3.24.3 { + # too many arguments + catchcmd "test.db" ".tables FOO BAD" +} {1 {Error: unknown command or invalid arguments: "tables". Enter ".help" for help}} + +# .timeout MS Try opening locked tables for MS milliseconds +do_test shell1-3.25.1 { + catchcmd "test.db" ".timeout" +} {1 {Error: unknown command or invalid arguments: "timeout". Enter ".help" for help}} +do_test shell1-3.25.2 { + catchcmd "test.db" ".timeout zzz" + # this should be treated the same as a '0' timeout +} {0 {}} +do_test shell1-3.25.3 { + catchcmd "test.db" ".timeout 1" +} {0 {}} +do_test shell1-3.25.4 { + # too many arguments + catchcmd "test.db" ".timeout 1 BAD" +} {1 {Error: unknown command or invalid arguments: "timeout". Enter ".help" for help}} + +# .width NUM NUM ... Set column widths for "column" mode +do_test shell1-3.26.1 { + catchcmd "test.db" ".width" +} {1 {Error: unknown command or invalid arguments: "width". Enter ".help" for help}} +do_test shell1-3.26.2 { + catchcmd "test.db" ".width xxx" + # this should be treated the same as a '0' width for col 1 +} {0 {}} +do_test shell1-3.26.3 { + catchcmd "test.db" ".width xxx yyy" + # this should be treated the same as a '0' width for col 1 and 2 +} {0 {}} +do_test shell1-3.26.4 { + catchcmd "test.db" ".width 1 1" + # this should be treated the same as a '1' width for col 1 and 2 +} {0 {}} + +# .timer ON|OFF Turn the CPU timer measurement on or off +do_test shell1-3.27.1 { + catchcmd "test.db" ".timer" +} {1 {Error: unknown command or invalid arguments: "timer". Enter ".help" for help}} +do_test shell1-3.27.2 { + catchcmd "test.db" ".timer ON" +} {0 {}} +do_test shell1-3.27.3 { + catchcmd "test.db" ".timer OFF" +} {0 {}} +do_test shell1-3.27.4 { + # too many arguments + catchcmd "test.db" ".timer OFF BAD" +} {1 {Error: unknown command or invalid arguments: "timer". Enter ".help" for help}} + +do_test shell1-3-28.1 { + catchcmd test.db \ + ".log stdout\nSELECT coalesce(sqlite_log(123,'hello'),'456');" +} "0 {(123) hello\n456}" + +puts "CLI tests completed successfully" ADDED tool/shell2.test Index: tool/shell2.test ================================================================== --- /dev/null +++ tool/shell2.test @@ -0,0 +1,222 @@ +# 2009 Nov 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the CLI shell tool. +# +# $Id: shell2.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ +# + +# Test plan: +# +# shell2-1.*: Misc. test of various tickets and reported errors. +# + +package require sqlite4 + +set CLI "./sqlite4" + +proc do_test {name cmd expected} { + puts -nonewline "$name ..." + set res [uplevel $cmd] + if {$res eq $expected} { + puts Ok + } else { + puts Error + puts " Got: $res" + puts " Expected: $expected" + exit + } +} + +proc execsql {sql} { + uplevel [list db eval $sql] +} + +proc catchsql {sql} { + set rc [catch {uplevel [list db eval $sql]} msg] + list $rc $msg +} + +proc catchcmd {db {cmd ""}} { + global CLI + set out [open cmds.txt w] + puts $out $cmd + close $out + set line "exec $CLI $db < cmds.txt" + set rc [catch { eval $line } msg] + list $rc $msg +} + +file delete -force test.db test.db.journal +sqlite4 db test.db + + +#---------------------------------------------------------------------------- +# shell2-1.*: Misc. test of various tickets and reported errors. +# + +# Batch mode not creating databases. +# Reported on mailing list by Ken Zalewski. +# Ticket [aeff892c57]. +do_test shell2-1.1.1 { + file delete -force foo.db + set rc [ catchcmd "-batch foo.db" "CREATE TABLE t1(a);" ] + set fexist [file exist foo.db] + list $rc $fexist +} {{0 {}} 1} + +# Shell silently ignores extra parameters. +# Ticket [f5cb008a65]. +do_test shell2-1.2.1 { + set rc [catch { eval exec $CLI \":memory:\" \"select 3\" \"select 4\" } msg] + list $rc \ + [regexp {Error: too many options: "select 4"} $msg] +} {1 1} + +# Test a problem reported on the mailing list. The shell was at one point +# returning the generic SQLITE_ERROR message ("SQL error or missing database") +# instead of the "too many levels..." message in the test below. +# +do_test shell2-1.3 { + catchcmd "-batch test.db" { + PRAGMA recursive_triggers = ON; + CREATE TABLE t5(a PRIMARY KEY, b, c); + INSERT INTO t5 VALUES(1, 2, 3); + CREATE TRIGGER au_tble AFTER UPDATE ON t5 BEGIN + UPDATE OR IGNORE t5 SET a = new.a, c = 10; + END; + + UPDATE OR REPLACE t5 SET a = 4 WHERE a = 1; + } +} {1 {Error: near line 9: too many levels of trigger recursion}} + + + +# Shell not echoing all commands with echo on. +# Ticket [eb620916be]. + +# Test with echo off +# NB. whitespace is important +do_test shell2-1.4.1 { + file delete -force foo.db + catchcmd "foo.db" {CREATE TABLE foo(a); +INSERT INTO foo(a) VALUES(1); +SELECT * FROM foo;} +} {0 1} + +# Test with echo on using command line option +# NB. whitespace is important +do_test shell2-1.4.2 { + file delete -force foo.db + catchcmd "-echo foo.db" {CREATE TABLE foo(a); +INSERT INTO foo(a) VALUES(1); +SELECT * FROM foo;} +} {0 {CREATE TABLE foo(a); +INSERT INTO foo(a) VALUES(1); +SELECT * FROM foo; +1}} + +# Test with echo on using dot command +# NB. whitespace is important +do_test shell2-1.4.3 { + file delete -force foo.db + catchcmd "foo.db" {.echo ON +CREATE TABLE foo(a); +INSERT INTO foo(a) VALUES(1); +SELECT * FROM foo;} +} {0 {CREATE TABLE foo(a); +INSERT INTO foo(a) VALUES(1); +SELECT * FROM foo; +1}} + +# Test with echo on using dot command and +# turning off mid- processing. +# NB. whitespace is important +do_test shell2-1.4.4 { + file delete -force foo.db + catchcmd "foo.db" {.echo ON +CREATE TABLE foo(a); +.echo OFF +INSERT INTO foo(a) VALUES(1); +SELECT * FROM foo;} +} {0 {CREATE TABLE foo(a); +.echo OFF +1}} + +# Test with echo on using dot command and +# multiple commands per line. +# NB. whitespace is important +do_test shell2-1.4.5 { + file delete -force foo.db + catchcmd "foo.db" {.echo ON +CREATE TABLE foo1(a); +INSERT INTO foo1(a) VALUES(1); +CREATE TABLE foo2(b); +INSERT INTO foo2(b) VALUES(1); +SELECT * FROM foo1; SELECT * FROM foo2; +INSERT INTO foo1(a) VALUES(2); INSERT INTO foo2(b) VALUES(2); +SELECT * FROM foo1; SELECT * FROM foo2; +} +} {0 {CREATE TABLE foo1(a); +INSERT INTO foo1(a) VALUES(1); +CREATE TABLE foo2(b); +INSERT INTO foo2(b) VALUES(1); +SELECT * FROM foo1; +1 +SELECT * FROM foo2; +1 +INSERT INTO foo1(a) VALUES(2); +INSERT INTO foo2(b) VALUES(2); +SELECT * FROM foo1; +1 +2 +SELECT * FROM foo2; +1 +2}} + +# Test with echo on and headers on using dot command and +# multiple commands per line. +# NB. whitespace is important +do_test shell2-1.4.6 { + file delete -force foo.db + catchcmd "foo.db" {.echo ON +.headers ON +CREATE TABLE foo1(a); +INSERT INTO foo1(a) VALUES(1); +CREATE TABLE foo2(b); +INSERT INTO foo2(b) VALUES(1); +SELECT * FROM foo1; SELECT * FROM foo2; +INSERT INTO foo1(a) VALUES(2); INSERT INTO foo2(b) VALUES(2); +SELECT * FROM foo1; SELECT * FROM foo2; +} +} {0 {.headers ON +CREATE TABLE foo1(a); +INSERT INTO foo1(a) VALUES(1); +CREATE TABLE foo2(b); +INSERT INTO foo2(b) VALUES(1); +SELECT * FROM foo1; +a +1 +SELECT * FROM foo2; +b +1 +INSERT INTO foo1(a) VALUES(2); +INSERT INTO foo2(b) VALUES(2); +SELECT * FROM foo1; +a +1 +2 +SELECT * FROM foo2; +b +1 +2}} + +puts "CLI tests completed successfully" ADDED tool/shell3.test Index: tool/shell3.test ================================================================== --- /dev/null +++ tool/shell3.test @@ -0,0 +1,124 @@ +# 2009 Dec 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the CLI shell tool. +# +# $Id: shell2.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ +# + +# Test plan: +# +# shell3-1.*: Basic tests for running SQL statments from command line. +# shell3-2.*: Basic tests for running SQL file from command line. +# + +package require sqlite4 + +set CLI "./sqlite4" + +proc do_test {name cmd expected} { + puts -nonewline "$name ..." + set res [uplevel $cmd] + if {$res eq $expected} { + puts Ok + } else { + puts Error + puts " Got: $res" + puts " Expected: $expected" + exit + } +} + +proc execsql {sql} { + uplevel [list db eval $sql] +} + +proc catchsql {sql} { + set rc [catch {uplevel [list db eval $sql]} msg] + list $rc $msg +} + +proc catchcmd {db {cmd ""}} { + global CLI + set out [open cmds.txt w] + puts $out $cmd + close $out + set line "exec $CLI $db < cmds.txt" + set rc [catch { eval $line } msg] + list $rc $msg +} + +file delete -force test.db test.db.journal +sqlite4 db test.db + + +#---------------------------------------------------------------------------- +# shell3-1.*: Basic tests for running SQL statments from command line. +# + +# Run SQL statement from command line +do_test shell3-1.1 { + file delete -force foo.db + set rc [ catchcmd "foo.db \"CREATE TABLE t1(a);\"" ] + set fexist [file exist foo.db] + list $rc $fexist +} {{0 {}} 1} +do_test shell3-1.2 { + catchcmd "foo.db" ".tables" +} {0 t1} +do_test shell3-1.3 { + catchcmd "foo.db \"DROP TABLE t1;\"" +} {0 {}} +do_test shell3-1.4 { + catchcmd "foo.db" ".tables" +} {0 {}} +do_test shell3-1.5 { + catchcmd "foo.db \"CREATE TABLE t1(a); DROP TABLE t1;\"" +} {0 {}} +do_test shell3-1.6 { + catchcmd "foo.db" ".tables" +} {0 {}} +do_test shell3-1.7 { + catchcmd "foo.db \"CREATE TABLE\"" +} {1 {Error: near "TABLE": syntax error}} + +#---------------------------------------------------------------------------- +# shell3-2.*: Basic tests for running SQL file from command line. +# + +# Run SQL file from command line +do_test shell3-2.1 { + file delete -force foo.db + set rc [ catchcmd "foo.db" "CREATE TABLE t1(a);" ] + set fexist [file exist foo.db] + list $rc $fexist +} {{0 {}} 1} +do_test shell3-2.2 { + catchcmd "foo.db" ".tables" +} {0 t1} +do_test shell3-2.3 { + catchcmd "foo.db" "DROP TABLE t1;" +} {0 {}} +do_test shell3-2.4 { + catchcmd "foo.db" ".tables" +} {0 {}} +do_test shell3-2.5 { + catchcmd "foo.db" "CREATE TABLE t1(a); DROP TABLE t1;" +} {0 {}} +do_test shell3-2.6 { + catchcmd "foo.db" ".tables" +} {0 {}} +do_test shell3-2.7 { + catchcmd "foo.db" "CREATE TABLE" +} {1 {Error: incomplete SQL: CREATE TABLE}} + + +puts "CLI tests completed successfully" ADDED tool/shell4.test Index: tool/shell4.test ================================================================== --- /dev/null +++ tool/shell4.test @@ -0,0 +1,129 @@ +# 2010 July 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the CLI shell tool. +# These tests are specific to the .stats command. +# +# $Id: shell4.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ +# + +# Test plan: +# +# shell4-1.*: Basic tests specific to the "stats" command. +# + +set CLI "./sqlite4" + +proc do_test {name cmd expected} { + puts -nonewline "$name ..." + set res [uplevel $cmd] + if {$res eq $expected} { + puts Ok + } else { + puts Error + puts " Got: $res" + puts " Expected: $expected" + exit + } +} + +proc catchcmd {db {cmd ""}} { + global CLI + set out [open cmds.txt w] + puts $out $cmd + close $out + set line "exec $CLI $db < cmds.txt" + set rc [catch { eval $line } msg] + list $rc $msg +} + +file delete -force test.db test.db.journal + +#---------------------------------------------------------------------------- +# Test cases shell4-1.*: Tests specific to the "stats" command. +# + +# should default to off +do_test shell4-1.1.1 { + set res [catchcmd "test.db" ".show"] + list [regexp {stats: off} $res] +} {1} + +do_test shell4-1.1.2 { + set res [catchcmd "test.db" ".show"] + list [regexp {stats: on} $res] +} {0} + +# -stats should turn it on +do_test shell4-1.2.1 { + set res [catchcmd "-stats test.db" ".show"] + list [regexp {stats: on} $res] +} {1} + +do_test shell4-1.2.2 { + set res [catchcmd "-stats test.db" ".show"] + list [regexp {stats: off} $res] +} {0} + +# .stats ON|OFF Turn stats on or off +do_test shell4-1.3.1 { + catchcmd "test.db" ".stats" +} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} +do_test shell4-1.3.2 { + catchcmd "test.db" ".stats ON" +} {0 {}} +do_test shell4-1.3.3 { + catchcmd "test.db" ".stats OFF" +} {0 {}} +do_test shell4-1.3.4 { + # too many arguments + catchcmd "test.db" ".stats OFF BAD" +} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} + +# NB. whitespace is important +do_test shell4-1.4.1 { + set res [catchcmd "test.db" {.show}] + list [regexp {stats: off} $res] +} {1} + +do_test shell4-1.4.2 { + set res [catchcmd "test.db" {.stats ON +.show +}] + list [regexp {stats: on} $res] +} {1} + +do_test shell4-1.4.3 { + set res [catchcmd "test.db" {.stats OFF +.show +}] + list [regexp {stats: off} $res] +} {1} + +# make sure stats not present when off +do_test shell4-1.5.1 { + set res [catchcmd "test.db" {SELECT 1;}] + list [regexp {Memory Used} $res] \ + [regexp {Heap Usage} $res] \ + [regexp {Autoindex Inserts} $res] +} {0 0 0} + +# make sure stats are present when on +do_test shell4-1.5.2 { + set res [catchcmd "test.db" {.stats ON +SELECT 1; +}] + list [regexp {Memory Used} $res] \ + [regexp {Heap Usage} $res] \ + [regexp {Autoindex Inserts} $res] +} {1 1 1} + +puts "CLI tests completed successfully" ADDED tool/shell5.test Index: tool/shell5.test ================================================================== --- /dev/null +++ tool/shell5.test @@ -0,0 +1,243 @@ +# 2010 August 4 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the CLI shell tool. +# These tests are specific to the .import command. +# +# $Id: shell5.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ +# + +# Test plan: +# +# shell5-1.*: Basic tests specific to the ".import" command. +# + +set CLI "./sqlite4" + +proc do_test {name cmd expected} { + puts -nonewline "$name ..." + set res [uplevel $cmd] + if {$res eq $expected} { + puts Ok + } else { + puts Error + puts " Got: $res" + puts " Expected: $expected" + exit + } +} + +proc catchcmd {db {cmd ""}} { + global CLI + set out [open cmds.txt w] + puts $out $cmd + close $out + set line "exec $CLI $db < cmds.txt" + set rc [catch { eval $line } msg] + list $rc $msg +} + +file delete -force test.db test.db.journal + +#---------------------------------------------------------------------------- +# Test cases shell5-1.*: Basic handling of the .import and .separator commands. +# + +# .import FILE TABLE Import data from FILE into TABLE +do_test shell5-1.1.1 { + catchcmd "test.db" ".import" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} +do_test shell5-1.1.2 { + catchcmd "test.db" ".import FOO" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} +do_test shell5-1.1.2 { + catchcmd "test.db" ".import FOO BAR" +} {1 {Error: no such table: BAR}} +do_test shell5-1.1.3 { + # too many arguments + catchcmd "test.db" ".import FOO BAR BAD" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} + +# .separator STRING Change separator used by output mode and .import +do_test shell1-1.2.1 { + catchcmd "test.db" ".separator" +} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} +do_test shell1-1.2.2 { + catchcmd "test.db" ".separator FOO" +} {0 {}} +do_test shell1-1.2.3 { + # too many arguments + catchcmd "test.db" ".separator FOO BAD" +} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} + +# separator should default to "|" +do_test shell5-1.3.1 { + set res [catchcmd "test.db" ".show"] + list [regexp {separator: \"\|\"} $res] +} {1} + +# set separator to different value. +# check that .show reports new value +do_test shell5-1.3.2 { + set res [catchcmd "test.db" {.separator , +.show}] + list [regexp {separator: \",\"} $res] +} {1} + +# import file doesn't exist +do_test shell5-1.4.1 { + file delete -force FOO + set res [catchcmd "test.db" {CREATE TABLE t1(a, b); +.import FOO t1}] +} {1 {Error: cannot open "FOO"}} + +# empty import file +do_test shell5-1.4.2 { + file delete -force shell5.csv + set in [open shell5.csv w] + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 0} + +# import file with 1 row, 1 column (expecting 2 cols) +do_test shell5-1.4.3 { + set in [open shell5.csv w] + puts $in "1" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1}] +} {1 {Error: shell5.csv line 1: expected 2 columns of data but found 1}} + +# import file with 1 row, 3 columns (expecting 2 cols) +do_test shell5-1.4.4 { + set in [open shell5.csv w] + puts $in "1|2|3" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1}] +} {1 {Error: shell5.csv line 1: expected 2 columns of data but found 3}} + +# import file with 1 row, 2 columns +do_test shell5-1.4.5 { + set in [open shell5.csv w] + puts $in "1|2" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 1} + +# import file with 2 rows, 2 columns +# note we end up with 3 rows because of the 1 row +# imported above. +do_test shell5-1.4.6 { + set in [open shell5.csv w] + puts $in "2|3" + puts $in "3|4" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 3} + +# import file with 1 row, 2 columns, using a comma +do_test shell5-1.4.7 { + set in [open shell5.csv w] + puts $in "4,5" + close $in + set res [catchcmd "test.db" {.separator , +.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 4} + +# import file with 1 row, 2 columns, text data +do_test shell5-1.4.8.1 { + set in [open shell5.csv w] + puts $in "5|Now is the time for all good men to come to the aid of their country." + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 5} + +do_test shell5-1.4.8.2 { + catchcmd "test.db" {SELECT b FROM t1 WHERE a='5';} +} {0 {Now is the time for all good men to come to the aid of their country.}} + +# import file with 1 row, 2 columns, quoted text data +# note that currently sqlite doesn't support quoted fields, and +# imports the entire field, quotes and all. +do_test shell5-1.4.9.1 { + set in [open shell5.csv w] + puts $in "6|'Now is the time for all good men to come to the aid of their country.'" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 6} + +do_test shell5-1.4.9.2 { + catchcmd "test.db" {SELECT b FROM t1 WHERE a='6';} +} {0 {'Now is the time for all good men to come to the aid of their country.'}} + +# import file with 1 row, 2 columns, quoted text data +do_test shell5-1.4.10.1 { + set in [open shell5.csv w] + puts $in "7|\"Now is the time for all good men to come to the aid of their country.\"" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 7} + +do_test shell5-1.4.10.2 { + catchcmd "test.db" {SELECT b FROM t1 WHERE a='7';} +} {0 {Now is the time for all good men to come to the aid of their country.}} + +# check importing very long field +do_test shell5-1.5.1 { + set str [string repeat X 999] + set in [open shell5.csv w] + puts $in "8|$str" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT length(b) FROM t1 WHERE a='8';}] +} {0 999} + +# try importing into a table with a large number of columns. +# This is limited by SQLITE_MAX_VARIABLE_NUMBER, which defaults to 999. +set cols 999 +do_test shell5-1.6.1 { + set sql {CREATE TABLE t2(} + set data {} + for {set i 1} {$i<$cols} {incr i} { + append sql "c$i," + append data "$i|" + } + append sql "c$cols);" + append data "$cols" + catchcmd "test.db" $sql + set in [open shell5.csv w] + puts $in $data + close $in + set res [catchcmd "test.db" {.import shell5.csv t2 +SELECT COUNT(*) FROM t2;}] +} {0 1} + +# try importing a large number of rows +set rows 999999 +do_test shell5-1.7.1 { + set in [open shell5.csv w] + for {set i 1} {$i<=$rows} {incr i} { + puts $in $i + } + close $in + set res [catchcmd "test.db" {CREATE TABLE t3(a); +.import shell5.csv t3 +SELECT COUNT(*) FROM t3;}] +} [list 0 $rows] + + +puts "CLI tests completed successfully" Index: www/index.wiki ================================================================== --- www/index.wiki +++ www/index.wiki @@ -6,8 +6,5 @@ * [./data_encoding.wiki | The Data Encoding] * [./key_encoding.wiki | The Key Encoding] * [./decimal.wiki | Internal representation of numeric values] * [./porting.wiki | Porting an app from SQLite3 to SQLite4] * [./storage.wiki | How to create a new storage engine] - * [./lsmusr.wiki | LSM User Manual] - * [./lsmperf.wiki | LSM Benchmarks] - * [./lsmapi.wiki | LSM API Reference] Index: www/key_encoding.wiki ================================================================== --- www/key_encoding.wiki +++ www/key_encoding.wiki @@ -90,11 +90,11 @@ byte of 0x23. Every other numeric value encoding begins with a smaller byte, ensuring that positive infinity always sorts last among numeric values. 0x0d is also smaller than 0x0e, the initial byte of a text value, ensuring that every numeric value sorts before every text value. -If the numeric value is exactly zero then it is encoded as a single +If the numeric value is exactly zero then then is encoded is a single byte of 0x15. Finite negative values will have initial bytes of 0x08 through 0x14 and finite positive values will have initial bytes of 0x16 through 0x22. For all values, we compute a mantissa M and an exponent E. The mantissa Index: www/lsm.wiki ================================================================== --- www/lsm.wiki +++ www/lsm.wiki @@ -430,12 +430,12 @@

  • Load the current tree-header from shared-memory.

  • Load the current snapshot from shared-memory.

    Steps 1 and 2 are similar. In both cases, there are two copies of the data structure being read in shared memory. No lock is held to prevent -another client updating them while the read is taking place. Updaters use the -following pattern: +another client updating them while the read is taking place using the following +pattern:

    1. Update copy 2.
    2. Invoke xShmBarrier().
    3. Update copy 1. DELETED www/lsmapi.wiki Index: www/lsmapi.wiki ================================================================== --- www/lsmapi.wiki +++ /dev/null @@ -1,546 +0,0 @@ - -LSM API Reference - -
      - -

      -This page contains the LSM API Reference Manual. It is intended to complement -the LSM User Manual. - -

      LSM API Topics

      -
        -
      1. Database Runtime Environment -
      2. LSM Error Codes -
      3. Creating and Destroying Database Connection Handles -
      4. Connecting to a Database -
      5. Obtaining pointers to database environments -
      6. Configuring a database connection. -
      7. Compression and/or Encryption Hooks -
      8. Allocating and Freeing Memory -
      9. Querying a Connection For Operational Data -
      10. Opening and Closing Write Transactions -
      11. Writing to a Database -
      12. Explicit Database Work and Checkpointing -
      13. Opening and Closing Database Cursors -
      14. Positioning Database Cursors -
      15. Extracting Data From Database Cursors -
      16. Change these!! - -
      -

      All LSM API Functions

      -lsm_begin -lsm_checkpoint -lsm_close -lsm_commit -lsm_config -lsm_config_log -lsm_config_work_hook -lsm_csr_close -lsm_csr_cmp -lsm_csr_first -lsm_csr_key -lsm_csr_last -lsm_csr_next -lsm_csr_open -lsm_csr_prev -lsm_csr_seek -lsm_csr_valid -lsm_csr_value -lsm_delete -lsm_delete_range -lsm_flush -lsm_free -lsm_info -lsm_insert -lsm_new -lsm_open -lsm_rollback -lsm_work -
      -

      All LSM API Types

      -lsm_compress -lsm_compress -lsm_env -
      -

      All LSM API Constants

      -LSM_BUSY -LSM_CANTOPEN -LSM_CONFIG_AUTOCHECKPOINT -LSM_CONFIG_AUTOFLUSH -LSM_CONFIG_AUTOMERGE -LSM_CONFIG_AUTOWORK -LSM_CONFIG_BLOCK_SIZE -LSM_CONFIG_GET_COMPRESSION -LSM_CONFIG_MAX_FREELIST -LSM_CONFIG_MMAP -LSM_CONFIG_MULTIPLE_PROCESSES -LSM_CONFIG_PAGE_SIZE -LSM_CONFIG_SAFETY -LSM_CONFIG_SET_COMPRESSION -LSM_CONFIG_SET_COMPRESSION_FACTORY -LSM_CONFIG_USE_LOG -LSM_CORRUPT -LSM_ERROR -LSM_FULL -LSM_INFO_ARRAY_PAGES -LSM_INFO_ARRAY_STRUCTURE -LSM_INFO_CHECKPOINT_SIZE -LSM_INFO_DB_STRUCTURE -LSM_INFO_FREELIST -LSM_INFO_FREELIST_SIZE -LSM_INFO_LOG_STRUCTURE -LSM_INFO_NREAD -LSM_INFO_NWRITE -LSM_INFO_PAGE_ASCII_DUMP -LSM_INFO_PAGE_HEX_DUMP -LSM_INFO_TREE_SIZE -LSM_IOERR -LSM_MISUSE -LSM_MUTEX_GLOBAL -LSM_MUTEX_HEAP -LSM_NOMEM -LSM_OK -LSM_PROTOCOL -LSM_SAFETY_FULL -LSM_SAFETY_NORMAL -LSM_SAFETY_OFF -LSM_SEEK_EQ -LSM_SEEK_GE -LSM_SEEK_LE -LSM_SEEK_LEFAST -
      -

      Database Runtime Environment

      -struct lsm_env { - int nByte; /* Size of this structure in bytes */ - int iVersion; /* Version number of this structure (1) */ - /****** file i/o ***********************************************/ - void *pVfsCtx; - int (*xFullpath)(lsm_env*, const char *, char *, int *); - int (*xOpen)(lsm_env*, const char *, lsm_file **); - int (*xRead)(lsm_file *, lsm_i64, void *, int); - int (*xWrite)(lsm_file *, lsm_i64, void *, int); - int (*xTruncate)(lsm_file *, lsm_i64); - int (*xSync)(lsm_file *); - int (*xSectorSize)(lsm_file *); - int (*xRemap)(lsm_file *, lsm_i64, void **, lsm_i64*); - int (*xFileid)(lsm_file *, void *pBuf, int *pnBuf); - int (*xClose)(lsm_file *); - int (*xUnlink)(lsm_env*, const char *); - int (*xLock)(lsm_file*, int, int); - int (*xShmMap)(lsm_file*, int, int, void **); - void (*xShmBarrier)(void); - int (*xShmUnmap)(lsm_file*, int); - /****** memory allocation ****************************************/ - void *pMemCtx; - void *(*xMalloc)(lsm_env*, int); /* malloc(3) function */ - void *(*xRealloc)(lsm_env*, void *, int); /* realloc(3) function */ - void (*xFree)(lsm_env*, void *); /* free(3) function */ - sqlite4_size_t (*xSize)(lsm_env*, void *); /* xSize function */ - /****** mutexes ****************************************************/ - void *pMutexCtx; - int (*xMutexStatic)(lsm_env*,int,lsm_mutex**); /* Obtain a static mutex */ - int (*xMutexNew)(lsm_env*, lsm_mutex**); /* Get a new dynamic mutex */ - void (*xMutexDel)(lsm_mutex *); /* Delete an allocated mutex */ - void (*xMutexEnter)(lsm_mutex *); /* Grab a mutex */ - int (*xMutexTry)(lsm_mutex *); /* Attempt to obtain a mutex */ - void (*xMutexLeave)(lsm_mutex *); /* Leave a mutex */ - int (*xMutexHeld)(lsm_mutex *); /* Return true if mutex is held */ - int (*xMutexNotHeld)(lsm_mutex *); /* Return true if mutex not held */ - /****** other ****************************************************/ - int (*xSleep)(lsm_env*, int microseconds); - /* New fields may be added in future releases, in which case the - ** iVersion value will increase. */ -}; -#define LSM_MUTEX_GLOBAL 1 -#define LSM_MUTEX_HEAP 2 - -

      Run-time environment used by LSM -Values that may be passed as the second argument to xMutexStatic. -

      LSM Error Codes

      -#define LSM_OK 0 -#define LSM_ERROR 1 -#define LSM_BUSY 5 -#define LSM_NOMEM 7 -#define LSM_IOERR 10 -#define LSM_CORRUPT 11 -#define LSM_FULL 13 -#define LSM_CANTOPEN 14 -#define LSM_PROTOCOL 15 -#define LSM_MISUSE 21 - -

      Creating and Destroying Database Connection Handles

      -int lsm_new(lsm_env*, lsm_db **ppDb); -int lsm_close(lsm_db *pDb); - -

      Open and close a database connection handle. -

      Connecting to a Database

      -int lsm_open(lsm_db *pDb, const char *zFilename); - -

      Obtaining pointers to database environments

      -lsm_env *lsm_get_env(lsm_db *pDb); -lsm_env *lsm_default_env(void); - -

      Return a pointer to the environment used by the database connection -passed as the first argument. Assuming the argument is valid, this -function always returns a valid environment pointer - it cannot fail. -The lsm_default_env() function returns a pointer to the default LSM -environment for the current platform. -

      Configuring a database connection.

      -int lsm_config(lsm_db *, int, ...); -#define LSM_CONFIG_AUTOFLUSH 1 -#define LSM_CONFIG_PAGE_SIZE 2 -#define LSM_CONFIG_SAFETY 3 -#define LSM_CONFIG_BLOCK_SIZE 4 -#define LSM_CONFIG_AUTOWORK 5 -#define LSM_CONFIG_MMAP 7 -#define LSM_CONFIG_USE_LOG 8 -#define LSM_CONFIG_AUTOMERGE 9 -#define LSM_CONFIG_MAX_FREELIST 10 -#define LSM_CONFIG_MULTIPLE_PROCESSES 11 -#define LSM_CONFIG_AUTOCHECKPOINT 12 -#define LSM_CONFIG_SET_COMPRESSION 13 -#define LSM_CONFIG_GET_COMPRESSION 14 -#define LSM_CONFIG_SET_COMPRESSION_FACTORY 15 -#define LSM_SAFETY_OFF 0 -#define LSM_SAFETY_NORMAL 1 -#define LSM_SAFETY_FULL 2 - -

      The lsm_config() function is used to configure a database connection. -The following values may be passed as the second argument to lsm_config(). -

      LSM_CONFIG_AUTOFLUSH
      A read/write integer parameter. -

      This value determines the amount of data allowed to accumulate in a -live in-memory tree before it is marked as old. After committing a -transaction, a connection checks if the size of the live in-memory tree, -including data structure overhead, is greater than the value of this -option in KB. If it is, and there is not already an old in-memory tree, -the live in-memory tree is marked as old. -

      The maximum allowable value is 1048576 (1GB). There is no minimum -value. If this parameter is set to zero, then an attempt is made to -mark the live in-memory tree as old after each transaction is committed. -

      The default value is 1024 (1MB). -

      LSM_CONFIG_PAGE_SIZE
      A read/write integer parameter. This parameter may only be set before -lsm_open() has been called. -

      LSM_CONFIG_BLOCK_SIZE
      A read/write integer parameter. -

      This parameter may only be set before lsm_open() has been called. It -must be set to a power of two between 64 and 65536, inclusive (block -sizes between 64KB and 64MB). -

      If the connection creates a new database, the block size of the new -database is set to the value of this option in KB. After lsm_open() -has been called, querying this parameter returns the actual block -size of the opened database. -

      The default value is 1024 (1MB blocks). -

      LSM_CONFIG_SAFETY
      A read/write integer parameter. Valid values are 0, 1 (the default) -and 2. This parameter determines how robust the database is in the -face of a system crash (e.g. a power failure or operating system -crash). As follows: -

      0 (off): No robustness. A system crash may corrupt the database. -

      1 (normal): Some robustness. A system crash may not corrupt the -database file, but recently committed transactions may -be lost following recovery. -

      2 (full): Full robustness. A system crash may not corrupt the -database file. Following recovery the database file -contains all successfully committed transactions. -

      LSM_CONFIG_AUTOWORK
      A read/write integer parameter. -

      LSM_CONFIG_AUTOCHECKPOINT
      A read/write integer parameter. -

      If this option is set to non-zero value N, then a checkpoint is -automatically attempted after each N KB of data have been written to -the database file. -

      The amount of uncheckpointed data already written to the database file -is a global parameter. After performing database work (writing to the -database file), the process checks if the total amount of uncheckpointed -data exceeds the value of this paramter. If so, a checkpoint is performed. -This means that this option may cause the connection to perform a -checkpoint even if the current connection has itself written very little -data into the database file. -

      The default value is 2048 (checkpoint every 2MB). -

      LSM_CONFIG_MMAP
      A read/write integer parameter. True to use mmap() to access the -database file. False otherwise. -

      LSM_CONFIG_USE_LOG
      A read/write boolean parameter. True (the default) to use the log -file normally. False otherwise. -

      LSM_CONFIG_AUTOMERGE
      A read/write integer parameter. The minimum number of segments to -merge together at a time. Default value 4. -

      LSM_CONFIG_MAX_FREELIST
      A read/write integer parameter. The maximum number of free-list -entries that are stored in a database checkpoint (the others are -stored elsewhere in the database). -

      There is no reason for an application to configure or query this -parameter. It is only present because configuring a small value -makes certain parts of the lsm code easier to test. -

      LSM_CONFIG_MULTIPLE_PROCESSES
      A read/write boolean parameter. This parameter may only be set before -lsm_open() has been called. If true, the library uses shared-memory -and posix advisory locks to co-ordinate access by clients from within -multiple processes. Otherwise, if false, all database clients must be -located in the same process. The default value is true. -

      LSM_CONFIG_SET_COMPRESSION
      Set the compression methods used to compress and decompress database -content. The argument to this option should be a pointer to a structure -of type lsm_compress. The lsm_config() method takes a copy of the -structures contents. -

      This option may only be used before lsm_open() is called. Invoking it -after lsm_open() has been called results in an LSM_MISUSE error. -

      LSM_CONFIG_GET_COMPRESSION
      Query the compression methods used to compress and decompress database -content. -

      LSM_CONFIG_SET_COMPRESSION_FACTORY
      Configure a factory method to be invoked in case of an LSM_MISMATCH -error. -

      Compression and/or Encryption Hooks

      -struct lsm_compress { - void *pCtx; - unsigned int iId; - int (*xBound)(void *, int nSrc); - int (*xCompress)(void *, char *, int *, const char *, int); - int (*xUncompress)(void *, char *, int *, const char *, int); - void (*xFree)(void *pCtx); -}; -struct lsm_compress_factory { - void *pCtx; - int (*xFactory)(void *, lsm_db *, u32); - void (*xFree)(void *pCtx); -}; - -

      Allocating and Freeing Memory

      -void *lsm_malloc(lsm_env*, size_t); -void *lsm_realloc(lsm_env*, void *, size_t); -void lsm_free(lsm_env*, void *); - -

      Invoke the memory allocation functions that belong to environment -pEnv. Or the system defaults if no memory allocation functions have -been registered. -

      Querying a Connection For Operational Data

      -int lsm_info(lsm_db *, int, ...); -#define LSM_INFO_NWRITE 1 -#define LSM_INFO_NREAD 2 -#define LSM_INFO_DB_STRUCTURE 3 -#define LSM_INFO_LOG_STRUCTURE 4 -#define LSM_INFO_ARRAY_STRUCTURE 5 -#define LSM_INFO_PAGE_ASCII_DUMP 6 -#define LSM_INFO_PAGE_HEX_DUMP 7 -#define LSM_INFO_FREELIST 8 -#define LSM_INFO_ARRAY_PAGES 9 -#define LSM_INFO_CHECKPOINT_SIZE 10 -#define LSM_INFO_TREE_SIZE 11 -#define LSM_INFO_FREELIST_SIZE 12 - -

      Query a database connection for operational statistics or data. -The following values may be passed as the second argument to lsm_info(). -

      LSM_INFO_NWRITE
      The third parameter should be of type (int *). The location pointed -to by the third parameter is set to the number of 4KB pages written to -the database file during the lifetime of this connection. -

      LSM_INFO_NREAD
      The third parameter should be of type (int *). The location pointed -to by the third parameter is set to the number of 4KB pages read from -the database file during the lifetime of this connection. -

      LSM_INFO_DB_STRUCTURE
      The third argument should be of type (char **). The location pointed -to is populated with a pointer to a nul-terminated string containing -the string representation of a Tcl data-structure reflecting the -current structure of the database file. Specifically, the current state -of the worker snapshot. The returned string should be eventually freed -by the caller using lsm_free(). -

      The returned list contains one element for each level in the database, -in order from most to least recent. Each element contains a -single element for each segment comprising the corresponding level, -starting with the lhs segment, then each of the rhs segments (if any) -in order from most to least recent. -

      Each segment element is itself a list of 4 integer values, as follows: -

      1. First page of segment -
      2. Last page of segment -
      3. Root page of segment (if applicable) -
      4. Total number of pages in segment -
      -

      LSM_INFO_ARRAY_STRUCTURE
      There should be two arguments passed following this option (i.e. a -total of four arguments passed to lsm_info()). The first argument -should be the page number of the first page in a database array -(perhaps obtained from an earlier INFO_DB_STRUCTURE call). The second -trailing argument should be of type (char **). The location pointed -to is populated with a pointer to a nul-terminated string that must -be eventually freed using lsm_free() by the caller. -

      The output string contains the text representation of a Tcl list of -integers. Each pair of integers represent a range of pages used by -the identified array. For example, if the array occupies database -pages 993 to 1024, then pages 2048 to 2777, then the returned string -will be "993 1024 2048 2777". -

      If the specified integer argument does not correspond to the first -page of any database array, LSM_ERROR is returned and the output -pointer is set to a NULL value. -

      LSM_INFO_LOG_STRUCTURE
      The third argument should be of type (char **). The location pointed -to is populated with a pointer to a nul-terminated string containing -the string representation of a Tcl data-structure. The returned -string should be eventually freed by the caller using lsm_free(). -

      The Tcl structure returned is a list of six integers that describe -the current structure of the log file. -

      LSM_INFO_ARRAY_PAGES

      LSM_INFO_PAGE_ASCII_DUMP
      As with LSM_INFO_ARRAY_STRUCTURE, there should be two arguments passed -with calls that specify this option - an integer page number and a -(char **) used to return a nul-terminated string that must be later -freed using lsm_free(). In this case the output string is populated -with a human-readable description of the page content. -

      If the page cannot be decoded, it is not an error. In this case the -human-readable output message will report the systems failure to -interpret the page data. -

      LSM_INFO_PAGE_HEX_DUMP
      This argument is similar to PAGE_ASCII_DUMP, except that keys and -values are represented using hexadecimal notation instead of ascii. -

      LSM_INFO_FREELIST
      The third argument should be of type (char **). The location pointed -to is populated with a pointer to a nul-terminated string containing -the string representation of a Tcl data-structure. The returned -string should be eventually freed by the caller using lsm_free(). -

      The Tcl structure returned is a list containing one element for each -free block in the database. The element itself consists of two -integers - the block number and the id of the snapshot that freed it. -

      LSM_INFO_CHECKPOINT_SIZE
      The third argument should be of type (int *). The location pointed to -by this argument is populated with the number of KB written to the -database file since the most recent checkpoint. -

      LSM_INFO_TREE_SIZE
      If this value is passed as the second argument to an lsm_info() call, it -should be followed by two arguments of type (int *) (for a total of four -arguments). -

      At any time, there are either one or two tree structures held in shared -memory that new database clients will access (there may also be additional -tree structures being used by older clients - this API does not provide -information on them). One tree structure - the current tree - is used to -accumulate new data written to the database. The other tree structure - -the old tree - is a read-only tree holding older data and may be flushed -to disk at any time. -

      Assuming no error occurs, the location pointed to by the first of the two -(int *) arguments is set to the size of the old in-memory tree in KB. -The second is set to the size of the current, or live in-memory tree. -

      Opening and Closing Write Transactions

      -int lsm_begin(lsm_db *pDb, int iLevel); -int lsm_commit(lsm_db *pDb, int iLevel); -int lsm_rollback(lsm_db *pDb, int iLevel); - -

      These functions are used to open and close transactions and nested -sub-transactions. -

      The lsm_begin() function is used to open transactions and sub-transactions. -A successful call to lsm_begin() ensures that there are at least iLevel -nested transactions open. To open a top-level transaction, pass iLevel=1. -To open a sub-transaction within the top-level transaction, iLevel=2. -Passing iLevel=0 is a no-op. -

      lsm_commit() is used to commit transactions and sub-transactions. A -successful call to lsm_commit() ensures that there are at most iLevel -nested transactions open. To commit a top-level transaction, pass iLevel=0. -To commit all sub-transactions inside the main transaction, pass iLevel=1. -

      Function lsm_rollback() is used to roll back transactions and -sub-transactions. A successful call to lsm_rollback() restores the database -to the state it was in when the iLevel'th nested sub-transaction (if any) -was first opened. And then closes transactions to ensure that there are -at most iLevel nested transactions open. Passing iLevel=0 rolls back and -closes the top-level transaction. iLevel=1 also rolls back the top-level -transaction, but leaves it open. iLevel=2 rolls back the sub-transaction -nested directly inside the top-level transaction (and leaves it open). -

      Writing to a Database

      -int lsm_insert(lsm_db*, const void *pKey, int nKey, const void *pVal, int nVal); -int lsm_delete(lsm_db *, const void *pKey, int nKey); -int lsm_delete_range(lsm_db *, - const void *pKey1, int nKey1, const void *pKey2, int nKey2 -); - -

      Write a new value into the database. If a value with a duplicate key -already exists it is replaced. -Delete a value from the database. No error is returned if the specified -key value does not exist in the database. -Delete all database entries with keys that are greater than (pKey1/nKey1) -and smaller than (pKey2/nKey2). Note that keys (pKey1/nKey1) and -(pKey2/nKey2) themselves, if they exist in the database, are not deleted. -

      Return LSM_OK if successful, or an LSM error code otherwise. -

      Explicit Database Work and Checkpointing

      -int lsm_work(lsm_db *pDb, int nMerge, int nKB, int *pnWrite); -int lsm_flush(lsm_db *pDb); -int lsm_checkpoint(lsm_db *pDb, int *pnKB); - -

      This function is called by a thread to work on the database structure. -Attempt to checkpoint the current database snapshot. Return an LSM -error code if an error occurs or LSM_OK otherwise. -

      If the current snapshot has already been checkpointed, calling this -function is a no-op. In this case if pnKB is not NULL, *pnKB is -set to 0. Or, if the current snapshot is successfully checkpointed -by this function and pbKB is not NULL, *pnKB is set to the number -of bytes written to the database file since the previous checkpoint -(the same measure as returned by the LSM_INFO_CHECKPOINT_SIZE query). -

      Opening and Closing Database Cursors

      -int lsm_csr_open(lsm_db *pDb, lsm_cursor **ppCsr); -int lsm_csr_close(lsm_cursor *pCsr); - -

      Open and close a database cursor. -

      Positioning Database Cursors

      -int lsm_csr_seek(lsm_cursor *pCsr, const void *pKey, int nKey, int eSeek); -int lsm_csr_first(lsm_cursor *pCsr); -int lsm_csr_last(lsm_cursor *pCsr); -int lsm_csr_next(lsm_cursor *pCsr); -int lsm_csr_prev(lsm_cursor *pCsr); -#define LSM_SEEK_LEFAST -2 -#define LSM_SEEK_LE -1 -#define LSM_SEEK_EQ 0 -#define LSM_SEEK_GE 1 - -

      If the fourth parameter is LSM_SEEK_EQ, LSM_SEEK_GE or LSM_SEEK_LE, -this function searches the database for an entry with key (pKey/nKey). -If an error occurs, an LSM error code is returned. Otherwise, LSM_OK. -

      If no error occurs and the requested key is present in the database, the -cursor is left pointing to the entry with the specified key. Or, if the -specified key is not present in the database the state of the cursor -depends on the value passed as the final parameter, as follows: -

      LSM_SEEK_EQ
      The cursor is left at EOF (invalidated). A call to lsm_csr_valid() -returns non-zero. -

      LSM_SEEK_LE
      The cursor is left pointing to the largest key in the database that -is smaller than (pKey/nKey). If the database contains no keys smaller -than (pKey/nKey), the cursor is left at EOF. -

      LSM_SEEK_GE
      The cursor is left pointing to the smallest key in the database that -is larger than (pKey/nKey). If the database contains no keys larger -than (pKey/nKey), the cursor is left at EOF. -

      If the fourth parameter is LSM_SEEK_LEFAST, this function searches the -database in a similar manner to LSM_SEEK_LE, with two differences: -

      1. Even if a key can be found (the cursor is not left at EOF), the -lsm_csr_value() function may not be used (attempts to do so return -LSM_MISUSE). -

      2. The key that the cursor is left pointing to may be one that has -been recently deleted from the database. In this case it is -guaranteed that the returned key is larger than any key currently -in the database that is less than or equal to (pKey/nKey). -
      -

      LSM_SEEK_LEFAST requests are intended to be used to allocate database -keys. -Advance the specified cursor to the next or previous key in the database. -Return LSM_OK if successful, or an LSM error code otherwise. -

      Functions lsm_csr_seek(), lsm_csr_first() and lsm_csr_last() are "seek" -functions. Whether or not lsm_csr_next and lsm_csr_prev may be called -successfully also depends on the most recent seek function called on -the cursor. Specifically: -

        -
      • At least one seek function must have been called on the cursor. -
      • To call lsm_csr_next(), the most recent call to a seek function must -have been either lsm_csr_first() or a call to lsm_csr_seek() specifying -LSM_SEEK_GE. -
      • To call lsm_csr_prev(), the most recent call to a seek function must -have been either lsm_csr_first() or a call to lsm_csr_seek() specifying -LSM_SEEK_GE. -
      -

      Otherwise, if the above conditions are not met when lsm_csr_next or -lsm_csr_prev is called, LSM_MISUSE is returned and the cursor position -remains unchanged. -Values that may be passed as the fourth argument to lsm_csr_seek(). -

      Extracting Data From Database Cursors

      -int lsm_csr_valid(lsm_cursor *pCsr); -int lsm_csr_key(lsm_cursor *pCsr, const void **ppKey, int *pnKey); -int lsm_csr_value(lsm_cursor *pCsr, const void **ppVal, int *pnVal); -int lsm_csr_cmp(lsm_cursor *pCsr, const void *pKey, int nKey, int *piRes); - -

      Retrieve data from a database cursor. -If no error occurs, this function compares the database key passed via -the pKey/nKey arguments with the key that the cursor passed as the first -argument currently points to. If the cursors key is less than, equal to -or greater than pKey/nKey, *piRes is set to less than, equal to or greater -than zero before returning. LSM_OK is returned in this case. -

      Or, if an error occurs, an LSM error code is returned and the final -value of *piRes is undefined. If the cursor does not point to a valid -key when this function is called, LSM_MISUSE is returned. -

      Change these!!

      -void lsm_config_log(lsm_db *, void (*)(void *, int, const char *), void *); -void lsm_config_work_hook(lsm_db *, void (*)(lsm_db *, void *), void *); - -

      Configure a callback to which debugging and other messages should -be directed. Only useful for debugging lsm. -Configure a callback that is invoked if the database connection ever -writes to the database file. - DELETED www/lsmperf.wiki Index: www/lsmperf.wiki ================================================================== --- www/lsmperf.wiki +++ /dev/null @@ -1,106 +0,0 @@ - -LSM Benchmarks - - -

      -This page contains the results of a benchmark test run on the LSM library -in several different configurations. - -

      -The test uses a single client database connection. It begins with an empty -database. The test runs for 200 iterations. Each iteration, the client: - -

        -
      • Inserts 50,000 new key-value pairs into the database. Each key is - a pseudo-randomly generated 12-byte blob. Each value is a - pseudo-randomly generated 100 byte blob. Each insert operation is - run in a separate implicit transaction. -
      • Runs 50,000 fetch queries to retrieve arbitrarily selected entries - from the database (all searches are hits - the queried keys are - selected from the set of keys that are present in the db). -
      - -

      -The time taken to insert or retrieve each batch of 50,000 keys is recorded, -and transformed to an operations-per-second rate (e.g. if it takes 0.5 -seconds to insert 50,000 keys, this is reported as 100,000 -operations-per-second). These values are the values reported below. -The final database size is roughly 1.2GB. - -

      -All tested configurations set the -LSM_CONFIG_SAFETY parameter to "normal" -and the -LSM_CONFIG_MULTIPLE_PROCESSES parameter to zero (single process mode). - -

      -Tests were run on a 3.3GHz dual-core PC with 4GB of RAM. The file-system is -ext4 on a 7200rpm HDD. - -

      Small Write Buffers Test

      - -

      -The following plot shows the performance of LSM in its default, single-threaded -configuration against a multi-threaded deployment. - -

      -In the single-threaded mode the in-memory tree is allowed to use up to 1MB of -memory before it is flushed to disk, and the database is checkpointed after -each 2MB of data is written into the database file. - -

      -The multi-threaded configuration launches two background threads as well as -the main client thread used to run the test case. One background thread is -dedicated to calling lsm_work() (writing data into the database file) and -the other to calling lsm_checkpoint() (writing to the database header and -syncing the database file). The in-memory tree is declared eligible to -be flushed to disk when it uses 1MB of memory. If it grows to 1.5MB, then -the client thread is blocked from continuing until the background thread -has flushed it to disk. After 2MB of data has been written into the database -file it is eligible for checkpointing. Once the database file contains 3MB of -uncheckpointed data, the worker thread is prevented from writing any further -data to the file until a checkpoint has been completed. - -

      -More detail regarding lsm_work() and lsm_checkpoint() is -available here. - -

      - - -

      Large Write Buffers Test

      -

      -This test is similar to the one above, except with larger write buffers. -Specifically, in the single-threaded mode: -

        -
      • The in-memory tree is allowed to use up to 4MB of memory before it - is flushed to disk, and -
      • The database is checkpointed after each 8MB of data is written to - the database file. -
      - -

      -And in multi-threaded mode: -

        -
      • The in-memory tree is eligible to be flushed to disk once it uses - 4MB of memory. The client thread is blocked if it reaches 6MB in - size. -
      • The database is still eligible for checkpointing after each 2MB of - data is written to it, but up to 8MB of uncheckpointed data is allowed - to accumulate in the database file before the worker thread is - blocked. -
      - -

      - - -

      Large Write Buffers Test With Pauses

      - -

      -This test is the same as the previous test, except that each iteration the -client thread pauses (sleeps) for 2.5 seconds after performing the database -inserts and queries. - -

      - - DELETED www/lsmperf1.gif Index: www/lsmperf1.gif ================================================================== --- www/lsmperf1.gif +++ /dev/null cannot compute difference between binary files DELETED www/lsmperf2.gif Index: www/lsmperf2.gif ================================================================== --- www/lsmperf2.gif +++ /dev/null cannot compute difference between binary files DELETED www/lsmperf3.gif Index: www/lsmperf3.gif ================================================================== --- www/lsmperf3.gif +++ /dev/null cannot compute difference between binary files DELETED www/lsmusr.wiki Index: www/lsmusr.wiki ================================================================== --- www/lsmusr.wiki +++ /dev/null @@ -1,1351 +0,0 @@ - -LSM Users Guide - - -

      Table of Contents

      - - - - - - - - - - -
      -      1. Introduction to LSM
      -      2. Using LSM in Applications
      -      3. Basic Usage
      -            3.1. Opening and Closing Database Connections
      -            3.2. Writing to a Database
      -            3.3. Reading from a Database
      -            3.4. Database Transactions and MVCC
      -      4. Data Durability
      -      5. Compressed and Encrypted Databases
      -      6. Performance Tuning
      -            6.1. Overview of LSM Architecture
      -            6.2. Performance Related Configuration Options
      -            6.3. Work and Checkpoint Scheduling
      -                  6.3.1. Automatic Scheduling
      -                  6.3.2. Explicit Scheduling
      -                  6.3.3. Compulsary Work and Checkpoints
      -            6.4. Database File Optimization
      - -
      - -

      Overview

      - -

      This document describes the LSM embedded database library and use thereof. -It is part user-manual and part tutorial. It is intended to complement the -LSM API reference manual. - -

      The first section of this document contains -a description of the LSM library and its features. -Section 2 describes how to use LSM from -within a C or C++ application (how to compile and link LSM, what to #include -etc.). The third section describes the essential APIs -that applications use to open and close database connections, and to read from -and write to databases. - -

      The three sections described above contain all the information required to -create applications that use LSM. The remaining sections discuss more -specialized topics. Section 4 discusses the -configuration parameter that influences transaction durability (the guarantees -offered with respect to recently committed transactions if a power failure -occurs). Section 5 explains -the interface provided by LSM that allows external data compression and/or -encryption functions to be used to create compressed and/or encrypted -databases. Section 6 deals with performance -tuning. - -

      1. Introduction to LSM

      - -

      LSM is an embedded database library for key-value data, roughly similar -in scope to -Berkeley DB, -LevelDB or -KyotoCabinet. -Both keys and -values are specified and stored as byte arrays. Duplicate keys are not -supported. Keys are always sorted in memcmp() order. LSM supports the following -operations for the manipulation and query of database data: - -

        -
      • Writing a new key and value into the database. -
      • Deleting an existing key from the database. -
      • Deleting a range of keys from the database. -
      • Querying the database for a specific key. -
      • Iterating through a range of database keys (either forwards or - backwards). -
      - -

      Other salient features are: - -

        -

      • A single-writer/multiple-reader MVCC based transactional - concurrency model. SQL style nested sub-transactions are supported. - Clients may concurrently access a single LSM database from within a - single process or multiple application processes. - -

      • An entire database is stored in a single file on disk. - -

      • Data durability in the face of application or power failure. - LSM may optionally use a write-ahead log file when writing to the - database to ensure committed transactions are not lost if an application - or power failure occurs. - -

      • An API that allows external data compression and/or encryption - routines to be used to create and access compressed and/or - encrypted databases. -
      - - -

      Many database systems that support range queries, including SQLite 3, Berkeley DB and Kyoto Cabinet, are -based on one of many variants of the -b-tree data structure. -B-trees are attractive because a b-tree structure minimizes the number of disk -sectors that must be read from disk when searching the database for a specific -key. However, b-tree implementations usually suffer from poor write -localization - updating the contents of a b-tree often involves modifying the -contents of nodes scattered throughout the database file. If the database is -stored on a spinning disk (HDD), then the disk heads must be moved between -writing non-contiguous sectors, which is extremely slow. If the database is -stored on solid state storage (SDD) a similar phenomena is encountered due to -the large erase-block sizes. In general, writing to a series of contiguous disk -sectors is orders of magnitude faster than updating to the same number of disk -sectors scattered randomly throughout a large file. Additionally, b-tree -structures are prone to fragmentation, reducing the speed of range queries. - -

      TODO: fix the link in the next paragraph to point to a description -of the log-structured-merge tree within lsm.wiki (or its successor). - -

      LSM uses a different data structure that makes the -following performance tradeoffs relative to a b-tree: - -

        -
      • A very large percentage of the disk sectors modified when writing to - the database are contiguous. - Additionally, in many cases the total number of sectors written - to disk is reduced. This makes writing to an LSM database faster - than the equivalent b-tree. - -
      • LSM databases do not suffer from fragmentation to the same degree - as b-trees. This means that the performance of large range queries - does not degrade as the database is updated as it may with a b-tree. - -
      • Under some circumstances searching an LSM database for a given key will - involve examining more disk sectors than it would with a b-tree. In - terms of disk sectors accessed when searching a database of size N, - both b-trees and LSM provide O(log(N)) efficiency, but the base of - the logarithm is generally larger for a b-tree than for LSM. -
      - -

      In other words, writing to an LSM database should be very fast and scanning -through large ranges of keys should also perform well, but searching the -database for specific keys may be slightly slower than when using a b-tree -based system. Additionally, avoiding random writes in favour of largely -contiguous updates (as LSM does) can significantly reduce the wear on SSD or -flash memory devices. - -

      Although it has quite different features to LSM in other respects, -LevelDB makes similar performance tradeoffs. - -

      Benchmark test results for LSM are available here. - -

      2. Using LSM in Applications

      - -

      LSM is not currently built or distributed independently. Instead, it -is part of the SQLite4 library. To use LSM in an application, the application -links against libsqlite4 and includes the header file "lsm.h" in any files -that access the LSM API. - -

      Pointer to build instructions for sqlite4 - -

      3. Basic Usage

      - -

      3.1. Opening and Closing Database Connections

      - -

      Opening a connection to a database is a two-step process. The -lsm_new() function is used to create a new -database handle, and the lsm_open() function -is used to connect an existing database handle to a database on disk. This -is because some database connection properties may only be configured -before the database is opened. In that case, one or more calls to the -lsm_config() method are made between the -calls to lsm_new() and lsm_open(). - -

      The functions are defined as follows: - - - int lsm_new(lsm_env *env, lsm_db **pDb); - int lsm_open(lsm_db *db, const char *zFile); - - -

      Like most lsm_xxx() functions that return type int (the exception is -lsm_csr_valid()), both of the above -return LSM_OK (0) if successful, or an LSM -error code otherwise. The first argument to lsm_new() may be passed either -a pointer to a database environment object -or NULL. Almost all applications should pass NULL. A database environment -object allows the application to supply custom implementations of the various -operating system calls that LSM uses to read and write files, allocate heap -memory, and coordinate between multiple application threads and processes. -This is normally only required if LSM is being used on a platform that is not -supported by default. Passing NULL instructs the library to use the default -implementations of all these things. The second argument to lsm_new() is an -output variable. Assuming the call is successful, *pDb is set to point to the -new database handle before returning. - -

      The first argument passed to lsm_open() must be an existing database -handle. The second is the name of the database file to connect to. Once -lsm_open() has been successfully called on a database handle, it can not be -called again on the same handle. Attempting to do so is an LSM_MISUSE error. - -

      For example, to create a new handle and connect it to database "test.db" -on disk: - - int rc; - lsm_db *db; - - /* Allocate a new database handle */ - rc = lsm_new(0, &db); - if( rc!=LSM_OK ) exit(1); - - /* Connect the database handle to database "test.db" */ - rc = lsm_open(db, "test.db"); - if( rc!=LSM_OK ) exit(1); - - -

      A database connection can be closed using the lsm_close() function. Calling -lsm_close() disconnects from the database (assuming lsm_open() has been -successfully called) and deletes the handle itself. Attempting to use a -database handle after it has been passed to lsm_close() results in undefined -behaviour (likely a segfault). - - - rc = lsm_close(db); - - -

      It is important that lsm_close() is called to close all database handles -created with lsm_new(), particularly if the connection has written to the -database. If an application writes to the database and then exits without -closing its database connection, then subsequent clients may have to run -"database recovery" when they open the database, slowing down the lsm_open() -call. Additionally, not matching each successful lsm_new() call with a call -to lsm_close() is a resource leak. - -

      Counter-intuitively, an lsm_close() call may fail. In this case the database -handle is not closed, so if the application exits it invites the "database -recovery" performance problem mentioned above. The usual reason for an -lsm_close() call failing is that the database handle has been used to create -database cursors that have not been -closed. Unless all database cursors are closed before lsm_close() is called, -it fails with an LSM_BUSY error and the database handle is not closed. - -

      3.2. Writing to a Database

      - -

      Three API functions are used to write to the database: - -

        -
      • lsm_insert(): insert a new key/value pair into the database, - overwriting any existing entry with the same key. -
      • lsm_delete(): remove a specific key from the database. -
      • lsm_delete_range(): remove an open-ended range (one that does not - include its endpoints) of keys from the database. -
      - -

      Each of these functions returns LSM_OK (0) if successful, or an LSM error -code otherwise (some non-zero value). - -

      The following example code inserts a key/value pair into the database. The -key is a 1-byte blob consisting of the value 0x61, and the value is a 3 byte -blob consisting of 0x6F, 0x6E and 0x65, in that order. An application might -interpret these as utf-8 or ASCII codepoints, but LSM treats them as opaque -blobs of data. - - rc = lsm_insert(db, "a", 1, "one", 3); - - -

      Remove the entry with the key "a" (single 0x61 octet) from the database: - - - rc = lsm_delete(db, "a", 1); - - -

      Remove all entries with keys that are greater than "c" but less than "f". -In this context key blobs are compared in the normal way - using memcmp(), -with longer keys being considered larger than their prefixes. - - - rc = lsm_delete_range(db, "c", 1, "f", 1); - - -

      The example above removes all keys between "c" and "f", but does not -remove the endpoints "c" and "f" themselves. To do this, requires three -separate calls - one to remove the open-ended range of keys and two to -remove the two endpoints. As follows: - - - /* Should be checking return codes! */ - lsm_delete(db, "c", 1); - lsm_delete_range(db, "c", 1, "f", 1); - lsm_delete(db, "f", 1); - - -

      3.3. Reading from a Database

      - -

      All data read from an LSM database is read via a cursor handle. Cursor -handles are opened using the -lsm_csr_open() API, as follows: - - - lsm_csr *csr; - rc = lsm_csr_open(db, &csr); - - -

      Once an application has finished using a database cursor, it must be closed -using the lsm_csr_close() API. The -lsm_csr_close() function does not return any value. It cannot fail. - - - lsm_csr_close(csr); - - -

      Database cursors support the following functions for positioning the cursor: - -

        -
      • lsm_csr_seek() - move the cursor to point to a nominated - database key. -
      • lsm_csr_first() - move the cursor to point to the first entry in - the database (the one with the smallest key). -
      • lsm_csr_last() - move the cursor to point to the last entry in - the database (the one with the largest key). -
      • lsm_csr_next() - move the cursor to point to the next entry in - the the database. -
      • lsm_csr_prev() - move the cursor to point to the previous entry - in the database. -
      - -

      Once a cursor has been positioned, it supports the following functions -for retrieving the details of the current entry: - -

        -
      • lsm_csr_valid() - determine whether or not the cursor currently - points to a valid entry. -
      • lsm_csr_key() - retrieve the key associated with the - database entry the cursor points to. -
      • lsm_csr_value() - retrieve the value associated with the - database entry the cursor points to. -
      • lsm_csr_cmp() - compare a key supplied by the application with - the key associated with the entry the cursor points to. -
      - -

      The following example demonstrates using the -lsm_csr_seek() function to search the -database for a specified key, -lsm_csr_valid() to check if the search -was successful, and lsm_csr_value() -to retrieve the value associated with the key within the database. - - - rc = lsm_csr_seek(csr, "b", 1, LSM_SEEK_EQ); - if( lsm_csr_valid(csr) ){ - const void *pVal; int nVal; - - rc = lsm_csr_value(csr, &pVal, &nVal); - if( rc==LSM_OK ){ - /* pVal now points to a buffer nVal bytes in size containing the - ** value associated with database key "b". */ - } - } - - -

      The example code below iterates forwards through all entries (in key -order, from smallest to largest) in the database. Function -lsm_csr_first() is used to position -the cursor to point to the first entry in the database, and -lsm_csr_next() is used to advance to -the next entry. After lsm_csr_next() is called to advance past the final -entry in the database, the cursor is left pointing to no entry at all, -lsm_csr_valid() returns 0, and the loop is finished. API function -lsm_csr_key() is used to retrieve the -key associated with each database entry visited. - - - for(rc=lsm_csr_first(csr); rc==LSM_OK && lsm_csr_valid(csr); rc=lsm_csr_next(csr)){ - const void *pKey; int nKey; - const void *pVal; int nVal; - - rc = lsm_csr_key(csr, &pKey, &nKey); - if( rc==LSM_OK ) rc = lsm_csr_value(csr, &pVal, &nVal); - if( rc==LSM_OK ) break; - - /* At this point pKey points to the current key (size nKey bytes) and - ** pVal points to the corresponding value (size nVal bytes). */ - } - - -

      The example code above could be modified to iterate backwards through -the entries in the database (again in key order, but this time from largest -to smallest) by replacing the call to lsm_csr_first() with -lsm_csr_last() and the call to lsm_csr_next() with lsm_csr_prev(). - -

      The signature of lsm_csr_seek() is: - - - int lsm_csr_seek(lsm_cursor *csr, const void *pKey, int nKey, int eSeek); - - -

      The second and third arguments passed to lsm_csr_seek() define the key -to search the database for (pKey must point to the buffer containing the -nKey byte key when this function is called). Assuming no error occurs, if -there an entry with the requested key is present in the database, the cursor -is left pointing to it. Otherwise, if no such entry is present, the final -position of the cursor depends on the value passed as the fourth parameter -to lsm_csr_seek(). Valid values for the fourth parameter to lsm_csr_seek() -are: - -

      -
      LSM_SEEK_EQ -

      - In this case, if the specified key is not present in the database, the - cursor is not left pointing to any database entry (i.e. calling - lsm_csr_valid() returns 0). - -

      LSM_SEEK_LE -

      - If the specified key is not present in the database and the fourth - argument to lsm_csr_seek() is LSM_SEEK_LE (Less than or Equal), the - cursor is left pointing to the database entry with the largest key - that is less than the specified key. Or, if there are no entries in - the database with keys smaller than the specified key, the cursor is - left pointing to no entry at all. - -

      LSM_SEEK_GE -

      - If the specified key is not present in the database and the fourth - argument to lsm_csr_seek() is LSM_SEEK_GE (Greater than or Equal), the - cursor is left pointing to the database entry with the smallest key - that is greater than the specified key. Or, if there are no entries - in the database with keys larger than the specified key, the cursor is - left pointing to no entry at all. -

      -

      - -

      Calls made to lsm_csr_seek() with LSM_SEEK_EQ as the final argument are -slightly more efficient than those made specifying LSM_SEEK_LE or LSM_SEEK_GE. -So to retrieve a specific entry from a database, LSM_SEEK_EQ should be -preferred. The other two values are primarily useful for implementing -range queries. For example, to iterate backwards through all keys from "ggg" -to "cc", inclusive: - - - for(rc = lsm_csr_seek(csr, "ggg", 3, LSM_SEEK_LE); lsm_csr_valid(csr); rc = lsm_csr_prev(csr)){ - const void *pKey; int nKey; - const void *pVal; int nVal; - int res; - - /* Compare the key that the cursor currently points to with "cc". If - ** the cursor key is less than "cc", break out of the loop. */ - rc = lsm_csr_cmp(csr, "cc", 2, &res); - if( rc!=LSM_OK || res<0 ) break; - - rc = lsm_csr_key(csr, &pKey, &nKey); - if( rc==LSM_OK ) rc = lsm_csr_value(csr, &pVal, &nVal); - if( rc!=LSM_OK ) break; - - /* At this point pKey points to the current key (size nKey bytes) and - ** pVal points to the corresponding value (size nVal bytes). */ - } - - -

      In the example code above, the call to lsm_csr_seek() positions the cursor -to point to the entry with key "ggg", if it exists, or to the largest entry -in the database with a key smaller than "ggg" if such a key can be found, -or to EOF otherwise. The lsm_csr_prev() call advances the cursor to the -next entry in the database file (in key order from largest to smallest), and -the lsm_csr_valid() call returns 0 to break out of the loop once the -cursor is advance past the entry with the smallest key in the database. So -on its own, the "for" statement serves to iterate the cursor in reverse order -through all keys in the database less than or equal to "ggg". - -

      The call to lsm_csr_cmp() call in the -body of the loop is used to enforce the lower bound (keys >= "cc") on the -range query by breaking out of the loop if an entry with a key smaller than -"cc" is ever visited. lsm_csr_cmp() has the following signature: - - - int lsm_csr_cmp(lsm_cursor *csr, const void *pKey, int nKey, int *piRes); - - -

      When lsm_csr_cmp() is called, the key specified by the second and third -arguments (pKey and nKey) is compared to the database key that the cursor -currently points to. Assuming no error occurs, depending on whether or not the -cursors key is less than, equal to, or greater than the specified key, *piRes -is set to a value less than, equal to, or greater than zero before returning. -In other words: - -

        *piRes = (cursors key) - (specified key)
      - - -

      3.4. Database Transactions and MVCC

      - -

      LSM supports a single-writer/multiple-reader -MVCC -based transactional concurrency model. This is the same model that SQLite -supports in WAL mode. - -

      A read-transaction must be opened in order to read from the database. -After a read-transaction has been opened, no writes to the database made -by other database connections are visible to the database reader. Instead, -the reader operates on a snapshot of the database as it existed when the -read transaction was first opened. Any number of clients may simultaneously -maintain open read-transactions. - -

      If one is not already open, a read-transaction is opened when a database -cursor is created (the lsm_csr_open() function). It is closed when the number -of open cursors drops to zero. - -

      A write-transaction is required to write to the database. At any point, -at most one database client may hold an open write transaction. If another -client already has an open write transaction, then attempting to open one -is an error (LSM_BUSY). If a read-transaction is already open when the -write-transaction is opened, then the snapshot read by the read-transaction -must correspond to the most recent version of the database. Otherwise, -the attempt to open the write-transaction fails (LSM_BUSY). In other words, -if any other client has written to the database since the current clients -read-transaction was opened, it will not be possible to upgrade to a -write-transaction. - -

      Write-transactions may be opened either implicitly or explicitly. If any -of the following functions are called to write to the database when there -is no write-transaction open, then an implicit write-transaction is opened and -closed (committed) within the call: - -

        -
      • lsm_insert() -
      • lsm_delete() -
      • lsm_delete_range() -
      - -

      This means, of course, that all three of the above may return LSM_BUSY. -Indicating either that another client currently has an open write-transaction, -or that there is currently an open read-transaction and some other client -has written to the database since it was opened. - -

      When an explicitly opened transaction is closed, it may either be -committed or rolled back (reverted - so that the state of the database is -unchanged). Within a write-transaction there may also be a hierarchy of -nested sub-transactions that may be rolled back or committed independently. -A write-transaction is a property of a database connection - all writes -made by the connection become part of the current transaction (and possibly -sub-transaction). - -

      The functions used to open, commit and rollback explicity transactions -and sub-transactions are, respectively: - - - int lsm_begin(lsm_db *, int); - int lsm_commit(lsm_db *, int); - int lsm_rollback(lsm_db *, int); - - -

      In all cases, the second parameter is either the maximum (lsm_commit(), -lsm_rollback()) or minimum (lsm_begin()) the number of nested -write-transactions that will exist following the call (assuming it succeeds). -If the second parameter passed is N, - -

        -

      • Calling lsm_begin(db, N) attempts opens zero or more - nested write-transactions so that the database connection is left with - at least N open nested write-transactions. If there are already - N or more open nested write-transactions open, then lsm_begin(db, - N) is a no-op. lsm_begin(db, 0) is always a no-op. Calling - lsm_begin(db, 1) when there is no open write-transaction opens a - top-level write-transaction. - -

      • Calling lsm_commit(db, N) commits zero or more nested - write-transactions so that the database connection is left with at most - N open write-transactions. If the connection has N or - fewer open nested write-transactions, then lsm_commit(db, N) is a - no-op. Calling lsm_commit(db, 0) commits the outermost transaction - (if any). - -

      • Calling lsm_rollback(db, 0) closes and rolls back the - top-level write-transaction. Calling lsm_rollback(db, N) - for any value of N greater than zero closes zero or more nested - write-transactions so that the database connection is left with at most - N open transactions. If, following this, the database connection - has exactly N open nested write-transactions, the outermost is - rolled back, but not closed. Calling lsm_rollback(db, 1) rolls back - (but does not close) the top-level transaction. -

      - -

      Examples follow. With error checking omitted for brevity's sake. - - - /* Open a write-transaction. Write some data to the database. Then - ** commit and close the write transaction. Following this, the database - ** contains: - ** - ** "j" -> "ten" - ** "k" -> "eleven" - */ - lsm_begin(db, 1); - lsm_insert(db, "j", 1, "ten", 3); - lsm_insert(db, "k", 1, "eleven", 6); - lsm_commit(db, 0); - - /* Open a write-transaction, perform all manner of writes and other - ** operations (not shown). Then roll the top-level transaction back. - ** Regardless of the write operations performed, the database remains - ** unchanged: - ** - ** "j" -> "ten" - ** "k" -> "eleven" - */ - lsm_begin(db, 1); - /* Do all manner of writes, sub-transactions etc. */ - lsm_rollback(db, 0); - - /* Open a write-transaction. Write some data to the database. Then - ** rollback the top level transaction but do not close it. Write - ** different data to the database and commit. Following this block, - ** the database is: - ** - ** "j" -> "ten" - ** "k" -> "eleven" - ** "m" -> "thirteen" - */ - lsm_begin(db, 1); - lsm_insert(db, "l", 1, "twelve", 3); - lsm_rollback(db, 1); - lsm_insert(db, "m", 1, "thirteen", 6); - lsm_commit(db, 0); - - /* Open a write-transaction and 2 nested sub-transactions. Delete a - ** database key. Then commit and close the outermost sub-transaction. - ** Open another sub-transaction (so that there are again 2 nested - ** sub-transactions). Delete a different database key. Then rollback - ** and close the outermost sub-transaction. Finally, delete yet another - ** db key and commit the outermost transaction. Leaving just: - ** - ** "k" -> "eleven" - */ - lsm_begin(db, 3); - lsm_delete(db, "j", 1); - lsm_commit(db, 2); - lsm_begin(db, 3); - lsm_delete(db, "k", 1); - lsm_rollback(db, 2); - lsm_delete(db, "m", 1); - lsm_commit(db, 0); - - - -

      4. Data Durability

      - -

      The value of the configuration parameter LSM_CONFIG_SAFETY determines -how often data is synced to disk by the LSM library. This is an important -tradeoff - syncing less often can lead to orders of magnitude better -performance, but also exposes the application to the risk of partial or total -data loss in the event of a power failure; - -

    • -
    LSM_SAFETY_OFF - (0) - Do not sync to disk at all. This is the fastest mode. -

    If a power failure occurs while writing to the database, - following recovery the database may be corrupt. All or some data may - be recoverable. - -

    LSM_SAFETY_NORMAL - (1) - Sync only as much as is necessary to prevent database corruption. - This is the default setting. Although slower than LSM_SAFETY_OFF, - this mode is still much faster than LSM_SAFETY_FULL. -

    If a power failure occurs while writing to the database, following - recovery some recently committed transactions may have been lost. - But the database file should not be corrupt and older data intact. - -

    LSM_SAFETY_FULL - (2) - Sync every transaction to disk as part of committing it. This is - the slowest mode. -

    If a power failure occurs while writing to the database, all - successfully committed transactions should be present. - The database file should not be corrupt. -

    - -

    The following example code sets the value of the LSM_CONFIG_SAFETY -parameter for connection db to LSM_SAFETY_FULL: - - - int iSafety = LSM_SAFETY_FULL; - lsm_config(db, LSM_CONFIG_SAFETY, &iSafety); - - -

    The current value of the LSM_CONFIG_SAFETY parameter can also be queried -by setting the initial value of the argument to -1 (or any other negative -value). For example: - - - int iSafety = -1; - lsm_config(db, LSM_CONFIG_SAFETY, &iSafety); - /* At this point, variable iSafety is set to the currently configured value - ** of the LSM_CONFIG_SAFETY parameter (either 0, 1 or 2). */ - - -

    The lsm_config() function may also be used to configure other database -connection parameters. - - -

    5. Compressed and Encrypted Databases

    - -

    LSM does not provide built-in methods for creating encrypted or compressed -databases. Instead, it allows the user to provide hooks to call external -functions to compress and/or encrypt data before it is written to the database -file, and to decrypt and/or uncompress data as it is read from the database -file. - -

    A database connection is configured to call compression functions using a -call to lsm_config() with the second argument set to -LSM_CONFIG_SET_COMPRESSION. The third argument should point to an instance -of type lsm_compress, which is defined as follows: - - - typedef struct lsm_compress lsm_compress; - struct lsm_compress { - u32 iId; - void *pCtx; - int (*xBound)(void *pCtx, int nIn); - int (*xCompress)(void *pCtx, void *pOut, int *pnOut, const void *pIn, int nIn); - int (*xUncompress)(void *pCtx, void *pOut, int *pnOut, const void *pIn, int nIn); - void (*xFree)(void *pCtx); - }; - - -

    Explain how the hooks work here (same as zipvfs) - - -

    Example code? Using zlib? Or something simple like an -RLE implementation? - -

    The database file header of any LSM database contains a 32-bit unsigned -"compression id" field. If the database is not a compressed database, this -field is set to 1. Otherwise, it is set to an application supplied value -identifying the compression and/or encryption scheme in use. Application -compression scheme ids must be greater than or equal to 10000. Values smaller -than 10000 are reserved for internal use. - -

    The lsm_info() API may be used to read the compression id from a database -connection as follows: - - - unsigned int iCompressionId; - rc = lsm_info(db, LSM_INFO_COMPRESSION_ID, &iCompressionId); - if( rc==LSM_OK ){ - /* Variable iCompressionId now contains the db compression id */ - } - - -Because the compression id is stored in the database -header, it may be read before any required compression or encryption hooks -are configured. - - - #define LSM_COMPRESSION_EMPTY 0 - #define LSM_COMPRESSION_NONE 1 - - -

    When a database is opened for the first time, before it is first written, -the compression id field is set to LSM_COMPRESSION_EMPTY (0). After data is -written into the database file, the database compression id is set to a copy -of the lsm_compress.iId field of the compression hooks for the database handle -doing the writing, or to LSM_COMPRESSION_NONE (1) if no compression hooks -are configured. - -

    Once the compression id is set to something other than -LSM_COMPRESSION_EMPTY, when a database handle attempts to read or write the -database file, the compression id is compared against the lsm_compress.iId -field of the configured compression hooks, or against LSM_COMPRESSION_NONE if -no compression hooks are configured. If the compression id does not match, then -an LSM_MISMATCH error is returned and the operation fails (no transaction or -database cursor is opened). - -

    It is also possible to register a compression factory callback with a -database handle. If one is registered, the compression factory callback is -invoked instead of returning LSM_MISMATCH if the configured compression hooks -do not match the compression id of a database. If the callback registers -compatible compression hooks with the database handle (using the normal -lsm_config() interface), then the database read or write operation resumes -after it returns. Otherwise, if the compression factory callback does not -register new, compatible, compression hooks with the database handle, -LSM_MISMATCH is returned to the user. - -

    A compression factory callback is registered with a database handle -by calling lsm_config() with the second argument set to -LSM_CONFIG_SET_COMPRESSION_FACTORY, and the third argument set to point to -an instance of structure lsm_compress_factory. The lsm_config() copies the -contents of the structure - it does not retain a pointer to it. - - - typedef struct lsm_compress_factory lsm_compress_factory; - struct lsm_compress_factory { - void *pCtx; - int (*xFactory)(void *pCtx, lsm_db *db, unsigned int iCompressionId); - void (*xFree)(void *pCtx); - }; - - -

    Explain how the xFactory hook works here. - -

    6. Performance Tuning

    - -

    This section describes the various measures that can be taken in order to -fine-tune LSM in order to improve performance in specific circumstances. -Sub-section 6.1 contains a high-level overview of the -system architecture -intended to help in understanding the various performance tradeoffs and -optimizations available to LSM applications. Sub-section 6.2 identifies the - configuration parameters -that can be used to influence database performance. -Sub-section 6.3 discusses options and methods for scheduling the time-consuming -processes of actually -writing and syncing the database file -to disk. Finally, 6.4 introduces "database optimization" -- the process of reorganizing a database file internally so that it is as small -as possible and optimized for search queries. - -

    6.1. Overview of LSM Architecture

    - -

    The following steps describe the journey taken by data written to the - database from the application to the database file on disk: - -

      -

    1. - When an application writes to an LSM database, the new data is first - written to a log file and stored in an in-memory tree structure. The - log file is used for database recovery - if an application crash or - power failure occurs and the contents of the in-memory tree are lost, - data is recovered by reading the log file. - -

    2. - Once sufficient data has been accumulated in an in-memory tree (by - default "sufficient data" means 1MB, including data structure - overhead), it is marked as "old" and a new "live" in-memory tree - created. An old in-memory tree is immutable - new data is always - inserted into the live tree. There may be at most one old tree - in memory at a time. - -

    3. - The contents of an old in-memory tree may be written into the - database file at any point. Once its contents have been written (or - "flushed") to the database file, the in-memory tree may be discarded. - Flushing an in-memory tree to the database file creates a new database - "segment". A database segment is an immutable b-tree structure stored - within the database file. A single database file may contain up to 64 - segments. - -

    4. - At any point, two or more existing segments within the database file - may be merged together into a single segment. Once their contents has - been merged into the new segment, the original segments may be - discarded. - -

    5. - After the set of segments in a database file has been modified (either - by flushing an in-memory tree to disk or by merging existing segments - together), the changes may be made persistent by "checkpointing" the - database. Checkpointing involves updating the database file header and - and (usually) syncing the contents of the database file to disk. -

    - -

    Steps 3 and 4 above are known as "working" on the database. Step 5 is -refered to as "checkpointing". By default, database connections perform work -and checkpoint operations periodically from within calls to API functions -lsm_insert, lsm_delete, lsm_delete_range -and lsm_commit (i.e. functions that write to the database). -Alternatively, work and checkpoint operations may be performed on demand using -the lsm_work and lsm_checkpoint APIs. By opening a -second database connection, these operations may be moved to a background -thread or process. - -

    Optimizing database write throughput and responsiveness is done by -configuring and scheduling work and checkpoint operations, and by configuring -a few other parameters, as described in the following two sections. - -

    The speed of database read operations is largely determined by the number -of segments in the database file. So optimizing read operations is also linked -to the configuring and scheduling of database write operations, as these -policies determine the number of segments that are present in the database -file at any time. - -

    Any data written to the database file since the last checkpoint may be -lost if a power or application failure occurs. As a result of this, regular -database checkpoints are required to ensure that unused space within the log -file and database file can be reused in a timely fashion. Specifically: - -

      -

    • Space within the log file cannot be recycled until the corresponding - data has been written into a database segment and a checkpoint - performed. - -

    • When two or more existing segments are merged into a new segment - within the database file, the space occupied by the original segments - may not be recycled until after a checkpoint has been performed. -

    - -

    -In other words, without checkpoints the system will function, but both the log and database files will grow indefinitely as the database is modified (even if the size of the dataset remains constant). Additionally, if a crash or power failure occurs, the next client to open the database file has to process all data written to the log file since the most recent checkpoint. If checkpoints are performed infrequently, this can be a time consuming exercise. - -

    If a connection attempts to open a write transaction on the database when -another connection already has an open write transaction, the attempt fails -and LSM_BUSY is returned to the caller. This is because to write to the -database, the connection must first obtain the WRITER lock - and at most -one connection may simultaneously hold the WRITER lock. As well as the WRITER -lock, there are two other exclusive locks that may be obtained on the database -- the WORKER and CHECKPOINTER locks. These are used, not surprisingly, to -ensure that at most one connection attempts to work on or checkpoint the -database at a time. More specifically, the roles of the three locks are: - - -
    WRITER -The WRITER lock is required to modify the contents of the in-memory tree. -Including marking an in-memory tree as "old" and starting a new live tree. -It is also required to write to the log file. - -
    WORKER -The WORKER lock is required to write to segments within the database file. -Either when merging two or more existing segments within the database, or -when flushing an in-memory tree to disk to create a new segment. - -
    CHECKPOINTER -The CHECKPOINTER lock is required to write to the database file header. -
    - -

    The three locks are independent. It is possible to simultaneously have one -client writing to the database, another working on the database file and a -third performing a checkpoint operation. - - -

    - -

    The options in this section are all set to integer values. They may be -set and queried using the lsm_config() -function. To set an option to a value, lsm_config() is used as follows: - - - /* Set the LSM_CONFIG_AUTOFLUSH option to 1MB (1024 KB) */ - int iVal = 1024; - rc = lsm_config(db, LSM_CONFIG_AUTOFLUSH, &iVal); - - -

    In order to query the current value of an option, the initial value of -the parameter (iVal in the example code above) should be set to a negative -value. Or any other value that happens to be out of range for the parameter - -negative values just happen to be out of range for all integer lsm_config() -parameters. - - - /* Set iVal to the current value of LSM_CONFIG_AUTOFLUSH */ - int iVal = -1; - rc = lsm_config(db, LSM_CONFIG_AUTOFLUSH, &iVal); - - -

    -
    LSM_CONFIG_AUTOCHECKPOINT -

    - This option determines how often the database is checkpointed (synced to - disk). A checkpoint is performed after N KB (approximately) have been - written to the database file, where N is the value of this option. - Increasing this value (say to 4MB or even 8MB) may improve overall write - throughput. - -

    The default value is 2048 (2MB). - -

    LSM_CONFIG_AUTOFLUSH -

    - This option determines how much data in KB is allowed to accumulate in a - live in-memory tree before it is marked as "old" (and made eligible to be - flushed through to the database file). Increasing this value may improve - overall write throughput. Decreasing it reduces memory usage. - -

    The default value is 1024 (1MB). - -

    LSM_CONFIG_AUTOMERGE -

    -

    If auto-work (the LSM_CONFIG_AUTOWORK option below) is enabled, then - this option is set to the number of segments that the library attempts - to merge simultaneously. Increasing this value may reduce the total - amount of data written to the database file. Decreasing it increases - the amount of data written to the file, but also decreases the average - number of segments present in the file, which can improve the performance - of database read operations. - -

    Additionally, whether or not auto-work is enabled, this option is used - to determine the maximum number of segments of a given age that are - allowed to accumulate in the database file. This is described in the - compulsary work and checkpoints - section below. - -

    The default value is 4. This option must be set to a value between - 2 and 8, inclusive. - -

    LSM_CONFIG_AUTOWORK -

    -

    This option may be set to either 1 (true) or 0 (false). If it is set - to true, then work and checkpoint operations are automatically scheduled - within calls to lsm_insert(), lsm_delete(), lsm_delete_range() and - lsm_commit(). Otherwise, if it is set to false, these operations must - be explicitly scheduled by the - application. - -

    The default value is 1. - -

    LSM_CONFIG_MMAP -

    - If LSM is running on a system with a 64-bit address space, this option - may be set to either 1 (true) or 0 (false). On a 32-bit platform, it is - always set to 0. -

    If it is set to true, the entire database file is memory mapped. Or, if - it is false, data is accessed using ordinary OS file read and write - primitives. Memory mapping the database file can significantly improve the - performance of read operations, as database pages do not have to be copied - from operating system buffers into user space buffers before they can be - examined. -

    This option may not be set if there is a read or write transaction - open on the database. -

    The default value is 1 (true) on a 64-bit platform, and 0 otherwise. - -

    LSM_CONFIG_MULTIPLE_PROCESSES -

    - This option may also be set to either 1 (true) or 0 (false). The default - value is 1 (true). If it is set to false, then the library assumes that all - database clients are located within the same process (have access to the - same memory space). Assuming this means the library can avoid using OS file - locking primitives to lock the database file, which speeds up opening and - closing read and write transactions. - -

    This option can only be set before lsm_open() is called on the database - connection. - -

    If this option is set to false and there is already a connection to the - database from another process when lsm_open() is called, the lsm_open() - call fails with error code LSM_BUSY. - -

    LSM_CONFIG_SAFETY -

    - The effect of this option on data durability - is described above. - -

    From a performance point of view, this option determines how often the - library pauses to wait for data written to the file-system to be stored - on the persistent media (e.g. hard disk or solid-state memory). This is - also known as "syncing" data to disk. Since this is orders of magnitude - slower than simply copying data into operating system buffers, the value - of this option has a large effect on write performance. - -

    If LSM_CONFIG_SAFETY is set to 2 (FULL), then the library syncs the - data written to the log file to disk whenever a transaction is committed. - Or, if LSM_CONFIG_SAFETY is set to 1 (NORMAL), then data is only synced - to disk when a checkpoint is performed (see above). Finally, if it is set - to 0 (OFF), then no data is ever synced to disk. - -

    The default value is 1 (NORMAL). - -

    LSM_CONFIG_USE_LOG -

    - This is another option that may be set to either 1 (true) or 0 (false). - The default value is 1 (true). If it is set to false, then the library - does not write data into the database log file. This makes writing faster, - but also means that if an application crash or power failure occurs, it is - very likely that any recently committed transactions will be lost. - -

    If this option is set to true, then an application crash cannot cause - data loss. Whether or not data loss may occur in the event of a power - failure depends on the value of the - LSM_CONFIG_SAFETY parameter. - -

    This option can only be set if the connection does not currently have - an open write transaction. - -

    - -

    6.3. Work and Checkpoint Scheduling

    - -

    6.3.1. Automatic Scheduling

    - -

    This section describes how work and checkpoint operations are scheduled -if the boolean LSM_CONFIG_AUTOWORK parameter is set to true. Automatic -work operations may occur within calls to any of the following API functions: - -

      -
    • lsm_insert() -
    • lsm_delete() -
    • lsm_delete_range() -
    • lsm_commit() -
    - -

    Each time a transaction is committed in auto-work mode, the library checks -to see if there exists an "old" in-memory tree (see the LSM_CONFIG_AUTOFLUSH -option above). If so, it attempts to flush it to disk immediately. Unlike -merges of existing segments, the entire in-memory tree must be flushed to disk -before control is returned to the user. It is not possible to incrementally -flush an in-memory tree in the same ways as it is possible to incrementally -merge existing database segments together. - -

    Each segment in the database file is assigned an "age" - an integer zero -or greater indicating how many times the data in the segment has been merged. -A segment created by flushing the in-memory tree to disk is assigned an age -of 1. When two or more segments with age=1 are merged together to create a -larger segment, it is assigned an age of 2. And so on. - -

    Assuming auto-work is enabled, the library periodically checks the state of -the database file to see if there exist N or more segments with the same age -value A, where N is the value assigned to the LSM_CONFIG_AUTOMERGE parameter. -If so, work is done to merge all such segments with age=A into a new, larger -segment assigned age=A+1. At present, "periodically" as used above means -roughly once for every 32KB of data (including overhead) written to the -in-memory tree. The merge operation is not necessarily completed within a -single call to a write API (this would result in blocking the writer thread for -too long in many cases - in large databases segments may grow to be many GB in -size). Currently, the amount of data written by a single auto-work operation is -roughly 32KB multiplied by the number of segments in the database file. This -formula may change - the point is that the library attempts to limit the amount -of data written in order to avoid blocking the writer thread for too long -within a single API call. - -

    Checkpoint operations are scheduled based on the value assigned to the -LSM_CONFIG_AUTOCHECKPOINT configuration parameter. - -

    In order to automatically perform work and checkpoint operations, the -client must obtain the WORKER and CHECKPOINTER locks, respectively. If an -attempt to obtain either of these locks fails (because some other client is -already holding them), it is not an error, the scheduled work or checkpoint -is simply not performed. - - - -

    6.3.2. Explicit Scheduling

    - -

    The alternative to automatic scheduling of work and checkpoint operations -is to explicitly schedule them - possibly in a background thread or dedicated -application process. In order to disable automatic work, a client must set -the LSM_CONFIG_AUTOWORK parameter to zero. This parameter is a property of -a database connection, not of a database itself, so it must be cleared -separately by all processes that may write to the database. Otherwise, they -may attempt automatic database work or checkpoints. - - - /* Disable auto-work on connection db */ - int iVal = 0; - lsm_config(db, LSM_CONFIG_AUTOWORK, &iVal); - - -

    The lsm_work() function is used to explicitly perform work on the database: - - - int lsm_work(lsm_db *db, int nMerge, int nKB, int *pnWrite); - - -

    Parameter nKB is passed a limit on the number of KB of data that -should be written to the database file before the call returns. It is a -hint only, the library does not honor this limit strictly. - -

    If the database has an old in-memory tree when lsm_work() is called, it is -flushed to disk. If this means that more than nKB KB of data is written -to the database file, no further work is performed. Otherwise, the number -of KB written is subtracted from nKB before proceeding. - -

    If parameter nMerge is greater than 1, then the library searches for -nMerge or more segments of the same age within the database file and performs -up to nKB KB of work to merge them together. If the merge is completed -before the nKB limit is exceeded, the library searches for another set of -nMerge or more segments to work on, and so on. If at any point no such set of -nMerge segments can be found, the call returns without performing any -further work. - -

    Calling lsm_work() with the nMerge argument set to 1 is used to "optimize" -the database (see below). Passing a value of zero or less for the nMerge -parameter is an error. - -

    In any case, before returning the value of *pnWrite is set to the actual -number of KB written to the database file. - -

    The example code below might be executed in a background thread or process -in order to perform database work and checkpointing. In this case all other -clients should set the LSM_CONFIG_AUTOWORK parameter to zero. - - - int rc; - lsm_db *db; - int nCkpt = 4*1024; /* 4096KB == 4MB */ - - /* Open a database connection to database "test.db". - ** - ** Configure the connection to automatically checkpoint the database after - ** writing each 4MB of data to it (instead of the default 2MB). As well - ** as to auto-work, the LSM_CONFIG_AUTOCHECKPOINT parameter applies to data - ** written by explicit calls to lsm_work(). - */ - lsm_new(0, &db); - lsm_config(db, LSM_CONFIG_AUTOCHECKPOINT, &nCkpt); - lsm_open(db, "test.db"); - - while( 1 ){ - int nWrite; - - /* Attempt up to 512KB of work. Set nWrite to the number of bytes - ** actually written to disk. */ - rc = lsm_work(db, 2, 512, &nWrite); - if( rc!=LSM_OK && rc!=LSM_BUSY ){ - /* Anything other than LSM_OK or LSM_BUSY is a problem. LSM_BUSY - ** indicates that some other client has taken the WORKER lock. Any - ** other error indicates something has gone quite wrong. */ - lsm_close(db); - return rc; - } - - /* nWrite may be set to zero here in two scenarios. lsm_work() - ** may have failed to obtain the WORKER lock and returned LSM_BUSY, - ** indicating that some other connection is working on the database. - ** Alternatively, it may be that there was no old in-memory tree to - ** flush and no two segments of the same age within the database file, - ** meaning the function could find no work to do. - ** - ** In either case, there is no point in calling lsm_work() again - ** immediately. Instead, sleep for a second before continuing. By that - ** time, things may have changed (the other process may have relinquished - ** the WORKER lock, or an in-memory tree may have been marked as old). - */ - if( nWrite==0 ) sleep(1); - } - - -

    The mechanism associated with the LSM_CONFIG_AUTOCHECKPOINT configuration -parameter applies to data written by both automatically scheduled work and -work performed by calls to the lsm_work() function. The amount of -uncheckpointed data that has been written into the database file is a property -of the database file, not a single connection, so checkpoints occur at the -configured interval even if multiple connections are used to work on the -database. - -

    Alternatively, checkpoint operations may be scheduled separately. If the -LSM_CONFIG_AUTOCHECKPOINT parameter is set to zero, then a connection never -performs a database checkpoint, regardless of how much data it or any other -connection writes into the database file. As with LSM_CONFIG_AUTOWORK, this -parameter must be zeroed for all connections that may perform work on the -database. Otherwise, they may perform a checkpoint operation. - -

    The lsm_checkpoint() API is used -to expicitly request a checkpoint. - - - int lsm_checkpoint(lsm_db *db, int *pnKB); - - -

    If no work has been performed on the database since the previous -checkpoint, lsm_checkpoint() sets *pnKB to zero and returns immediately. -Otherwise, it checkpoints the database and sets *pnKB to the number of KB of -data written to the database file since the previous checkpoint. - -

    A database may be queried for the number of KB written to the database -since the most recent checkpoint using the -lsm_info() API function. As follows: - - - int nCkpt; - rc = lsm_info(db, LSM_INFO_CHECKPOINT_SIZE, &nCkpt); - - -

    It may also be queried for the size of the in-memory tree or trees. The -following block of code sets variable nLive to the size of the current -live in-memory tree in KB, and nOld to the size of the old in-memory tree in -KB (or 0 if there is no old in-memory tree). - - - int nOld, nLive; - rc = lsm_info(db, LSM_INFO_TREE_SIZE, &nOld, &nLive); - - -

    6.3.3. Compulsary Work and Checkpoints

    - -

    There are three scenarios where database work or checkpointing may be -performed automatically, regardless of the value of the LSM_CONFIG_AUTOWORK -parameter. - -

      -
    • When closing a database connection, and -
    • When the number of segments with a common age in the database - file grows unacceptably high. -
    • When the total number of segments in the database file grows - unacceptably high. -
    - -

    Whenever an lsm_close() call would mean that the total number of -connections to a database drops to zero, the connection checks if the -in-memory tree is empty. If not, it is flushed to disk. Both the live and -old in-memory trees are flushed to disk in this case. It also checks if the -database file has been modified since the most recent checkpoint was -performed. If so, it also performs a checkpoint. Finally, assuming no error -has occurred, it deletes the log file. - -

    Additionally, whenever a worker wishes to flush an in-memory tree to a new -age=1 segment, it must first ensure that there are less than (N+1) existing -age=1 segments, where N is the value that the LSM_CONFIG_AUTOMERGE parameter is -set to. If there are already (N+1) or more age=1 segments, they must be merged -into an age=2 segment before a new age=1 segment can be created within the -database file. Similar rules apply to segments of other ages - it is not -possible to create a new age=I segment if there are already (N+1) segments -with age=I in the database file. This has two implications: - -

    This scenario should never come about if all connections that write to the -database have auto-work enabled. It only occurs if auto-work is disabled and -the lsm_work() function is called too infrequently. In this case it is possible -that flushing an in-memory tree may require writing a tremendous amount of -data to disk (possibly even rewriting the entire database file). - -

    Finally, regardless of age, a database is limited to a maximum of 64 -segments in total. If an attempt is made to flush an in-memory tree to disk -when the database already contains 64 segments, two or more existing segments -must be merged together before the new segment can be created. - -

    6.4. Database File Optimization

    - -

    Database optimization transforms the contents of database file so that -the following are true: - -

      -

    • All database content is stored in a single - segment. This makes the - data structure equivalent to an optimally packed b-tree stucture - for search operations - minimizing the number of disk sectors that need - to be visted when searching the database. - -

    • The database file contains no (or as little as possible) free space. - In other words, it is no larger than required to contain the single - segment. -

    - -

    In order to optimize the database, lsm_work() should be called with -the nMerge argument set to 1 and the third parameter set to a negative value -(interpreted as "keep working until there is no more work to do"). For -example: - - - rc = lsm_work(db, 1, -1, 0); - - -

    When optimizing the database as above, either the LSM_CONFIG_AUTOCHECKPOINT -parameter should be set to a non-zero value or lsm_checkpoint() should be -called periodically. Otherwise, no checkpoints will be performed, preventing -the library from reusing any space occupied by old segments even after their -content has been merged into the new segment. The result - a database file that -is optimized, except that it is up to twice as large as it otherwise would be. - - - Index: www/varint.wiki ================================================================== --- www/varint.wiki +++ www/varint.wiki @@ -1,19 +1,19 @@ Variable-Length Integers A variable length integer is an encoding of 64-bit unsigned integers into between 1 and 9 bytes. The encoding has the following properties: - 1. Smaller (and more common) values use fewer bytes and take up less space + 1. Smaller (and common) values use fewer bytes and take up less space than larger (and less common) values. 2. The length of any varint can be determined by looking at just the first byte of the encoding. 3. Lexicographical and numeric ordering for varints are the same. Hence if a group of varints are order lexicographically (that is to say, if - they are order by memcmp() with shorter varints coming first) then + they are order by memcmp() with shorted varints coming first) then those varints will also be in numeric order. This property means that varints can be used as keys in the key/value backend storage and the records will occur in numerical order of the keys. The encoding is described by algorithms to decode (convert from