Index: lsm-test/README ================================================================== --- lsm-test/README +++ lsm-test/README @@ -26,9 +26,14 @@ The difference from lsmtest2.c is that this file tests live-recovery (recovery from a failure that occurs while other clients are still running) whereas lsmtest2.c tests recovery from a system or power failure. + + lsmtest9.c: More data tests. These focus on testing that calling + lsm_work(nMerge=1) to compact the database does not corrupt it. + In other words, that databases containing block-redirects + can be read and written. Index: lsm-test/lsmtest.h ================================================================== --- lsm-test/lsmtest.h +++ lsm-test/lsmtest.h @@ -63,14 +63,24 @@ */ int test_kc_open(const char *zFilename, int bClear, TestDb **ppDb); int test_kc_close(TestDb *); int test_kc_write(TestDb *, void *, int , void *, int); int test_kc_delete(TestDb *, void *, int); +int test_kc_delete_range(TestDb *, void *, int, void *, int); int test_kc_fetch(TestDb *, void *, int, void **, int *); int test_kc_scan(TestDb *, void *, int, void *, int, void *, int, void (*)(void *, void *, int , void *, int) ); + +int test_mdb_open(const char *zFilename, int bClear, TestDb **ppDb); +int test_mdb_close(TestDb *); +int test_mdb_write(TestDb *, void *, int , void *, int); +int test_mdb_delete(TestDb *, void *, int); +int test_mdb_fetch(TestDb *, void *, int, void **, int *); +int test_mdb_scan(TestDb *, void *, int, void *, int, void *, int, + void (*)(void *, void *, int , void *, int) +); /* ** Functions in wrapper3.c. This file contains the tdb wrapper for lsm. ** The wrapper for lsm is a bit more involved than the others, as it ** includes code for a couple of different lsm configurations, and for @@ -207,10 +217,13 @@ void test_data_3(const char *, const char *, int *pRc); void testDbContents(TestDb *, Datasource *, int, int, int, int, int, int *); void testCaseProgress(int, int, int, int *); int testCaseNDot(void); +void testCompareDb(Datasource *, int, int, TestDb *, TestDb *, int *); +int testControlDb(TestDb **ppDb); + typedef struct CksumDb CksumDb; CksumDb *testCksumArrayNew(Datasource *, int, int, int); char *testCksumArrayGet(CksumDb *, int); void testCksumArrayFree(CksumDb *); void testCaseStart(int *pRc, char *zFmt, ...); @@ -222,10 +235,13 @@ int testCksumDatabase(TestDb *pDb, char *zOut); int testCountDatabase(TestDb *pDb); void testCompareInt(int, int, int *); void testCompareStr(const char *z1, const char *z2, int *pRc); +/* lsmtest9.c */ +void test_data_4(const char *, const char *, int *pRc); + /* ** Similar to the Tcl_GetIndexFromObjStruct() Tcl library function. */ #define testArgSelect(w,x,y,z) testArgSelectX(w,x,sizeof(w[0]),y,z) Index: lsm-test/lsmtest1.c ================================================================== --- lsm-test/lsmtest1.c +++ lsm-test/lsmtest1.c @@ -90,11 +90,11 @@ ); testFree(zData); return zRet; } -static int testControlDb(TestDb **ppDb){ +int testControlDb(TestDb **ppDb){ #ifdef HAVE_KYOTOCABINET return tdb_open("kyotocabinet", "tmp.db", 1, ppDb); #else return tdb_open("sqlite3", ":memory:", 1, ppDb); #endif @@ -161,11 +161,11 @@ int *pRc /* IN/OUT: Error code */ ){ int j; int rc = *pRc; - if( rc==0 && nScanTest ){ + if( 0 && rc==0 && nScanTest ){ TestDb *pDb2 = 0; /* Open a control db (i.e. one that we assume works) */ rc = testControlDb(&pDb2); @@ -346,11 +346,11 @@ } testFree(zName); } } -static void testCompareDb( +void testCompareDb( Datasource *pData, int nData, int iSeed, TestDb *pControl, TestDb *pDb, @@ -407,11 +407,11 @@ pData = testDatasourceNew(&p->defn); rc = testControlDb(&pControl); if( tdb_lsm(pDb) ){ int nBuf = 32 * 1024 * 1024; - lsm_config(tdb_lsm(pDb), LSM_CONFIG_WRITE_BUFFER, &nBuf); + lsm_config(tdb_lsm(pDb), LSM_CONFIG_AUTOFLUSH, &nBuf); } for(i=0; rc==0 && inIter; i++){ void *pKey1; int nKey1; void *pKey2; int nKey2; Index: lsm-test/lsmtest2.c ================================================================== --- lsm-test/lsmtest2.c +++ lsm-test/lsmtest2.c @@ -265,12 +265,12 @@ CksumDb *pCksumDb; TestDb *pDb; char *zCfg; const char *azConfig[2] = { - "page_size=1024 block_size=65536 write_buffer=16384 safety=2 mmap=0", - "page_size=1024 block_size=65536 write_buffer=16384 safety=2 " + "page_size=1024 block_size=65536 autoflush=16384 safety=2 mmap=0", + "page_size=1024 block_size=65536 autoflush=16384 safety=2 " " compression=1 mmap=0" }; assert( bCompress==0 || bCompress==1 ); /* Allocate datasource. And calculate the expected checksums. */ @@ -277,11 +277,11 @@ pData = testDatasourceNew(&defn); pCksumDb = testCksumArrayNew(pData, nRow, nRow, 1); /* Setup and save the initial database. */ - zCfg = testMallocPrintf("%s nmerge=7", azConfig[bCompress]); + zCfg = testMallocPrintf("%s automerge=7", azConfig[bCompress]); testSetupSavedLsmdb(zCfg, DBNAME, pData, 5000, pRc); testFree(zCfg); for(i=0; inRec / 3); + int iData = 0; + + /* Start the test case, open a database and allocate the datasource. */ + rc = testControlDb(&pControl); + pDb = testOpen(zSystem, 1, &rc); + pData = testDatasourceNew(&p->defn); + if( rc==0 ) db = tdb_lsm(pDb); + + testWriteDatasourceRange(pControl, pData, iData, nRecOn3*3, &rc); + testWriteDatasourceRange(pDb, pData, iData, nRecOn3*3, &rc); + + for(i=0; rc==0 && inRepeat; i++){ + + testDeleteDatasourceRange(pControl, pData, iData, nRecOn3*2, &rc); + testDeleteDatasourceRange(pDb, pData, iData, nRecOn3*2, &rc); + + if( db ){ + int nDone; +#if 0 + fprintf(stderr, "lsm_work() start...\n"); fflush(stderr); +#endif + do { + nDone = 0; + rc = lsm_work(db, 1, (1<<30), &nDone); + }while( rc==0 && nDone>0 ); +#if 0 + fprintf(stderr, "lsm_work() done...\n"); fflush(stderr); +#endif + } + +if( i+1nRepeat ){ + iData += (nRecOn3*2); + testWriteDatasourceRange(pControl, pData, iData+nRecOn3, nRecOn3*2, &rc); + testWriteDatasourceRange(pDb, pData, iData+nRecOn3, nRecOn3*2, &rc); + + testCompareDb(pData, nRecOn3*3, iData, pControl, pDb, &rc); + + /* If Datatest4.bReopen is true, close and reopen the database */ + if( p->bReopen ){ + testReopen(&pDb, &rc); + if( rc==0 ) db = tdb_lsm(pDb); + } +} + + /* Update the progress dots... */ + testCaseProgress(i, p->nRepeat, testCaseNDot(), &iDot); + } + + testClose(&pDb); + testClose(&pControl); + testDatasourceFree(pData); + testCaseFinish(rc); + *pRc = rc; +} + +static char *getName4(const char *zSystem, Datatest4 *pTest){ + char *zRet; + char *zData; + zData = testDatasourceName(&pTest->defn); + zRet = testMallocPrintf("data4.%s.%s.%d.%d.%d", + zSystem, zData, pTest->nRec, pTest->nRepeat, pTest->bReopen + ); + testFree(zData); + return zRet; +} + +void test_data_4( + const char *zSystem, /* Database system name */ + const char *zPattern, /* Run test cases that match this pattern */ + int *pRc /* IN/OUT: Error code */ +){ + Datatest4 aTest[] = { + /* defn, nRec, nRepeat, bReopen */ + { {DATA_RANDOM, 20,25, 500,600}, 10000, 10, 0 }, + { {DATA_RANDOM, 20,25, 500,600}, 10000, 10, 1 }, + }; + + int i; + + for(i=0; *pRc==LSM_OK && ixOpen(pEnv, zDb, &pOut); + rc = pEnv->xOpen(pEnv, zDb, 0, &pOut); if( rc!=LSM_OK ) return rc; while( feof(pInput)==0 ){ char zLine[80]; fgets(zLine, sizeof(zLine)-1, pInput); Index: lsm-test/lsmtest_tdb.c ================================================================== --- lsm-test/lsmtest_tdb.c +++ lsm-test/lsmtest_tdb.c @@ -239,10 +239,18 @@ } static int kc_delete(TestDb *pTestDb, void *pKey, int nKey){ return test_kc_delete(pTestDb, pKey, nKey); } + +static int kc_delete_range( + TestDb *pTestDb, + void *pKey1, int nKey1, + void *pKey2, int nKey2 +){ + return test_kc_delete_range(pTestDb, pKey1, nKey1, pKey2, nKey2); +} static int kc_fetch( TestDb *pTestDb, void *pKey, int nKey, @@ -269,10 +277,11 @@ static int kc_open(const char *zFilename, int bClear, TestDb **ppDb){ static const DatabaseMethods KcdbMethods = { kc_close, kc_write, kc_delete, + kc_delete_range, kc_fetch, kc_scan, error_transaction_function, error_transaction_function, error_transaction_function @@ -293,10 +302,80 @@ #endif /* HAVE_KYOTOCABINET */ /* ** End wrapper for Kyoto cabinet. *************************************************************************/ +#ifdef HAVE_MDB +static int mdb_close(TestDb *pTestDb){ + return test_mdb_close(pTestDb); +} + +static int mdb_write( + TestDb *pTestDb, + void *pKey, + int nKey, + void *pVal, + int nVal +){ + return test_mdb_write(pTestDb, pKey, nKey, pVal, nVal); +} + +static int mdb_delete(TestDb *pTestDb, void *pKey, int nKey){ + return test_mdb_delete(pTestDb, pKey, nKey); +} + +static int mdb_fetch( + TestDb *pTestDb, + void *pKey, + int nKey, + void **ppVal, + int *pnVal +){ + if( pKey==0 ) return LSM_OK; + return test_mdb_fetch(pTestDb, pKey, nKey, ppVal, pnVal); +} + +static int mdb_scan( + TestDb *pTestDb, + void *pCtx, + int bReverse, + void *pFirst, int nFirst, + void *pLast, int nLast, + void (*xCallback)(void *, void *, int , void *, int) +){ + return test_mdb_scan( + pTestDb, pCtx, bReverse, pFirst, nFirst, pLast, nLast, xCallback + ); +} + +static int mdb_open(const char *zFilename, int bClear, TestDb **ppDb){ + static const DatabaseMethods KcdbMethods = { + mdb_close, + mdb_write, + mdb_delete, + 0, + mdb_fetch, + mdb_scan, + error_transaction_function, + error_transaction_function, + error_transaction_function + }; + + int rc; + TestDb *pTestDb = 0; + + rc = test_mdb_open(zFilename, bClear, &pTestDb); + if( rc!=0 ){ + *ppDb = 0; + return rc; + } + pTestDb->pMethods = &KcdbMethods; + *ppDb = pTestDb; + return 0; +} +#endif /* HAVE_MDB */ + /************************************************************************* ** Begin wrapper for SQLite. */ /* @@ -623,11 +702,14 @@ #endif #ifdef HAVE_LEVELDB { "leveldb", "testdb.leveldb", test_leveldb_open }, #endif #ifdef HAVE_KYOTOCABINET - { "kyotocabinet", "testdb.kc", kc_open } + { "kyotocabinet", "testdb.kc", kc_open }, +#endif +#ifdef HAVE_MDB + { "mdb", "./testdb.mdb", mdb_open } #endif }; const char *tdb_system_name(int i){ if( i<0 || i>=ArraySize(aLib) ) return 0; Index: lsm-test/lsmtest_tdb2.cc ================================================================== --- lsm-test/lsmtest_tdb2.cc +++ lsm-test/lsmtest_tdb2.cc @@ -1,156 +1,10 @@ #include "lsmtest.h" #include -#ifdef HAVE_LEVELDB -#include "leveldb/db.h" -extern "C" { - struct LevelDb { - TestDb base; - leveldb::DB* db; - std::string *pRes; - }; -} - -int test_leveldb_open(const char *zFilename, int bClear, TestDb **ppDb){ - LevelDb *pLevelDb; - - if( bClear ){ - char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); - system(zCmd); - sqlite3_free(zCmd); - } - - pLevelDb = (LevelDb *)malloc(sizeof(LevelDb)); - memset(pLevelDb, 0, sizeof(LevelDb)); - - leveldb::Options options; - options.create_if_missing = 1; - leveldb::Status s = leveldb::DB::Open(options, zFilename, &pLevelDb->db); - - if( s.ok() ){ - *ppDb = (TestDb *)pLevelDb; - pLevelDb->pRes = new std::string(""); - }else{ - test_leveldb_close((TestDb *)pLevelDb); - *ppDb = 0; - return 1; - } - return 0; -} - -int test_leveldb_close(TestDb *pDb){ - LevelDb *pLevelDb = (LevelDb *)pDb; - if( pDb ){ - if( pLevelDb->db ) delete pLevelDb->db; - } - free((void *)pDb); - return 0; -} - -int test_leveldb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ - LevelDb *pLevelDb = (LevelDb *)pDb; - leveldb::Status s = pLevelDb->db->Put(leveldb::WriteOptions(), - leveldb::Slice((const char *)pKey, nKey), - leveldb::Slice((const char *)pVal, nVal) - ); - return !s.ok(); -} - -int test_leveldb_delete(TestDb *pDb, void *pKey, int nKey){ - LevelDb *pLevelDb = (LevelDb *)pDb; - leveldb::Status s = pLevelDb->db->Delete(leveldb::WriteOptions(), - leveldb::Slice((const char *)pKey, nKey) - ); - return !s.ok(); -} - -int test_leveldb_fetch( - TestDb *pDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - LevelDb *pLevelDb = (LevelDb *)pDb; - pLevelDb->pRes->assign(""); - leveldb::Status s = pLevelDb->db->Get( - leveldb::ReadOptions(), - leveldb::Slice((const char *)pKey, nKey), - pLevelDb->pRes - ); - if( s.ok() ){ - *ppVal = (void *)pLevelDb->pRes->data(); - *pnVal = (int)pLevelDb->pRes->length(); - }else{ - *pnVal = -1; - *ppVal = 0; - } - return 0; -} - -int test_leveldb_scan( - TestDb *pDb, /* Database handle */ - void *pCtx, /* Context pointer to pass to xCallback */ - int bReverse, /* True to iterate in reverse order */ - void *pKey1, int nKey1, /* Start of search */ - void *pKey2, int nKey2, /* End of search */ - void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) -){ - LevelDb *pLevelDb = (LevelDb *)pDb; - leveldb::Iterator *pIter; - - leveldb::Slice sKey1((const char *)pKey1, nKey1); - leveldb::Slice sKey2((const char *)pKey2, nKey2); - - pIter = pLevelDb->db->NewIterator(leveldb::ReadOptions()); - if( bReverse==0 ){ - if( pKey1 ){ - pIter->Seek(sKey1); - }else{ - pIter->SeekToFirst(); - } - }else{ - if( pKey2 ){ - pIter->Seek(sKey2); - if( pIter->Valid()==0 ){ - pIter->SeekToLast(); - }else{ - int res = pIter->key().compare(sKey2); - assert( res>=0 ); - if( res>0 ){ - pIter->Prev(); - } - assert( pIter->Valid()==0 || pIter->key().compare(sKey2)<=0 ); - } - }else{ - pIter->SeekToLast(); - } - } - - while( pIter->Valid() ){ - if( (bReverse==0 && pKey2 && pIter->key().compare(sKey2)>0) ) break; - if( (bReverse!=0 && pKey1 && pIter->key().compare(sKey1)<0) ) break; - - leveldb::Slice k = pIter->key(); - leveldb::Slice v = pIter->value(); - xCallback(pCtx, (void *)k.data(), k.size(), (void *)v.data(), v.size()); - - if( bReverse==0 ){ - pIter->Next(); - }else{ - pIter->Prev(); - } - } - - delete pIter; - return LSM_OK; -} -#endif - #ifdef HAVE_KYOTOCABINET #include "kcpolydb.h" extern "C" { struct KcDb { TestDb base; @@ -216,10 +70,54 @@ int ok; ok = pKcDb->db->remove((const char *)pKey, nKey); return (ok ? 0 : 1); } + +int test_kc_delete_range( + TestDb *pDb, + void *pKey1, int nKey1, + void *pKey2, int nKey2 +){ + int res; + KcDb *pKcDb = (KcDb *)pDb; + kyotocabinet::DB::Cursor* pCur = pKcDb->db->cursor(); + + if( pKey1 ){ + res = pCur->jump((const char *)pKey1, nKey1); + }else{ + res = pCur->jump(); + } + + while( 1 ){ + const char *pKey; size_t nKey; + const char *pVal; size_t nVal; + + pKey = pCur->get(&nKey, &pVal, &nVal); + if( pKey==0 ) break; + +#ifndef NDEBUG + if( pKey1 ){ + res = memcmp(pKey, pKey1, MIN((size_t)nKey1, nKey)); + assert( res>0 || (res==0 && nKey>nKey1) ); + } +#endif + + if( pKey2 ){ + res = memcmp(pKey, pKey2, MIN((size_t)nKey2, nKey)); + if( res>0 || (res==0 && (size_t)nKey2remove(); + delete [] pKey; + } + + delete pCur; + return 0; +} int test_kc_fetch( TestDb *pDb, void *pKey, int nKey, @@ -302,7 +200,166 @@ } delete pCur; return 0; } -#endif +#endif /* HAVE_KYOTOCABINET */ + +#ifdef HAVE_MDB +#include "lmdb.h" + +extern "C" { + struct MdbDb { + TestDb base; + MDB_env *env; + MDB_dbi dbi; + }; +} + +int test_mdb_open(const char *zFilename, int bClear, TestDb **ppDb){ + MDB_txn *txn; + MdbDb *pMdb; + int rc; + + if( bClear ){ + char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); + system(zCmd); + sqlite3_free(zCmd); + } + + pMdb = (MdbDb *)malloc(sizeof(MdbDb)); + memset(pMdb, 0, sizeof(MdbDb)); + + rc = mdb_env_create(&pMdb->env); + if( rc==0 ) rc = mdb_env_set_mapsize(pMdb->env, 1*1024*1024*1024); + if( rc==0 ) rc = mdb_env_open(pMdb->env, zFilename, MDB_NOSYNC|MDB_NOSUBDIR, 0600); + if( rc==0 ) rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); + if( rc==0 ){ + rc = mdb_open(txn, NULL, 0, &pMdb->dbi); + mdb_txn_commit(txn); + } + + *ppDb = (TestDb *)pMdb; + return rc; +} + +int test_mdb_close(TestDb *pDb){ + MdbDb *pMdb = (MdbDb *)pDb; + + mdb_close(pMdb->env, pMdb->dbi); + mdb_env_close(pMdb->env); + free(pMdb); + return 0; +} + +int test_mdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ + int rc; + MdbDb *pMdb = (MdbDb *)pDb; + MDB_val val; + MDB_val key; + MDB_txn *txn; + + val.mv_size = nVal; + val.mv_data = pVal; + key.mv_size = nKey; + key.mv_data = pKey; + + rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); + if( rc==0 ){ + rc = mdb_put(txn, pMdb->dbi, &key, &val, 0); + if( rc==0 ){ + rc = mdb_txn_commit(txn); + }else{ + mdb_txn_abort(txn); + } + } + + return rc; +} + +int test_mdb_delete(TestDb *pDb, void *pKey, int nKey){ + int rc; + MdbDb *pMdb = (MdbDb *)pDb; + MDB_val key; + MDB_txn *txn; + + key.mv_size = nKey; + key.mv_data = pKey; + rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); + if( rc==0 ){ + rc = mdb_del(txn, pMdb->dbi, &key, 0); + if( rc==0 ){ + rc = mdb_txn_commit(txn); + }else{ + mdb_txn_abort(txn); + } + } + + return rc; +} + +int test_mdb_fetch( + TestDb *pDb, + void *pKey, + int nKey, + void **ppVal, + int *pnVal +){ + int rc; + MdbDb *pMdb = (MdbDb *)pDb; + MDB_val key; + MDB_txn *txn; + + key.mv_size = nKey; + key.mv_data = pKey; + + rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn); + if( rc==0 ){ + MDB_val val = {0, 0}; + rc = mdb_get(txn, pMdb->dbi, &key, &val); + if( rc==MDB_NOTFOUND ){ + rc = 0; + *ppVal = 0; + *pnVal = -1; + }else{ + *ppVal = val.mv_data; + *pnVal = val.mv_size; + } + mdb_txn_commit(txn); + } + + return rc; +} + +int test_mdb_scan( + TestDb *pDb, /* Database handle */ + void *pCtx, /* Context pointer to pass to xCallback */ + int bReverse, /* True for a reverse order scan */ + void *pKey1, int nKey1, /* Start of search */ + void *pKey2, int nKey2, /* End of search */ + void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) +){ + MdbDb *pMdb = (MdbDb *)pDb; + int rc; + MDB_cursor_op op = bReverse ? MDB_PREV : MDB_NEXT; + MDB_txn *txn; + + rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn); + if( rc==0 ){ + MDB_cursor *csr; + MDB_val key = {0, 0}; + MDB_val val = {0, 0}; + + rc = mdb_cursor_open(txn, pMdb->dbi, &csr); + if( rc==0 ){ + while( mdb_cursor_get(csr, &key, &val, op)==0 ){ + xCallback(pCtx, key.mv_data, key.mv_size, val.mv_data, val.mv_size); + } + mdb_cursor_close(csr); + } + } + + return rc; +} + +#endif /* HAVE_MDB */ Index: lsm-test/lsmtest_tdb3.c ================================================================== --- lsm-test/lsmtest_tdb3.c +++ lsm-test/lsmtest_tdb3.c @@ -1,9 +1,8 @@ #include "lsmtest_tdb.h" #include "lsm.h" - #include "lsmtest.h" #include #include #include @@ -14,27 +13,41 @@ typedef struct LsmDb LsmDb; typedef struct LsmWorker LsmWorker; typedef struct LsmFile LsmFile; +#define LSMTEST_DFLT_MT_MAX_CKPT (8*1024) +#define LSMTEST_DFLT_MT_MIN_CKPT (2*1024) + #ifdef LSM_MUTEX_PTHREADS #include + +#define LSMTEST_THREAD_CKPT 1 +#define LSMTEST_THREAD_WORKER 2 +#define LSMTEST_THREAD_WORKER_AC 3 + +/* +** There are several different types of worker threads that run in different +** test configurations, depending on the value of LsmWorker.eType. +** +** 1. Checkpointer. +** 2. Worker with auto-checkpoint. +** 3. Worker without auto-checkpoint. +*/ struct LsmWorker { LsmDb *pDb; /* Main database structure */ lsm_db *pWorker; /* Worker database handle */ pthread_t worker_thread; /* Worker thread */ pthread_cond_t worker_cond; /* Condition var the worker waits on */ pthread_mutex_t worker_mutex; /* Mutex used with worker_cond */ int bDoWork; /* Set to true by client when there is work */ int worker_rc; /* Store error code here */ - - int lsm_work_flags; /* Flags to pass to lsm_work() */ - int lsm_work_npage; /* nPage parameter to pass to lsm_work() */ - int bCkpt; /* True to call lsm_checkpoint() */ + int eType; /* LSMTEST_THREAD_XXX constant */ + int bBlock; }; #else -struct LsmWorker { int worker_rc; }; +struct LsmWorker { int worker_rc; int bBlock; }; #endif static void mt_shutdown(LsmDb *); lsm_env *tdb_lsm_env(void){ @@ -105,13 +118,21 @@ /* IO logging hook */ void (*xWriteHook)(void *, int, lsm_i64, int, int); void *pWriteCtx; /* Worker threads (for lsm_mt) */ + int nMtMinCkpt; + int nMtMaxCkpt; + int eMode; int nWorker; LsmWorker *aWorker; }; + +#define LSMTEST_MODE_SINGLETHREAD 1 +#define LSMTEST_MODE_BACKGROUND_CKPT 2 +#define LSMTEST_MODE_BACKGROUND_WORK 3 +#define LSMTEST_MODE_BACKGROUND_BOTH 4 /************************************************************************* ************************************************************************** ** Begin test VFS code. */ @@ -133,10 +154,11 @@ } static int testEnvOpen( lsm_env *pEnv, /* Environment for current LsmDb */ const char *zFile, /* Name of file to open */ + int flags, lsm_file **ppFile /* OUT: New file handle object */ ){ lsm_env *pRealEnv = tdb_lsm_env(); LsmDb *pDb = (LsmDb *)pEnv->pVfsCtx; int rc; /* Return Code */ @@ -146,11 +168,11 @@ nFile = strlen(zFile); pRet = (LsmFile *)testMalloc(sizeof(LsmFile)); pRet->pDb = pDb; pRet->bLog = (nFile > 4 && 0==memcmp("-log", &zFile[nFile-4], 4)); - rc = pRealEnv->xOpen(pRealEnv, zFile, &pRet->pReal); + rc = pRealEnv->xOpen(pRealEnv, zFile, flags, &pRet->pReal); if( rc!=LSM_OK ){ testFree(pRet); pRet = 0; } @@ -357,11 +379,11 @@ for(iFile=0; iFile<2; iFile++){ lsm_file *pFile = 0; int i; - pEnv->xOpen(pEnv, zFile, &pFile); + pEnv->xOpen(pEnv, zFile, 0, &pFile); for(i=0; iaFile[iFile].nSector; i++){ u8 *aOld = pDb->aFile[iFile].aSector[i].aOld; if( aOld ){ int iOpt = testPrngValue(iSeed++) % 3; switch( iOpt ){ @@ -431,12 +453,12 @@ return (rc==Z_OK ? 0 : LSM_ERROR); } static int testConfigureCompression(lsm_db *pDb){ static lsm_compress zip = { - 1, sizeof(lsm_compress), 0, /* Context pointer (unused) */ + 1, /* Id value */ testZipBound, /* xBound method */ testZipCompress, /* xCompress method */ testZipUncompress /* xUncompress method */ }; return lsm_config(pDb, LSM_CONFIG_SET_COMPRESSION, &zip); @@ -474,38 +496,76 @@ memset(pDb, sizeof(LsmDb), 0x11); testFree((char *)pDb->pBuf); testFree((char *)pDb); return rc; } + +static int waitOnCheckpointer(LsmDb *pDb, lsm_db *db){ + int nSleep = 0; + int nKB; + int rc; + + do { + nKB = 0; + rc = lsm_info(db, LSM_INFO_CHECKPOINT_SIZE, &nKB); + if( rc!=LSM_OK || nKBnMtMaxCkpt ) break; + usleep(5000); + nSleep += 5; + }while( 1 ); + +#if 0 + if( nSleep ) printf("# waitOnCheckpointer(): nSleep=%d\n", nSleep); +#endif + + return rc; +} + +static int waitOnWorker(LsmDb *pDb){ + int rc; + int nLimit = -1; + int nSleep = 0; + + rc = lsm_config(pDb->db, LSM_CONFIG_AUTOFLUSH, &nLimit); + do { + int nOld, nNew, rc; + rc = lsm_info(pDb->db, LSM_INFO_TREE_SIZE, &nOld, &nNew); + if( rc!=LSM_OK ) return rc; + if( nOld==0 || nNew<(nLimit/2) ) break; + usleep(5000); + nSleep += 5; + }while( 1 ); + +#if 0 + if( nSleep ) printf("# waitOnWorker(): nSleep=%d\n", nSleep); +#endif + + return rc; +} static int test_lsm_write( TestDb *pTestDb, void *pKey, int nKey, - void *pVal, + void *pVal, int nVal ){ LsmDb *pDb = (LsmDb *)pTestDb; - - if( pDb->aWorker ){ - int nLimit = -1; - int nSleep = 0; - lsm_config(pDb->db, LSM_CONFIG_WRITE_BUFFER, &nLimit); - do { - int bOld, nNew, rc; - rc = lsm_tree_size(pDb->db, &bOld, &nNew); - if( rc!=LSM_OK ) return rc; - if( bOld==0 || nNewdb, pKey, nKey, pVal, nVal); + int rc = LSM_OK; + + if( pDb->eMode==LSMTEST_MODE_BACKGROUND_CKPT ){ + rc = waitOnCheckpointer(pDb, pDb->db); + }else if( + pDb->eMode==LSMTEST_MODE_BACKGROUND_WORK + || pDb->eMode==LSMTEST_MODE_BACKGROUND_BOTH + ){ + rc = waitOnWorker(pDb); + } + + if( rc==LSM_OK ){ + rc = lsm_insert(pDb->db, pKey, nKey, pVal, nVal); + } + return rc; } static int test_lsm_delete(TestDb *pTestDb, void *pKey, int nKey){ LsmDb *pDb = (LsmDb *)pTestDb; return lsm_delete(pDb->db, pKey, nKey); @@ -670,12 +730,15 @@ LsmDb *p = (LsmDb *)pArg; if( p->xWork ) p->xWork(db, p->pWorkCtx); } #define TEST_NO_RECOVERY -1 -#define TEST_THREADS -2 #define TEST_COMPRESSION -3 + +#define TEST_MT_MODE -2 +#define TEST_MT_MIN_CKPT -4 +#define TEST_MT_MAX_CKPT -5 int test_lsm_config_str( LsmDb *pLsm, lsm_db *db, int bWorker, @@ -685,25 +748,29 @@ struct CfgParam { const char *zParam; int bWorker; int eParam; } aParam[] = { - { "write_buffer", 0, LSM_CONFIG_WRITE_BUFFER }, + { "autoflush", 0, LSM_CONFIG_AUTOFLUSH }, { "page_size", 0, LSM_CONFIG_PAGE_SIZE }, { "block_size", 0, LSM_CONFIG_BLOCK_SIZE }, { "safety", 0, LSM_CONFIG_SAFETY }, { "autowork", 0, LSM_CONFIG_AUTOWORK }, { "autocheckpoint", 0, LSM_CONFIG_AUTOCHECKPOINT }, - { "log_size", 0, LSM_CONFIG_LOG_SIZE }, { "mmap", 0, LSM_CONFIG_MMAP }, { "use_log", 0, LSM_CONFIG_USE_LOG }, - { "nmerge", 0, LSM_CONFIG_NMERGE }, + { "automerge", 0, LSM_CONFIG_AUTOMERGE }, { "max_freelist", 0, LSM_CONFIG_MAX_FREELIST }, { "multi_proc", 0, LSM_CONFIG_MULTIPLE_PROCESSES }, - { "worker_nmerge", 1, LSM_CONFIG_NMERGE }, + { "worker_automerge", 1, LSM_CONFIG_AUTOMERGE }, { "test_no_recovery", 0, TEST_NO_RECOVERY }, - { "threads", 0, TEST_THREADS }, + { "bg_min_ckpt", 0, TEST_NO_RECOVERY }, + + { "mt_mode", 0, TEST_MT_MODE }, + { "mt_min_ckpt", 0, TEST_MT_MIN_CKPT }, + { "mt_max_ckpt", 0, TEST_MT_MAX_CKPT }, + #ifdef HAVE_ZLIB { "compression", 0, TEST_COMPRESSION }, #endif { 0, 0 } }; @@ -758,13 +825,19 @@ }else{ switch( eParam ){ case TEST_NO_RECOVERY: if( pLsm ) pLsm->bNoRecovery = iVal; break; - case TEST_THREADS: + case TEST_MT_MODE: if( pLsm ) nThread = iVal; break; + case TEST_MT_MIN_CKPT: + if( pLsm && iVal>0 ) pLsm->nMtMinCkpt = iVal; + break; + case TEST_MT_MAX_CKPT: + if( pLsm && iVal>0 ) pLsm->nMtMaxCkpt = iVal; + break; #ifdef HAVE_ZLIB case TEST_COMPRESSION: testConfigureCompression(db); break; #endif @@ -774,10 +847,14 @@ goto syntax_error; } } if( pnThread ) *pnThread = nThread; + if( pLsm && pLsm->nMtMaxCkpt < pLsm->nMtMinCkpt ){ + pLsm->nMtMinCkpt = pLsm->nMtMaxCkpt; + } + return 0; syntax_error: testPrintError("syntax error at: \"%s\"\n", z); return 1; } @@ -841,10 +918,14 @@ ** Todo: There should be an OS method to obtain this value - just as ** there is in SQLite. For now, LSM assumes that it is smaller than ** the page size (default 4KB). */ pDb->szSector = 256; + + /* Default values for the mt_min_ckpt and mt_max_ckpt parameters. */ + pDb->nMtMinCkpt = LSMTEST_DFLT_MT_MIN_CKPT; + pDb->nMtMaxCkpt = LSMTEST_DFLT_MT_MAX_CKPT; memcpy(&pDb->env, tdb_lsm_env(), sizeof(lsm_env)); pDb->env.pVfsCtx = (void *)pDb; pDb->env.xFullpath = testEnvFullpath; pDb->env.xOpen = testEnvOpen; @@ -870,13 +951,14 @@ lsm_config_work_hook(pDb->db, xWorkHook, (void *)pDb); rc = test_lsm_config_str(pDb, pDb->db, 0, zCfg, &nThread); if( rc==LSM_OK ) rc = lsm_open(pDb->db, zFilename); + pDb->eMode = nThread; #ifdef LSM_MUTEX_PTHREADS - if( rc==LSM_OK && (nThread==2 || nThread==3) ){ - testLsmStartWorkers(pDb, nThread-1, zFilename, zCfg); + if( rc==LSM_OK && nThread>1 ){ + testLsmStartWorkers(pDb, nThread, zFilename, zCfg); } #endif if( rc!=LSM_OK ){ test_lsm_close((TestDb *)pDb); @@ -895,22 +977,23 @@ int test_lsm_small_open( const char *zFile, int bClear, TestDb **ppDb ){ - const char *zCfg = "page_size=256 block_size=65536"; + const char *zCfg = "page_size=256 block_size=64"; return testLsmOpen(zCfg, zFile, bClear, ppDb); } int test_lsm_lomem_open( const char *zFilename, int bClear, TestDb **ppDb ){ + /* "max_freelist=4 autocheckpoint=32" */ const char *zCfg = - "page_size=256 block_size=65536 write_buffer=16384 " - "max_freelist=2 autocheckpoint=32768 " + "page_size=256 block_size=64 autoflush=16 " + "autocheckpoint=32" "mmap=0 " ; return testLsmOpen(zCfg, zFilename, bClear, ppDb); } @@ -918,13 +1001,12 @@ const char *zFilename, int bClear, TestDb **ppDb ){ const char *zCfg = - "page_size=256 block_size=65536 write_buffer=16384 " - "max_freelist=2 autocheckpoint=32768 compression=1" - "mmap=0 " + "page_size=256 block_size=64 autoflush=16 " + "autocheckpoint=32 compression=1 mmap=0 " ; return testLsmOpen(zCfg, zFilename, bClear, ppDb); } lsm_db *tdb_lsm(TestDb *pDb){ @@ -935,12 +1017,10 @@ } void tdb_lsm_enable_log(TestDb *pDb, int bEnable){ lsm_db *db = tdb_lsm(pDb); if( db ){ - LsmDb *p = (LsmDb *)pDb; - int i; lsm_config_log(db, (bEnable ? xLog : 0), (void *)"client"); } } void tdb_lsm_application_crash(TestDb *pDb){ @@ -1025,10 +1105,13 @@ p->bDoWork = 1; pthread_cond_signal(&p->worker_cond); pthread_mutex_unlock(&p->worker_mutex); } +/* +** This routine is used as the main() for all worker threads. +*/ static void *worker_main(void *pArg){ LsmWorker *p = (LsmWorker *)pArg; lsm_db *pWorker; /* Connection to access db through */ pthread_mutex_lock(&p->worker_mutex); @@ -1035,49 +1118,43 @@ while( (pWorker = p->pWorker) ){ int rc = LSM_OK; int nCkpt = -1; /* Do some work. If an error occurs, exit. */ + pthread_mutex_unlock(&p->worker_mutex); - - if( p->bCkpt ){ - rc = lsm_checkpoint(pWorker, 0); + if( p->eType==LSMTEST_THREAD_CKPT ){ + int nKB = 0; + rc = lsm_info(pWorker, LSM_INFO_CHECKPOINT_SIZE, &nKB); + if( rc==LSM_OK && nKB>=p->pDb->nMtMinCkpt ){ + rc = lsm_checkpoint(pWorker, 0); + } }else{ - int nWrite = 0; /* Pages written by lsm_work() call */ - int nAuto = -1; /* Configured AUTOCHECKPOINT value */ - int nLimit = -1; /* Configured WRITE_BUFFER value */ - - lsm_config(pWorker, LSM_CONFIG_WRITE_BUFFER, &nLimit); - lsm_config(pWorker, LSM_CONFIG_AUTOCHECKPOINT, &nAuto); + int nWrite; do { - int nSleep = 0; - lsm_ckpt_size(pWorker, &nCkpt); - while( nAuto==0 && nCkpt>(nLimit*4) ){ - usleep(1000); - mt_signal_worker(p->pDb, 1); - nSleep++; - lsm_ckpt_size(pWorker, &nCkpt); - } -#if 0 - if( nSleep ) printf("nLimit=%d nSleep=%d (worker)\n", nLimit, nSleep); -#endif - - rc = lsm_work(pWorker, p->lsm_work_flags, p->lsm_work_npage, &nWrite); - if( nAuto==0 && nWrite && rc==LSM_OK ) mt_signal_worker(p->pDb, 1); + + if( p->eType==LSMTEST_THREAD_WORKER ){ + waitOnCheckpointer(p->pDb, pWorker); + } + + nWrite = 0; + rc = lsm_work(pWorker, 0, 256, &nWrite); + + if( p->eType==LSMTEST_THREAD_WORKER && nWrite ){ + mt_signal_worker(p->pDb, 1); + } }while( nWrite && p->pWorker ); } pthread_mutex_lock(&p->worker_mutex); if( rc!=LSM_OK && rc!=LSM_BUSY ){ p->worker_rc = rc; break; } - /* If the call to lsm_work() indicates that there is nothing more - ** to do at this point, wait on the condition variable. The thread will - ** wake up when it is signaled either because the client thread has - ** flushed an in-memory tree into the db file or when the connection + /* The thread will wake up when it is signaled either because another + ** thread has created some work for this one or because the connection ** is being closed. */ if( p->pWorker && p->bDoWork==0 ){ pthread_cond_wait(&p->worker_cond, &p->worker_mutex); } p->bDoWork = 0; @@ -1124,16 +1201,15 @@ ** This implies there may be work to do on the database, so signal ** the worker threads. */ static void mt_client_work_hook(lsm_db *db, void *pArg){ LsmDb *pDb = (LsmDb *)pArg; /* LsmDb database handle */ - int i; /* Iterator variable */ /* Invoke the user level work-hook, if any. */ if( pDb->xWork ) pDb->xWork(db, pDb->pWorkCtx); - /* Signal the lsm_work() thread */ + /* Wake up worker thread 0. */ mt_signal_worker(pDb, 0); } static void mt_worker_work_hook(lsm_db *db, void *pArg){ LsmDb *pDb = (LsmDb *)pArg; /* LsmDb database handle */ @@ -1147,24 +1223,24 @@ */ static int mt_start_worker( LsmDb *pDb, /* Main database structure */ int iWorker, /* Worker number to start */ const char *zFilename, /* File name of database to open */ - const char *zCfg, - int flags, /* flags parameter to lsm_work() */ - int nPage, /* nPage parameter to lsm_work() */ - int bCkpt /* True to call lsm_checkpoint() */ + const char *zCfg, /* Connection configuration string */ + int eType /* Type of worker thread */ ){ int rc = 0; /* Return code */ LsmWorker *p; /* Object to initialize */ assert( iWorkernWorker ); + assert( eType==LSMTEST_THREAD_CKPT + || eType==LSMTEST_THREAD_WORKER + || eType==LSMTEST_THREAD_WORKER_AC + ); p = &pDb->aWorker[iWorker]; - p->lsm_work_flags = flags; - p->lsm_work_npage = nPage; - p->bCkpt = bCkpt; + p->eType = eType; p->pDb = pDb; /* Open the worker connection */ if( rc==0 ) rc = lsm_new(&pDb->env, &p->pWorker); if( zCfg ){ @@ -1175,10 +1251,14 @@ /* Configure the work-hook */ if( rc==0 ){ lsm_config_work_hook(p->pWorker, mt_worker_work_hook, (void *)pDb); } + + if( eType==LSMTEST_THREAD_WORKER ){ + test_lsm_config_str(0, p->pWorker, 1, "autocheckpoint=0", 0); + } /* Kick off the worker thread. */ if( rc==0 ) rc = pthread_cond_init(&p->worker_cond, 0); if( rc==0 ) rc = pthread_mutex_init(&p->worker_mutex, 0); if( rc==0 ) rc = pthread_create(&p->worker_thread, 0, worker_main, (void *)p); @@ -1186,43 +1266,60 @@ return rc; } static int testLsmStartWorkers( - LsmDb *pDb, int nWorker, const char *zFilename, const char *zCfg + LsmDb *pDb, int eModel, const char *zFilename, const char *zCfg ){ int rc; - int bAutowork = 0; - assert( nWorker==1 || nWorker==2 ); + + if( eModel<1 || eModel>4 ) return 1; + if( eModel==1 ) return 0; - /* Configure a work-hook for the client connection. */ + /* Configure a work-hook for the client connection. Worker 0 is signalled + ** every time the users connection writes to the database. */ lsm_config_work_hook(pDb->db, mt_client_work_hook, (void *)pDb); - pDb->aWorker = (LsmWorker *)testMalloc(sizeof(LsmWorker) * nWorker); - memset(pDb->aWorker, 0, sizeof(LsmWorker) * nWorker); - pDb->nWorker = nWorker; - - if( nWorker==1 ){ - rc = mt_start_worker(pDb, 0, zFilename, zCfg, 0, 256, 0); - }else{ - rc = mt_start_worker(pDb, 0, zFilename, zCfg, 0, 256, 0); - if( rc==LSM_OK ){ - rc = mt_start_worker(pDb, 1, zFilename, zCfg, 0, 0, 1); - } + /* Allocate space for two worker connections. They may not both be + ** used, but both are allocated. */ + pDb->aWorker = (LsmWorker *)testMalloc(sizeof(LsmWorker) * 2); + memset(pDb->aWorker, 0, sizeof(LsmWorker) * 2); + + switch( eModel ){ + case LSMTEST_MODE_BACKGROUND_CKPT: + pDb->nWorker = 1; + test_lsm_config_str(0, pDb->db, 0, "autocheckpoint=0", 0); + rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_CKPT); + break; + + case LSMTEST_MODE_BACKGROUND_WORK: + pDb->nWorker = 1; + test_lsm_config_str(0, pDb->db, 0, "autowork=0", 0); + rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_WORKER_AC); + break; + + case LSMTEST_MODE_BACKGROUND_BOTH: + pDb->nWorker = 2; + test_lsm_config_str(0, pDb->db, 0, "autowork=0", 0); + rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_WORKER); + if( rc==0 ){ + rc = mt_start_worker(pDb, 1, zFilename, zCfg, LSMTEST_THREAD_CKPT); + } + break; } return rc; } int test_lsm_mt2(const char *zFilename, int bClear, TestDb **ppDb){ - const char *zCfg = "threads=2"; + const char *zCfg = "mt_mode=2"; return testLsmOpen(zCfg, zFilename, bClear, ppDb); } int test_lsm_mt3(const char *zFilename, int bClear, TestDb **ppDb){ - const char *zCfg = "threads=3"; + const char *zCfg = "mt_mode=4"; return testLsmOpen(zCfg, zFilename, bClear, ppDb); } #else static void mt_shutdown(LsmDb *pDb) { Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -62,36 +62,31 @@ # TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 TCCX += -I$(TOP)/ext/async - -# Object files for the SQLite library. -# -FTS3_OBJ = fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ - fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o \ - fts3_write.o -# To remove fts3 (if it won't compile for you), unset FTS3_OBJ: -# FTS3_OBJ = - -LIBOBJ+= alter.o analyze.o attach.o auth.o \ +TCPPX = g++ -Wall -g -I. -I$(TOP)/src $(OPTS) + + +LIBOBJ+= vdbe.o parse.o \ + alter.o analyze.o attach.o auth.o \ build.o \ callback.o complete.o ctime.o date.o delete.o expr.o fault.o fkey.o \ - $(FTS3_OBJ) \ + fts5.o fts5func.o \ func.o global.o hash.o \ icu.o insert.o kv.o kvlsm.o kvmem.o legacy.o \ lsm_ckpt.o lsm_file.o lsm_log.o lsm_main.o lsm_mem.o lsm_mutex.o \ lsm_shared.o lsm_str.o lsm_sorted.o lsm_tree.o \ lsm_unix.o lsm_varint.o \ main.o malloc.o math.o mem0.o mem1.o mem2.o mem3.o mem5.o \ mutex.o mutex_noop.o mutex_unix.o mutex_w32.o \ opcodes.o os.o \ - parse.o pragma.o prepare.o printf.o \ + pragma.o prepare.o printf.o \ random.o resolve.o rowset.o rtree.o select.o status.o \ tokenize.o trigger.o \ update.o util.o varint.o \ - vdbe.o vdbeapi.o vdbeaux.o vdbecodec.o vdbecursor.o \ + vdbeapi.o vdbeaux.o vdbecodec.o vdbecursor.o \ vdbemem.o vdbetrace.o \ walker.o where.o utf.o # All of the source code files. # @@ -107,10 +102,12 @@ $(TOP)/src/date.c \ $(TOP)/src/delete.c \ $(TOP)/src/expr.c \ $(TOP)/src/fault.c \ $(TOP)/src/fkey.c \ + $(TOP)/src/fts5.c \ + $(TOP)/src/fts5func.c \ $(TOP)/src/func.c \ $(TOP)/src/global.c \ $(TOP)/src/hash.c \ $(TOP)/src/hash.h \ $(TOP)/src/hwtime.h \ @@ -218,12 +215,10 @@ # Source code to the test files. # TESTSRC = \ - $(TOP)/ext/fts3/fts3_term.c \ - $(TOP)/ext/fts3/fts3_test.c \ $(TOP)/test/test_main.c \ $(TOP)/test/test_thread0.c \ $(TOP)/test/test_utf.c \ $(TOP)/test/test_misc1.c \ $(TOP)/test/test_config.c \ @@ -244,10 +239,12 @@ TESTSRC2 = \ $(TOP)/src/attach.c \ $(TOP)/src/build.c \ $(TOP)/src/date.c \ $(TOP)/src/expr.c \ + $(TOP)/src/fts5.c \ + $(TOP)/src/fts5func.c \ $(TOP)/src/func.c \ $(TOP)/src/insert.c \ $(TOP)/src/mem5.c \ $(TOP)/src/os.c \ $(TOP)/src/pragma.c \ @@ -262,15 +259,10 @@ $(TOP)/src/vdbeaux.c \ $(TOP)/src/vdbe.c \ $(TOP)/src/vdbemem.c \ $(TOP)/src/where.c \ parse.c \ - $(TOP)/ext/fts3/fts3.c \ - $(TOP)/ext/fts3/fts3_aux.c \ - $(TOP)/ext/fts3/fts3_expr.c \ - $(TOP)/ext/fts3/fts3_tokenizer.c \ - $(TOP)/ext/fts3/fts3_write.c # Header files used by all library source files. # HDR = \ $(TOP)/src/hash.h \ @@ -301,10 +293,11 @@ LSMTESTSRC = $(TOP)/lsm-test/lsmtest1.c $(TOP)/lsm-test/lsmtest2.c \ $(TOP)/lsm-test/lsmtest3.c $(TOP)/lsm-test/lsmtest4.c \ $(TOP)/lsm-test/lsmtest5.c $(TOP)/lsm-test/lsmtest6.c \ $(TOP)/lsm-test/lsmtest7.c $(TOP)/lsm-test/lsmtest8.c \ + $(TOP)/lsm-test/lsmtest9.c \ $(TOP)/lsm-test/lsmtest_datasource.c \ $(TOP)/lsm-test/lsmtest_func.c $(TOP)/lsm-test/lsmtest_io.c \ $(TOP)/lsm-test/lsmtest_main.c $(TOP)/lsm-test/lsmtest_mem.c \ $(TOP)/lsm-test/lsmtest_tdb.c $(TOP)/lsm-test/lsmtest_tdb3.c \ $(TOP)/lsm-test/lsmtest_util.c @@ -519,11 +512,12 @@ ./testfixture$(EXE) $(TOP)/test/src4.test # Rules to build the 'lsmtest' application. # lsmtest$(EXE): libsqlite4.a $(LSMTESTSRC) $(LSMTESTHDR) - $(TCCX) $(LSMTESTSRC) libsqlite4.a -o lsmtest$(EXE) $(THREADLIB) -lsqlite3 + $(TCPPX) -c $(TOP)/lsm-test/lsmtest_tdb2.cc + $(TCCX) $(LSMTESTSRC) lsmtest_tdb2.o libsqlite4.a -o lsmtest$(EXE) $(THREADLIB) -lsqlite3 varint$(EXE): $(TOP)/src/varint.c $(TCCX) -DVARINT_TOOL -o varint$(EXE) $(TOP)/src/varint.c Index: src/alter.c ================================================================== --- src/alter.c +++ src/alter.c @@ -77,11 +77,11 @@ assert( len>0 ); } while( token!=TK_LP && token!=TK_USING ); zRet = sqlite4MPrintf(db, "%.*s\"%w\"%s", ((u8*)tname.z) - zSql, zSql, zTableName, tname.z+tname.n); - sqlite4_result_text(context, zRet, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zRet, -1, SQLITE4_TRANSIENT, 0); sqlite4DbFree(db, zRet); } } /* @@ -127,11 +127,11 @@ }while( token==TK_SPACE ); zParent = sqlite4DbStrNDup(db, (const char *)z, n); if( zParent==0 ) break; sqlite4Dequote(zParent); - if( 0==sqlite4StrICmp((const char *)zOld, zParent) ){ + if( 0==sqlite4_stricmp((const char *)zOld, zParent) ){ char *zOut = sqlite4MPrintf(db, "%s%.*s\"%w\"", (zOutput?zOutput:""), z-zInput, zInput, (const char *)zNew ); sqlite4DbFree(db, zOutput); zOutput = zOut; @@ -140,11 +140,11 @@ sqlite4DbFree(db, zParent); } } zResult = sqlite4MPrintf(db, "%s%s", (zOutput?zOutput:""), zInput), - sqlite4_result_text(context, zResult, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zResult, -1, SQLITE4_TRANSIENT, 0); sqlite4DbFree(db, zOutput); sqlite4DbFree(db, zResult); } #endif @@ -218,11 +218,11 @@ /* Variable tname now contains the token that is the old table-name ** in the CREATE TRIGGER statement. */ zRet = sqlite4MPrintf(db, "%.*s\"%w\"%s", ((u8*)tname.z) - zSql, zSql, zTableName, tname.z+tname.n); - sqlite4_result_text(context, zRet, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zRet, -1, SQLITE4_TRANSIENT, 0); sqlite4DbFree(db, zRet); } } #endif /* !SQLITE4_OMIT_TRIGGER */ @@ -379,11 +379,11 @@ ** in pParse->zErr (system tables may not be altered) and returns non-zero. ** ** Or, if zName is not a system table, zero is returned. */ static int isSystemTable(Parse *pParse, const char *zName){ - if( sqlite4Strlen30(zName)>6 && 0==sqlite4StrNICmp(zName, "sqlite_", 7) ){ + if( sqlite4Strlen30(zName)>6 && 0==sqlite4_strnicmp(zName, "sqlite_", 7) ){ sqlite4ErrorMsg(pParse, "table %s may not be altered", zName); return 1; } return 0; } Index: src/attach.c ================================================================== --- src/attach.c +++ src/attach.c @@ -100,11 +100,11 @@ goto attach_error; } for(i=0; inDb; i++){ char *z = db->aDb[i].zName; assert( z && zName ); - if( sqlite4StrICmp(z, zName)==0 ){ + if( sqlite4_stricmp(z, zName)==0 ){ zErrDyn = sqlite4MPrintf(db, "database %s is already in use", zName); goto attach_error; } } @@ -218,11 +218,11 @@ if( zName==0 ) zName = ""; for(i=0; inDb; i++){ pDb = &db->aDb[i]; if( pDb->pKV==0 ) continue; - if( sqlite4StrICmp(pDb->zName, zName)==0 ) break; + if( sqlite4_stricmp(pDb->zName, zName)==0 ) break; } if( i>=db->nDb ){ sqlite4_snprintf(zErr,sizeof(zErr), "no such database: %s", zName); goto detach_error; @@ -241,10 +241,12 @@ goto detach_error; } sqlite4KVStoreClose(pDb->pKV); pDb->pKV = 0; + sqlite4SchemaClear(db->pEnv, pDb->pSchema); + sqlite4DbFree(db, pDb->pSchema); pDb->pSchema = 0; sqlite4ResetInternalSchema(db, -1); return; detach_error: @@ -420,11 +422,11 @@ if( NEVER(pList==0) ) return 0; zDb = pFix->zDb; for(i=0, pItem=pList->a; inSrc; i++, pItem++){ if( pItem->zDatabase==0 ){ pItem->zDatabase = sqlite4DbStrDup(pFix->pParse->db, zDb); - }else if( sqlite4StrICmp(pItem->zDatabase,zDb)!=0 ){ + }else if( sqlite4_stricmp(pItem->zDatabase,zDb)!=0 ){ sqlite4ErrorMsg(pFix->pParse, "%s %T cannot reference objects in database %s", pFix->zType, pFix->pName, pItem->zDatabase); return 1; } Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -215,11 +215,11 @@ assert( zName!=0 ); nName = sqlite4Strlen30(zName); /* All mutexes are required for schema access. Make sure we hold them. */ for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ - if( zDatabase!=0 && sqlite4StrICmp(zDatabase, db->aDb[j].zName) ) continue; + if( zDatabase!=0 && sqlite4_stricmp(zDatabase, db->aDb[j].zName) ) continue; p = sqlite4HashFind(&db->aDb[j].pSchema->tblHash, zName, nName); if( p ) break; } return p; } @@ -280,11 +280,11 @@ /* All mutexes are required for schema access. Make sure we hold them. */ for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ Schema *pSchema = db->aDb[j].pSchema; assert( pSchema ); - if( zDb && sqlite4StrICmp(zDb, db->aDb[j].zName) ) continue; + if( zDb && sqlite4_stricmp(zDb, db->aDb[j].zName) ) continue; p = sqlite4HashFind(&pSchema->idxHash, zName, nName); if( p ) break; } return p; } @@ -294,10 +294,11 @@ */ static void freeIndex(sqlite4 *db, Index *p){ #ifndef SQLITE4_OMIT_ANALYZE sqlite4DeleteIndexSamples(db, p); #endif + sqlite4Fts5IndexFree(db, p); sqlite4DbFree(db, p->zColAff); sqlite4DbFree(db, p); } /* @@ -547,11 +548,11 @@ if( zName ){ Db *pDb; int n = sqlite4Strlen30(zName); for(i=(db->nDb-1), pDb=&db->aDb[i]; i>=0; i--, pDb--){ if( (!OMIT_TEMPDB || i!=1 ) && n==sqlite4Strlen30(pDb->zName) && - 0==sqlite4StrICmp(pDb->zName, zName) ){ + 0==sqlite4_stricmp(pDb->zName, zName) ){ break; } } } return i; @@ -626,11 +627,11 @@ ** is reserved for internal use. */ int sqlite4CheckObjectName(Parse *pParse, const char *zName){ if( !pParse->db->init.busy && pParse->nested==0 && (pParse->db->flags & SQLITE4_WriteSchema)==0 - && 0==sqlite4StrNICmp(zName, "sqlite_", 7) ){ + && 0==sqlite4_strnicmp(zName, "sqlite_", 7) ){ sqlite4ErrorMsg(pParse, "object name reserved for internal use: %s", zName); return SQLITE4_ERROR; } return SQLITE4_OK; } @@ -837,20 +838,20 @@ return; } /* ** This macro is used to compare two strings in a case-insensitive manner. -** It is slightly faster than calling sqlite4StrICmp() directly, but +** It is slightly faster than calling sqlite4_stricmp() directly, but ** produces larger code. ** ** WARNING: This macro is not compatible with the strcmp() family. It ** returns true if the two strings are equal, otherwise false. */ #define STRICMP(x, y) (\ sqlite4UpperToLower[*(unsigned char *)(x)]== \ sqlite4UpperToLower[*(unsigned char *)(y)] \ -&& sqlite4StrICmp((x)+1,(y)+1)==0 ) +&& sqlite4_stricmp((x)+1,(y)+1)==0 ) /* ** Add a new column to the table currently being constructed. ** ** The parser calls this routine once for each column declaration @@ -1037,31 +1038,21 @@ ** most recently added column of the table is the primary key. ** ** A table can have at most one primary key. If the table already has ** a primary key (and this is the second primary key) then create an ** error. -** -** If the PRIMARY KEY is on a single column whose datatype is INTEGER, -** then we will try to use that column as the rowid. Set the Table.iPKey -** field of the table under construction to be the index of the -** INTEGER PRIMARY KEY column. Table.iPKey is set to -1 if there is -** no INTEGER PRIMARY KEY. -** -** If the key is not an INTEGER PRIMARY KEY, then create a unique -** index for the key. No index is created for INTEGER PRIMARY KEYs. */ void sqlite4AddPrimaryKey( Parse *pParse, /* Parsing context */ ExprList *pList, /* List of field names to be indexed */ int onError, /* What to do with a uniqueness conflict */ int autoInc, /* True if the AUTOINCREMENT keyword is present */ int sortOrder /* SQLITE4_SO_ASC or SQLITE4_SO_DESC */ ){ Table *pTab = pParse->pNewTable; -#if 0 - char *zType = 0; -#endif + Index *pPk; /* Primary key index */ + char *zType = 0; /* Primary key column type */ int iCol = -1, i; if( pTab==0 || IN_DECLARE_VTAB ) goto primary_key_exit; if( pTab->tabFlags & TF_HasPrimaryKey ){ sqlite4ErrorMsg(pParse, "table \"%s\" has more than one primary key", pTab->zName); @@ -1073,47 +1064,39 @@ pTab->aCol[iCol].isPrimKey = 1; pTab->aCol[iCol].notNull = 1; }else{ for(i=0; inExpr; i++){ for(iCol=0; iColnCol; iCol++){ - if( sqlite4StrICmp(pList->a[i].zName, pTab->aCol[iCol].zName)==0 ){ + if( sqlite4_stricmp(pList->a[i].zName, pTab->aCol[iCol].zName)==0 ){ break; } } if( iColnCol ){ - pTab->aCol[iCol].isPrimKey = 1; + pTab->aCol[iCol].isPrimKey = i+1; pTab->aCol[iCol].notNull = 1; } } if( pList->nExpr>1 ) iCol = -1; } - -#if 0 - if( iCol>=0 && iColnCol ){ - zType = pTab->aCol[iCol].zType; - } - - if( zType && sqlite4StrICmp(zType, "INTEGER")==0 - && sortOrder==SQLITE4_SO_ASC ){ - pTab->iPKey = iCol; - pTab->keyConf = (u8)onError; + pPk = sqlite4CreateIndex( + pParse, 0, 0, 0, pList, onError, 0, 0, sortOrder, 0, 1 + ); + + if( iCol>=0 && iColnCol + && (zType = pTab->aCol[iCol].zType)!=0 + && sqlite4_stricmp(zType, "INTEGER")==0 + && sortOrder==SQLITE4_SO_ASC + && pPk + ){ + pPk->fIndex |= IDX_IntPK; assert( autoInc==0 || autoInc==1 ); - pTab->tabFlags |= autoInc*TF_Autoincrement; + pTab->tabFlags |= (-autoInc)&TF_Autoincrement; }else if( autoInc ){ -#ifndef SQLITE4_OMIT_AUTOINCREMENT - sqlite4ErrorMsg(pParse, "AUTOINCREMENT is only allowed on an " - "INTEGER PRIMARY KEY"); -#endif - }else -#endif - - { - sqlite4CreateIndex( - pParse, 0, 0, 0, pList, onError, 0, 0, sortOrder, 0, 1 - ); - pList = 0; - } + sqlite4ErrorMsg(pParse, + "AUTOINCREMENT permitted on INTEGER PRIMARY KEY ASC only"); + } + pList = 0; primary_key_exit: sqlite4ExprListDelete(pParse->db, pList); return; } @@ -2062,12 +2045,12 @@ if( sqlite4AuthCheck(pParse, SQLITE4_DELETE, pTab->zName, 0, zDb) ){ goto exit_drop_table; } } #endif - if( sqlite4StrNICmp(pTab->zName, "sqlite_", 7)==0 - && sqlite4StrNICmp(pTab->zName, "sqlite_stat", 11)!=0 ){ + if( sqlite4_strnicmp(pTab->zName, "sqlite_", 7)==0 + && sqlite4_strnicmp(pTab->zName, "sqlite_stat", 11)!=0 ){ sqlite4ErrorMsg(pParse, "table %s may not be dropped", pTab->zName); goto exit_drop_table; } #ifndef SQLITE4_OMIT_VIEW @@ -2175,11 +2158,11 @@ pFKey->aCol[0].iFrom = p->nCol-1; }else{ for(i=0; inCol; j++){ - if( sqlite4StrICmp(p->aCol[j].zName, pFromCol->a[i].zName)==0 ){ + if( sqlite4_stricmp(p->aCol[j].zName, pFromCol->a[i].zName)==0 ){ pFKey->aCol[i].iFrom = j; break; } } if( j>=p->nCol ){ @@ -2286,29 +2269,305 @@ if( bCreate ) sqlite4VdbeChangeP5(v, 1); /* Loop through the contents of the PK index. At each row, insert the ** corresponding entry into the auxiliary index. */ addr1 = sqlite4VdbeAddOp2(v, OP_Rewind, iTab, 0); - sqlite4GetTempRange(pParse,2); - regKey = sqlite4GetTempReg(pParse); - sqlite4EncodeIndexKey(pParse, pPk, iTab, pIdx, iIdx, 0, regKey); - if( pIdx->onError!=OE_None ){ - const char *zErr = "indexed columns are not unique"; - int addrTest; - - addrTest = sqlite4VdbeAddOp4Int(v, OP_IsUnique, iIdx, 0, regKey, 0); - sqlite4HaltConstraint(pParse, OE_Abort, (char *)zErr, P4_STATIC); - sqlite4VdbeJumpHere(v, addrTest); - } - sqlite4VdbeAddOp3(v, OP_IdxInsert, iIdx, 0, regKey); + + if( pIdx->eIndexType==SQLITE4_INDEX_FTS5 ){ + int regData; + int i; + + regKey = sqlite4GetTempRange(pParse, pTab->nCol+1); + regData = regKey+1; + + sqlite4VdbeAddOp2(v, OP_RowKey, iTab, regKey); + for(i=0; inCol; i++){ + sqlite4VdbeAddOp3(v, OP_Column, iTab, i, regData+i); + } + sqlite4Fts5CodeUpdate(pParse, pIdx, pParse->iNewidxReg, regKey, regData, 0); + }else{ + sqlite4GetTempRange(pParse,2); + regKey = sqlite4GetTempReg(pParse); + sqlite4EncodeIndexKey(pParse, pPk, iTab, pIdx, iIdx, 0, regKey); + if( pIdx->onError!=OE_None ){ + const char *zErr = "indexed columns are not unique"; + int addrTest; + + addrTest = sqlite4VdbeAddOp4Int(v, OP_IsUnique, iIdx, 0, regKey, 0); + sqlite4HaltConstraint(pParse, OE_Abort, (char *)zErr, P4_STATIC); + sqlite4VdbeJumpHere(v, addrTest); + } + sqlite4VdbeAddOp3(v, OP_IdxInsert, iIdx, 0, regKey); + } + sqlite4VdbeAddOp2(v, OP_Next, iTab, addr1+1); sqlite4VdbeJumpHere(v, addr1); sqlite4ReleaseTempReg(pParse, regKey); sqlite4VdbeAddOp1(v, OP_Close, iTab); sqlite4VdbeAddOp1(v, OP_Close, iIdx); } + +/* +** The CreateIndex structure indicated by the first argument contains the +** results of parsing the first part of a CREATE INDEX statement. +** Specifically, everything up to and including the "ON tblname" clause. +** The index may be an ordinary index, or it may be a "USING fts5" index. +** This function performs processing common to both. +*/ +static Table *createIndexFindTable( + Parse *pParse, /* Parsing context */ + CreateIndex *p, /* First part of CREATE INDEX statement */ + Token **ppIdxName, /* OUT: Pointer to index name token */ + char **pzIdxName, /* OUT: DbMalloc'd copy of index name */ + int *piDb /* OUT: Database to create index in */ +){ + DbFixer sFix; /* For assigning database names to pTblName */ + sqlite4 *db = pParse->db; /* Database handle */ + Token *pName = 0; /* Token containing unqualified index name */ + char *zName; /* Name of index being created */ + int iDb; /* Index of database in db->aDb[] */ + Table *pTab; /* Table object to return */ + + /* Use the two-part index name to determine the database + ** to search for the table. 'Fix' the table name to this db + ** before looking up the table. */ + iDb = sqlite4TwoPartName(pParse, &p->tName1, &p->tName2, &pName); + if( iDb<0 ) return 0; + assert( pName && pName->z ); + +#ifndef SQLITE4_OMIT_TEMPDB + /* If the index name was unqualified, check if the the table + ** is a temp table. If so, set the database to 1. Do not do this + ** if initialising a database schema. */ + if( !db->init.busy ){ + pTab = sqlite4SrcListLookup(pParse, p->pTblName); + if( p->tName2.n==0 && pTab && pTab->pSchema==db->aDb[1].pSchema ){ + iDb = 1; + } + } +#endif + + if( sqlite4FixInit(&sFix, pParse, iDb, "index", pName) && + sqlite4FixSrcList(&sFix, p->pTblName) + ){ + /* Because the parser constructs pTblName from a single identifier, + ** sqlite4FixSrcList can never fail. */ + assert(0); + } + + pTab = sqlite4SrcListLookup(pParse, p->pTblName); + if( !pTab || db->mallocFailed ) return 0; + assert( db->aDb[iDb].pSchema==pTab->pSchema ); + assert( pParse->nErr==0 ); + + /* TODO: We will need to reinstate this block when sqlite_master is + ** modified to use an implicit primary key. */ +#if 0 + if( sqlite4_strnicmp(pTab->zName, "sqlite_", 7)==0 + && memcmp(&pTab->zName[7],"altertab_",9)!=0 ){ + sqlite4ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); + goto exit_create_index; + } +#endif + + /* Verify that this is not an attempt to create an index on a view or + ** virtual table. */ + if( IsView(pTab) ){ + sqlite4ErrorMsg(pParse, "views may not be indexed"); + return 0; + } + if( IsVirtual(pTab) ){ + sqlite4ErrorMsg(pParse, "virtual tables may not be indexed"); + return 0; + } + + /* Ensure that the proposed index name is not reserved. */ + assert( pName->z!=0 ); + zName = sqlite4NameFromToken(db, pName); + if( zName==0 || sqlite4CheckObjectName(pParse, zName) ) return 0; + + /* Unless SQLite is currently parsing an existing database schema, check + ** that there is not already an index or table using the proposed name. */ + if( !db->init.busy ){ + char *zDb = db->aDb[iDb].zName; + if( sqlite4FindTable(db, zName, zDb)!=0 ){ + sqlite4ErrorMsg(pParse, "there is already a table named %s", zName); + } + else if( sqlite4FindIndex(db, zName, zDb)!=0 ){ + if( p->bIfnotexist ){ + assert( !db->init.busy ); + sqlite4CodeVerifySchema(pParse, iDb); + pTab = 0; + }else{ + sqlite4ErrorMsg(pParse, "index %s already exists", zName); + } + } + } + + if( pParse->nErr || pTab==0 ){ + sqlite4DbFree(db, zName); + pTab = 0; + zName = 0; + } + *ppIdxName = pName; + *pzIdxName = zName; + *piDb = iDb; + return pTab; +} + +#ifndef SQLITE4_OMIT_AUTHORIZATION +/* +** Check for authorization to create index zIdx on table pTab. If +** authorization is granted, return zero. Otherwise, return non-zero +** and leave an error in pParse. +*/ +static int createIndexAuth( + Parse *pParse, /* Parser context */ + Table *pTab, /* Table index is being created on */ + const char *zIdx /* Name of index being created */ +){ + sqlite4 *db; + int iDb; + int iOp; + const char *zDb; + + db = pParse->db; + iDb = sqlite4SchemaToIndex(db, pTab->pSchema); + zDb = db->aDb[iDb].zName; + + if( sqlite4AuthCheck(pParse, SQLITE4_INSERT, SCHEMA_TABLE(iDb), 0, zDb) ){ + return 1; + } + + iOp = (!OMIT_TEMPDB && iDb==1)?SQLITE4_CREATE_TEMP_INDEX:SQLITE4_CREATE_INDEX; + if( sqlite4AuthCheck(pParse, iOp, zIdx, pTab->zName, zDb) ){ + return 1; + } + + return 0; +} +#else +# define createIndexAuth(a, b, c) 0 +#endif // SQLITE4_OMIT_AUTHORIZATION + +/* +** This function is called when parsing a CREATE statement executed by the +** user (not when parsing the schema of an existing database). It generates +** the VDBE code to assign a root page number to the new index and, if +** required, to write a new entry into the sqlite_master table. +*/ +static void createIndexWriteSchema( + Parse *pParse, /* Parser context */ + Index *pIdx, /* New index object */ + Token *pName, /* Token containing name of new index */ + Token *pEnd /* Token for final closing paren of CREATE */ +){ + sqlite4 *db = pParse->db; + int iDb; + + assert( db->init.busy==0 ); + iDb = sqlite4SchemaToIndex(db, pIdx->pSchema); + pIdx->tnum = ++pParse->nMem; + allocateTableNumber(pParse, iDb, pIdx->tnum); + + if( pIdx->eIndexType!=SQLITE4_INDEX_PRIMARYKEY ){ + Vdbe *v; + char *zStmt; + + v = sqlite4GetVdbe(pParse); + if( v==0 ) return; + + sqlite4BeginWriteOperation(pParse, 1, iDb); + + /* Unless this index is an automatic index created by a UNIQUE + ** constraint, assemble a CREATE INDEX statement to write into the + ** sqlite_master table. */ + if( pIdx->eIndexType!=SQLITE4_INDEX_UNIQUE ){ + int n = (int)(pEnd->z - pName->z) + pEnd->n; + const char *zUnique = (pIdx->onError==OE_None ? "" : " UNIQUE"); + zStmt = sqlite4MPrintf(db, "CREATE%s INDEX %.*s", zUnique, n, pName->z); + }else{ + /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ + zStmt = 0; + } + + /* Add an entry in sqlite_master for this index */ + sqlite4NestedParse(pParse, + "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#%d,%Q);", + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + pIdx->zName, + pIdx->pTable->zName, + pIdx->tnum, + zStmt + ); + sqlite4DbFree(db, zStmt); + + /* Fill the index with data and reparse the schema. Code an OP_Expire + ** to invalidate all pre-compiled statements. + */ + if( pIdx->eIndexType!=SQLITE4_INDEX_UNIQUE ){ + sqlite4RefillIndex(pParse, pIdx, 1); + sqlite4ChangeCookie(pParse, iDb); + sqlite4VdbeAddParseSchemaOp(v, iDb, + sqlite4MPrintf(db, "name='%q' AND type='index'", pIdx->zName)); + sqlite4VdbeAddOp1(v, OP_Expire, 0); + } + } +} + +void sqlite4CreateUsingIndex( + Parse *pParse, /* Parsing context */ + CreateIndex *p, /* First part of CREATE INDEX statement */ + ExprList *pList, + Token *pUsing, /* Token following USING keyword */ + Token *pEnd /* Final '(' in CREATE INDEX */ +){ + sqlite4 *db = pParse->db; + if( p->bUnique ){ + sqlite4ErrorMsg(pParse, "USING %.*s index may not be UNIQUE", + pUsing->n, pUsing->z + ); + }else{ + Index *pIdx = 0; /* New index object */ + Table *pTab; + Token *pIdxName = 0; + char *zIdx = 0; + int iDb = 0; + + pTab = createIndexFindTable(pParse, p, &pIdxName, &zIdx, &iDb); + if( pTab && 0==createIndexAuth(pParse, pTab, zIdx) ){ + int nExtra = sqlite4Fts5IndexSz(); + char *zExtra = 0; + pIdx = newIndex(pParse, pTab, zIdx, 0, 0, nExtra, &zExtra); + if( pIdx ){ + pIdx->pFts = (Fts5Index *)zExtra; + sqlite4Fts5IndexInit(pParse, pIdx, pList); + pIdx->eIndexType = SQLITE4_INDEX_FTS5; + } + } + + if( pIdx ){ + if( db->init.busy ){ + db->flags |= SQLITE4_InternChanges; + pIdx->tnum = db->init.newTnum; + pIdx->pNext = pTab->pIndex; + pTab->pIndex = pIdx; + addIndexToHash(db, pIdx); + pIdx = 0; + }else{ + createIndexWriteSchema(pParse, pIdx, pIdxName, pEnd); + } + } + + if( pIdx ) freeIndex(db, pIdx); + sqlite4DbFree(db, zIdx); + } + + sqlite4ExprListDelete(db, pList); + sqlite4SrcListDelete(db, p->pTblName); +} /* ** Create a new index for an SQL table. pName1.pName2 is the name of the index ** and pTblList is the name of the table that is to be indexed. Both will ** be NULL for a primary key or an index that is created to satisfy a @@ -2363,46 +2622,23 @@ /* ** Find the table that is to be indexed. Return early if not found. */ if( pTblName!=0 ){ - - /* Use the two-part index name to determine the database - ** to search for the table. 'Fix' the table name to this db - ** before looking up the table. - */ - assert( !bPrimaryKey ); - assert( pName1 && pName2 ); - iDb = sqlite4TwoPartName(pParse, pName1, pName2, &pName); - if( iDb<0 ) goto exit_create_index; - assert( pName && pName->z ); - -#ifndef SQLITE4_OMIT_TEMPDB - /* If the index name was unqualified, check if the the table - ** is a temp table. If so, set the database to 1. Do not do this - ** if initialising a database schema. - */ - if( !db->init.busy ){ - pTab = sqlite4SrcListLookup(pParse, pTblName); - if( pName2->n==0 && pTab && pTab->pSchema==db->aDb[1].pSchema ){ - iDb = 1; - } - } -#endif - - if( sqlite4FixInit(&sFix, pParse, iDb, "index", pName) && - sqlite4FixSrcList(&sFix, pTblName) - ){ - /* Because the parser constructs pTblName from a single identifier, - ** sqlite4FixSrcList can never fail. */ - assert(0); - } - pTab = sqlite4LocateTable(pParse, 0, pTblName->a[0].zName, - pTblName->a[0].zDatabase); - if( !pTab || db->mallocFailed ) goto exit_create_index; - assert( db->aDb[iDb].pSchema==pTab->pSchema ); + CreateIndex sCreate; + sCreate.bUnique = onError; + sCreate.bIfnotexist = ifNotExist; + sCreate.tCreate = *pStart; + sCreate.tName1 = *pName1; + sCreate.tName2 = *pName2; + sCreate.pTblName = pTblName; + + pTab = createIndexFindTable(pParse, &sCreate, &pName, &zName, &iDb); + if( !pTab ) goto exit_create_index; + }else{ + assert( pName==0 ); assert( pStart==0 ); pTab = pParse->pNewTable; if( !pTab ) goto exit_create_index; iDb = sqlite4SchemaToIndex(db, pTab->pSchema); @@ -2409,99 +2645,32 @@ } pDb = &db->aDb[iDb]; assert( pTab!=0 ); assert( pParse->nErr==0 ); - - /* TODO: We will need to reinstate this block when sqlite_master is - ** modified to use an implicit primary key. */ -#if 0 - if( sqlite4StrNICmp(pTab->zName, "sqlite_", 7)==0 - && memcmp(&pTab->zName[7],"altertab_",9)!=0 ){ - sqlite4ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); - goto exit_create_index; - } -#endif - -#ifndef SQLITE4_OMIT_VIEW - if( pTab->pSelect ){ - assert( !bPrimaryKey ); - sqlite4ErrorMsg(pParse, "views may not be indexed"); - goto exit_create_index; - } -#endif -#ifndef SQLITE4_OMIT_VIRTUALTABLE - if( IsVirtual(pTab) ){ - assert( !bPrimaryKey ); - sqlite4ErrorMsg(pParse, "virtual tables may not be indexed"); - goto exit_create_index; - } -#endif - - /* - ** Find the name of the index. Make sure there is not already another - ** index or table with the same name. - ** - ** Exception: If we are reading the names of permanent indices from the - ** sqlite_master table (because some other process changed the schema) and - ** one of the index names collides with the name of a temporary table or - ** index, then we will continue to process this index. - ** - ** If pName==0 it means that we are - ** dealing with a primary key or UNIQUE constraint. We have to invent our - ** own name. - */ - if( pName ){ - assert( !bPrimaryKey ); - zName = sqlite4NameFromToken(db, pName); - if( zName==0 ) goto exit_create_index; - assert( pName->z!=0 ); - if( SQLITE4_OK!=sqlite4CheckObjectName(pParse, zName) ){ - goto exit_create_index; - } - if( !db->init.busy ){ - if( sqlite4FindTable(db, zName, 0)!=0 ){ - sqlite4ErrorMsg(pParse, "there is already a table named %s", zName); - goto exit_create_index; - } - } - if( sqlite4FindIndex(db, zName, pDb->zName)!=0 ){ - if( !ifNotExist ){ - sqlite4ErrorMsg(pParse, "index %s already exists", zName); - }else{ - assert( !db->init.busy ); - sqlite4CodeVerifySchema(pParse, iDb); - } - goto exit_create_index; - } - }else if( !bPrimaryKey ){ - int n; - Index *pLoop; - for(pLoop=pTab->pIndex, n=1; pLoop; pLoop=pLoop->pNext, n++){} - zName = sqlite4MPrintf(db, "sqlite_autoindex_%s_%d", pTab->zName, n); - }else{ - zName = sqlite4MPrintf(db, "%s", pTab->zName); - } - if( zName==0 ){ - goto exit_create_index; - } - - /* Check for authorization to create an index. - */ -#ifndef SQLITE4_OMIT_AUTHORIZATION - if( bPrimaryKey==0 ){ - const char *zDb = pDb->zName; - if( sqlite4AuthCheck(pParse, SQLITE4_INSERT, SCHEMA_TABLE(iDb), 0, zDb) ){ - goto exit_create_index; - } - i = SQLITE4_CREATE_INDEX; - if( !OMIT_TEMPDB && iDb==1 ) i = SQLITE4_CREATE_TEMP_INDEX; - if( sqlite4AuthCheck(pParse, i, zName, pTab->zName, zDb) ){ - goto exit_create_index; - } - } -#endif + assert( IsVirtual(pTab)==0 && IsView(pTab)==0 ); + + /* If pName==0 it means that we are dealing with a primary key or + ** UNIQUE constraint. We have to invent our own name. */ + if( pName==0 ){ + if( !bPrimaryKey ){ + int n; + Index *pLoop; + for(pLoop=pTab->pIndex, n=1; pLoop; pLoop=pLoop->pNext, n++){} + zName = sqlite4MPrintf(db, "sqlite_%s_unique%d", pTab->zName, n); + }else{ + zName = sqlite4MPrintf(db, "%s", pTab->zName); + } + if( zName==0 ){ + goto exit_create_index; + } + } + + /* Check for authorization to create the index. */ + if( bPrimaryKey==0 && createIndexAuth(pParse, pTab, zName) ){ + goto exit_create_index; + } /* If pList==0, it means this routine was called as a result of a PRIMARY ** KEY or UNIQUE constraint attached to the last column added to the table ** under construction. So create a fake list to simulate this. ** @@ -2549,35 +2718,28 @@ ** load the column indices into the Index structure. Report an error ** if any column is not found. ** ** TODO: Add a test to make sure that the same column is not named ** more than once within the same index. Only the first instance of - ** the column will ever be used by the optimizer. Note that using the - ** same column more than once cannot be an error because that would - ** break backwards compatibility - it needs to be a warning. + ** the column will ever be used by the optimizer. */ for(i=0, pListItem=pList->a; inExpr; i++, pListItem++){ const char *zColName = pListItem->zName; Column *pTabCol; char *zColl; /* Collation sequence name */ for(j=0, pTabCol=pTab->aCol; jnCol; j++, pTabCol++){ - if( sqlite4StrICmp(zColName, pTabCol->zName)==0 ) break; + if( sqlite4_stricmp(zColName, pTabCol->zName)==0 ) break; } if( j>=pTab->nCol ){ sqlite4ErrorMsg(pParse, "table %s has no column named %s", pTab->zName, zColName); pParse->checkSchema = 1; goto exit_create_index; } pIndex->aiColumn[i] = j; - /* Justification of the ALWAYS(pListItem->pExpr->pColl): Because of - ** the way the "idxlist" non-terminal is constructed by the parser, - ** if pListItem->pExpr is not null then either pListItem->pExpr->pColl - ** must exist or else there must have been an OOM error. But if there - ** was an OOM error, we would never reach this point. */ - if( pListItem->pExpr && ALWAYS(pListItem->pExpr->pColl) ){ + if( pListItem->pExpr && pListItem->pExpr->pColl ){ int nColl; zColl = pListItem->pExpr->pColl->zName; nColl = sqlite4Strlen30(zColl) + 1; assert( nExtra>=nColl ); memcpy(zExtra, zColl, nColl); @@ -2632,11 +2794,11 @@ const char *z1; const char *z2; if( pIdx->aiColumn[k]!=pIndex->aiColumn[k] ) break; z1 = pIdx->azColl[k]; z2 = pIndex->azColl[k]; - if( z1!=z2 && sqlite4StrICmp(z1, z2) ) break; + if( z1!=z2 && sqlite4_stricmp(z1, z2) ) break; } if( k==pIdx->nColumn ){ if( pIdx->onError!=pIndex->onError ){ /* This constraint creates the same index as a previous ** constraint specified somewhere in the CREATE TABLE statement. @@ -2690,62 +2852,11 @@ ** or UNIQUE constraint of a CREATE TABLE statement. Since the table ** has just been created, it contains no data and the index initialization ** step can be skipped. */ else{ - pIndex->tnum = ++pParse->nMem; - allocateTableNumber(pParse, iDb, pIndex->tnum); - if( bPrimaryKey==0 ){ - Vdbe *v; - char *zStmt; - - v = sqlite4GetVdbe(pParse); - if( v==0 ) goto exit_create_index; - - /* Create the rootpage for the index - */ - sqlite4BeginWriteOperation(pParse, 1, iDb); - - /* Gather the complete text of the CREATE INDEX statement into - ** the zStmt variable - */ - if( pStart ){ - assert( pEnd!=0 ); - /* A named index with an explicit CREATE INDEX statement */ - zStmt = sqlite4MPrintf(db, "CREATE%s INDEX %.*s", - onError==OE_None ? "" : " UNIQUE", - (int)(pEnd->z - pName->z) + 1, - pName->z); - }else{ - /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ - /* zStmt = sqlite4MPrintf(""); */ - zStmt = 0; - } - - /* Add an entry in sqlite_master for this index - */ - sqlite4NestedParse(pParse, - "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#%d,%Q);", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), - pIndex->zName, - pTab->zName, - pIndex->tnum, - zStmt - ); - sqlite4DbFree(db, zStmt); - - /* Fill the index with data and reparse the schema. Code an OP_Expire - ** to invalidate all pre-compiled statements. - */ - if( pTblName ){ - sqlite4RefillIndex(pParse, pIndex, 1); - sqlite4ChangeCookie(pParse, iDb); - sqlite4VdbeAddParseSchemaOp(v, iDb, - sqlite4MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); - sqlite4VdbeAddOp1(v, OP_Expire, 0); - } - } + createIndexWriteSchema(pParse, pIndex, pName, pEnd); } /* When adding an index to the list of indices for a table, make ** sure all indices labeled OE_Replace come after all those labeled ** OE_Ignore. This is necessary for the correct constraint check @@ -2842,11 +2953,13 @@ sqlite4CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); } pParse->checkSchema = 1; goto exit_drop_index; } - if( pIndex->eIndexType!=SQLITE4_INDEX_USER ){ + if( pIndex->eIndexType!=SQLITE4_INDEX_USER + && pIndex->eIndexType!=SQLITE4_INDEX_FTS5 + ){ sqlite4ErrorMsg(pParse, "index associated with UNIQUE " "or PRIMARY KEY constraint cannot be dropped", 0); goto exit_drop_index; } iDb = sqlite4SchemaToIndex(db, pIndex->pSchema); @@ -2977,11 +3090,11 @@ */ int sqlite4IdListIndex(IdList *pList, const char *zName){ int i; if( pList==0 ) return -1; for(i=0; inId; i++){ - if( sqlite4StrICmp(pList->a[i].zName, zName)==0 ) return i; + if( sqlite4_stricmp(pList->a[i].zName, zName)==0 ) return i; } return -1; } /* @@ -3400,11 +3513,11 @@ void sqlite4CodeVerifyNamedSchema(Parse *pParse, const char *zDb){ sqlite4 *db = pParse->db; int i; for(i=0; inDb; i++){ Db *pDb = &db->aDb[i]; - if( pDb->pKV && (!zDb || 0==sqlite4StrICmp(zDb, pDb->zName)) ){ + if( pDb->pKV && (!zDb || 0==sqlite4_stricmp(zDb, pDb->zName)) ){ sqlite4CodeVerifySchema(pParse, i); } } } @@ -3483,11 +3596,11 @@ int i; assert( zColl!=0 ); for(i=0; inColumn; i++){ const char *z = pIndex->azColl[i]; assert( z!=0 ); - if( 0==sqlite4StrICmp(z, zColl) ){ + if( 0==sqlite4_stricmp(z, zColl) ){ return 1; } } return 0; } Index: src/callback.c ================================================================== --- src/callback.c +++ src/callback.c @@ -30,11 +30,11 @@ } #ifndef SQLITE4_OMIT_UTF16 if( db->xCollNeeded16 ){ char const *zExternal; sqlite4_value *pTmp = sqlite4ValueNew(db); - sqlite4ValueSetStr(pTmp, -1, zName, SQLITE4_UTF8, SQLITE4_STATIC); + sqlite4ValueSetStr(pTmp, -1, zName, SQLITE4_UTF8, SQLITE4_STATIC, 0); zExternal = sqlite4ValueText(pTmp, SQLITE4_UTF16NATIVE); if( zExternal ){ db->xCollNeeded16(db->pCollNeededArg, db, (int)ENC(db), zExternal); } sqlite4ValueFree(pTmp); @@ -268,11 +268,11 @@ int nFunc /* Number of bytes in zFunc */ ){ FuncDef *p; if( nFunc<0 ) nFunc = sqlite4Strlen30(zFunc); for(p=pFuncTab->pFirst; p; p=p->pNextName){ - if( sqlite4StrNICmp(p->zName, zFunc, nFunc)==0 && p->zName[nFunc]==0 ){ + if( sqlite4_strnicmp(p->zName, zFunc, nFunc)==0 && p->zName[nFunc]==0 ){ return p; } } return 0; } @@ -297,11 +297,11 @@ if( pFuncTab->pFirst==0 ){ pFuncTab->pFirst = pDef; pFuncTab->pLast = pDef; pFuncTab->pSame = pDef; }else if( isBuiltIn - && sqlite4StrICmp(pDef->zName, pFuncTab->pLast->zName)==0 ){ + && sqlite4_stricmp(pDef->zName, pFuncTab->pLast->zName)==0 ){ assert( pFuncTab->pSame->pSameName==0 || pFuncTab->pSame->pSameName==pDef ); pFuncTab->pSame->pSameName = pDef; pFuncTab->pSame = pDef; }else if( !isBuiltIn && (pOther=functionSearch(pFuncTab,pDef->zName,-1))!=0 ){ pDef->pSameName = pOther->pSameName; @@ -422,17 +422,17 @@ Hash temp2; HashElem *pElem; temp1 = pSchema->tblHash; temp2 = pSchema->trigHash; - sqlite4HashInit(pEnv, &pSchema->trigHash); + sqlite4HashInit(pEnv, &pSchema->trigHash, 0); sqlite4HashClear(&pSchema->idxHash); for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){ sqlite4DeleteTrigger(0, (Trigger*)sqliteHashData(pElem)); } sqlite4HashClear(&temp2); - sqlite4HashInit(pEnv, &pSchema->tblHash); + sqlite4HashInit(pEnv, &pSchema->tblHash, 0); for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ Table *pTab = sqliteHashData(pElem); sqlite4DeleteTable(0, pTab); } sqlite4HashClear(&temp1); @@ -452,13 +452,13 @@ Schema * p; p = (Schema *)sqlite4DbMallocZero(0, sizeof(Schema)); if( !p ){ db->mallocFailed = 1; }else if ( 0==p->file_format ){ - sqlite4HashInit(db->pEnv, &p->tblHash); - sqlite4HashInit(db->pEnv, &p->idxHash); - sqlite4HashInit(db->pEnv, &p->trigHash); - sqlite4HashInit(db->pEnv, &p->fkeyHash); + sqlite4HashInit(db->pEnv, &p->tblHash, 0); + sqlite4HashInit(db->pEnv, &p->idxHash, 0); + sqlite4HashInit(db->pEnv, &p->trigHash, 0); + sqlite4HashInit(db->pEnv, &p->fkeyHash, 0); p->enc = SQLITE4_UTF8; } return p; } Index: src/complete.c ================================================================== --- src/complete.c +++ src/complete.c @@ -198,35 +198,35 @@ #ifdef SQLITE4_OMIT_TRIGGER token = tkOTHER; #else switch( *zSql ){ case 'c': case 'C': { - if( nId==6 && sqlite4StrNICmp(zSql, "create", 6)==0 ){ + if( nId==6 && sqlite4_strnicmp(zSql, "create", 6)==0 ){ token = tkCREATE; }else{ token = tkOTHER; } break; } case 't': case 'T': { - if( nId==7 && sqlite4StrNICmp(zSql, "trigger", 7)==0 ){ + if( nId==7 && sqlite4_strnicmp(zSql, "trigger", 7)==0 ){ token = tkTRIGGER; - }else if( nId==4 && sqlite4StrNICmp(zSql, "temp", 4)==0 ){ + }else if( nId==4 && sqlite4_strnicmp(zSql, "temp", 4)==0 ){ token = tkTEMP; - }else if( nId==9 && sqlite4StrNICmp(zSql, "temporary", 9)==0 ){ + }else if( nId==9 && sqlite4_strnicmp(zSql, "temporary", 9)==0 ){ token = tkTEMP; }else{ token = tkOTHER; } break; } case 'e': case 'E': { - if( nId==3 && sqlite4StrNICmp(zSql, "end", 3)==0 ){ + if( nId==3 && sqlite4_strnicmp(zSql, "end", 3)==0 ){ token = tkEND; }else #ifndef SQLITE4_OMIT_EXPLAIN - if( nId==7 && sqlite4StrNICmp(zSql, "explain", 7)==0 ){ + if( nId==7 && sqlite4_strnicmp(zSql, "explain", 7)==0 ){ token = tkEXPLAIN; }else #endif { token = tkOTHER; @@ -267,11 +267,11 @@ #ifndef SQLITE4_OMIT_AUTOINIT rc = sqlite4_initialize(0); if( rc ) return rc; #endif pVal = sqlite4ValueNew(0); - sqlite4ValueSetStr(pVal, -1, zSql, SQLITE4_UTF16NATIVE, SQLITE4_STATIC); + sqlite4ValueSetStr(pVal, -1, zSql, SQLITE4_UTF16NATIVE, SQLITE4_STATIC, 0); zSql8 = sqlite4ValueText(pVal, SQLITE4_UTF8); if( zSql8 ){ rc = sqlite4_complete(zSql8); }else{ rc = SQLITE4_NOMEM; Index: src/ctime.c ================================================================== --- src/ctime.c +++ src/ctime.c @@ -344,17 +344,17 @@ ** The name can optionally begin with "SQLITE4_" but the "SQLITE4_" prefix ** is not required for a match. */ int sqlite4_compileoption_used(const char *zOptName){ int i, n; - if( sqlite4StrNICmp(zOptName, "SQLITE4_", 8)==0 ) zOptName += 8; + if( sqlite4_strnicmp(zOptName, "SQLITE4_", 8)==0 ) zOptName += 8; n = sqlite4Strlen30(zOptName); /* Since ArraySize(azCompileOpt) is normally in single digits, a ** linear search is adequate. No need for a binary search. */ for(i=0; iiJD = (sqlite4_int64)(r*86400000.0 + 0.5); p->validJD = 1; return 0; @@ -812,11 +812,11 @@ if( isDate(context, argc, argv, &x)==0 ){ char zBuf[100]; computeYMD_HMS(&x); sqlite4_snprintf(zBuf,sizeof(zBuf), "%04d-%02d-%02d %02d:%02d:%02d", x.Y, x.M, x.D, x.h, x.m, (int)(x.s)); - sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT, 0); } } /* ** time( TIMESTRING, MOD, MOD, ...) @@ -831,11 +831,11 @@ DateTime x; if( isDate(context, argc, argv, &x)==0 ){ char zBuf[100]; computeHMS(&x); sqlite4_snprintf(zBuf,sizeof(zBuf), "%02d:%02d:%02d", x.h, x.m, (int)x.s); - sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT, 0); } } /* ** date( TIMESTRING, MOD, MOD, ...) @@ -850,11 +850,11 @@ DateTime x; if( isDate(context, argc, argv, &x)==0 ){ char zBuf[100]; computeYMD(&x); sqlite4_snprintf(zBuf,sizeof(zBuf), "%04d-%02d-%02d", x.Y, x.M, x.D); - sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT, 0); } } /* ** strftime( FORMAT, TIMESTRING, MOD, MOD, ...) @@ -998,11 +998,11 @@ } } } z[j] = 0; sqlite4_result_text(context, z, -1, - z==zBuf ? SQLITE4_TRANSIENT : SQLITE4_DYNAMIC); + z==zBuf ? SQLITE4_TRANSIENT : SQLITE4_DYNAMIC, 0); } /* ** current_time() ** @@ -1085,11 +1085,11 @@ if( pTm ) memcpy(&sNow, pTm, sizeof(sNow)); sqlite4_mutex_leave(sqlite4MutexAlloc(SQLITE4_MUTEX_STATIC_MASTER)); #endif if( pTm ){ strftime(zBuf, 20, zFormat, &sNow); - sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT, 0); } } #endif /* Index: src/delete.c ================================================================== --- src/delete.c +++ src/delete.c @@ -551,11 +551,11 @@ } /* Build the index key. If bAddSeq is true, append a sequence number to ** the end of the key to ensure it is unique. */ sqlite4VdbeAddOp3(v, OP_MakeIdxKey, iIdxCsr, regTmp, regOut); - if( bAddSeq ) sqlite4VdbeChangeP5(v, 1); + if( bAddSeq ) sqlite4VdbeChangeP5(v, OPFLAG_SEQCOUNT); /* Release temp registers */ sqlite4ReleaseTempRange(pParse, regTmp, nTmpReg); } @@ -583,21 +583,32 @@ int *aRegIdx /* Only delete if (aRegIdx && aRegIdx[i]>0) */ ){ Vdbe *v = pParse->pVdbe; Index *pPk; int iPk; + int iPkCsr; int i; int regKey; Index *pIdx; regKey = sqlite4GetTempReg(pParse); pPk = sqlite4FindPrimaryKey(pTab, &iPk); + iPkCsr = baseCur+iPk; for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ - if( pIdx!=pPk && (aRegIdx==0 || aRegIdx[i]>0) ){ + if( pIdx->eIndexType==SQLITE4_INDEX_FTS5 ){ + int iCol; + int iReg = pParse->nMem+1; + pParse->nMem += (1 + pTab->nCol); + for(iCol=0; iColnCol; iCol++){ + sqlite4VdbeAddOp3(v, OP_Column, iPkCsr, iCol, iReg+iCol); + } + sqlite4VdbeAddOp2(v, OP_RowKey, iPkCsr, iReg+pTab->nCol); + sqlite4Fts5CodeUpdate(pParse, pIdx, 0, iReg+pTab->nCol, iReg, 1); + }else if( pIdx!=pPk && (aRegIdx==0 || aRegIdx[i]>0) ){ int addrNotFound; - sqlite4EncodeIndexKey(pParse, pPk, baseCur+iPk, pIdx, baseCur+i,0,regKey); + sqlite4EncodeIndexKey(pParse, pPk, baseCur+iPk,pIdx,baseCur+i,0,regKey); addrNotFound = sqlite4VdbeAddOp4(v, OP_NotFound, baseCur+i, 0, regKey, 0, P4_INT32 ); sqlite4VdbeAddOp1(v, OP_Delete, baseCur+i); sqlite4VdbeJumpHere(v, addrNotFound); Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -364,12 +364,11 @@ ** is responsible for making sure the node eventually gets freed. ** ** If dequote is true, then the token (if it exists) is dequoted. ** If dequote is false, no dequoting is performance. The deQuote ** parameter is ignored if pToken is NULL or if the token does not -** appear to be quoted. If the quotes were of the form "..." (double-quotes) -** then the EP_DblQuoted flag is set on the expression node. +** appear to be quoted. ** ** Special case: If op==TK_INTEGER and pToken points to a string that ** can be translated into a 32-bit integer, then the token is not ** stored in u.zToken. Instead, the integer values is written ** into u.iValue and the EP_IntValue flag is set. No extra storage @@ -405,13 +404,12 @@ pNew->u.zToken = (char*)&pNew[1]; assert( pToken->z!=0 || pToken->n==0 ); if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); pNew->u.zToken[pToken->n] = 0; if( dequote && nExtra>=3 - && ((c = pToken->z[0])=='\'' || c=='"' || c=='[' || c=='`') ){ + && ((c = pToken->z[0])=='\'' || c=='"' || c=='[') ){ sqlite4Dequote(pNew->u.zToken); - if( c=='"' ) pNew->flags |= EP_DblQuoted; } } } #if SQLITE4_MAX_EXPR_DEPTH>0 pNew->nHeight = 1; @@ -1132,10 +1130,11 @@ ** and pWalker->u.i==2 */ case TK_FUNCTION: if( pWalker->u.i==2 ) return 0; /* Fall through */ case TK_ID: + case TK_MATCH: case TK_COLUMN: case TK_AGG_FUNCTION: case TK_AGG_COLUMN: testcase( pExpr->op==TK_ID ); testcase( pExpr->op==TK_COLUMN ); @@ -2587,10 +2586,13 @@ } if( pDef->flags & SQLITE4_FUNC_NEEDCOLL ){ if( !pColl ) pColl = db->pDfltColl; sqlite4VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); } + if( pDef->bMatchinfo ){ + sqlite4VdbeAddOp1(v, OP_Mifunction, pFarg->a[0].pExpr->iTable); + } sqlite4VdbeAddOp4(v, OP_Function, constMask, r1, target, (char*)pDef, P4_FUNCDEF); sqlite4VdbeChangeP5(v, (u8)nFarg); if( nFarg ){ sqlite4ReleaseTempRange(pParse, r1, nFarg); @@ -2713,10 +2715,15 @@ sqlite4VdbeAddOp1(v, OP_RealAffinity, target); } #endif break; } + + case TK_MATCH: { + sqlite4ErrorMsg(pParse, "no index to process MATCH operator"); + return 0; + } /* ** Form A: ** CASE x WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END @@ -3183,10 +3190,11 @@ */ static int isAppropriateForFactoring(Expr *p){ if( !sqlite4ExprIsConstantNotJoin(p) ){ return 0; /* Only constant expressions are appropriate for factoring */ } + if( p->op==TK_MATCH || p->op==TK_TABLE ) return 0; if( (p->flags & EP_FixedDest)==0 ){ return 1; /* Any constant without a fixed destination is appropriate */ } while( p->op==TK_UPLUS ) p = p->pLeft; switch( p->op ){ Index: src/fkey.c ================================================================== --- src/fkey.c +++ src/fkey.c @@ -239,15 +239,15 @@ ** unusable. Bail out early in this case. */ zDfltColl = pParent->aCol[iCol].zColl; if( !zDfltColl ){ zDfltColl = "BINARY"; } - if( sqlite4StrICmp(pIdx->azColl[i], zDfltColl) ) break; + if( sqlite4_stricmp(pIdx->azColl[i], zDfltColl) ) break; zIdxCol = pParent->aCol[iCol].zName; for(j=0; jaCol[j].zCol, zIdxCol)==0 ){ + if( sqlite4_stricmp(pFKey->aCol[j].zCol, zIdxCol)==0 ){ if( aiCol ) aiCol[i] = pFKey->aCol[j].iFrom; break; } } if( j==nCol ) break; @@ -866,11 +866,11 @@ for(i=0; inCol; i++){ char *zKey = p->aCol[i].zCol; int iKey; for(iKey=0; iKeynCol; iKey++){ Column *pCol = &pTab->aCol[iKey]; - if( (zKey ? !sqlite4StrICmp(pCol->zName, zKey) : pCol->isPrimKey) ){ + if( (zKey ? !sqlite4_stricmp(pCol->zName, zKey) : pCol->isPrimKey) ){ if( aChange[iKey]>=0 ) return 1; } } } } ADDED src/fts5.c Index: src/fts5.c ================================================================== --- /dev/null +++ src/fts5.c @@ -0,0 +1,3425 @@ +/* +** 2012 December 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#include "sqliteInt.h" +#include "vdbeInt.h" + +/* +** Stream numbers must be lower than this. +** +** For optimization purposes, it is assumed that a given tokenizer uses +** a set of contiguous stream numbers starting with 0. And that most +** tokens belong to stream 0. +** +** The hard limit is 63 (due to the format of "row size" records). +*/ +#define SQLITE4_FTS5_NSTREAM 32 + +/* +** Records stored within the index: +** +** Row size record: +** There is one "row size" record in the index for each row in the +** indexed table. The "row size" record contains the number of tokens +** in the associated row for each combination of a stream and column +** number (i.e. contains the data required to find the number of +** tokens associated with stream S present in column C of the row for +** all S and C). +** +** The key for the row size record is a single 0x00 byte followed by +** a copy of the PK blob for the table row. +** +** The value is a series of varints. Each column of the table is +** represented by one or more varints packed into the array. +** +** If a column contains only stream 0 tokens, then it is represented +** by a single varint - (nToken << 1), where nToken is the number of +** stream 0 tokens stored in the column. +** +** Or, if the column contains tokens from multiple streams, the first +** varint contains a bitmask indicating which of the streams are present +** (stored as ((bitmask << 1) | 0x01)). Following the bitmask is a +** varint containing the number of tokens for each stream present, in +** ascending order of stream number. +** +** TODO: The format above is not currently implemented! Instead, there +** is a simpler place-holder format (which consumes more space). +** +** Global size record: +** There is a single "global size" record stored in the database. The +** database key for this record is a single byte - 0x00. +** +** The data for this record is a series of varint values. The first +** varint is the total number of rows in the table. The subsequent +** varints make up a "row size" record containing the total number of +** tokens for each S/C combination in all rows of the table. +** +** FTS index records: +** The FTS index records implement the following mapping: +** +** (token, document-pk) -> (list of instances) +** +** The key for each index record is in the same format as the keys for +** regular text indexes. An 0x24 byte, followed by the utf-8 representation +** of the token, followed by 0x00, followed by the PK blob for the table +** row. +** +** TODO: Describe value format. +*/ + +/* +** Default distance value for NEAR operators. +*/ +#define FTS5_DEFAULT_NEAR 10 + +/* +** Token types used by expression parser. +*/ +#define TOKEN_EOF 0 /* end of expression - no more tokens */ +#define TOKEN_PRIMITIVE 1 /* quoted string or non-keyword */ +#define TOKEN_STAR 2 /* * */ +#define TOKEN_PLUS 3 /* + */ +#define TOKEN_NEAR 4 /* NEAR/nnn */ +#define TOKEN_COLON 5 /* : */ +#define TOKEN_NOT 6 /* NOT */ +#define TOKEN_AND 7 /* AND */ +#define TOKEN_OR 8 /* OR */ +#define TOKEN_LP 9 /* ( */ +#define TOKEN_RP 10 /* ) */ + +/* +** Each tokenizer registered with a database handle is stored as an object +** of the following type. All objects associated with a single database +** connection are stored in the singly-linked list starting at +** sqlite4.pTokenizer and connected by Fts5Tokenizer.pNext. +*/ +struct Fts5Tokenizer { + char *zName; /* Name of tokenizer (nul-terminated) */ + void *pCtx; + int (*xCreate)(void*, const char**, int, sqlite4_tokenizer**); + int (*xTokenize)(void*, sqlite4_tokenizer*, + const char*, int, int(*x)(void*, int, int, const char*, int, int, int) + ); + int (*xDestroy)(sqlite4_tokenizer *); + Fts5Tokenizer *pNext; +}; + +/* +** FTS5 specific index data. +** +** This object is part of a database schema, so it may be shared between +** multiple connections. +*/ +struct Fts5Index { + int nTokenizer; /* Elements in azTokenizer[] array */ + char **azTokenizer; /* Name and arguments for tokenizer */ +}; + +/* +** Expression grammar: +** +** phrase := PRIMITIVE +** phrase := PRIMITIVE * +** phrase := phrase + phrase +** phrase := phrase NEAR phrase +** +** expr := phrase +** expr := PRIMITIVE COLON phrase +** +** expr := expr NOT expr +** expr := expr AND expr +** expr := expr OR expr +** expr := LP expr RP +*/ + +/* +** Structure types used by this module. +*/ +typedef struct Fts5Expr Fts5Expr; +typedef struct Fts5ExprNode Fts5ExprNode; +typedef struct Fts5List Fts5List; +typedef struct Fts5MatchIter Fts5MatchIter; +typedef struct Fts5Parser Fts5Parser; +typedef struct Fts5ParserToken Fts5ParserToken; +typedef struct Fts5Phrase Fts5Phrase; +typedef struct Fts5Prefix Fts5Prefix; +typedef struct Fts5Size Fts5Size; +typedef struct Fts5Str Fts5Str; +typedef struct Fts5Token Fts5Token; + + +struct Fts5ParserToken { + int eType; /* Token type */ + int n; /* Size of z[] in bytes */ + const char *z; /* Token value */ +}; + +struct Fts5Parser { + Fts5Tokenizer *pTokenizer; + sqlite4_tokenizer *p; + sqlite4 *db; /* Database handle */ + + char *zErr; /* Error message (or NULL) */ + + const char *zExpr; /* Pointer to expression text (nul-term) */ + int iExpr; /* Current offset in zExpr */ + Fts5ParserToken next; /* Next token */ + + char **azCol; /* Column names of indexed table */ + int nCol; /* Size of azCol[] in bytes */ + int iRoot; /* Root page number of FTS index */ + + /* Space for dequoted copies of strings */ + char *aSpace; + int iSpace; + int nSpace; /* Total size of aSpace in bytes */ +}; + +struct Fts5List { + u8 *aData; + int nData; +}; + +struct Fts5Prefix { + u8 *aPk; /* Buffer containing PK */ + int nPk; /* Size of PK in bytes */ + Fts5Prefix *pNext; /* Next entry in query-time list */ + u8 *aList; + int nList; + int nAlloc; +}; + +struct Fts5Token { + /* TODO: The first three members are redundant in some senses, since the + ** same information is encoded in the aPrefix[]/nPrefix key. */ + int bPrefix; /* True for a prefix search */ + int n; /* Size of z[] in bytes */ + char *z; /* Token value */ + + KVByteArray *aPrefix; /* KV prefix to iterate through */ + KVSize nPrefix; /* Size of aPrefix in bytes */ + KVCursor *pCsr; /* Cursor to iterate thru entries for token */ + Fts5Prefix *pPrefix; /* Head of prefix list */ +}; + +struct Fts5Str { + Fts5Token *aToken; + int nToken; + u8 *aList; + int nList; + int nListAlloc; +}; + +struct Fts5Phrase { + int iCol; /* Column of table to search (-1 -> all) */ + int nStr; + Fts5Str *aStr; + int *aiNear; +}; + +struct Fts5ExprNode { + int eType; + Fts5Phrase *pPhrase; + Fts5ExprNode *pLeft; + Fts5ExprNode *pRight; + const u8 *aPk; /* Primary key of current entry (or null) */ + int nPk; /* Size of aPk[] in bytes */ +}; + +struct Fts5Expr { + Fts5ExprNode *pRoot; /* Root node of expression */ + int nPhrase; /* Number of Fts5Str objects in query */ + Fts5Str **apPhrase; /* All Fts5Str objects */ +}; + +/* +** FTS5 specific cursor data. +*/ +struct Fts5Cursor { + sqlite4 *db; + Fts5Info *pInfo; + Fts5Expr *pExpr; /* MATCH expression for this cursor */ + char *zExpr; /* Full text of MATCH expression */ + KVByteArray *aKey; /* Buffer for primary key */ + int nKeyAlloc; /* Bytes allocated at aKey[] */ + + KVCursor *pCsr; /* Cursor used to retrive values */ + Mem *aMem; /* Array of column values */ + int bMemValid; /* True if contents of aMem[] are valid */ + + Fts5Size *pSz; /* Local size data */ + Fts5Size *pGlobal; /* Global size data */ + i64 nGlobal; /* Total number of rows in table */ + + /* Arrays used by sqlite4_mi_row_count(). */ + int *anRowCS; + int *anRowC; + int *anRowS; + int *anRow; + + Fts5MatchIter *pIter; /* Used by mi_match_detail() */ +}; + +/* +** A deserialized 'size record' (see above). +*/ +struct Fts5Size { + int nCol; /* Number of columns in indexed table */ + int nStream; /* Number of streams */ + i64 *aSz; /* Token count for each C/S */ +}; + +/* +** This type is used when reading (decoding) an instance-list. +*/ +typedef struct InstanceList InstanceList; +struct InstanceList { + u8 *aList; + int nList; + int iList; + + /* The current entry */ + int iCol; + int iStream; + int iOff; +}; + +/* +** An instance of this structure is used by the sqlite4_mi_match_detail() +** API to iterate through matches. +*/ +struct Fts5MatchIter { + int bValid; /* True if aList[] is current row */ + int iCurrent; /* Current index in aList[] (or -1) */ + int iMatch; /* Current iMatch value */ + InstanceList *aList; /* One iterator for each phrase in expr */ +}; + +/* +** Return true for EOF, or false if the next entry is valid. +*/ +static int fts5InstanceListNext(InstanceList *p){ + int i = p->iList; + int bRet = 1; + + while( bRet && inList ){ + u32 iVal; + i += getVarint32(&p->aList[i], iVal); + if( (iVal & 0x03)==0x01 ){ + p->iCol = (iVal>>2); + p->iOff = 0; + } + else if( (iVal & 0x03)==0x03 ){ + p->iStream = (iVal>>2); + } + else{ + p->iOff += (iVal>>1); + bRet = 0; + } + } + if( bRet ){ + p->aList = 0; + } + + p->iList = i; + return bRet; +} + +static int fts5InstanceListEof(InstanceList *p){ + return (p->aList==0); +} + +static void fts5InstanceListAppend( + InstanceList *p, /* Instance list to append to */ + int iCol, /* Column of new entry */ + int iStream, /* Weight of new entry */ + int iOff /* Offset of new entry */ +){ + assert( iCol>=p->iCol ); + assert( iCol>p->iCol || iOff>=p->iOff ); + + if( iCol!=p->iCol ){ + p->iList += putVarint32(&p->aList[p->iList], (iCol<<2)|0x01); + p->iCol = iCol; + p->iOff = 0; + } + + if( iStream!=p->iStream ){ + p->iList += putVarint32(&p->aList[p->iList], (iStream<<2)|0x03); + p->iStream = iStream; + } + + p->iList += putVarint32(&p->aList[p->iList], (iOff-p->iOff)<<1); + p->iOff = iOff; + + assert( p->iList<=p->nList ); +} + +static void fts5InstanceListInit(u8 *aList, int nList, InstanceList *p){ + memset(p, 0, sizeof(InstanceList)); + p->aList = aList; + p->nList = nList; +} + +/* +** Return true if argument c is one of the special non-whitespace +** characters that ends an unquoted expression token. +*/ +static int fts5IsSpecial(char c){ + return (c==':' || c=='(' || c==')' || c=='+' || c=='"' || c=='*'); +} + +static int fts5NextToken( + Fts5Parser *pParse, /* Parser context */ + Fts5ParserToken *p /* OUT: Populate this object */ +){ + const char *z = pParse->zExpr; + char c; + + memset(p, 0, sizeof(Fts5ParserToken)); + + /* Skip past any whitespace */ + while( sqlite4Isspace(z[pParse->iExpr]) ) pParse->iExpr++; + + c = z[pParse->iExpr]; + if( c=='\0' ){ + p->eType = TOKEN_EOF; + } + + else if( c=='(' ){ + pParse->iExpr++; + p->eType = TOKEN_LP; + } + + else if( c==')' ){ + pParse->iExpr++; + p->eType = TOKEN_RP; + } + + else if( c==':' ){ + pParse->iExpr++; + p->eType = TOKEN_COLON; + } + + else if( c=='+' ){ + pParse->iExpr++; + p->eType = TOKEN_PLUS; + } + + else if( c=='*' ){ + pParse->iExpr++; + p->eType = TOKEN_STAR; + } + + else if( c=='"' ){ + char *zOut = &pParse->aSpace[pParse->iSpace]; + const char *zPrimitive = zOut; + int i = pParse->iExpr+1; + + while( z[i] ){ + if( z[i]=='"' ){ + if( z[i+1]=='"' ){ + i++; + }else{ + break; + } + } + *zOut++ = z[i]; + i++; + } + if( z[i]!='"' ){ + /* Mismatched quotation mark */ + return SQLITE4_ERROR; + } + + pParse->iExpr = i+1; + p->eType = TOKEN_PRIMITIVE; + p->z = zPrimitive; + p->n = (zOut - zPrimitive); + pParse->iSpace += (zOut - zPrimitive); + } + + else{ + const char *zPrimitive = &z[pParse->iExpr]; + int n = 0; + while( zPrimitive[n] + && fts5IsSpecial(zPrimitive[n])==0 + && sqlite4Isspace(zPrimitive[n])==0 + ){ + n++; + } + pParse->iExpr += n; + + if( n>=4 && memcmp(zPrimitive, "NEAR", 4)==0 ){ + int nNear = FTS5_DEFAULT_NEAR; + if( n>4 ){ + int i; + nNear = 0; + for(i=5; ieType = TOKEN_NEAR; + p->n = nNear; + p->z = 0; + }else if( n==3 && memcmp(zPrimitive, "NOT", 3)==0 ){ + p->eType = TOKEN_NOT; + } + else if( n==2 && memcmp(zPrimitive, "OR", 2)==0 ){ + p->eType = TOKEN_OR; + } + else if( n==3 && memcmp(zPrimitive, "AND", 3)==0 ){ + p->eType = TOKEN_AND; + }else{ + p->eType = TOKEN_PRIMITIVE; + p->z = zPrimitive; + p->n = n; + } + } + + return SQLITE4_OK; +} + +static int fts5NextToken2( + Fts5Parser *pParse, + Fts5ParserToken *p +){ + int rc = SQLITE4_OK; + if( pParse->iExpr==0 ){ + rc = fts5NextToken(pParse, p); + }else{ + *p = pParse->next; + } + + if( rc==SQLITE4_OK && p->eType!=TOKEN_EOF ){ + rc = fts5NextToken(pParse, &pParse->next); + } + + return rc; +} + +static int fts5PhraseNewStr( + Fts5Parser *pParse, /* Expression parsing context */ + Fts5Phrase *pPhrase, /* Phrase to add a new Fts5Str to */ + int nNear /* Value of nnn in NEAR/nnn operator */ +){ + const int nIncr = 4; + + if( (pPhrase->nStr % nIncr)==0 ){ + Fts5Str *aNew; + aNew = (Fts5Str *)sqlite4DbRealloc(pParse->db, + pPhrase->aStr, (pPhrase->nStr+nIncr)*sizeof(Fts5Str) + ); + if( !aNew ) return SQLITE4_NOMEM; + memset(&aNew[pPhrase->nStr], 0, nIncr*sizeof(Fts5Str)); + pPhrase->aStr = aNew; + } + if( pPhrase->nStr>0 ){ + if( ((pPhrase->nStr-1) % nIncr)==0 ){ + int *aNew; + aNew = (int *)sqlite4DbRealloc(pParse->db, + pPhrase->aiNear, (pPhrase->nStr+nIncr-1)*sizeof(int) + ); + if( !aNew ) return SQLITE4_NOMEM; + pPhrase->aiNear = aNew; + } + pPhrase->aiNear[pPhrase->nStr-1] = nNear; + } + + pPhrase->nStr++; + return SQLITE4_OK; +} + +/* +** Callback for fts5CountTokens(). +*/ +static int fts5CountTokensCb( + void *pCtx, + int iStream, + int iOff, + const char *z, int n, + int iSrc, int nSrc +){ + (*((int *)pCtx))++; + return 0; +} + +/* +** Count the number of tokens in document zDoc/nDoc using the tokenizer and +** tokenizer instance supplied as the first two arguments. Set *pnToken to +** the result before returning. +*/ +static int fts5CountTokens( + Fts5Tokenizer *pTokenizer, + sqlite4_tokenizer *p, + const char *zDoc, + int nDoc, + int *pnToken +){ + int nToken = 0; + int rc; + rc = pTokenizer->xTokenize((void *)&nToken, p, zDoc, nDoc, fts5CountTokensCb); + *pnToken = nToken; + return rc; +} + +struct AppendTokensCtx { + Fts5Parser *pParse; + Fts5Str *pStr; +}; + +static int fts5AppendTokensCb( + void *pCtx, + int iStream, + int iOff, + const char *z, int n, + int iSrc, int nSrc +){ + struct AppendTokensCtx *p = (struct AppendTokensCtx *)pCtx; + Fts5Parser *pParse = p->pParse; + Fts5Token *pToken; + char *zSpace; + int nUsed; + + pToken = &p->pStr->aToken[p->pStr->nToken]; + + zSpace = &pParse->aSpace[pParse->iSpace]; + nUsed = putVarint32((u8 *)zSpace, pParse->iRoot); + zSpace[nUsed++] = 0x24; + pToken->bPrefix = 0; + pToken->pPrefix = 0; + pToken->z = &zSpace[nUsed]; + pToken->n = n; + memcpy(pToken->z, z, n); + pToken->z[n] = '\0'; + + nUsed += (n+1); + pToken->aPrefix = (u8 *)zSpace; + pToken->nPrefix = nUsed; + pToken->pCsr = 0; + pParse->iSpace += nUsed; + p->pStr->nToken++; + + assert( pParse->iSpace<=pParse->nSpace ); + return 0; +} + +static int fts5AppendTokens( + Fts5Parser *pParse, + Fts5Str *pStr, const char *zPrim, + int nPrim +){ + struct AppendTokensCtx ctx; + ctx.pParse = pParse; + ctx.pStr = pStr; + + return pParse->pTokenizer->xTokenize( + (void *)&ctx, pParse->p , zPrim, nPrim, fts5AppendTokensCb + ); +} + +/* +** Append a new token to the current phrase. +*/ +static int fts5PhraseAppend( + Fts5Parser *pParse, + Fts5Phrase *pPhrase, + const char *zPrim, + int nPrim +){ + Fts5Tokenizer *pTok = pParse->pTokenizer; + int nToken; + int rc; + + rc = fts5CountTokens(pTok, pParse->p, zPrim, nPrim, &nToken); + if( rc==SQLITE4_OK && nToken>0 ){ + /* Extend the size of the token array by nToken entries */ + Fts5Str *pStr = &pPhrase->aStr[pPhrase->nStr-1]; + + pStr->aToken = sqlite4DbReallocOrFree(pParse->db, pStr->aToken, + (pStr->nToken + nToken) * sizeof(Fts5Token) + ); + if( !pStr->aToken ){ + rc = SQLITE4_NOMEM; + }else{ + rc = fts5AppendTokens(pParse, pStr, zPrim, nPrim); + } + } + + return rc; +} + +static int fts5PhraseAppendStar( + Fts5Parser *pParse, + Fts5Phrase *pPhrase +){ + Fts5Str *pStr = &pPhrase->aStr[pPhrase->nStr-1]; + Fts5Token *p = &pStr->aToken[pStr->nToken-1]; + + if( p->bPrefix ){ + return SQLITE4_ERROR; + } + p->bPrefix = 1; + p->nPrefix--; + return SQLITE4_OK; +} + +static void fts5PhraseFree(sqlite4 *db, Fts5Phrase *p){ + if( p ){ + int i; + for(i=0; inStr; i++){ + int iTok; + for(iTok=0; iTokaStr[i].nToken; iTok++){ + sqlite4KVCursorClose(p->aStr[i].aToken[iTok].pCsr); + } + sqlite4DbFree(db, p->aStr[i].aToken); + sqlite4DbFree(db, p->aStr[i].aList); + } + sqlite4DbFree(db, p->aiNear); + sqlite4DbFree(db, p->aStr); + sqlite4DbFree(db, p); + } +} + +static int fts5NextTokenOrPhrase( + Fts5Parser *pParse, /* Parser context */ + int *peType, /* OUT: Token type */ + Fts5Phrase **ppPhrase /* OUT: New phrase object */ +){ + int rc; + Fts5Phrase *pPhrase = 0; + Fts5ParserToken t; + + rc = fts5NextToken2(pParse, &t); + *peType = t.eType; + if( rc==SQLITE4_OK && t.eType==TOKEN_PRIMITIVE ){ + + /* Allocate the Fts5Phrase object */ + pPhrase = sqlite4DbMallocZero(pParse->db, sizeof(Fts5Phrase)); + if( pPhrase==0 ){ + rc = SQLITE4_NOMEM; + goto token_or_phrase_out; + } + pPhrase->iCol = -1; + + /* Check if this first primitive is a column name or not. */ + if( pParse->next.eType==TOKEN_COLON ){ + int iCol; + for(iCol=0; iColnCol; iCol++){ + if( sqlite4_strnicmp(pParse->azCol[iCol], t.z, t.n)==0 ) break; + } + if( iCol==pParse->nCol ){ + pParse->zErr = sqlite4MPrintf(pParse->db, + "fts5: no such column: %.*s", t.n, t.z + ); + rc = SQLITE4_ERROR; + goto token_or_phrase_out; + } + pPhrase->iCol = iCol; + + rc = fts5NextToken2(pParse, &t); + if( rc==SQLITE4_OK ) rc = fts5NextToken2(pParse, &t); + if( rc==SQLITE4_OK && t.eType!=TOKEN_PRIMITIVE ){ + rc = SQLITE4_ERROR; + } + if( rc!=SQLITE4_OK ) goto token_or_phrase_out; + } + + /* Add the first Fts5Str to the new phrase object. Populate it with the + ** results of tokenizing t.z/t.n. */ + rc = fts5PhraseNewStr(pParse, pPhrase, 0); + if( rc==SQLITE4_OK ){ + rc = fts5PhraseAppend(pParse, pPhrase, t.z, t.n); + } + if( rc==SQLITE4_OK && pParse->next.eType==TOKEN_STAR ){ + fts5NextToken2(pParse, &t); + rc = fts5PhraseAppendStar(pParse, pPhrase); + } + + /* Add any further primitives connected by "+" or NEAR operators. */ + while( rc==SQLITE4_OK && + (pParse->next.eType==TOKEN_PLUS || pParse->next.eType==TOKEN_NEAR) + ){ + rc = fts5NextToken2(pParse, &t); + if( rc==SQLITE4_OK ){ + if( t.eType==TOKEN_NEAR ){ + rc = fts5PhraseNewStr(pParse, pPhrase, t.n); + if( rc!=SQLITE4_OK ) goto token_or_phrase_out; + } + rc = fts5NextToken2(pParse, &t); + if( rc!=SQLITE4_OK ) goto token_or_phrase_out; + if( t.eType!=TOKEN_PRIMITIVE ){ + rc = SQLITE4_ERROR; + }else{ + rc = fts5PhraseAppend(pParse, pPhrase, t.z, t.n); + if( rc==SQLITE4_OK && pParse->next.eType==TOKEN_STAR ){ + fts5NextToken2(pParse, &t); + rc = fts5PhraseAppendStar(pParse, pPhrase); + } + } + } + } + } + + token_or_phrase_out: + if( rc!=SQLITE4_OK ){ + fts5PhraseFree(pParse->db, pPhrase); + }else{ + *ppPhrase = pPhrase; + } + return rc; +} + +static void fts5FreeExprNode(sqlite4 *db, Fts5ExprNode *pNode){ + if( pNode ){ + fts5PhraseFree(db, pNode->pPhrase); + fts5FreeExprNode(db, pNode->pLeft); + fts5FreeExprNode(db, pNode->pRight); + sqlite4DbFree(db, pNode); + } +} + +static void fts5ExpressionFree(sqlite4 *db, Fts5Expr *pExpr){ + if( pExpr ){ + fts5FreeExprNode(db, pExpr->pRoot); + sqlite4DbFree(db, pExpr->apPhrase); + sqlite4DbFree(db, pExpr); + } +} + +typedef struct ExprHier ExprHier; +struct ExprHier { + Fts5ExprNode **ppNode; + int nOpen; +}; + +static int fts5GrowExprHier( + sqlite4 *db, + int *pnAlloc, + ExprHier **paHier, + int nReq +){ + int rc = SQLITE4_OK; + int nAlloc = *pnAlloc; + if( nAlloceType = eType; + + pp = (*paHier)[*pnHier-1].ppNode; + pNode->pLeft = *pp; + *pp = pNode; + (*paHier)[*pnHier].ppNode = &pNode->pRight; + (*paHier)[*pnHier].nOpen = 0; + (*pnHier)++; + + return SQLITE4_OK; +} + +static void fts5FindStrings(Fts5ExprNode *p, Fts5Str ***papStr){ + if( p ){ + if( p->eType==TOKEN_PRIMITIVE ){ + int i; + Fts5Str *aStr = p->pPhrase->aStr; + for(i=0; ipPhrase->nStr; i++){ + **papStr = &aStr[i]; + (*papStr)++; + } + } + fts5FindStrings(p->pLeft, papStr); + fts5FindStrings(p->pRight, papStr); + } +} + +static int fts5ParseExpression( + sqlite4 *db, /* Database handle */ + Fts5Tokenizer *pTokenizer, /* Tokenizer module */ + sqlite4_tokenizer *p, /* Tokenizer instance */ + int iRoot, /* Root page number of FTS index */ + char **azCol, /* Array of column names (nul-term'd) */ + int nCol, /* Size of array azCol[] */ + const char *zExpr, /* FTS expression text */ + Fts5Expr **ppExpr, /* OUT: Expression object */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE4_OK; + Fts5Parser sParse; + int nStr = 0; + int nExpr; + int i; + Fts5Expr *pExpr; + + int nHier = 0; + int nHierAlloc = 0; + ExprHier *aHier = 0; + + nExpr = sqlite4Strlen30(zExpr); + memset(&sParse, 0, sizeof(Fts5Parser)); + sParse.zExpr = zExpr; + sParse.azCol = azCol; + sParse.nCol = nCol; + sParse.pTokenizer = pTokenizer; + sParse.p = p; + sParse.db = db; + sParse.iRoot = iRoot; + + pExpr = sqlite4DbMallocZero(db, sizeof(Fts5Expr) + nExpr*4); + if( !pExpr ) return SQLITE4_NOMEM; + sParse.aSpace = (char *)&pExpr[1]; + sParse.nSpace = nExpr*4; + + rc = fts5GrowExprHier(db, &nHierAlloc, &aHier, 1); + if( rc==SQLITE4_OK ){ + aHier[0].ppNode = &pExpr->pRoot; + aHier[0].nOpen = 0; + nHier = 1; + } + + while( rc==SQLITE4_OK ){ + int eType = 0; + Fts5Phrase *pPhrase = 0; + Fts5ExprNode *pNode = 0; + + rc = fts5NextTokenOrPhrase(&sParse, &eType, &pPhrase); + if( rc!=SQLITE4_OK || eType==TOKEN_EOF ) break; + + switch( eType ){ + case TOKEN_PRIMITIVE: { + Fts5ExprNode **pp = aHier[nHier-1].ppNode; + if( *pp ){ + rc = fts5AddBinary(db, TOKEN_AND, &nHier, &nHierAlloc, &aHier); + pp = aHier[nHier-1].ppNode; + } + if( rc==SQLITE4_OK ){ + pNode = sqlite4DbMallocZero(db, sizeof(Fts5ExprNode)); + if( pNode==0 ){ + rc = SQLITE4_NOMEM; + }else{ + pNode->eType = TOKEN_PRIMITIVE; + pNode->pPhrase = pPhrase; + *pp = pNode; + } + } + nStr += pPhrase->nStr; + break; + } + + case TOKEN_AND: + case TOKEN_OR: + case TOKEN_NOT: { + Fts5ExprNode **pp = aHier[nHier-1].ppNode; + + if( *pp==0 ){ + rc = SQLITE4_ERROR; + }else{ + while( nHier>1 + && aHier[nHier-1].nOpen==0 + && (*aHier[nHier-2].ppNode)->eType < eType + ){ + nHier--; + } + + rc = fts5AddBinary(db, eType, &nHier, &nHierAlloc, &aHier); + } + break; + } + + case TOKEN_LP: { + Fts5ExprNode **pp = aHier[nHier-1].ppNode; + if( *pp ){ + rc = SQLITE4_ERROR; + }else{ + aHier[nHier-1].nOpen++; + } + break; + } + + case TOKEN_RP: { + Fts5ExprNode **pp = aHier[nHier-1].ppNode; + if( *pp==0 ){ + rc = SQLITE4_ERROR; + }else{ + for(i=nHier-1; i>=0; i--){ + if( aHier[i].nOpen>0 ) break; + } + if( i<0 ){ + rc = SQLITE4_ERROR; + }else{ + aHier[i].nOpen--; + nHier = i+1; + } + } + break; + } + + default: + rc = SQLITE4_ERROR; + break; + } + + if( rc!=SQLITE4_OK ){ + sqlite4DbFree(db, pNode); + break; + } + } + + if( rc==SQLITE4_OK && *aHier[nHier-1].ppNode==0 ){ + rc = SQLITE4_ERROR; + } + for(i=0; rc==SQLITE4_OK && i0 ) rc = SQLITE4_ERROR; + } + + if( rc==SQLITE4_OK ){ + pExpr->nPhrase = nStr; + pExpr->apPhrase = (Fts5Str**)sqlite4DbMallocZero(db, sizeof(Fts5Str*)*nStr); + if( pExpr->apPhrase==0 ){ + rc = SQLITE4_NOMEM; + }else{ + Fts5Str **a = pExpr->apPhrase; + fts5FindStrings(pExpr->pRoot, &a); + } + } + + if( rc!=SQLITE4_OK ){ + fts5ExpressionFree(db, pExpr); + *pzErr = sParse.zErr; + pExpr = 0; + } + *ppExpr = pExpr; + sqlite4DbFree(db, aHier); + return rc; +} + +/* +** Search for the Fts5Tokenizer object named zName. Return a pointer to it +** if it exists, or NULL otherwise. +*/ +static Fts5Tokenizer *fts5FindTokenizer(sqlite4 *db, const char *zName){ + Fts5Tokenizer *p; + for(p=db->pTokenizer; p; p=p->pNext){ + if( 0==sqlite4_stricmp(zName, p->zName) ) break; + } + return p; +} + +static void fts5TokenizerCreate( + Parse *pParse, + Fts5Index *pFts, + Fts5Tokenizer **ppTokenizer, + sqlite4_tokenizer **pp +){ + Fts5Tokenizer *pTok; + char *zTok; /* Tokenizer name */ + const char **azArg; /* Tokenizer arguments */ + int nArg; /* Number of elements in azArg */ + + if( pFts->nTokenizer ){ + zTok = pFts->azTokenizer[0]; + azArg = (const char **)&pFts->azTokenizer[1]; + nArg = pFts->nTokenizer-1; + }else{ + zTok = "simple"; + azArg = 0; + nArg = 0; + } + + *ppTokenizer = pTok = fts5FindTokenizer(pParse->db, zTok); + if( !pTok ){ + sqlite4ErrorMsg(pParse, "no such tokenizer: \"%s\"", zTok); + }else{ + int rc = pTok->xCreate(pTok->pCtx, azArg, nArg, pp); + if( rc!=SQLITE4_OK ){ + assert( *pp==0 ); + sqlite4ErrorMsg(pParse, "error creating tokenizer"); + } + } +} + +static void fts5TokenizerDestroy(Fts5Tokenizer *pTok, sqlite4_tokenizer *p){ + if( p ) pTok->xDestroy(p); +} + +void sqlite4ShutdownFts5(sqlite4 *db){ + Fts5Tokenizer *p; + Fts5Tokenizer *pNext; + for(p=db->pTokenizer; p; p=pNext){ + pNext = p->pNext; + sqlite4DbFree(db, p); + } +} + +/* +** This function is used to install custom FTS tokenizers. +*/ +int sqlite4_create_tokenizer( + sqlite4 *db, + const char *zName, + void *pCtx, + int (*xCreate)(void*, const char**, int, sqlite4_tokenizer**), + int (*xTokenize)(void*, sqlite4_tokenizer*, + const char*, int, int(*x)(void*, int, int, const char*, int, int, int) + ), + int (*xDestroy)(sqlite4_tokenizer *) +){ + int rc = SQLITE4_OK; + sqlite4_mutex_enter(db->mutex); + + /* It is not possible to override an existing tokenizer */ + if( fts5FindTokenizer(db, zName) ){ + rc = SQLITE4_ERROR; + }else{ + int nName = sqlite4Strlen30(zName); + Fts5Tokenizer *pTokenizer = (Fts5Tokenizer *)sqlite4DbMallocZero(db, + sizeof(Fts5Tokenizer) + nName+1 + ); + if( !pTokenizer ){ + rc = SQLITE4_NOMEM; + }else{ + pTokenizer->pCtx = pCtx; + pTokenizer->xCreate = xCreate; + pTokenizer->xTokenize = xTokenize; + pTokenizer->xDestroy = xDestroy; + pTokenizer->zName = (char *)&pTokenizer[1]; + memcpy(pTokenizer->zName, zName, nName+1); + + pTokenizer->pNext = db->pTokenizer; + db->pTokenizer = pTokenizer; + } + } + + rc = sqlite4ApiExit(db, rc); + sqlite4_mutex_leave(db->mutex); + return rc; +} + +/* +** Return the size of an Fts5Index structure, in bytes. +*/ +int sqlite4Fts5IndexSz(void){ + return sizeof(Fts5Index); +} + +/* +** Initialize the fts5 specific part of the index object passed as the +** second argument. +*/ +void sqlite4Fts5IndexInit(Parse *pParse, Index *pIdx, ExprList *pArgs){ + Fts5Index *pFts = pIdx->pFts; + + if( pArgs ){ + int i; + for(i=0; pParse->nErr==0 && inExpr; i++){ + char *zArg = pArgs->a[i].zName; + char *zVal = pArgs->a[i].pExpr->u.zToken; + + if( zArg && sqlite4_stricmp(zArg, "tokenizer")==0 ){ + /* zVal is the name of the tokenizer to use. Any subsequent arguments + ** that do not contain assignment operators (=) are also passed to + ** the tokenizer. Figure out how many bytes of space are required for + ** all. */ + int j; + char *pSpace; + int nByte = sqlite4Strlen30(zVal) + 1; + for(j=i+1; jnExpr; j++){ + ExprListItem *pItem = &pArgs->a[j]; + if( pItem->zName ) break; + nByte += sqlite4Strlen30(pItem->pExpr->u.zToken) + 1; + } + nByte += sizeof(char *) * (j-i); + pFts->azTokenizer = (char **)sqlite4DbMallocZero(pParse->db, nByte); + if( pFts->azTokenizer==0 ) return; + pFts->nTokenizer = (j-i); + + pSpace = (char *)&pFts->azTokenizer[j-i]; + for(j=i; jnExpr; j++){ + ExprListItem *pItem = &pArgs->a[j]; + if( pItem->zName && j>i ){ + break; + }else{ + int nToken = sqlite4Strlen30(pItem->pExpr->u.zToken); + memcpy(pSpace, pItem->pExpr->u.zToken, nToken+1); + pFts->azTokenizer[j-i] = pSpace; + pSpace += nToken+1; + } + } + + /* If this function is being called as part of a CREATE INDEX statement + ** issued by the user (to create a new index) check if the tokenizer + ** is valid. If not, return an error. Do not do this if this function + ** is being called as part of parsing an existing database schema. + */ + if( pParse->db->init.busy==0 ){ + Fts5Tokenizer *pTok = 0; + sqlite4_tokenizer *t = 0; + + fts5TokenizerCreate(pParse, pFts, &pTok, &t); + fts5TokenizerDestroy(pTok, t); + } + } + else{ + sqlite4ErrorMsg(pParse,"unrecognized argument: \"%s\"", zArg?zArg:zVal); + } + } + } +} + +void sqlite4Fts5IndexFree(sqlite4 *db, Index *pIdx){ + if( pIdx->pFts ){ + sqlite4DbFree(db, pIdx->pFts->azTokenizer); + } +} + + +/* +** Context structure passed to tokenizer callback when tokenizing a document. +** +** The hash table maps between tokens and TokenizeTerm structures. +** +** TokenizeTerm structures are allocated using sqlite4DbMalloc(). Immediately +** following the structure in memory is the token itself (TokenizeTerm.nToken +** bytes of data). Following this is the list of token instances in the same +** format as it is stored in the database. +** +** All of the above is a single allocation, size TokenizeTerm.nAlloc bytes. +** If the initial allocation is too small, it is extended using +** sqlite4DbRealloc(). +*/ +typedef struct TokenizeCtx TokenizeCtx; +typedef struct TokenizeTerm TokenizeTerm; +struct TokenizeCtx { + int rc; + int iCol; + int nCol; /* Number of columns in table */ + sqlite4 *db; + int nMax; + i64 *aSz; /* Number of tokens in each column/stream */ + int nStream; /* Number of streams in document */ + Hash hash; +}; +struct TokenizeTerm { + int iStream; /* Weight of previous entry */ + int iCol; /* Column containing previous entry */ + int iOff; /* Token offset of previous entry */ + int nToken; /* Size of token in bytes */ + int nData; /* Bytes of data in value */ + int nAlloc; /* Bytes of data allocated */ +}; + +TokenizeTerm *fts5TokenizeAppendInt( + TokenizeCtx *p, + TokenizeTerm *pTerm, + int iVal +){ + unsigned char *a; + int nSpace = pTerm->nAlloc-pTerm->nData-pTerm->nToken-sizeof(TokenizeTerm); + + if( nSpace < 5 ){ + int nAlloc = (pTerm->nAlloc<256) ? 256 : pTerm->nAlloc * 2; + pTerm = sqlite4DbReallocOrFree(p->db, pTerm, nAlloc); + if( !pTerm ) return 0; + pTerm->nAlloc = sqlite4DbMallocSize(p->db, pTerm); + } + + a = &(((unsigned char *)&pTerm[1])[pTerm->nToken+pTerm->nData]); + pTerm->nData += putVarint32(a, iVal); + return pTerm; +} + +static int fts5TokenizeCb( + void *pCtx, + int iStream, + int iOff, + const char *zToken, + int nToken, + int iSrc, + int nSrc +){ + TokenizeCtx *p = (TokenizeCtx *)pCtx; + sqlite4 *db = p->db; + TokenizeTerm *pTerm = 0; + + /* TODO: Error here if iStream is out of range */ + + if( nToken>p->nMax ) p->nMax = nToken; + + if( iStream>=p->nStream ){ + int iCol; + int nOld = p->nStream; + int nNew = 4; + i64 *aNew; + + while( nNew<=iStream ) nNew = nNew*2; + aNew = sqlite4DbMallocZero(db, nNew*p->nCol*sizeof(i64)); + if( aNew==0 ) goto tokenize_cb_out; + + for(iCol=0; iColnCol; iCol++){ + int iStr; + for(iStr=0; iStraSz[nOld*iCol + iStr]; + } + } + + sqlite4DbFree(db, p->aSz); + p->aSz = aNew; + p->nStream = nNew; + } + p->aSz[p->iCol * p->nStream + iStream]++; + + pTerm = (TokenizeTerm *)sqlite4HashFind(&p->hash, zToken, nToken); + if( pTerm==0 ){ + /* Size the initial allocation so that it fits in the lookaside buffer */ + int nAlloc = sizeof(TokenizeTerm) + nToken + 32; + + pTerm = sqlite4DbMallocZero(p->db, nAlloc); + if( pTerm ){ + pTerm->nAlloc = sqlite4DbMallocSize(p->db, pTerm); + pTerm->nToken = nToken; + memcpy(&pTerm[1], zToken, nToken); + } + if( pTerm==0 ) goto tokenize_cb_out; + }else{ + sqlite4HashInsert(&p->hash, zToken, nToken, 0); + } + + if( iStream!=pTerm->iStream ){ + pTerm = fts5TokenizeAppendInt(p, pTerm, (iStream << 2) | 0x00000003); + if( !pTerm ) goto tokenize_cb_out; + pTerm->iStream = iStream; + } + + if( pTerm && p->iCol!=pTerm->iCol ){ + pTerm = fts5TokenizeAppendInt(p, pTerm, (p->iCol << 2) | 0x00000001); + if( !pTerm ) goto tokenize_cb_out; + pTerm->iCol = p->iCol; + pTerm->iOff = 0; + } + + pTerm = fts5TokenizeAppendInt(p, pTerm, (iOff-pTerm->iOff) << 1); + if( !pTerm ) goto tokenize_cb_out; + pTerm->iOff = iOff; + +tokenize_cb_out: + sqlite4HashInsert(&p->hash, (char *)&pTerm[1], nToken, pTerm); + if( !pTerm ){ + p->rc = SQLITE4_NOMEM; + return 1; + } + + return 0; +} + +static int fts5LoadSizeRecord( + sqlite4 *db, /* Database handle */ + u8 *aKey, int nKey, /* KVStore key */ + int nMinStream, /* Space for at least this many streams */ + Fts5Info *pInfo, /* Info record */ + i64 *pnRow, /* non-NULL when reading global record */ + Fts5Size **ppSz /* OUT: Loaded size record */ +){ + Fts5Size *pSz = 0; /* Size object */ + KVCursor *pCsr = 0; /* Cursor used to read global record */ + int rc; + + rc = sqlite4KVStoreOpenCursor(db->aDb[pInfo->iDb].pKV, &pCsr); + if( rc==SQLITE4_OK ){ + rc = sqlite4KVCursorSeek(pCsr, aKey, nKey, 0); + if( rc==SQLITE4_NOTFOUND ){ + if( pnRow ){ + int nByte = sizeof(Fts5Size) + sizeof(i64) * pInfo->nCol * nMinStream; + pSz = sqlite4DbMallocZero(db, nByte); + if( pSz==0 ){ + rc = SQLITE4_NOMEM; + }else{ + pSz->aSz = (i64 *)&pSz[1]; + pSz->nStream = nMinStream; + pSz->nCol = pInfo->nCol; + *pnRow = 0; + rc = SQLITE4_OK; + } + }else{ + rc = SQLITE4_CORRUPT_BKPT; + } + }else if( rc==SQLITE4_OK ){ + const u8 *aData = 0; + int nData = 0; + rc = sqlite4KVCursorData(pCsr, 0, -1, &aData, &nData); + if( rc==SQLITE4_OK ){ + int iOff = 0; + int nStream = 0; + int nAlloc; + + /* If pnRow is not NULL, then this is the global record. Read the + ** number of documents in the table from the start of the record. */ + if( pnRow ){ + iOff += sqlite4GetVarint(&aData[iOff], (u64 *)pnRow); + } + iOff += getVarint32(&aData[iOff], nStream); + nAlloc = (nStream < nMinStream ? nMinStream : nStream); + + pSz = sqlite4DbMallocZero(db, + sizeof(Fts5Size) + sizeof(i64) * pInfo->nCol * nAlloc + ); + if( pSz==0 ){ + rc = SQLITE4_NOMEM; + }else{ + int iCol = 0; + pSz->aSz = (i64 *)&pSz[1]; + pSz->nCol = pInfo->nCol; + pSz->nStream = nAlloc; + while( iOffaSz[iCol*nAlloc]; + for(i=0; i=0 ){ + iOff += sqlite4PutVarint(&a[iOff], nRow); + } + iOff += sqlite4PutVarint(&a[iOff], pSz->nStream); + + for(iCol=0; iColnCol; iCol++){ + int i; + for(i=0; inStream; i++){ + iOff += sqlite4PutVarint(&a[iOff], pSz->aSz[iCol*pSz->nStream+i]); + } + } + + return sqlite4KVStoreReplace(p, aKey, nKey, a, iOff); +} + +static int fts5CsrLoadGlobal(Fts5Cursor *pCsr){ + int rc = SQLITE4_OK; + if( pCsr->pGlobal==0 ){ + int nKey; + u8 aKey[10]; + nKey = putVarint32(aKey, pCsr->pInfo->iRoot); + aKey[nKey++] = 0x00; + rc = fts5LoadSizeRecord( + pCsr->db, aKey, nKey, 0, pCsr->pInfo, &pCsr->nGlobal, &pCsr->pGlobal + ); + } + return rc; +} + +static int fts5CsrLoadSz(Fts5Cursor *pCsr){ + int rc = SQLITE4_OK; + if( pCsr->pSz==0 ){ + sqlite4 *db = pCsr->db; + Fts5Info *pInfo = pCsr->pInfo; + u8 *aKey; + int nKey = 0; + int nPk = pCsr->pExpr->pRoot->nPk; + + aKey = (u8 *)sqlite4DbMallocZero(db, 10 + nPk); + if( !aKey ) return SQLITE4_NOMEM; + + nKey = putVarint32(aKey, pInfo->iRoot); + aKey[nKey++] = 0x00; + memcpy(&aKey[nKey], pCsr->pExpr->pRoot->aPk, nPk); + nKey += nPk; + + rc = fts5LoadSizeRecord(pCsr->db, aKey, nKey, 0, pInfo, 0, &pCsr->pSz); + sqlite4DbFree(db, aKey); + } + + return rc; +} + + +/* +** Update an fts index. +*/ +int sqlite4Fts5Update( + sqlite4 *db, /* Database handle */ + Fts5Info *pInfo, /* Description of fts index to update */ + int iRoot, + Mem *pKey, /* Primary key blob */ + Mem *aArg, /* Array of arguments (see above) */ + int bDel, /* True for a delete, false for insert */ + char **pzErr /* OUT: Error message */ +){ + int i; + int rc = SQLITE4_OK; + KVStore *pStore; + TokenizeCtx sCtx; + int nTnum = 0; + u32 dummy = 0; + + u8 *aSpace = 0; + int nSpace = 0; + + const u8 *pPK; + int nPK; + HashElem *pElem; + + if( iRoot==0 ) iRoot = pInfo->iRoot; + pStore = db->aDb[pInfo->iDb].pKV; + + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.db = db; + sCtx.nCol = pInfo->nCol; + sqlite4HashInit(db->pEnv, &sCtx.hash, 1); + + pPK = (const u8 *)sqlite4_value_blob(pKey); + nPK = sqlite4_value_bytes(pKey); + + nTnum = getVarint32(pPK, dummy); + nPK -= nTnum; + pPK += nTnum; + + for(i=0; rc==SQLITE4_OK && inCol; i++){ + sqlite4_value *pArg = (sqlite4_value *)(&aArg[i]); + if( pArg->flags & MEM_Str ){ + const char *zText; + int nText; + + zText = (const char *)sqlite4_value_text(pArg); + nText = sqlite4_value_bytes(pArg); + sCtx.iCol = i; + rc = pInfo->pTokenizer->xTokenize( + &sCtx, pInfo->p, zText, nText, fts5TokenizeCb + ); + } + } + + /* Allocate enough space to serialize all the stuff that needs to + ** be inserted into the database. Specifically: + ** + ** * Space for index record keys, + ** * space for the size record and key for this document, and + ** * space for the updated global size record for the document set. + ** + ** To make it easier, the below allocates enough space to simultaneously + ** store the largest index record key and the largest possible global + ** size record. + */ + nSpace = (sqlite4VarintLen(iRoot) + 2 + sCtx.nMax + nPK) + + (9 * (2 + pInfo->nCol * sCtx.nStream)); + aSpace = sqlite4DbMallocRaw(db, nSpace); + if( aSpace==0 ) rc = SQLITE4_NOMEM; + + for(pElem=sqliteHashFirst(&sCtx.hash); pElem; pElem=sqliteHashNext(pElem)){ + TokenizeTerm *pTerm = (TokenizeTerm *)sqliteHashData(pElem); + if( rc==SQLITE4_OK ){ + int nToken = sqliteHashKeysize(pElem); + char *zToken = (char *)sqliteHashKey(pElem); + u8 *aKey = aSpace; + int nKey; + + nKey = putVarint32(aKey, iRoot); + aKey[nKey++] = 0x24; + memcpy(&aKey[nKey], zToken, nToken); + nKey += nToken; + aKey[nKey++] = 0x00; + memcpy(&aKey[nKey], pPK, nPK); + nKey += nPK; + + if( bDel ){ + /* delete key aKey/nKey from the index */ + rc = sqlite4KVStoreReplace(pStore, aKey, nKey, 0, -1); + }else{ + /* Insert a new entry for aKey/nKey into the fts index */ + const KVByteArray *aData = (const KVByteArray *)&pTerm[1]; + aData += pTerm->nToken; + rc = sqlite4KVStoreReplace(pStore, aKey, nKey, aData, pTerm->nData); + } + } + sqlite4DbFree(db, pTerm); + } + + /* Write the size record into the db */ + if( rc==SQLITE4_OK ){ + u8 *aKey = aSpace; + int nKey; + + nKey = putVarint32(aKey, iRoot); + aKey[nKey++] = 0x00; + memcpy(&aKey[nKey], pPK, nPK); + nKey += nPK; + + if( bDel==0 ){ + Fts5Size sSz; + sSz.nCol = pInfo->nCol; + sSz.nStream = sCtx.nStream; + sSz.aSz = sCtx.aSz; + rc = fts5StoreSizeRecord(pStore, aKey, nKey, &sSz, -1, &aKey[nKey]); + }else{ + rc = sqlite4KVStoreReplace(pStore, aKey, nKey, 0, -1); + } + } + + /* Update the global record */ + if( rc==SQLITE4_OK ){ + Fts5Size *pSz; /* Deserialized global size record */ + i64 nRow; /* Number of rows in indexed table */ + u8 *aKey = aSpace; /* Space to format the global record key */ + int nKey; /* Size of global record key in bytes */ + + nKey = putVarint32(aKey, iRoot); + aKey[nKey++] = 0x00; + rc = fts5LoadSizeRecord(db, aKey, nKey, sCtx.nStream, pInfo, &nRow, &pSz); + assert( rc!=SQLITE4_OK || pSz->nStream>=sCtx.nStream ); + + if( rc==SQLITE4_OK ){ + int iCol; + for(iCol=0; iColnCol; iCol++){ + int iStr; + i64 *aIn = &sCtx.aSz[iCol * sCtx.nStream]; + i64 *aOut = &pSz->aSz[iCol * pSz->nStream]; + for(iStr=0; iStrdb; + Fts5Info *pInfo; /* p4 argument for FtsUpdate opcode */ + int nByte; + + nByte = sizeof(Fts5Info); + if( bCol ){ + int i; + int nCol = pIdx->pTable->nCol; + for(i=0; ipTable->aCol[i].zName; + nByte += sqlite4Strlen30(zCol) + 1; + } + nByte += nCol * sizeof(char *); + } + + pInfo = sqlite4DbMallocZero(db, nByte); + if( pInfo ){ + pInfo->iDb = sqlite4SchemaToIndex(db, pIdx->pSchema); + pInfo->iRoot = pIdx->tnum; + pInfo->iTbl = sqlite4FindPrimaryKey(pIdx->pTable, 0)->tnum; + pInfo->nCol = pIdx->pTable->nCol; + fts5TokenizerCreate(pParse, pIdx->pFts, &pInfo->pTokenizer, &pInfo->p); + + if( pInfo->p==0 ){ + assert( pParse->nErr ); + sqlite4DbFree(db, pInfo); + pInfo = 0; + } + else if( bCol ){ + int i; + char *p; + int nCol = pIdx->pTable->nCol; + + pInfo->azCol = (char **)(&pInfo[1]); + p = (char *)(&pInfo->azCol[nCol]); + for(i=0; ipTable->aCol[i].zName; + int n = sqlite4Strlen30(zCol) + 1; + pInfo->azCol[i] = p; + memcpy(p, zCol, n); + p += n; + } + } + } + + return pInfo; +} + +void sqlite4Fts5CodeUpdate( + Parse *pParse, + Index *pIdx, + int iRegRoot, + int iRegPk, + int iRegData, + int bDel +){ + Vdbe *v; + Fts5Info *pInfo; /* p4 argument for FtsUpdate opcode */ + + if( 0==(pInfo = fts5InfoCreate(pParse, pIdx, 0)) ) return; + + v = sqlite4GetVdbe(pParse); + sqlite4VdbeAddOp3(v, OP_FtsUpdate, iRegPk, iRegRoot, iRegData); + sqlite4VdbeChangeP4(v, -1, (const char *)pInfo, P4_FTS5INFO); + sqlite4VdbeChangeP5(v, (u8)bDel); +} + +void sqlite4Fts5CodeQuery( + Parse *pParse, + Index *pIdx, + int iCsr, + int iJump, + int iRegMatch +){ + Vdbe *v; + Fts5Info *pInfo; /* p4 argument for FtsOpen opcode */ + + if( 0==(pInfo = fts5InfoCreate(pParse, pIdx, 1)) ) return; + + v = sqlite4GetVdbe(pParse); + sqlite4VdbeAddOp3(v, OP_FtsOpen, iCsr, iJump, iRegMatch); + sqlite4VdbeChangeP4(v, -1, (const char *)pInfo, P4_FTS5INFO); +} + +void sqlite4Fts5FreeInfo(sqlite4 *db, Fts5Info *p){ + if( db->pnBytesFreed==0 ){ + if( p->p ) p->pTokenizer->xDestroy(p->p); + sqlite4DbFree(db, p); + } +} + +void sqlite4Fts5CodeCksum( + Parse *pParse, + Index *pIdx, + int iCksum, + int iReg, + int bIdx /* True for fts index, false for table */ +){ + Vdbe *v; + Fts5Info *pInfo; /* p4 argument for FtsCksum opcode */ + + if( 0==(pInfo = fts5InfoCreate(pParse, pIdx, 0)) ) return; + + v = sqlite4GetVdbe(pParse); + sqlite4VdbeAddOp3(v, OP_FtsCksum, iCksum, 0, iReg); + sqlite4VdbeChangeP4(v, -1, (const char *)pInfo, P4_FTS5INFO); + sqlite4VdbeChangeP5(v, bIdx); +} + +/* +** Calculate a 64-bit checksum for a term instance. The index checksum is +** the XOR of the checksum for each term instance in the table. A term +** instance checksum is calculated based on: +** +** * the term itself, +** * the pk of the row the instance appears in, +** * the weight assigned to the instance, +** * the column number, and +** * the term offset. +*/ +static i64 fts5TermInstanceCksum( + const u8 *aTerm, int nTerm, + const u8 *aPk, int nPk, + int iStream, + int iCol, + int iOff +){ + int i; + i64 cksum = 0; + + /* Add the term to the checksum */ + for(i=0; ipPK, p->nPK, + (const u8 *)zToken, nToken, iStream, p->iCol, iOff + ); + + p->cksum = (p->cksum ^ cksum); + return 0; +} + +int sqlite4Fts5RowCksum( + sqlite4 *db, /* Database handle */ + Fts5Info *pInfo, /* Index description */ + Mem *pKey, /* Primary key blob */ + Mem *aArg, /* Array of column values */ + i64 *piCksum /* OUT: Checksum value */ +){ + int i; + int rc = SQLITE4_OK; + CksumCtx sCtx; + int nTnum = 0; + u32 dummy = 0; + + sCtx.cksum = 0; + + sCtx.pPK = (const u8 *)sqlite4_value_blob(pKey); + sCtx.nPK = sqlite4_value_bytes(pKey); + nTnum = getVarint32(sCtx.pPK, dummy); + sCtx.nPK -= nTnum; + sCtx.pPK += nTnum; + + for(i=0; rc==SQLITE4_OK && inCol; i++){ + sqlite4_value *pArg = (sqlite4_value *)(&aArg[i]); + if( pArg->flags & MEM_Str ){ + const char *zText; + int nText; + + zText = (const char *)sqlite4_value_text(pArg); + nText = sqlite4_value_bytes(pArg); + sCtx.iCol = i; + rc = pInfo->pTokenizer->xTokenize( + &sCtx, pInfo->p, zText, nText, fts5CksumCb + ); + } + } + + *piCksum = sCtx.cksum; + return rc; +} + +/* +** Obtain the primary key value from the entry cursor pToken->pCsr currently +** points to. Set *paPk to point to a buffer containing the PK, and *pnPk +** to the size of the buffer in bytes before returning. +** +** Return SQLITE4_OK if everything goes according to plan, or an error code +** if an error occurs. If an error occurs the final values of *paPk and *pnPk +** are undefined. +*/ +static int fts5TokenPk(Fts5Token *p, const u8 **paPk, int *pnPk){ + int rc; + + if( p->pCsr ){ + const u8 *aKey; + int nKey; + + rc = sqlite4KVCursorKey(p->pCsr, &aKey, &nKey); + if( rc==SQLITE4_OK ){ + if( nKeynPrefix || memcmp(p->aPrefix, aKey, p->nPrefix) ){ + rc = SQLITE4_NOTFOUND; + }else if( p->bPrefix==0 ){ + *paPk = &aKey[p->nPrefix]; + *pnPk = nKey - p->nPrefix; + }else{ + const u8 *z = &aKey[p->nPrefix]; + while( *(z++)!='\0' ); + *paPk = z; + *pnPk = nKey - (z-aKey); + } + } + }else{ + if( p->pPrefix ){ + *paPk = p->pPrefix->aPk; + *pnPk = p->pPrefix->nPk; + rc = SQLITE4_OK; + }else{ + rc = SQLITE4_NOTFOUND; + } + } + + return rc; +} + +static int fts5TokenAdvance(sqlite4 *db, Fts5Token *pToken){ + int rc; + if( pToken->pCsr ){ + rc = sqlite4KVCursorNext(pToken->pCsr); + }else if( pToken->pPrefix ){ + Fts5Prefix *pDel = pToken->pPrefix; + pToken->pPrefix = pDel->pNext; + sqlite4DbFree(db, pDel->aList); + sqlite4DbFree(db, pDel); + rc = SQLITE4_OK; + }else{ + rc = SQLITE4_NOTFOUND; + } + return rc; +} + +static int fts5TokenData(Fts5Token *pToken, const u8 **paData, int *pnData){ + int rc; + if( pToken->pCsr ){ + rc = sqlite4KVCursorData(pToken->pCsr, 0, -1, paData, pnData); + }else if( pToken->pPrefix ){ + *paData = pToken->pPrefix->aList; + *pnData = pToken->pPrefix->nList; + rc = SQLITE4_OK; + }else{ + rc = SQLITE4_NOTFOUND; + } + + return rc; +} + +/* +** Compare keys (aLeft/nLeft) and (aRight/nRight) using the ordinary memcmp() +** method. Except, if either aLeft or aRight are NULL, consider them larger +** than all other values. +*/ +static int fts5KeyCompare( + const u8 *aLeft, int nLeft, + const u8 *aRight, int nRight +){ + int res; + int nMin; + + res = (aLeft==0) - (aRight==0); + if( res==0 ){ + nMin = (nLeft > nRight) ? nRight : nLeft; + res = memcmp(aLeft, aRight, nMin); + } + return (res ? res : (nLeft-nRight)); +} + +static int fts5ListMerge(sqlite4 *db, Fts5List *p1, Fts5List *p2, int bFree){ + InstanceList in1; + InstanceList in2; + InstanceList out; + + memset(&out, 0, sizeof(InstanceList)); + if( p1->nData==0 && p2->nData==0 ) return SQLITE4_OK; + out.nList = p1->nData+p2->nData; + out.aList = sqlite4DbMallocRaw(db, out.nList); + if( !out.aList ) return SQLITE4_NOMEM; + fts5InstanceListInit(p1->aData, p1->nData, &in1); + fts5InstanceListInit(p2->aData, p2->nData, &in2); + + fts5InstanceListNext(&in1); + fts5InstanceListNext(&in2); + + while( fts5InstanceListEof(&in1)==0 || fts5InstanceListEof(&in2)==0 ){ + InstanceList *pAdv; + + if( fts5InstanceListEof(&in1) ){ + pAdv = &in2; + }else if( fts5InstanceListEof(&in2) ){ + pAdv = &in1; + }else if( in1.iCol==in2.iCol && in1.iOff==in2.iOff ){ + pAdv = &in1; + fts5InstanceListNext(&in2); + }else if( in1.iColiCol, pAdv->iStream, pAdv->iOff); + fts5InstanceListNext(pAdv); + } + + if( bFree ){ + sqlite4DbFree(db, p1->aData); + sqlite4DbFree(db, p2->aData); + } + memset(p2, 0, sizeof(Fts5List)); + p1->aData = out.aList; + p1->nData = out.iList; + return SQLITE4_OK; +} + +static void fts5PrefixMerge(Fts5Prefix **pp, Fts5Prefix *p2){ + Fts5Prefix *p1 = *pp; + Fts5Prefix *pRet = 0; + Fts5Prefix **ppWrite = &pRet; + + while( p1 || p2 ){ + Fts5Prefix **ppAdv = 0; + if( p1==0 ){ + ppAdv = &p2; + }else if( p2==0 ){ + ppAdv = &p1; + }else{ + int res = fts5KeyCompare(p1->aPk, p1->nPk, p2->aPk, p2->nPk); + assert( res!=0 ); + if( res<0 ){ + ppAdv = &p1; + }else{ + ppAdv = &p2; + } + } + + *ppWrite = *ppAdv; + ppWrite = &((*ppWrite)->pNext); + *ppAdv = (*ppAdv)->pNext; + *ppWrite = 0; + } + + *pp = pRet; +} + +static int fts5FindPrefixes(sqlite4 *db, Fts5Info *pInfo, Fts5Token *pToken){ + int rc = SQLITE4_OK; + HashElem *pElem; + Hash hash; + + assert( pToken->bPrefix ); + assert( pToken->aPrefix[pToken->nPrefix-1]!='\0' ); + sqlite4HashInit(db->pEnv, &hash, 1); + + do { + const u8 *aData; + int nData; + const u8 *aPk; + int nPk; + + rc = fts5TokenPk(pToken, &aPk, &nPk); + if( rc==SQLITE4_OK ){ + rc = fts5TokenData(pToken, &aData, &nData); + } + if( rc==SQLITE4_OK ){ + Fts5Prefix *p; + + p = (Fts5Prefix *)sqlite4HashFind(&hash, (const char *)aPk, nPk); + if( !p ){ + p = (Fts5Prefix *)sqlite4DbMallocZero(db, sizeof(Fts5Prefix) + nPk); + if( !p ){ + rc = SQLITE4_NOMEM; + }else{ + void *pFree; + p->aPk = (u8 *)&p[1]; + p->nPk = nPk; + memcpy(p->aPk, aPk, nPk); + pFree = sqlite4HashInsert(&hash, (const char *)p->aPk, p->nPk, p); + if( pFree ){ + assert( pFree==(void *)p ); + rc = SQLITE4_NOMEM; + sqlite4DbFree(db, pFree); + } + } + } + + if( rc==SQLITE4_OK ){ + int nReq = nData + sqlite4VarintLen(nData); + while( (p->nList + nReq) > p->nAlloc ){ + int nAlloc = (p->nAlloc ? p->nAlloc*2 : 64); + p->aList = sqlite4DbReallocOrFree(db, p->aList, nAlloc); + if( !p->aList ){ + rc = SQLITE4_NOMEM; + break; + } + p->nAlloc = nAlloc; + } + } + + if( rc==SQLITE4_OK ){ + p->nList += putVarint32(&p->aList[p->nList], nData); + memcpy(&p->aList[p->nList], aData, nData); + p->nList += nData; + } + + if( rc==SQLITE4_OK ){ + rc = fts5TokenAdvance(db, pToken); + } + } + }while( rc==SQLITE4_OK ); + if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK; + + if( rc==SQLITE4_OK ){ + Fts5List *aMerge; + aMerge = (Fts5List *)sqlite4DbMallocZero(db, sizeof(Fts5List) * 32); + if( !aMerge ) rc = SQLITE4_NOMEM; + + for(pElem=sqliteHashFirst(&hash); pElem; pElem=sqliteHashNext(pElem)){ + Fts5Prefix *p = (Fts5Prefix *)sqliteHashData(pElem); + Fts5List list = {0, 0}; + int i = 0; + int iLevel; + + memset(aMerge, 0, sizeof(Fts5List)*32); + while( inList && rc==SQLITE4_OK ){ + u32 n; + i += getVarint32(&p->aList[i], n); + list.aData = &p->aList[i]; + list.nData = n; + i += n; + + for(iLevel=0; rc==SQLITE4_OK && iLevel<32; iLevel++){ + if( aMerge[iLevel].aData==0 ){ + aMerge[iLevel] = list; + break; + }else{ + rc = fts5ListMerge(db, &list, &aMerge[iLevel], (iLevel>0)); + } + } + assert( iLevel<32 ); + } + + list.aData = 0; + list.nData = 0; + for(iLevel=0; rc==SQLITE4_OK && iLevel<32; iLevel++){ + rc = fts5ListMerge(db, &list, &aMerge[iLevel], (iLevel>0)); + } + + if( rc==SQLITE4_OK ){ + sqlite4DbFree(db, p->aList); + p->aList = list.aData; + p->nAlloc = p->nList = list.nData; + }else{ + sqlite4DbFree(db, list.aData); + } + } + + sqlite4DbFree(db, aMerge); + } + + if( rc==SQLITE4_OK ){ + Fts5Prefix **aMerge; + Fts5Prefix *pPrefix = 0; + + aMerge = (Fts5Prefix **)sqlite4DbMallocZero(db, sizeof(Fts5List) * 32); + if( !aMerge ){ + rc = SQLITE4_NOMEM; + }else{ + int iLevel; + for(pElem=sqliteHashFirst(&hash); pElem; pElem=sqliteHashNext(pElem)){ + pPrefix = (Fts5Prefix *)sqliteHashData(pElem); + for(iLevel=0; iLevel<32; iLevel++){ + if( aMerge[iLevel] ){ + fts5PrefixMerge(&pPrefix, aMerge[iLevel]); + aMerge[iLevel] = 0; + }else{ + aMerge[iLevel] = pPrefix; + break; + } + } + assert( iLevel<32 ); + } + pPrefix = 0; + for(iLevel=0; iLevel<32; iLevel++){ + fts5PrefixMerge(&pPrefix, aMerge[iLevel]); + } + sqlite4HashClear(&hash); + sqlite4DbFree(db, aMerge); + } + pToken->pPrefix = pPrefix; + } + + for(pElem=sqliteHashFirst(&hash); pElem; pElem=sqliteHashNext(pElem)){ + Fts5Prefix *pPrefix = (Fts5Prefix *)sqliteHashData(pElem); + sqlite4DbFree(db, pPrefix->aList); + sqlite4DbFree(db, pPrefix); + } + sqlite4KVCursorClose(pToken->pCsr); + pToken->pCsr = 0; + + return rc; +} + +static int fts5OpenExprCursors(sqlite4 *db, Fts5Info *pInfo, Fts5ExprNode *p){ + int rc = SQLITE4_OK; + if( p ){ + if( p->eType==TOKEN_PRIMITIVE ){ + KVStore *pStore = db->aDb[pInfo->iDb].pKV; + Fts5Phrase *pPhrase = p->pPhrase; + int iStr; + + for(iStr=0; rc==SQLITE4_OK && iStrnStr; iStr++){ + Fts5Str *pStr = &pPhrase->aStr[iStr]; + int i; + for(i=0; rc==SQLITE4_OK && inToken; i++){ + Fts5Token *pToken = &pStr->aToken[i]; + rc = sqlite4KVStoreOpenCursor(pStore, &pToken->pCsr); + rc = sqlite4KVCursorSeek( + pToken->pCsr, pToken->aPrefix, pToken->nPrefix, 1 + ); + if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK; + if( rc==SQLITE4_OK && pToken->bPrefix ){ + rc = fts5FindPrefixes(db, pInfo, pToken); + } + } + } + } + if( rc==SQLITE4_OK ) rc = fts5OpenExprCursors(db, pInfo, p->pLeft); + if( rc==SQLITE4_OK ) rc = fts5OpenExprCursors(db, pInfo, p->pRight); + } + + return rc; +} + +/* +** Open a cursor for each token in the expression. +*/ +static int fts5OpenCursors(sqlite4 *db, Fts5Info *pInfo, Fts5Cursor *pCsr){ + return fts5OpenExprCursors(db, pInfo, pCsr->pExpr->pRoot); +} + +void sqlite4Fts5Close(Fts5Cursor *pCsr){ + if( pCsr ){ + sqlite4 *db = pCsr->db; + if( pCsr->aMem ){ + int i; + for(i=0; ipInfo->nCol; i++){ + sqlite4DbFree(db, pCsr->aMem[i].zMalloc); + } + sqlite4DbFree(db, pCsr->aMem); + } + + sqlite4KVCursorClose(pCsr->pCsr); + fts5ExpressionFree(db, pCsr->pExpr); + sqlite4DbFree(db, pCsr->pIter); + sqlite4DbFree(db, pCsr->aKey); + sqlite4DbFree(db, pCsr->anRow); + sqlite4DbFree(db, pCsr); + } +} + +static int fts5TokenAdvanceToMatch( + InstanceList *p, + InstanceList *pFirst, + int iOff, + int *pbEof +){ + int iReq = pFirst->iOff + iOff; + + while( p->iColiCol || (p->iCol==pFirst->iCol && p->iOff < iReq) ){ + int bEof = fts5InstanceListNext(p); + if( bEof ){ + *pbEof = 1; + return 0; + } + } + + return (p->iCol==pFirst->iCol && p->iOff==iReq); +} + +static int fts5StringFindInstances(sqlite4 *db, int iCol, Fts5Str *pStr){ + int i; + int rc = SQLITE4_OK; + int bEof = 0; + int nByte = sizeof(InstanceList) * pStr->nToken; + InstanceList *aIn; + InstanceList out; + + pStr->nList = 0; + memset(&out, 0, sizeof(InstanceList)); + + aIn = (InstanceList *)sqlite4DbMallocZero(db, nByte); + if( !aIn ) rc = SQLITE4_NOMEM; + for(i=0; rc==SQLITE4_OK && inToken; i++){ + const u8 *aData; + int nData; + rc = fts5TokenData(&pStr->aToken[i], &aData, &nData); + if( rc==SQLITE4_OK ){ + fts5InstanceListInit((u8 *)aData, nData, &aIn[i]); + fts5InstanceListNext(&aIn[i]); + } + } + + /* Allocate the output list */ + if( rc==SQLITE4_OK ){ + int nReq = aIn[0].nList; + if( nReq<=pStr->nListAlloc ){ + out.aList = pStr->aList; + out.nList = pStr->nListAlloc; + }else{ + pStr->aList = out.aList = sqlite4DbReallocOrFree(db, pStr->aList, nReq*2); + pStr->nListAlloc = out.nList = nReq*2; + if( out.aList==0 ) rc = SQLITE4_NOMEM; + } + } + + while( rc==SQLITE4_OK && bEof==0 ){ + for(i=1; inToken; i++){ + int bMatch = fts5TokenAdvanceToMatch(&aIn[i], &aIn[0], i, &bEof); + if( bMatch==0 || bEof ) break; + } + if( i==pStr->nToken && (iCol<0 || aIn[0].iCol==iCol) ){ + /* Record a match here */ + fts5InstanceListAppend(&out, aIn[0].iCol, aIn[0].iStream, aIn[0].iOff); + } + bEof = fts5InstanceListNext(&aIn[0]); + } + + pStr->nList = out.iList; + sqlite4DbFree(db, aIn); + + return rc; +} + +static int fts5IsNear(InstanceList *p1, InstanceList *p2, int nNear){ + if( p1->iCol==p2->iCol && p1->iOffiOff && (p1->iOff+nNear)>=p2->iOff ){ + return 1; + } + return 0; +} + +static int fts5StringNearTrim( + Fts5Str *pTrim, /* Trim this instance list */ + Fts5Str *pNext, /* According to this one */ + int nNear +){ + if( pNext->nList==0 ){ + pTrim->nList = 0; + }else{ + int bEof = 0; + int nTrail = nNear + (pNext->nToken-1) + 1; + int nLead = nNear + (pTrim->nToken-1) + 1; + + InstanceList lNear; + InstanceList in; + InstanceList out; + + fts5InstanceListInit(pNext->aList, pNext->nList, &lNear); + fts5InstanceListInit(pTrim->aList, pTrim->nList, &in); + fts5InstanceListInit(pTrim->aList, pTrim->nList, &out); + fts5InstanceListNext(&lNear); + fts5InstanceListNext(&in); + + while( bEof==0 ){ + if( fts5IsNear(&lNear, &in, nTrail) + || fts5IsNear(&in, &lNear, nLead) + ){ + /* The current position is a match. Append an entry to the output + ** and advance the input cursor. */ + fts5InstanceListAppend(&out, in.iCol, in.iStream, in.iOff); + bEof = fts5InstanceListNext(&in); + }else{ + if( lNear.iColnList = out.iList; + } + return SQLITE4_OK; +} + +/* +** This function tests if the cursors embedded in the Fts5Phrase object +** currently point to a match for the entire phrase. If so, *pbMatch +** is set to true before returning. +** +** If the cursors do not point to a match, then *ppAdvance is set to +** the token of the individual cursor that should be advanced before +** retrying this function. +*/ +static int fts5PhraseIsMatch( + sqlite4 *db, /* Database handle */ + Fts5Phrase *pPhrase, /* Phrase to test */ + int *pbMatch, /* OUT: True for a match, false otherwise */ + Fts5Token **ppAdvance /* OUT: Token to advance before retrying */ +){ + const u8 *aPk1 = 0; + int nPk1 = 0; + int rc = SQLITE4_OK; + int i; + + *pbMatch = 0; + *ppAdvance = &pPhrase->aStr[0].aToken[0]; + + rc = fts5TokenPk(*ppAdvance, &aPk1, &nPk1); + for(i=0; rc==SQLITE4_OK && inStr; i++){ + int j; + for(j=(i==0); jaStr[i].nToken; j++){ + const u8 *aPk = 0; + int nPk = 0; + Fts5Token *pToken = &pPhrase->aStr[i].aToken[j]; + rc = fts5TokenPk(pToken, &aPk, &nPk); + if( rc==SQLITE4_OK ){ + int res = fts5KeyCompare(aPk1, nPk1, aPk, nPk); + if( res<0 ){ + return SQLITE4_OK; + } + if( res>0 ){ + *ppAdvance = pToken; + return SQLITE4_OK; + } + } + } + } + + /* At this point, it is established that all of the token cursors in the + ** phrase point to an entry with the same primary key. Now figure out if + ** the various string constraints are met. Along the way, synthesize a + ** position list for each Fts5Str object. */ + for(i=0; rc==SQLITE4_OK && inStr; i++){ + Fts5Str *pStr = &pPhrase->aStr[i]; + rc = fts5StringFindInstances(db, pPhrase->iCol, pStr); + } + + /* Trim the instance lists according to any NEAR constraints. */ + for(i=1; rc==SQLITE4_OK && inStr; i++){ + int n = pPhrase->aiNear[i-1]; + rc = fts5StringNearTrim(&pPhrase->aStr[i], &pPhrase->aStr[i-1], n); + } + for(i=pPhrase->nStr-1; rc==SQLITE4_OK && i>0; i--){ + int n = pPhrase->aiNear[i-1]; + rc = fts5StringNearTrim(&pPhrase->aStr[i-1], &pPhrase->aStr[i], n); + } + + *pbMatch = (pPhrase->aStr[0].nList>0); + return rc; +} + +static int fts5PhraseAdvanceToMatch(sqlite4 *db, Fts5Phrase *pPhrase){ + int rc; + do { + int bMatch; + Fts5Token *pAdvance = 0; + rc = fts5PhraseIsMatch(db, pPhrase, &bMatch, &pAdvance); + if( rc!=SQLITE4_OK || bMatch ) break; + rc = fts5TokenAdvance(db, pAdvance); + }while( rc==SQLITE4_OK ); + return rc; +} + +static int fts5ExprAdvance(sqlite4 *db, Fts5ExprNode *p, int bFirst){ + int rc = SQLITE4_OK; + + switch( p->eType ){ + case TOKEN_PRIMITIVE: { + Fts5Phrase *pPhrase = p->pPhrase; + if( bFirst==0 ){ + rc = fts5TokenAdvance(db, &pPhrase->aStr[0].aToken[0]); + } + if( rc==SQLITE4_OK ) rc = fts5PhraseAdvanceToMatch(db, pPhrase); + if( rc==SQLITE4_OK ){ + rc = fts5TokenPk(&pPhrase->aStr[0].aToken[0], &p->aPk, &p->nPk); + }else{ + p->aPk = 0; + p->nPk = 0; + if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK; + } + break; + } + + case TOKEN_AND: + p->aPk = 0; + p->nPk = 0; + rc = fts5ExprAdvance(db, p->pLeft, bFirst); + if( rc==SQLITE4_OK ) rc = fts5ExprAdvance(db, p->pRight, bFirst); + while( rc==SQLITE4_OK && p->pLeft->aPk && p->pRight->aPk ){ + int res = fts5KeyCompare( + p->pLeft->aPk, p->pLeft->nPk, p->pRight->aPk, p->pRight->nPk + ); + if( res<0 ){ + rc = fts5ExprAdvance(db, p->pLeft, 0); + }else if( res>0 ){ + rc = fts5ExprAdvance(db, p->pRight, 0); + }else{ + p->aPk = p->pLeft->aPk; + p->nPk = p->pLeft->nPk; + break; + } + } + break; + + case TOKEN_OR: { + int res = 0; + if( bFirst==0 ){ + res = fts5KeyCompare( + p->pLeft->aPk, p->pLeft->nPk, p->pRight->aPk, p->pRight->nPk + ); + } + + if( res<=0 ) rc = fts5ExprAdvance(db, p->pLeft, bFirst); + if( rc==SQLITE4_OK && res>=0 ){ + rc = fts5ExprAdvance(db, p->pRight, bFirst); + } + + res = fts5KeyCompare( + p->pLeft->aPk, p->pLeft->nPk, p->pRight->aPk, p->pRight->nPk + ); + if( res>0 ){ + p->aPk = p->pRight->aPk; + p->nPk = p->pRight->nPk; + }else{ + p->aPk = p->pLeft->aPk; + p->nPk = p->pLeft->nPk; + } + assert( p->aPk!=0 || (p->pLeft->aPk==0 && p->pRight->aPk==0) ); + break; + } + + + default: assert( p->eType==TOKEN_NOT ); + + p->aPk = 0; + p->nPk = 0; + + rc = fts5ExprAdvance(db, p->pLeft, bFirst); + if( bFirst && rc==SQLITE4_OK ){ + rc = fts5ExprAdvance(db, p->pRight, bFirst); + } + + while( rc==SQLITE4_OK && p->pLeft->aPk && p->pRight->aPk ){ + int res = fts5KeyCompare( + p->pLeft->aPk, p->pLeft->nPk, p->pRight->aPk, p->pRight->nPk + ); + if( res<0 ){ + break; + }else if( res>0 ){ + rc = fts5ExprAdvance(db, p->pRight, 0); + }else{ + rc = fts5ExprAdvance(db, p->pLeft, 0); + } + } + + p->aPk = p->pLeft->aPk; + p->nPk = p->pLeft->nPk; + break; + } + + assert( rc!=SQLITE4_NOTFOUND ); + return rc; +} + +int sqlite4Fts5Next(Fts5Cursor *pCsr){ + sqlite4DbFree(pCsr->db, pCsr->pSz); + pCsr->pSz = 0; + pCsr->bMemValid = 0; + return fts5ExprAdvance(pCsr->db, pCsr->pExpr->pRoot, 0); +} + +int sqlite4Fts5Open( + sqlite4 *db, /* Database handle */ + Fts5Info *pInfo, /* Index description */ + const char *zMatch, /* Match expression */ + int bDesc, /* True to iterate in desc. order of PK */ + Fts5Cursor **ppCsr, /* OUT: New FTS cursor object */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE4_OK; + Fts5Cursor *pCsr; + int nMatch = sqlite4Strlen30(zMatch); + + pCsr = sqlite4DbMallocZero(db, sizeof(Fts5Cursor) + nMatch + 1); + + if( !pCsr ){ + rc = SQLITE4_NOMEM; + }else{ + pCsr->zExpr = (char *)&pCsr[1]; + memcpy(pCsr->zExpr, zMatch, nMatch); + pCsr->pInfo = pInfo; + pCsr->db = db; + rc = fts5ParseExpression(db, pInfo->pTokenizer, pInfo->p, + pInfo->iRoot, pInfo->azCol, pInfo->nCol, zMatch, &pCsr->pExpr, pzErr + ); + } + + if( rc==SQLITE4_OK ){ + /* Open a KV cursor for each term in the expression. Set each cursor + ** to point to the first entry in the range it will scan. */ + rc = fts5OpenCursors(db, pInfo, pCsr); + } + if( rc!=SQLITE4_OK ){ + sqlite4Fts5Close(pCsr); + pCsr = 0; + }else{ + rc = fts5ExprAdvance(db, pCsr->pExpr->pRoot, 1); + } + *ppCsr = pCsr; + return rc; +} + +/* +** Return true if the cursor passed as the second argument currently points +** to a valid entry, or false otherwise. +*/ +int sqlite4Fts5Valid(Fts5Cursor *pCsr){ + return( pCsr->pExpr->pRoot->aPk!=0 ); +} + +int sqlite4Fts5Pk( + Fts5Cursor *pCsr, + int iTbl, + KVByteArray **paKey, + KVSize *pnKey +){ + int i; + int nReq; + const u8 *aPk; + int nPk; + + aPk = pCsr->pExpr->pRoot->aPk; + nPk = pCsr->pExpr->pRoot->nPk; + + nReq = sqlite4VarintLen(iTbl) + nPk; + if( nReq>pCsr->nKeyAlloc ){ + pCsr->aKey = sqlite4DbReallocOrFree(pCsr->db, pCsr->aKey, nReq*2); + if( !pCsr->aKey ) return SQLITE4_NOMEM; + pCsr->nKeyAlloc = nReq*2; + } + + i = putVarint32(pCsr->aKey, iTbl); + memcpy(&pCsr->aKey[i], aPk, nPk); + + *paKey = pCsr->aKey; + *pnKey = nReq; + return SQLITE4_OK; +} + +int sqlite4_mi_column_count(sqlite4_context *pCtx, int *pn){ + int rc = SQLITE4_OK; + if( pCtx->pFts ){ + *pn = pCtx->pFts->pInfo->nCol; + }else{ + rc = SQLITE4_MISUSE; + } + return rc; +} + +int sqlite4_mi_phrase_count(sqlite4_context *pCtx, int *pn){ + int rc = SQLITE4_OK; + if( pCtx->pFts ){ + *pn = pCtx->pFts->pExpr->nPhrase; + }else{ + rc = SQLITE4_MISUSE; + } + return rc; +} + +int sqlite4_mi_phrase_token_count(sqlite4_context *pCtx, int iP, int *pn){ + int rc = SQLITE4_OK; + if( pCtx->pFts ){ + Fts5Expr *pExpr = pCtx->pFts->pExpr; + if( iP>pExpr->nPhrase || iP<0 ){ + *pn = 0; + }else{ + *pn = pExpr->apPhrase[iP]->nToken; + } + }else{ + rc = SQLITE4_MISUSE; + } + return rc; +} + +int sqlite4_mi_stream_count(sqlite4_context *pCtx, int *pn){ + int rc = SQLITE4_OK; + Fts5Cursor *pCsr = pCtx->pFts; + if( pCsr ){ + rc = fts5CsrLoadGlobal(pCtx->pFts); + if( rc==SQLITE4_OK ) *pn = pCsr->pGlobal->nStream; + }else{ + rc = SQLITE4_MISUSE; + } + return rc; +} + +static int fts5GetSize(Fts5Size *pSz, int iC, int iS){ + int nToken = 0; + int i; + + if( iC<0 && iS<0 ){ + int nFin = pSz->nCol * pSz->nStream; + for(i=0; iaSz[i]; + }else if( iC<0 ){ + for(i=0; inCol; i++) nToken += pSz->aSz[i*pSz->nStream + iS]; + }else if( iS<0 ){ + for(i=0; inStream; i++) nToken += pSz->aSz[pSz->nStream*iC + i]; + }else if( iCnCol && iSnStream ){ + nToken = pSz->aSz[iC * pSz->nStream + iS]; + } + + return nToken; +} + +int sqlite4_mi_size(sqlite4_context *pCtx, int iC, int iS, int *pn){ + int rc = SQLITE4_OK; + Fts5Cursor *pCsr = pCtx->pFts; + + if( pCsr==0 ){ + rc = SQLITE4_MISUSE; + }else{ + rc = fts5CsrLoadSz(pCsr); + if( rc==SQLITE4_OK ){ + *pn = fts5GetSize(pCsr->pSz, iC, iS); + } + } + return rc; +} + +int sqlite4_mi_total_size(sqlite4_context *pCtx, int iC, int iS, int *pn){ + int rc = SQLITE4_OK; + Fts5Cursor *pCsr = pCtx->pFts; + + if( pCsr==0 ){ + rc = SQLITE4_MISUSE; + }else{ + rc = fts5CsrLoadGlobal(pCsr); + if( rc==SQLITE4_OK ){ + *pn = fts5GetSize(pCsr->pGlobal, iC, iS); + } + } + return rc; +} + +int sqlite4_mi_total_rows(sqlite4_context *pCtx, int *pn){ + int rc = SQLITE4_OK; + Fts5Cursor *pCsr = pCtx->pFts; + if( pCsr==0 ){ + rc = SQLITE4_MISUSE; + }else{ + rc = fts5CsrLoadGlobal(pCsr); + if( rc==SQLITE4_OK ) *pn = pCsr->nGlobal; + } + return rc; +} + +int sqlite4_mi_column_value( + sqlite4_context *pCtx, + int iCol, + sqlite4_value **ppVal +){ + int rc = SQLITE4_OK; + Fts5Cursor *pCsr = pCtx->pFts; + if( pCsr==0 ){ + rc = SQLITE4_MISUSE; + }else{ + if( pCsr->bMemValid==0 ){ + sqlite4 *db = pCsr->db; + + Fts5Info *pInfo = pCsr->pInfo; + if( pCsr->aMem==0 ){ + int nByte = sizeof(Mem) * pInfo->nCol; + pCsr->aMem = (Mem *)sqlite4DbMallocZero(db, nByte); + if( pCsr->aMem==0 ){ + rc = SQLITE4_NOMEM; + }else{ + int i; + for(i=0; inCol; i++){ + pCsr->aMem[i].db = db; + } + } + } + + if( pCsr->pCsr==0 && rc==SQLITE4_OK ){ + KVStore *pStore = db->aDb[pInfo->iDb].pKV; + rc = sqlite4KVStoreOpenCursor(pStore, &pCsr->pCsr); + } + + if( rc==SQLITE4_OK ){ + u8 *aKey = 0; int nKey; /* Primary key for current row */ + const u8 *aData; int nData; /* Data record for current row */ + + rc = sqlite4Fts5Pk(pCsr, pInfo->iTbl, &aKey, &nKey); + if( rc==SQLITE4_OK ){ + rc = sqlite4KVCursorSeek(pCsr->pCsr, aKey, nKey, 0); + if( rc==SQLITE4_NOTFOUND ){ + rc = SQLITE4_CORRUPT_BKPT; + } + } + + if( rc==SQLITE4_OK ){ + rc = sqlite4KVCursorData(pCsr->pCsr, 0, -1, &aData, &nData); + } + + if( rc==SQLITE4_OK ){ + int i; + ValueDecoder *pCodec; /* The decoder object */ + + rc = sqlite4VdbeCreateDecoder(db, aData, nData, pInfo->nCol, &pCodec); + for(i=0; rc==SQLITE4_OK && inCol; i++){ + rc = sqlite4VdbeDecodeValue(pCodec, i, 0, &pCsr->aMem[i]); + } + sqlite4VdbeDestroyDecoder(pCodec); + } + + if( rc==SQLITE4_OK ) pCsr->bMemValid = 1; + } + } + + if( rc==SQLITE4_OK ){ + assert( pCsr->bMemValid ); + *ppVal = &pCsr->aMem[iCol]; + } + } + + return rc; +} + +int sqlite4_mi_tokenize( + sqlite4_context *pCtx, + const char *zText, + int nText, + void *p, + int(*x)(void *, int, int, const char *, int, int, int) +){ + int rc = SQLITE4_OK; + Fts5Cursor *pCsr = pCtx->pFts; + + if( pCsr==0 ){ + rc = SQLITE4_MISUSE; + }else{ + Fts5Info *pInfo = pCsr->pInfo; + rc = pInfo->pTokenizer->xTokenize(p, pInfo->p, zText, nText, x); + } + return rc; +} + +static Fts5Str *fts5FindStr( + const u8 *aPk, int nPk, + Fts5ExprNode *p, + int *piStr +){ + Fts5Str *pRet = 0; + if( p->eType==TOKEN_PRIMITIVE ){ + int iStr = *piStr; + if( iStrpPhrase->nStr && iStr>=0 + && p->nPk==nPk && 0==memcmp(p->aPk, aPk, nPk) + ){ + pRet = &p->pPhrase->aStr[iStr]; + }else{ + *piStr = iStr - p->pPhrase->nStr; + } + }else{ + pRet = fts5FindStr(aPk, nPk, p->pLeft, piStr); + if( pRet==0 ) pRet = fts5FindStr(aPk, nPk, p->pRight, piStr); + } + return pRet; +} + +int sqlite4_mi_match_count( + sqlite4_context *pCtx, + int iC, + int iS, + int iPhrase, + int *pnMatch +){ + int rc = SQLITE4_OK; + Fts5Cursor *pCsr = pCtx->pFts; + if( pCsr ){ + Fts5ExprNode *pRoot = pCsr->pExpr->pRoot; + int nMatch = 0; + Fts5Str *pStr; + int iCopy = iPhrase; + InstanceList sList; + + pStr = fts5FindStr(pRoot->aPk, pRoot->nPk, pRoot, &iCopy); + if( pStr ){ + fts5InstanceListInit(pStr->aList, pStr->nList, &sList); + while( 0==fts5InstanceListNext(&sList) ){ + if( (iC<0 || sList.iCol==iC) && (iS<0 || sList.iStream==iS) ) nMatch++; + } + } + *pnMatch = nMatch; + }else{ + rc = SQLITE4_MISUSE; + } + return rc; +} + +int sqlite4_mi_match_offset( + sqlite4_context *pCtx, + int iCol, + int iPhrase, + int iMatch, + int *piOff +){ + return SQLITE4_OK; +} + +int sqlite4_mi_total_match_count( + sqlite4_context *pCtx, + int iCol, + int iPhrase, + int *pnMatch, + int *pnDoc, + int *pnRelevant +){ + return SQLITE4_OK; +} + +static void fts5StrLoadRowcounts( + Fts5Str *pStr, + int nStream, + int *anRow, + int *anRowC, + int *anRowS, + int *pnRowCS +){ + u32 mask = 0; + int iPrevCol = -1; + InstanceList sList; + + fts5InstanceListInit(pStr->aList, pStr->nList, &sList); + while( 0==fts5InstanceListNext(&sList) ){ + if( iPrevCol<0 ) (*pnRowCS)++; + if( sList.iCol!=iPrevCol ){ + mask = 0; + anRowC[sList.iCol]++; + } + if( (mask & (1<pInfo; + + if( pNode->eType==TOKEN_PRIMITIVE ){ + Fts5Phrase *pPhrase = pNode->pPhrase; + int iStr = *piStr; + + rc = fts5ExprAdvance(db, pNode, 1); + while( rc==SQLITE4_OK && pNode->aPk ){ + int i; + for(i=0; inStr; i++){ + int *anRow = &pCsr->anRow[(iStr+i) * pInfo->nCol * nStream]; + int *anRowC = &pCsr->anRowC[(iStr+i) * pInfo->nCol]; + int *anRowS = &pCsr->anRowS[(iStr+i) * nStream]; + int *pnRowCS = &pCsr->anRowCS[iStr+i]; + fts5StrLoadRowcounts( + &pPhrase->aStr[i], nStream, anRow, anRowC, anRowS, pnRowCS + ); + } + rc = fts5ExprAdvance(db, pNode, 0); + } + + *piStr = iStr + pPhrase->nStr; + } + + if( rc==SQLITE4_OK ){ + rc = fts5ExprLoadRowcounts(db, pCsr, nStream, pNode->pLeft, piStr); + } + if( rc==SQLITE4_OK ){ + rc = fts5ExprLoadRowcounts(db, pCsr, nStream, pNode->pRight, piStr); + } + } + + return rc; +} + +static int fts5CsrLoadRowcounts(Fts5Cursor *pCsr){ + int rc = SQLITE4_OK; + + if( pCsr->anRow==0 ){ + int nStream = pCsr->pGlobal->nStream; + sqlite4 *db = pCsr->db; + Fts5Expr *pCopy; + Fts5Expr *pExpr = pCsr->pExpr; + Fts5Info *pInfo = pCsr->pInfo; + int *anRow; + int iPhrase = 0; + + pCsr->anRow = anRow = (int *)sqlite4DbMallocZero(db, sizeof(int) * ( + pExpr->nPhrase * pInfo->nCol * pCsr->pGlobal->nStream + + pExpr->nPhrase * pInfo->nCol + + pExpr->nPhrase * pCsr->pGlobal->nStream + + pExpr->nPhrase + )); + if( !anRow ) return SQLITE4_NOMEM; + pCsr->anRowC = &anRow[pExpr->nPhrase*pInfo->nCol*pCsr->pGlobal->nStream]; + pCsr->anRowS = &pCsr->anRowC[pExpr->nPhrase * pInfo->nCol]; + pCsr->anRowCS = &pCsr->anRowS[pExpr->nPhrase * pCsr->pGlobal->nStream]; + + rc = fts5ParseExpression(db, pInfo->pTokenizer, pInfo->p, + pInfo->iRoot, pInfo->azCol, pInfo->nCol, pCsr->zExpr, &pCopy, 0 + ); + if( rc==SQLITE4_OK ){ + rc = fts5OpenExprCursors(db, pInfo, pCopy->pRoot); + } + if( rc==SQLITE4_OK ){ + rc = fts5ExprLoadRowcounts(db, pCsr, nStream, pCopy->pRoot, &iPhrase); + } + + fts5ExpressionFree(db, pCopy); + } + + return rc; +} + +int sqlite4_mi_row_count( + sqlite4_context *pCtx, /* Context object passed to mi function */ + int iC, /* Specific column (or -ve for all columns) */ + int iS, /* Specific stream (or -ve for all streams) */ + int iP, /* Specific phrase */ + int *pn /* Total number of rows containing C/S/P */ +){ + int rc = SQLITE4_OK; + Fts5Cursor *pCsr = pCtx->pFts; + if( pCsr==0 ){ + rc = SQLITE4_MISUSE; + }else{ + rc = fts5CsrLoadGlobal(pCsr); + if( rc==SQLITE4_OK ) rc = fts5CsrLoadRowcounts(pCsr); + + if( rc==SQLITE4_OK ){ + int i; + int nRow = 0; + int nStream = pCsr->pGlobal->nStream; + int nCol = pCsr->pInfo->nCol; + int *aRow = &pCsr->anRow[iP * nStream * nCol]; + + if( iC<0 && iS<0 ){ + nRow = pCsr->anRowCS[iP]; + }else if( iC<0 ){ + for(i=0; ianRowC[iP*nCol + iC]; + }else if( iCaList[i]; + if( fts5InstanceListEof(p)==0 ){ + if( (pBest==0) + || (p->iColiCol) + || (p->iCol==pBest->iCol && p->iOffiOff) + ){ + pBest = p; + } + } + } + + if( pBest==0 ){ + pIter->iCurrent = -1; + }else{ + pIter->iCurrent = pBest - pIter->aList; + } +} + +static void fts5InitExprIterator( + const u8 *aPk, + int nPk, + Fts5ExprNode *p, + Fts5MatchIter *pIter +){ + if( p ){ + if( p->eType==TOKEN_PRIMITIVE ){ + if( p->nPk==nPk && 0==memcmp(aPk, p->aPk, nPk) ){ + int i; + for(i=0; ipPhrase->nStr; i++){ + Fts5Str *pStr = &p->pPhrase->aStr[i]; + InstanceList *pList = &pIter->aList[pIter->iCurrent++]; + fts5InstanceListInit(pStr->aList, pStr->nList, pList); + fts5InstanceListNext(pList); + } + }else{ + memset(&pIter->aList[pIter->iCurrent], 0, sizeof(InstanceList)); + pIter->iCurrent += p->pPhrase->nStr; + } + } + fts5InitExprIterator(aPk, nPk, p->pLeft, pIter); + fts5InitExprIterator(aPk, nPk, p->pRight, pIter); + } +} + +static void fts5InitIterator(Fts5Cursor *pCsr){ + Fts5MatchIter *pIter = pCsr->pIter; + Fts5ExprNode *pRoot = pCsr->pExpr->pRoot; + + pIter->iCurrent = 0; + fts5InitExprIterator(pRoot->aPk, pRoot->nPk, pRoot, pIter); + pIter->iMatch = 0; + pIter->bValid = 1; + fts5IterSetCurrent(pIter, pCsr->pExpr->nPhrase); +} + +int sqlite4_mi_match_detail( + sqlite4_context *pCtx, /* Context object passed to mi function */ + int iMatch, /* Index of match */ + int *piOff, /* OUT: Token offset of match */ + int *piC, /* OUT: Column number of match iMatch */ + int *piS, /* OUT: Stream number of match iMatch */ + int *piP /* OUT: Phrase number of match iMatch */ +){ + int rc = SQLITE4_OK; + Fts5Cursor *pCsr = pCtx->pFts; + if( pCsr==0 ){ + rc = SQLITE4_MISUSE; + }else{ + int nPhrase = pCsr->pExpr->nPhrase; + Fts5MatchIter *pIter = pCsr->pIter; + if( pIter==0 ){ + pCsr->pIter = pIter = (Fts5MatchIter *)sqlite4DbMallocZero( + pCsr->db, sizeof(Fts5MatchIter) + sizeof(InstanceList)*nPhrase + ); + if( pIter ){ + pIter->aList = (InstanceList *)&pIter[1]; + }else{ + rc = SQLITE4_NOMEM; + } + } + + if( rc==SQLITE4_OK && (pIter->bValid==0 || iMatchiMatch) ){ + fts5InitIterator(pCsr); +#if 0 + int i; + for(i=0; ipExpr->nPhrase; i++){ + Fts5Str *pStr = pCsr->pExpr->apPhrase[i]; + fts5InstanceListInit(pStr->aList, pStr->nList, &pIter->aList[i]); + fts5InstanceListNext(&pIter->aList[i]); + } + pIter->iMatch = 0; + fts5IterSetCurrent(pIter, pCsr->pExpr->nPhrase); +#endif + } + + if( rc==SQLITE4_OK ){ + assert( pIter->iMatch<=iMatch ); + while( pIter->iCurrent>=0 && pIter->iMatchaList[pIter->iCurrent]); + fts5IterSetCurrent(pIter, pCsr->pExpr->nPhrase); + pIter->iMatch++; + } + if( pIter->iCurrent<0 ){ + rc = SQLITE4_NOTFOUND; + }else{ + InstanceList *p = &pIter->aList[pIter->iCurrent]; + *piOff = p->iOff; + *piC = p->iCol; + *piS = p->iStream; + *piP = pIter->iCurrent; + } + } + } + return rc; +} + +/************************************************************************** +*************************************************************************** +** Below this point is test code. +*/ +#ifdef SQLITE4_TEST +static int fts5PrintExprNode(sqlite4 *, const char **, Fts5ExprNode *, char **); +static int fts5PrintExprNodeParen( + sqlite4 *db, const char **azCol, + Fts5ExprNode *pNode, + char **pzRet +){ + int bParen = (pNode->eType!=TOKEN_PRIMITIVE || pNode->pPhrase->nStr>1); + sqlite4_env *pEnv = sqlite4_db_env(db); + char *zRet = *pzRet; + + if( bParen ) zRet = sqlite4_mprintf(pEnv, "%z(", zRet); + fts5PrintExprNode(db, azCol, pNode, &zRet); + if( bParen ) zRet = sqlite4_mprintf(pEnv, "%z)", zRet); + + *pzRet = zRet; + return SQLITE4_OK; +} +static int fts5PrintExprNode( + sqlite4 *db, + const char **azCol, + Fts5ExprNode *pNode, + char **pzRet +){ + sqlite4_env *pEnv = sqlite4_db_env(db); + char *zRet = *pzRet; + + assert( + pNode->eType==TOKEN_AND || pNode->eType==TOKEN_OR + || pNode->eType==TOKEN_NOT || pNode->eType==TOKEN_PRIMITIVE + ); + assert( (pNode->eType==TOKEN_PRIMITIVE)==(pNode->pPhrase!=0) ); + + if( pNode->eType==TOKEN_PRIMITIVE ){ + int iStr; + Fts5Phrase *pPhrase = pNode->pPhrase; + if( pPhrase->iCol>=0 ){ + zRet = sqlite4_mprintf(pEnv, "%z\"%s\":", zRet, azCol[pPhrase->iCol]); + } + for(iStr=0; iStrnStr; iStr++){ + int iToken; + Fts5Str *pStr = &pPhrase->aStr[iStr]; + if( iStr>0 ){ + zRet = sqlite4_mprintf( + pEnv, "%z NEAR/%d ", zRet, pPhrase->aiNear[iStr-1] + ); + } + for(iToken=0; iTokennToken; iToken++){ + int nRet = sqlite4Strlen30(zRet); + const char *z = pStr->aToken[iToken].z; + int n = pStr->aToken[iToken].n; + int i; + + zRet = (char *)sqlite4_realloc(pEnv, zRet, nRet + n*2+4); + if( iToken>0 ) zRet[nRet++] = '+'; + zRet[nRet++] = '"'; + + for(i=0; iaToken[iToken].bPrefix ){ + zRet[nRet++] = '*'; + } + zRet[nRet++] = '\0'; + } + } + }else{ + fts5PrintExprNodeParen(db, azCol, pNode->pLeft, &zRet); + switch( pNode->eType ){ + case TOKEN_AND: + zRet = sqlite4_mprintf(pEnv, "%z AND ", zRet); + break; + case TOKEN_OR: + zRet = sqlite4_mprintf(pEnv, "%z OR ", zRet); + break; + case TOKEN_NOT: + zRet = sqlite4_mprintf(pEnv, "%z NOT ", zRet); + break; + } + fts5PrintExprNodeParen(db, azCol, pNode->pRight, &zRet); + } + + *pzRet = zRet; + return SQLITE4_OK; +} +static int fts5PrintExpr( + sqlite4 *db, + const char **azCol, + Fts5Expr *pExpr, + char **pzRet +){ + return fts5PrintExprNode(db, azCol, pExpr->pRoot, pzRet); +} + +/* +** A user defined function used to test the fts5 expression parser. As follows: +** +** fts5_parse_expr(, ); +*/ +static void fts5_parse_expr( + sqlite4_context *pCtx, + int nVal, + sqlite4_value **aVal +){ + int rc; + Fts5Expr *pExpr = 0; + Fts5Tokenizer *pTok; + sqlite4_tokenizer *p = 0; + sqlite4 *db; + + const char *zTokenizer; + const char *zExpr; + const char *zTbl; + char *zErr = 0; + char *zRet = 0; + const char **azCol = 0; + int nCol = 0; + sqlite4_stmt *pStmt = 0; + + db = sqlite4_context_db_handle(pCtx); + assert( nVal==3 ); + zTokenizer = (const char *)sqlite4_value_text(aVal[0]); + zExpr = (const char *)sqlite4_value_text(aVal[1]); + zTbl = (const char *)sqlite4_value_text(aVal[2]); + + if( sqlite4Strlen30(zTbl)>0 ){ + int i; + char *zSql = sqlite4MPrintf(db, "SELECT * FROM '%q'", zTbl); + rc = sqlite4_prepare(db, zSql, -1, &pStmt, 0); + sqlite4DbFree(db, zSql); + if( rc!=SQLITE4_OK ){ + sqlite4_result_error(pCtx, sqlite4_errmsg(db), -1); + sqlite4_result_error_code(pCtx, rc); + return; + } + nCol = sqlite4_column_count(pStmt); + azCol = sqlite4DbMallocZero(db, sizeof(char *)*nCol); + for(i=0; ixCreate(pTok->pCtx, 0, 0, &p); + if( rc!=SQLITE4_OK ){ + zErr = sqlite4MPrintf(db, "error creating tokenizer: %d", rc); + goto fts5_parse_expr_out; + } + } + + rc = fts5ParseExpression( + db, pTok, p, 0, (char **)azCol, nCol, zExpr, &pExpr, &zErr); + if( rc!=SQLITE4_OK ){ + if( zErr==0 ){ + zErr = sqlite4MPrintf(db, "error parsing expression: %d", rc); + } + goto fts5_parse_expr_out; + } + + fts5PrintExpr(db, azCol, pExpr, &zRet); + sqlite4_result_text(pCtx, zRet, -1, SQLITE4_TRANSIENT, 0); + fts5ExpressionFree(db, pExpr); + sqlite4_free(sqlite4_db_env(db), zRet); + + fts5_parse_expr_out: + if( p ) pTok->xDestroy(p); + sqlite4DbFree(db, azCol); + sqlite4_finalize(pStmt); + if( zErr ){ + sqlite4_result_error(pCtx, zErr, -1); + sqlite4DbFree(db, zErr); + } +} +#endif + +/* +** Register the default FTS5 tokenizer and functions with handle db. +*/ +int sqlite4InitFts5(sqlite4 *db){ +#ifdef SQLITE4_TEST + int rc = sqlite4_create_function( + db, "fts5_parse_expr", 3, SQLITE4_UTF8, 0, fts5_parse_expr, 0, 0 + ); + if( rc!=SQLITE4_OK ) return rc; +#endif + return sqlite4InitFts5Func(db); +} ADDED src/fts5func.c Index: src/fts5func.c ================================================================== --- /dev/null +++ src/fts5func.c @@ -0,0 +1,672 @@ +/* +** 2012 December 17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +/* +** The BM25 and BM25F implementations in this file are based on information +** found in: +** +** Stephen Robertson and Hugo Zaragoza: "The Probablistic Relevance +** Framework: BM25 and Beyond", 2009. +*/ + +#include "sqliteInt.h" +#include /* temporary: For log() */ + +static char fts5Tolower(char c){ + if( c>='A' && c<='Z' ) c = c + ('a' - 'A'); + return c; +} + +static int fts5SimpleCreate( + void *pCtx, + const char **azArg, + int nArg, + sqlite4_tokenizer **pp +){ + *pp = (sqlite4_tokenizer *)pCtx; + return SQLITE4_OK; +} + +static int fts5SimpleDestroy(sqlite4_tokenizer *p){ + return SQLITE4_OK; +} + +typedef struct Fts5RankCtx Fts5RankCtx; +struct Fts5RankCtx { + sqlite4 *db; + double *aAvgdl; /* Average document size of each field */ + int nPhrase; /* Number of phrases in query */ + double *aIdf; /* IDF weights for each phrase in query */ +}; + +static void fts5RankFreeCtx(void *pNotUsed, void *pCtx){ + if( pCtx ){ + Fts5RankCtx *p = (Fts5RankCtx *)pCtx; + sqlite4DbFree(p->db, p); + } +} + +#define BM25_EXPLAIN 0x01 +#define BM25_FCOLUMNS 0x02 +#define BM25_FSTREAMS 0x04 + +static int fts5GetSizeFreqScale( + sqlite4_context *pCtx, + int nArg, sqlite4_value **apArg,/* Function arguments */ + int bm25mask, /* bm25 configuration mask */ + int iPhrase, /* Phrase number */ + int iField, /* Field number */ + int *pnSize, /* OUT: Size of field in tokens */ + int *pnFreq, /* OUT: Occurences of phrase in field */ + double *pdScale /* OUT: Scale to use with this field */ +){ + int rc; + double scale = 1.0; + int nSize = 0; + int nFreq = 0; + + if( bm25mask & BM25_FCOLUMNS ){ + rc = sqlite4_mi_match_count(pCtx, iField, -1, iPhrase, &nFreq); + if( rc==SQLITE4_OK ) rc = sqlite4_mi_size(pCtx, iField, -1, &nSize); + if( nArg>iField ) scale = sqlite4_value_double(apArg[iField]); + }else if( bm25mask & BM25_FSTREAMS ){ + rc = sqlite4_mi_match_count(pCtx, -1, iField, iPhrase, &nFreq); + if( rc==SQLITE4_OK ) rc = sqlite4_mi_size(pCtx, -1, iField, &nSize); + if( nArg>iField ) scale = sqlite4_value_double(apArg[iField]); + }else{ + rc = sqlite4_mi_match_count(pCtx, -1, -1, iPhrase, &nFreq); + if( rc==SQLITE4_OK ) rc = sqlite4_mi_size(pCtx, -1, -1, &nSize); + } + + *pnSize = nSize; + *pnFreq = nFreq; + *pdScale = scale; + return rc; +} + +/* +** A BM25(F) based ranking function for fts5. +** +** This is based on the information in the Robertson/Zaragoza paper +** referenced above. As there is no way to provide relevance feedback +** IDF weights (equation 3.3 in R/Z) are used instead of RSJ for each phrase. +** The rest of the implementation is as presented in equations 3.19-21. +** +** R and Z observe that the experimental evidence suggests that reasonable +** values for free parameters "b" and "k1" are often in the ranges +** (0.5 < b < 0.8) and (1.2 < k1 < 2), although the optimal values depend +** on the nature of both the documents and queries. The implementation +** below sets each parameter to the midpoint of the suggested range. +*/ +static void fts5Rank(sqlite4_context *pCtx, int nArg, sqlite4_value **apArg){ + const double b = 0.65; + const double k1 = 1.6; + + sqlite4 *db = sqlite4_context_db_handle(pCtx); + int rc = SQLITE4_OK; /* Error code */ + Fts5RankCtx *p; /* Structure to store reusable values */ + int i; /* Used to iterate through phrases */ + double rank = 0.0; /* UDF return value */ + + int bExplain; /* True to run in explain mode */ + char *zExplain = 0; /* String to return in explain mode */ + int nField = 1; /* Number of fields in collection */ + + int bm25mask = SQLITE4_PTR_TO_INT(sqlite4_user_data(pCtx)); + bExplain = (bm25mask & BM25_EXPLAIN); + + if( bm25mask & BM25_FCOLUMNS ) sqlite4_mi_column_count(pCtx, &nField); + if( bm25mask & BM25_FSTREAMS ) sqlite4_mi_stream_count(pCtx, &nField); + + p = sqlite4_get_auxdata(pCtx, 0); + if( p==0 ){ + int nPhrase; /* Number of phrases in query expression */ + int nByte; /* Number of bytes of data to allocate */ + + sqlite4_mi_phrase_count(pCtx, &nPhrase); + nByte = sizeof(Fts5RankCtx) + (nPhrase+nField) * sizeof(double); + p = (Fts5RankCtx *)sqlite4DbMallocZero(db, nByte); + sqlite4_set_auxdata(pCtx, 0, (void *)p, fts5RankFreeCtx, 0); + p = sqlite4_get_auxdata(pCtx, 0); + + if( !p ){ + rc = SQLITE4_NOMEM; + }else{ + int N; /* Total number of docs in collection */ + int ni; /* Number of docs with phrase i */ + + p->db = db; + p->nPhrase = nPhrase; + p->aIdf = (double *)&p[1]; + p->aAvgdl = &p->aIdf[nPhrase]; + + /* Determine the IDF weight for each phrase in the query. */ + rc = sqlite4_mi_total_rows(pCtx, &N); + for(i=0; rc==SQLITE4_OK && iaIdf[i] = log((0.5 + N - ni) / (0.5 + ni)); + } + } + + /* Determine the average document length. For bm25f, determine the + ** average length of each field. */ + if( rc==SQLITE4_OK ){ + int iField; + for(iField=0; iFieldaAvgdl[iField] = (double)nTotal / (double)N; + } + } + } + } + } + + if( bExplain ){ + int iField; + zExplain = sqlite4MAppendf( + db, zExplain, "%s
StreamScaleavgslsl", + zExplain + ); + for(i=0; inPhrase; i++){ + zExplain = sqlite4MAppendf( + db, zExplain, "%stf%d", zExplain, i + ); + } + for(iField=0; rc==SQLITE4_OK && iField%d%.2f%.2f%d", + zExplain, iField, scale, p->aAvgdl[iField], dl + ); + for(i=0; rc==SQLITE4_OK && inPhrase; i++){ + rc = fts5GetSizeFreqScale( + pCtx, nArg, apArg, bm25mask, i, iField, &dl, &tf, &scale + ); + zExplain = sqlite4MAppendf(db, zExplain, "%s%d", zExplain, tf); + } + } + zExplain = sqlite4MAppendf( + db, zExplain, "%s
PhraseIDF", zExplain + ); + for(i=0; i" + "tfs%d", zExplain, i + ); + } + zExplain = sqlite4MAppendf(db, zExplain, "%srank", zExplain); + } + + for(i=0; rc==SQLITE4_OK && inPhrase; i++){ + int iField; + double tfns = 0.0; /* Sum of tfn for all fields */ + double prank; /* Contribution to rank of this phrase */ + + if( bExplain ){ + zExplain = sqlite4MAppendf( + db, zExplain, "%s
%d%.2f", zExplain, i, p->aIdf[i] + ); + } + + for(iField = 0; iFieldaAvgdl[iField]; /* 3.20 */ + tfn = scale * (double)tf / B; + tfns += tfn; /* 3.19 */ + + + if( bExplain ){ + zExplain = sqlite4MAppendf(db, zExplain, "%s%.2f", zExplain, tfn); + } + } + + prank = p->aIdf[i] * tfns / (k1 + tfns); /* 3.21 */ + if( bExplain ){ + zExplain = sqlite4MAppendf(db, zExplain, "%s%.2f", zExplain, prank); + } + + /* Add it to the overall rank */ + rank += prank; + } + + if( rc==SQLITE4_OK ){ + if( bExplain ){ + zExplain = sqlite4MAppendf( + db, zExplain, "%s
overall rank=%.2f", zExplain, rank + ); + sqlite4_result_text(pCtx, zExplain, -1, SQLITE4_TRANSIENT, 0); + }else{ + sqlite4_result_double(pCtx, rank); + } + }else{ + sqlite4_result_error_code(pCtx, rc); + } + sqlite4DbFree(db, zExplain); +} + +typedef struct Snippet Snippet; +typedef struct SnippetText SnippetText; + +struct Snippet { + int iCol; + int iOff; + u64 hlmask; +}; + +struct SnippetText { + char *zOut; /* Pointer to snippet text */ + int nOut; /* Size of zOut in bytes */ + int nAlloc; /* Bytes of space allocated at zOut */ +}; + +typedef struct SnippetCtx SnippetCtx; +struct SnippetCtx { + sqlite4 *db; /* Database handle */ + int nToken; /* Number of tokens in snippet */ + int iOff; /* First token in snippet */ + u64 mask; /* Snippet mask. Highlight these terms */ + const char *zStart; + const char *zEnd; + const char *zEllipses; + + SnippetText *pOut; + + int iFrom; + int iTo; + const char *zText; /* Document to extract snippet from */ + int rc; /* Set to NOMEM if OOM is encountered */ +}; + +static void fts5SnippetAppend(SnippetCtx *p, const char *z, int n){ + if( p->rc==SQLITE4_OK ){ + SnippetText *pOut = p->pOut; + if( n<0 ) n = strlen(z); + if( (pOut->nOut + n) > pOut->nAlloc ){ + int nNew = (pOut->nOut+n) * 2; + + pOut->zOut = sqlite4DbReallocOrFree(p->db, pOut->zOut, nNew); + if( pOut->zOut==0 ){ + p->rc = SQLITE4_NOMEM; + return; + } + pOut->nAlloc = sqlite4DbMallocSize(p->db, pOut->zOut); + } + + memcpy(&pOut->zOut[pOut->nOut], z, n); + pOut->nOut += n; + } +} + +static int fts5SnippetCb( + void *pCtx, + int iStream, + int iOff, + const char *z, int n, + int iSrc, int nSrc +){ + SnippetCtx *p = (SnippetCtx *)pCtx; + + if( iOffiOff ){ + return 0; + }else if( iOff>=(p->iOff + p->nToken) ){ + fts5SnippetAppend(p, &p->zText[p->iFrom], p->iTo - p->iFrom); + fts5SnippetAppend(p, p->zEllipses, -1); + p->iFrom = -1; + return 1; + }else{ + int bHighlight; /* True to highlight term */ + + bHighlight = (p->mask & ((u64)1 << (iOff-p->iOff))) ? 1 : 0; + + if( p->iFrom==0 && p->iOff!=0 ){ + p->iFrom = iSrc; + if( p->pOut->nOut==0 ) fts5SnippetAppend(p, p->zEllipses, -1); + } + + if( bHighlight ){ + fts5SnippetAppend(p, &p->zText[p->iFrom], iSrc - p->iFrom); + fts5SnippetAppend(p, p->zStart, -1); + fts5SnippetAppend(p, &p->zText[iSrc], nSrc); + fts5SnippetAppend(p, p->zEnd, -1); + p->iTo = p->iFrom = iSrc+nSrc; + }else{ + p->iTo = iSrc + nSrc; + } + } + + return 0; +} + +static int fts5SnippetText( + sqlite4_context *pCtx, + Snippet *pSnip, + SnippetText *pText, + int nToken, + const char *zStart, + const char *zEnd, + const char *zEllipses +){ + int rc; + sqlite4_value *pVal = 0; + + u64 mask = pSnip->hlmask; + int iOff = pSnip->iOff; + int iCol = pSnip->iCol; + + rc = sqlite4_mi_column_value(pCtx, iCol, &pVal); + if( rc==SQLITE4_OK ){ + SnippetCtx sCtx; + int nText; + + nText = sqlite4_value_bytes(pVal); + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.zText = (const char *)sqlite4_value_text(pVal); + sCtx.db = sqlite4_context_db_handle(pCtx); + sCtx.nToken = nToken; + sCtx.iOff = iOff; + sCtx.mask = mask; + sCtx.zStart = zStart; + sCtx.zEnd = zEnd; + sCtx.zEllipses = zEllipses; + sCtx.pOut = pText; + + sqlite4_mi_tokenize(pCtx, sCtx.zText, nText, &sCtx, fts5SnippetCb); + if( sCtx.rc==SQLITE4_OK && sCtx.iFrom>0 ){ + fts5SnippetAppend(&sCtx, &sCtx.zText[sCtx.iFrom], nText - sCtx.iFrom); + } + rc = sCtx.rc; + } + + return rc; +} + +static int fts5BestSnippet( + sqlite4_context *pCtx, /* Context snippet() was called in */ + int iColumn, /* In this column (-1 means any column) */ + u64 *pMask, /* IN/OUT: Mask of high-priority phrases */ + int nToken, /* Number of tokens in requested snippet */ + Snippet *pSnip /* Populate this object */ +){ + sqlite4 *db = sqlite4_context_db_handle(pCtx); + int nPhrase; + int rc = SQLITE4_OK; + int i; + int iPrev = 0; + int iPrevCol = 0; + u64 *aMask; + u64 mask = *pMask; + u64 allmask = 0; + + int iBestOff = nToken-1; + int iBestCol = (iColumn >= 0 ? iColumn : 0); + int nBest = 0; + u64 hlmask = 0; /* Highlight mask associated with iBestOff */ + u64 missmask = 0; /* Mask of missing terms in iBestOff snip. */ + + sqlite4_mi_phrase_count(pCtx, &nPhrase); + aMask = sqlite4DbMallocZero(db, sizeof(u64) * nPhrase); + if( !aMask ) return SQLITE4_NOMEM; + + /* Iterate through all matches for all phrases */ + for(i=0; rc==SQLITE4_OK; i++){ + int iOff; + int iCol; + int iStream; + int iPhrase; + + rc = sqlite4_mi_match_detail(pCtx, i, &iOff, &iCol, &iStream, &iPhrase); + if( rc==SQLITE4_OK ){ + u64 tmask = 0; + u64 miss = 0; + int iMask; + int nShift; + int nScore = 0; + + int nPTok; + int iPTok; + + if( iColumn>=0 && iColumn!=iCol ) continue; + + allmask |= ((u64)1 << iPhrase); + + nShift = ((iPrevCol==iCol) ? (iOff-iPrev) : 100); + + for(iMask=0; iMask> nShift; + }else{ + aMask[iMask] = 0; + } + } + sqlite4_mi_phrase_token_count(pCtx, iPhrase, &nPTok); + for(iPTok=0; iPToknBest ){ + hlmask = tmask; + missmask = miss; + nBest = nScore; + iBestOff = iOff; + iBestCol = iCol; + } + + iPrev = iOff; + iPrevCol = iCol; + } + } + if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK; + + pSnip->iOff = iBestOff-nToken+1; + pSnip->iCol = iBestCol; + pSnip->hlmask = hlmask; + *pMask = mask & missmask & allmask; + + sqlite4DbFree(db, aMask); + return rc; +} + +static void fts5SnippetImprove( + sqlite4_context *pCtx, + int nToken, /* Size of required snippet */ + int nSz, /* Total size of column in tokens */ + Snippet *pSnip +){ + int i; + int nLead = 0; + int nShift = 0; + + u64 mask = pSnip->hlmask; + int iOff = pSnip->iOff; + + if( mask==0 ) return; + assert( mask & ((u64)1 << (nToken-1)) ); + + for(i=0; (mask & ((u64)1 << i))==0; i++); + nLead = i; + + nShift = (nLead/2); + if( iOff+nShift > nSz-nToken ) nShift = (nSz-nToken) - iOff; + if( iOff+nShift < 0 ) nShift = -1 * iOff; + + iOff += nShift; + mask = mask >> nShift; + + pSnip->iOff = iOff; + pSnip->hlmask = mask; +} + +static void fts5Snippet(sqlite4_context *pCtx, int nArg, sqlite4_value **apArg){ + Snippet aSnip[4]; + int nSnip; + int iCol = -1; + int nToken = -15; + int rc; + int nPhrase; + + const char *zStart = ""; + const char *zEnd = ""; + const char *zEllipses = "..."; + + if( nArg>0 ) zStart = (const char *)sqlite4_value_text(apArg[0]); + if( nArg>1 ) zEnd = (const char *)sqlite4_value_text(apArg[1]); + if( nArg>2 ) zEllipses = (const char *)sqlite4_value_text(apArg[2]); + if( nArg>3 ) iCol = sqlite4_value_int(apArg[3]); + if( nArg>4 ) nToken = sqlite4_value_int(apArg[4]); + + rc = sqlite4_mi_phrase_count(pCtx, &nPhrase); + for(nSnip=1; rc==SQLITE4_OK && nSnip<5; nSnip = ((nSnip==2) ? 3 : (nSnip+1))){ + int nTok; + int i; + u64 mask = ((u64)1 << nPhrase) - 1; + + if( nToken<0 ){ + nTok = nToken * -1; + }else{ + nTok = (nToken + (nSnip-1)) / nSnip; + } + + memset(aSnip, 0, sizeof(aSnip)); + for(i=0; rc==SQLITE4_OK && i0 ){ + brk = x(pCtx, 0, iOff++, aBuf, iBuf, i-iBuf, iBuf); + iBuf = 0; + } + } + if( iBuf>0 ) x(pCtx, 0, iOff++, aBuf, iBuf, i-iBuf, iBuf); + + sqlite4_free(pEnv, aBuf); + return SQLITE4_OK; +} + +int sqlite4InitFts5Func(sqlite4 *db){ + int rc; + int i; + sqlite4_env *pEnv = sqlite4_db_env(db); + + struct RankFunction { + const char *zName; + int mask; + } aRank[] = { + { "rank", 0 }, + { "erank", BM25_EXPLAIN }, + { "rankc", BM25_FCOLUMNS }, + { "erankc", BM25_FCOLUMNS|BM25_EXPLAIN }, + { "ranks", BM25_FSTREAMS }, + { "eranks", BM25_FSTREAMS|BM25_EXPLAIN } + }; + + rc = sqlite4_create_tokenizer(db, "simple", (void *)pEnv, + fts5SimpleCreate, fts5SimpleTokenize, fts5SimpleDestroy + ); + if( rc==SQLITE4_OK ){ + rc = sqlite4_create_mi_function( + db, "snippet", -1, SQLITE4_UTF8, 0, fts5Snippet, 0); + } + + for(i=0; rc==SQLITE4_OK && ilen ){ p2 = len-p1; if( p2<0 ) p2 = 0; } - sqlite4_result_blob(context, (char*)&z[p1], (int)p2, SQLITE4_TRANSIENT); + sqlite4_result_blob(context, (char*)&z[p1], (int)p2, SQLITE4_TRANSIENT, 0); } } /* ** Implementation of the round() function @@ -334,11 +334,11 @@ z1 = contextMalloc(context, ((i64)n)+1); if( z1 ){ for(i=0; i>4)&0x0F]; zText[(i*2)+3] = hexdigits[(zBlob[i])&0x0F]; } zText[(nBlob*2)+2] = '\''; zText[(nBlob*2)+3] = '\0'; - zText[0] = 'X'; + zText[0] = 'x'; zText[1] = '\''; - sqlite4_result_text(context, zText, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zText, -1, SQLITE4_TRANSIENT, 0); sqlite4_free(sqlite4_context_env(context), zText); } break; } case SQLITE4_TEXT: { @@ -899,17 +900,17 @@ z[j++] = '\''; } } z[j++] = '\''; z[j] = 0; - sqlite4_result_text(context, z, j, SQLITE4_DYNAMIC); + sqlite4_result_text(context, z, j, SQLITE4_DYNAMIC, 0); } break; } default: { assert( sqlite4_value_type(argv[0])==SQLITE4_NULL ); - sqlite4_result_text(context, "NULL", 4, SQLITE4_STATIC); + sqlite4_result_text(context, "NULL", 4, SQLITE4_STATIC, 0); break; } } } @@ -936,11 +937,11 @@ unsigned char c = *pBlob; *(z++) = hexdigits[(c>>4)&0xf]; *(z++) = hexdigits[c&0xf]; } *z = 0; - sqlite4_result_text(context, zHex, n*2, SQLITE4_DYNAMIC); + sqlite4_result_text(context, zHex, n*2, SQLITE4_DYNAMIC, 0); } } /* ** The zeroblob(N) function returns a zero-filled blob of size N bytes. @@ -1045,11 +1046,11 @@ assert( j+nStr-i+1==nOut ); memcpy(&zOut[j], &zStr[i], nStr-i); j += nStr - i; assert( j<=nOut ); zOut[j] = 0; - sqlite4_result_text(context, (char*)zOut, j, SQLITE4_DYNAMIC); + sqlite4_result_text(context, (char*)zOut, j, SQLITE4_DYNAMIC, 0); } /* ** Implementation of the TRIM(), LTRIM(), and RTRIM() functions. ** The userdata is 0x1 for left trim, 0x2 for right trim, 0x3 for both. @@ -1129,11 +1130,11 @@ } if( zCharSet ){ sqlite4_free(sqlite4_context_env(context), azChar); } } - sqlite4_result_text(context, (char*)zIn, nIn, SQLITE4_TRANSIENT); + sqlite4_result_text(context, (char*)zIn, nIn, SQLITE4_TRANSIENT, 0); } /* IMP: R-25361-16150 This function is omitted from SQLite by default. It ** is only available if the SQLITE4_SOUNDEX compile-time option is used @@ -1184,15 +1185,15 @@ } while( j<4 ){ zResult[j++] = '0'; } zResult[j] = 0; - sqlite4_result_text(context, zResult, 4, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zResult, 4, SQLITE4_TRANSIENT, 0); }else{ /* IMP: R-64894-50321 The string "?000" is returned if the argument ** is NULL or contains no ASCII alphabetic characters. */ - sqlite4_result_text(context, "?000", 4, SQLITE4_STATIC); + sqlite4_result_text(context, "?000", 4, SQLITE4_STATIC, 0); } } #endif /* SQLITE4_SOUNDEX */ #if 0 /*ndef SQLITE4_OMIT_LOAD_EXTENSION*/ @@ -1416,11 +1417,11 @@ sqlite4_result_error_toobig(context); }else if( pAccum->mallocFailed ){ sqlite4_result_error_nomem(context); }else{ sqlite4_result_text(context, sqlite4StrAccumFinish(pAccum), -1, - SQLITE4_DYNAMIC); + SQLITE4_DYNAMIC, 0); } } } /* Index: src/hash.c ================================================================== --- src/hash.c +++ src/hash.c @@ -12,23 +12,61 @@ ** This is the implementation of generic hash-tables ** used in SQLite. */ #include "sqliteInt.h" #include + +/* +** The hashing function. +*/ +static unsigned int strHash(const char *z, int nKey){ + int h = 0; + assert( nKey>=0 ); + while( nKey > 0 ){ + h = (h<<3) ^ h ^ sqlite4UpperToLower[(unsigned char)*z++]; + nKey--; + } + return h; +} + +static int strCmp(const char *z1, const char *z2, int n){ + return sqlite4_strnicmp(z1, z2, n); +} + +static unsigned int binHash(const char *z, int nKey){ + int h = 0; + assert( nKey>=0 ); + while( nKey > 0 ){ + h = (h<<3) ^ h ^ ((unsigned char)*z++); + nKey--; + } + return h; +} + +static int binCmp(const char *z1, const char *z2, int n){ + return memcmp(z1, z2, n); +} /* Turn bulk memory into a hash table object by initializing the ** fields of the Hash structure. ** ** "pNew" is a pointer to the hash table that is to be initialized. */ -void sqlite4HashInit(sqlite4_env *pEnv, Hash *pNew){ +void sqlite4HashInit(sqlite4_env *pEnv, Hash *pNew, int bBin){ assert( pNew!=0 ); pNew->first = 0; pNew->count = 0; pNew->htsize = 0; pNew->ht = 0; pNew->pEnv = pEnv; + if( bBin ){ + pNew->xHash = binHash; + pNew->xCmp = binCmp; + }else{ + pNew->xHash = strHash; + pNew->xCmp = strCmp; + } } /* Remove all entries from a hash table. Reclaim all memory. ** Call this routine to delete a hash table or to reset a hash table ** to the empty state. @@ -47,24 +85,10 @@ sqlite4_free(pH->pEnv, elem); elem = next_elem; } pH->count = 0; } - -/* -** The hashing function. -*/ -static unsigned int strHash(const char *z, int nKey){ - int h = 0; - assert( nKey>=0 ); - while( nKey > 0 ){ - h = (h<<3) ^ h ^ sqlite4UpperToLower[(unsigned char)*z++]; - nKey--; - } - return h; -} - /* Link pNew element into the hash table pH. If pEntry!=0 then also ** insert pNew into the pEntry hash bucket. */ static void insertElement( @@ -124,11 +148,11 @@ sqlite4_free(pH->pEnv, pH->ht); pH->ht = new_ht; pH->htsize = new_size = sqlite4MallocSize(pH->pEnv, new_ht)/sizeof(struct _ht); memset(new_ht, 0, new_size*sizeof(struct _ht)); for(elem=pH->first, pH->first=0; elem; elem = next_elem){ - unsigned int h = strHash(elem->pKey, elem->nKey) % new_size; + unsigned int h = pH->xHash(elem->pKey, elem->nKey) % new_size; next_elem = elem->next; insertElement(pH, &new_ht[h], elem); } return 1; } @@ -153,11 +177,11 @@ }else{ elem = pH->first; count = pH->count; } while( count-- && ALWAYS(elem) ){ - if( elem->nKey==nKey && sqlite4StrNICmp(elem->pKey,pKey,nKey)==0 ){ + if( elem->nKey==nKey && pH->xCmp(elem->pKey,pKey,nKey)==0 ){ return elem; } elem = elem->next; } return 0; @@ -207,11 +231,11 @@ assert( pH!=0 ); assert( pKey!=0 ); assert( nKey>=0 ); if( pH->ht ){ - h = strHash(pKey, nKey) % pH->htsize; + h = pH->xHash(pKey, nKey) % pH->htsize; }else{ h = 0; } elem = findElementGivenHash(pH, pKey, nKey, h); return elem ? elem->data : 0; @@ -238,11 +262,11 @@ assert( pH!=0 ); assert( pKey!=0 ); assert( nKey>=0 ); if( pH->htsize ){ - h = strHash(pKey, nKey) % pH->htsize; + h = pH->xHash(pKey, nKey) % pH->htsize; }else{ h = 0; } elem = findElementGivenHash(pH,pKey,nKey,h); if( elem ){ @@ -264,15 +288,15 @@ new_elem->data = data; pH->count++; if( pH->count>=10 && pH->count > 2*pH->htsize ){ if( rehash(pH, pH->count*2) ){ assert( pH->htsize>0 ); - h = strHash(pKey, nKey) % pH->htsize; + h = pH->xHash(pKey, nKey) % pH->htsize; } } if( pH->ht ){ insertElement(pH, &pH->ht[h], new_elem); }else{ insertElement(pH, 0, new_elem); } return 0; } Index: src/hash.h ================================================================== --- src/hash.h +++ src/hash.h @@ -40,10 +40,12 @@ ** in the table, it is faster to do a linear search than to manage ** the hash table. */ struct Hash { sqlite4_env *pEnv; /* Memory allocation environment */ + unsigned int (*xHash)(const char *, int); + int (*xCmp)(const char *, const char *, int); unsigned int htsize; /* Number of buckets in the hash table */ unsigned int count; /* Number of entries in this table */ HashElem *first; /* The first element of the array */ struct _ht { /* the hash table */ int count; /* Number of entries with this hash */ @@ -64,11 +66,11 @@ }; /* ** Access routines. To delete, insert a NULL pointer. */ -void sqlite4HashInit(sqlite4_env *pEnv, Hash*); +void sqlite4HashInit(sqlite4_env *pEnv, Hash*, int); void *sqlite4HashInsert(Hash*, const char *pKey, int nKey, void *pData); void *sqlite4HashFind(const Hash*, const char *pKey, int nKey); void sqlite4HashClear(Hash*); /* @@ -84,14 +86,14 @@ ** } */ #define sqliteHashFirst(H) ((H)->first) #define sqliteHashNext(E) ((E)->next) #define sqliteHashData(E) ((E)->data) -/* #define sqliteHashKey(E) ((E)->pKey) // NOT USED */ -/* #define sqliteHashKeysize(E) ((E)->nKey) // NOT USED */ +#define sqliteHashKey(E) ((E)->pKey) +#define sqliteHashKeysize(E) ((E)->nKey) /* ** Number of entries in a hash table */ /* #define sqliteHashCount(H) ((H)->count) // NOT USED */ #endif /* _SQLITE4_HASH_H_ */ Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -22,11 +22,12 @@ int iCur, /* The cursor number of the table */ int iDb, /* The database index in sqlite4.aDb[] */ Table *pTab, /* The table to be opened */ int opcode /* OP_OpenRead or OP_OpenWrite */ ){ - assert( 0 ); + Index *pPk = sqlite4FindPrimaryKey(pTab, 0); + sqlite4OpenIndex(p, iCur, iDb, pPk, opcode); } /* ** Open VDBE cursor iCur to access index pIdx. pIdx is guaranteed to be ** a part of database iDb. @@ -246,11 +247,11 @@ ** (3) Register to hold the rowid in sqlite_sequence of pTab ** ** The 2nd register is the one that is returned. That is all the ** insert routine needs to know about. */ -/*static FIXME: make static when this function gets used. */ int autoIncBegin( +static int autoIncBegin( Parse *pParse, /* Parsing context */ int iDb, /* Index of the database holding pTab */ Table *pTab /* The table we are writing to */ ){ int memId = 0; /* Register holding maximum rowid */ @@ -320,11 +321,11 @@ ** This routine should be called when the top of the stack holds a ** new rowid that is about to be inserted. If that new rowid is ** larger than the maximum rowid in the memId memory cell, then the ** memory cell is updated. The stack is unchanged. */ -/*static FIXME: make static when this function gets used. */ void autoIncStep(Parse *pParse, int memId, int regRowid){ +static void autoIncStep(Parse *pParse, int memId, int regRowid){ if( memId>0 ){ sqlite4VdbeAddOp2(pParse->pVdbe, OP_MemMax, memId, regRowid); } } @@ -515,22 +516,22 @@ int addrSelect = 0; /* Address of coroutine that implements the SELECT */ SelectDest dest; /* Destination for SELECT on rhs of INSERT */ int iDb; /* Index of database holding TABLE */ Db *pDb; /* The database containing table being inserted into */ int appendFlag = 0; /* True if the insert is likely to be an append */ + int iPk; /* Cursor offset of PK index cursor */ + Index *pPk; /* Primary key for table pTab */ + int iIntPKCol = -1; /* Column of INTEGER PRIMARY KEY or -1 */ + int bImplicitPK; /* True if table pTab has an implicit PK */ /* Register allocations */ int regFromSelect = 0;/* Base register for data coming from SELECT */ int regEof = 0; /* Register recording end of SELECT data */ int *aRegIdx = 0; /* One register allocated to each index */ - - int iPk; /* Cursor offset of PK index cursor */ - Index *pPk; /* Primary key for table pTab */ - int bImplicitPK; /* True if table pTab has an implicit PK */ - int regContent; /* First register in column value array */ - int regRowid; /* If bImplicitPK, register holding IPK */ - + int regContent; /* First register in column value array */ + int regRowid; /* If bImplicitPK, register holding IPK */ + int regAutoinc; /* Register holding the AUTOINCREMENT counter */ #ifndef SQLITE4_OMIT_TRIGGER int isView; /* True if attempting to insert into a view */ Trigger *pTrigger; /* List of triggers on pTab, if required */ int tmask; /* Mask of trigger times */ @@ -562,11 +563,20 @@ ** Also set pPk to point to the primary key, and iPk to the cursor offset ** of the primary key cursor (i.e. so that the cursor opened on the primary ** key index is VDBE cursor (baseCur+iPk). */ pPk = sqlite4FindPrimaryKey(pTab, &iPk); assert( (pPk==0)==IsView(pTab) ); - bImplicitPK = (pPk && pPk->aiColumn[0]==-1); + if( pPk ){ + bImplicitPK = pPk->aiColumn[0]==(-1); + if( pPk->fIndex & IDX_IntPK ){ + assert( pPk->nColumn==1 ); + iIntPKCol = pPk->aiColumn[0]; + } + }else{ + bImplicitPK = 0; + } + /* Figure out if we have any triggers and if the table being ** inserted into is a view. */ #ifndef SQLITE4_OMIT_TRIGGER pTrigger = sqlite4TriggersExist(pParse, pTab, TK_INSERT, 0, &tmask); @@ -617,10 +627,15 @@ assert( !pTrigger ); assert( pList==0 ); goto insert_end; } #endif /* SQLITE4_OMIT_XFER_OPT */ + + /* If this is an AUTOINCREMENT table, look up the sequence number in the + ** sqlite_sequence table and store it in memory cell regAutoinc. + */ + regAutoinc = autoIncBegin(pParse, iDb, pTab); /* Figure out how many columns of data are supplied. If the data ** is coming from a SELECT statement, then generate a co-routine that ** produces a single row of the SELECT on each invocation. The ** co-routine is the common header to the 3rd and 4th templates. @@ -766,11 +781,11 @@ pColumn->a[i].idx = -1; } for(i=0; inId; i++){ char *zTest = pColumn->a[i].zName; for(j=0; jnCol; j++){ - if( sqlite4StrICmp(zTest, pTab->aCol[j].zName)==0 ){ + if( sqlite4_stricmp(zTest, pTab->aCol[j].zName)==0 ){ pColumn->a[i].idx = j; break; } } if( j==pTab->nCol ){ @@ -832,11 +847,11 @@ addrInsTop = sqlite4VdbeAddOp1(v, OP_If, regEof); } /* Allocate an array of registers in which to assemble the values for the ** new row. If the table has an explicit primary key, we need one register - ** for each table column. If the table uses an implicit primary key, the + ** for each table column. If the table uses an implicit primary key, then ** nCol+1 registers are required. */ regRowid = ++pParse->nMem; regContent = pParse->nMem+1; pParse->nMem += pTab->nCol; @@ -848,26 +863,27 @@ } endOfLoop = sqlite4VdbeMakeLabel(v); for(i=0; inCol; i++){ + int regDest = regContent+i; j = i; if( pColumn ){ for(j=0; jnId; j++){ if( pColumn->a[j].idx==i ) break; } } if( nColumn==0 || (pColumn && j>=pColumn->nId) ){ - sqlite4ExprCode(pParse, pTab->aCol[i].pDflt, regContent+i); + sqlite4ExprCode(pParse, pTab->aCol[i].pDflt, regDest); }else if( useTempTable ){ - sqlite4VdbeAddOp3(v, OP_Column, srcTab, j, regContent+i); + sqlite4VdbeAddOp3(v, OP_Column, srcTab, j, regDest); }else if( pSelect ){ - sqlite4VdbeAddOp2(v, OP_SCopy, regFromSelect+j, regContent+i); + sqlite4VdbeAddOp2(v, OP_SCopy, regFromSelect+j, regDest); }else{ assert( pSelect==0 ); /* Otherwise useTempTable is true */ - sqlite4ExprCodeAndCache(pParse, pList->a[j].pExpr, regContent+i); + sqlite4ExprCodeAndCache(pParse, pList->a[j].pExpr, regDest); } } if( !isView ){ sqlite4VdbeAddOp2(v, OP_Affinity, regContent, pTab->nCol); @@ -881,10 +897,19 @@ sqlite4CodeRowTrigger( pParse, pTrigger, TK_INSERT, 0, TRIGGER_BEFORE, pTab, (regRowid - pTab->nCol - 1), onError, endOfLoop ); } + + if( iIntPKCol>=0 ){ + int regDest = regContent+iIntPKCol; + int a1; + a1 = sqlite4VdbeAddOp1(v, OP_NotNull, regDest); + sqlite4VdbeAddOp3(v, OP_NewRowid, baseCur, regDest, regAutoinc); + sqlite4VdbeJumpHere(v, a1); + autoIncStep(pParse, regAutoinc, regDest); + } if( bImplicitPK ){ assert( !isView ); sqlite4VdbeAddOp2(v, OP_NewRowid, baseCur+iPk, regRowid); } @@ -1060,10 +1085,13 @@ } #else /* !defined(SQLITE4_OMIT_CHECK) */ # define generateCheckChecks(a,b,c,d,e) #endif +/* +** Locate the primary key index for a table. +*/ Index *sqlite4FindPrimaryKey( Table *pTab, /* Table to locate primary key for */ int *piPk /* OUT: Index of PRIMARY KEY */ ){ Index *p; @@ -1289,10 +1317,13 @@ int idx = pPk->aiColumn[i]; sqlite4VdbeAddOp2(v, OP_SCopy, regContent+idx, regTmp+i+pIdx->nColumn); } } sqlite4VdbeAddOp3(v, OP_MakeIdxKey, iIdx, regTmp, regKey); + if( pIdx==pPk && (pPk->fIndex & IDX_IntPK)!=0 ){ + sqlite4VdbeChangeP5(v, OPFLAG_LASTROWID); + } VdbeComment((v, "key for %s", pIdx->zName)); /* If Index.onError==OE_None, then pIdx is not a UNIQUE or PRIMARY KEY ** index. In this case there is no need to test the index for uniqueness ** - all that is required is to populate the regKey register. Jump @@ -1417,11 +1448,16 @@ sqlite4ExprCacheAffinityChange(pParse, regContent, pTab->nCol); /* Write the entry to each index. */ for(i=0, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ assert( pIdx->eIndexType!=SQLITE4_INDEX_PRIMARYKEY || aRegIdx[i] ); - if( aRegIdx[i] ){ + if( pIdx->eIndexType==SQLITE4_INDEX_FTS5 ){ + int iPK; + sqlite4FindPrimaryKey(pTab, &iPK); + sqlite4Fts5CodeUpdate(pParse, pIdx, 0, aRegIdx[iPK], regContent, 0); + } + else if( aRegIdx[i] ){ int regData = 0; int flags = 0; if( pIdx->eIndexType==SQLITE4_INDEX_PRIMARYKEY ){ regData = regRec; flags = pik_flags; @@ -1501,11 +1537,11 @@ return z2==0; } if( z2==0 ){ return 0; } - return sqlite4StrICmp(z1, z2)==0; + return sqlite4_stricmp(z1, z2)==0; } /* ** Check to see if index pSrc is compatible as a source of data Index: src/kv.c ================================================================== --- src/kv.c +++ src/kv.c @@ -88,11 +88,11 @@ KVStore *pNew = 0; int rc; sqlite4_env *pEnv = &sqlite4DefaultEnv; /* OR db->pEnv */ const char *zStorageName; KVFactory *pMkr; - int (*xFactory)(sqlite4_env*,sqlite4_kvstore**,const char*,unsigned); + sqlite4_kvfactory xFactory; if( (flags & SQLITE4_KVOPEN_TEMPORARY)!=0 || zUri==0 || zUri[0]==0 ){ zStorageName = "temp"; }else{ zStorageName = sqlite4_uri_parameter(zName, "kv"); Index: src/kvlsm.c ================================================================== --- src/kvlsm.c +++ src/kvlsm.c @@ -307,12 +307,11 @@ KVCursor *pKVCursor, /* The cursor whose key is desired */ const KVByteArray **paKey, /* Make this point to the key */ KVSize *pN /* Make this point to the size of the key */ ){ KVLsmCsr *pCsr = (KVLsmCsr *)pKVCursor; - - assert( lsm_csr_valid(pCsr->pCsr) ); + if( 0==lsm_csr_valid(pCsr->pCsr) ) return SQLITE4_DONE; return lsm_csr_key(pCsr->pCsr, (const void **)paKey, (int *)pN); } /* ** Return the data of the node the cursor is pointing to. @@ -452,11 +451,12 @@ }else{ struct Config { const char *zParam; int eParam; } aConfig[] = { - { "lsm_block_size", LSM_CONFIG_BLOCK_SIZE } + { "lsm_block_size", LSM_CONFIG_BLOCK_SIZE }, + { "lsm_multiple_processes", LSM_CONFIG_MULTIPLE_PROCESSES } }; memset(pNew, 0, sizeof(KVLsm)); pNew->base.pStoreVfunc = &kvlsmMethods; pNew->base.pEnv = pEnv; Index: src/lsm.h ================================================================== --- src/lsm.h +++ src/lsm.h @@ -22,10 +22,11 @@ /* ** Opaque handle types. */ typedef struct lsm_compress lsm_compress; /* Compression library functions */ +typedef struct lsm_compress_factory lsm_compress_factory; typedef struct lsm_cursor lsm_cursor; /* Database cursor handle */ typedef struct lsm_db lsm_db; /* Database connection handle */ typedef struct lsm_env lsm_env; /* Runtime environment */ typedef struct lsm_file lsm_file; /* OS file handle */ typedef struct lsm_mutex lsm_mutex; /* Mutex handle */ @@ -36,10 +37,13 @@ /* Candidate values for the 3rd argument to lsm_env.xLock() */ #define LSM_LOCK_UNLOCK 0 #define LSM_LOCK_SHARED 1 #define LSM_LOCK_EXCL 2 +/* Flags for lsm_env.xOpen() */ +#define LSM_OPEN_READONLY 0x0001 + /* ** CAPI: Database Runtime Environment ** ** Run-time environment used by LSM */ @@ -47,11 +51,11 @@ int nByte; /* Size of this structure in bytes */ int iVersion; /* Version number of this structure (1) */ /****** file i/o ***********************************************/ void *pVfsCtx; int (*xFullpath)(lsm_env*, const char *, char *, int *); - int (*xOpen)(lsm_env*, const char *, lsm_file **); + int (*xOpen)(lsm_env*, const char *, int flags, lsm_file **); int (*xRead)(lsm_file *, lsm_i64, void *, int); int (*xWrite)(lsm_file *, lsm_i64, void *, int); int (*xTruncate)(lsm_file *, lsm_i64); int (*xSync)(lsm_file *); int (*xSectorSize)(lsm_file *); @@ -97,17 +101,23 @@ */ #define LSM_OK 0 #define LSM_ERROR 1 #define LSM_BUSY 5 #define LSM_NOMEM 7 +#define LSM_READONLY 8 #define LSM_IOERR 10 #define LSM_CORRUPT 11 #define LSM_FULL 13 #define LSM_CANTOPEN 14 #define LSM_PROTOCOL 15 #define LSM_MISUSE 21 +#define LSM_MISMATCH 50 + + +#define LSM_IOERR_NOENT (LSM_IOERR | (1<<8)) + /* ** CAPI: Creating and Destroying Database Connection Handles ** ** Open and close a database connection handle. */ @@ -118,11 +128,11 @@ ** CAPI: Connecting to a Database */ int lsm_open(lsm_db *pDb, const char *zFilename); /* -** CAPI: Obtaining pointers to databases environments +** CAPI: Obtaining pointers to database environments ** ** Return a pointer to the environment used by the database connection ** passed as the first argument. Assuming the argument is valid, this ** function always returns a valid environment pointer - it cannot fail. */ @@ -143,25 +153,43 @@ int lsm_config(lsm_db *, int, ...); /* ** The following values may be passed as the second argument to lsm_config(). ** -** LSM_CONFIG_WRITE_BUFFER: -** A read/write integer parameter. This value determines the maximum amount -** of space (in bytes) used to accumulate writes in main-memory before -** they are flushed to a level 0 segment. +** LSM_CONFIG_AUTOFLUSH: +** A read/write integer parameter. +** +** This value determines the amount of data allowed to accumulate in a +** live in-memory tree before it is marked as old. After committing a +** transaction, a connection checks if the size of the live in-memory tree, +** including data structure overhead, is greater than the value of this +** option in KB. If it is, and there is not already an old in-memory tree, +** the live in-memory tree is marked as old. +** +** The maximum allowable value is 1048576 (1GB). There is no minimum +** value. If this parameter is set to zero, then an attempt is made to +** mark the live in-memory tree as old after each transaction is committed. +** +** The default value is 1024 (1MB). ** ** LSM_CONFIG_PAGE_SIZE: ** A read/write integer parameter. This parameter may only be set before ** lsm_open() has been called. ** ** LSM_CONFIG_BLOCK_SIZE: -** A read/write integer parameter. This parameter may only be set before -** lsm_open() has been called. +** A read/write integer parameter. ** -** LSM_CONFIG_LOG_SIZE: -** A read/write integer parameter. +** This parameter may only be set before lsm_open() has been called. It +** must be set to a power of two between 64 and 65536, inclusive (block +** sizes between 64KB and 64MB). +** +** If the connection creates a new database, the block size of the new +** database is set to the value of this option in KB. After lsm_open() +** has been called, querying this parameter returns the actual block +** size of the opened database. +** +** The default value is 1024 (1MB blocks). ** ** LSM_CONFIG_SAFETY: ** A read/write integer parameter. Valid values are 0, 1 (the default) ** and 2. This parameter determines how robust the database is in the ** face of a system crash (e.g. a power failure or operating system @@ -181,19 +209,33 @@ ** A read/write integer parameter. ** ** LSM_CONFIG_AUTOCHECKPOINT: ** A read/write integer parameter. ** +** If this option is set to non-zero value N, then a checkpoint is +** automatically attempted after each N KB of data have been written to +** the database file. +** +** The amount of uncheckpointed data already written to the database file +** is a global parameter. After performing database work (writing to the +** database file), the process checks if the total amount of uncheckpointed +** data exceeds the value of this paramter. If so, a checkpoint is performed. +** This means that this option may cause the connection to perform a +** checkpoint even if the current connection has itself written very little +** data into the database file. +** +** The default value is 2048 (checkpoint every 2MB). +** ** LSM_CONFIG_MMAP: ** A read/write integer parameter. True to use mmap() to access the ** database file. False otherwise. ** ** LSM_CONFIG_USE_LOG: ** A read/write boolean parameter. True (the default) to use the log ** file normally. False otherwise. ** -** LSM_CONFIG_NMERGE: +** LSM_CONFIG_AUTOMERGE: ** A read/write integer parameter. The minimum number of segments to ** merge together at a time. Default value 4. ** ** LSM_CONFIG_MAX_FREELIST: ** A read/write integer parameter. The maximum number of free-list @@ -221,25 +263,34 @@ ** after lsm_open() has been called results in an LSM_MISUSE error. ** ** LSM_CONFIG_GET_COMPRESSION: ** Query the compression methods used to compress and decompress database ** content. +** +** LSM_CONFIG_SET_COMPRESSION_FACTORY: +** Configure a factory method to be invoked in case of an LSM_MISMATCH +** error. +** +** LSM_CONFIG_READONLY: +** A read/write boolean parameter. This parameter may only be set before +** lsm_open() is called. */ -#define LSM_CONFIG_WRITE_BUFFER 1 -#define LSM_CONFIG_PAGE_SIZE 2 -#define LSM_CONFIG_SAFETY 3 -#define LSM_CONFIG_BLOCK_SIZE 4 -#define LSM_CONFIG_AUTOWORK 5 -#define LSM_CONFIG_LOG_SIZE 6 -#define LSM_CONFIG_MMAP 7 -#define LSM_CONFIG_USE_LOG 8 -#define LSM_CONFIG_NMERGE 9 -#define LSM_CONFIG_MAX_FREELIST 10 -#define LSM_CONFIG_MULTIPLE_PROCESSES 11 -#define LSM_CONFIG_AUTOCHECKPOINT 12 -#define LSM_CONFIG_SET_COMPRESSION 13 -#define LSM_CONFIG_GET_COMPRESSION 14 +#define LSM_CONFIG_AUTOFLUSH 1 +#define LSM_CONFIG_PAGE_SIZE 2 +#define LSM_CONFIG_SAFETY 3 +#define LSM_CONFIG_BLOCK_SIZE 4 +#define LSM_CONFIG_AUTOWORK 5 +#define LSM_CONFIG_MMAP 7 +#define LSM_CONFIG_USE_LOG 8 +#define LSM_CONFIG_AUTOMERGE 9 +#define LSM_CONFIG_MAX_FREELIST 10 +#define LSM_CONFIG_MULTIPLE_PROCESSES 11 +#define LSM_CONFIG_AUTOCHECKPOINT 12 +#define LSM_CONFIG_SET_COMPRESSION 13 +#define LSM_CONFIG_GET_COMPRESSION 14 +#define LSM_CONFIG_SET_COMPRESSION_FACTORY 15 +#define LSM_CONFIG_READONLY 16 #define LSM_SAFETY_OFF 0 #define LSM_SAFETY_NORMAL 1 #define LSM_SAFETY_FULL 2 @@ -250,12 +301,21 @@ void *pCtx; unsigned int iId; int (*xBound)(void *, int nSrc); int (*xCompress)(void *, char *, int *, const char *, int); int (*xUncompress)(void *, char *, int *, const char *, int); + void (*xFree)(void *pCtx); +}; + +struct lsm_compress_factory { + void *pCtx; + int (*xFactory)(void *, lsm_db *, unsigned int); + void (*xFree)(void *pCtx); }; +#define LSM_COMPRESSION_EMPTY 0 +#define LSM_COMPRESSION_NONE 1 /* ** CAPI: Allocating and Freeing Memory ** ** Invoke the memory allocation functions that belong to environment @@ -360,10 +420,37 @@ ** string should be eventually freed by the caller using lsm_free(). ** ** The Tcl structure returned is a list containing one element for each ** free block in the database. The element itself consists of two ** integers - the block number and the id of the snapshot that freed it. +** +** LSM_INFO_CHECKPOINT_SIZE: +** The third argument should be of type (int *). The location pointed to +** by this argument is populated with the number of KB written to the +** database file since the most recent checkpoint. +** +** LSM_INFO_TREE_SIZE: +** If this value is passed as the second argument to an lsm_info() call, it +** should be followed by two arguments of type (int *) (for a total of four +** arguments). +** +** At any time, there are either one or two tree structures held in shared +** memory that new database clients will access (there may also be additional +** tree structures being used by older clients - this API does not provide +** information on them). One tree structure - the current tree - is used to +** accumulate new data written to the database. The other tree structure - +** the old tree - is a read-only tree holding older data and may be flushed +** to disk at any time. +** +** Assuming no error occurs, the location pointed to by the first of the two +** (int *) arguments is set to the size of the old in-memory tree in KB. +** The second is set to the size of the current, or live in-memory tree. +** +** LSM_INFO_COMPRESSION_ID: +** This value should be followed by a single argument of type +** (unsigned int *). If successful, the location pointed to is populated +** with the database compression id before returning. */ #define LSM_INFO_NWRITE 1 #define LSM_INFO_NREAD 2 #define LSM_INFO_DB_STRUCTURE 3 #define LSM_INFO_LOG_STRUCTURE 4 @@ -370,10 +457,14 @@ #define LSM_INFO_ARRAY_STRUCTURE 5 #define LSM_INFO_PAGE_ASCII_DUMP 6 #define LSM_INFO_PAGE_HEX_DUMP 7 #define LSM_INFO_FREELIST 8 #define LSM_INFO_ARRAY_PAGES 9 +#define LSM_INFO_CHECKPOINT_SIZE 10 +#define LSM_INFO_TREE_SIZE 11 +#define LSM_INFO_FREELIST_SIZE 12 +#define LSM_INFO_COMPRESSION_ID 13 /* ** CAPI: Opening and Closing Write Transactions ** @@ -432,26 +523,26 @@ /* ** CAPI: Explicit Database Work and Checkpointing ** ** This function is called by a thread to work on the database structure. */ -int lsm_work(lsm_db *pDb, int nMerge, int nPage, int *pnWrite); +int lsm_work(lsm_db *pDb, int nMerge, int nKB, int *pnWrite); int lsm_flush(lsm_db *pDb); /* ** Attempt to checkpoint the current database snapshot. Return an LSM ** error code if an error occurs or LSM_OK otherwise. ** ** If the current snapshot has already been checkpointed, calling this -** function is a no-op. In this case if pnByte is not NULL, *pnByte is +** function is a no-op. In this case if pnKB is not NULL, *pnKB is ** set to 0. Or, if the current snapshot is successfully checkpointed -** by this function and pbCkpt is not NULL, *pnByte is set to the number +** by this function and pbKB is not NULL, *pnKB is set to the number ** of bytes written to the database file since the previous checkpoint -** (the same measure as returned by lsm_ckpt_size()). +** (the same measure as returned by the LSM_INFO_CHECKPOINT_SIZE query). */ -int lsm_checkpoint(lsm_db *pDb, int *pnByte); +int lsm_checkpoint(lsm_db *pDb, int *pnKB); /* ** CAPI: Opening and Closing Database Cursors ** ** Open and close a database cursor. @@ -573,49 +664,11 @@ /* ** Configure a callback that is invoked if the database connection ever ** writes to the database file. */ void lsm_config_work_hook(lsm_db *, void (*)(lsm_db *, void *), void *); - -/* -** The lsm_tree_size() function reports on the current state of the -** in-memory tree data structure. -** -** At any time, there are either one or two tree structures held in shared -** memory that new database clients will access (there may also be additional -** tree structures being used by older clients - this API does not provide -** information on them). One tree structure - the current tree - is used to -** accumulate new data written to the database. The other tree structure - the -** old tree - is a read-only tree holding older data and may be flushed to disk -** at any time. -** -** If successful, this function sets *pnNew to the number of bytes of shared -** memory space used by the current tree. *pbOld is set to true if the old -** tree exists, or false if it does not. -** -** If no error occurs, LSM_OK is returned. Otherwise an LSM error code. -** -** RACE CONDITION: -** Describe the race condition this function is subject to. -*/ -int lsm_tree_size(lsm_db *, int *pbOld, int *pnNew); - -/* -** This function is used to query the amount of data that has been written -** to the database file but not checkpointed (synced). If successful, *pnByte -** is set to the number of bytes before returning. -** -** LSM_OK is returned if successful. Or if an error occurs, an LSM error -** code is returned. -** -** RACE CONDITION: -** Describe the race condition this function is subject to. Or remove -** it somehow. -*/ -int lsm_ckpt_size(lsm_db *, int *pnByte); - /* ENDOFAPI */ #ifdef __cplusplus } /* End of the 'extern "C"' block */ #endif #endif /* ifndef _LSM_H */ Index: src/lsmInt.h ================================================================== --- src/lsmInt.h +++ src/lsmInt.h @@ -40,16 +40,21 @@ /* ** Default values for various data structure parameters. These may be ** overridden by calls to lsm_config(). */ -#define LSM_DFLT_PAGE_SIZE (4 * 1024) -#define LSM_DFLT_BLOCK_SIZE (2 * 1024 * 1024) -#define LSM_DFLT_WRITE_BUFFER (2 * 1024 * 1024) -#define LSM_DFLT_AUTOCHECKPOINT (4 * 1024 * 1024) -#define LSM_DFLT_LOG_SIZE (128*1024) -#define LSM_DFLT_NMERGE 4 +#define LSM_DFLT_PAGE_SIZE (4 * 1024) +#define LSM_DFLT_BLOCK_SIZE (1 * 1024 * 1024) +#define LSM_DFLT_AUTOFLUSH (1 * 1024 * 1024) +#define LSM_DFLT_AUTOCHECKPOINT (i64)(2 * 1024 * 1024) +#define LSM_DFLT_AUTOWORK 1 +#define LSM_DFLT_LOG_SIZE (128*1024) +#define LSM_DFLT_AUTOMERGE 4 +#define LSM_DFLT_SAFETY LSM_SAFETY_NORMAL +#define LSM_DFLT_MMAP LSM_IS_64_BIT +#define LSM_DFLT_MULTIPLE_PROCESSES 1 +#define LSM_DFLT_USE_LOG 1 /* Initial values for log file checksums. These are only used if the ** database file does not contain a valid checkpoint. */ #define LSM_CKSUM0_INIT 42 #define LSM_CKSUM1_INIT 42 @@ -76,10 +81,11 @@ typedef struct Merge Merge; typedef struct MergeInput MergeInput; typedef struct MetaPage MetaPage; typedef struct MultiCursor MultiCursor; typedef struct Page Page; +typedef struct Redirect Redirect; typedef struct Segment Segment; typedef struct SegmentMerger SegmentMerger; typedef struct ShmChunk ShmChunk; typedef struct ShmHeader ShmHeader; typedef struct ShmReader ShmReader; @@ -89,15 +95,17 @@ typedef struct TreeCursor TreeCursor; typedef struct TreeHeader TreeHeader; typedef struct TreeMark TreeMark; typedef struct TreeRoot TreeRoot; +#ifndef _SQLITEINT_H_ typedef unsigned char u8; typedef unsigned short int u16; typedef unsigned int u32; typedef lsm_i64 i64; typedef unsigned long long int u64; +#endif /* A page number is a 64-bit integer. */ typedef i64 Pgno; #ifdef LSM_DEBUG @@ -104,14 +112,15 @@ int lsmErrorBkpt(int); #else # define lsmErrorBkpt(x) (x) #endif -#define LSM_IOERR_BKPT lsmErrorBkpt(LSM_IOERR) -#define LSM_NOMEM_BKPT lsmErrorBkpt(LSM_NOMEM) -#define LSM_CORRUPT_BKPT lsmErrorBkpt(LSM_CORRUPT) -#define LSM_MISUSE_BKPT lsmErrorBkpt(LSM_MISUSE) +#define LSM_PROTOCOL_BKPT lsmErrorBkpt(LSM_PROTOCOL) +#define LSM_IOERR_BKPT lsmErrorBkpt(LSM_IOERR) +#define LSM_NOMEM_BKPT lsmErrorBkpt(LSM_NOMEM) +#define LSM_CORRUPT_BKPT lsmErrorBkpt(LSM_CORRUPT) +#define LSM_MISUSE_BKPT lsmErrorBkpt(LSM_MISUSE) #define unused_parameter(x) (void)(x) #define array_size(x) (sizeof(x)/sizeof(x[0])) @@ -122,24 +131,30 @@ #define LSM_SHM_CHUNK_HDR (sizeof(ShmChunk)) /* The number of available read locks. */ #define LSM_LOCK_NREADER 6 -/* Lock definitions */ +/* The number of available read-write client locks. */ +#define LSM_LOCK_NRWCLIENT 16 + +/* Lock definitions. */ #define LSM_LOCK_DMS1 1 #define LSM_LOCK_DMS2 2 #define LSM_LOCK_WRITER 3 #define LSM_LOCK_WORKER 4 #define LSM_LOCK_CHECKPOINTER 5 #define LSM_LOCK_READER(i) ((i) + LSM_LOCK_CHECKPOINTER + 1) +#define LSM_LOCK_RWCLIENT(i) ((i) + LSM_LOCK_READER(LSM_LOCK_NREADER)) /* ** Hard limit on the number of free-list entries that may be stored in ** a checkpoint (the remainder are stored as a system record in the LSM). ** See also LSM_CONFIG_MAX_FREELIST. */ #define LSM_MAX_FREELIST_ENTRIES 24 + +#define LSM_MAX_BLOCK_REDIRECTS 16 #define LSM_ATTEMPTS_BEFORE_PROTOCOL 10000 /* @@ -183,10 +198,18 @@ struct IntArray { int nAlloc; int nArray; u32 *aArray; }; + +struct Redirect { + int n; /* Number of redirects */ + struct RedirectEntry { + int iFrom; + int iTo; + } *a; +}; /* ** An instance of this structure represents a point in the history of the ** tree structure to roll back to. Refer to comments in lsm_tree.c for ** details. @@ -235,10 +258,11 @@ }; struct TreeRoot { u32 iRoot; u32 nHeight; + u32 nByte; /* Total size of this tree in bytes */ u32 iTransId; }; /* ** Tree header structure. @@ -248,11 +272,10 @@ u32 iNextShmid; /* Shm-id of next chunk allocated */ u32 iFirst; /* Chunk number of smallest shm-id */ u32 nChunk; /* Number of chunks in shared-memory file */ TreeRoot root; /* Root and height of current tree */ u32 iWrite; /* Write offset in shm file */ - u32 nByte; /* Size of current tree structure in bytes */ TreeRoot oldroot; /* Root and height of the previous tree */ u32 iOldShmid; /* Last shm-id used by previous tree */ i64 iOldLog; /* Log offset associated with old tree */ u32 oldcksum0; u32 oldcksum1; @@ -264,18 +287,18 @@ ** Database handle structure. ** ** mLock: ** A bitmask representing the locks currently held by the connection. ** An LSM database supports N distinct locks, where N is some number less -** than or equal to 16. Locks are numbered starting from 1 (see the +** than or equal to 32. Locks are numbered starting from 1 (see the ** definitions for LSM_LOCK_WRITER and co.). ** -** The least significant 16-bits in mLock represent EXCLUSIVE locks. The +** The least significant 32-bits in mLock represent EXCLUSIVE locks. The ** most significant are SHARED locks. So, if a connection holds a SHARED ** lock on lock region iLock, then the following is true: ** -** (mLock & ((iLock+16-1) << 1)) +** (mLock & ((iLock+32-1) << 1)) ** ** Or for an EXCLUSIVE lock: ** ** (mLock & ((iLock-1) << 1)) */ @@ -286,50 +309,58 @@ int (*xCmp)(void *, int, void *, int); /* Compare function */ /* Values configured by calls to lsm_config */ int eSafety; /* LSM_SAFETY_OFF, NORMAL or FULL */ int bAutowork; /* Configured by LSM_CONFIG_AUTOWORK */ - int nTreeLimit; /* Configured by LSM_CONFIG_WRITE_BUFFER */ - int nMerge; /* Configured by LSM_CONFIG_NMERGE */ - int nLogSz; /* Configured by LSM_CONFIG_LOG_SIZE */ + int nTreeLimit; /* Configured by LSM_CONFIG_AUTOFLUSH */ + int nMerge; /* Configured by LSM_CONFIG_AUTOMERGE */ int bUseLog; /* Configured by LSM_CONFIG_USE_LOG */ int nDfltPgsz; /* Configured by LSM_CONFIG_PAGE_SIZE */ int nDfltBlksz; /* Configured by LSM_CONFIG_BLOCK_SIZE */ int nMaxFreelist; /* Configured by LSM_CONFIG_MAX_FREELIST */ int bMmap; /* Configured by LSM_CONFIG_MMAP */ - int nAutockpt; /* Configured by LSM_CONFIG_AUTOCHECKPOINT */ + i64 nAutockpt; /* Configured by LSM_CONFIG_AUTOCHECKPOINT */ int bMultiProc; /* Configured by L_C_MULTIPLE_PROCESSES */ + int bReadonly; /* Configured by LSM_CONFIG_READONLY */ lsm_compress compress; /* Compression callbacks */ + lsm_compress_factory factory; /* Compression callback factory */ /* Sub-system handles */ FileSystem *pFS; /* On-disk portion of database */ Database *pDatabase; /* Database shared data */ + int iRwclient; /* Read-write client lock held (-1 == none) */ + /* Client transaction context */ - Snapshot *pClient; /* Client snapshot (non-NULL in read trans) */ + Snapshot *pClient; /* Client snapshot */ int iReader; /* Read lock held (-1 == unlocked) */ + int bRoTrans; /* True if a read-only db trans is open */ MultiCursor *pCsr; /* List of all open cursors */ LogWriter *pLogWriter; /* Context for writing to the log file */ int nTransOpen; /* Number of opened write transactions */ int nTransAlloc; /* Allocated size of aTrans[] array */ TransMark *aTrans; /* Array of marks for transaction rollback */ IntArray rollback; /* List of tree-nodes to roll back */ + int bDiscardOld; /* True if lsmTreeDiscardOld() was called */ /* Worker context */ Snapshot *pWorker; /* Worker snapshot (or NULL) */ Freelist *pFreelist; /* See sortedNewToplevel() */ int bUseFreelist; /* True to use pFreelist */ + int bIncrMerge; /* True if currently doing a merge */ + + int bInFactory; /* True if within factory.xFactory() */ /* Debugging message callback */ void (*xLog)(void *, int, const char *); void *pLogCtx; /* Work done notification callback */ void (*xWork)(lsm_db *, void *); void *pWorkCtx; - u32 mLock; /* Mask of current locks. See lsmShmLock(). */ + u64 mLock; /* Mask of current locks. See lsmShmLock(). */ lsm_db *pNext; /* Next connection to same database */ int nShm; /* Size of apShm[] array */ void **apShm; /* Shared memory chunks */ ShmHeader *pShmhdr; /* Live shared-memory header */ @@ -340,10 +371,12 @@ struct Segment { Pgno iFirst; /* First page of this run */ Pgno iLastPg; /* Last page of this run */ Pgno iRoot; /* Root page number (if any) */ int nSize; /* Size of this run in pages */ + + Redirect *pRedirect; /* Block redirects (or NULL) */ }; /* ** iSplitTopic/pSplitKey/nSplitKey: ** If nRight>0, this buffer contains a copy of the largest key that has @@ -454,12 +487,18 @@ struct ShmChunk { u32 iShmid; u32 iNext; }; +/* +** Maximum number of shared-memory chunks allowed in the *-shm file. Since +** each shared-memory chunk is 32KB in size, this is a theoretical limit only. +*/ +#define LSM_MAX_SHMCHUNKS (1<<30) + /* Return true if shm-sequence "a" is larger than or equal to "b" */ -#define shm_sequence_ge(a, b) (((u32)a-(u32)b) < (1<<30)) +#define shm_sequence_ge(a, b) (((u32)a-(u32)b) < LSM_MAX_SHMCHUNKS) #define LSM_APPLIST_SZ 4 /* ** An instance of the following structure stores the in-memory part of @@ -498,13 +537,15 @@ ** to read or write a database file on disk. See the description of struct ** Database below for futher details. */ struct Snapshot { Database *pDatabase; /* Database this snapshot belongs to */ + u32 iCmpId; /* Id of compression scheme */ Level *pLevel; /* Pointer to level 0 of snapshot (or NULL) */ i64 iId; /* Snapshot id */ i64 iLogOff; /* Log file offset */ + Redirect redirect; /* Block redirection array */ /* Used by worker snapshots only */ int nBlock; /* Number of blocks in database file */ Pgno aiAppend[LSM_APPLIST_SZ]; /* Append point list */ Freelist freelist; /* Free block list */ @@ -513,11 +554,11 @@ #define LSM_INITIAL_SNAPSHOT_ID 11 /* ** Functions from file "lsm_ckpt.c". */ -int lsmCheckpointWrite(lsm_db *, u32 *); +int lsmCheckpointWrite(lsm_db *, int, u32 *); int lsmCheckpointLevels(lsm_db *, int, void **, int *); int lsmCheckpointLoadLevels(lsm_db *pDb, void *pVal, int nVal); int lsmCheckpointRecover(lsm_db *); int lsmCheckpointDeserialize(lsm_db *, int, u32 *, Snapshot **); @@ -527,10 +568,11 @@ int lsmCheckpointLoad(lsm_db *pDb, int *); int lsmCheckpointLoadOk(lsm_db *pDb, int); int lsmCheckpointClientCacheOk(lsm_db *); +u32 lsmCheckpointNBlock(u32 *); i64 lsmCheckpointId(u32 *, int); u32 lsmCheckpointNWrite(u32 *, int); i64 lsmCheckpointLogOffset(u32 *); int lsmCheckpointPgsz(u32 *); int lsmCheckpointBlksz(u32 *); @@ -539,10 +581,13 @@ int lsmCheckpointSaveWorker(lsm_db *pDb, int); int lsmDatabaseFull(lsm_db *pDb); int lsmCheckpointSynced(lsm_db *pDb, i64 *piId, i64 *piLog, u32 *pnWrite); +int lsmCheckpointSize(lsm_db *db, int *pnByte); + +int lsmInfoCompressionId(lsm_db *db, u32 *piCmpId); /* ** Functions from file "lsm_tree.c". */ int lsmTreeNew(lsm_env *, int (*)(void *, int, void *, int), Tree **ppTree); @@ -558,10 +603,11 @@ int lsmTreeEndTransaction(lsm_db *pDb, int bCommit); int lsmTreeLoadHeader(lsm_db *pDb, int *); int lsmTreeLoadHeaderOk(lsm_db *, int); int lsmTreeInsert(lsm_db *pDb, void *pKey, int nKey, void *pVal, int nVal); +int lsmTreeDelete(lsm_db *db, void *pKey1, int nKey1, void *pKey2, int nKey2); void lsmTreeRollback(lsm_db *pDb, TreeMark *pMark); void lsmTreeMark(lsm_db *pDb, TreeMark *pMark); int lsmTreeCursorNew(lsm_db *pDb, int, TreeCursor **); void lsmTreeCursorDestroy(TreeCursor *); @@ -580,19 +626,10 @@ void lsmFlagsToString(int flags, char *zFlags); /* ** Functions from file "mem.c". */ -int lsmPoolNew(lsm_env *pEnv, Mempool **ppPool); -void lsmPoolDestroy(lsm_env *pEnv, Mempool *pPool); -void *lsmPoolMalloc(lsm_env *pEnv, Mempool *pPool, int nByte); -void *lsmPoolMallocZero(lsm_env *pEnv, Mempool *pPool, int nByte); -int lsmPoolUsed(Mempool *pPool); - -void lsmPoolMark(Mempool *pPool, void **, int *); -void lsmPoolRollback(lsm_env *pEnv, Mempool *pPool, void *, int); - void *lsmMalloc(lsm_env*, size_t); void lsmFree(lsm_env*, void *); void *lsmRealloc(lsm_env*, void *, size_t); void *lsmReallocOrFree(lsm_env*, void *, size_t); void *lsmReallocOrFreeRc(lsm_env *, void *, size_t, int *); @@ -619,12 +656,16 @@ #endif /************************************************************************** ** Start of functions from "lsm_file.c". */ -int lsmFsOpen(lsm_db *, const char *); +int lsmFsOpen(lsm_db *, const char *, int); +int lsmFsOpenLog(lsm_db *, int *); +void lsmFsCloseLog(lsm_db *); void lsmFsClose(FileSystem *); + +int lsmFsConfigure(lsm_db *db); int lsmFsBlockSize(FileSystem *); void lsmFsSetBlockSize(FileSystem *, int); int lsmFsPageSize(FileSystem *); @@ -634,11 +675,11 @@ /* Creating, populating, gobbling and deleting sorted runs. */ void lsmFsGobble(lsm_db *, Segment *, Pgno *, int); int lsmFsSortedDelete(FileSystem *, Snapshot *, int, Segment *); int lsmFsSortedFinish(FileSystem *, Segment *); -int lsmFsSortedAppend(FileSystem *, Snapshot *, Segment *, Page **); +int lsmFsSortedAppend(FileSystem *, Snapshot *, Level *, int, Page **); int lsmFsSortedPadding(FileSystem *, Snapshot *, Segment *); /* Functions to retrieve the lsm_env pointer from a FileSystem or Page object */ lsm_env *lsmFsEnv(FileSystem *); lsm_env *lsmPageEnv(Page *); @@ -648,11 +689,11 @@ void lsmSortedSplitkey(lsm_db *, Level *, int *); /* Reading sorted run content. */ int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg); -int lsmFsDbPageGet(FileSystem *, Pgno, Page **); +int lsmFsDbPageGet(FileSystem *, Segment *, Pgno, Page **); int lsmFsDbPageNext(Segment *, Page *, int eDir, Page **); u8 *lsmFsPageData(Page *, int *); int lsmFsPageRelease(Page *); int lsmFsPagePersist(Page *); @@ -676,32 +717,40 @@ /* Functions to read, write and sync the log file. */ int lsmFsWriteLog(FileSystem *pFS, i64 iOff, LsmString *pStr); int lsmFsSyncLog(FileSystem *pFS); int lsmFsReadLog(FileSystem *pFS, i64 iOff, int nRead, LsmString *pStr); int lsmFsTruncateLog(FileSystem *pFS, i64 nByte); +int lsmFsTruncateDb(FileSystem *pFS, i64 nByte); int lsmFsCloseAndDeleteLog(FileSystem *pFS); void lsmFsDeferClose(FileSystem *pFS, LsmFile **pp); /* And to sync the db file */ -int lsmFsSyncDb(FileSystem *); +int lsmFsSyncDb(FileSystem *, int); + +void lsmFsFlushWaiting(FileSystem *, int *); /* Used by lsm_info(ARRAY_STRUCTURE) and lsm_config(MMAP) */ -int lsmInfoArrayStructure(lsm_db *pDb, Pgno iFirst, char **pzOut); +int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, Pgno iFirst, char **pzOut); int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut); int lsmConfigMmap(lsm_db *pDb, int *piParam); -int lsmEnvOpen(lsm_env *, const char *, lsm_file **); +int lsmEnvOpen(lsm_env *, const char *, int, lsm_file **); int lsmEnvClose(lsm_env *pEnv, lsm_file *pFile); int lsmEnvLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int eLock); int lsmEnvShmMap(lsm_env *, lsm_file *, int, int, void **); void lsmEnvShmBarrier(lsm_env *); void lsmEnvShmUnmap(lsm_env *, lsm_file *, int); void lsmEnvSleep(lsm_env *, int); +int lsmFsReadSyncedId(lsm_db *db, int, i64 *piVal); + +int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, Pgno, int *); +Pgno lsmFsRedirectPage(FileSystem *, Redirect *, Pgno); + /* ** End of functions from "lsm_file.c". **************************************************************************/ /* @@ -709,11 +758,11 @@ */ int lsmInfoPageDump(lsm_db *, Pgno, int, char **); void lsmSortedCleanup(lsm_db *); int lsmSortedAutoWork(lsm_db *, int nUnit); -int lsmSortedWalkFreelist(lsm_db *, int (*)(void *, int, i64), void *); +int lsmSortedWalkFreelist(lsm_db *, int, int (*)(void *, int, i64), void *); int lsmSaveWorker(lsm_db *, int); int lsmFlushTreeToDisk(lsm_db *pDb); @@ -730,11 +779,11 @@ void lsmSortedSaveTreeCursors(lsm_db *); int lsmMCursorNew(lsm_db *, MultiCursor **); void lsmMCursorClose(MultiCursor *); -int lsmMCursorSeek(MultiCursor *, void *, int , int); +int lsmMCursorSeek(MultiCursor *, int, void *, int , int); int lsmMCursorFirst(MultiCursor *); int lsmMCursorPrev(MultiCursor *); int lsmMCursorLast(MultiCursor *); int lsmMCursorValid(MultiCursor *); int lsmMCursorNext(MultiCursor *); @@ -747,13 +796,15 @@ int lsmRestoreCursors(lsm_db *pDb); void lsmSortedDumpStructure(lsm_db *pDb, Snapshot *, int, int, const char *); void lsmFsDumpBlocklists(lsm_db *); +void lsmSortedExpandBtreePage(Page *pPg, int nOrig); void lsmPutU32(u8 *, u32); u32 lsmGetU32(u8 *); +u64 lsmGetU64(u8 *); /* ** Functions from "lsm_varint.c". */ int lsmVarintPut32(u8 *, int); @@ -766,10 +817,11 @@ /* ** Functions from file "main.c". */ void lsmLogMessage(lsm_db *, int, const char *, ...); +int lsmInfoFreelist(lsm_db *pDb, char **pzOut); /* ** Functions from file "lsm_log.c". */ int lsmLogBegin(lsm_db *pDb); @@ -776,14 +828,14 @@ int lsmLogWrite(lsm_db *, void *, int, void *, int); int lsmLogCommit(lsm_db *); void lsmLogEnd(lsm_db *pDb, int bCommit); void lsmLogTell(lsm_db *, LogMark *); void lsmLogSeek(lsm_db *, LogMark *); +void lsmLogClose(lsm_db *); int lsmLogRecover(lsm_db *); int lsmInfoLogStructure(lsm_db *pDb, char **pzVal); -int lsmInfoFreelist(lsm_db *, char **pzVal); /************************************************************************** ** Functions from file "lsm_shared.c". */ @@ -813,11 +865,11 @@ Level *lsmDbSnapshotLevel(Snapshot *); void lsmDbSnapshotSetLevel(Snapshot *, Level *); void lsmDbRecoveryComplete(lsm_db *, int); -int lsmBlockAllocate(lsm_db *, int *); +int lsmBlockAllocate(lsm_db *, int, int *); int lsmBlockFree(lsm_db *, int); int lsmBlockRefree(lsm_db *, int); void lsmFreelistDeltaBegin(lsm_db *); void lsmFreelistDeltaEnd(lsm_db *); @@ -837,11 +889,11 @@ /* Candidate values for the 3rd argument to lsmShmLock() */ #define LSM_LOCK_UNLOCK 0 #define LSM_LOCK_SHARED 1 #define LSM_LOCK_EXCL 2 -int lsmShmChunk(lsm_db *db, int iChunk, void **ppData); +int lsmShmCacheChunks(lsm_db *db, int nChunk); int lsmShmLock(lsm_db *db, int iLock, int eOp, int bBlock); void lsmShmBarrier(lsm_db *db); #ifdef LSM_DEBUG void lsmShmHasLock(lsm_db *db, int iLock, int eOp); @@ -858,11 +910,13 @@ int lsmDbMultiProc(lsm_db *); void lsmDbDeferredClose(lsm_db *, lsm_file *, LsmFile *); LsmFile *lsmDbRecycleFd(lsm_db *); -int lsmWalkFreelist(lsm_db *, int (*)(void *, int, i64), void *); +int lsmWalkFreelist(lsm_db *, int, int (*)(void *, int, i64), void *); + +int lsmCheckCompressionId(lsm_db *, u32); /************************************************************************** ** functions in lsm_str.c */ Index: src/lsm_ckpt.c ================================================================== --- src/lsm_ckpt.c +++ src/lsm_ckpt.c @@ -30,14 +30,16 @@ ** ** 1. The checkpoint id MSW. ** 2. The checkpoint id LSW. ** 3. The number of integer values in the entire checkpoint, including ** the two checksum values. -** 4. The total number of blocks in the database. -** 5. The block size. -** 6. The number of levels. -** 7. The nominal database page size. +** 4. The compression scheme id. +** 5. The total number of blocks in the database. +** 6. The block size. +** 7. The number of levels. +** 8. The nominal database page size. +** 9. The number of pages (in total) written to the database file. ** ** Log pointer: ** ** 1. The log offset MSW. ** 2. The log offset LSW. @@ -64,17 +66,26 @@ ** is 64-bits - 2 integers) ** 5b. Cell number of next cell to read during merge ** 7. Page containing current split-key (64-bits - 2 integers). ** 8. Cell within page containing current split-key. ** 9. Current pointer value (64-bits - 2 integers). +** +** The block redirect array: +** +** 1. Number of redirections (maximum LSM_MAX_BLOCK_REDIRECTS). +** 2. For each redirection: +** a. "from" block number +** b. "to" block number ** ** The in-memory freelist entries. Each entry is either an insert or a ** delete. The in-memory freelist is to the free-block-list as the ** in-memory tree is to the users database content. ** ** 1. Number of free-list entries stored in checkpoint header. -** 2. For each entry: +** 2. Number of free blocks (in total). +** 3. Total number of blocks freed during database lifetime. +** 4. For each entry: ** 2a. Block number of free block. ** 2b. A 64-bit integer (MSW followed by LSW). -1 for a delete entry, ** or the associated checkpoint id for an insert. ** ** The checksum: @@ -162,28 +173,29 @@ static const int one = 1; #define LSM_LITTLE_ENDIAN (*(u8 *)(&one)) /* Sizes, in integers, of various parts of the checkpoint. */ -#define CKPT_HDR_SIZE 8 +#define CKPT_HDR_SIZE 9 #define CKPT_LOGPTR_SIZE 4 #define CKPT_APPENDLIST_SIZE (LSM_APPLIST_SZ * 2) /* A #define to describe each integer in the checkpoint header. */ #define CKPT_HDR_ID_MSW 0 #define CKPT_HDR_ID_LSW 1 #define CKPT_HDR_NCKPT 2 -#define CKPT_HDR_NBLOCK 3 -#define CKPT_HDR_BLKSZ 4 -#define CKPT_HDR_NLEVEL 5 -#define CKPT_HDR_PGSZ 6 -#define CKPT_HDR_NWRITE 7 - -#define CKPT_HDR_LO_MSW 8 -#define CKPT_HDR_LO_LSW 9 -#define CKPT_HDR_LO_CKSUM1 10 -#define CKPT_HDR_LO_CKSUM2 11 +#define CKPT_HDR_CMPID 3 +#define CKPT_HDR_NBLOCK 4 +#define CKPT_HDR_BLKSZ 5 +#define CKPT_HDR_NLEVEL 6 +#define CKPT_HDR_PGSZ 7 +#define CKPT_HDR_NWRITE 8 + +#define CKPT_HDR_LO_MSW 9 +#define CKPT_HDR_LO_LSW 10 +#define CKPT_HDR_LO_CKSUM1 11 +#define CKPT_HDR_LO_CKSUM2 12 typedef struct CkptBuffer CkptBuffer; /* ** Dynamic buffer used to accumulate data for a checkpoint. @@ -414,10 +426,17 @@ iLevel = 0; for(pLevel=lsmDbSnapshotLevel(pSnap); iLevelpNext){ ckptExportLevel(pLevel, &ckpt, &iOut, &rc); iLevel++; } + + /* Write the block-redirect list */ + ckptSetValue(&ckpt, iOut++, pSnap->redirect.n, &rc); + for(i=0; iredirect.n; i++){ + ckptSetValue(&ckpt, iOut++, pSnap->redirect.a[i].iFrom, &rc); + ckptSetValue(&ckpt, iOut++, pSnap->redirect.a[i].iTo, &rc); + } /* Write the freelist */ assert( pSnap->freelist.nEntry<=pDb->nMaxFreelist ); if( rc==LSM_OK ){ int nFree = pSnap->freelist.nEntry; @@ -430,13 +449,17 @@ } } /* Write the checkpoint header */ assert( iId>=0 ); + assert( pSnap->iCmpId==pDb->compress.iId + || pSnap->iCmpId==LSM_COMPRESSION_EMPTY + ); ckptSetValue(&ckpt, CKPT_HDR_ID_MSW, (u32)(iId>>32), &rc); ckptSetValue(&ckpt, CKPT_HDR_ID_LSW, (u32)(iId&0xFFFFFFFF), &rc); ckptSetValue(&ckpt, CKPT_HDR_NCKPT, iOut+2, &rc); + ckptSetValue(&ckpt, CKPT_HDR_CMPID, pDb->compress.iId, &rc); ckptSetValue(&ckpt, CKPT_HDR_NBLOCK, pSnap->nBlock, &rc); ckptSetValue(&ckpt, CKPT_HDR_BLKSZ, lsmFsBlockSize(pFS), &rc); ckptSetValue(&ckpt, CKPT_HDR_NLEVEL, nLevel, &rc); ckptSetValue(&ckpt, CKPT_HDR_PGSZ, lsmFsPageSize(pFS), &rc); ckptSetValue(&ckpt, CKPT_HDR_NWRITE, pSnap->nWrite, &rc); @@ -742,23 +765,24 @@ ** Initialize the shared-memory header with an empty snapshot. This function ** is called when no valid snapshot can be found in the database header. */ static void ckptLoadEmpty(lsm_db *pDb){ u32 aCkpt[] = { - 0, /* CKPT_HDR_ID_MSW */ - 10, /* CKPT_HDR_ID_LSW */ - 0, /* CKPT_HDR_NCKPT */ - 0, /* CKPT_HDR_NBLOCK */ - 0, /* CKPT_HDR_BLKSZ */ - 0, /* CKPT_HDR_NLEVEL */ - 0, /* CKPT_HDR_PGSZ */ - 0, /* CKPT_HDR_OVFL */ - 0, /* CKPT_HDR_NWRITE */ - 0, 0, 1234, 5678, /* The log pointer and initial checksum */ - 0,0,0,0, 0,0,0,0, /* The append list */ - 0, /* The free block list */ - 0, 0 /* Space for checksum values */ + 0, /* CKPT_HDR_ID_MSW */ + 10, /* CKPT_HDR_ID_LSW */ + 0, /* CKPT_HDR_NCKPT */ + LSM_COMPRESSION_EMPTY, /* CKPT_HDR_CMPID */ + 0, /* CKPT_HDR_NBLOCK */ + 0, /* CKPT_HDR_BLKSZ */ + 0, /* CKPT_HDR_NLEVEL */ + 0, /* CKPT_HDR_PGSZ */ + 0, /* CKPT_HDR_OVFL */ + 0, /* CKPT_HDR_NWRITE */ + 0, 0, 1234, 5678, /* The log pointer and initial checksum */ + 0,0,0,0, 0,0,0,0, /* The append list */ + 0, /* The free block list */ + 0, 0 /* Space for checksum values */ }; u32 nCkpt = array_size(aCkpt); ShmHeader *pShm = pDb->pShmhdr; aCkpt[CKPT_HDR_NCKPT] = nCkpt; @@ -857,11 +881,23 @@ } } lsmShmBarrier(pDb); } - return LSM_PROTOCOL; + return LSM_PROTOCOL_BKPT; +} + +int lsmInfoCompressionId(lsm_db *db, u32 *piCmpId){ + int rc; + + assert( db->pClient==0 && db->pWorker==0 ); + rc = lsmCheckpointLoad(db, 0); + if( rc==LSM_OK ){ + *piCmpId = db->aSnapshot[CKPT_HDR_CMPID]; + } + + return rc; } int lsmCheckpointLoadOk(lsm_db *pDb, int iSnap){ u32 *aShm; assert( iSnap==1 || iSnap==2 ); @@ -881,12 +917,15 @@ int rc; ShmHeader *pShm = pDb->pShmhdr; int nInt1; int nInt2; - /* Must be holding the WORKER lock to do this */ - assert( lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL) ); + /* Must be holding the WORKER lock to do this. Or DMS2. */ + assert( + lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL) + || lsmShmAssertLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_EXCL) + ); /* Check that the two snapshots match. If not, repair them. */ nInt1 = pShm->aSnap1[CKPT_HDR_NCKPT]; nInt2 = pShm->aSnap2[CKPT_HDR_NCKPT]; if( nInt1!=nInt2 || memcmp(pShm->aSnap1, pShm->aSnap2, nInt2*sizeof(u32)) ){ @@ -893,16 +932,24 @@ if( ckptChecksumOk(pShm->aSnap1) ){ memcpy(pShm->aSnap2, pShm->aSnap1, sizeof(u32)*nInt1); }else if( ckptChecksumOk(pShm->aSnap2) ){ memcpy(pShm->aSnap1, pShm->aSnap2, sizeof(u32)*nInt2); }else{ - return LSM_PROTOCOL; + return LSM_PROTOCOL_BKPT; } } rc = lsmCheckpointDeserialize(pDb, 1, pShm->aSnap1, &pDb->pWorker); + if( pDb->pWorker ) pDb->pWorker->pDatabase = pDb->pDatabase; + + if( rc==LSM_OK ){ + rc = lsmCheckCompressionId(pDb, pDb->pWorker->iCmpId); + } + +#if 0 assert( rc!=LSM_OK || lsmFsIntegrityCheck(pDb) ); +#endif return rc; } int lsmCheckpointDeserialize( lsm_db *pDb, @@ -913,10 +960,11 @@ int rc = LSM_OK; Snapshot *pNew; pNew = (Snapshot *)lsmMallocZeroRc(pDb->pEnv, sizeof(Snapshot), &rc); if( rc==LSM_OK ){ + Level *pLvl; int nFree; int i; int nLevel = (int)aCkpt[CKPT_HDR_NLEVEL]; int iIn = CKPT_HDR_SIZE + CKPT_APPENDLIST_SIZE + CKPT_LOGPTR_SIZE; @@ -923,19 +971,40 @@ pNew->iId = lsmCheckpointId(aCkpt, 0); pNew->nBlock = aCkpt[CKPT_HDR_NBLOCK]; pNew->nWrite = aCkpt[CKPT_HDR_NWRITE]; rc = ckptLoadLevels(pDb, aCkpt, &iIn, nLevel, &pNew->pLevel); pNew->iLogOff = lsmCheckpointLogOffset(aCkpt); + pNew->iCmpId = aCkpt[CKPT_HDR_CMPID]; /* Make a copy of the append-list */ for(i=0; iaiAppend[i] = ckptRead64(a); } + + /* Read the block-redirect list */ + pNew->redirect.n = aCkpt[iIn++]; + if( pNew->redirect.n ){ + pNew->redirect.a = lsmMallocZeroRc(pDb->pEnv, + (sizeof(struct RedirectEntry) * LSM_MAX_BLOCK_REDIRECTS), &rc + ); + if( rc==LSM_OK ){ + for(i=0; iredirect.n; i++){ + pNew->redirect.a[i].iFrom = aCkpt[iIn++]; + pNew->redirect.a[i].iTo = aCkpt[iIn++]; + } + } + for(pLvl=pNew->pLevel; pLvl->pNext; pLvl=pLvl->pNext); + if( pLvl->nRight ){ + pLvl->aRhs[pLvl->nRight-1].pRedirect = &pNew->redirect; + }else{ + pLvl->lhs.pRedirect = &pNew->redirect; + } + } /* Copy the free-list */ - if( bInclFreelist ){ + if( rc==LSM_OK && bInclFreelist ){ nFree = aCkpt[iIn++]; if( nFree ){ pNew->freelist.aEntry = (FreelistEntry *)lsmMallocZeroRc( pDb->pEnv, sizeof(FreelistEntry)*nFree, &rc ); @@ -1003,11 +1072,12 @@ ShmHeader *pShm = pDb->pShmhdr; void *p = 0; int n = 0; int rc; - rc = ckptExportSnapshot(pDb, bFlush, pSnap->iId+1, 1, &p, &n); + pSnap->iId++; + rc = ckptExportSnapshot(pDb, bFlush, pSnap->iId, 1, &p, &n); if( rc!=LSM_OK ) return rc; assert( ckptChecksumOk((u32 *)p) ); assert( n<=LSM_META_PAGE_SIZE ); memcpy(pShm->aSnap2, p, n); @@ -1086,10 +1156,14 @@ }else{ iId = ((i64)aCkpt[CKPT_HDR_ID_MSW] << 32) + (i64)aCkpt[CKPT_HDR_ID_LSW]; } return iId; } + +u32 lsmCheckpointNBlock(u32 *aCkpt){ + return aCkpt[CKPT_HDR_NBLOCK]; +} u32 lsmCheckpointNWrite(u32 *aCkpt, int bDisk){ if( bDisk ){ return lsmGetU32((u8 *)&aCkpt[CKPT_HDR_NWRITE]); }else{ @@ -1134,20 +1208,27 @@ memcpy(pDb->pShmhdr->aSnap1, pDb->aSnapshot, nCkpt*sizeof(u32)); memcpy(pDb->pShmhdr->aSnap2, pDb->aSnapshot, nCkpt*sizeof(u32)); } -int lsm_ckpt_size(lsm_db *db, int *pnByte){ +/* +** Set the output variable to the number of KB of data written into the +** database file since the most recent checkpoint. +*/ +int lsmCheckpointSize(lsm_db *db, int *pnKB){ ShmHeader *pShm = db->pShmhdr; int rc = LSM_OK; u32 nSynced; + /* Set nSynced to the number of pages that had been written when the + ** database was last checkpointed. */ rc = lsmCheckpointSynced(db, 0, 0, &nSynced); + if( rc==LSM_OK ){ u32 nPgsz = db->pShmhdr->aSnap1[CKPT_HDR_PGSZ]; u32 nWrite = db->pShmhdr->aSnap1[CKPT_HDR_NWRITE]; - *pnByte = (int)((nWrite - nSynced) * nPgsz); + *pnKB = (int)(( ((i64)(nWrite - nSynced) * nPgsz) + 1023) / 1024); } return rc; } Index: src/lsm_file.c ================================================================== --- src/lsm_file.c +++ src/lsm_file.c @@ -167,11 +167,12 @@ ** are carrying pointers into the database file mapping (pMap/nMap). If the ** file has to be unmapped and then remapped (required to grow the mapping ** as the file grows), the Page.aData pointers are updated by iterating ** through the contents of this list. ** -** In non-mmap() mode, this list is an LRU list of cached pages with nRef==0. +** In non-mmap() mode, this list is an LRU list of cached pages with +** nRef==0. */ struct FileSystem { lsm_db *pDb; /* Database handle that owns this object */ lsm_env *pEnv; /* Environment pointer */ char *zDb; /* Database file name */ @@ -179,11 +180,11 @@ int nMetasize; /* Size of meta pages in bytes */ int nPagesize; /* Database page-size in bytes */ int nBlocksize; /* Database block-size in bytes */ /* r/w file descriptors for both files. */ - LsmFile *pLsmFile; + LsmFile *pLsmFile; /* Used after lsm_close() to link into list */ lsm_file *fdDb; /* Database file */ lsm_file *fdLog; /* Log file */ int szSector; /* Database file sector size */ /* If this is a compressed database, a pointer to the compression methods. @@ -196,10 +197,12 @@ /* mmap() mode things */ int bUseMmap; /* True to use mmap() to access db file */ void *pMap; /* Current mapping of database file */ i64 nMap; /* Bytes mapped at pMap */ Page *pFree; + + Page *pWaiting; /* b-tree pages waiting to be written */ /* Statistics */ int nWrite; /* Total number of pages written */ int nRead; /* Total number of pages read */ @@ -239,10 +242,13 @@ /* Only used in compressed database mode: */ int nCompress; /* Compressed size (or 0 for uncomp. db) */ int nCompressPrev; /* Compressed size of prev page */ Segment *pSeg; /* Segment this page will be written to */ + + /* Fix this up somehow */ + Page *pNextWaiting; }; /* ** Meta-data page handle. There are two meta-data pages at the start of ** the database file, each FileSystem.nMetasize bytes in size. @@ -295,12 +301,12 @@ ** lsmEnvClose() ** lsmEnvTruncate() ** lsmEnvUnlink() ** lsmEnvRemap() */ -int lsmEnvOpen(lsm_env *pEnv, const char *zFile, lsm_file **ppNew){ - return pEnv->xOpen(pEnv, zFile, ppNew); +int lsmEnvOpen(lsm_env *pEnv, const char *zFile, int flags, lsm_file **ppNew){ + return pEnv->xOpen(pEnv, zFile, flags, ppNew); } static int lsmEnvRead( lsm_env *pEnv, lsm_file *pFile, lsm_i64 iOff, @@ -408,10 +414,18 @@ */ int lsmFsTruncateLog(FileSystem *pFS, i64 nByte){ if( pFS->fdLog==0 ) return LSM_OK; return lsmEnvTruncate(pFS->pEnv, pFS->fdLog, nByte); } + +/* +** Truncate the log file to nByte bytes in size. +*/ +int lsmFsTruncateDb(FileSystem *pFS, i64 nByte){ + if( pFS->fdDb==0 ) return LSM_OK; + return lsmEnvTruncate(pFS->pEnv, pFS->fdDb, nByte); +} /* ** Close the log file. Then delete it from the file-system. This function ** is called during database shutdown only. */ @@ -443,16 +457,20 @@ ** This is a helper function for lsmFsOpen(). It opens a single file on ** disk (either the database or log file). */ static lsm_file *fsOpenFile( FileSystem *pFS, /* File system object */ + int bReadonly, /* True to open this file read-only */ int bLog, /* True for log, false for db */ int *pRc /* IN/OUT: Error code */ ){ lsm_file *pFile = 0; if( *pRc==LSM_OK ){ - *pRc = lsmEnvOpen(pFS->pEnv, (bLog ? pFS->zLog : pFS->zDb), &pFile); + int flags = (bReadonly ? LSM_OPEN_READONLY : 0); + const char *zPath = (bLog ? pFS->zLog : pFS->zDb); + + *pRc = lsmEnvOpen(pFS->pEnv, zPath, flags, &pFile); } return pFile; } /* @@ -464,21 +482,51 @@ ** ** lsmFsWriteLog ** lsmFsSyncLog ** lsmFsReadLog */ -int lsmFsOpenLog(FileSystem *pFS){ +int lsmFsOpenLog(lsm_db *db, int *pbOpen){ int rc = LSM_OK; - if( 0==pFS->fdLog ){ pFS->fdLog = fsOpenFile(pFS, 1, &rc); } + FileSystem *pFS = db->pFS; + + if( 0==pFS->fdLog ){ + pFS->fdLog = fsOpenFile(pFS, db->bReadonly, 1, &rc); + + if( rc==LSM_IOERR_NOENT && db->bReadonly ){ + rc = LSM_OK; + } + } + + if( pbOpen ) *pbOpen = (pFS->fdLog!=0); return rc; } + +void lsmFsCloseLog(lsm_db *db){ + FileSystem *pFS = db->pFS; + if( pFS->fdLog ){ + lsmEnvClose(pFS->pEnv, pFS->fdLog); + pFS->fdLog = 0; + } +} /* ** Open a connection to a database stored within the file-system (the ** "system of files"). +** +** If parameter bReadonly is true, then open a read-only file-descriptor +** on the database file. It is possible that bReadonly will be false even +** if the user requested that pDb be opened read-only. This is because the +** file-descriptor may later on be recycled by a read-write connection. +** If the db file can be opened for read-write access, it always is. Parameter +** bReadonly is only ever true if it has already been determined that the +** db can only be opened for read-only access. */ -int lsmFsOpen(lsm_db *pDb, const char *zDb){ +int lsmFsOpen( + lsm_db *pDb, /* Database connection to open fd for */ + const char *zDb, /* Full path to database file */ + int bReadonly /* True to open db file read-only */ +){ FileSystem *pFS; int rc = LSM_OK; int nDb = strlen(zDb); int nByte; @@ -494,15 +542,10 @@ pFS->nPagesize = LSM_DFLT_PAGE_SIZE; pFS->nBlocksize = LSM_DFLT_BLOCK_SIZE; pFS->nMetasize = 4 * 1024; pFS->pDb = pDb; pFS->pEnv = pDb->pEnv; - if( pDb->compress.xCompress ){ - pFS->pCompress = &pDb->compress; - }else{ - pFS->bUseMmap = pDb->bMmap; - } /* Make a copy of the database and log file names. */ memcpy(pFS->zDb, zDb, nDb+1); memcpy(pFS->zLog, zDb, nDb); memcpy(&pFS->zLog[nDb], "-log", 5); @@ -520,11 +563,11 @@ pFS->fdDb = pLsmFile->pFile; memset(pLsmFile, 0, sizeof(LsmFile)); }else{ pFS->pLsmFile = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmFile), &rc); if( rc==LSM_OK ){ - pFS->fdDb = fsOpenFile(pFS, 0, &rc); + pFS->fdDb = fsOpenFile(pFS, bReadonly, 0, &rc); } } if( rc!=LSM_OK ){ lsmFsClose(pFS); @@ -535,10 +578,62 @@ } pDb->pFS = pFS; return rc; } + +/* +** Configure the file-system object according to the current values of +** the LSM_CONFIG_MMAP and LSM_CONFIG_SET_COMPRESSION options. +*/ +int lsmFsConfigure(lsm_db *db){ + FileSystem *pFS = db->pFS; + if( pFS ){ + lsm_env *pEnv = pFS->pEnv; + Page *pPg; + + assert( pFS->nOut==0 ); + assert( pFS->pWaiting==0 ); + + /* Reset any compression/decompression buffers already allocated */ + lsmFree(pEnv, pFS->aIBuffer); + lsmFree(pEnv, pFS->aOBuffer); + pFS->nBuffer = 0; + + /* Unmap the file, if it is currently mapped */ + if( pFS->pMap ){ + lsmEnvRemap(pEnv, pFS->fdDb, -1, &pFS->pMap, &pFS->nMap); + pFS->bUseMmap = 0; + } + + /* Free all allocate page structures */ + pPg = pFS->pLruFirst; + while( pPg ){ + Page *pNext = pPg->pLruNext; + if( pPg->flags & PAGE_FREE ) lsmFree(pEnv, pPg->aData); + lsmFree(pEnv, pPg); + pPg = pNext; + } + + /* Zero pointers that point to deleted page objects */ + pFS->nCacheAlloc = 0; + pFS->pLruFirst = 0; + pFS->pLruLast = 0; + pFS->pFree = 0; + + /* Configure the FileSystem object */ + if( db->compress.xCompress ){ + pFS->pCompress = &db->compress; + pFS->bUseMmap = 0; + }else{ + pFS->pCompress = 0; + pFS->bUseMmap = db->bMmap; + } + } + + return LSM_OK; +} /* ** Close and destroy a FileSystem object. */ void lsmFsClose(FileSystem *pFS){ @@ -728,11 +823,11 @@ /* ** Return the page number of a page. */ Pgno lsmFsPageNumber(Page *pPage){ - assert( (pPage->flags & PAGE_DIRTY)==0 ); + /* assert( (pPage->flags & PAGE_DIRTY)==0 ); */ return pPage ? pPage->iPg : 0; } /* ** Page pPg is currently part of the LRU list belonging to pFS. Remove @@ -849,44 +944,108 @@ } *pRc = rc; } } -static int fsPageGet(FileSystem *, Pgno, int, Page **, int *); + +/* +** fsync() the database file. +*/ +int lsmFsSyncDb(FileSystem *pFS, int nBlock){ + if( nBlock && pFS->bUseMmap ){ + int rc = LSM_OK; + i64 nMin = (i64)nBlock * (i64)pFS->nBlocksize; + fsGrowMapping(pFS, nMin, &rc); + if( rc!=LSM_OK ) return rc; + } + return lsmEnvSync(pFS->pEnv, pFS->fdDb); +} + +static int fsPageGet(FileSystem *, Segment *, Pgno, int, Page **, int *); + +static int fsRedirectBlock(Redirect *p, int iBlk){ + if( p ){ + int i; + for(i=0; in; i++){ + if( iBlk==p->a[i].iFrom ) return p->a[i].iTo; + } + } + assert( iBlk!=0 ); + return iBlk; +} + +Pgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, Pgno iPg){ + Pgno iReal = iPg; + + if( pRedir ){ + const int nPagePerBlock = ( + pFS->pCompress ? pFS->nBlocksize : (pFS->nBlocksize / pFS->nPagesize) + ); + int iBlk = fsPageToBlock(pFS, iPg); + int i; + for(i=0; in; i++){ + int iFrom = pRedir->a[i].iFrom; + if( iFrom>iBlk ) break; + if( iFrom==iBlk ){ + int iTo = pRedir->a[i].iTo; + iReal = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock; + if( iTo==1 ){ + iReal += (fsFirstPageOnBlock(pFS, 1)-1); + } + break; + } + } + } + + assert( iReal!=0 ); + return iReal; +} /* ** Parameter iBlock is a database file block. This function reads the value ** stored in the blocks "next block" pointer and stores it in *piNext. ** LSM_OK is returned if everything is successful, or an LSM error code ** otherwise. */ static int fsBlockNext( FileSystem *pFS, /* File-system object handle */ + Segment *pSeg, /* Use this segment for block redirects */ int iBlock, /* Read field from this block */ int *piNext /* OUT: Next block in linked list */ ){ int rc; + int iRead; /* Read block from here */ + + if( pSeg ){ + iRead = fsRedirectBlock(pSeg->pRedirect, iBlock); + }else{ + iRead = iBlock; + } assert( pFS->bUseMmap==0 || pFS->pCompress==0 ); if( pFS->pCompress ){ i64 iOff; /* File offset to read data from */ u8 aNext[4]; /* 4-byte pointer read from db file */ - iOff = (i64)iBlock * pFS->nBlocksize - sizeof(aNext); + iOff = (i64)iRead * pFS->nBlocksize - sizeof(aNext); rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aNext, sizeof(aNext)); if( rc==LSM_OK ){ *piNext = (int)lsmGetU32(aNext); } }else{ const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); Page *pLast; - rc = fsPageGet(pFS, iBlock*nPagePerBlock, 0, &pLast, 0); + rc = fsPageGet(pFS, 0, iRead*nPagePerBlock, 0, &pLast, 0); if( rc==LSM_OK ){ *piNext = lsmGetU32(&pLast->aData[pFS->nPagesize-4]); lsmFsPageRelease(pLast); } } + + if( pSeg ){ + *piNext = fsRedirectBlock(pSeg->pRedirect, *piNext); + } return rc; } /* ** Return the page number of the last page on the same block as page iPg. @@ -898,10 +1057,11 @@ /* ** This function is only called in compressed database mode. */ static int fsReadData( FileSystem *pFS, /* File-system handle */ + Segment *pSeg, /* Block redirection */ i64 iOff, /* Read data from this offset */ u8 *aData, /* Buffer to read data into */ int nData /* Number of bytes to read */ ){ i64 iEob; /* End of block */ @@ -915,11 +1075,11 @@ rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aData, nRead); if( rc==LSM_OK && nRead!=nData ){ int iBlk; - rc = fsBlockNext(pFS, fsPageToBlock(pFS, iOff), &iBlk); + rc = fsBlockNext(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); if( rc==LSM_OK ){ i64 iOff2 = fsFirstPageOnBlock(pFS, iBlk); rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff2, &aData[nRead], nData-nRead); } } @@ -933,10 +1093,11 @@ ** LSM_OK is returned if everything is successful, or an LSM error code ** otherwise. */ static int fsBlockPrev( FileSystem *pFS, /* File-system object handle */ + Segment *pSeg, /* Use this segment for block redirects */ int iBlock, /* Read field from this block */ int *piPrev /* OUT: Previous block in linked list */ ){ int rc = LSM_OK; /* Return code */ @@ -946,11 +1107,12 @@ if( pFS->pCompress ){ i64 iOff = fsFirstPageOnBlock(pFS, iBlock) - 4; u8 aPrev[4]; /* 4-byte pointer read from db file */ rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aPrev, sizeof(aPrev)); if( rc==LSM_OK ){ - *piPrev = (int)lsmGetU32(aPrev); + Redirect *pRedir = (pSeg ? pSeg->pRedirect : 0); + *piPrev = fsRedirectBlock(pRedir, (int)lsmGetU32(aPrev)); } }else{ assert( 0 ); } return rc; @@ -971,13 +1133,19 @@ nByte += (aBuf[2] & 0x7F); *pbFree = !(aBuf[1] & 0x80); return nByte; } -static int fsSubtractOffset(FileSystem *pFS, i64 iOff, int iSub, i64 *piRes){ +static int fsSubtractOffset( + FileSystem *pFS, + Segment *pSeg, + i64 iOff, + int iSub, + i64 *piRes +){ i64 iStart; - int iBlk; + int iBlk = 0; int rc; assert( pFS->pCompress ); iStart = fsFirstPageOnBlock(pFS, fsPageToBlock(pFS, iOff)); @@ -984,16 +1152,22 @@ if( (iOff-iSub)>=iStart ){ *piRes = (iOff-iSub); return LSM_OK; } - rc = fsBlockPrev(pFS, fsPageToBlock(pFS, iOff), &iBlk); + rc = fsBlockPrev(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); *piRes = fsLastPageOnBlock(pFS, iBlk) - iSub + (iOff - iStart + 1); return rc; } -static int fsAddOffset(FileSystem *pFS, i64 iOff, int iAdd, i64 *piRes){ +static int fsAddOffset( + FileSystem *pFS, + Segment *pSeg, + i64 iOff, + int iAdd, + i64 *piRes +){ i64 iEob; int iBlk; int rc; assert( pFS->pCompress ); @@ -1002,11 +1176,11 @@ if( (iOff+iAdd)<=iEob ){ *piRes = (iOff+iAdd); return LSM_OK; } - rc = fsBlockNext(pFS, fsPageToBlock(pFS, iOff), &iBlk); + rc = fsBlockNext(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); *piRes = fsFirstPageOnBlock(pFS, iBlk) + iAdd - (iEob - iOff + 1); return rc; } static int fsAllocateBuffer(FileSystem *pFS, int bWrite){ @@ -1040,10 +1214,11 @@ ** ** LSM_OK is returned if successful, or an LSM error code otherwise. */ static int fsReadPagedata( FileSystem *pFS, /* File-system handle */ + Segment *pSeg, /* pPg is part of this segment */ Page *pPg, /* Page to read and uncompress data for */ int *pnSpace /* OUT: Total bytes of free space */ ){ lsm_compress *p = pFS->pCompress; i64 iOff = pPg->iPg; @@ -1052,11 +1227,11 @@ assert( p && pPg->nCompress==0 ); if( fsAllocateBuffer(pFS, 0) ) return LSM_NOMEM; - rc = fsReadData(pFS, iOff, aSz, sizeof(aSz)); + rc = fsReadData(pFS, pSeg, iOff, aSz, sizeof(aSz)); if( rc==LSM_OK ){ int bFree; if( aSz[0] & 0x80 ){ pPg->nCompress = (int)getRecordSize(aSz, &bFree); @@ -1069,16 +1244,16 @@ *pnSpace = pPg->nCompress + sizeof(aSz)*2; }else{ rc = LSM_CORRUPT_BKPT; } }else{ - rc = fsAddOffset(pFS, iOff, 3, &iOff); + rc = fsAddOffset(pFS, pSeg, iOff, 3, &iOff); if( rc==LSM_OK ){ if( pPg->nCompress>pFS->nBuffer ){ rc = LSM_CORRUPT_BKPT; }else{ - rc = fsReadData(pFS, iOff, pFS->aIBuffer, pPg->nCompress); + rc = fsReadData(pFS, pSeg, iOff, pFS->aIBuffer, pPg->nCompress); } if( rc==LSM_OK ){ int n = pFS->nPagesize; rc = p->xUncompress(p->pCtx, (char *)pPg->aData, &n, @@ -1097,28 +1272,46 @@ /* ** Return a handle for a database page. */ static int fsPageGet( FileSystem *pFS, /* File-system handle */ + Segment *pSeg, /* Block redirection to use (or NULL) */ Pgno iPg, /* Page id */ int noContent, /* True to not load content from disk */ Page **ppPg, /* OUT: New page handle */ int *pnSpace /* OUT: Bytes of free space */ ){ Page *p; int iHash; int rc = LSM_OK; + /* In most cases iReal is the same as iPg. Except, if pSeg->pRedirect is + ** not NULL, and the block containing iPg has been redirected, then iReal + ** is the page number after redirection. */ + Pgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg); + assert( iPg>=fsFirstPageOnBlock(pFS, 1) ); + assert( iReal>=fsFirstPageOnBlock(pFS, 1) ); *ppPg = 0; assert( pFS->bUseMmap==0 || pFS->pCompress==0 ); if( pFS->bUseMmap ){ - i64 iEnd = (i64)iPg * pFS->nPagesize; + Page *pTest; + i64 iEnd = (i64)iReal * pFS->nPagesize; fsGrowMapping(pFS, iEnd, &rc); if( rc!=LSM_OK ) return rc; + p = 0; + for(pTest=pFS->pWaiting; pTest; pTest=pTest->pNextWaiting){ + if( pTest->iPg==iReal ){ + assert( iReal==iPg ); + p = pTest; + p->nRef++; + *ppPg = p; + return LSM_OK; + } + } if( pFS->pFree ){ p = pFS->pFree; pFS->pFree = p->pHashNext; assert( p->nRef==0 ); }else{ @@ -1125,25 +1318,25 @@ p = lsmMallocZeroRc(pFS->pEnv, sizeof(Page), &rc); if( rc ) return rc; fsPageAddToLru(pFS, p); p->pFS = pFS; } - p->aData = &((u8 *)pFS->pMap)[pFS->nPagesize * (i64)(iPg-1)]; - p->iPg = iPg; + p->aData = &((u8 *)pFS->pMap)[pFS->nPagesize * (iReal-1)]; + p->iPg = iReal; }else{ /* Search the hash-table for the page */ - iHash = fsHashKey(pFS->nHash, iPg); + iHash = fsHashKey(pFS->nHash, iReal); for(p=pFS->apHash[iHash]; p; p=p->pHashNext){ - if( p->iPg==iPg) break; + if( p->iPg==iReal) break; } if( p==0 ){ rc = fsPageBuffer(pFS, 1, &p); if( rc==LSM_OK ){ int nSpace = 0; - p->iPg = iPg; + p->iPg = iReal; p->nRef = 0; p->pFS = pFS; assert( p->flags==0 || p->flags==PAGE_FREE ); #ifdef LSM_DEBUG @@ -1150,14 +1343,14 @@ memset(p->aData, 0x56, pFS->nPagesize); #endif assert( p->pLruNext==0 && p->pLruPrev==0 ); if( noContent==0 ){ if( pFS->pCompress ){ - rc = fsReadPagedata(pFS, p, &nSpace); + rc = fsReadPagedata(pFS, pSeg, p, &nSpace); }else{ int nByte = pFS->nPagesize; - i64 iOff = (i64)(iPg-1) * pFS->nPagesize; + i64 iOff = (i64)(iReal-1) * pFS->nPagesize; rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, p->aData, nByte); } pFS->nRead++; } @@ -1181,13 +1374,13 @@ || (rc!=LSM_OK && p==0) ); } if( rc==LSM_OK && p ){ - if( pFS->pCompress==0 && (fsIsLast(pFS, iPg) || fsIsFirst(pFS, iPg)) ){ + if( pFS->pCompress==0 && (fsIsLast(pFS, iReal) || fsIsFirst(pFS, iReal)) ){ p->nData = pFS->nPagesize - 4; - if( fsIsFirst(pFS, iPg) && p->nRef==0 ){ + if( fsIsFirst(pFS, iReal) && p->nRef==0 ){ p->aData += 4; p->flags |= PAGE_HASPREV; } }else{ p->nData = pFS->nPagesize; @@ -1194,10 +1387,42 @@ } pFS->nOut += (p->nRef==0); p->nRef++; } *ppPg = p; + return rc; +} + +/* +** Read the 64-bit checkpoint id of the checkpoint currently stored on meta +** page iMeta of the database file. If no error occurs, store the id value +** in *piVal and return LSM_OK. Otherwise, return an LSM error code and leave +** *piVal unmodified. +** +** If a checkpointer connection is currently updating meta-page iMeta, or an +** earlier checkpointer crashed while doing so, the value read into *piVal +** may be garbage. It is the callers responsibility to deal with this. +*/ +int lsmFsReadSyncedId(lsm_db *db, int iMeta, i64 *piVal){ + FileSystem *pFS = db->pFS; + int rc = LSM_OK; + + assert( iMeta==1 || iMeta==2 ); + if( pFS->bUseMmap ){ + fsGrowMapping(pFS, iMeta*LSM_META_PAGE_SIZE, &rc); + if( rc==LSM_OK ){ + *piVal = (i64)lsmGetU64(&((u8 *)pFS->pMap)[(iMeta-1)*LSM_META_PAGE_SIZE]); + } + }else{ + MetaPage *pMeta = 0; + rc = lsmFsMetaPageGet(pFS, 0, iMeta, &pMeta); + if( rc==LSM_OK ){ + *piVal = (i64)lsmGetU64(pMeta->aData); + lsmFsMetaPageRelease(pMeta); + } + } + return rc; } static int fsRunEndsBetween( @@ -1291,17 +1516,22 @@ /* Mark all blocks currently used by this sorted run as free */ while( iBlk && rc==LSM_OK ){ int iNext = 0; if( iBlk!=iLastBlk ){ - rc = fsBlockNext(pFS, iBlk, &iNext); + rc = fsBlockNext(pFS, pDel, iBlk, &iNext); }else if( bZero==0 && pDel->iLastPg!=fsLastPageOnBlock(pFS, iLastBlk) ){ break; } rc = fsFreeBlock(pFS, pSnapshot, pDel, iBlk); iBlk = iNext; } + + if( pDel->pRedirect ){ + assert( pDel->pRedirect==&pSnapshot->redirect ); + pSnapshot->redirect.n = 0; + } if( bZero ) memset(pDel, 0, sizeof(Segment)); } return LSM_OK; } @@ -1316,10 +1546,32 @@ } } return iRet; } +#ifndef NDEBUG +/* +** Return true if page iPg, which is a part of segment p, lies on +** a redirected block. +*/ +static int fsPageRedirects(FileSystem *pFS, Segment *p, Pgno iPg){ + return (iPg!=0 && iPg!=lsmFsRedirectPage(pFS, p->pRedirect, iPg)); +} + +/* +** Return true if the second argument is not NULL and any of the first +** last or root pages lie on a redirected block. +*/ +static int fsSegmentRedirects(FileSystem *pFS, Segment *p){ + return (p && ( + fsPageRedirects(pFS, p, p->iFirst) + || fsPageRedirects(pFS, p, p->iRoot) + || fsPageRedirects(pFS, p, p->iLastPg) + )); +} +#endif + /* ** Argument aPgno is an array of nPgno page numbers. All pages belong to ** the segment pRun. This function gobbles from the start of the run to the ** first page that appears in aPgno[] (i.e. so that the aPgno[] entry is ** the new first page of the run). @@ -1334,10 +1586,13 @@ FileSystem *pFS = pDb->pFS; Snapshot *pSnapshot = pDb->pWorker; int iBlk; assert( pRun->nSize>0 ); + assert( 0==fsSegmentRedirects(pFS, pRun) ); + assert( nPgno>0 && 0==fsPageRedirects(pFS, pRun, aPgno[0]) ); + iBlk = fsPageToBlock(pFS, pRun->iFirst); pRun->nSize += (pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk)); while( rc==LSM_OK ){ int iNext = 0; @@ -1344,11 +1599,11 @@ Pgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno); if( iFirst ){ pRun->iFirst = iFirst; break; } - rc = fsBlockNext(pFS, iBlk, &iNext); + rc = fsBlockNext(pFS, pRun, iBlk, &iNext); if( rc==LSM_OK ) rc = fsFreeBlock(pFS, pSnapshot, pRun, iBlk); pRun->nSize -= ( 1 + fsLastPageOnBlock(pFS, iBlk) - fsFirstPageOnBlock(pFS, iBlk) ); iBlk = iNext; @@ -1356,10 +1611,24 @@ pRun->nSize -= (pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk)); assert( pRun->nSize>0 ); } +/* +** This function is only used in compressed database mode. +** +** Argument iPg is the page number (byte offset) of a page within segment +** pSeg. The page record, including all headers, is nByte bytes in size. +** Before returning, set *piNext to the page number of the next page in +** the segment, or to zero if iPg is the last. +** +** In other words, do: +** +** *piNext = iPg + nByte; +** +** But take block overflow and redirection into account. +*/ static int fsNextPageOffset( FileSystem *pFS, /* File system object */ Segment *pSeg, /* Segment to move within */ Pgno iPg, /* Offset of current page */ int nByte, /* Size of current page including headers */ @@ -1368,28 +1637,33 @@ Pgno iNext; int rc; assert( pFS->pCompress ); - rc = fsAddOffset(pFS, iPg, nByte-1, &iNext); + rc = fsAddOffset(pFS, pSeg, iPg, nByte-1, &iNext); if( pSeg && iNext==pSeg->iLastPg ){ iNext = 0; }else if( rc==LSM_OK ){ - rc = fsAddOffset(pFS, iNext, 1, &iNext); + rc = fsAddOffset(pFS, pSeg, iNext, 1, &iNext); } *piNext = iNext; return rc; } -static int fsGetPageBefore(FileSystem *pFS, i64 iOff, Pgno *piPrev){ +static int fsGetPageBefore( + FileSystem *pFS, + Segment *pSeg, + i64 iOff, + Pgno *piPrev +){ u8 aSz[3]; int rc; i64 iRead; - rc = fsSubtractOffset(pFS, iOff, sizeof(aSz), &iRead); - if( rc==LSM_OK ) rc = fsReadData(pFS, iRead, aSz, sizeof(aSz)); + rc = fsSubtractOffset(pFS, pSeg, iOff, sizeof(aSz), &iRead); + if( rc==LSM_OK ) rc = fsReadData(pFS, pSeg, iRead, aSz, sizeof(aSz)); if( rc==LSM_OK ){ int bFree; int nSz; if( aSz[2] & 0x80 ){ @@ -1396,11 +1670,11 @@ nSz = getRecordSize(aSz, &bFree) + sizeof(aSz)*2; }else{ nSz = (int)(aSz[2] & 0x7F); bFree = 1; } - rc = fsSubtractOffset(pFS, iOff, nSz, piPrev); + rc = fsSubtractOffset(pFS, pSeg, iOff, nSz, piPrev); } return rc; } @@ -1428,10 +1702,11 @@ int lsmFsDbPageNext(Segment *pRun, Page *pPg, int eDir, Page **ppNext){ int rc = LSM_OK; FileSystem *pFS = pPg->pFS; Pgno iPg = pPg->iPg; + assert( 0==fsSegmentRedirects(pFS, pRun) ); if( pFS->pCompress ){ int nSpace = pPg->nCompress + 2*3; do { if( eDir>0 ){ @@ -1438,24 +1713,25 @@ rc = fsNextPageOffset(pFS, pRun, iPg, nSpace, &iPg); }else{ if( iPg==pRun->iFirst ){ iPg = 0; }else{ - rc = fsGetPageBefore(pFS, iPg, &iPg); + rc = fsGetPageBefore(pFS, pRun, iPg, &iPg); } } nSpace = 0; if( iPg!=0 ){ - rc = fsPageGet(pFS, iPg, 0, ppNext, &nSpace); + rc = fsPageGet(pFS, pRun, iPg, 0, ppNext, &nSpace); assert( (*ppNext==0)==(rc!=LSM_OK || nSpace>0) ); }else{ *ppNext = 0; } }while( nSpace>0 && rc==LSM_OK ); }else{ + Redirect *pRedir = pRun ? pRun->pRedirect : 0; assert( eDir==1 || eDir==-1 ); if( eDir<0 ){ if( pRun && iPg==pRun->iFirst ){ *ppNext = 0; return LSM_OK; @@ -1464,55 +1740,77 @@ iPg = fsLastPageOnBlock(pFS, lsmGetU32(&pPg->aData[-4])); }else{ iPg--; } }else{ - if( pRun && iPg==pRun->iLastPg ){ - *ppNext = 0; - return LSM_OK; - }else if( fsIsLast(pFS, iPg) ){ - iPg = fsFirstPageOnBlock(pFS, lsmGetU32(&pPg->aData[pFS->nPagesize-4])); + if( pRun ){ + if( iPg==pRun->iLastPg ){ + *ppNext = 0; + return LSM_OK; + } + } + + if( fsIsLast(pFS, iPg) ){ + int iBlk = fsRedirectBlock( + pRedir, lsmGetU32(&pPg->aData[pFS->nPagesize-4]) + ); + iPg = fsFirstPageOnBlock(pFS, iBlk); }else{ iPg++; } } - rc = fsPageGet(pFS, iPg, 0, ppNext, 0); + rc = fsPageGet(pFS, pRun, iPg, 0, ppNext, 0); } return rc; } -static Pgno findAppendPoint(FileSystem *pFS){ +static Pgno findAppendPoint(FileSystem *pFS, Level *pLvl){ int i; Pgno *aiAppend = pFS->pDb->pWorker->aiAppend; u32 iRet = 0; for(i=LSM_APPLIST_SZ-1; iRet==0 && i>=0; i--){ - if( (iRet = aiAppend[i]) ) aiAppend[i] = 0; + if( (iRet = aiAppend[i]) ){ + if( pLvl ){ + int iBlk = fsPageToBlock(pFS, iRet); + int j; + for(j=0; iRet && jnRight; j++){ + if( fsPageToBlock(pFS, pLvl->aRhs[j].iLastPg)==iBlk ){ + iRet = 0; + } + } + } + if( iRet ) aiAppend[i] = 0; + } } return iRet; } /* -** Append a page to file iFile. Set the ref-count to 1 and return a pointer -** to it. The page is writable until either lsmFsPagePersist() is called on -** it or the ref-count drops to zero. +** Append a page to the left-hand-side of pLvl. Set the ref-count to 1 and +** return a pointer to it. The page is writable until either +** lsmFsPagePersist() is called on it or the ref-count drops to zero. */ int lsmFsSortedAppend( FileSystem *pFS, Snapshot *pSnapshot, - Segment *p, + Level *pLvl, + int bDefer, Page **ppOut ){ int rc = LSM_OK; Page *pPg = 0; *ppOut = 0; int iApp = 0; int iNext = 0; + Segment *p = &pLvl->lhs; int iPrev = p->iLastPg; - if( pFS->pCompress ){ + assert( p->pRedirect==0 ); + + if( pFS->pCompress || bDefer ){ /* In compressed database mode the page is not assigned a page number ** or location in the database file at this point. This will be done ** by the lsmFsPagePersist() call. */ rc = fsPageBuffer(pFS, 1, &pPg); if( rc==LSM_OK ){ @@ -1520,32 +1818,33 @@ pPg->pSeg = p; pPg->iPg = 0; pPg->flags |= PAGE_DIRTY; pPg->nData = pFS->nPagesize; assert( pPg->aData ); + if( pFS->pCompress==0 ) pPg->nData -= 4; pPg->nRef = 1; pFS->nOut++; } }else{ if( iPrev==0 ){ - iApp = findAppendPoint(pFS); + iApp = findAppendPoint(pFS, pLvl); }else if( fsIsLast(pFS, iPrev) ){ int iNext; - rc = fsBlockNext(pFS, fsPageToBlock(pFS, iPrev), &iNext); + rc = fsBlockNext(pFS, 0, fsPageToBlock(pFS, iPrev), &iNext); if( rc!=LSM_OK ) return rc; iApp = fsFirstPageOnBlock(pFS, iNext); }else{ iApp = iPrev + 1; } /* If this is the first page allocated, or if the page allocated is the - ** last in the block, allocate a new block here. */ + ** last in the block, also allocate the next block here. */ if( iApp==0 || fsIsLast(pFS, iApp) ){ int iNew; /* New block number */ - rc = lsmBlockAllocate(pFS->pDb, &iNew); + rc = lsmBlockAllocate(pFS->pDb, 0, &iNew); if( rc!=LSM_OK ) return rc; if( iApp==0 ){ iApp = fsFirstPageOnBlock(pFS, iNew); }else{ iNext = fsFirstPageOnBlock(pFS, iNew); @@ -1552,11 +1851,11 @@ } } /* Grab the new page. */ pPg = 0; - rc = fsPageGet(pFS, iApp, 1, &pPg, 0); + rc = fsPageGet(pFS, 0, iApp, 1, &pPg, 0); assert( rc==LSM_OK || pPg==0 ); /* If this is the first or last page of a block, fill in the pointer ** value at the end of the new page. */ if( rc==LSM_OK ){ @@ -1581,20 +1880,19 @@ ** Mark the sorted run passed as the second argument as finished. */ int lsmFsSortedFinish(FileSystem *pFS, Segment *p){ int rc = LSM_OK; if( p && p->iLastPg ){ - int iBlk; + assert( p->pRedirect==0 ); /* Check if the last page of this run happens to be the last of a block. ** If it is, then an extra block has already been allocated for this run. ** Shift this extra block back to the free-block list. ** ** Otherwise, add the first free page in the last block used by the run ** to the lAppend list. */ - iBlk = fsPageToBlock(pFS, p->iLastPg); if( fsLastPageOnPagesBlock(pFS, p->iLastPg)!=p->iLastPg ){ int i; Pgno *aiAppend = pFS->pDb->pWorker->aiAppend; for(i=0; ipCompress==0 ){ Page *pLast; - rc = fsPageGet(pFS, p->iLastPg, 0, &pLast, 0); + rc = fsPageGet(pFS, 0, p->iLastPg, 0, &pLast, 0); if( rc==LSM_OK ){ int iBlk = (int)lsmGetU32(&pLast->aData[pFS->nPagesize-4]); lsmBlockRefree(pFS->pDb, iBlk); lsmFsPageRelease(pLast); } }else{ int iBlk = 0; - rc = fsBlockNext(pFS, fsPageToBlock(pFS, p->iLastPg), &iBlk); + rc = fsBlockNext(pFS, p, fsPageToBlock(pFS, p->iLastPg), &iBlk); if( rc==LSM_OK ){ lsmBlockRefree(pFS->pDb, iBlk); } } } @@ -1622,13 +1920,13 @@ } /* ** Obtain a reference to page number iPg. */ -int lsmFsDbPageGet(FileSystem *pFS, Pgno iPg, Page **ppPg){ +int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, Pgno iPg, Page **ppPg){ assert( pFS ); - return fsPageGet(pFS, iPg, 0, ppPg, 0); + return fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0); } /* ** Obtain a reference to the last page in the segment passed as the ** second argument. @@ -1639,18 +1937,18 @@ if( pFS->pCompress ){ int nSpace; iPg++; do { nSpace = 0; - rc = fsGetPageBefore(pFS, iPg, &iPg); + rc = fsGetPageBefore(pFS, pSeg, iPg, &iPg); if( rc==LSM_OK ){ - rc = fsPageGet(pFS, iPg, 0, ppPg, &nSpace); + rc = fsPageGet(pFS, pSeg, iPg, 0, ppPg, &nSpace); } }while( rc==LSM_OK && nSpace>0 ); }else{ - rc = fsPageGet(pFS, iPg, 0, ppPg, 0); + rc = fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0); } return rc; } /* @@ -1746,10 +2044,85 @@ ** Return true if page is currently writable. */ int lsmFsPageWritable(Page *pPg){ return (pPg->flags & PAGE_DIRTY) ? 1 : 0; } + +static void fsMovePage( + FileSystem *pFS, + Segment *pSeg, + int iTo, + int iFrom, + Pgno *piPg +){ + Pgno iPg = *piPg; + if( iFrom==fsPageToBlock(pFS, iPg) ){ + const int nPagePerBlock = ( + pFS->pCompress ? pFS ->nBlocksize : (pFS->nBlocksize / pFS->nPagesize) + ); + *piPg = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock; + } +} + +/* +** Copy the contents of block iFrom to block iTo. +** +** It is safe to assume that there are no outstanding references to pages +** on block iTo. And that block iFrom is not currently being written. In +** other words, the data can be read and written directly. +*/ +int lsmFsMoveBlock(FileSystem *pFS, Segment *pSeg, int iTo, int iFrom){ + Snapshot *p = pFS->pDb->pWorker; + int rc = LSM_OK; + + i64 iFromOff = (i64)(iFrom-1) * pFS->nBlocksize; + i64 iToOff = (i64)(iTo-1) * pFS->nBlocksize; + + assert( iTo!=1 ); + assert( iFrom>iTo ); + + if( pFS->bUseMmap ){ + fsGrowMapping(pFS, (i64)iFrom * pFS->nBlocksize, &rc); + if( rc==LSM_OK ){ + u8 *aMap = (u8 *)(pFS->pMap); + memcpy(&aMap[iToOff], &aMap[iFromOff], pFS->nBlocksize); + } + }else{ + int nSz = pFS->nPagesize; + u8 *aData = (u8 *)lsmMallocRc(pFS->pEnv, nSz, &rc); + if( rc==LSM_OK ){ + const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); + int i; + for(i=0; rc==LSM_OK && ipEnv, pFS->fdDb, iOff, aData, nSz); + if( rc==LSM_OK ){ + iOff = iToOff + i*nSz; + rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, aData, nSz); + } + } + } + } + + /* Update append-point list if necessary */ + if( rc==LSM_OK ){ + int i; + for(i=0; iaiAppend[i])==iFrom ){ + const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); + p->aiAppend[i] -= (i64)(iFrom-iTo) * nPagePerBlock; + } + } + } + + /* Update the Segment structure itself */ + fsMovePage(pFS, pSeg, iTo, iFrom, &pSeg->iFirst); + fsMovePage(pFS, pSeg, iTo, iFrom, &pSeg->iLastPg); + fsMovePage(pFS, pSeg, iTo, iFrom, &pSeg->iRoot); + + return rc; +} /* ** Append raw data to a segment. This function is only used in compressed ** database mode. */ @@ -1770,14 +2143,14 @@ Pgno iApp = pSeg->iLastPg+1; /* If this is the first data written into the segment, find an append-point ** or allocate a new block. */ if( iApp==1 ){ - pSeg->iFirst = iApp = findAppendPoint(pFS); + pSeg->iFirst = iApp = findAppendPoint(pFS, 0); if( iApp==0 ){ int iBlk; - rc = lsmBlockAllocate(pFS->pDb, &iBlk); + rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); pSeg->iFirst = iApp = fsFirstPageOnBlock(pFS, iBlk); } } iRet = iApp; @@ -1802,11 +2175,11 @@ u8 aPtr[4]; /* Space to serialize a u32 */ int iBlk; /* New block number */ if( nWrite>0 ){ /* Allocate a new block. */ - rc = lsmBlockAllocate(pFS->pDb, &iBlk); + rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); /* Set the "next" pointer on the old block */ if( rc==LSM_OK ){ assert( iApp==(fsPageToBlock(pFS, iApp)*pFS->nBlocksize)-4 ); lsmPutU32(aPtr, iBlk); @@ -1822,11 +2195,12 @@ if( nRem>0 ) iApp = iWrite; } }else{ /* The next block is already allocated. */ assert( nRem>0 ); - rc = fsBlockNext(pFS, fsPageToBlock(pFS, iApp), &iBlk); + assert( pSeg->pRedirect==0 ); + rc = fsBlockNext(pFS, 0, fsPageToBlock(pFS, iApp), &iBlk); iRet = iApp = fsFirstPageOnBlock(pFS, iBlk); } /* Write the remaining data into the new block */ if( rc==LSM_OK && nRem>0 ){ @@ -1861,10 +2235,69 @@ return p->xCompress(p->pCtx, (char *)pFS->aOBuffer, &pPg->nCompress, (const char *)pPg->aData, pPg->nData ); } + +static int fsAppendPage( + FileSystem *pFS, + Segment *pSeg, + Pgno *piNew, + int *piPrev, + int *piNext +){ + Pgno iPrev = pSeg->iLastPg; + int rc; + assert( iPrev!=0 ); + + *piPrev = 0; + *piNext = 0; + + if( fsIsLast(pFS, iPrev) ){ + /* Grab the first page on the next block (which has already be + ** allocated). In this case set *piPrev to tell the caller to set + ** the "previous block" pointer in the first 4 bytes of the page. + */ + int iNext; + int iBlk = fsPageToBlock(pFS, iPrev); + assert( pSeg->pRedirect==0 ); + rc = fsBlockNext(pFS, 0, iBlk, &iNext); + if( rc!=LSM_OK ) return rc; + *piNew = fsFirstPageOnBlock(pFS, iNext); + *piPrev = iBlk; + }else{ + *piNew = iPrev+1; + if( fsIsLast(pFS, *piNew) ){ + /* Allocate the next block here. */ + int iBlk; + rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); + if( rc!=LSM_OK ) return rc; + *piNext = iBlk; + } + } + + pSeg->nSize++; + pSeg->iLastPg = *piNew; + return LSM_OK; +} + +void lsmFsFlushWaiting(FileSystem *pFS, int *pRc){ + int rc = *pRc; + Page *pPg; + + pPg = pFS->pWaiting; + pFS->pWaiting = 0; + + while( pPg ){ + Page *pNext = pPg->pNextWaiting; + if( rc==LSM_OK ) rc = lsmFsPagePersist(pPg); + assert( pPg->nRef==1 ); + lsmFsPageRelease(pPg); + pPg = pNext; + } + *pRc = rc; +} /* ** If the page passed as an argument is dirty, update the database file ** (or mapping of the database file) with its current contents and mark ** the page as clean. @@ -1898,30 +2331,86 @@ pPg->pHashNext = pFS->apHash[iHash]; pFS->apHash[iHash] = pPg; pPg->pSeg->nSize += (sizeof(aSz) * 2) + pPg->nCompress; - }else{ - i64 iOff; /* Offset to write within database file */ - iOff = (i64)pFS->nPagesize * (i64)(pPg->iPg-1); - if( pFS->bUseMmap==0 ){ - u8 *aData = pPg->aData - (pPg->flags & PAGE_HASPREV); - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, aData, pFS->nPagesize); - }else if( pPg->flags & PAGE_FREE ){ - fsGrowMapping(pFS, iOff + pFS->nPagesize, &rc); - if( rc==LSM_OK ){ - u8 *aTo = &((u8 *)(pFS->pMap))[iOff]; - memcpy(aTo, pPg->aData, pFS->nPagesize); - lsmFree(pFS->pEnv, pPg->aData); - pPg->aData = aTo; - pPg->flags &= ~PAGE_FREE; - fsPageAddToLru(pFS, pPg); - } - } - } - pPg->flags &= ~PAGE_DIRTY; - pFS->nWrite++; + pPg->flags &= ~PAGE_DIRTY; + pFS->nWrite++; + }else{ + + if( pPg->iPg==0 ){ + /* No page number has been assigned yet. This occurs with pages used + ** in the b-tree hierarchy. They were not assigned page numbers when + ** they were created as doing so would cause this call to + ** lsmFsPagePersist() to write an out-of-order page. Instead a page + ** number is assigned here so that the page data will be appended + ** to the current segment. + */ + Page **pp; + int iPrev = 0; + int iNext = 0; + + assert( pPg->pSeg->iFirst ); + assert( pPg->flags & PAGE_FREE ); + assert( (pPg->flags & PAGE_HASPREV)==0 ); + assert( pPg->nData==pFS->nPagesize-4 ); + + rc = fsAppendPage(pFS, pPg->pSeg, &pPg->iPg, &iPrev, &iNext); + if( rc!=LSM_OK ) return rc; + + if( pFS->bUseMmap==0 ){ + int iHash = fsHashKey(pFS->nHash, pPg->iPg); + pPg->pHashNext = pFS->apHash[iHash]; + pFS->apHash[iHash] = pPg; + assert( pPg->pHashNext==0 || pPg->pHashNext->iPg!=pPg->iPg ); + } + + if( iPrev ){ + assert( iNext==0 ); + memmove(&pPg->aData[4], pPg->aData, pPg->nData); + lsmPutU32(pPg->aData, iPrev); + pPg->flags |= PAGE_HASPREV; + pPg->aData += 4; + }else if( iNext ){ + assert( iPrev==0 ); + lsmPutU32(&pPg->aData[pPg->nData], iNext); + }else{ + int nData = pPg->nData; + pPg->nData += 4; + lsmSortedExpandBtreePage(pPg, nData); + } + + pPg->nRef++; + for(pp=&pFS->pWaiting; *pp; pp=&(*pp)->pNextWaiting); + *pp = pPg; + assert( pPg->pNextWaiting==0 ); + + }else{ + i64 iOff; /* Offset to write within database file */ + + iOff = (i64)pFS->nPagesize * (i64)(pPg->iPg-1); + if( pFS->bUseMmap==0 ){ + u8 *aData = pPg->aData - (pPg->flags & PAGE_HASPREV); + rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, aData, pFS->nPagesize); + }else if( pPg->flags & PAGE_FREE ){ + fsGrowMapping(pFS, iOff + pFS->nPagesize, &rc); + if( rc==LSM_OK ){ + u8 *aTo = &((u8 *)(pFS->pMap))[iOff]; + u8 *aFrom = pPg->aData - (pPg->flags & PAGE_HASPREV); + memcpy(aTo, aFrom, pFS->nPagesize); + lsmFree(pFS->pEnv, aFrom); + pPg->aData = aTo + (pPg->flags & PAGE_HASPREV); + pPg->flags &= ~PAGE_FREE; + fsPageAddToLru(pFS, pPg); + } + } + + lsmFsFlushWaiting(pFS, &rc); + pPg->flags &= ~PAGE_DIRTY; + pFS->nWrite++; + } + } } return rc; } @@ -1997,11 +2486,11 @@ int lsmFsPageRelease(Page *pPg){ int rc = LSM_OK; if( pPg ){ assert( pPg->nRef>0 ); pPg->nRef--; - if( pPg->nRef==0 && pPg->iPg!=0 ){ + if( pPg->nRef==0 ){ FileSystem *pFS = pPg->pFS; rc = lsmFsPagePersist(pPg); pFS->nOut--; assert( pPg->pFS->pCompress @@ -2034,17 +2523,10 @@ /* ** Return the total number of pages written to the database file. */ int lsmFsNWrite(FileSystem *pFS){ return pFS->nWrite; } -/* -** fsync() the database file. -*/ -int lsmFsSyncDb(FileSystem *pFS){ - return lsmEnvSync(pFS->pEnv, pFS->fdDb); -} - /* ** Return a copy of the environment pointer used by the file-system object. */ lsm_env *lsmFsEnv(FileSystem *pFS) { return pFS->pEnv; @@ -2098,11 +2580,16 @@ ** containing the array structure and LSM_OK is returned. The caller should ** eventually free the string using lsmFree(). ** ** If an error occurs, *pzOut is set to NULL and an LSM error code returned. */ -int lsmInfoArrayStructure(lsm_db *pDb, Pgno iFirst, char **pzOut){ +int lsmInfoArrayStructure( + lsm_db *pDb, + int bBlock, /* True for block numbers only */ + Pgno iFirst, + char **pzOut +){ int rc = LSM_OK; Snapshot *pWorker; /* Worker snapshot */ Segment *pArray = 0; /* Array to report on */ int bUnlock = 0; @@ -2132,17 +2619,25 @@ iBlk = fsPageToBlock(pFS, pArray->iFirst); iLastBlk = fsPageToBlock(pFS, pArray->iLastPg); lsmStringInit(&str, pDb->pEnv); - lsmStringAppendf(&str, "%d", pArray->iFirst); - while( iBlk!=iLastBlk ){ - lsmStringAppendf(&str, " %d", fsLastPageOnBlock(pFS, iBlk)); - fsBlockNext(pFS, iBlk, &iBlk); - lsmStringAppendf(&str, " %d", fsFirstPageOnBlock(pFS, iBlk)); + if( bBlock ){ + lsmStringAppendf(&str, "%d", iBlk); + while( iBlk!=iLastBlk ){ + fsBlockNext(pFS, pArray, iBlk, &iBlk); + lsmStringAppendf(&str, " %d", iBlk); + } + }else{ + lsmStringAppendf(&str, "%d", pArray->iFirst); + while( iBlk!=iLastBlk ){ + lsmStringAppendf(&str, " %d", fsLastPageOnBlock(pFS, iBlk)); + fsBlockNext(pFS, pArray, iBlk, &iBlk); + lsmStringAppendf(&str, " %d", fsFirstPageOnBlock(pFS, iBlk)); + } + lsmStringAppendf(&str, " %d", pArray->iLastPg); } - lsmStringAppendf(&str, " %d", pArray->iLastPg); *pzOut = str.z; } if( bUnlock ){ @@ -2149,10 +2644,34 @@ int rcwork = LSM_BUSY; lsmFinishWork(pDb, 0, &rcwork); } return rc; } + +int lsmFsSegmentContainsPg( + FileSystem *pFS, + Segment *pSeg, + Pgno iPg, + int *pbRes +){ + Redirect *pRedir = pSeg->pRedirect; + int rc = LSM_OK; + int iBlk; + int iLastBlk; + int iPgBlock; /* Block containing page iPg */ + + iPgBlock = fsPageToBlock(pFS, pSeg->iFirst); + iBlk = fsRedirectBlock(pRedir, fsPageToBlock(pFS, pSeg->iFirst)); + iLastBlk = fsRedirectBlock(pRedir, fsPageToBlock(pFS, pSeg->iLastPg)); + + while( iBlk!=iLastBlk && iBlk!=iPgBlock && rc==LSM_OK ){ + rc = fsBlockNext(pFS, pSeg, iBlk, &iBlk); + } + + *pbRes = (iBlk==iPgBlock); + return rc; +} /* ** This function implements the lsm_info(LSM_INFO_ARRAY_PAGES) request. ** If successful, *pzOut is set to point to a nul-terminated string ** containing the array structure and LSM_OK is returned. The caller should @@ -2161,11 +2680,11 @@ ** If an error occurs, *pzOut is set to NULL and an LSM error code returned. */ int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut){ int rc = LSM_OK; Snapshot *pWorker; /* Worker snapshot */ - Segment *pArray = 0; /* Array to report on */ + Segment *pSeg = 0; /* Array to report on */ int bUnlock = 0; *pzOut = 0; if( iFirst==0 ) return LSM_ERROR; @@ -2177,26 +2696,26 @@ pWorker = pDb->pWorker; bUnlock = 1; } /* Search for the array that starts on page iFirst */ - pArray = findSegment(pWorker, iFirst); + pSeg = findSegment(pWorker, iFirst); - if( pArray==0 ){ + if( pSeg==0 ){ /* Could not find the requested array. This is an error. */ rc = LSM_ERROR; }else{ Page *pPg = 0; FileSystem *pFS = pDb->pFS; LsmString str; lsmStringInit(&str, pDb->pEnv); - rc = lsmFsDbPageGet(pFS, iFirst, &pPg); + rc = lsmFsDbPageGet(pFS, pSeg, iFirst, &pPg); while( rc==LSM_OK && pPg ){ Page *pNext = 0; lsmStringAppendf(&str, " %lld", lsmFsPageNumber(pPg)); - rc = lsmFsDbPageNext(pArray, pPg, 1, &pNext); + rc = lsmFsDbPageNext(pSeg, pPg, 1, &pNext); lsmFsPageRelease(pPg); pPg = pNext; } if( rc!=LSM_OK ){ @@ -2247,31 +2766,33 @@ u8 *aUsed ){ if( pSeg ){ if( pSeg && pSeg->nSize>0 ){ int rc; - Pgno iLast = pSeg->iLastPg; - int iBlk; - int iLastBlk; - int bLastIsLastOnBlock; + int iBlk; /* Current block (during iteration) */ + int iLastBlk; /* Last block of segment */ + int iFirstBlk; /* First block of segment */ + int bLastIsLastOnBlock; /* True iLast is the last on its block */ - iBlk = fsPageToBlock(pFS, pSeg->iFirst); + assert( 0==fsSegmentRedirects(pFS, pSeg) ); + iBlk = iFirstBlk = fsPageToBlock(pFS, pSeg->iFirst); iLastBlk = fsPageToBlock(pFS, pSeg->iLastPg); - bLastIsLastOnBlock = (fsLastPageOnBlock(pFS, iLastBlk)==iLast); - assert( iBlk>0 ); - - /* If the first page of this run is also the first page of its first - ** block, set the flag to indicate that the first page of iBlk is - ** in use. */ - if( fsFirstPageOnBlock(pFS, iBlk)==pSeg->iFirst ){ - assert( (aUsed[iBlk-1] & INTEGRITY_CHECK_FIRST_PG)==0 ); - aUsed[iBlk-1] |= INTEGRITY_CHECK_FIRST_PG; - } + + bLastIsLastOnBlock = (fsLastPageOnBlock(pFS, iLastBlk)==pSeg->iLastPg); + assert( iBlk>0 ); do { /* iBlk is a part of this sorted run. */ aUsed[iBlk-1] |= INTEGRITY_CHECK_USED; + + /* If the first page of this block is also part of the segment, + ** set the flag to indicate that the first page of iBlk is in use. + */ + if( fsFirstPageOnBlock(pFS, iBlk)==pSeg->iFirst || iBlk!=iFirstBlk ){ + assert( (aUsed[iBlk-1] & INTEGRITY_CHECK_FIRST_PG)==0 ); + aUsed[iBlk-1] |= INTEGRITY_CHECK_FIRST_PG; + } /* Unless the sorted run finishes before the last page on this block, ** the last page of this block is also in use. */ if( iBlk!=iLastBlk || bLastIsLastOnBlock ){ assert( (aUsed[iBlk-1] & INTEGRITY_CHECK_LAST_PG)==0 ); @@ -2282,11 +2803,11 @@ ** a level currently undergoing an incremental merge. The sorted ** run ends on the last page of iBlk, but the next block has already ** been allocated. So mark it as in use as well. */ if( iBlk==iLastBlk && bLastIsLastOnBlock && bExtra ){ int iExtra = 0; - rc = fsBlockNext(pFS, iBlk, &iExtra); + rc = fsBlockNext(pFS, pSeg, iBlk, &iExtra); assert( rc==LSM_OK ); assert( aUsed[iExtra-1]==0 ); aUsed[iExtra-1] |= INTEGRITY_CHECK_USED; aUsed[iExtra-1] |= INTEGRITY_CHECK_FIRST_PG; @@ -2297,11 +2818,11 @@ ** in order to break out of the loop if this was the last block in ** the run. */ if( iBlk==iLastBlk ){ iBlk = 0; }else{ - rc = fsBlockNext(pFS, iBlk, &iBlk); + rc = fsBlockNext(pFS, pSeg, iBlk, &iBlk); assert( rc==LSM_OK ); } }while( iBlk ); } } @@ -2343,10 +2864,16 @@ Freelist freelist = {0, 0, 0}; u8 *aUsed; Level *pLevel; Snapshot *pWorker = pDb->pWorker; int nBlock = pWorker->nBlock; + +#if 0 + static int nCall = 0; + nCall++; + printf("%d calls\n", nCall); +#endif aUsed = lsmMallocZero(pDb->pEnv, nBlock); if( aUsed==0 ){ /* Malloc has failed. Since this function is only called within debug ** builds, this probably means the user is running an OOM injection test. @@ -2364,11 +2891,11 @@ } /* Mark all blocks in the free-list as used */ ctx.aUsed = aUsed; ctx.nBlock = nBlock; - rc = lsmWalkFreelist(pDb, checkFreelistCb, (void *)&ctx); + rc = lsmWalkFreelist(pDb, 0, checkFreelistCb, (void *)&ctx); if( rc==LSM_OK ){ for(i=0; ipShmhdr->iMetaPage ){ + int iMeta; + + iMeta = (int)pDb->pShmhdr->iMetaPage; + if( iMeta==1 || iMeta==2 ){ DbLog *pLog = &pDb->treehdr.log; - i64 iSnapshotId = 0; - i64 iOff = 0; - rc = lsmCheckpointSynced(pDb, &iSnapshotId, &iOff, 0); - if( rc==LSM_OK && pLog->iSnapshotIdaRegion[iRegion]; - if( iOff>=p->iStart && iOff<=p->iEnd ) break; - p->iStart = 0; - p->iEnd = 0; - } - assert( iRegion<3 ); - pLog->aRegion[iRegion].iStart = iOff; - pLog->iSnapshotId = iSnapshotId; + i64 iSyncedId; + + /* Read the snapshot-id of the snapshot stored on meta-page iMeta. Note + ** that in theory, the value read is untrustworthy (due to a race + ** condition - see comments above lsmFsReadSyncedId()). So it is only + ** ever used to conclude that no log space can be reclaimed. If it seems + ** to indicate that it may be possible to reclaim log space, a + ** second call to lsmCheckpointSynced() (which does return trustworthy + ** values) is made below to confirm. */ + rc = lsmFsReadSyncedId(pDb, iMeta, &iSyncedId); + + if( rc==LSM_OK && pLog->iSnapshotId!=iSyncedId ){ + i64 iSnapshotId = 0; + i64 iOff = 0; + rc = lsmCheckpointSynced(pDb, &iSnapshotId, &iOff, 0); + if( rc==LSM_OK && pLog->iSnapshotIdaRegion[iRegion]; + if( iOff>=p->iStart && iOff<=p->iEnd ) break; + p->iStart = 0; + p->iEnd = 0; + } + assert( iRegion<3 ); + pLog->aRegion[iRegion].iStart = iOff; + pLog->iSnapshotId = iSnapshotId; + } } } return rc; } @@ -336,16 +355,27 @@ int rc = LSM_OK; LogWriter *pNew; LogRegion *aReg; if( pDb->bUseLog==0 ) return LSM_OK; - rc = lsmFsOpenLog(pDb->pFS); - pNew = lsmMallocZeroRc(pDb->pEnv, sizeof(LogWriter), &rc); - if( pNew ){ - lsmStringInit(&pNew->buf, pDb->pEnv); - rc = lsmStringExtend(&pNew->buf, 2); + + /* If the log file has not yet been opened, open it now. Also allocate + ** the LogWriter structure, if it has not already been allocated. */ + rc = lsmFsOpenLog(pDb, 0); + if( pDb->pLogWriter==0 ){ + pNew = lsmMallocZeroRc(pDb->pEnv, sizeof(LogWriter), &rc); + if( pNew ){ + lsmStringInit(&pNew->buf, pDb->pEnv); + rc = lsmStringExtend(&pNew->buf, 2); + } + }else{ + pNew = pDb->pLogWriter; + assert( (u8 *)(&pNew[1])==(u8 *)(&((&pNew->buf)[1])) ); + memset(pNew, 0, ((u8 *)&pNew->buf) - (u8 *)pNew); + pNew->buf.n = 0; } + if( rc==LSM_OK ){ /* The following call detects whether or not a new snapshot has been ** synced into the database file. If so, it updates the contents of ** the pDb->treehdr.log structure to reclaim any space in the log ** file that is no longer required. @@ -357,12 +387,11 @@ ** the log file. */ rc = logReclaimSpace(pDb); } if( rc!=LSM_OK ){ - if( pNew ) lsmFree(pDb->pEnv, pNew->buf.z); - lsmFree(pDb->pEnv, pNew); + lsmLogClose(pDb); return rc; } /* Set the effective sector-size for this transaction. Sectors are assumed ** to be one byte in size if the safety-mode is OFF or NORMAL, or as @@ -375,13 +404,12 @@ } /* There are now three scenarios: ** ** 1) Regions 0 and 1 are both zero bytes in size and region 2 begins - ** at a file offset greater than N, where N is the value configured - ** by LSM_CONFIG_LOG_SIZE. In this case, wrap around to the start - ** and write data into the start of the log file. + ** at a file offset greater than LSM_MIN_LOGWRAP. In this case, wrap + ** around to the start and write data into the start of the log file. ** ** 2) Region 1 is zero bytes in size and region 2 occurs earlier in the ** file than region 0. In this case, append data to region 2, but ** remember to jump over region 1 if required. ** @@ -393,11 +421,11 @@ assert( aReg[1].iEnd==0 || aReg[1].iEnd>aReg[1].iStart ); pNew->cksum0 = pDb->treehdr.log.cksum0; pNew->cksum1 = pDb->treehdr.log.cksum1; - if( aReg[0].iEnd==0 && aReg[1].iEnd==0 && aReg[2].iStart>=pDb->nLogSz ){ + if( aReg[0].iEnd==0 && aReg[1].iEnd==0 && aReg[2].iStart>=LSM_MIN_LOGWRAP ){ /* Case 1. Wrap around to the start of the file. Write an LSM_LOG_JUMP ** into the log file in this case. Pad it out to 8 bytes using a PAD2 ** record so that the checksums can be updated immediately. */ u8 aJump[] = { LSM_LOG_PAD2, 0x04, 0x00, 0x00, 0x00, 0x00, LSM_LOG_JUMP, 0x00 @@ -464,13 +492,10 @@ pLog->aRegion[1].iStart = pLog->aRegion[2].iStart; pLog->aRegion[1].iEnd = p->iRegion1End; pLog->aRegion[2].iStart = p->iRegion2Start; } } - lsmStringClear(&p->buf); - lsmFree(pDb->pEnv, p); - pDb->pLogWriter = 0; } static int jumpIfRequired( lsm_db *pDb, LogWriter *pLog, @@ -931,12 +956,13 @@ int rc = LSM_OK; /* Return code */ int nCommit = 0; /* Number of transactions to recover */ int iPass; int nJump = 0; /* Number of LSM_LOG_JUMP records in pass 0 */ DbLog *pLog; + int bOpen; - rc = lsmFsOpenLog(pDb->pFS); + rc = lsmFsOpenLog(pDb, &bOpen); if( rc!=LSM_OK ) return rc; rc = lsmTreeInit(pDb); if( rc!=LSM_OK ) return rc; @@ -949,124 +975,126 @@ /* The outer for() loop runs at most twice. The first iteration is to ** count the number of committed transactions in the log. The second ** iterates through those transactions and updates the in-memory tree ** structure with their contents. */ - for(iPass=0; iPass<2 && rc==LSM_OK; iPass++){ - int bEof = 0; - - while( rc==LSM_OK && !bEof ){ - u8 eType = 0; - logReaderByte(&reader, &eType, &rc); - - switch( eType ){ - case LSM_LOG_PAD1: - break; - - case LSM_LOG_PAD2: { - int nPad; - logReaderVarint(&reader, &buf1, &nPad, &rc); - logReaderBlob(&reader, &buf1, nPad, 0, &rc); - break; - } - - case LSM_LOG_WRITE: - case LSM_LOG_WRITE_CKSUM: { - int nKey; - int nVal; - u8 *aVal; - logReaderVarint(&reader, &buf1, &nKey, &rc); - logReaderVarint(&reader, &buf2, &nVal, &rc); - - if( eType==LSM_LOG_WRITE_CKSUM ){ - logReaderCksum(&reader, &buf1, &bEof, &rc); - }else{ - bEof = logRequireCksum(&reader, nKey+nVal); - } - if( bEof ) break; - - logReaderBlob(&reader, &buf1, nKey, 0, &rc); - logReaderBlob(&reader, &buf2, nVal, &aVal, &rc); - if( iPass==1 && rc==LSM_OK ){ - rc = lsmTreeInsert(pDb, (u8 *)buf1.z, nKey, aVal, nVal); - } - break; - } - - case LSM_LOG_DELETE: - case LSM_LOG_DELETE_CKSUM: { - int nKey; u8 *aKey; - logReaderVarint(&reader, &buf1, &nKey, &rc); - - if( eType==LSM_LOG_DELETE_CKSUM ){ - logReaderCksum(&reader, &buf1, &bEof, &rc); - }else{ - bEof = logRequireCksum(&reader, nKey); - } - if( bEof ) break; - - logReaderBlob(&reader, &buf1, nKey, &aKey, &rc); - if( iPass==1 && rc==LSM_OK ){ - rc = lsmTreeInsert(pDb, aKey, nKey, NULL, -1); - } - break; - } - - case LSM_LOG_COMMIT: - logReaderCksum(&reader, &buf1, &bEof, &rc); - if( bEof==0 ){ - nCommit++; - assert( nCommit>0 || iPass==1 ); - if( nCommit==0 ) bEof = 1; - } - break; - - case LSM_LOG_JUMP: { - int iOff = 0; - logReaderVarint(&reader, &buf1, &iOff, &rc); - if( rc==LSM_OK ){ - if( iPass==1 ){ - if( pLog->aRegion[2].iStart==0 ){ - assert( pLog->aRegion[1].iStart==0 ); - pLog->aRegion[1].iEnd = reader.iOff; - }else{ - assert( pLog->aRegion[0].iStart==0 ); - pLog->aRegion[0].iStart = pLog->aRegion[2].iStart; - pLog->aRegion[0].iEnd = reader.iOff - reader.buf.n+reader.iBuf; - } - pLog->aRegion[2].iStart = iOff; - }else{ - if( (nJump++)==2 ){ - bEof = 1; - } - } - - reader.iOff = iOff; - reader.buf.n = reader.iBuf; - } - break; - } - - default: - /* Including LSM_LOG_EOF */ - bEof = 1; - break; - } - } - - if( rc==LSM_OK && iPass==0 ){ - if( nCommit==0 ){ - if( pLog->aRegion[2].iStart==0 ){ - iPass = 1; - }else{ - pLog->aRegion[2].iStart = 0; - iPass = -1; - lsmCheckpointZeroLogoffset(pDb); - } - } - logReaderInit(pDb, pLog, 0, &reader); - nCommit = nCommit * -1; + if( bOpen ){ + for(iPass=0; iPass<2 && rc==LSM_OK; iPass++){ + int bEof = 0; + + while( rc==LSM_OK && !bEof ){ + u8 eType = 0; + logReaderByte(&reader, &eType, &rc); + + switch( eType ){ + case LSM_LOG_PAD1: + break; + + case LSM_LOG_PAD2: { + int nPad; + logReaderVarint(&reader, &buf1, &nPad, &rc); + logReaderBlob(&reader, &buf1, nPad, 0, &rc); + break; + } + + case LSM_LOG_WRITE: + case LSM_LOG_WRITE_CKSUM: { + int nKey; + int nVal; + u8 *aVal; + logReaderVarint(&reader, &buf1, &nKey, &rc); + logReaderVarint(&reader, &buf2, &nVal, &rc); + + if( eType==LSM_LOG_WRITE_CKSUM ){ + logReaderCksum(&reader, &buf1, &bEof, &rc); + }else{ + bEof = logRequireCksum(&reader, nKey+nVal); + } + if( bEof ) break; + + logReaderBlob(&reader, &buf1, nKey, 0, &rc); + logReaderBlob(&reader, &buf2, nVal, &aVal, &rc); + if( iPass==1 && rc==LSM_OK ){ + rc = lsmTreeInsert(pDb, (u8 *)buf1.z, nKey, aVal, nVal); + } + break; + } + + case LSM_LOG_DELETE: + case LSM_LOG_DELETE_CKSUM: { + int nKey; u8 *aKey; + logReaderVarint(&reader, &buf1, &nKey, &rc); + + if( eType==LSM_LOG_DELETE_CKSUM ){ + logReaderCksum(&reader, &buf1, &bEof, &rc); + }else{ + bEof = logRequireCksum(&reader, nKey); + } + if( bEof ) break; + + logReaderBlob(&reader, &buf1, nKey, &aKey, &rc); + if( iPass==1 && rc==LSM_OK ){ + rc = lsmTreeInsert(pDb, aKey, nKey, NULL, -1); + } + break; + } + + case LSM_LOG_COMMIT: + logReaderCksum(&reader, &buf1, &bEof, &rc); + if( bEof==0 ){ + nCommit++; + assert( nCommit>0 || iPass==1 ); + if( nCommit==0 ) bEof = 1; + } + break; + + case LSM_LOG_JUMP: { + int iOff = 0; + logReaderVarint(&reader, &buf1, &iOff, &rc); + if( rc==LSM_OK ){ + if( iPass==1 ){ + if( pLog->aRegion[2].iStart==0 ){ + assert( pLog->aRegion[1].iStart==0 ); + pLog->aRegion[1].iEnd = reader.iOff; + }else{ + assert( pLog->aRegion[0].iStart==0 ); + pLog->aRegion[0].iStart = pLog->aRegion[2].iStart; + pLog->aRegion[0].iEnd = reader.iOff - reader.buf.n+reader.iBuf; + } + pLog->aRegion[2].iStart = iOff; + }else{ + if( (nJump++)==2 ){ + bEof = 1; + } + } + + reader.iOff = iOff; + reader.buf.n = reader.iBuf; + } + break; + } + + default: + /* Including LSM_LOG_EOF */ + bEof = 1; + break; + } + } + + if( rc==LSM_OK && iPass==0 ){ + if( nCommit==0 ){ + if( pLog->aRegion[2].iStart==0 ){ + iPass = 1; + }else{ + pLog->aRegion[2].iStart = 0; + iPass = -1; + lsmCheckpointZeroLogoffset(pDb); + } + } + logReaderInit(pDb, pLog, 0, &reader); + nCommit = nCommit * -1; + } } } /* Initialize DbLog object */ if( rc==LSM_OK ){ @@ -1078,11 +1106,25 @@ if( rc==LSM_OK ){ rc = lsmFinishRecovery(pDb); }else{ lsmFinishRecovery(pDb); } + + if( pDb->bRoTrans ){ + lsmFsCloseLog(pDb); + } lsmStringClear(&buf1); lsmStringClear(&buf2); lsmStringClear(&reader.buf); return rc; } + +void lsmLogClose(lsm_db *db){ + if( db->pLogWriter ){ + lsmFree(db->pEnv, db->pLogWriter->buf.z); + lsmFree(db->pEnv, db->pLogWriter); + db->pLogWriter = 0; + } +} + + Index: src/lsm_main.c ================================================================== --- src/lsm_main.c +++ src/lsm_main.c @@ -37,11 +37,14 @@ /* If there is at least one cursor or a write transaction open, the database ** handle must be holding a pointer to a client snapshot. And the reverse ** - if there are no open cursors and no write transactions then there must ** not be a client snapshot. */ - assert( (pDb->pCsr!=0 || pDb->nTransOpen>0)==(pDb->iReader>=0) ); + + assert( (pDb->pCsr!=0||pDb->nTransOpen>0)==(pDb->iReader>=0||pDb->bRoTrans) ); + + assert( (pDb->iReader<0 && pDb->bRoTrans==0) || pDb->pClient!=0 ); assert( pDb->nTransOpen>=0 ); } #else # define assert_db_state(x) @@ -78,25 +81,26 @@ *ppDb = pDb = (lsm_db *)lsmMallocZero(pEnv, sizeof(lsm_db)); if( pDb==0 ) return LSM_NOMEM_BKPT; /* Initialize the new object */ pDb->pEnv = pEnv; - pDb->nTreeLimit = LSM_DFLT_WRITE_BUFFER; + pDb->nTreeLimit = LSM_DFLT_AUTOFLUSH; pDb->nAutockpt = LSM_DFLT_AUTOCHECKPOINT; - pDb->bAutowork = 1; - pDb->eSafety = LSM_SAFETY_NORMAL; + pDb->bAutowork = LSM_DFLT_AUTOWORK; + pDb->eSafety = LSM_DFLT_SAFETY; pDb->xCmp = xCmp; - pDb->nLogSz = LSM_DFLT_LOG_SIZE; pDb->nDfltPgsz = LSM_DFLT_PAGE_SIZE; pDb->nDfltBlksz = LSM_DFLT_BLOCK_SIZE; - pDb->nMerge = LSM_DFLT_NMERGE; + pDb->nMerge = LSM_DFLT_AUTOMERGE; pDb->nMaxFreelist = LSM_MAX_FREELIST_ENTRIES; - pDb->bUseLog = 1; + pDb->bUseLog = LSM_DFLT_USE_LOG; pDb->iReader = -1; - pDb->bMultiProc = 1; - pDb->bMmap = LSM_IS_64_BIT; + pDb->iRwclient = -1; + pDb->bMultiProc = LSM_DFLT_MULTIPLE_PROCESSES; + pDb->bMmap = LSM_DFLT_MMAP; pDb->xLog = xLog; + pDb->compress.iId = LSM_COMPRESSION_NONE; return LSM_OK; } lsm_env *lsm_get_env(lsm_db *pDb){ assert( pDb->pEnv ); @@ -158,27 +162,33 @@ ** path is required to ensure that the correct files are operated ** on even if the application changes the cwd. */ rc = getFullpathname(pDb->pEnv, zFilename, &zFull); assert( rc==LSM_OK || zFull==0 ); - /* Connect to the database */ + /* Connect to the database. */ if( rc==LSM_OK ){ rc = lsmDbDatabaseConnect(pDb, zFull); } - /* Configure the file-system connection with the page-size and block-size - ** of this database. Even if the database file is zero bytes in size - ** on disk, these values have been set in shared-memory by now, and so are - ** guaranteed not to change during the lifetime of this connection. */ - if( rc==LSM_OK && LSM_OK==(rc = lsmCheckpointLoad(pDb, 0)) ){ - lsmFsSetPageSize(pDb->pFS, lsmCheckpointPgsz(pDb->aSnapshot)); - lsmFsSetBlockSize(pDb->pFS, lsmCheckpointBlksz(pDb->aSnapshot)); + if( pDb->bReadonly==0 ){ + /* Configure the file-system connection with the page-size and block-size + ** of this database. Even if the database file is zero bytes in size + ** on disk, these values have been set in shared-memory by now, and so + ** are guaranteed not to change during the lifetime of this connection. + */ + if( rc==LSM_OK && LSM_OK==(rc = lsmCheckpointLoad(pDb, 0)) ){ + lsmFsSetPageSize(pDb->pFS, lsmCheckpointPgsz(pDb->aSnapshot)); + lsmFsSetBlockSize(pDb->pFS, lsmCheckpointBlksz(pDb->aSnapshot)); + } } lsmFree(pDb->pEnv, zFull); } + assert( pDb->bReadonly==0 || pDb->bReadonly==1 ); + assert( rc!=LSM_OK || (pDb->pShmhdr==0)==(pDb->bReadonly==1) ); + return rc; } int lsm_close(lsm_db *pDb){ @@ -188,12 +198,22 @@ if( pDb->pCsr || pDb->nTransOpen ){ rc = LSM_MISUSE_BKPT; }else{ lsmFreeSnapshot(pDb->pEnv, pDb->pClient); pDb->pClient = 0; + lsmDbDatabaseRelease(pDb); + lsmLogClose(pDb); lsmFsClose(pDb->pFS); + assert( pDb->mLock==0 ); + + /* Invoke any destructors registered for the compression or + ** compression factory callbacks. */ + if( pDb->factory.xFree ) pDb->factory.xFree(pDb->factory.pCtx); + if( pDb->compress.xFree ) pDb->compress.xFree(pDb->compress.pCtx); + + lsmFree(pDb->pEnv, pDb->rollback.aArray); lsmFree(pDb->pEnv, pDb->aTrans); lsmFree(pDb->pEnv, pDb->apShm); lsmFree(pDb->pEnv, pDb); } } @@ -204,16 +224,19 @@ int rc = LSM_OK; va_list ap; va_start(ap, eParam); switch( eParam ){ - case LSM_CONFIG_WRITE_BUFFER: { + case LSM_CONFIG_AUTOFLUSH: { + /* This parameter is read and written in KB. But all internal + ** processing is done in bytes. */ int *piVal = va_arg(ap, int *); - if( *piVal>=0 ){ - pDb->nTreeLimit = *piVal; + int iVal = *piVal; + if( iVal>=0 && iVal<=(1024*1024) ){ + pDb->nTreeLimit = iVal*1024; } - *piVal = pDb->nTreeLimit; + *piVal = (pDb->nTreeLimit / 1024); break; } case LSM_CONFIG_AUTOWORK: { int *piVal = va_arg(ap, int *); @@ -223,24 +246,18 @@ *piVal = pDb->bAutowork; break; } case LSM_CONFIG_AUTOCHECKPOINT: { + /* This parameter is read and written in KB. But all internal processing + ** (including the lsm_db.nAutockpt variable) is done in bytes. */ int *piVal = va_arg(ap, int *); if( *piVal>=0 ){ - pDb->nAutockpt = *piVal; - } - *piVal = pDb->nAutockpt; - break; - } - - case LSM_CONFIG_LOG_SIZE: { - int *piVal = va_arg(ap, int *); - if( *piVal>0 ){ - pDb->nLogSz = *piVal; - } - *piVal = pDb->nLogSz; + int iVal = *piVal; + pDb->nAutockpt = (i64)iVal * 1024; + } + *piVal = (int)(pDb->nAutockpt / 1024); break; } case LSM_CONFIG_PAGE_SIZE: { int *piVal = va_arg(ap, int *); @@ -258,21 +275,24 @@ } break; } case LSM_CONFIG_BLOCK_SIZE: { + /* This parameter is read and written in KB. But all internal + ** processing is done in bytes. */ int *piVal = va_arg(ap, int *); if( pDb->pDatabase ){ /* If lsm_open() has been called, this is a read-only parameter. - ** Set the output variable to the page-size according to the - ** FileSystem object. */ - *piVal = lsmFsBlockSize(pDb->pFS); - }else{ - if( *piVal>=65536 && ((*piVal-1) & *piVal)==0 ){ - pDb->nDfltBlksz = *piVal; - }else{ - *piVal = pDb->nDfltBlksz; + ** Set the output variable to the block-size in KB according to the + ** FileSystem object. */ + *piVal = lsmFsBlockSize(pDb->pFS) / 1024; + }else{ + int iVal = *piVal; + if( iVal>=64 && iVal<=65536 && ((iVal-1) & iVal)==0 ){ + pDb->nDfltBlksz = iVal * 1024; + }else{ + *piVal = pDb->nDfltBlksz / 1024; } } break; } @@ -285,12 +305,13 @@ break; } case LSM_CONFIG_MMAP: { int *piVal = va_arg(ap, int *); - if( pDb->pDatabase==0 ){ - pDb->bMmap = (LSM_IS_64_BIT && *piVal); + if( pDb->iReader<0 && *piVal>=0 && *piVal<=1 ){ + pDb->bMmap = *piVal; + rc = lsmFsConfigure(pDb); } *piVal = pDb->bMmap; break; } @@ -301,11 +322,11 @@ } *piVal = pDb->bUseLog; break; } - case LSM_CONFIG_NMERGE: { + case LSM_CONFIG_AUTOMERGE: { int *piVal = va_arg(ap, int *); if( *piVal>1 ) pDb->nMerge = *piVal; *piVal = pDb->nMerge; break; } @@ -329,24 +350,54 @@ }else{ pDb->bMultiProc = *piVal = (*piVal!=0); } break; } + + case LSM_CONFIG_READONLY: { + int *piVal = va_arg(ap, int *); + /* If lsm_open() has been called, this is a read-only parameter. */ + if( pDb->pDatabase==0 && *piVal>=0 ){ + pDb->bReadonly = *piVal = (*piVal!=0); + } + *piVal = pDb->bReadonly; + break; + } case LSM_CONFIG_SET_COMPRESSION: { - int *p = va_arg(ap, lsm_compress *); - if( pDb->pDatabase ){ - /* If lsm_open() has been called, this call is against the rules. */ + lsm_compress *p = va_arg(ap, lsm_compress *); + if( pDb->iReader>=0 && pDb->bInFactory==0 ){ + /* May not change compression schemes with an open transaction */ rc = LSM_MISUSE_BKPT; }else{ - memcpy(&pDb->compress, p, sizeof(lsm_compress)); + if( pDb->compress.xFree ){ + /* Invoke any destructor belonging to the current compression. */ + pDb->compress.xFree(pDb->compress.pCtx); + } + if( p->xBound==0 ){ + memset(&pDb->compress, 0, sizeof(lsm_compress)); + pDb->compress.iId = LSM_COMPRESSION_NONE; + }else{ + memcpy(&pDb->compress, p, sizeof(lsm_compress)); + } + rc = lsmFsConfigure(pDb); + } + break; + } + + case LSM_CONFIG_SET_COMPRESSION_FACTORY: { + lsm_compress_factory *p = va_arg(ap, lsm_compress_factory *); + if( pDb->factory.xFree ){ + /* Invoke any destructor belonging to the current factory. */ + pDb->factory.xFree(pDb->factory.pCtx); } + memcpy(&pDb->factory, p, sizeof(lsm_compress_factory)); break; } case LSM_CONFIG_GET_COMPRESSION: { - int *p = va_arg(ap, lsm_compress *); + lsm_compress *p = va_arg(ap, lsm_compress *); memcpy(p, &pDb->compress, sizeof(lsm_compress)); break; } default: @@ -371,11 +422,11 @@ if( !pDb->pWorker ){ rc = lsmBeginWork(pDb); if( rc!=LSM_OK ) return rc; *pbUnlock = 1; } - *pp = pDb->pWorker; + if( pp ) *pp = pDb->pWorker; return rc; } static void infoFreeWorker(lsm_db *pDb, int bUnlock){ if( bUnlock ){ @@ -402,12 +453,12 @@ /* Format the contents of the snapshot as text */ pTopLevel = lsmDbSnapshotLevel(pWorker); lsmStringInit(&s, pDb->pEnv); for(p=pTopLevel; rc==LSM_OK && p; p=p->pNext){ int i; - lsmStringAppendf(&s, "%s{", (s.n ? " " : "")); - lsmAppendSegmentList(&s, "", &p->lhs); + lsmStringAppendf(&s, "%s{%d", (s.n ? " " : ""), (int)p->iAge); + lsmAppendSegmentList(&s, " ", &p->lhs); for(i=0; rc==LSM_OK && inRight; i++){ lsmAppendSegmentList(&s, " ", &p->aRhs[i]); } lsmStringAppend(&s, "}", 1); } @@ -427,19 +478,18 @@ int lsmInfoFreelist(lsm_db *pDb, char **pzOut){ Snapshot *pWorker; /* Worker snapshot */ int bUnlock = 0; LsmString s; - int i; int rc; /* Obtain the worker snapshot */ rc = infoGetWorker(pDb, &pWorker, &bUnlock); if( rc!=LSM_OK ) return rc; lsmStringInit(&s, pDb->pEnv); - rc = lsmWalkFreelist(pDb, infoFreelistCb, &s); + rc = lsmWalkFreelist(pDb, 0, infoFreelistCb, &s); if( rc!=LSM_OK ){ lsmFree(pDb->pEnv, s.z); }else{ *pzOut = s.z; } @@ -446,10 +496,47 @@ /* Release the snapshot and return */ infoFreeWorker(pDb, bUnlock); return rc; } + +static int infoTreeSize(lsm_db *db, int *pnOldKB, int *pnNewKB){ + ShmHeader *pShm = db->pShmhdr; + TreeHeader *p = &pShm->hdr1; + + /* The following code suffers from two race conditions, as it accesses and + ** trusts the contents of shared memory without verifying checksums: + ** + ** * The two values read - TreeHeader.root.nByte and oldroot.nByte - are + ** 32-bit fields. It is assumed that reading from one of these + ** is atomic - that it is not possible to read a partially written + ** garbage value. However the two values may be mutually inconsistent. + ** + ** * TreeHeader.iLogOff is a 64-bit value. And lsmCheckpointLogOffset() + ** reads a 64-bit value from a snapshot stored in shared memory. It + ** is assumed that in each case it is possible to read a partially + ** written garbage value. If this occurs, then the value returned + ** for the size of the "old" tree may reflect the size of an "old" + ** tree that was recently flushed to disk. + ** + ** Given the context in which this function is called (as a result of an + ** lsm_info(LSM_INFO_TREE_SIZE) request), neither of these are considered to + ** be problems. + */ + *pnNewKB = ((int)p->root.nByte + 1023) / 1024; + if( p->iOldShmid ){ + if( p->iOldLog==lsmCheckpointLogOffset(pShm->aSnap1) ){ + *pnOldKB = 0; + }else{ + *pnOldKB = ((int)p->oldroot.nByte + 1023) / 1024; + } + }else{ + *pnOldKB = 0; + } + + return LSM_OK; +} int lsm_info(lsm_db *pDb, int eParam, ...){ int rc = LSM_OK; va_list ap; va_start(ap, eParam); @@ -474,11 +561,11 @@ } case LSM_INFO_ARRAY_STRUCTURE: { Pgno pgno = va_arg(ap, Pgno); char **pzVal = va_arg(ap, char **); - rc = lsmInfoArrayStructure(pDb, pgno, pzVal); + rc = lsmInfoArrayStructure(pDb, 0, pgno, pzVal); break; } case LSM_INFO_ARRAY_PAGES: { Pgno pgno = va_arg(ap, Pgno); @@ -489,11 +576,17 @@ case LSM_INFO_PAGE_HEX_DUMP: case LSM_INFO_PAGE_ASCII_DUMP: { Pgno pgno = va_arg(ap, Pgno); char **pzVal = va_arg(ap, char **); - rc = lsmInfoPageDump(pDb, pgno, (eParam==LSM_INFO_PAGE_HEX_DUMP), pzVal); + int bUnlock = 0; + rc = infoGetWorker(pDb, 0, &bUnlock); + if( rc==LSM_OK ){ + int bHex = (eParam==LSM_INFO_PAGE_HEX_DUMP); + rc = lsmInfoPageDump(pDb, pgno, bHex, pzVal); + } + infoFreeWorker(pDb, bUnlock); break; } case LSM_INFO_LOG_STRUCTURE: { char **pzVal = va_arg(ap, char **); @@ -504,10 +597,33 @@ case LSM_INFO_FREELIST: { char **pzVal = va_arg(ap, char **); rc = lsmInfoFreelist(pDb, pzVal); break; } + + case LSM_INFO_CHECKPOINT_SIZE: { + int *pnKB = va_arg(ap, int *); + rc = lsmCheckpointSize(pDb, pnKB); + break; + } + + case LSM_INFO_TREE_SIZE: { + int *pnOld = va_arg(ap, int *); + int *pnNew = va_arg(ap, int *); + rc = infoTreeSize(pDb, pnOld, pnNew); + break; + } + + case LSM_INFO_COMPRESSION_ID: { + unsigned int *piOut = va_arg(ap, unsigned int *); + if( pDb->pClient ){ + *piOut = pDb->pClient->iCmpId; + }else{ + rc = lsmInfoCompressionId(pDb, piOut); + } + break; + } default: rc = LSM_MISUSE; break; } @@ -621,11 +737,18 @@ int rc; /* Return code */ MultiCursor *pCsr = 0; /* New cursor object */ /* Open a read transaction if one is not already open. */ assert_db_state(pDb); - rc = lsmBeginReadTrans(pDb); + + if( pDb->pShmhdr==0 ){ + assert( pDb->bReadonly ); + rc = lsmBeginRoTrans(pDb); + }else{ + assert( pDb->bRoTrans==0 ); + rc = lsmBeginReadTrans(pDb); + } /* Allocate the multi-cursor. */ if( rc==LSM_OK ) rc = lsmMCursorNew(pDb, &pCsr); /* If an error has occured, set the output to NULL and delete any partially @@ -659,11 +782,11 @@ ** Attempt to seek the cursor to the database entry specified by pKey/nKey. ** If an error occurs (e.g. an OOM or IO error), return an LSM error code. ** Otherwise, return LSM_OK. */ int lsm_csr_seek(lsm_cursor *pCsr, const void *pKey, int nKey, int eSeek){ - return lsmMCursorSeek((MultiCursor *)pCsr, (void *)pKey, nKey, eSeek); + return lsmMCursorSeek((MultiCursor *)pCsr, 0, (void *)pKey, nKey, eSeek); } int lsm_csr_next(lsm_cursor *pCsr){ return lsmMCursorNext((MultiCursor *)pCsr); } @@ -724,17 +847,17 @@ lsmStringClear(&s); } } int lsm_begin(lsm_db *pDb, int iLevel){ - int rc = LSM_OK; + int rc; assert_db_state( pDb ); + rc = (pDb->bReadonly ? LSM_READONLY : LSM_OK); /* A value less than zero means open one more transaction. */ if( iLevel<0 ) iLevel = pDb->nTransOpen + 1; - if( iLevel>pDb->nTransOpen ){ int i; /* Extend the pDb->aTrans[] array if required. */ if( rc==LSM_OK && pDb->nTransAllocnTransOpen - 1); if( iLevelnTransOpen ){ if( iLevel==0 ){ - int bAutowork = 0; - /* Commit the transaction to disk. */ if( rc==LSM_OK ) rc = lsmLogCommit(pDb); if( rc==LSM_OK && pDb->eSafety==LSM_SAFETY_FULL ){ rc = lsmFsSyncLog(pDb->pFS); } @@ -814,21 +935,6 @@ } return rc; } -int lsm_tree_size(lsm_db *db, int *pnOld, int *pnNew){ - ShmHeader *pShm = db->pShmhdr; - - *pnNew = (int)pShm->hdr1.nByte; - *pnOld = 0; - if( pShm->hdr1.iOldShmid ){ - i64 iOff = pShm->hdr1.iOldLog; - if( iOff!=lsmCheckpointLogOffset(pShm->aSnap1) ){ - *pnOld = 1; - } - } - - return LSM_OK; -} - Index: src/lsm_mem.c ================================================================== --- src/lsm_mem.c +++ src/lsm_mem.c @@ -12,28 +12,10 @@ ** ** Helper routines for memory allocation. */ #include "lsmInt.h" -/* Default allocation size. */ -#define CHUNKSIZE 16*1024 - -typedef struct Chunk Chunk; - -struct Chunk { - int iOff; /* Offset of free space within pSpace */ - u8 *aData; /* Pointer to space for user allocations */ - int nData; /* Size of buffer aData, in bytes */ - Chunk *pNext; -}; - -struct Mempool { - Chunk *pFirst; /* First in list of chunks */ - Chunk *pLast; /* Last in list of chunks */ - int nUsed; /* Total number of bytes allocated */ -}; - /* ** The following routines are called internally by LSM sub-routines. In ** this case a valid environment pointer must be supplied. */ void *lsmMalloc(lsm_env *pEnv, size_t N){ @@ -118,125 +100,5 @@ if( zRet ){ memcpy(zRet, zIn, nByte+1); } return zRet; } - - -/* -** Allocate a new Chunk structure (using lsmMalloc()). -*/ -static Chunk * poolChunkNew(lsm_env *pEnv, int nMin){ - Chunk *pChunk; - int nAlloc = LSM_MAX(CHUNKSIZE, nMin + sizeof(Chunk)); - - pChunk = (Chunk *)lsmMalloc(pEnv, nAlloc); - if( pChunk ){ - pChunk->pNext = 0; - pChunk->iOff = 0; - pChunk->aData = (u8 *)&pChunk[1]; - pChunk->nData = nAlloc - sizeof(Chunk); - } - - return pChunk; -} - -/* -** Allocate sz bytes from chunk pChunk. -*/ -static u8 *poolChunkAlloc(Chunk *pChunk, int sz){ - u8 *pRet; /* Pointer value to return */ - assert( sz<=(pChunk->nData - pChunk->iOff) ); - pRet = &pChunk->aData[pChunk->iOff]; - pChunk->iOff += sz; - return pRet; -} - - -int lsmPoolNew(lsm_env *pEnv, Mempool **ppPool){ - int rc = LSM_NOMEM; - Mempool *pPool = 0; - Chunk *pChunk; - - pChunk = poolChunkNew(pEnv, sizeof(Mempool)); - if( pChunk ){ - pPool = (Mempool *)poolChunkAlloc(pChunk, sizeof(Mempool)); - pPool->pFirst = pChunk; - pPool->pLast = pChunk; - pPool->nUsed = 0; - rc = LSM_OK; - } - - *ppPool = pPool; - return rc; -} - -void lsmPoolDestroy(lsm_env *pEnv, Mempool *pPool){ - if( pPool ){ - Chunk *pChunk = pPool->pFirst; - while( pChunk ){ - Chunk *pFree = pChunk; - pChunk = pChunk->pNext; - lsmFree(pEnv, pFree); - } - } -} - -void *lsmPoolMalloc(lsm_env *pEnv, Mempool *pPool, int nByte){ - u8 *pRet = 0; - Chunk *pLast = pPool->pLast; - - nByte = ROUND8(nByte); - if( nByte > (pLast->nData - pLast->iOff) ){ - Chunk *pNew = poolChunkNew(pEnv, nByte); - if( !pNew ) return 0; - pLast->pNext = pNew; - pPool->pLast = pNew; - pLast = pNew; - } - - if( pLast ){ - pRet = poolChunkAlloc(pLast, nByte); - pPool->nUsed += nByte; - } - return (void *)pRet; -} - -void *lsmPoolMallocZero(lsm_env *pEnv, Mempool *pPool, int nByte){ - void *pRet = lsmPoolMalloc(pEnv, pPool, nByte); - if( pRet ) memset(pRet, 0, nByte); - return pRet; -} - -/* -** Return the amount of memory currently allocated from this pool. -*/ -int lsmPoolUsed(Mempool *pPool){ - return pPool->nUsed; -} - -void lsmPoolMark(Mempool *pPool, void **ppChunk, int *piOff){ - *ppChunk = (void *)pPool->pLast; - *piOff = pPool->pLast->iOff; -} - -void lsmPoolRollback(lsm_env *pEnv, Mempool *pPool, void *pChunk, int iOff){ - Chunk *pLast = (Chunk *)pChunk; - Chunk *p; - Chunk *pNext; - -#ifdef LSM_EXPENSIVE_DEBUG - /* Check that pLast is actually in the list of chunks */ - for(p=pPool->pFirst; p!=pLast; p=p->pNext); -#endif - - pPool->nUsed -= (pLast->iOff - iOff); - for(p=pLast->pNext; p; p=pNext){ - pPool->nUsed -= p->iOff; - pNext = p->pNext; - lsmFree(pEnv, p); - } - - pLast->pNext = 0; - pLast->iOff = iOff; - pPool->pLast = pLast; -} Index: src/lsm_shared.c ================================================================== --- src/lsm_shared.c +++ src/lsm_shared.c @@ -32,19 +32,27 @@ ** Database structure. There is one such structure for each distinct ** database accessed by this process. They are stored in the singly linked ** list starting at global variable gShared.pDatabase. Database objects are ** reference counted. Once the number of connections to the associated ** database drops to zero, they are removed from the linked list and deleted. +** +** pFile: +** In multi-process mode, this file descriptor is used to obtain locks +** and to access shared-memory. In single process mode, its only job is +** to hold the exclusive lock on the file. +** */ struct Database { /* Protected by the global mutex (enterGlobalMutex/leaveGlobalMutex): */ char *zName; /* Canonical path to database file */ int nName; /* strlen(zName) */ int nDbRef; /* Number of associated lsm_db handles */ Database *pDbNext; /* Next Database structure in global list */ /* Protected by the local mutex (pClientMutex) */ + int bReadonly; /* True if Database.pFile is read-only */ + int bMultiProc; /* True if running in multi-process mode */ lsm_file *pFile; /* Used for locks/shm in multi-proc mode */ LsmFile *pLsmFile; /* List of deferred closes */ lsm_mutex *pClientMutex; /* Protects the apShmChunk[] and pConn */ int nShmChunk; /* Number of entries in apShmChunk[] array */ void **apShmChunk; /* Array of "shared" memory regions */ @@ -148,10 +156,90 @@ /* Free the memory allocated for the Database struct itself */ lsmFree(pEnv, p); } } + +typedef struct DbTruncateCtx DbTruncateCtx; +struct DbTruncateCtx { + int nBlock; + i64 iInUse; +}; + +static int dbTruncateCb(void *pCtx, int iBlk, i64 iSnapshot){ + DbTruncateCtx *p = (DbTruncateCtx *)pCtx; + if( iBlk!=p->nBlock || (p->iInUse>=0 && iSnapshot>=p->iInUse) ) return 1; + p->nBlock--; + return 0; +} + +static int dbTruncate(lsm_db *pDb, i64 iInUse){ + int rc = LSM_OK; +#if 0 + int i; + DbTruncateCtx ctx; + + assert( pDb->pWorker ); + ctx.nBlock = pDb->pWorker->nBlock; + ctx.iInUse = iInUse; + + rc = lsmWalkFreelist(pDb, 1, dbTruncateCb, (void *)&ctx); + for(i=ctx.nBlock+1; rc==LSM_OK && i<=pDb->pWorker->nBlock; i++){ + rc = freelistAppend(pDb, i, -1); + } + + if( rc==LSM_OK ){ +#ifdef LSM_LOG_FREELIST + if( ctx.nBlock!=pDb->pWorker->nBlock ){ + lsmLogMessage(pDb, 0, + "dbTruncate(): truncated db to %d blocks",ctx.nBlock + ); + } +#endif + pDb->pWorker->nBlock = ctx.nBlock; + } +#endif + return rc; +} + + +/* +** This function is called during database shutdown (when the number of +** connections drops from one to zero). It truncates the database file +** to as small a size as possible without truncating away any blocks that +** contain data. +*/ +static int dbTruncateFile(lsm_db *pDb){ + int rc; + + assert( pDb->pWorker==0 ); + assert( lsmShmAssertLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_EXCL) ); + rc = lsmCheckpointLoadWorker(pDb); + + if( rc==LSM_OK ){ + DbTruncateCtx ctx; + + /* Walk the database free-block-list in reverse order. Set ctx.nBlock + ** to the block number of the last block in the database that actually + ** contains data. */ + ctx.nBlock = pDb->pWorker->nBlock; + ctx.iInUse = -1; + rc = lsmWalkFreelist(pDb, 1, dbTruncateCb, (void *)&ctx); + + /* If the last block that contains data is not already the last block in + ** the database file, truncate the database file so that it is. */ + if( rc==LSM_OK && ctx.nBlock!=pDb->pWorker->nBlock ){ + rc = lsmFsTruncateDb( + pDb->pFS, (i64)ctx.nBlock*lsmFsBlockSize(pDb->pFS) + ); + } + } + + lsmFreeSnapshot(pDb->pEnv, pDb->pWorker); + pDb->pWorker = 0; + return rc; +} static void doDbDisconnect(lsm_db *pDb){ int rc; /* Block for an exclusive lock on DMS1. This lock serializes all calls @@ -178,21 +266,27 @@ rc = lsmFlushTreeToDisk(pDb); } /* Write a checkpoint to disk. */ if( rc==LSM_OK ){ - rc = lsmCheckpointWrite(pDb, 0); + rc = lsmCheckpointWrite(pDb, 1, 0); } - /* If the checkpoint was written successfully, delete the log file */ + /* If the checkpoint was written successfully, delete the log file + ** and, if possible, truncate the database file. */ if( rc==LSM_OK ){ Database *p = pDb->pDatabase; + dbTruncateFile(pDb); lsmFsCloseAndDeleteLog(pDb->pFS); - if( p->pFile ) lsmEnvShmUnmap(pDb->pEnv, p->pFile, 1); + if( p->pFile && p->bMultiProc ) lsmEnvShmUnmap(pDb->pEnv, p->pFile, 1); } } } + + if( pDb->iRwclient>=0 ){ + lsmShmLock(pDb, LSM_LOCK_RWCLIENT(pDb->iRwclient), LSM_LOCK_UNLOCK, 0); + } lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_UNLOCK, 0); lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); pDb->pShmhdr = 0; } @@ -202,12 +296,13 @@ int nUs = 1000; /* us to wait between DMS1 attempts */ int rc; /* Obtain a pointer to the shared-memory header */ assert( pDb->pShmhdr==0 ); - rc = lsmShmChunk(pDb, 0, (void **)&pDb->pShmhdr); + rc = lsmShmCacheChunks(pDb, 1); if( rc!=LSM_OK ) return rc; + pDb->pShmhdr = (ShmHeader *)pDb->apShm[0]; /* Block for an exclusive lock on DMS1. This lock serializes all calls ** to doDbConnect() and doDbDisconnect() across all processes. */ while( 1 ){ rc = lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL, 1); @@ -233,25 +328,52 @@ } }else if( rc==LSM_BUSY ){ rc = LSM_OK; } - /* Take a shared lock on DMS2. This lock "cannot" fail, as connections - ** may only hold an exclusive lock on DMS2 if they first hold an exclusive - ** lock on DMS1. And this connection is currently holding the exclusive - ** lock on DSM1. */ + /* Take a shared lock on DMS2. In multi-process mode this lock "cannot" + ** fail, as connections may only hold an exclusive lock on DMS2 if they + ** first hold an exclusive lock on DMS1. And this connection is currently + ** holding the exclusive lock on DSM1. + ** + ** However, if some other connection has the database open in single-process + ** mode, this operation will fail. In this case, return the error to the + ** caller - the attempt to connect to the db has failed. + */ if( rc==LSM_OK ){ rc = lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_SHARED, 0); - assert( rc!=LSM_BUSY ); } - /* If anything went wrong, unlock DMS2. Unlock DMS1 in any case. */ + /* If anything went wrong, unlock DMS2. Otherwise, try to take an exclusive + ** lock on one of the LSM_LOCK_RWCLIENT() locks. Unlock DMS1 in any case. */ if( rc!=LSM_OK ){ lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_UNLOCK, 0); pDb->pShmhdr = 0; + }else{ + int i; + for(i=0; iiRwclient = i; + if( rc2!=LSM_BUSY ){ + rc = rc2; + break; + } + } } lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); + return rc; +} + +static int dbOpenSharedFd(lsm_env *pEnv, Database *p, int bRoOk){ + int rc; + + rc = lsmEnvOpen(pEnv, p->zName, 0, &p->pFile); + if( rc==LSM_IOERR && bRoOk ){ + rc = lsmEnvOpen(pEnv, p->zName, LSM_OPEN_READONLY, &p->pFile); + p->bReadonly = 1; + } + return rc; } /* ** Return a reference to the shared Database handle for the database @@ -278,11 +400,11 @@ assert( pDb->pDatabase==0 ); rc = enterGlobalMutex(pEnv); if( rc==LSM_OK ){ /* Search the global list for an existing object. TODO: Need something - ** better than the strcmp() below to figure out if a given Database + ** better than the memcmp() below to figure out if a given Database ** object represents the requested file. */ for(p=gShared.pDatabase; p; p=p->pDbNext){ if( nName==p->nName && 0==memcmp(zName, p->zName, nName) ) break; } @@ -291,20 +413,28 @@ p = (Database *)lsmMallocZeroRc(pEnv, sizeof(Database)+nName+1, &rc); /* If the allocation was successful, fill in other fields and ** allocate the client mutex. */ if( rc==LSM_OK ){ + p->bMultiProc = pDb->bMultiProc; p->zName = (char *)&p[1]; p->nName = nName; memcpy((void *)p->zName, zName, nName+1); rc = lsmMutexNew(pEnv, &p->pClientMutex); } - /* If running in multi-process mode and nothing has gone wrong so far, - ** open the shared fd */ - if( rc==LSM_OK && pDb->bMultiProc ){ - rc = lsmEnvOpen(pDb->pEnv, p->zName, &p->pFile); + /* If nothing has gone wrong so far, open the shared fd. And if that + ** succeeds and this connection requested single-process mode, + ** attempt to take the exclusive lock on DMS2. */ + if( rc==LSM_OK ){ + int bReadonly = (pDb->bReadonly && pDb->bMultiProc); + rc = dbOpenSharedFd(pDb->pEnv, p, bReadonly); + } + + if( rc==LSM_OK && p->bMultiProc==0 ){ + assert( p->bReadonly==0 ); + rc = lsmEnvLock(pDb->pEnv, p->pFile, LSM_LOCK_DMS2, LSM_LOCK_EXCL); } if( rc==LSM_OK ){ p->pDbNext = gShared.pDatabase; gShared.pDatabase = p; @@ -328,14 +458,24 @@ } pDb->pDatabase = p; if( rc==LSM_OK ){ assert( p ); - rc = lsmFsOpen(pDb, zName); + rc = lsmFsOpen(pDb, zName, p->bReadonly); } - if( rc==LSM_OK ){ - rc = doDbConnect(pDb); + + /* If the db handle is read-write, then connect to the system now. Run + ** recovery as necessary. Or, if this is a read-only database handle, + ** defer attempting to connect to the system until a read-transaction + ** is opened. */ + if( pDb->bReadonly==0 ){ + if( rc==LSM_OK ){ + rc = doDbConnect(pDb); + } + if( rc==LSM_OK ){ + rc = lsmFsConfigure(pDb); + } } return rc; } @@ -375,38 +515,37 @@ } lsmMutexEnter(pDb->pEnv, p->pClientMutex); for(ppDb=&p->pConn; *ppDb!=pDb; ppDb=&((*ppDb)->pNext)); *ppDb = pDb->pNext; - if( lsmDbMultiProc(pDb) ){ - dbDeferClose(pDb); - } + dbDeferClose(pDb); lsmMutexLeave(pDb->pEnv, p->pClientMutex); enterGlobalMutex(pDb->pEnv); p->nDbRef--; if( p->nDbRef==0 ){ + LsmFile *pIter; + LsmFile *pNext; Database **pp; /* Remove the Database structure from the linked list. */ for(pp=&gShared.pDatabase; *pp!=p; pp=&((*pp)->pDbNext)); *pp = p->pDbNext; - /* Free the Database object and shared memory buffers. */ - if( p->pFile==0 ){ + /* If they were allocated from the heap, free the shared memory chunks */ + if( p->bMultiProc==0 ){ int i; for(i=0; inShmChunk; i++){ lsmFree(pDb->pEnv, p->apShmChunk[i]); } - }else{ - LsmFile *pIter; - LsmFile *pNext; - for(pIter=p->pLsmFile; pIter; pIter=pNext){ - pNext = pIter->pNext; - lsmEnvClose(pDb->pEnv, pIter->pFile); - lsmFree(pDb->pEnv, pIter); - } + } + + /* Close any outstanding file descriptors */ + for(pIter=p->pLsmFile; pIter; pIter=pNext){ + pNext = pIter->pNext; + lsmEnvClose(pDb->pEnv, pIter->pFile); + lsmFree(pDb->pEnv, pIter); } freeDatabase(pDb->pEnv, p); } leaveGlobalMutex(pDb->pEnv); } @@ -427,41 +566,52 @@ ** Context object used by the lsmWalkFreelist() utility. */ typedef struct WalkFreelistCtx WalkFreelistCtx; struct WalkFreelistCtx { lsm_db *pDb; + int bReverse; Freelist *pFreelist; int iFree; int (*xUsr)(void *, int, i64); /* User callback function */ void *pUsrctx; /* User callback context */ + int bDone; /* Set to true after xUsr() returns true */ }; /* ** Callback used by lsmWalkFreelist(). */ static int walkFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ WalkFreelistCtx *p = (WalkFreelistCtx *)pCtx; + const int iDir = (p->bReverse ? -1 : 1); Freelist *pFree = p->pFreelist; + assert( p->bDone==0 ); if( pFree ){ - while( (p->iFree < pFree->nEntry) ){ + while( (p->iFree < pFree->nEntry) && p->iFree>=0 ){ FreelistEntry *pEntry = &pFree->aEntry[p->iFree]; - if( pEntry->iBlk>iBlk ){ + if( (p->bReverse==0 && pEntry->iBlk>iBlk) + || (p->bReverse!=0 && pEntry->iBlkiFree++; + p->iFree += iDir; if( pEntry->iId>=0 && p->xUsr(p->pUsrctx, pEntry->iBlk, pEntry->iId) ){ + p->bDone = 1; return 1; } if( pEntry->iBlk==iBlk ) return 0; } } } - return p->xUsr(p->pUsrctx, iBlk, iSnapshot); + if( p->xUsr(p->pUsrctx, iBlk, iSnapshot) ){ + p->bDone = 1; + return 1; + } + return 0; } /* ** The database handle passed as the first argument must be the worker ** connection. This function iterates through the contents of the current @@ -473,69 +623,92 @@ ** stored within the LSM. This function also merges in any in-memory ** elements. */ int lsmWalkFreelist( lsm_db *pDb, /* Database handle (must be worker) */ + int bReverse, /* True to iterate from largest to smallest */ int (*x)(void *, int, i64), /* Callback function */ void *pCtx /* First argument to pass to callback */ ){ + const int iDir = (bReverse ? -1 : 1); int rc; int iCtx; WalkFreelistCtx ctx[2]; ctx[0].pDb = pDb; + ctx[0].bReverse = bReverse; ctx[0].pFreelist = &pDb->pWorker->freelist; - ctx[0].iFree = 0; + if( ctx[0].pFreelist && bReverse ){ + ctx[0].iFree = ctx[0].pFreelist->nEntry-1; + }else{ + ctx[0].iFree = 0; + } ctx[0].xUsr = walkFreelistCb; ctx[0].pUsrctx = (void *)&ctx[1]; + ctx[0].bDone = 0; ctx[1].pDb = pDb; + ctx[1].bReverse = bReverse; ctx[1].pFreelist = pDb->pFreelist; - ctx[1].iFree = 0; + if( ctx[1].pFreelist && bReverse ){ + ctx[1].iFree = ctx[1].pFreelist->nEntry-1; + }else{ + ctx[1].iFree = 0; + } ctx[1].xUsr = x; ctx[1].pUsrctx = pCtx; - - rc = lsmSortedWalkFreelist(pDb, walkFreelistCb, (void *)&ctx[0]); - - for(iCtx=0; iCtx<2; iCtx++){ - int i; - WalkFreelistCtx *p = &ctx[iCtx]; - for(i=p->iFree; p->pFreelist && rc==LSM_OK && ipFreelist->nEntry; i++){ - FreelistEntry *pEntry = &p->pFreelist->aEntry[i]; - if( pEntry->iId>=0 && p->xUsr(p->pUsrctx, pEntry->iBlk, pEntry->iId) ){ - return LSM_OK; + ctx[1].bDone = 0; + + rc = lsmSortedWalkFreelist(pDb, bReverse, walkFreelistCb, (void *)&ctx[0]); + + if( ctx[0].bDone==0 ){ + for(iCtx=0; iCtx<2; iCtx++){ + int i; + WalkFreelistCtx *p = &ctx[iCtx]; + for(i=p->iFree; + p->pFreelist && rc==LSM_OK && ipFreelist->nEntry && i>=0; + i += iDir + ){ + FreelistEntry *pEntry = &p->pFreelist->aEntry[i]; + if( pEntry->iId>=0 && p->xUsr(p->pUsrctx, pEntry->iBlk, pEntry->iId) ){ + return LSM_OK; + } } } } return rc; } + typedef struct FindFreeblockCtx FindFreeblockCtx; struct FindFreeblockCtx { i64 iInUse; int iRet; + int bNotOne; }; static int findFreeblockCb(void *pCtx, int iBlk, i64 iSnapshot){ FindFreeblockCtx *p = (FindFreeblockCtx *)pCtx; - if( iSnapshotiInUse ){ + if( iSnapshotiInUse && (iBlk!=1 || p->bNotOne==0) ){ p->iRet = iBlk; return 1; } return 0; } -static int findFreeblock(lsm_db *pDb, i64 iInUse, int *piRet){ +static int findFreeblock(lsm_db *pDb, i64 iInUse, int bNotOne, int *piRet){ int rc; /* Return code */ FindFreeblockCtx ctx; /* Context object */ ctx.iInUse = iInUse; ctx.iRet = 0; - rc = lsmWalkFreelist(pDb, findFreeblockCb, (void *)&ctx); + ctx.bNotOne = bNotOne; + rc = lsmWalkFreelist(pDb, 0, findFreeblockCb, (void *)&ctx); *piRet = ctx.iRet; + return rc; } /* ** Allocate a new database file block to write data to, either by extending @@ -543,52 +716,84 @@ ** must be held in order to call this function. ** ** If successful, *piBlk is set to the block number allocated and LSM_OK is ** returned. Otherwise, *piBlk is zeroed and an lsm error code returned. */ -int lsmBlockAllocate(lsm_db *pDb, int *piBlk){ +int lsmBlockAllocate(lsm_db *pDb, int iBefore, int *piBlk){ Snapshot *p = pDb->pWorker; int iRet = 0; /* Block number of allocated block */ int rc = LSM_OK; i64 iInUse = 0; /* Snapshot id still in use */ + i64 iSynced = 0; /* Snapshot id synced to disk */ assert( p ); + +#ifdef LSM_LOG_FREELIST + { + static int nCall = 0; + char *zFree = 0; + nCall++; + rc = lsmInfoFreelist(pDb, &zFree); + if( rc!=LSM_OK ) return rc; + lsmLogMessage(pDb, 0, "lsmBlockAllocate(): %d freelist: %s", nCall, zFree); + lsmFree(pDb->pEnv, zFree); + } +#endif /* Set iInUse to the smallest snapshot id that is either: ** ** * Currently in use by a database client, ** * May be used by a database client in the future, or ** * Is the most recently checkpointed snapshot (i.e. the one that will ** be used following recovery if a failure occurs at this point). */ - rc = lsmCheckpointSynced(pDb, &iInUse, 0, 0); - if( rc==LSM_OK && iInUse==0 ) iInUse = p->iId; - if( rc==LSM_OK && pDb->pClient ) iInUse = LSM_MIN(iInUse, pDb->pClient->iId); + rc = lsmCheckpointSynced(pDb, &iSynced, 0, 0); + if( rc==LSM_OK && iSynced==0 ) iSynced = p->iId; + iInUse = iSynced; + if( rc==LSM_OK && pDb->iReader>=0 ){ + assert( pDb->pClient ); + iInUse = LSM_MIN(iInUse, pDb->pClient->iId); + } if( rc==LSM_OK ) rc = firstSnapshotInUse(pDb, &iInUse); + +#ifdef LSM_LOG_FREELIST + { + lsmLogMessage(pDb, 0, "lsmBlockAllocate(): " + "snapshot-in-use: %lld (iSynced=%lld) (client-id=%lld)", + iInUse, iSynced, (pDb->iReader>=0 ? pDb->pClient->iId : 0) + ); + } +#endif /* Query the free block list for a suitable block */ - if( rc==LSM_OK ) rc = findFreeblock(pDb, iInUse, &iRet); + if( rc==LSM_OK ) rc = findFreeblock(pDb, iInUse, (iBefore>0), &iRet); - /* If a block was found in the free block list, use it and remove it from - ** the list. Otherwise, if no suitable block was found, allocate one from - ** the end of the file. */ - if( rc==LSM_OK ){ + if( iBefore>0 && (iRet<=0 || iRet>=iBefore) ){ + iRet = 0; + + }else if( rc==LSM_OK ){ + /* If a block was found in the free block list, use it and remove it from + ** the list. Otherwise, if no suitable block was found, allocate one from + ** the end of the file. */ if( iRet>0 ){ #ifdef LSM_LOG_FREELIST lsmLogMessage(pDb, 0, "reusing block %d (snapshot-in-use=%lld)", iRet, iInUse); #endif rc = freelistAppend(pDb, iRet, -1); + if( rc==LSM_OK ){ + rc = dbTruncate(pDb, iInUse); + } }else{ iRet = ++(p->nBlock); #ifdef LSM_LOG_FREELIST lsmLogMessage(pDb, 0, "extending file to %d blocks", iRet); #endif } } - assert( iRet>0 || rc!=LSM_OK ); + assert( iBefore>0 || iRet>0 || rc!=LSM_OK ); *piBlk = iRet; return rc; } /* @@ -619,11 +824,10 @@ ** block may be reused immediately. Whereas a freed block can not be reused ** until (at least) after the next checkpoint. */ int lsmBlockRefree(lsm_db *pDb, int iBlk){ int rc = LSM_OK; /* Return code */ - Snapshot *p = pDb->pWorker; #ifdef LSM_LOG_FREELIST lsmLogMessage(pDb, LSM_OK, "lsmBlockRefree(): Refree block %d", iBlk); #endif @@ -638,11 +842,11 @@ ** The WORKER lock must not be held when this is called. This is because ** this function may indirectly call fsync(). And the WORKER lock should ** not be held that long (in case it is required by a client flushing an ** in-memory tree to disk). */ -int lsmCheckpointWrite(lsm_db *pDb, u32 *pnWrite){ +int lsmCheckpointWrite(lsm_db *pDb, int bTruncate, u32 *pnWrite){ int rc; /* Return Code */ u32 nWrite = 0; assert( pDb->pWorker==0 ); assert( 1 || pDb->pClient==0 ); @@ -651,10 +855,11 @@ rc = lsmShmLock(pDb, LSM_LOCK_CHECKPOINTER, LSM_LOCK_EXCL, 0); if( rc!=LSM_OK ) return rc; rc = lsmCheckpointLoad(pDb, 0); if( rc==LSM_OK ){ + int nBlock = lsmCheckpointNBlock(pDb->aSnapshot); ShmHeader *pShm = pDb->pShmhdr; int bDone = 0; /* True if checkpoint is already stored */ /* Check if this checkpoint has already been written to the database ** file. If so, set variable bDone to true. */ @@ -675,30 +880,31 @@ bDone = (iDisk>=iCkpt); } if( rc==LSM_OK && bDone==0 ){ int iMeta = (pShm->iMetaPage % 2) + 1; -#if 0 - lsmLogMessage(pDb, 0, "starting checkpoint"); -#endif if( pDb->eSafety!=LSM_SAFETY_OFF ){ - rc = lsmFsSyncDb(pDb->pFS); + rc = lsmFsSyncDb(pDb->pFS, nBlock); } if( rc==LSM_OK ) rc = lsmCheckpointStore(pDb, iMeta); if( rc==LSM_OK && pDb->eSafety!=LSM_SAFETY_OFF){ - rc = lsmFsSyncDb(pDb->pFS); + rc = lsmFsSyncDb(pDb->pFS, 0); } if( rc==LSM_OK ){ pShm->iMetaPage = iMeta; nWrite = lsmCheckpointNWrite(pDb->aSnapshot, 0) - nWrite; } #ifdef LSM_LOG_WORK - lsmLogMessage(pDb, 0, "finish checkpoint %d", - (int)lsmCheckpointId(pDb->aSnapshot, 0) - ); + lsmLogMessage(pDb, 0, "finish checkpoint %d", + (int)lsmCheckpointId(pDb->aSnapshot, 0) + ); #endif } + + if( rc==LSM_OK && bTruncate ){ + rc = lsmFsTruncateDb(pDb->pFS, (i64)nBlock*lsmFsBlockSize(pDb->pFS)); + } } lsmShmLock(pDb, LSM_LOCK_CHECKPOINTER, LSM_LOCK_UNLOCK, 0); if( pnWrite && rc==LSM_OK ) *pnWrite = nWrite; return rc; @@ -711,65 +917,160 @@ rc = lsmShmLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL, 0); /* Deserialize the current worker snapshot */ if( rc==LSM_OK ){ rc = lsmCheckpointLoadWorker(pDb); - if( pDb->pWorker ) pDb->pWorker->pDatabase = pDb->pDatabase; } return rc; } void lsmFreeSnapshot(lsm_env *pEnv, Snapshot *p){ if( p ){ lsmSortedFreeLevel(pEnv, p->pLevel); lsmFree(pEnv, p->freelist.aEntry); + lsmFree(pEnv, p->redirect.a); lsmFree(pEnv, p); } } + +/* +** Attempt to populate one of the read-lock slots to contain lock values +** iLsm/iShm. Or, if such a slot exists already, this function is a no-op. +** +** It is not an error if no slot can be populated because the write-lock +** cannot be obtained. If any other error occurs, return an LSM error code. +** Otherwise, LSM_OK. +** +** This function is called at various points to try to ensure that there +** always exists at least one read-lock slot that can be used by a read-only +** client. And so that, in the usual case, there is an "exact match" available +** whenever a read transaction is opened by any client. At present this +** function is called when: +** +** * A write transaction that called lsmTreeDiscardOld() is committed, and +** * Whenever the working snapshot is updated (i.e. lsmFinishWork()). +*/ +static int dbSetReadLock(lsm_db *db, i64 iLsm, u32 iShm){ + int rc = LSM_OK; + ShmHeader *pShm = db->pShmhdr; + int i; + + /* Check if there is already a slot containing the required values. */ + for(i=0; iaReader[i]; + if( p->iLsmId==iLsm && p->iTreeId==iShm ) return LSM_OK; + } + + /* Iterate through all read-lock slots, attempting to take a write-lock + ** on each of them. If a write-lock succeeds, populate the locked slot + ** with the required values and break out of the loop. */ + for(i=0; rc==LSM_OK && iaReader[i]; + p->iLsmId = iLsm; + p->iTreeId = iShm; + lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_UNLOCK, 0); + break; + } + } + + return rc; +} + /* ** Argument bFlush is true if the contents of the in-memory tree has just ** been flushed to disk. The significance of this is that once the snapshot ** created to hold the updated state of the database is synced to disk, log ** file space can be recycled. */ void lsmFinishWork(lsm_db *pDb, int bFlush, int *pRc){ - assert( *pRc!=0 || pDb->pWorker ); + int rc = *pRc; + assert( rc!=0 || pDb->pWorker ); if( pDb->pWorker ){ /* If no error has occurred, serialize the worker snapshot and write ** it to shared memory. */ - if( *pRc==LSM_OK ){ - *pRc = lsmSaveWorker(pDb, bFlush); + if( rc==LSM_OK ){ + rc = lsmSaveWorker(pDb, bFlush); + } + + /* Assuming no error has occurred, update a read lock slot with the + ** new snapshot id (see comments above function dbSetReadLock()). */ + if( rc==LSM_OK ){ + if( pDb->iReader<0 ){ + rc = lsmTreeLoadHeader(pDb, 0); + } + if( rc==LSM_OK ){ + rc = dbSetReadLock(pDb, pDb->pWorker->iId, pDb->treehdr.iUsedShmid); + } } + + /* Free the snapshot object. */ lsmFreeSnapshot(pDb->pEnv, pDb->pWorker); pDb->pWorker = 0; } lsmShmLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_UNLOCK, 0); + *pRc = rc; } - /* ** Called when recovery is finished. */ int lsmFinishRecovery(lsm_db *pDb){ lsmTreeEndTransaction(pDb, 1); return LSM_OK; } + +/* +** Check if the currently configured compression functions +** (LSM_CONFIG_SET_COMPRESSION) are compatible with a database that has its +** compression id set to iReq. Compression routines are compatible if iReq +** is zero (indicating the database is empty), or if it is equal to the +** compression id of the configured compression routines. +** +** If the check shows that the current compression are incompatible and there +** is a compression factory registered, give it a chance to install new +** compression routines. +** +** If, after any registered factory is invoked, the compression functions +** are still incompatible, return LSM_MISMATCH. Otherwise, LSM_OK. +*/ +int lsmCheckCompressionId(lsm_db *pDb, u32 iReq){ + if( iReq!=LSM_COMPRESSION_EMPTY && pDb->compress.iId!=iReq ){ + if( pDb->factory.xFactory ){ + pDb->bInFactory = 1; + pDb->factory.xFactory(pDb->factory.pCtx, pDb, iReq); + pDb->bInFactory = 0; + } + if( pDb->compress.iId!=iReq ){ + /* Incompatible */ + return LSM_MISMATCH; + } + } + /* Compatible */ + return LSM_OK; +} /* ** Begin a read transaction. This function is a no-op if the connection ** passed as the only argument already has an open read transaction. */ int lsmBeginReadTrans(lsm_db *pDb){ const int MAX_READLOCK_ATTEMPTS = 10; + const int nMaxAttempt = (pDb->bRoTrans ? 1 : MAX_READLOCK_ATTEMPTS); + int rc = LSM_OK; /* Return code */ int iAttempt = 0; + assert( pDb->pWorker==0 ); - while( rc==LSM_OK && pDb->iReader<0 && (iAttempt++)iReader<0 && (iAttempt++)pCsr==0 && pDb->nTransOpen==0 ); /* Load the in-memory tree header. */ @@ -790,11 +1091,11 @@ ** that the shared-memory still contains the same values. If so, proceed. ** Otherwise, relinquish the read-lock and retry the whole procedure ** (starting with loading the in-memory tree header). */ if( rc==LSM_OK ){ u32 iShmMax = pDb->treehdr.iUsedShmid; - u32 iShmMin = pDb->treehdr.iNextShmid+1-(1<<10); + u32 iShmMin = pDb->treehdr.iNextShmid+1-LSM_MAX_SHMCHUNKS; rc = lsmReadlock( pDb, lsmCheckpointId(pDb->aSnapshot, 0), iShmMin, iShmMax ); if( rc==LSM_OK ){ if( lsmTreeLoadHeaderOk(pDb, iTreehdr) @@ -806,15 +1107,22 @@ ** version of the snapshot. */ if( pDb->pClient==0 ){ rc = lsmCheckpointDeserialize(pDb, 0, pDb->aSnapshot,&pDb->pClient); } assert( (rc==LSM_OK)==(pDb->pClient!=0) ); - assert( pDb->iReader>=0 ); + assert( pDb->iReader>=0 || pDb->bRoTrans ); + + /* Check that the client has the right compression hooks loaded. + ** If not, set rc to LSM_MISMATCH. */ + if( rc==LSM_OK ){ + rc = lsmCheckCompressionId(pDb, pDb->pClient->iCmpId); + } }else{ rc = lsmReleaseReadlock(pDb); } } + if( rc==LSM_BUSY ){ rc = LSM_OK; } } #if 0 @@ -828,14 +1136,55 @@ ); } #endif } + if( rc==LSM_OK ){ + rc = lsmShmCacheChunks(pDb, pDb->treehdr.nChunk); + } if( rc!=LSM_OK ){ lsmReleaseReadlock(pDb); } if( pDb->pClient==0 && rc==LSM_OK ) rc = LSM_BUSY; + return rc; +} + +/* +** db is a read-only database handle in the disconnected state. This function +** attempts to open a read-transaction on the database. This may involve +** connecting to the database system (opening shared memory etc.). +*/ +int lsmBeginRoTrans(lsm_db *db){ + int rc = LSM_OK; + + assert( db->bReadonly && db->pShmhdr==0 ); + assert( db->iReader<0 ); + + if( db->bRoTrans==0 ){ + if( 1 ){ + rc = lsmShmLock(db, LSM_LOCK_CHECKPOINTER, LSM_LOCK_SHARED, 0); + if( rc==LSM_OK ){ + db->bRoTrans = 1; + rc = lsmShmCacheChunks(db, 1); + if( rc==LSM_OK ){ + db->pShmhdr = (ShmHeader *)db->apShm[0]; + memset(db->pShmhdr, 0, sizeof(ShmHeader)); + rc = lsmCheckpointRecover(db); + if( rc==LSM_OK ){ + rc = lsmLogRecover(db); + } + } + } + }else{ + /* lock(DMS2, SHARED) etc. */ + } + + if( rc==LSM_OK ){ + rc = lsmBeginReadTrans(db); + } + } + return rc; } /* ** Close the currently open read transaction. @@ -847,25 +1196,24 @@ ** transactions have been closed. Finally pClient should be non-NULL ** only iff pDb->iReader>=0. */ assert( pDb->pWorker==0 ); assert( pDb->pCsr==0 && pDb->nTransOpen==0 ); -#if 0 - if( pClient ){ - lsmFreeSnapshot(pDb->pEnv, pDb->pClient); - pDb->pClient = 0; - } -#endif - -#if 0 -if( pDb->pClient && pDb->iReader>=0 ){ - fprintf(stderr, - "finished reading %p: snapshot:%d\n", (void *)pDb, (int)pDb->pClient->iId - ); -} -#endif - if( pDb->iReader>=0 ) lsmReleaseReadlock(pDb); + lsmReleaseReadlock(pDb); + if( pDb->bRoTrans ){ + int i; + for(i=0; inShm; i++){ + lsmFree(pDb->pEnv, pDb->apShm[i]); + } + lsmFree(pDb->pEnv, pDb->apShm); + pDb->apShm = 0; + pDb->nShm = 0; + pDb->pShmhdr = 0; + + lsmShmLock(pDb, LSM_LOCK_CHECKPOINTER, LSM_LOCK_UNLOCK, 0); + pDb->bRoTrans = 0; + } } /* ** Open a write transaction. */ @@ -872,10 +1220,11 @@ int lsmBeginWriteTrans(lsm_db *pDb){ int rc; /* Return code */ ShmHeader *pShm = pDb->pShmhdr; /* Shared memory header */ assert( pDb->nTransOpen==0 ); + assert( pDb->bDiscardOld==0 ); /* If there is no read-transaction open, open one now. */ rc = lsmBeginReadTrans(pDb); /* Attempt to take the WRITER lock */ @@ -906,10 +1255,11 @@ TreeHeader *p = &pDb->treehdr; pShm->bWriter = 1; p->root.iTransId++; if( lsmTreeHasOld(pDb) && p->iOldLog==pDb->pClient->iLogOff ){ lsmTreeDiscardOld(pDb); + pDb->bDiscardOld = 1; } }else{ lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_UNLOCK, 0); if( pDb->pCsr==0 ) lsmFinishReadTrans(pDb); } @@ -938,14 +1288,20 @@ bFlush = 1; lsmTreeMakeOld(pDb); } lsmTreeEndTransaction(pDb, bCommit); - if( rc==LSM_OK && bFlush && pDb->bAutowork ){ - rc = lsmSortedAutoWork(pDb, 1); + if( rc==LSM_OK ){ + if( bFlush && pDb->bAutowork ){ + rc = lsmSortedAutoWork(pDb, 1); + }else if( bCommit && pDb->bDiscardOld ){ + rc = dbSetReadLock(pDb, pDb->pClient->iId, pDb->treehdr.iUsedShmid); + } } + pDb->bDiscardOld = 0; lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_UNLOCK, 0); + if( bFlush && pDb->bAutowork==0 && pDb->xWork ){ pDb->xWork(pDb, pDb->pWorkCtx); } return rc; } @@ -978,10 +1334,13 @@ ShmHeader *pShm = db->pShmhdr; int i; assert( db->iReader<0 ); assert( shm_sequence_ge(iShmMax, iShmMin) ); + + /* This is a no-op if the read-only transaction flag is set. */ + if( db->bRoTrans ) return LSM_OK; /* Search for an exact match. */ for(i=0; db->iReader<0 && rc==LSM_OK && iaReader[i]; if( p->iLsmId==iLsm && p->iTreeId==iShmMax ){ @@ -1139,11 +1498,11 @@ ** This function may only be called after a successful call to ** lsmDbDatabaseConnect(). It returns true if the connection is in ** multi-process mode, or false otherwise. */ int lsmDbMultiProc(lsm_db *pDb){ - return pDb->pDatabase && (pDb->pDatabase->pFile!=0); + return pDb->pDatabase && pDb->pDatabase->bMultiProc; } /************************************************************************* ************************************************************************** @@ -1151,69 +1510,94 @@ ************************************************************************** ************************************************************************** *************************************************************************/ /* -** Retrieve a pointer to shared-memory chunk iChunk. Chunks are numbered -** starting from 0 (i.e. the header chunk is chunk 0). -*/ -int lsmShmChunk(lsm_db *db, int iChunk, void **ppData){ - int rc = LSM_OK; - void *pRet = 0; - Database *p = db->pDatabase; - lsm_env *pEnv = db->pEnv; - - while( iChunk>=db->nShm ){ - void **apShm; - apShm = lsmRealloc(pEnv, db->apShm, sizeof(void*)*(db->nShm+16)); - if( !apShm ) return LSM_NOMEM_BKPT; - memset(&apShm[db->nShm], 0, sizeof(void*)*16); - db->apShm = apShm; - db->nShm += 16; - } - if( db->apShm[iChunk] ){ - *ppData = db->apShm[iChunk]; - return rc; - } - - /* Enter the client mutex */ - assert( iChunk>=0 ); - lsmMutexEnter(pEnv, p->pClientMutex); - - if( iChunk>=p->nShmChunk ){ - int nNew = iChunk+1; - void **apNew; - apNew = (void **)lsmRealloc(pEnv, p->apShmChunk, sizeof(void*) * nNew); - if( apNew==0 ){ - rc = LSM_NOMEM_BKPT; - }else{ - memset(&apNew[p->nShmChunk], 0, sizeof(void*) * (nNew-p->nShmChunk)); - p->apShmChunk = apNew; - p->nShmChunk = nNew; - } - } - - if( rc==LSM_OK && p->apShmChunk[iChunk]==0 ){ - void *pChunk = 0; - if( p->pFile==0 ){ - /* Single process mode */ - pChunk = lsmMallocZeroRc(pEnv, LSM_SHM_CHUNK_SIZE, &rc); - }else{ - /* Multi-process mode */ - rc = lsmEnvShmMap(pEnv, p->pFile, iChunk, LSM_SHM_CHUNK_SIZE, &pChunk); - } - p->apShmChunk[iChunk] = pChunk; - } - - if( rc==LSM_OK ){ - pRet = p->apShmChunk[iChunk]; - } - - /* Release the client mutex */ - lsmMutexLeave(pEnv, p->pClientMutex); - - *ppData = db->apShm[iChunk] = pRet; +** Ensure that database connection db has cached pointers to at least the +** first nChunk chunks of shared memory. +*/ +int lsmShmCacheChunks(lsm_db *db, int nChunk){ + int rc = LSM_OK; + if( nChunk>db->nShm ){ + static const int NINCR = 16; + Database *p = db->pDatabase; + lsm_env *pEnv = db->pEnv; + int nAlloc; + int i; + + /* Ensure that the db->apShm[] array is large enough. If an attempt to + ** allocate memory fails, return LSM_NOMEM immediately. The apShm[] array + ** is always extended in multiples of 16 entries - so the actual allocated + ** size can be inferred from nShm. */ + nAlloc = ((db->nShm + NINCR - 1) / NINCR) * NINCR; + while( nChunk>=nAlloc ){ + void **apShm; + nAlloc += NINCR; + apShm = lsmRealloc(pEnv, db->apShm, sizeof(void*)*nAlloc); + if( !apShm ) return LSM_NOMEM_BKPT; + db->apShm = apShm; + } + + if( db->bRoTrans ){ + for(i=db->nShm; rc==LSM_OK && iapShm[i] = lsmMallocZeroRc(pEnv, LSM_SHM_CHUNK_SIZE, &rc); + db->nShm++; + } + + }else{ + + /* Enter the client mutex */ + lsmMutexEnter(pEnv, p->pClientMutex); + + /* Extend the Database objects apShmChunk[] array if necessary. Using the + ** same pattern as for the lsm_db.apShm[] array above. */ + nAlloc = ((p->nShmChunk + NINCR - 1) / NINCR) * NINCR; + while( nChunk>=nAlloc ){ + void **apShm; + nAlloc += NINCR; + apShm = lsmRealloc(pEnv, p->apShmChunk, sizeof(void*)*nAlloc); + if( !apShm ){ + rc = LSM_NOMEM_BKPT; + break; + } + p->apShmChunk = apShm; + } + + for(i=db->nShm; rc==LSM_OK && i=p->nShmChunk ){ + void *pChunk = 0; + if( p->bMultiProc==0 ){ + /* Single process mode */ + pChunk = lsmMallocZeroRc(pEnv, LSM_SHM_CHUNK_SIZE, &rc); + }else{ + /* Multi-process mode */ + rc = lsmEnvShmMap(pEnv, p->pFile, i, LSM_SHM_CHUNK_SIZE, &pChunk); + } + if( rc==LSM_OK ){ + p->apShmChunk[i] = pChunk; + p->nShmChunk++; + } + } + if( rc==LSM_OK ){ + db->apShm[i] = p->apShmChunk[i]; + db->nShm++; + } + } + + /* Release the client mutex */ + lsmMutexLeave(pEnv, p->pClientMutex); + } + } + + return rc; +} + +static int lockSharedFile(lsm_env *pEnv, Database *p, int iLock, int eOp){ + int rc = LSM_OK; + if( p->bMultiProc ){ + rc = lsmEnvLock(pEnv, p->pFile, iLock, eOp); + } return rc; } /* ** Attempt to obtain the lock identified by the iLock and bExcl parameters. @@ -1229,17 +1613,18 @@ int iLock, int eOp, /* One of LSM_LOCK_UNLOCK, SHARED or EXCL */ int bBlock /* True for a blocking lock */ ){ lsm_db *pIter; - const u32 me = (1 << (iLock-1)); - const u32 ms = (1 << (iLock+16-1)); + const u64 me = ((u64)1 << (iLock-1)); + const u64 ms = ((u64)1 << (iLock+32-1)); int rc = LSM_OK; Database *p = db->pDatabase; - assert( iLock>=1 && iLock<=LSM_LOCK_READER(LSM_LOCK_NREADER-1) ); - assert( iLock<=16 ); + assert( eOp!=LSM_LOCK_EXCL || db->bReadonly==0 ); + assert( iLock>=1 && iLock<=LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT-1) ); + assert( LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT-1)<=32 ); assert( eOp==LSM_LOCK_UNLOCK || eOp==LSM_LOCK_SHARED || eOp==LSM_LOCK_EXCL ); /* Check for a no-op. Proceed only if this is not one of those. */ if( (eOp==LSM_LOCK_UNLOCK && (db->mLock & (me|ms))!=0) || (eOp==LSM_LOCK_SHARED && (db->mLock & (me|ms))!=ms) @@ -1266,21 +1651,21 @@ assert( nExcl==0 || (db->mLock & (me|ms))==0 ); switch( eOp ){ case LSM_LOCK_UNLOCK: if( nShared==0 ){ - lsmEnvLock(db->pEnv, p->pFile, iLock, LSM_LOCK_UNLOCK); + lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_UNLOCK); } db->mLock &= ~(me|ms); break; case LSM_LOCK_SHARED: if( nExcl ){ rc = LSM_BUSY; }else{ if( nShared==0 ){ - rc = lsmEnvLock(db->pEnv, p->pFile, iLock, LSM_LOCK_SHARED); + rc = lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_SHARED); } db->mLock |= ms; db->mLock &= ~me; } break; @@ -1288,11 +1673,11 @@ default: assert( eOp==LSM_LOCK_EXCL ); if( nExcl || nShared ){ rc = LSM_BUSY; }else{ - rc = lsmEnvLock(db->pEnv, p->pFile, iLock, LSM_LOCK_EXCL); + rc = lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_EXCL); db->mLock |= (me|ms); } break; } @@ -1303,12 +1688,12 @@ } #ifdef LSM_DEBUG int shmLockType(lsm_db *db, int iLock){ - const u32 me = (1 << (iLock-1)); - const u32 ms = (1 << (iLock+16-1)); + const u64 me = ((u64)1 << (iLock-1)); + const u64 ms = ((u64)1 << (iLock+32-1)); if( db->mLock & me ) return LSM_LOCK_EXCL; if( db->mLock & ms ) return LSM_LOCK_SHARED; return LSM_LOCK_UNLOCK; } @@ -1396,26 +1781,26 @@ void lsmShmBarrier(lsm_db *db){ lsmEnvShmBarrier(db->pEnv); } -int lsm_checkpoint(lsm_db *pDb, int *pnByte){ +int lsm_checkpoint(lsm_db *pDb, int *pnKB){ int rc; /* Return code */ u32 nWrite = 0; /* Number of pages checkpointed */ /* Attempt the checkpoint. If successful, nWrite is set to the number of ** pages written between this and the previous checkpoint. */ - rc = lsmCheckpointWrite(pDb, &nWrite); + rc = lsmCheckpointWrite(pDb, 0, &nWrite); - /* If required, calculate the output variable (bytes of data checkpointed). + /* If required, calculate the output variable (KB of data checkpointed). ** Set it to zero if an error occured. */ - if( pnByte ){ - int nByte = 0; + if( pnKB ){ + int nKB = 0; if( rc==LSM_OK && nWrite ){ - nByte = (int)nWrite * lsmFsPageSize(pDb->pFS); + nKB = (((i64)nWrite * lsmFsPageSize(pDb->pFS)) + 1023) / 1024; } - *pnByte = nByte; + *pnKB = nKB; } return rc; } Index: src/lsm_sorted.c ================================================================== --- src/lsm_sorted.c +++ src/lsm_sorted.c @@ -70,10 +70,13 @@ #ifndef _LSM_INT_H # include "lsmInt.h" #endif #include "sqlite4.h" /* only for sqlite4_snprintf() */ +#define LSM_LOG_STRUCTURE 0 +#define LSM_LOG_DATA 0 + /* ** Macros to help decode record types. */ #define rtTopic(eType) ((eType) & LSM_SYSTEMKEY) #define rtIsDelete(eType) (((eType) & 0x0F)==LSM_POINT_DELETE) @@ -222,11 +225,11 @@ ** cursor of a multi-cursor. */ #define CURSOR_DATA_TREE0 0 /* Current tree cursor (apTreeCsr[0]) */ #define CURSOR_DATA_TREE1 1 /* The "old" tree, if any (apTreeCsr[1]) */ #define CURSOR_DATA_SYSTEM 2 /* Free-list entries (new-toplevel only) */ -#define CURSOR_DATA_SEGMENT 3 +#define CURSOR_DATA_SEGMENT 3 /* First segment pointer (aPtr[0]) */ /* ** CURSOR_IGNORE_DELETE ** If set, this cursor will not visit SORTED_DELETE keys. ** @@ -340,11 +343,11 @@ + ((u32)aOut[1] << 16) + ((u32)aOut[2] << 8) + ((u32)aOut[3]); } -u32 lsmGetU64(u8 *aOut){ +u64 lsmGetU64(u8 *aOut){ return ((u64)aOut[0] << 56) + ((u64)aOut[1] << 48) + ((u64)aOut[2] << 40) + ((u64)aOut[3] << 32) + ((u64)aOut[4] << 24) @@ -393,10 +396,11 @@ if( pBlob->pData ) lsmFree(pBlob->pEnv, pBlob->pData); memset(pBlob, 0, sizeof(Blob)); } static int sortedReadData( + Segment *pSeg, Page *pPg, int iOff, int nByte, void **ppData, Blob *pBlob @@ -446,11 +450,11 @@ i -= iEnd; /* Grab the next page in the segment */ do { - rc = lsmFsDbPageNext(0, pPg, 1, &pNext); + rc = lsmFsDbPageNext(pSeg, pPg, 1, &pNext); if( rc==LSM_OK && pNext==0 ){ rc = LSM_CORRUPT_BKPT; } if( rc ) break; lsmFsPageRelease(pPg); @@ -507,10 +511,11 @@ lsmVarintGet64(&aCell[1], &iRet); return iRet; } static u8 *pageGetKey( + Segment *pSeg, /* Segment pPg belongs to */ Page *pPg, /* Page to read from */ int iCell, /* Index of cell on page to read */ int *piTopic, /* OUT: Topic associated with this key */ int *pnKey, /* OUT: Size of key in bytes */ Blob *pBlob /* If required, use this for dynamic memory */ @@ -533,26 +538,27 @@ if( rtIsWrite(eType) ){ pKey += lsmVarintGet32(pKey, &nDummy); } *piTopic = rtTopic(eType); - sortedReadData(pPg, pKey-aData, *pnKey, (void **)&pKey, pBlob); + sortedReadData(pSeg, pPg, pKey-aData, *pnKey, (void **)&pKey, pBlob); return pKey; } static int pageGetKeyCopy( lsm_env *pEnv, /* Environment handle */ + Segment *pSeg, /* Segment pPg belongs to */ Page *pPg, /* Page to read from */ int iCell, /* Index of cell on page to read */ int *piTopic, /* OUT: Topic associated with this key */ Blob *pBlob /* If required, use this for dynamic memory */ ){ int rc = LSM_OK; int nKey; u8 *aKey; - aKey = pageGetKey(pPg, iCell, piTopic, &nKey, pBlob); + aKey = pageGetKey(pSeg, pPg, iCell, piTopic, &nKey, pBlob); assert( (void *)aKey!=pBlob->pData || nKey==pBlob->nData ); if( (void *)aKey!=pBlob->pData ){ rc = sortedBlobSet(pEnv, pBlob, aKey, nKey); } @@ -577,10 +583,11 @@ #define GETVARINT64(a, i) (((i)=((u8*)(a))[0])<=240?1:lsmVarintGet64((a), &(i))) #define GETVARINT32(a, i) (((i)=((u8*)(a))[0])<=240?1:lsmVarintGet32((a), &(i))) static int pageGetBtreeKey( + Segment *pSeg, /* Segment page pPg belongs to */ Page *pPg, int iKey, Pgno *piPtr, int *piTopic, void **ppKey, @@ -603,13 +610,13 @@ if( eType==0 ){ int rc; Pgno iRef; /* Page number of referenced page */ Page *pRef; aCell += GETVARINT64(aCell, iRef); - rc = lsmFsDbPageGet(lsmPageFS(pPg), iRef, &pRef); + rc = lsmFsDbPageGet(lsmPageFS(pPg), pSeg, iRef, &pRef); if( rc!=LSM_OK ) return rc; - pageGetKeyCopy(lsmPageEnv(pPg), pRef, 0, &eType, pBlob); + pageGetKeyCopy(lsmPageEnv(pPg), pSeg, pRef, 0, &eType, pBlob); lsmFsPageRelease(pRef); *ppKey = pBlob->pData; *pnKey = pBlob->nData; }else{ aCell += GETVARINT32(aCell, *pnKey); @@ -634,10 +641,11 @@ iCell = pCsr->aPg[iPg].iCell-1; } if( iPg<0 || iCell<0 ) return LSM_CORRUPT_BKPT; rc = pageGetBtreeKey( + pCsr->pSeg, pCsr->aPg[iPg].pPage, iCell, &dummy, &pCsr->eType, &pCsr->pKey, &pCsr->nKey, &pCsr->blob ); pCsr->eType |= LSM_SEPARATOR; } @@ -695,11 +703,11 @@ iLoad = btreeCursorPtr(aData, nData, pPg->iCell); do { Page *pLoad; pCsr->iPg++; - rc = lsmFsDbPageGet(pCsr->pFS, iLoad, &pLoad); + rc = lsmFsDbPageGet(pCsr->pFS, pCsr->pSeg, iLoad, &pLoad); pCsr->aPg[pCsr->iPg].pPage = pLoad; pCsr->aPg[pCsr->iPg].iCell = 0; if( rc==LSM_OK ){ if( pCsr->iPg==(pCsr->nDepth-1) ) break; aData = fsPageData(pLoad, &nData); @@ -740,11 +748,11 @@ Page *pPg = 0; FileSystem *pFS = pCsr->pFS; int iPg = pCsr->pSeg->iRoot; do { - rc = lsmFsDbPageGet(pFS, iPg, &pPg); + rc = lsmFsDbPageGet(pFS, pCsr->pSeg, iPg, &pPg); assert( (rc==LSM_OK)==(pPg!=0) ); if( rc==LSM_OK ){ u8 *aData; int nData; int flags; @@ -830,10 +838,11 @@ if( p->iPg ){ lsm_env *pEnv = lsmFsEnv(pCsr->pFS); int iCell; /* Current cell number on leaf page */ Pgno iLeaf; /* Page number of current leaf page */ int nDepth; /* Depth of b-tree structure */ + Segment *pSeg = pCsr->pSeg; /* Decode the MergeInput structure */ iLeaf = p->iPg; nDepth = (p->iCell & 0x00FF); iCell = (p->iCell >> 8) - 1; @@ -842,24 +851,25 @@ assert( pCsr->aPg==0 ); pCsr->aPg = (BtreePg *)lsmMallocZeroRc(pEnv, sizeof(BtreePg) * nDepth, &rc); /* Populate the last entry of the aPg[] array */ if( rc==LSM_OK ){ + Page **pp = &pCsr->aPg[nDepth-1].pPage; pCsr->iPg = nDepth-1; pCsr->nDepth = nDepth; pCsr->aPg[pCsr->iPg].iCell = iCell; - rc = lsmFsDbPageGet(pCsr->pFS, iLeaf, &pCsr->aPg[nDepth-1].pPage); + rc = lsmFsDbPageGet(pCsr->pFS, pSeg, iLeaf, pp); } /* Populate any other aPg[] array entries */ if( rc==LSM_OK && nDepth>1 ){ Blob blob = {0,0,0}; void *pSeek; int nSeek; int iTopicSeek; int iPg = 0; - int iLoad = pCsr->pSeg->iRoot; + int iLoad = pSeg->iRoot; Page *pPg = pCsr->aPg[nDepth-1].pPage; if( pageObjGetNRec(pPg)==0 ){ /* This can happen when pPg is the right-most leaf in the b-tree. ** In this case, set the iTopicSeek/pSeek/nSeek key to a value @@ -868,18 +878,18 @@ iTopicSeek = 1000; pSeek = 0; nSeek = 0; }else{ Pgno dummy; - rc = pageGetBtreeKey(pPg, + rc = pageGetBtreeKey(pSeg, pPg, 0, &dummy, &iTopicSeek, &pSeek, &nSeek, &pCsr->blob ); } do { Page *pPg; - rc = lsmFsDbPageGet(pCsr->pFS, iLoad, &pPg); + rc = lsmFsDbPageGet(pCsr->pFS, pSeg, iLoad, &pPg); assert( rc==LSM_OK || pPg==0 ); if( rc==LSM_OK ){ u8 *aData; /* Buffer containing page data */ int nData; /* Size of aData[] in bytes */ int iMin; @@ -899,11 +909,13 @@ void *pKey; int nKey; /* Key for cell iTry */ int iTopic; /* Topic for key pKeyT/nKeyT */ Pgno iPtr; /* Pointer for cell iTry */ int res; /* (pSeek - pKeyT) */ - rc = pageGetBtreeKey(pPg, iTry, &iPtr, &iTopic, &pKey, &nKey,&blob); + rc = pageGetBtreeKey( + pSeg, pPg, iTry, &iPtr, &iTopic, &pKey, &nKey, &blob + ); if( rc!=LSM_OK ) break; res = sortedKeyCompare( xCmp, iTopicSeek, pSeek, nSeek, iTopic, pKey, nKey ); @@ -919,11 +931,13 @@ } pCsr->aPg[iPg].pPage = pPg; pCsr->aPg[iPg].iCell = iCell; iPg++; - assert( iPg!=nDepth-1 || iLoad==iLeaf ); + assert( iPg!=nDepth-1 + || lsmFsRedirectPage(pCsr->pFS, pSeg->pRedirect, iLoad)==iLeaf + ); } }while( rc==LSM_OK && iPg<(nDepth-1) ); sortedBlobFree(&blob); } @@ -941,11 +955,11 @@ int i; for(i=pCsr->iPg-1; i>=0; i--){ if( pCsr->aPg[i].iCell>0 ) break; } assert( i>=0 ); - rc = pageGetBtreeKey( + rc = pageGetBtreeKey(pSeg, pCsr->aPg[i].pPage, pCsr->aPg[i].iCell-1, &dummy, &pCsr->eType, &pCsr->pKey, &pCsr->nKey, &pCsr->blob ); pCsr->eType |= LSM_SEPARATOR; @@ -998,11 +1012,11 @@ int iNew /* Page number of new page */ ){ Page *pPg = 0; /* The new page */ int rc; /* Return Code */ - rc = lsmFsDbPageGet(pFS, iNew, &pPg); + rc = lsmFsDbPageGet(pFS, pPtr->pSeg, iNew, &pPg); assert( rc==LSM_OK || pPg==0 ); segmentPtrSetPage(pPtr, pPg); return rc; } @@ -1012,11 +1026,11 @@ int iOff, int nByte, void **ppData, Blob *pBlob ){ - return sortedReadData(pPtr->pPg, iOff, nByte, ppData, pBlob); + return sortedReadData(pPtr->pSeg, pPtr->pPg, iOff, nByte, ppData, pBlob); } static int segmentPtrNextPage( SegmentPtr *pPtr, /* Load page into this SegmentPtr object */ int eDir /* +1 for next(), -1 for prev() */ @@ -1071,18 +1085,39 @@ } return rc; } -void lsmSortedSplitkey(lsm_db *pDb, Level *pLevel, int *pRc){ + +static Segment *sortedSplitkeySegment(Level *pLevel){ + Merge *pMerge = pLevel->pMerge; + MergeInput *p = &pMerge->splitkey; + Segment *pSeg; + int i; + + for(i=0; inInput; i++){ + if( p->iPg==pMerge->aInput[i].iPg ) break; + } + if( pMerge->nInput==(pLevel->nRight+1) && i>=(pMerge->nInput-1) ){ + pSeg = &pLevel->pNext->lhs; + }else{ + pSeg = &pLevel->aRhs[i]; + } + + return pSeg; +} + +static void sortedSplitkey(lsm_db *pDb, Level *pLevel, int *pRc){ + Segment *pSeg; Page *pPg = 0; lsm_env *pEnv = pDb->pEnv; /* Environment handle */ int rc = *pRc; Merge *pMerge = pLevel->pMerge; + pSeg = sortedSplitkeySegment(pLevel); if( rc==LSM_OK ){ - rc = lsmFsDbPageGet(pDb->pFS, pMerge->splitkey.iPg, &pPg); + rc = lsmFsDbPageGet(pDb->pFS, pSeg, pMerge->splitkey.iPg, &pPg); } if( rc==LSM_OK ){ int iTopic; Blob blob = {0, 0, 0, 0}; u8 *aData; @@ -1091,18 +1126,20 @@ aData = lsmFsPageData(pPg, &nData); if( pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG ){ void *pKey; int nKey; Pgno dummy; - rc = pageGetBtreeKey( + rc = pageGetBtreeKey(pSeg, pPg, pMerge->splitkey.iCell, &dummy, &iTopic, &pKey, &nKey, &blob ); if( rc==LSM_OK && blob.pData!=pKey ){ rc = sortedBlobSet(pEnv, &blob, pKey, nKey); } }else{ - rc = pageGetKeyCopy(pEnv, pPg, pMerge->splitkey.iCell, &iTopic, &blob); + rc = pageGetKeyCopy( + pEnv, pSeg, pPg, pMerge->splitkey.iCell, &iTopic, &blob + ); } pLevel->iSplitTopic = iTopic; pLevel->pSplitKey = blob.pData; pLevel->nSplitKey = blob.nData; @@ -1170,14 +1207,17 @@ rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey ); if( res<0 ) segmentPtrReset(pPtr); } + if( pPtr->pPg==0 && (svFlags & LSM_END_DELETE) ){ - rc = lsmFsDbPageGet(pCsr->pDb->pFS, pPtr->pSeg->iFirst, &pPtr->pPg); + Segment *pSeg = pPtr->pSeg; + rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, pSeg->iFirst, &pPtr->pPg); if( rc!=LSM_OK ) return rc; - pPtr->eType = LSM_START_DELETE | (pLvl->iSplitTopic ? LSM_SYSTEMKEY : 0); + pPtr->eType = LSM_START_DELETE | LSM_POINT_DELETE; + pPtr->eType |= (pLvl->iSplitTopic ? LSM_SYSTEMKEY : 0); pPtr->pKey = pLvl->pSplitKey; pPtr->nKey = pLvl->nSplitKey; } }while( pCsr @@ -1194,15 +1234,16 @@ SegmentPtr *pPtr, int bLast, int *pRc ){ if( *pRc==LSM_OK ){ + Segment *pSeg = pPtr->pSeg; Page *pNew = 0; if( bLast ){ - *pRc = lsmFsDbPageLast(pFS, pPtr->pSeg, &pNew); + *pRc = lsmFsDbPageLast(pFS, pSeg, &pNew); }else{ - *pRc = lsmFsDbPageGet(pFS, pPtr->pSeg->iFirst, &pNew); + *pRc = lsmFsDbPageGet(pFS, pSeg, pSeg->iFirst, &pNew); } segmentPtrSetPage(pPtr, pNew); } } @@ -1228,18 +1269,25 @@ rc = segmentPtrNextPage(pPtr, (bLast ? -1 : 1)); } if( rc==LSM_OK && pPtr->pPg ){ rc = segmentPtrLoadCell(pPtr, bLast ? (pPtr->nCell-1) : 0); + if( rc==LSM_OK && bLast && pPtr->pSeg!=&pLvl->lhs ){ + int res = sortedKeyCompare(pCsr->pDb->xCmp, + rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, + pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey + ); + if( res<0 ) segmentPtrReset(pPtr); + } } bIgnore = segmentPtrIgnoreSeparators(pCsr, pPtr); if( rc==LSM_OK && pPtr->pPg && bIgnore && rtIsSeparator(pPtr->eType) ){ rc = segmentPtrAdvance(pCsr, pPtr, bLast); } - +#if 0 if( bLast && rc==LSM_OK && pPtr->pPg && pPtr->pSeg==&pLvl->lhs && pLvl->nRight && (pPtr->eType & LSM_START_DELETE) ){ pPtr->iCell++; @@ -1247,10 +1295,11 @@ pPtr->pKey = pLvl->pSplitKey; pPtr->nKey = pLvl->nSplitKey; pPtr->pVal = 0; pPtr->nVal = 0; } +#endif return rc; } static void segmentPtrKey(SegmentPtr *pPtr, void **ppKey, int *pnKey){ @@ -1291,13 +1340,14 @@ for(eDir=-1; eDir<=1; eDir+=2){ Page *pTest = pPtr->pPg; lsmFsPageRef(pTest); while( pTest ){ + Segment *pSeg = pPtr->pSeg; Page *pNext; - int rc = lsmFsDbPageNext(pPtr->pSeg, pTest, eDir, &pNext); + int rc = lsmFsDbPageNext(pSeg, pTest, eDir, &pNext); lsmFsPageRelease(pTest); if( rc ) return 1; pTest = pNext; if( pTest ){ @@ -1311,11 +1361,11 @@ u8 *pPgKey; int res; int iCell; iCell = ((eDir < 0) ? (nCell-1) : 0); - pPgKey = pageGetKey(pTest, iCell, &iPgTopic, &nPgKey, &blob); + pPgKey = pageGetKey(pSeg, pTest, iCell, &iPgTopic, &nPgKey, &blob); res = iTopic - iPgTopic; if( res==0 ) res = pCsr->pDb->xCmp(pKey, nKey, pPgKey, nPgKey); if( (eDir==1 && res>0) || (eDir==-1 && res<0) ){ /* Taking this branch means something has gone wrong. */ char *zMsg = lsmMallocPrintf(pEnv, "Key \"%s\" is not on page %d", @@ -1361,10 +1411,11 @@ #endif static int segmentPtrSearchOversized( MultiCursor *pCsr, /* Cursor context */ SegmentPtr *pPtr, /* Pointer to seek */ + int iTopic, /* Topic of key to search for */ void *pKey, int nKey /* Key to seek to */ ){ int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; int rc = LSM_OK; @@ -1380,19 +1431,19 @@ int iLastTopic; int res; /* Result of comparison */ Page *pNext; /* Load the last key on the current page. */ - pLastKey = pageGetKey( + pLastKey = pageGetKey(pPtr->pSeg, pPtr->pPg, pPtr->nCell-1, &iLastTopic, &nLastKey, &pPtr->blob1 ); /* If the loaded key is >= than (pKey/nKey), break out of the loop. ** If (pKey/nKey) is present in this array, it must be on the current ** page. */ res = sortedKeyCompare( - xCmp, iLastTopic, pLastKey, nLastKey, 0, pKey, nKey + xCmp, iLastTopic, pLastKey, nLastKey, iTopic, pKey, nKey ); if( res>=0 ) break; /* Advance to the next page that contains at least one key. */ pNext = pPtr->pPg; @@ -1568,10 +1619,11 @@ } static int segmentPtrSeek( MultiCursor *pCsr, /* Cursor context */ SegmentPtr *pPtr, /* Pointer to seek */ + int iTopic, /* Key topic to seek to */ void *pKey, int nKey, /* Key to seek to */ int eSeek, /* Search bias - see above */ int *piPtr, /* OUT: FC pointer */ int *pbStop ){ @@ -1579,17 +1631,16 @@ int res; /* Result of comparison operation */ int rc = LSM_OK; int iMin; int iMax; Pgno iPtrOut = 0; - const int iTopic = 0; /* If the current page contains an oversized entry, then there are no ** pointers to one or more of the subsequent pages in the sorted run. ** The following call ensures that the segment-ptr points to the correct ** page in this case. */ - rc = segmentPtrSearchOversized(pCsr, pPtr, pKey, nKey); + rc = segmentPtrSearchOversized(pCsr, pPtr, iTopic, pKey, nKey); iPtrOut = pPtr->iPtr; /* Assert that this page is the right page of this segment for the key ** that we are searching for. Do this by loading page (iPg-1) and testing ** that pKey/nKey is greater than all keys on that page, and then by @@ -1704,30 +1755,30 @@ } static int seekInBtree( MultiCursor *pCsr, /* Multi-cursor object */ Segment *pSeg, /* Seek within this segment */ + int iTopic, void *pKey, int nKey, /* Key to seek to */ Pgno *aPg, /* OUT: Page numbers */ Page **ppPg /* OUT: Leaf (sorted-run) page reference */ ){ int i = 0; int rc; int iPg; Page *pPg = 0; Blob blob = {0, 0, 0}; - int iTopic = 0; /* TODO: Fix me */ iPg = pSeg->iRoot; do { Pgno *piFirst = 0; if( aPg ){ aPg[i++] = iPg; piFirst = &aPg[i]; } - rc = lsmFsDbPageGet(pCsr->pDb->pFS, iPg, &pPg); + rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, iPg, &pPg); assert( rc==LSM_OK || pPg==0 ); if( rc==LSM_OK ){ u8 *aData; /* Buffer containing page data */ int nData; /* Size of aData[] in bytes */ int iMin; @@ -1749,11 +1800,13 @@ void *pKeyT; int nKeyT; /* Key for cell iTry */ int iTopicT; /* Topic for key pKeyT/nKeyT */ Pgno iPtr; /* Pointer associated with cell iTry */ int res; /* (pKey - pKeyT) */ - rc = pageGetBtreeKey(pPg, iTry, &iPtr, &iTopicT, &pKeyT, &nKeyT, &blob); + rc = pageGetBtreeKey( + pSeg, pPg, iTry, &iPtr, &iTopicT, &pKeyT, &nKeyT, &blob + ); if( rc!=LSM_OK ) break; if( piFirst && pKeyT==blob.pData ){ *piFirst = pageGetBtreeRef(pPg, iTry); piFirst = 0; i++; @@ -1785,10 +1838,11 @@ } static int seekInSegment( MultiCursor *pCsr, SegmentPtr *pPtr, + int iTopic, void *pKey, int nKey, int iPg, /* Page to search */ int eSeek, /* Search bias - see above */ int *piPtr, /* OUT: FC pointer */ int *pbStop /* OUT: Stop search flag */ @@ -1797,11 +1851,11 @@ int rc = LSM_OK; if( pPtr->pSeg->iRoot ){ Page *pPg; assert( pPtr->pSeg->iRoot!=0 ); - rc = seekInBtree(pCsr, pPtr->pSeg, pKey, nKey, 0, &pPg); + rc = seekInBtree(pCsr, pPtr->pSeg, iTopic, pKey, nKey, 0, &pPg); if( rc==LSM_OK ) segmentPtrSetPage(pPtr, pPg); }else{ if( iPtr==0 ){ iPtr = pPtr->pSeg->iFirst; } @@ -1809,11 +1863,11 @@ rc = segmentPtrLoadPage(pCsr->pDb->pFS, pPtr, iPtr); } } if( rc==LSM_OK ){ - rc = segmentPtrSeek(pCsr, pPtr, pKey, nKey, eSeek, piPtr, pbStop); + rc = segmentPtrSeek(pCsr, pPtr, iTopic, pKey, nKey, eSeek, piPtr, pbStop); } return rc; } /* @@ -1832,10 +1886,11 @@ */ static int seekInLevel( MultiCursor *pCsr, /* Sorted cursor object to seek */ SegmentPtr *aPtr, /* Pointer to array of (nRhs+1) SPs */ int eSeek, /* Search bias - see above */ + int iTopic, /* Key topic to search for */ void *pKey, int nKey, /* Key to search for */ Pgno *piPgno, /* IN/OUT: fraction cascade pointer (or 0) */ int *pbStop /* OUT: See above */ ){ Level *pLvl = aPtr[0].pLevel; /* Level to seek within */ @@ -1847,11 +1902,11 @@ /* If this is a composite level (one currently undergoing an incremental ** merge), figure out if the search key is larger or smaller than the ** levels split-key. */ if( nRhs ){ - res = sortedKeyCompare(pCsr->pDb->xCmp, 0, pKey, nKey, + res = sortedKeyCompare(pCsr->pDb->xCmp, iTopic, pKey, nKey, pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey ); } /* If (res<0), then key pKey/nKey is smaller than the split-key (or this @@ -1859,26 +1914,44 @@ ** left-hand-side of the level in this case. */ if( res<0 ){ int iPtr = 0; if( nRhs==0 ) iPtr = *piPgno; - rc = seekInSegment(pCsr, &aPtr[0], pKey, nKey, iPtr, eSeek, &iOut, &bStop); + rc = seekInSegment( + pCsr, &aPtr[0], iTopic, pKey, nKey, iPtr, eSeek, &iOut, &bStop + ); if( rc==LSM_OK && nRhs>0 && eSeek==LSM_SEEK_GE && aPtr[0].pPg==0 ){ res = 0; } } if( res>=0 ){ + int bHit = 0; /* True if at least one rhs is not EOF */ int iPtr = *piPgno; int i; for(i=1; rc==LSM_OK && i<=nRhs && bStop==0; i++){ + SegmentPtr *pPtr = &aPtr[i]; iOut = 0; - rc = seekInSegment(pCsr, &aPtr[i], pKey, nKey, iPtr, eSeek, &iOut,&bStop); + rc = seekInSegment( + pCsr, pPtr, iTopic, pKey, nKey, iPtr, eSeek, &iOut, &bStop + ); iPtr = iOut; + + /* If the segment-pointer has settled on a key that is smaller than + ** the splitkey, invalidate the segment-pointer. */ + if( pPtr->pPg ){ + res = sortedKeyCompare(pCsr->pDb->xCmp, + rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, + pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey + ); + if( res<0 ) segmentPtrReset(pPtr); + } + + if( aPtr[i].pKey ) bHit = 1; } - if( rc==LSM_OK && eSeek==LSM_SEEK_LE ){ + if( rc==LSM_OK && eSeek==LSM_SEEK_LE && bHit==0 ){ rc = segmentPtrEnd(pCsr, &aPtr[0], 1); } } assert( eSeek==LSM_SEEK_EQ || bStop==0 ); @@ -2084,16 +2157,47 @@ int bReverse ){ int rc; SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; Level *pLvl = pPtr->pLevel; - int bComposite; + int bComposite; /* True if pPtr is part of composite level */ + /* Advance the segment-pointer object. */ rc = segmentPtrAdvance(pCsr, pPtr, bReverse); if( rc!=LSM_OK ) return rc; + bComposite = (pLvl->nRight>0 && pCsr->nPtr>pLvl->nRight); + if( bComposite && pPtr->pPg==0 ){ + int bFix = 0; + if( (bReverse==0)==(pPtr->pSeg==&pLvl->lhs) ){ + int i; + if( bReverse ){ + SegmentPtr *pLhs = &pCsr->aPtr[iPtr - 1 - (pPtr->pSeg - pLvl->aRhs)]; + for(i=0; inRight; i++){ + if( pLhs[i+1].pPg ) break; + } + if( i==pLvl->nRight ){ + bFix = 1; + rc = segmentPtrEnd(pCsr, pLhs, 1); + } + }else{ + bFix = 1; + for(i=0; rc==LSM_OK && inRight; i++){ + rc = sortedRhsFirst(pCsr, pLvl, &pCsr->aPtr[iPtr+1+i]); + } + } + } + if( bFix ){ + int i; + for(i=pCsr->nTree-1; i>0; i--){ + multiCursorDoCompare(pCsr, i, bReverse); + } + } + } + +#if 0 if( bComposite && pPtr->pSeg==&pLvl->lhs /* lhs of composite level */ && bReverse==0 /* csr advanced forwards */ && pPtr->pPg==0 /* segment at EOF */ ){ int i; @@ -2102,10 +2206,11 @@ } for(i=pCsr->nTree-1; i>0; i--){ multiCursorDoCompare(pCsr, i, 0); } } +#endif return rc; } static void mcursorFreeComponents(MultiCursor *pCsr){ @@ -2208,10 +2313,30 @@ pCsr->aPtr[i].pLevel = pLvl; } return LSM_OK; } + +static void multiCursorAddOne(MultiCursor *pCsr, Level *pLvl, int *pRc){ + if( *pRc==LSM_OK ){ + int iPtr = pCsr->nPtr; + int i; + pCsr->aPtr[iPtr].pLevel = pLvl; + pCsr->aPtr[iPtr].pSeg = &pLvl->lhs; + iPtr++; + for(i=0; inRight; i++){ + pCsr->aPtr[iPtr].pLevel = pLvl; + pCsr->aPtr[iPtr].pSeg = &pLvl->aRhs[i]; + iPtr++; + } + + if( pLvl->nRight && pLvl->pSplitKey==0 ){ + sortedSplitkey(pCsr->pDb, pLvl, pRc); + } + pCsr->nPtr = iPtr; + } +} static int multiCursorAddAll(MultiCursor *pCsr, Snapshot *pSnap){ Level *pLvl; int nPtr = 0; int iPtr = 0; @@ -2227,26 +2352,14 @@ nPtr += (1 + pLvl->nRight); } assert( pCsr->aPtr==0 ); pCsr->aPtr = lsmMallocZeroRc(pCsr->pDb->pEnv, sizeof(SegmentPtr) * nPtr, &rc); - if( rc==LSM_OK ) pCsr->nPtr = nPtr; - - for(pLvl=pSnap->pLevel; pLvl && rc==LSM_OK; pLvl=pLvl->pNext){ - int i; - if( pLvl->flags & LEVEL_INCOMPLETE ) continue; - pCsr->aPtr[iPtr].pLevel = pLvl; - pCsr->aPtr[iPtr].pSeg = &pLvl->lhs; - iPtr++; - for(i=0; inRight; i++){ - pCsr->aPtr[iPtr].pLevel = pLvl; - pCsr->aPtr[iPtr].pSeg = &pLvl->aRhs[i]; - iPtr++; - } - - if( pLvl->nRight && pLvl->pSplitKey==0 ){ - lsmSortedSplitkey(pCsr->pDb, pLvl, &rc); + + for(pLvl=pSnap->pLevel; pLvl; pLvl=pLvl->pNext){ + if( (pLvl->flags & LEVEL_INCOMPLETE)==0 ){ + multiCursorAddOne(pCsr, pLvl, &rc); } } return rc; } @@ -2386,35 +2499,47 @@ assert( rc==LSM_OK || (*ppVal==0 && *pnVal==0) ); return rc; } +static int multiCursorAdvance(MultiCursor *pCsr, int bReverse); + /* ** This function is called by worker connections to walk the part of the ** free-list stored within the LSM data structure. */ int lsmSortedWalkFreelist( lsm_db *pDb, /* Database handle */ + int bReverse, /* True to iterate from largest to smallest */ int (*x)(void *, int, i64), /* Callback function */ void *pCtx /* First argument to pass to callback */ ){ MultiCursor *pCsr; /* Cursor used to read db */ int rc = LSM_OK; /* Return Code */ Snapshot *pSnap = 0; assert( pDb->pWorker ); - rc = lsmCheckpointDeserialize(pDb, 0, pDb->pShmhdr->aSnap1, &pSnap); - if( rc!=LSM_OK ) return rc; + if( pDb->bIncrMerge ){ + rc = lsmCheckpointDeserialize(pDb, 0, pDb->pShmhdr->aSnap1, &pSnap); + if( rc!=LSM_OK ) return rc; + }else{ + pSnap = pDb->pWorker; + } pCsr = multiCursorNew(pDb, &rc); if( pCsr ){ rc = multiCursorAddAll(pCsr, pSnap); pCsr->flags |= CURSOR_IGNORE_DELETE; } if( rc==LSM_OK ){ - rc = lsmMCursorLast(pCsr); + if( bReverse==0 ){ + rc = lsmMCursorLast(pCsr); + }else{ + rc = lsmMCursorSeek(pCsr, 1, "", 0, LSM_SEEK_GE); + } + while( rc==LSM_OK && lsmMCursorValid(pCsr) && rtIsSystem(pCsr->eType) ){ void *pKey; int nKey; void *pVal; int nVal; rc = lsmMCursorKey(pCsr, &pKey, &nKey); @@ -2425,17 +2550,19 @@ int iBlk; i64 iSnap; iBlk = (int)(~(lsmGetU32((u8 *)pKey))); iSnap = (i64)lsmGetU64((u8 *)pVal); if( x(pCtx, iBlk, iSnap) ) break; - rc = lsmMCursorPrev(pCsr); + rc = multiCursorAdvance(pCsr, !bReverse); } } } lsmMCursorClose(pCsr); - lsmFreeSnapshot(pDb->pEnv, pSnap); + if( pSnap!=pDb->pWorker ){ + lsmFreeSnapshot(pDb->pEnv, pSnap); + } return rc; } int lsmSortedLoadFreelist( @@ -2503,32 +2630,118 @@ int nKey; multiCursorGetKey(pCsr, pCsr->aTree[1], &pCsr->eType, &pKey, &nKey); *pRc = sortedBlobSet(pCsr->pDb->pEnv, &pCsr->key, pKey, nKey); } } + +#ifndef NDEBUG +static void assertCursorTree(MultiCursor *pCsr){ + int bRev = !!(pCsr->flags & CURSOR_PREV_OK); + int *aSave = pCsr->aTree; + int nSave = pCsr->nTree; + int rc; + + pCsr->aTree = 0; + pCsr->nTree = 0; + rc = multiCursorAllocTree(pCsr); + if( rc==LSM_OK ){ + int i; + for(i=pCsr->nTree-1; i>0; i--){ + multiCursorDoCompare(pCsr, i, bRev); + } + + assert( nSave==pCsr->nTree + && 0==memcmp(aSave, pCsr->aTree, sizeof(int)*nSave) + ); + + lsmFree(pCsr->pDb->pEnv, pCsr->aTree); + } + + pCsr->aTree = aSave; + pCsr->nTree = nSave; +} +#else +# define assertCursorTree(x) +#endif static int mcursorLocationOk(MultiCursor *pCsr, int bDeleteOk){ int eType = pCsr->eType; int iKey; int i; - int rdmask = 0; + int rdmask; assert( pCsr->flags & (CURSOR_NEXT_OK|CURSOR_PREV_OK) ); - if( pCsr->flags & CURSOR_NEXT_OK ){ - rdmask = LSM_END_DELETE; - }else{ - rdmask = LSM_START_DELETE; - } + assertCursorTree(pCsr); + rdmask = (pCsr->flags & CURSOR_NEXT_OK) ? LSM_END_DELETE : LSM_START_DELETE; + + /* If the cursor does not currently point to an actual database key (i.e. + ** it points to a delete key, or the start or end of a range-delete), and + ** the CURSOR_IGNORE_DELETE flag is set, skip past this entry. */ if( (pCsr->flags & CURSOR_IGNORE_DELETE) && bDeleteOk==0 ){ if( (eType & LSM_INSERT)==0 ) return 0; } + + /* If the cursor points to a system key (free-list entry), and the + ** CURSOR_IGNORE_SYSTEM flag is set, skip thie entry. */ if( (pCsr->flags & CURSOR_IGNORE_SYSTEM) && rtTopic(eType)!=0 ){ return 0; } - /* Check if this key has already been deleted by a range-delete */ +#ifndef NDEBUG + /* This block fires assert() statements to check one of the assumptions + ** in the comment below - that if the lhs sub-cursor of a level undergoing + ** a merge is valid, then all the rhs sub-cursors must be at EOF. + ** + ** Also assert that all rhs sub-cursors are either at EOF or point to + ** a key that is not less than the level split-key. */ + for(i=0; inPtr; i++){ + SegmentPtr *pPtr = &pCsr->aPtr[i]; + Level *pLvl = pPtr->pLevel; + if( pLvl->nRight && pPtr->pPg ){ + if( pPtr->pSeg==&pLvl->lhs ){ + int j; + for(j=0; jnRight; j++) assert( pPtr[j+1].pPg==0 ); + }else{ + int res = sortedKeyCompare(pCsr->pDb->xCmp, + rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, + pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey + ); + assert( res>=0 ); + } + } + } +#endif + + /* Now check if this key has already been deleted by a range-delete. If + ** so, skip past it. + ** + ** Assume, for the moment, that the tree contains no levels currently + ** undergoing incremental merge, and that this cursor is iterating forwards + ** through the database keys. The cursor currently points to a key in + ** level L. This key has already been deleted if any of the sub-cursors + ** that point to levels newer than L (or to the in-memory tree) point to + ** a key greater than the current key with the LSM_END_DELETE flag set. + ** + ** Or, if the cursor is iterating backwards through data keys, if any + ** such sub-cursor points to a key smaller than the current key with the + ** LSM_START_DELETE flag set. + ** + ** Why it works with levels undergoing a merge too: + ** + ** When a cursor iterates forwards, the sub-cursors for the rhs of a + ** level are only activated once the lhs reaches EOF. So when iterating + ** forwards, the keys visited are the same as if the level was completely + ** merged. + ** + ** If the cursor is iterating backwards, then the lhs sub-cursor is not + ** initialized until the last of the rhs sub-cursors has reached EOF. + ** Additionally, if the START_DELETE flag is set on the last entry (in + ** reverse order - so the entry with the smallest key) of a rhs sub-cursor, + ** then a pseudo-key equal to the levels split-key with the END_DELETE + ** flag set is visited by the sub-cursor. + */ iKey = pCsr->aTree[1]; for(i=0; i0 && (rdmask & lsmTreeCursorFlags(pCsr->apTreeCsr[0]))) - || (iKey>1 && (rdmask & lsmTreeCursorFlags(pCsr->apTreeCsr[1]))) - ){ - return 0; - } - if( iKey>CURSOR_DATA_SYSTEM && (pCsr->flags & CURSOR_FLUSH_FREELIST) ){ - int eType; - multiCursorGetKey(pCsr, CURSOR_DATA_SYSTEM, &eType, 0, 0); - if( rdmask & eType ) return 0; - } - - for(i=CURSOR_DATA_SEGMENT; iaPtr[iPtr].pPg && (pCsr->aPtr[iPtr].eType & rdmask) ){ - return 0; - } - } -#endif - + /* The current cursor position is one this cursor should visit. Return 1. */ return 1; } + +static int multiCursorSetupTree(MultiCursor *pCsr, int bRev){ + int rc; + + rc = multiCursorAllocTree(pCsr); + if( rc==LSM_OK ){ + int i; + for(i=pCsr->nTree-1; i>0; i--){ + multiCursorDoCompare(pCsr, i, bRev); + } + } + + assertCursorTree(pCsr); + multiCursorCacheKey(pCsr, &rc); + + if( rc==LSM_OK && mcursorLocationOk(pCsr, 0)==0 ){ + rc = multiCursorAdvance(pCsr, bRev); + } + return rc; +} + static int multiCursorEnd(MultiCursor *pCsr, int bLast){ int rc = LSM_OK; int i; pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK); pCsr->flags |= (bLast ? CURSOR_PREV_OK : CURSOR_NEXT_OK); - - if( pCsr->apTreeCsr[0] ){ - rc = lsmTreeCursorEnd(pCsr->apTreeCsr[0], bLast); - } - if( rc==LSM_OK && pCsr->apTreeCsr[1] ){ - rc = lsmTreeCursorEnd(pCsr->apTreeCsr[1], bLast); - } - pCsr->iFree = 0; + + /* Position the two in-memory tree cursors */ + for(i=0; rc==LSM_OK && i<2; i++){ + if( pCsr->apTreeCsr[i] ){ + rc = lsmTreeCursorEnd(pCsr->apTreeCsr[i], bLast); + } + } for(i=0; rc==LSM_OK && inPtr; i++){ SegmentPtr *pPtr = &pCsr->aPtr[i]; Level *pLvl = pPtr->pLevel; - - rc = segmentPtrEnd(pCsr, pPtr, bLast); - if( rc==LSM_OK && bLast==0 && pLvl->nRight && pPtr->pSeg==&pLvl->lhs ){ - int iRhs; - for(iRhs=1+i; rc==LSM_OK && iRhs<1+i+pLvl->nRight; iRhs++){ - SegmentPtr *pRhs = &pCsr->aPtr[iRhs]; - if( pPtr->pPg==0 ){ - rc = sortedRhsFirst(pCsr, pLvl, pRhs); + int iRhs; + int bHit = 0; + + if( bLast ){ + for(iRhs=0; iRhsnRight && rc==LSM_OK; iRhs++){ + rc = segmentPtrEnd(pCsr, &pPtr[iRhs+1], 1); + if( pPtr[iRhs+1].pPg ) bHit = 1; + } + if( bHit==0 && rc==LSM_OK ){ + rc = segmentPtrEnd(pCsr, pPtr, 1); + }else{ + segmentPtrReset(pPtr); + } + }else{ + int bLhs = (pPtr->pSeg==&pLvl->lhs); + assert( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[0] ); + + if( bLhs ){ + rc = segmentPtrEnd(pCsr, pPtr, 0); + if( pPtr->pKey ) bHit = 1; + } + for(iRhs=0; iRhsnRight && rc==LSM_OK; iRhs++){ + if( bHit ){ + segmentPtrReset(&pPtr[iRhs+1]); }else{ - segmentPtrReset(pRhs); + rc = sortedRhsFirst(pCsr, pLvl, &pPtr[iRhs+bLhs]); } } - i += pLvl->nRight; } + i += pLvl->nRight; } + /* And the b-tree cursor, if applicable */ if( rc==LSM_OK && pCsr->pBtCsr ){ assert( bLast==0 ); rc = btreeCursorFirst(pCsr->pBtCsr); } if( rc==LSM_OK ){ - rc = multiCursorAllocTree(pCsr); - } - if( rc==LSM_OK ){ - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, bLast); - } - } - - multiCursorCacheKey(pCsr, &rc); - if( rc==LSM_OK && mcursorLocationOk(pCsr, 0)==0 ){ - if( bLast ){ - rc = lsmMCursorPrev(pCsr); - }else{ - rc = lsmMCursorNext(pCsr); - } - } - + rc = multiCursorSetupTree(pCsr, bLast); + } + return rc; } int mcursorSave(MultiCursor *pCsr){ @@ -2648,11 +2865,13 @@ int mcursorRestore(lsm_db *pDb, MultiCursor *pCsr){ int rc; rc = multiCursorInit(pCsr, pDb->pClient); if( rc==LSM_OK && pCsr->key.pData ){ - rc = lsmMCursorSeek(pCsr, pCsr->key.pData, pCsr->key.nData, +1); + rc = lsmMCursorSeek(pCsr, + rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, +1 + ); } return rc; } int lsmSaveCursors(lsm_db *pDb){ @@ -2747,16 +2966,24 @@ /* ** Seek the cursor. */ -int lsmMCursorSeek(MultiCursor *pCsr, void *pKey, int nKey, int eSeek){ +int lsmMCursorSeek( + MultiCursor *pCsr, + int iTopic, + void *pKey, int nKey, + int eSeek +){ int eESeek = eSeek; /* Effective eSeek parameter */ int bStop = 0; /* Set to true to halt search operation */ int rc = LSM_OK; /* Return code */ int iPtr = 0; /* Used to iterate through pCsr->aPtr[] */ Pgno iPgno = 0; /* FC pointer value */ + + assert( pCsr->apTreeCsr[0]==0 || iTopic==0 ); + assert( pCsr->apTreeCsr[1]==0 || iTopic==0 ); if( eESeek==LSM_SEEK_LEFAST ) eESeek = LSM_SEEK_LE; assert( eESeek==LSM_SEEK_EQ || eESeek==LSM_SEEK_LE || eESeek==LSM_SEEK_GE ); assert( (pCsr->flags & CURSOR_FLUSH_FREELIST)==0 ); @@ -2770,11 +2997,11 @@ /* Seek all segment pointers. */ for(iPtr=0; iPtrnPtr && rc==LSM_OK && bStop==0; iPtr++){ SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; assert( pPtr->pSeg==&pPtr->pLevel->lhs ); - rc = seekInLevel(pCsr, pPtr, eESeek, pKey, nKey, &iPgno, &bStop); + rc = seekInLevel(pCsr, pPtr, eESeek, iTopic, pKey, nKey, &iPgno, &bStop); iPtr += pPtr->pLevel->nRight; } if( eSeek!=LSM_SEEK_EQ ){ if( rc==LSM_OK ){ @@ -2859,11 +3086,11 @@ ** cursor points to a SORTED_DELETE entry, then the cursor has not been ** successfully advanced. ** ** Similarly, if the cursor is configured to skip system keys and the ** current cursor points to a system key, it has not yet been advanced. - */ + */ if( *pRc==LSM_OK && 0==mcursorLocationOk(pCsr, 0) ) return 0; } return 1; } @@ -2897,10 +3124,12 @@ static int multiCursorAdvance(MultiCursor *pCsr, int bReverse){ int rc = LSM_OK; /* Return Code */ if( lsmMCursorValid(pCsr) ){ do { int iKey = pCsr->aTree[1]; + + assertCursorTree(pCsr); /* If this multi-cursor is advancing forwards, and the sub-cursor ** being advanced is the one that separator keys may be being read ** from, record the current absolute pointer value. */ if( pCsr->pPrevMergePtr ){ @@ -2935,10 +3164,11 @@ if( rc==LSM_OK ){ int i; for(i=(iKey+pCsr->nTree)/2; i>0; i=i/2){ multiCursorDoCompare(pCsr, i, bReverse); } + assertCursorTree(pCsr); } }while( mcursorAdvanceOk(pCsr, bReverse, &rc)==0 ); } return rc; } @@ -3081,32 +3311,46 @@ */ static int mergeWorkerMoveHierarchy( MergeWorker *pMW, /* Merge worker */ int bSep /* True for separators run */ ){ - Segment *pSeg; /* Segment being written */ lsm_db *pDb = pMW->pDb; /* Database handle */ int rc = LSM_OK; /* Return code */ int i; Page **apHier = pMW->hier.apHier; int nHier = pMW->hier.nHier; - pSeg = &pMW->pLevel->lhs; - for(i=0; rc==LSM_OK && ipFS, pDb->pWorker, pSeg, &pNew); + rc = lsmFsSortedAppend(pDb->pFS, pDb->pWorker, pMW->pLevel, 1, &pNew); assert( rc==LSM_OK ); if( rc==LSM_OK ){ u8 *a1; int n1; u8 *a2; int n2; a1 = fsPageData(pNew, &n1); a2 = fsPageData(apHier[i], &n2); + + assert( n1==n2 || n1+4==n2 ); + + if( n1==n2 ){ + memcpy(a1, a2, n2); + }else{ + int nEntry = pageGetNRec(a2, n2); + int iEof1 = SEGMENT_EOF(n1, nEntry); + int iEof2 = SEGMENT_EOF(n2, nEntry); + + memcpy(a1, a2, iEof2 - 4); + memcpy(&a1[iEof1], &a2[iEof2], n2 - iEof2); + } + + lsmFsPageRelease(apHier[i]); + apHier[i] = pNew; + +#if 0 assert( n1==n2 || n1+4==n2 || n2+4==n1 ); - if( n1>=n2 ){ /* If n1 (size of the new page) is equal to or greater than n2 (the ** size of the old page), then copy the data into the new page. If ** n1==n2, this could be done with a single memcpy(). However, ** since sometimes n1>n2, the page content and footer must be copied @@ -3123,10 +3367,11 @@ lsmPutU16(&a1[SEGMENT_NRECORD_OFFSET(n1)], 0); lsmPutU64(&a1[SEGMENT_POINTER_OFFSET(n1)], 0); i = i - 1; lsmFsPageRelease(pNew); } +#endif } } #ifdef LSM_DEBUG if( rc==LSM_OK ){ @@ -3159,11 +3404,11 @@ Page *pPg = 0; u8 *aData; int nData; int flags; - rc = lsmFsDbPageGet(pFS, iPg, &pPg); + rc = lsmFsDbPageGet(pFS, pSeg, iPg, &pPg); if( rc!=LSM_OK ) break; aData = fsPageData(pPg, &nData); flags = pageGetFlags(aData, nData); if( flags&SEGMENT_BTREE_FLAG ){ @@ -3252,11 +3497,10 @@ Pgno iPtr, Pgno iKeyPg, void *pKey, int nKey ){ - Segment *pSeg = &pMW->pLevel->lhs; Hierarchy *p = &pMW->hier; lsm_db *pDb = pMW->pDb; /* Database handle */ int rc = LSM_OK; /* Return Code */ int iLevel; /* Level of b-tree hierachy to write to */ int nData; /* Size of aData[] in bytes */ @@ -3308,10 +3552,11 @@ if( nByte<=nFree ) break; /* Otherwise, this page is full. Set the right-hand-child pointer ** to iPtr and release it. */ lsmPutU64(&aData[SEGMENT_POINTER_OFFSET(nData)], iPtr); + assert( lsmFsPageNumber(pOld)==0 ); rc = lsmFsPagePersist(pOld); if( rc==LSM_OK ){ iPtr = lsmFsPageNumber(pOld); lsmFsPageRelease(pOld); } @@ -3319,11 +3564,11 @@ /* Allocate a new page for apHier[iLevel]. */ p->apHier[iLevel] = 0; if( rc==LSM_OK ){ rc = lsmFsSortedAppend( - pDb->pFS, pDb->pWorker, pSeg, &p->apHier[iLevel] + pDb->pFS, pDb->pWorker, pMW->pLevel, 1, &p->apHier[iLevel] ); } if( rc!=LSM_OK ) return rc; aData = fsPageData(p->apHier[iLevel], &nData); @@ -3387,18 +3632,10 @@ u8 *aData; /* Page data for level iLevel */ int iOff; /* Offset on b-tree page to write record to */ int nRec; /* Initial number of records on b-tree page */ Pgno iPtr; /* Pointer value to accompany pKey/nKey */ - Hierarchy *p; - Segment *pSeg; - - /* If there exists a b-tree hierarchy and it is not loaded into - ** memory, load it now. */ - pSeg = &pMW->pLevel->lhs; - p = &pMW->hier; - assert( pMW->aSave[0].bStore==0 ); assert( pMW->aSave[1].bStore==0 ); rc = mergeWorkerBtreeIndirect(pMW); /* Obtain the absolute pointer value to store along with the key in the @@ -3527,13 +3764,12 @@ int rc = LSM_OK; /* Return code */ Page *pNext = 0; /* New page appended to run */ lsm_db *pDb = pMW->pDb; /* Database handle */ Segment *pSeg; /* Run to append to */ - pSeg = &pMW->pLevel->lhs; - rc = lsmFsSortedAppend(pDb->pFS, pDb->pWorker, pSeg, &pNext); - assert( rc!=LSM_OK || pSeg->iFirst>0 || pMW->pDb->compress.xCompress ); + rc = lsmFsSortedAppend(pDb->pFS, pDb->pWorker, pMW->pLevel, 0, &pNext); + assert( rc || pMW->pLevel->lhs.iFirst>0 || pMW->pDb->compress.xCompress ); if( rc==LSM_OK ){ u8 *aData; /* Data buffer belonging to page pNext */ int nData; /* Size of aData[] in bytes */ @@ -3614,11 +3850,11 @@ rc = LSM_OK; iFPtr = pMW->pLevel->pNext->lhs.iFirst; }else if( pCsr->nPtr>0 ){ Segment *pSeg; pSeg = pCsr->aPtr[pCsr->nPtr-1].pSeg; - rc = lsmFsDbPageGet(pMW->pDb->pFS, pSeg->iFirst, &pPg); + rc = lsmFsDbPageGet(pMW->pDb->pFS, pSeg, pSeg->iFirst, &pPg); if( rc==LSM_OK ){ u8 *aData; /* Buffer for page pPg */ int nData; /* Size of aData[] in bytes */ aData = fsPageData(pPg, &nData); iFPtr = pageGetPtr(aData, nData); @@ -3750,27 +3986,10 @@ return rc; } -static int multiCursorSetupTree(MultiCursor *pCsr, int bRev){ - int rc; - - assert( pCsr->aTree==0 ); - - rc = multiCursorAllocTree(pCsr); - if( rc==LSM_OK ){ - int i; - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, bRev); - } - } - - multiCursorCacheKey(pCsr, &rc); - return rc; -} - /* ** Free all resources allocated by mergeWorkerInit(). */ static void mergeWorkerShutdown(MergeWorker *pMW, int *pRc){ int i; /* Iterator variable */ @@ -3821,10 +4040,11 @@ /* Persist and release the output page. */ if( rc==LSM_OK ) rc = mergeWorkerPersistAndRelease(pMW); if( rc==LSM_OK ) rc = mergeWorkerBtreeIndirect(pMW); if( rc==LSM_OK ) rc = mergeWorkerFinishHierarchy(pMW); if( rc==LSM_OK ) rc = mergeWorkerAddPadding(pMW); + lsmFsFlushWaiting(pMW->pDb->pFS, &rc); mergeWorkerReleaseAll(pMW); lsmFree(pMW->pDb->pEnv, pMW->aGobble); pMW->aGobble = 0; pMW->pCsr = 0; @@ -3955,11 +4175,11 @@ mergeRangeDeletes(pCsr, &iVal, &eType); if( eType!=0 ){ if( pMW->aGobble ){ int iGobble = pCsr->aTree[1] - CURSOR_DATA_SEGMENT; - if( iGobblenPtr ){ + if( iGobblenPtr && iGobble>=0 ){ SegmentPtr *pGobble = &pCsr->aPtr[iGobble]; if( (pGobble->flags & PGFTR_SKIP_THIS_FLAG)==0 ){ pMW->aGobble[iGobble] = lsmFsPageNumber(pGobble->pPg); } } @@ -3985,34 +4205,10 @@ /* Advance the cursor to the next input record (assuming one exists). */ assert( lsmMCursorValid(pMW->pCsr) ); if( rc==LSM_OK ) rc = lsmMCursorNext(pMW->pCsr); - /* If the cursor is at EOF, the merge is finished. Release all page - ** references currently held by the merge worker and inform the - ** FileSystem object that no further pages will be appended to either - ** the main or separators array. - */ - if( rc==LSM_OK && !lsmMCursorValid(pMW->pCsr) ){ - - mergeWorkerShutdown(pMW, &rc); - if( pSeg->iFirst ){ - rc = lsmFsSortedFinish(pDb->pFS, pSeg); - } - -#ifdef LSM_DEBUG_EXPENSIVE - if( rc==LSM_OK ){ -#if 0 - rc = assertBtreeOk(pDb, pSeg); - if( pMW->pCsr->pBtCsr ){ - Segment *pNext = &pMW->pLevel->pNext->lhs; - rc = assertPointersOk(pDb, pSeg, pNext, 0); - } -#endif - } -#endif - } return rc; } static int mergeWorkerDone(MergeWorker *pMW){ return pMW->pCsr==0 || !lsmMCursorValid(pMW->pCsr); @@ -4040,13 +4236,18 @@ ){ int rc = LSM_OK; /* Return Code */ MultiCursor *pCsr = 0; Level *pNext = 0; /* The current top level */ Level *pNew; /* The new level itself */ - Segment *pDel = 0; /* Delete separators from this segment */ + Segment *pLinked = 0; /* Delete separators from this segment */ + Level *pDel = 0; /* Delete this entire level */ int nWrite = 0; /* Number of database pages written */ Freelist freelist; + + if( eTree!=TREE_NONE ){ + rc = lsmShmCacheChunks(pDb, pDb->treehdr.nChunk); + } assert( pDb->bUseFreelist==0 ); pDb->pFreelist = &freelist; pDb->bUseFreelist = 1; memset(&freelist, 0, sizeof(freelist)); @@ -4067,22 +4268,23 @@ pCsr->pDb = pDb; rc = multiCursorVisitFreelist(pCsr); if( rc==LSM_OK ){ rc = multiCursorAddTree(pCsr, pDb->pWorker, eTree); } - if( rc==LSM_OK - && pNext && pNext->pMerge==0 && pNext->lhs.iRoot - && (eTree!=TREE_NONE || (pNext->flags & LEVEL_FREELIST_ONLY)) - ){ - pDel = &pNext->lhs; - rc = btreeCursorNew(pDb, pDel, &pCsr->pBtCsr); + if( rc==LSM_OK && pNext && pNext->pMerge==0 ){ + if( (pNext->flags & LEVEL_FREELIST_ONLY) ){ + pDel = pNext; + pCsr->aPtr = lsmMallocZeroRc(pDb->pEnv, sizeof(SegmentPtr), &rc); + multiCursorAddOne(pCsr, pNext, &rc); + }else if( eTree!=TREE_NONE && pNext->lhs.iRoot ){ + pLinked = &pNext->lhs; + rc = btreeCursorNew(pDb, pLinked, &pCsr->pBtCsr); + } } - if( pNext==0 ){ - multiCursorIgnoreDelete(pCsr); - } - + /* If this will be the only segment in the database, discard any delete + ** markers present in the in-memory tree. */ if( pNext==0 ){ multiCursorIgnoreDelete(pCsr); } } @@ -4109,14 +4311,16 @@ /* Do the work to create the new merged segment on disk */ if( rc==LSM_OK ) rc = lsmMCursorFirst(pCsr); while( rc==LSM_OK && mergeWorkerDone(&mergeworker)==0 ){ rc = mergeWorkerStep(&mergeworker); } + mergeWorkerShutdown(&mergeworker, &rc); assert( rc!=LSM_OK || mergeworker.nWork==0 || pNew->lhs.iFirst ); - + if( rc==LSM_OK && pNew->lhs.iFirst ){ + rc = lsmFsSortedFinish(pDb->pFS, &pNew->lhs); + } nWrite = mergeworker.nWork; - mergeWorkerShutdown(&mergeworker, &rc); pNew->flags &= ~LEVEL_INCOMPLETE; if( eTree==TREE_NONE ){ pNew->flags |= LEVEL_FREELIST_ONLY; } pNew->pMerge = 0; @@ -4125,14 +4329,21 @@ if( rc!=LSM_OK || pNew->lhs.iFirst==0 ){ assert( rc!=LSM_OK || pDb->pWorker->freelist.nEntry==0 ); lsmDbSnapshotSetLevel(pDb->pWorker, pNext); sortedFreeLevel(pDb->pEnv, pNew); }else{ - if( pDel ) pDel->iRoot = 0; + if( pLinked ){ + pLinked->iRoot = 0; + }else if( pDel ){ + assert( pNew->pNext==pDel ); + pNew->pNext = pDel->pNext; + lsmFsSortedDelete(pDb->pFS, pDb->pWorker, 1, &pDel->lhs); + sortedFreeLevel(pDb->pEnv, pDel); + } -#if 0 - lsmSortedDumpStructure(pDb, pDb->pWorker, 0, 0, "new-toplevel"); +#if LSM_LOG_STRUCTURE + lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, "new-toplevel"); #endif if( freelist.nEntry ){ Freelist *p = &pDb->pWorker->freelist; lsmFree(pDb->pEnv, p->aEntry); @@ -4218,11 +4429,13 @@ for(pp=&pTopLevel; *pp!=pLevel; pp=&((*pp)->pNext)); *pp = pNew; lsmDbSnapshotSetLevel(pDb->pWorker, pTopLevel); /* Determine whether or not the next separators will be linked in */ - if( pNext && pNext->pMerge==0 && pNext->lhs.iRoot ){ + if( pNext && pNext->pMerge==0 && pNext->lhs.iRoot && pNext + && (bFreeOnly==0 || (pNext->flags & LEVEL_FREELIST_ONLY)) + ){ bUseNext = 1; } } /* Allocate the merge object */ @@ -4267,10 +4480,11 @@ ** If the new level is the lowest (oldest) in the db, discard any ** delete keys. Key annihilation. */ pCsr = multiCursorNew(pDb, &rc); if( pCsr ){ + pCsr->flags |= CURSOR_NEXT_OK; rc = multiCursorAddRhs(pCsr, pLevel); } if( rc==LSM_OK && pMerge->nInput > pLevel->nRight ){ rc = btreeCursorNew(pDb, &pNext->lhs, &pCsr->pBtCsr); }else if( pNext ){ @@ -4345,11 +4559,13 @@ ** gobbled up to (but not including) the first of these page numbers. */ assert( pSeg->iRoot>0 ); aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(Pgno)*32, &rc); if( rc==LSM_OK ){ - rc = seekInBtree(pCsr, pSeg, pCsr->key.pData, pCsr->key.nData, aPg, 0); + rc = seekInBtree(pCsr, pSeg, + rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, aPg, 0 + ); } if( rc==LSM_OK ){ for(nPg=0; aPg[nPg]; nPg++); lsmFsGobble(pDb, pSeg, aPg, nPg); @@ -4393,12 +4609,12 @@ if( pLevel->nRight==0 && pThis && pLevel->iAge==pThis->iAge ){ nThis++; }else{ if( nThis>nBest ){ if( (pLevel->iAge!=pThis->iAge+1) - || (pLevel->nRight==0 && sortedCountLevels(pLevel)<=pDb->nMerge) - ){ + || (pLevel->nRight==0 && sortedCountLevels(pLevel)<=pDb->nMerge) + ){ pBest = pThis; nBest = nThis; } } if( pLevel->nRight ){ @@ -4457,10 +4673,176 @@ ){ return 1; } return 0; } + +typedef struct MoveBlockCtx MoveBlockCtx; +struct MoveBlockCtx { + int iSeen; /* Previous free block on list */ + int iFrom; /* Total number of blocks in file */ +}; + +static int moveBlockCb(void *pCtx, int iBlk, i64 iSnapshot){ + MoveBlockCtx *p = (MoveBlockCtx *)pCtx; + assert( p->iFrom==0 ); + if( iBlk==(p->iSeen-1) ){ + p->iSeen = iBlk; + return 0; + } + p->iFrom = p->iSeen-1; + return 1; +} + +/* +** This function is called to further compact a database for which all +** of the content has already been merged into a single segment. If +** possible, it moves the contents of a single block from the end of the +** file to a free-block that lies closer to the start of the file (allowing +** the file to be eventually truncated). +*/ +static int sortedMoveBlock(lsm_db *pDb, int *pnWrite){ + Snapshot *p = pDb->pWorker; + Level *pLvl = lsmDbSnapshotLevel(p); + int iFrom; /* Block to move */ + int iTo; /* Destination to move block to */ + int rc; /* Return code */ + + MoveBlockCtx sCtx; + + assert( pLvl->pNext==0 && pLvl->nRight==0 ); + assert( p->redirect.n<=LSM_MAX_BLOCK_REDIRECTS ); + + *pnWrite = 0; + + /* Check that the redirect array is not already full. If it is, return + ** without moving any database content. */ + if( p->redirect.n>=LSM_MAX_BLOCK_REDIRECTS ) return LSM_OK; + + /* Find the last block of content in the database file. Do this by + ** traversing the free-list in reverse (descending block number) order. + ** The first block not on the free list is the one that will be moved. + ** Since the db consists of a single segment, there is no ambiguity as + ** to which segment the block belongs to. */ + sCtx.iSeen = p->nBlock+1; + sCtx.iFrom = 0; + rc = lsmWalkFreelist(pDb, 1, moveBlockCb, &sCtx); + if( rc!=LSM_OK || sCtx.iFrom==0 ) return rc; + iFrom = sCtx.iFrom; + + /* Find the first free block in the database, ignoring block 1. Block + ** 1 is tricky as it is smaller than the other blocks. */ + rc = lsmBlockAllocate(pDb, iFrom, &iTo); + if( rc!=LSM_OK || iTo==0 ) return rc; + assert( iTo!=1 && iTopFS, &pLvl->lhs, iTo, iFrom); + if( rc==LSM_OK ){ + if( p->redirect.a==0 ){ + int nByte = sizeof(struct RedirectEntry) * LSM_MAX_BLOCK_REDIRECTS; + p->redirect.a = lsmMallocZeroRc(pDb->pEnv, nByte, &rc); + } + if( rc==LSM_OK ){ + + /* Check if the block just moved was already redirected. */ + int i; + for(i=0; iredirect.n; i++){ + if( p->redirect.a[i].iTo==iFrom ) break; + } + + if( i==p->redirect.n ){ + /* Block iFrom was not already redirected. Add a new array entry. */ + memmove(&p->redirect.a[1], &p->redirect.a[0], + sizeof(struct RedirectEntry) * p->redirect.n + ); + p->redirect.a[0].iFrom = iFrom; + p->redirect.a[0].iTo = iTo; + p->redirect.n++; + }else{ + /* Block iFrom was already redirected. Overwrite existing entry. */ + p->redirect.a[i].iTo = iTo; + } + + rc = lsmBlockFree(pDb, iFrom); + + *pnWrite = lsmFsBlockSize(pDb->pFS) / lsmFsPageSize(pDb->pFS); + pLvl->lhs.pRedirect = &p->redirect; + } + } + +#if LSM_LOG_STRUCTURE + if( rc==LSM_OK ){ + char aBuf[64]; + sprintf(aBuf, "move-block %d/%d", p->redirect.n-1, LSM_MAX_BLOCK_REDIRECTS); + lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, aBuf); + } +#endif + return rc; +} + +/* +*/ +static int mergeInsertFreelistSegments( + lsm_db *pDb, + int nFree, + MergeWorker *pMW +){ + int rc = LSM_OK; + if( nFree>0 ){ + MultiCursor *pCsr = pMW->pCsr; + Level *pLvl = pMW->pLevel; + SegmentPtr *aNew1; + Segment *aNew2; + + Level *pIter; + Level *pNext; + int i = 0; + + aNew1 = (SegmentPtr *)lsmMallocZeroRc( + pDb->pEnv, sizeof(SegmentPtr) * (pCsr->nPtr+nFree), &rc + ); + if( rc ) return rc; + memcpy(&aNew1[nFree], pCsr->aPtr, sizeof(SegmentPtr)*pCsr->nPtr); + pCsr->nPtr += nFree; + lsmFree(pDb->pEnv, pCsr->aTree); + lsmFree(pDb->pEnv, pCsr->aPtr); + pCsr->aTree = 0; + pCsr->aPtr = aNew1; + + aNew2 = (Segment *)lsmMallocZeroRc( + pDb->pEnv, sizeof(Segment) * (pLvl->nRight+nFree), &rc + ); + if( rc ) return rc; + memcpy(&aNew2[nFree], pLvl->aRhs, sizeof(Segment)*pLvl->nRight); + pLvl->nRight += nFree; + lsmFree(pDb->pEnv, pLvl->aRhs); + pLvl->aRhs = aNew2; + + for(pIter=pDb->pWorker->pLevel; rc==LSM_OK && pIter!=pLvl; pIter=pNext){ + Segment *pSeg = &pLvl->aRhs[i]; + memcpy(pSeg, &pIter->lhs, sizeof(Segment)); + + pCsr->aPtr[i].pSeg = pSeg; + pCsr->aPtr[i].pLevel = pLvl; + rc = segmentPtrEnd(pCsr, &pCsr->aPtr[i], 0); + + pDb->pWorker->pLevel = pNext = pIter->pNext; + sortedFreeLevel(pDb->pEnv, pIter); + i++; + } + assert( i==nFree ); + assert( rc!=LSM_OK || pDb->pWorker->pLevel==pLvl ); + + for(i=nFree; inPtr; i++){ + pCsr->aPtr[i].pSeg = &pLvl->aRhs[i]; + } + + lsmFree(pDb->pEnv, pMW->aGobble); + pMW->aGobble = 0; + } + return rc; +} static int sortedWork( lsm_db *pDb, /* Database handle. Must be worker. */ int nWork, /* Number of pages of work to do */ int nMerge, /* Try to merge this many levels at once */ @@ -4480,33 +4862,81 @@ /* Find a level to work on. */ rc = sortedSelectLevel(pDb, nMerge, &pLevel); assert( rc==LSM_OK || pLevel==0 ); if( pLevel==0 ){ + int nDone = 0; + Level *pTopLevel = lsmDbSnapshotLevel(pDb->pWorker); + if( bFlush==0 && nMerge==1 && pTopLevel && pTopLevel->pNext==0 ){ + rc = sortedMoveBlock(pDb, &nDone); + } + nRemaining -= nDone; + /* Could not find any work to do. Finished. */ - break; + if( nDone==0 ) break; }else{ + int bSave = 0; + Freelist freelist = {0, 0, 0}; MergeWorker mergeworker; /* State used to work on the level merge */ + assert( pDb->bIncrMerge==0 ); + assert( pDb->pFreelist==0 && pDb->bUseFreelist==0 ); + + pDb->bIncrMerge = 1; rc = mergeWorkerInit(pDb, pLevel, &mergeworker); assert( mergeworker.nWork==0 ); + while( rc==LSM_OK && 0==mergeWorkerDone(&mergeworker) - && mergeworker.nWorkbUseFreelist) ){ + int eType = rtTopic(mergeworker.pCsr->eType); rc = mergeWorkerStep(&mergeworker); + + /* If the cursor now points at the first entry past the end of the + ** user data (i.e. either to EOF or to the first free-list entry + ** that will be added to the run), then check if it is possible to + ** merge in any free-list entries that are either in-memory or in + ** free-list-only blocks. */ + if( rc==LSM_OK && nMerge==1 && eType==0 + && (rtTopic(mergeworker.pCsr->eType) || mergeWorkerDone(&mergeworker)) + ){ + int nFree = 0; /* Number of free-list-only levels to merge */ + Level *pLvl; + assert( pDb->pFreelist==0 && pDb->bUseFreelist==0 ); + + /* Now check if all levels containing data newer than this one + ** are single-segment free-list only levels. If so, they will be + ** merged in now. */ + for(pLvl=pDb->pWorker->pLevel; + pLvl!=mergeworker.pLevel && (pLvl->flags & LEVEL_FREELIST_ONLY); + pLvl=pLvl->pNext + ){ + assert( pLvl->nRight==0 ); + nFree++; + } + if( pLvl==mergeworker.pLevel ){ + + rc = mergeInsertFreelistSegments(pDb, nFree, &mergeworker); + if( rc==LSM_OK ){ + rc = multiCursorVisitFreelist(mergeworker.pCsr); + } + if( rc==LSM_OK ){ + rc = multiCursorSetupTree(mergeworker.pCsr, 0); + pDb->pFreelist = &freelist; + pDb->bUseFreelist = 1; + } + } + } } nRemaining -= LSM_MAX(mergeworker.nWork, 1); - /* Check if the merge operation is completely finished. If so, the - ** Merge object and the right-hand-side of the level can be deleted. - ** - ** Otherwise, gobble up (declare eligible for recycling) any pages - ** from rhs segments for which the content has been completely merged - ** into the lhs of the level. - */ if( rc==LSM_OK ){ + /* Check if the merge operation is completely finished. If not, + ** gobble up (declare eligible for recycling) any pages from rhs + ** segments for which the content has been completely merged into + ** the lhs of the level. */ if( mergeWorkerDone(&mergeworker)==0 ){ int i; for(i=0; inRight; i++){ SegmentPtr *pGobble = &mergeworker.pCsr->aPtr[i]; if( pGobble->pSeg->iRoot ){ @@ -4513,65 +4943,86 @@ rc = sortedBtreeGobble(pDb, mergeworker.pCsr, i); }else if( mergeworker.aGobble[i] ){ lsmFsGobble(pDb, pGobble->pSeg, &mergeworker.aGobble[i], 1); } } - }else if( pLevel->lhs.iFirst==0 ){ - /* If the new level is completely empty, remove it from the - ** database snapshot. This can only happen if all input keys were - ** annihilated. Since keys are only annihilated if the new level - ** is the last in the linked list (contains the most ancient of - ** database content), this guarantees that pLevel->pNext==0. */ - - Level *pTop; /* Top level of worker snapshot */ - Level **pp; /* Read/write iterator for Level.pNext list */ - int i; - assert( pLevel->pNext==0 ); - - /* Remove the level from the worker snapshot. */ - pTop = lsmDbSnapshotLevel(pWorker); - for(pp=&pTop; *pp!=pLevel; pp=&((*pp)->pNext)); - *pp = pLevel->pNext; - lsmDbSnapshotSetLevel(pWorker, pTop); - - /* Free the Level structure. */ - lsmFsSortedDelete(pDb->pFS, pWorker, 1, &pLevel->lhs); - for(i=0; inRight; i++){ - lsmFsSortedDelete(pDb->pFS, pWorker, 1, &pLevel->aRhs[i]); - } - sortedFreeLevel(pDb->pEnv, pLevel); - }else{ - int i; - - /* Free the separators of the next level, if required. */ - if( pLevel->pMerge->nInput > pLevel->nRight ){ - assert( pLevel->pNext->lhs.iRoot ); - pLevel->pNext->lhs.iRoot = 0; - } - - /* Free the right-hand-side of pLevel */ - for(i=0; inRight; i++){ - lsmFsSortedDelete(pDb->pFS, pWorker, 1, &pLevel->aRhs[i]); - } - lsmFree(pDb->pEnv, pLevel->aRhs); - pLevel->nRight = 0; - pLevel->aRhs = 0; - - /* Free the Merge object */ - lsmFree(pDb->pEnv, pLevel->pMerge); - pLevel->pMerge = 0; + }else{ + int i; + int bEmpty; + mergeWorkerShutdown(&mergeworker, &rc); + bEmpty = (pLevel->lhs.iFirst==0); + + if( bEmpty==0 && rc==LSM_OK ){ + rc = lsmFsSortedFinish(pDb->pFS, &pLevel->lhs); + } + + if( pDb->bUseFreelist ){ + Freelist *p = &pDb->pWorker->freelist; + lsmFree(pDb->pEnv, p->aEntry); + memcpy(p, &freelist, sizeof(freelist)); + pDb->bUseFreelist = 0; + pDb->pFreelist = 0; + bSave = 1; + } + + for(i=0; inRight; i++){ + lsmFsSortedDelete(pDb->pFS, pWorker, 1, &pLevel->aRhs[i]); + } + + if( bEmpty ){ + /* If the new level is completely empty, remove it from the + ** database snapshot. This can only happen if all input keys were + ** annihilated. Since keys are only annihilated if the new level + ** is the last in the linked list (contains the most ancient of + ** database content), this guarantees that pLevel->pNext==0. */ + Level *pTop; /* Top level of worker snapshot */ + Level **pp; /* Read/write iterator for Level.pNext list */ + + assert( pLevel->pNext==0 ); + + /* Remove the level from the worker snapshot. */ + pTop = lsmDbSnapshotLevel(pWorker); + for(pp=&pTop; *pp!=pLevel; pp=&((*pp)->pNext)); + *pp = pLevel->pNext; + lsmDbSnapshotSetLevel(pWorker, pTop); + + /* Free the Level structure. */ + sortedFreeLevel(pDb->pEnv, pLevel); + }else{ + + /* Free the separators of the next level, if required. */ + if( pLevel->pMerge->nInput > pLevel->nRight ){ + assert( pLevel->pNext->lhs.iRoot ); + pLevel->pNext->lhs.iRoot = 0; + } + + /* Zero the right-hand-side of pLevel */ + lsmFree(pDb->pEnv, pLevel->aRhs); + pLevel->nRight = 0; + pLevel->aRhs = 0; + + /* Free the Merge object */ + lsmFree(pDb->pEnv, pLevel->pMerge); + pLevel->pMerge = 0; + } + + if( bSave && rc==LSM_OK ){ + pDb->bIncrMerge = 0; + rc = lsmSaveWorker(pDb, 0); + } } } /* Clean up the MergeWorker object initialized above. If no error ** has occurred, invoke the work-hook to inform the application that ** the database structure has changed. */ mergeWorkerShutdown(&mergeworker, &rc); + pDb->bIncrMerge = 0; if( rc==LSM_OK ) sortedInvokeWorkHook(pDb); -#if 0 - lsmSortedDumpStructure(pDb, pDb->pWorker, 0, 0, "work"); +#if LSM_LOG_STRUCTURE + lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, "work"); #endif assertBtreeOk(pDb, &pLevel->lhs); assertRunInOrder(pDb, &pLevel->lhs); /* If bFlush is true and the database is no longer considered "full", @@ -4602,13 +5053,10 @@ int rc = LSM_OK; int bRet = 0; assert( pDb->pWorker ); if( *pRc==LSM_OK ){ - if( pDb->nTransOpen==0 ){ - rc = lsmTreeLoadHeader(pDb, 0); - } if( rc==LSM_OK && pDb->treehdr.iOldShmid && pDb->treehdr.iOldLog!=pDb->pWorker->iLogOff ){ bRet = 1; @@ -4618,15 +5066,23 @@ *pRc = rc; } assert( *pRc==LSM_OK || bRet==0 ); return bRet; } + +/* +** Create a new free-list only top-level segment. Return LSM_OK if successful +** or an LSM error code if some error occurs. +*/ +static int sortedNewFreelistOnly(lsm_db *pDb){ + return sortedNewToplevel(pDb, TREE_NONE, 0); +} int lsmSaveWorker(lsm_db *pDb, int bFlush){ Snapshot *p = pDb->pWorker; if( p->freelist.nEntry>pDb->nMaxFreelist ){ - int rc = sortedNewToplevel(pDb, TREE_NONE, 0); + int rc = sortedNewFreelistOnly(pDb); if( rc!=LSM_OK ) return rc; } return lsmCheckpointSaveWorker(pDb, bFlush); } @@ -4636,21 +5092,25 @@ int nMerge, /* Minimum segments to merge together */ int nPage, /* Number of pages to write to disk */ int *pnWrite, /* OUT: Pages actually written to disk */ int *pbCkpt /* OUT: True if an auto-checkpoint is req. */ ){ + Snapshot *pWorker; /* Worker snapshot */ int rc = LSM_OK; /* Return code */ int bDirty = 0; int nMax = nPage; /* Maximum pages to write to disk */ int nRem = nPage; int bCkpt = 0; + + assert( nPage>0 ); /* Open the worker 'transaction'. It will be closed before this function ** returns. */ assert( pDb->pWorker==0 ); rc = lsmBeginWork(pDb); if( rc!=LSM_OK ) return rc; + pWorker = pDb->pWorker; /* If this connection is doing auto-checkpoints, set nMax (and nRem) so ** that this call stops writing when the auto-checkpoint is due. The ** caller will do the checkpoint, then possibly call this function again. */ if( bShutdown==0 && pDb->nAutockpt ){ @@ -4660,19 +5120,22 @@ lsmCheckpointSynced(pDb, 0, 0, &nSync); nUnsync = lsmCheckpointNWrite(pDb->pShmhdr->aSnap1, 0); nPgsz = lsmCheckpointPgsz(pDb->pShmhdr->aSnap1); - nMax = LSM_MIN(nMax, (pDb->nAutockpt/nPgsz) - (nUnsync-nSync)); + nMax = LSM_MIN(nMax, (pDb->nAutockpt/nPgsz) - (int)(nUnsync-nSync)); if( nMaxnTransOpen==0 ){ + rc = lsmTreeLoadHeader(pDb, 0); + } if( sortedTreeHasOld(pDb, &rc) ){ /* sortedDbIsFull() returns non-zero if either (a) there are too many ** levels in total in the db, or (b) there are too many levels with the ** the same age in the db. Either way, call sortedWork() to merge ** existing segments together until this condition is cleared. */ @@ -4708,55 +5171,64 @@ /* If the in-memory part of the free-list is too large, write a new ** top-level containing just the in-memory free-list entries to disk. */ if( rc==LSM_OK && pDb->pWorker->freelist.nEntry > pDb->nMaxFreelist ){ int nPg = 0; - while( rc==LSM_OK && sortedDbIsFull(pDb) ){ + while( rc==LSM_OK && lsmDatabaseFull(pDb) ){ rc = sortedWork(pDb, 16, nMerge, 1, &nPg); nRem -= nPg; } if( rc==LSM_OK ){ - rc = sortedNewToplevel(pDb, TREE_NONE, &nPg); + rc = sortedNewFreelistOnly(pDb); } nRem -= nPg; if( nPg ) bDirty = 1; } + + if( rc==LSM_OK ){ + *pnWrite = (nMax - nRem); + *pbCkpt = (bCkpt && nRem<=0); + if( nMerge==1 && pDb->nAutockpt>0 && *pnWrite>0 + && pWorker->pLevel + && pWorker->pLevel->nRight==0 + && pWorker->pLevel->pNext==0 + ){ + *pbCkpt = 1; + } + } if( rc==LSM_OK && bDirty ){ lsmFinishWork(pDb, 0, &rc); }else{ int rcdummy = LSM_BUSY; lsmFinishWork(pDb, 0, &rcdummy); + *pnWrite = 0; } assert( pDb->pWorker==0 ); - - if( rc==LSM_OK ){ - if( pnWrite ) *pnWrite = (nMax - nRem); - if( pbCkpt ) *pbCkpt = (bCkpt && nRem<=0); - }else{ - if( pnWrite ) *pnWrite = 0; - if( pbCkpt ) *pbCkpt = 0; - } - return rc; } static int doLsmWork(lsm_db *pDb, int nMerge, int nPage, int *pnWrite){ - int rc; - int nWrite = 0; - int bCkpt = 0; + int rc = LSM_OK; /* Return code */ + int nWrite = 0; /* Number of pages written */ assert( nMerge>=1 ); - do { - int nThis = 0; - bCkpt = 0; - rc = doLsmSingleWork(pDb, 0, nMerge, nPage-nWrite, &nThis, &bCkpt); - nWrite += nThis; - if( rc==LSM_OK && bCkpt ){ - rc = lsm_checkpoint(pDb, 0); - } - }while( rc==LSM_OK && (nWrite=0) ? (nPage-nWrite) : ((int)0x7FFFFFFF); + + bCkpt = 0; + rc = doLsmSingleWork(pDb, 0, nMerge, nReq, &nThis, &bCkpt); + nWrite += nThis; + if( rc==LSM_OK && bCkpt ){ + rc = lsm_checkpoint(pDb, 0); + } + }while( rc==LSM_OK && bCkpt && (nWritenTransOpen || pDb->pCsr ) return LSM_MISUSE_BKPT; - if( nMerge<=0 ) nMerge = pDb->nMerge; - return doLsmWork(pDb, nMerge, nPage, pnWrite); + + /* Convert from KB to pages */ + nPgsz = lsmFsPageSize(pDb->pFS); + if( nKB>=0 ){ + nPage = ((i64)nKB * 1024 + nPgsz - 1) / nPgsz; + }else{ + nPage = -1; + } + + rc = doLsmWork(pDb, nMerge, nPage, &nWrite); + + if( pnWrite ){ + /* Convert back from pages to KB */ + *pnWrite = (int)(((i64)nWrite * 1024 + nPgsz - 1) / nPgsz); + } + return rc; } int lsm_flush(lsm_db *db){ int rc; @@ -4842,10 +5332,11 @@ nRemaining = nUnit * nDepth; #ifdef LSM_LOG_WORK lsmLogMessage(pDb, rc, "lsmSortedAutoWork(): %d*%d = %d pages", nUnit, nDepth, nRemaining); #endif + assert( nRemaining>=0 ); rc = doLsmWork(pDb, pDb->nMerge, nRemaining, 0); if( rc==LSM_BUSY ) rc = LSM_OK; if( bRestore && pDb->pCsr ){ lsmFreeSnapshot(pDb->pEnv, pDb->pClient); @@ -4876,10 +5367,11 @@ } if( rc==LSM_OK ){ rc = sortedNewToplevel(pDb, TREE_BOTH, 0); } + lsmFinishWork(pDb, 1, &rc); return rc; } /* @@ -4918,22 +5410,32 @@ return z; } static int fileToString( - lsm_env *pEnv, /* For xMalloc() */ + lsm_db *pDb, /* For xMalloc() */ char *aBuf, int nBuf, int nMin, Segment *pSeg ){ int i = 0; - char *zSeg; + if( pSeg ){ + char *zSeg; - zSeg = segToString(pEnv, pSeg, nMin); - i += sqlite4_snprintf(&aBuf[i], nBuf-i, "%s", zSeg); - lsmFree(pEnv, zSeg); + zSeg = segToString(pDb->pEnv, pSeg, nMin); + i += sqlite4_snprintf(&aBuf[i], nBuf-i, "%s", zSeg); + lsmFree(pDb->pEnv, zSeg); + +#ifdef LSM_LOG_FREELIST + lsmInfoArrayStructure(pDb, 1, pSeg->iFirst, &zSeg); + i += sqlite4_snprintf(&aBuf[i], nBuf-i, " (%s)", zSeg); + lsmFree(pDb->pEnv, zSeg); +#endif + }else{ + aBuf[0] = '\0'; + } return i; } void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){ @@ -4973,16 +5475,16 @@ aCell += lsmVarintGet32(aCell, &iPgPtr); if( eType==0 ){ Pgno iRef; /* Page number of referenced page */ aCell += lsmVarintGet64(aCell, &iRef); - lsmFsDbPageGet(pDb->pFS, iRef, &pRef); - aKey = pageGetKey(pRef, 0, &iTopic, &nKey, &blob); + lsmFsDbPageGet(pDb->pFS, pRun, iRef, &pRef); + aKey = pageGetKey(pRun, pRef, 0, &iTopic, &nKey, &blob); }else{ aCell += lsmVarintGet32(aCell, &nKey); if( rtIsWrite(eType) ) aCell += lsmVarintGet32(aCell, &nVal); - sortedReadData(pPg, (aCell-aData), nKey+nVal, (void **)&aKey, &blob); + sortedReadData(0, pPg, (aCell-aData), nKey+nVal, (void **)&aKey, &blob); aVal = &aKey[nKey]; iTopic = eType; } lsmStringAppendf(&s, "%s%2X:", (i==0?"":" "), iTopic); @@ -5006,11 +5508,12 @@ sortedBlobFree(&blob); } static void infoCellDump( - lsm_db *pDb, + lsm_db *pDb, /* Database handle */ + Segment *pSeg, /* Segment page belongs to */ int bIndirect, /* True to follow indirect refs */ Page *pPg, int iCell, int *peType, int *piPgPtr, @@ -5035,12 +5538,12 @@ if( eType==0 ){ int dummy; Pgno iRef; /* Page number of referenced page */ aCell += lsmVarintGet64(aCell, &iRef); if( bIndirect ){ - lsmFsDbPageGet(pDb->pFS, iRef, &pRef); - pageGetKeyCopy(pDb->pEnv, pRef, 0, &dummy, pBlob); + lsmFsDbPageGet(pDb->pFS, pSeg, iRef, &pRef); + pageGetKeyCopy(pDb->pEnv, pSeg, pRef, 0, &dummy, pBlob); aKey = (u8 *)pBlob->pData; nKey = pBlob->nData; lsmFsPageRelease(pRef); }else{ aKey = (u8 *)""; @@ -5047,11 +5550,11 @@ nKey = 11; } }else{ aCell += lsmVarintGet32(aCell, &nKey); if( rtIsWrite(eType) ) aCell += lsmVarintGet32(aCell, &nVal); - sortedReadData(pPg, (aCell-aData), nKey+nVal, (void **)&aKey, pBlob); + sortedReadData(pSeg, pPg, (aCell-aData), nKey+nVal, (void **)&aKey, pBlob); aVal = &aKey[nKey]; } if( peType ) *peType = eType; if( piPgPtr ) *piPgPtr = iPgPtr; @@ -5086,20 +5589,42 @@ ){ int rc = LSM_OK; /* Return code */ Page *pPg = 0; /* Handle for page iPg */ int i, j; /* Loop counters */ const int perLine = 16; /* Bytes per line in the raw hex dump */ + Segment *pSeg = 0; + Snapshot *pSnap; int bValues = (flags & INFO_PAGE_DUMP_VALUES); int bHex = (flags & INFO_PAGE_DUMP_HEX); int bData = (flags & INFO_PAGE_DUMP_DATA); int bIndirect = (flags & INFO_PAGE_DUMP_INDIRECT); *pzOut = 0; if( iPg==0 ) return LSM_ERROR; - rc = lsmFsDbPageGet(pDb->pFS, iPg, &pPg); + assert( pDb->pClient || pDb->pWorker ); + pSnap = pDb->pClient; + if( pSnap==0 ) pSnap = pDb->pWorker; + if( pSnap->redirect.n>0 ){ + Level *pLvl; + int bUse = 0; + for(pLvl=pSnap->pLevel; pLvl->pNext; pLvl=pLvl->pNext); + pSeg = (pLvl->nRight==0 ? &pLvl->lhs : &pLvl->aRhs[pLvl->nRight-1]); + rc = lsmFsSegmentContainsPg(pDb->pFS, pSeg, iPg, &bUse); + if( bUse==0 ){ + pSeg = 0; + } + } + + /* iPg is a real page number (not subject to redirection). So it is safe + ** to pass a NULL in place of the segment pointer as the second argument + ** to lsmFsDbPageGet() here. */ + if( rc==LSM_OK ){ + rc = lsmFsDbPageGet(pDb->pFS, 0, iPg, &pPg); + } + if( rc==LSM_OK ){ Blob blob = {0, 0, 0, 0}; int nKeyWidth = 0; LsmString str; int nRec; @@ -5112,33 +5637,34 @@ nRec = pageGetNRec(aData, nData); iPtr = pageGetPtr(aData, nData); flags = pageGetFlags(aData, nData); lsmStringInit(&str, pDb->pEnv); - lsmStringAppendf(&str, "Page : %d (%d bytes)\n", iPg, nData); + lsmStringAppendf(&str, "Page : %lld (%d bytes)\n", iPg, nData); lsmStringAppendf(&str, "nRec : %d\n", nRec); lsmStringAppendf(&str, "iPtr : %d\n", iPtr); lsmStringAppendf(&str, "flags: %04x\n", flags); lsmStringAppendf(&str, "\n"); for(iCell=0; iCellnKeyWidth ) nKeyWidth = nKey; } if( bHex ) nKeyWidth = nKeyWidth * 2; for(iCell=0; iCellpEnv, pRun, 0); lsmLogMessage(pDb, LSM_OK, "Segment: %s", zSeg); lsmFree(pDb->pEnv, zSeg); - lsmFsDbPageGet(pDb->pFS, pRun->iFirst, &pPg); + lsmFsDbPageGet(pDb->pFS, pRun, pRun->iFirst, &pPg); while( pPg ){ Page *pNext; char *z = 0; infoPageDump(pDb, lsmFsPageNumber(pPg), flags, &z); lsmLogMessage(pDb, LSM_OK, "%s", z); @@ -5247,18 +5773,26 @@ int bVals, /* Output the values from each segment */ const char *zWhy /* Caption to print near top of dump */ ){ Snapshot *pDump = pSnap; Level *pTopLevel; + char *zFree = 0; + assert( pSnap ); pTopLevel = lsmDbSnapshotLevel(pDump); if( pDb->xLog && pTopLevel ){ + static int nCall = 0; Level *pLevel; int iLevel = 0; - lsmLogMessage(pDb, LSM_OK, "Database structure (%s)", zWhy); + nCall++; + lsmLogMessage(pDb, LSM_OK, "Database structure %d (%s)", nCall, zWhy); + +#if 0 + if( nCall==1031 || nCall==1032 ) bKeys=1; +#endif for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ char zLeft[1024]; char zRight[1024]; int i = 0; @@ -5273,22 +5807,30 @@ aLeft[nLeft++] = pSeg; for(i=0; inRight; i++){ aRight[nRight++] = &pLevel->aRhs[i]; } + +#ifdef LSM_LOG_FREELIST + if( nRight ){ + memmove(&aRight[1], aRight, sizeof(aRight[0])*nRight); + aRight[0] = 0; + nRight++; + } +#endif for(i=0; ipEnv, zLeft, sizeof(zLeft), 28, aLeft[i]); + fileToString(pDb, zLeft, sizeof(zLeft), 24, aLeft[i]); } if( ipEnv, zRight, sizeof(zRight), 28, aRight[i]); + fileToString(pDb, zRight, sizeof(zRight), 24, aRight[i]); } if( i==0 ){ sqlite4_snprintf(zLevel, sizeof(zLevel), "L%d: (age=%d) (flags=%.4x)", iLevel, (int)pLevel->iAge, (int)pLevel->flags @@ -5296,14 +5838,14 @@ }else{ zLevel[0] = '\0'; } if( nRight==0 ){ - iPad = 28 - (strlen(zLeft)/2) ; + iPad = 10; } - lsmLogMessage(pDb, LSM_OK, "% 7s % *s% -35s %s", + lsmLogMessage(pDb, LSM_OK, "% 25s % *s% -35s %s", zLevel, iPad, "", zLeft, zRight ); } iLevel++; @@ -5317,10 +5859,16 @@ sortedDumpSegment(pDb, &pLevel->aRhs[i], bVals); } } } } + + lsmInfoFreelist(pDb, &zFree); + lsmLogMessage(pDb, LSM_OK, "Freelist: %s", zFree); + lsmFree(pDb->pEnv, zFree); + + assert( lsmFsIntegrityCheck(pDb) ); } void lsmSortedFreeLevel(lsm_env *pEnv, Level *pLevel){ Level *pNext; Level *p; @@ -5336,18 +5884,30 @@ for(pCsr=pDb->pCsr; pCsr; pCsr=pCsr->pNext){ lsmTreeCursorSave(pCsr->apTreeCsr[0]); lsmTreeCursorSave(pCsr->apTreeCsr[1]); } } + +void lsmSortedExpandBtreePage(Page *pPg, int nOrig){ + u8 *aData; + int nData; + int nEntry; + int iHdr; + + aData = lsmFsPageData(pPg, &nData); + nEntry = pageGetNRec(aData, nOrig); + iHdr = SEGMENT_EOF(nOrig, nEntry); + memmove(&aData[iHdr + (nData-nOrig)], &aData[iHdr], nOrig-iHdr); +} #ifdef LSM_DEBUG_EXPENSIVE static void assertRunInOrder(lsm_db *pDb, Segment *pSeg){ Page *pPg = 0; Blob blob1 = {0, 0, 0, 0}; Blob blob2 = {0, 0, 0, 0}; - lsmFsDbPageGet(pDb->pFS, pSeg->iFirst, &pPg); + lsmFsDbPageGet(pDb->pFS, pSeg, pSeg->iFirst, &pPg); while( pPg ){ u8 *aData; int nData; Page *pNext; aData = lsmFsPageData(pPg, &nData); @@ -5354,21 +5914,21 @@ if( 0==(pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG) ){ int i; int nRec = pageGetNRec(aData, nData); for(i=0; ipEnv, pPg, i, &iTopic1, &blob1); + pageGetKeyCopy(pDb->pEnv, pSeg, pPg, i, &iTopic1, &blob1); if( i==0 && blob2.nData ){ assert( sortedKeyCompare( pDb->xCmp, iTopic2, blob2.pData, blob2.nData, iTopic1, blob1.pData, blob1.nData )<0 ); } if( i<(nRec-1) ){ - pageGetKeyCopy(pDb->pEnv, pPg, i+1, &iTopic2, &blob2); + pageGetKeyCopy(pDb->pEnv, pSeg, pPg, i+1, &iTopic2, &blob2); assert( sortedKeyCompare( pDb->xCmp, iTopic1, blob1.pData, blob1.nData, iTopic2, blob2.pData, blob2.nData )<0 ); } @@ -5498,11 +6058,11 @@ rc = btreeCursorNew(pDb, pSeg, &pCsr); if( rc==LSM_OK ){ rc = btreeCursorFirst(pCsr); } if( rc==LSM_OK ){ - rc = lsmFsDbPageGet(pFS, pSeg->iFirst, &pPg); + rc = lsmFsDbPageGet(pFS, pSeg, pSeg->iFirst, &pPg); } while( rc==LSM_OK ){ Page *pNext; u8 *aData; @@ -5520,11 +6080,11 @@ && 0!=pageGetNRec(aData, nData) ){ u8 *pKey; int nKey; int iTopic; - pKey = pageGetKey(pPg, 0, &iTopic, &nKey, &blob); + pKey = pageGetKey(pSeg, pPg, 0, &iTopic, &nKey, &blob); assert( nKey==pCsr->nKey && 0==memcmp(pKey, pCsr->pKey, nKey) ); assert( lsmFsPageNumber(pPg)==pCsr->iPtr ); rc = btreeCursorNext(pCsr); } } Index: src/lsm_tree.c ================================================================== --- src/lsm_tree.c +++ src/lsm_tree.c @@ -239,12 +239,11 @@ /* ** Zero the IntArray object. */ static void intArrayFree(lsm_env *pEnv, IntArray *p){ - lsmFree(pEnv, p->aArray); - memset(p, 0, sizeof(IntArray)); + p->nArray = 0; } /* ** Return the number of entries currently in the int-array object. */ @@ -290,32 +289,28 @@ /* ** Return a pointer to the mapped memory location associated with *-shm ** file offset iPtr. */ -static void *treeShmptr(lsm_db *pDb, u32 iPtr, int *pRc){ - /* TODO: This will likely be way too slow. If it is, chunks should be - ** cached as part of the db handle. */ - void *pChunk; - int iChunk = (iPtr>>15); - assert( LSM_SHM_CHUNK_SIZE==(1<<15) ); - - if( (pDb->nShm<=iChunk || 0==(pChunk = pDb->apShm[iChunk])) ){ - *pRc = lsmShmChunk(pDb, iChunk, &pChunk); - } - - if( iPtr==0 || *pRc ) return 0; - return &((u8 *)pChunk)[iPtr & (LSM_SHM_CHUNK_SIZE-1)]; +static void *treeShmptr(lsm_db *pDb, u32 iPtr){ + + assert( (iPtr>>15)nShm ); + assert( pDb->apShm[iPtr>>15] ); + + return iPtr?(&((u8*)(pDb->apShm[iPtr>>15]))[iPtr & (LSM_SHM_CHUNK_SIZE-1)]):0; } static ShmChunk * treeShmChunk(lsm_db *pDb, int iChunk){ - int rcdummy = LSM_OK; - return (ShmChunk *)treeShmptr(pDb, iChunk*LSM_SHM_CHUNK_SIZE, &rcdummy); + return (ShmChunk *)(pDb->apShm[iChunk]); } static ShmChunk * treeShmChunkRc(lsm_db *pDb, int iChunk, int *pRc){ - return (ShmChunk *)treeShmptr(pDb, iChunk*LSM_SHM_CHUNK_SIZE, pRc); + assert( *pRc==LSM_OK ); + if( iChunknShm || LSM_OK==(*pRc = lsmShmCacheChunks(pDb, iChunk+1)) ){ + return (ShmChunk *)(pDb->apShm[iChunk]); + } + return 0; } #ifndef NDEBUG static void assertIsWorkingChild( @@ -325,12 +320,12 @@ int iCell ){ TreeNode *p; int rc = LSM_OK; u32 iPtr = getChildPtr(pParent, WORKING_VERSION, iCell); - p = treeShmptr(db, iPtr, &rc); - assert( p==pNode || rc!=LSM_OK ); + p = treeShmptr(db, iPtr); + assert( p==pNode ); } #else # define assertIsWorkingChild(w,x,y,z) #endif @@ -346,11 +341,11 @@ int *pRc /* IN/OUT: Error code */ ){ TreeKey *pRet; assert( eLoad==TKV_LOADKEY || eLoad==TKV_LOADVAL ); - pRet = (TreeKey *)treeShmptr(pDb, iPtr, pRc); + pRet = (TreeKey *)treeShmptr(pDb, iPtr); if( pRet ){ int nReq; /* Bytes of space required at pRet */ int nAvail; /* Bytes of space available at pRet */ nReq = sizeof(TreeKey) + pRet->nKey; @@ -363,11 +358,11 @@ if( nAvaila[nLoad], p, n); nLoad += n; if( nLoad==nReq ) break; @@ -490,11 +485,11 @@ int rc = LSM_OK; LsmString s; TreeNode *pNode; TreeBlob b = {0, 0}; - pNode = (TreeNode *)treeShmptr(pDb, iNode, &rc); + pNode = (TreeNode *)treeShmptr(pDb, iNode); if( nHeight==0 ){ /* Append the nIndent bytes of space to string s. */ lsmStringInit(&s, pDb->pEnv); @@ -676,21 +671,21 @@ *pRc = rc; return 0; } /* Set the header values for the chunk just finished */ - pHdr = (ShmChunk *)treeShmptr(pDb, iChunk*CHUNK_SIZE, pRc); + pHdr = (ShmChunk *)treeShmptr(pDb, iChunk*CHUNK_SIZE); pHdr->iNext = iNext; /* Advance to the next chunk */ iWrite = iNext * CHUNK_SIZE + CHUNK_HDR; } /* Allocate space at iWrite. */ iRet = iWrite; pDb->treehdr.iWrite = iWrite + nByte; - pDb->treehdr.nByte += nByte; + pDb->treehdr.root.nByte += nByte; } return iRet; } /* @@ -698,11 +693,11 @@ */ static void *treeShmallocZero(lsm_db *pDb, int nByte, u32 *piPtr, int *pRc){ u32 iPtr; void *p; iPtr = treeShmalloc(pDb, 1, nByte, pRc); - p = treeShmptr(pDb, iPtr, pRc); + p = treeShmptr(pDb, iPtr); if( p ){ assert( *pRc==LSM_OK ); memset(p, 0, nByte); *piPtr = iPtr; } @@ -730,11 +725,11 @@ u8 *a; int n; /* Allocate space for the TreeKey structure itself */ *piPtr = iPtr = treeShmalloc(pDb, 1, sizeof(TreeKey), pRc); - p = treeShmptr(pDb, iPtr, pRc); + p = treeShmptr(pDb, iPtr); if( *pRc ) return 0; p->nKey = nKey; p->nValue = nVal; /* Allocate and populate the space required for the key and value. */ @@ -748,11 +743,11 @@ iWrite = (pDb->treehdr.iWrite & (LSM_SHM_CHUNK_SIZE-1)); iWrite = LSM_MAX(iWrite, LSM_SHM_CHUNK_HDR); nAlloc = LSM_MIN((LSM_SHM_CHUNK_SIZE-iWrite), nRem); - aAlloc = treeShmptr(pDb, treeShmalloc(pDb, 0, nAlloc, pRc), pRc); + aAlloc = treeShmptr(pDb, treeShmalloc(pDb, 0, nAlloc, pRc)); if( aAlloc==0 ) break; memcpy(aAlloc, &a[n-nRem], nAlloc); nRem -= nAlloc; } a = pVal; @@ -1060,11 +1055,11 @@ memcpy(&pDb->treehdr.oldroot, &pDb->treehdr.root, sizeof(TreeRoot)); pDb->treehdr.root.iTransId = 1; pDb->treehdr.root.iRoot = 0; pDb->treehdr.root.nHeight = 0; - pDb->treehdr.nByte = 0; + pDb->treehdr.root.nByte = 0; } } void lsmTreeDiscardOld(lsm_db *pDb){ assert( lsmShmAssertLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_EXCL) @@ -1377,11 +1372,11 @@ while( iNode>=0 ){ TreeNode *pNode = pCsr->apTreeNode[iNode]; if( iCell<3 && pNode->aiKeyPtr[iCell] ){ int rc = LSM_OK; - TreeKey *pKey = treeShmptr(db, pNode->aiKeyPtr[iCell], &rc); + TreeKey *pKey = treeShmptr(db, pNode->aiKeyPtr[iCell]); assert( rc==LSM_OK ); return ((pKey->flags & LSM_END_DELETE) ? 1 : 0); } iNode--; iCell = pCsr->aiCell[iNode]; @@ -1400,11 +1395,11 @@ while( iNode>=0 ){ TreeNode *pNode = pCsr->apTreeNode[iNode]; int iCell = pCsr->aiCell[iNode]-1; if( iCell>=0 && pNode->aiKeyPtr[iCell] ){ int rc = LSM_OK; - TreeKey *pKey = treeShmptr(db, pNode->aiKeyPtr[iCell], &rc); + TreeKey *pKey = treeShmptr(db, pNode->aiKeyPtr[iCell]); assert( rc==LSM_OK ); return ((pKey->flags & LSM_START_DELETE) ? 1 : 0); } iNode--; } @@ -1641,11 +1636,11 @@ iDir = -1; }else{ iDir = +1; } iPeer = getChildPtr(pParent, WORKING_VERSION, iPSlot+iDir); - pPeer = (TreeNode *)treeShmptr(db, iPeer, &rc); + pPeer = (TreeNode *)treeShmptr(db, iPeer); assertIsWorkingChild(db, pNode, pParent, iPSlot); /* Allocate the first new leaf node. This is always required. */ if( bLeaf ){ pNew1 = (TreeNode *)newTreeLeaf(db, &iNew1, &rc); @@ -1882,11 +1877,11 @@ /* ** Return, in bytes, the amount of memory currently used by the tree ** structure. */ int lsmTreeSize(lsm_db *pDb){ - return pDb->treehdr.nByte; + return pDb->treehdr.root.nByte; } /* ** Open a cursor on the in-memory tree pTree. */ @@ -1970,11 +1965,11 @@ while( iNodePtr ){ TreeNode *pNode; /* Node at location iNodePtr */ int iTest; /* Index of second key to test (0 or 2) */ TreeKey *pTreeKey; /* Key to compare against */ - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr, &rc); + pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); iNode++; pCsr->apTreeNode[iNode] = pNode; /* Compare (pKey/nKey) with the key in the middle slot of B-tree node ** pNode. The middle slot is never empty. If the comparison is a match, @@ -2064,11 +2059,11 @@ if( pCsr->iNodeiTransId, iCell) ){ do { u32 iNodePtr; pCsr->iNode++; iNodePtr = getChildPtr(pNode, pRoot->iTransId, iCell); - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr, &rc); + pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); pCsr->apTreeNode[pCsr->iNode] = pNode; iCell = pCsr->aiCell[pCsr->iNode] = (pNode->aiKeyPtr[0]==0); }while( pCsr->iNode < iLeaf ); } @@ -2129,11 +2124,11 @@ if( pCsr->iNodeiTransId, iCell) ){ do { u32 iNodePtr; pCsr->iNode++; iNodePtr = getChildPtr(pNode, pRoot->iTransId, iCell); - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr, &rc); + pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); if( rc!=LSM_OK ) break; pCsr->apTreeNode[pCsr->iNode] = pNode; iCell = 1 + (pNode->aiKeyPtr[2]!=0) + (pCsr->iNode < iLeaf); pCsr->aiCell[pCsr->iNode] = iCell; }while( pCsr->iNode < iLeaf ); @@ -2179,11 +2174,11 @@ iNodePtr = pRoot->iRoot; while( iNodePtr ){ int iCell; TreeNode *pNode; - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr, &rc); + pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); if( rc ) break; if( bLast ){ iCell = ((pNode->aiKeyPtr[2]==0) ? 2 : 3); }else{ @@ -2206,11 +2201,11 @@ int lsmTreeCursorFlags(TreeCursor *pCsr){ int flags = 0; if( pCsr && pCsr->iNode>=0 ){ int rc = LSM_OK; TreeKey *pKey = (TreeKey *)treeShmptr(pCsr->pDb, - pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[pCsr->aiCell[pCsr->iNode]], &rc + pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[pCsr->aiCell[pCsr->iNode]] ); assert( rc==LSM_OK ); flags = pKey->flags; } return flags; @@ -2283,11 +2278,10 @@ /* ** Roll back to mark pMark. Structure *pMark should have been previously ** populated by a call to lsmTreeMark(). */ void lsmTreeRollback(lsm_db *pDb, TreeMark *pMark){ - int rcdummy = LSM_OK; int iIdx; int nIdx; u32 iNext; ShmChunk *pChunk; u32 iChunk; @@ -2295,12 +2289,12 @@ /* Revert all required v2 pointers. */ nIdx = intArraySize(&pDb->rollback); for(iIdx = pMark->iRollback; iIdxrollback, iIdx), &rcdummy); - assert( pNode && rcdummy==LSM_OK ); + pNode = treeShmptr(pDb, intArrayEntry(&pDb->rollback, iIdx)); + assert( pNode ); pNode->iV2 = 0; pNode->iV2Child = 0; pNode->iV2Ptr = 0; } intArrayTruncate(&pDb->rollback, pMark->iRollback); @@ -2361,11 +2355,11 @@ return LSM_OK; } lsmShmBarrier(pDb); } - return LSM_PROTOCOL; + return LSM_PROTOCOL_BKPT; } int lsmTreeLoadHeaderOk(lsm_db *pDb, int iRead){ TreeHeader *p = (iRead==1) ? &pDb->pShmhdr->hdr1 : &pDb->pShmhdr->hdr2; assert( iRead==1 || iRead==2 ); @@ -2377,16 +2371,14 @@ ** is true, the transaction is committed. Otherwise it is rolled back. */ int lsmTreeEndTransaction(lsm_db *pDb, int bCommit){ ShmHeader *pShm = pDb->pShmhdr; - if( bCommit ){ - treeHeaderChecksum(&pDb->treehdr, pDb->treehdr.aCksum); - memcpy(&pShm->hdr2, &pDb->treehdr, sizeof(TreeHeader)); - lsmShmBarrier(pDb); - memcpy(&pShm->hdr1, &pDb->treehdr, sizeof(TreeHeader)); - } + treeHeaderChecksum(&pDb->treehdr, pDb->treehdr.aCksum); + memcpy(&pShm->hdr2, &pDb->treehdr, sizeof(TreeHeader)); + lsmShmBarrier(pDb); + memcpy(&pShm->hdr1, &pDb->treehdr, sizeof(TreeHeader)); pShm->bWriter = 0; intArrayFree(pDb->pEnv, &pDb->rollback); return LSM_OK; } Index: src/lsm_unix.c ================================================================== --- src/lsm_unix.c +++ src/lsm_unix.c @@ -36,10 +36,15 @@ #include #include #include "lsmInt.h" +/* There is no fdatasync() call on Android */ +#ifdef __ANDROID__ +# define fdatasync(x) fsync(x) +#endif + /* ** An open file is an instance of the following object */ typedef struct PosixFile PosixFile; struct PosixFile { @@ -51,12 +56,10 @@ off_t nMap; /* Size of mapping at pMap in bytes */ int nShm; /* Number of entries in array apShm[] */ void **apShm; /* Array of 32K shared memory segments */ }; -static int lsm_ioerr(void){ return LSM_IOERR; } - static char *posixShmFile(PosixFile *p){ char *zShm; int nName = strlen(p->zName); zShm = (char *)lsmMalloc(p->pEnv, nName+4+1); if( zShm ){ @@ -66,28 +69,35 @@ return zShm; } static int lsmPosixOsOpen( lsm_env *pEnv, - const char *zFile, + const char *zFile, + int flags, lsm_file **ppFile ){ int rc = LSM_OK; PosixFile *p; p = lsm_malloc(pEnv, sizeof(PosixFile)); if( p==0 ){ rc = LSM_NOMEM; }else{ + int bReadonly = (flags & LSM_OPEN_READONLY); + int oflags = (bReadonly ? O_RDONLY : (O_RDWR|O_CREAT)); memset(p, 0, sizeof(PosixFile)); p->zName = zFile; p->pEnv = pEnv; - p->fd = open(zFile, O_RDWR|O_CREAT, 0644); + p->fd = open(zFile, oflags, 0644); if( p->fd<0 ){ lsm_free(pEnv, p); p = 0; - rc = lsm_ioerr(); + if( errno==ENOENT ){ + rc = lsmErrorBkpt(LSM_IOERR_NOENT); + }else{ + rc = LSM_IOERR_BKPT; + } } } *ppFile = (lsm_file *)p; return rc; @@ -103,29 +113,33 @@ PosixFile *p = (PosixFile *)pFile; off_t offset; offset = lseek(p->fd, (off_t)iOff, SEEK_SET); if( offset!=iOff ){ - rc = lsm_ioerr(); + rc = LSM_IOERR_BKPT; }else{ ssize_t prc = write(p->fd, pData, (size_t)nData); - if( prc<0 ) rc = lsm_ioerr(); + if( prc<0 ) rc = LSM_IOERR_BKPT; } return rc; } static int lsmPosixOsTruncate( lsm_file *pFile, /* File to write to */ lsm_i64 nSize /* Size to truncate file to */ ){ - int rc = LSM_OK; + PosixFile *p = (PosixFile *)pFile; + int rc = LSM_OK; /* Return code */ int prc; /* Posix Return Code */ - PosixFile *p = (PosixFile *)pFile; - - prc = ftruncate(p->fd, (off_t)nSize); - if( prc<0 ) rc = lsm_ioerr(); + struct stat sStat; /* Result of fstat() invocation */ + + prc = fstat(p->fd, &sStat); + if( prc==0 && sStat.st_size>nSize ){ + prc = ftruncate(p->fd, (off_t)nSize); + } + if( prc<0 ) rc = LSM_IOERR_BKPT; return rc; } static int lsmPosixOsRead( @@ -138,15 +152,15 @@ PosixFile *p = (PosixFile *)pFile; off_t offset; offset = lseek(p->fd, (off_t)iOff, SEEK_SET); if( offset!=iOff ){ - rc = lsm_ioerr(); + rc = LSM_IOERR_BKPT; }else{ ssize_t prc = read(p->fd, pData, (size_t)nData); if( prc<0 ){ - rc = lsm_ioerr(); + rc = LSM_IOERR_BKPT; }else if( prcpMap ){ prc = msync(p->pMap, p->nMap, MS_SYNC); } if( prc==0 ) prc = fdatasync(p->fd); - if( prc<0 ) rc = lsm_ioerr(); + if( prc<0 ) rc = LSM_IOERR_BKPT; #else (void)pFile; #endif return rc; @@ -188,26 +202,28 @@ PosixFile *p = (PosixFile *)pFile; struct stat buf; if( p->pMap ){ munmap(p->pMap, p->nMap); - p->pMap = 0; - p->nMap = 0; - } - - memset(&buf, 0, sizeof(buf)); - prc = fstat(p->fd, &buf); - if( prc!=0 ) return LSM_IOERR_BKPT; - iSz = buf.st_size; - if( iSzfd, iSz); - if( prc!=0 ) return LSM_IOERR_BKPT; - } - - p->pMap = mmap(0, iSz, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd, 0); - p->nMap = iSz; + *ppOut = p->pMap = 0; + *pnOut = p->nMap = 0; + } + + if( iMin>=0 ){ + memset(&buf, 0, sizeof(buf)); + prc = fstat(p->fd, &buf); + if( prc!=0 ) return LSM_IOERR_BKPT; + iSz = buf.st_size; + if( iSzfd, iSz); + if( prc!=0 ) return LSM_IOERR_BKPT; + } + + p->pMap = mmap(0, iSz, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd, 0); + p->nMap = iSz; + } *ppOut = p->pMap; *pnOut = p->nMap; return LSM_OK; } @@ -293,11 +309,11 @@ assert( aType[LSM_LOCK_UNLOCK]==F_UNLCK ); assert( aType[LSM_LOCK_SHARED]==F_RDLCK ); assert( aType[LSM_LOCK_EXCL]==F_WRLCK ); assert( eType>=0 && eType0 && iLock<=16 ); + assert( iLock>0 && iLock<=32 ); memset(&lock, 0, sizeof(lock)); lock.l_whence = SEEK_SET; lock.l_len = 1; lock.l_type = aType[eType]; @@ -306,11 +322,11 @@ if( fcntl(p->fd, F_SETLK, &lock) ){ int e = errno; if( e==EACCES || e==EAGAIN ){ rc = LSM_BUSY; }else{ - rc = LSM_IOERR; + rc = LSM_IOERR_BKPT; } } return rc; } @@ -360,11 +376,11 @@ if( p->apShm[iChunk]==0 ){ p->apShm[iChunk] = mmap(0, LSM_SHM_CHUNK_SIZE, PROT_READ|PROT_WRITE, MAP_SHARED, p->shmfd, iChunk*LSM_SHM_CHUNK_SIZE ); - if( p->apShm[iChunk]==0 ) return LSM_IOERR; + if( p->apShm[iChunk]==0 ) return LSM_IOERR_BKPT; } *ppShm = p->apShm[iChunk]; return LSM_OK; } @@ -403,20 +419,21 @@ lsm_free(p->pEnv, p); return LSM_OK; } static int lsmPosixOsSleep(lsm_env *pEnv, int us){ - if( usleep(us) ){ - return LSM_IOERR; - } +#if 0 + /* Apparently on Android usleep() returns void */ + if( usleep(us) ) return LSM_IOERR; +#endif + usleep(us); return LSM_OK; } /**************************************************************************** ** Memory allocation routines. */ -#define ROUND8(x) (((x)+7)&~7) #define BLOCK_HDR_SIZE ROUND8( sizeof(sqlite4_size_t) ) static void *lsmPosixOsMalloc(lsm_env *pEnv, int N){ unsigned char * m; N += BLOCK_HDR_SIZE; @@ -460,11 +477,10 @@ static sqlite4_size_t lsmPosixOsMSize(lsm_env *pEnv, void *p){ unsigned char * m = (unsigned char *)p; return *((sqlite4_size_t*)(m-BLOCK_HDR_SIZE)); } -#undef ROUND8 #undef BLOCK_HDR_SIZE #ifdef LSM_MUTEX_PTHREADS /************************************************************************* Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -27,11 +27,11 @@ #endif /* ** Dummy function used as a unique symbol for SQLITE4_DYNAMIC */ -void sqlite4_dynamic(void *p){ (void)p; } +void sqlite4_dynamic(void *pArg,void *p){ (void)pArg; (void)p; } #ifndef SQLITE4_AMALGAMATION /* IMPLEMENTATION-OF: R-46656-45156 The sqlite4_version[] string constant ** contains the text of SQLITE4_VERSION macro. */ @@ -366,27 +366,26 @@ pEnv->pLogArg = va_arg(ap, void*); break; } /* - ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_KVSTORE_PUSH, zName, xFactory); + ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_KVSTORE_PUSH, zName,xFactory); ** ** Push a new KVStore factory onto the factory stack. The new factory ** takes priority over prior factories. */ case SQLITE4_ENVCONFIG_KVSTORE_PUSH: { - typedef int(*PFactory)(sqlite4_env*, KVStore **, const char *, unsigned int); const char *zName = va_arg(ap, const char*); int nName = sqlite4Strlen30(zName); KVFactory *pMkr = sqlite4_malloc(pEnv, sizeof(*pMkr)+nName+1); char *z; if( pMkr==0 ) return SQLITE4_NOMEM; z = (char*)&pMkr[1]; memcpy(z, zName, nName+1); memset(pMkr, 0, sizeof(*pMkr)); pMkr->zName = z; - pMkr->xFactory = va_arg(ap, PFactory); + pMkr->xFactory = va_arg(ap, sqlite4_kvfactory); sqlite4_mutex_enter(pEnv->pFactoryMutex); pMkr->pNext = pEnv->pFactory; pEnv->pFactory = pMkr; sqlite4_mutex_leave(pEnv->pFactoryMutex); break; @@ -640,11 +639,11 @@ static int collNocaseCmp( void *NotUsed, int nKey1, const void *pKey1, int nKey2, const void *pKey2 ){ - int r = sqlite4StrNICmp( + int r = sqlite4_strnicmp( (const char *)pKey1, (const char *)pKey2, (nKey1=nIn ){ int i; u8 *aIn = (u8 *)pKey1; u8 *aOut = (u8 *)pKey2; for(i=0; inDb<=2 ); assert( db->aDb==db->aDbStatic ); { FuncDef *pNext, *pSame, *p; @@ -1051,10 +1053,34 @@ out: rc = sqlite4ApiExit(db, rc); sqlite4_mutex_leave(db->mutex); return rc; } + +int sqlite4_create_mi_function( + sqlite4 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xFunc)(sqlite4_context*,int,sqlite4_value **), + void (*xDestroy)(void *) +){ + int rc; + int n; + + n = nArg + (nArg>=0); + sqlite4_mutex_enter(db->mutex); + rc = sqlite4_create_function_v2(db, zFunc, n, enc, p, xFunc, 0,0,xDestroy); + if( rc==SQLITE4_OK ){ + FuncDef *p = sqlite4FindFunction(db, zFunc, -1, n, enc, 0); + p->bMatchinfo = 1; + } + rc = sqlite4ApiExit(db, rc); + sqlite4_mutex_leave(db->mutex); + return rc; +} #ifndef SQLITE4_OMIT_UTF16 int sqlite4_create_function16( sqlite4 *db, const void *zFunctionName, @@ -1239,11 +1265,11 @@ z = (void *)outOfMem; }else{ z = sqlite4_value_text16(db->pErr); if( z==0 ){ sqlite4ValueSetStr(db->pErr, -1, sqlite4ErrStr(db->errCode), - SQLITE4_UTF8, SQLITE4_STATIC); + SQLITE4_UTF8, SQLITE4_STATIC, 0); z = sqlite4_value_text16(db->pErr); } /* A malloc() may have failed within the call to sqlite4_value_text16() ** above. If this is the case, then the db->mallocFailed flag needs to ** be cleared before returning. Do this directly, instead of via @@ -1704,13 +1730,13 @@ db->flags |= SQLITE4_AutoIndex | SQLITE4_EnableTrigger | SQLITE4_ForeignKeys ; - sqlite4HashInit(pEnv, &db->aCollSeq); + sqlite4HashInit(pEnv, &db->aCollSeq, 0); #ifndef SQLITE4_OMIT_VIRTUALTABLE - sqlite4HashInit(pEnv, &db->aModule); + sqlite4HashInit(pEnv, &db->aModule, 0); #endif /* Add the default collation sequence BINARY. BINARY works for both UTF-8 ** and UTF-16, so add a version for each to avoid any unnecessary ** conversions. The only error that can occur here is a malloc() failure. @@ -1782,15 +1808,13 @@ if( rc!=SQLITE4_OK ){ goto opendb_out; } } -#ifdef SQLITE4_ENABLE_FTS3 if( !db->mallocFailed && rc==SQLITE4_OK ){ - rc = sqlite4Fts3Init(db); + rc = sqlite4InitFts5(db); } -#endif #ifdef SQLITE4_ENABLE_ICU if( !db->mallocFailed && rc==SQLITE4_OK ){ rc = sqlite4IcuInit(db); } @@ -1988,11 +2012,11 @@ sqlite4_mutex_enter(db->mutex); /* Find the named key-value store */ for(i=0; inDb; i++){ Db *pDb = &db->aDb[i]; - if( pDb->pKV && (0==zDbName || 0==sqlite4StrICmp(zDbName, pDb->zName)) ){ + if( pDb->pKV && (0==zDbName || 0==sqlite4_stricmp(zDbName, pDb->zName)) ){ pKV = pDb->pKV; break; } } Index: src/math.c ================================================================== --- src/math.c +++ src/math.c @@ -213,10 +213,24 @@ if( r.approx==0 && A.m%B.m!=0 ) r.approx = 1; r.m = A.m/B.m; r.e = A.e - B.e; return r; } + +/* +** Test if A is infinite. +*/ +int sqlite4_num_isinf(sqlite4_num A){ + return A.e>SQLITE4_MX_EXP && A.m!=0; +} + +/* +** Test if A is NaN. +*/ +int sqlite4_num_isnan(sqlite4_num A){ + return A.e>SQLITE4_MX_EXP && A.m==0; +} /* ** Compare numbers A and B. Return: ** ** 1 if ASQLITE4_MX_EXP ){ if( B.m==0 ) return 0; return B.sign ? 3 : 1; } if( A.sign!=B.sign ){ + if ( A.m==0 && B.m==0 ) return 2; return A.sign ? 1 : 3; } adjustExponent(&A, &B); if( A.sign ){ sqlite4_num t = A; @@ -321,11 +336,11 @@ i = incr; }else{ i = 0; } if( nIn<=0 ) goto not_a_valid_number; - if( nIn>=incr*2 + if( nIn>=incr*3 && ((c=zIn[i])=='i' || c=='I') && ((c=zIn[i+incr])=='n' || c=='N') && ((c=zIn[i+incr*2])=='f' || c=='F') ){ r.e = SQLITE4_MX_EXP+1; @@ -333,11 +348,11 @@ return r; } while( i='0' && c<='9' ){ - if( c==0 && nDigit==0 ){ + if( c=='0' && nDigit==0 ){ if( seenRadix && r.e > -(SQLITE4_MX_EXP+1000) ) r.e--; continue; } nDigit++; if( nDigit<=18 ){ @@ -347,44 +362,61 @@ if( c!='0' ) r.approx = 1; if( !seenRadix ) r.e++; } }else if( c=='.' ){ seenRadix = 1; - }else{ + }else if( c=='e' || c=='E' ){ + int exp = 0; + int expsign = 0; + int nEDigit = 0; + if( zIn[i]=='-' ){ + expsign = 1; + i += incr; + }else if( zIn[i]=='+' ){ + i += incr; + } + if( i>=nIn ) goto not_a_valid_number; + while( i'9' ) goto not_a_valid_number; + if( c=='0' && nEDigit==0 ) continue; + nEDigit++; + if( nEDigit>3 ) goto not_a_valid_number; + exp = exp*10 + c - '0'; + } + if( expsign ) exp = -exp; + r.e += exp; break; - } - } - if( c=='e' || c=='E' ){ - int exp = 0; - int expsign = 0; - int nEDigit = 0; - if( zIn[i]=='-' ){ - expsign = 1; - i += incr; - }else if( zIn[i]=='+' ){ - i += incr; - } - if( i>=nIn ) goto not_a_valid_number; - while( i'9' ) break; - if( c=='0' && nEDigit==0 ) continue; - nEDigit++; - if( nEDigit>3 ) goto not_a_valid_number; - exp = exp*10 + c - '0'; - } - if( expsign ) exp = -exp; - r.e += exp; - } - if( c!=0 ) goto not_a_valid_number; + }else{ + goto not_a_valid_number; + } + } return r; not_a_valid_number: r.e = SQLITE4_MX_EXP+1; r.m = 0; return r; } + +/* +** Convert an sqlite4_int64 to a number and return that number. +*/ +sqlite4_num sqlite4_num_from_int64(sqlite4_int64 n){ + sqlite4_num r; + r.approx = 0; + r.e = 0; + r.sign = n < 0; + if( n>=0 ){ + r.m = n; + }else if( n!=SMALLEST_INT64 ){ + r.m = -n; + }else{ + r.m = 1+(u64)LARGEST_INT64; + } + return r; +} /* ** Convert an integer into text in the buffer supplied. The ** text is zero-terminated and right-justified in the buffer. ** A pointer to the first character of text is returned. @@ -467,12 +499,14 @@ removeTrailingZeros(zNum, &n); if( n>0 ){ zOut[0] = '.'; memcpy(zOut+1, zNum, n); nOut += n; + zOut[n+1] = 0; + }else{ + zOut[0] = 0; } - zOut[n+1] = 0; return nOut; } if( x.e<0 && x.e >= -n-5 ){ /* Values less than 1 and with no more than 5 subsequent zeros prior ** to the first significant digit. Ex: 0.0000012345 */ ADDED src/mem.c Index: src/mem.c ================================================================== --- /dev/null +++ src/mem.c @@ -0,0 +1,434 @@ +/* +** 2013-01-01 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the implementation of the "sqlite4_mm" memory +** allocator object. +*/ +#include "sqlite4.h" +#include "mem.h" +#include + +/************************************************************************* +** The SQLITE4_MM_SYSTEM memory allocator. This allocator uses the +** malloc/realloc/free from the system library. It also tries to use +** the memory allocation sizer from the system library if such a routine +** exists. If there is no msize in the system library, then each allocation +** is increased in size by 8 bytes and the size of the allocation is stored +** in those initial 8 bytes. +** +** C-preprocessor macro summary: +** +** HAVE_MALLOC_USABLE_SIZE The configure script sets this symbol if +** the malloc_usable_size() interface exists +** on the target platform. Or, this symbol +** can be set manually, if desired. +** If an equivalent interface exists by +** a different name, using a separate -D +** option to rename it. This symbol will +** be enabled automatically on windows +** systems, and malloc_usable_size() will +** be redefined to _msize(), unless the +** SQLITE4_WITHOUT_MSIZE macro is defined. +** +** SQLITE4_WITHOUT_ZONEMALLOC Some older macs lack support for the zone +** memory allocator. Set this symbol to enable +** building on older macs. +** +** SQLITE4_WITHOUT_MSIZE Set this symbol to disable the use of +** _msize() on windows systems. This might +** be necessary when compiling for Delphi, +** for example. +*/ + +/* +** Windows systems have malloc_usable_size() but it is called _msize(). +** The use of _msize() is automatic, but can be disabled by compiling +** with -DSQLITE4_WITHOUT_MSIZE +*/ +#if !defined(HAVE_MALLOC_USABLE_SIZE) && SQLITE4_OS_WIN \ + && !defined(SQLITE4_WITHOUT_MSIZE) +# define HAVE_MALLOC_USABLE_SIZE 1 +# define SQLITE4_MALLOCSIZE _msize +#endif + +#if defined(__APPLE__) && !defined(SQLITE4_WITHOUT_ZONEMALLOC) + +/* +** Use the zone allocator available on apple products unless the +** SQLITE4_WITHOUT_ZONEMALLOC symbol is defined. +*/ +#include +#include +#include +static malloc_zone_t* _sqliteZone_; +#define SQLITE4_MALLOC(x) malloc_zone_malloc(_sqliteZone_, (x)) +#define SQLITE4_FREE(x) malloc_zone_free(_sqliteZone_, (x)); +#define SQLITE4_REALLOC(x,y) malloc_zone_realloc(_sqliteZone_, (x), (y)) +#define SQLITE4_MALLOCSIZE(x) \ + (_sqliteZone_ ? _sqliteZone_->size(_sqliteZone_,x) : malloc_size(x)) + +#else /* if not __APPLE__ */ + +/* +** Use standard C library malloc and free on non-Apple systems. +** Also used by Apple systems if SQLITE4_WITHOUT_ZONEMALLOC is defined. +*/ +#define SQLITE4_MALLOC(x) malloc(x) +#define SQLITE4_FREE(x) free(x) +#define SQLITE4_REALLOC(x,y) realloc((x),(y)) + +#ifdef HAVE_MALLOC_USABLE_SIZE +# ifndef SQLITE4_MALLOCSIZE +# include +# define SQLITE4_MALLOCSIZE(x) malloc_usable_size(x) +# endif +#else +# undef SQLITE4_MALLOCSIZE +#endif + +#endif /* __APPLE__ or not __APPLE__ */ + +/* +** Implementations of core routines +*/ +static void *mmSysMalloc(sqlite4_mm *pMM, sqlite4_int64 iSize){ +#ifdef SQLITE4_MALLOCSIZE + return SQLITE4_MALLOC(iSize); +#else + unsigned char *pRes = SQLITE4_MALLOC(iSize+8); + if( pRes ){ + *(sqlite4_int64*)pRes = iSize; + pRes += 8; + } + return pRes; +#endif +} +static void *mmSysRealloc(sqlite4_mm *pMM, void *pOld, sqlite4_int64 iSz){ +#ifdef SQLITE4_MALLOCSIZE + return SQLITE4_REALLOC(pOld, iSz); +#else + unsigned char *pRes; + if( pOld==0 ) return mmSysMalloc(pMM, iSz); + pRes = (unsigned char*)pOld; + pRes -= 8; + pRes = SQLITE4_REALLOC(pRes, iSz+8); + if( pRes ){ + *(sqlite4_int64*)pRes = iSz; + pRes += 8; + } + return pRes; +#endif +} +static void mmSysFree(sqlite4_mm *pNotUsed, void *pOld){ +#ifdef SQLITE4_MALLOCSIZE + SQLITE4_FREE(pOld); +#else + unsigned char *pRes; + if( pOld==0 ) return; + pRes = (unsigned char *)pOld; + pRes -= 8; + SQLITE4_FREE(pRes); +#endif +} +static sqlite4_int64 mmSysMsize(sqlite4_mm *pNotUsed, void *pOld){ +#ifdef SQLITE4_MALLOCSIZE + return SQLITE4_MALLOCSIZE(pOld); +#else + unsigned char *pX; + if( pOld==0 ) return 0; + pX = (unsigned char *)pOld; + pX -= 8; + return *(sqlite4_int64*)pX; +#endif +} + +static const sqlite4_mm_methods mmSysMethods = { + /* iVersion */ 1, + /* xMalloc */ mmSysMalloc, + /* xRealloc */ mmSysRealloc, + /* xFree */ mmSysFree, + /* xMsize */ mmSysMsize, + /* xMember */ 0, + /* xBenign */ 0, + /* xFinal */ 0 +}; +static sqlite4_mm mmSystem = { + /* pMethods */ &mmSysMethods +}; + +/************************************************************************* +** The SQLITE4_MM_OVERFLOW memory allocator. +** +** This memory allocator has two child memory allocators, A and B. Always +** try to fulfill the request using A first, then overflow to B if the request +** on A fails. The A allocator must support the xMember method. +*/ +struct mmOvfl { + sqlite4_mm base; /* Base class - must be first */ + int (*xMemberOfA)(sqlite4_mm*, const void*); + sqlite4_mm *pA; /* Primary memory allocator */ + sqlite4_mm *pB; /* Backup memory allocator in case pA fails */ +}; + +static void *mmOvflMalloc(sqlite4_mm *pMM, sqlite4_int64 iSz){ + const struct mmOvfl *pOvfl = (const struct mmOvfl*)pMM; + void *pRes; + pRes = pOvfl->pA->pMethods->xMalloc(pOvfl->pA, iSz); + if( pRes==0 ){ + pRes = pOvfl->pB->pMethods->xMalloc(pOvfl->pB, iSz); + } + return pRes; +} +static void *mmOvflRealloc(sqlite4_mm *pMM, void *pOld, sqlite4_int64 iSz){ + const struct mmOvfl *pOvfl; + void *pRes; + if( pOld==0 ) return mmOvflMalloc(pMM, iSz); + pOvfl = (const struct mmOvfl*)pMM; + if( pOvfl->xMemberOfA(pOvfl->pA, pOld) ){ + pRes = pOvfl->pA->pMethods->xRealloc(pOvfl->pA, pOld, iSz); + }else{ + pRes = pOvfl->pB->pMethods->xRealloc(pOvfl->pB, pOld, iSz); + } + return pRes; +} +static void mmOvflFree(sqlite4_mm *pMM, void *pOld){ + const struct mmOvfl *pOvfl; + if( pOld==0 ) return; + pOvfl = (const struct mmOvfl*)pMM; + if( pOvfl->xMemberOfA(pOvfl->pA, pOld) ){ + pOvfl->pA->pMethods->xFree(pOvfl->pA, pOld); + }else{ + pOvfl->pB->pMethods->xFree(pOvfl->pB, pOld); + } +} +static sqlite4_int64 mmOvflMsize(sqlite4_mm *pMM, void *pOld){ + const struct mmOvfl *pOvfl; + sqlite4_int64 iSz; + if( pOld==0 ) return 0; + pOvfl = (const struct mmOvfl*)pMM; + if( pOvfl->xMemberOfA(pOvfl->pA, pOld) ){ + iSz = sqlite4_mm_msize(pOvfl->pA, pOld); + }else{ + iSz = sqlite4_mm_msize(pOvfl->pB, pOld); + } + return iSz; +} +static int mmOvflMember(sqlite4_mm *pMM, const void *pOld){ + const struct mmOvfl *pOvfl; + int iRes; + if( pOld==0 ) return 0; + pOvfl = (const struct mmOvfl*)pMM; + if( pOvfl->xMemberOfA(pOvfl->pA, pOld) ){ + iRes = 1; + }else{ + iRes = sqlite4_mm_member(pOvfl->pB, pOld); + } + return iRes; +} +static void mmOvflBenign(sqlite4_mm *pMM, int bEnable){ + struct mmOvfl *pOvfl = (struct mmOvfl*)pMM; + sqlite4_mm_benign_failures(pOvfl->pA, bEnable); + sqlite4_mm_benign_failures(pOvfl->pB, bEnable); +} +static void mmOvflFinal(sqlite4_mm *pMM){ + struct mmOvfl *pOvfl = (struct mmOvfl*)pMM; + sqlite4_mm *pA = pOvfl->pA; + sqlite4_mm *pB = pOvfl->pB; + mmOvflFree(pMM, pMM); + sqlite4_mm_destroy(pA); + sqlite4_mm_destroy(pB); +} +static const sqlite4_mm_methods mmOvflMethods = { + /* iVersion */ 1, + /* xMalloc */ mmOvflMalloc, + /* xRealloc */ mmOvflRealloc, + /* xFree */ mmOvflFree, + /* xMsize */ mmOvflMsize, + /* xMember */ mmOvflMember, + /* xBenign */ mmOvflBenign, + /* xFinal */ mmOvflFinal +}; +static sqlite4_mm *mmOvflNew(sqlite4_mm *pA, sqlite4_mm *pB){ + struct mmOvfl *pOvfl; + if( pA->pMethods->xMember==0 ) return 0; + pOvfl = sqlite4_mm_malloc(pA, sizeof(*pOvfl)); + if( pOvfl==0 ){ + pOvfl = sqlite4_mm_malloc(pB, sizeof(*pOvfl)); + } + if( pOvfl ){ + pOvfl->base.pMethods = &mmOvflMethods; + pOvfl->xMemberOfA = pA->pMethods->xMember; + pOvfl->pA = pA; + pOvfl->pB = pB; + } + return &pOvfl->base; +} + + +/************************************************************************* +** The SQLITE4_MM_ONESIZE memory allocator. +** +** All memory allocations are rounded up to a single size, "sz". A request +** for an allocation larger than sz bytes fails. All allocations come out +** of a single initial buffer with "cnt" chunks of "sz" bytes each. +** +** Space to hold the sqlite4_mm object comes from the first block in the +** allocation space. +*/ +struct mmOnesz { + sqlite4_mm base; /* Base class. Must be first. */ + const void *pSpace; /* Space to allocate */ + const void *pLast; /* Last possible allocation */ + struct mmOneszBlock *pFree; /* List of free blocks */ + int sz; /* Size of each allocation */ +}; + +/* A free block in the buffer */ +struct mmOneszBlock { + struct mmOneszBlock *pNext; /* Next on the freelist */ +}; + +static void *mmOneszMalloc(sqlite4_mm *pMM, sqlite4_int64 iSz){ + struct mmOnesz *pOnesz = (struct mmOnesz*)pMM; + void *pRes; + if( iSz>pOnesz->sz ) return 0; + if( pOnesz->pFree==0 ) return 0; + pRes = pOnesz->pFree; + pOnesz->pFree = pOnesz->pFree->pNext; + return pRes; +} +static void mmOneszFree(sqlite4_mm *pMM, void *pOld){ + struct mmOnesz *pOnesz = (struct mmOnesz*)pMM; + if( pOld ){ + struct mmOneszBlock *pBlock = (struct mmOneszBlock*)pOld; + pBlock->pNext = pOnesz->pFree; + pOnesz->pFree = pBlock; + } +} +static void *mmOneszRealloc(sqlite4_mm *pMM, void *pOld, sqlite4_int64 iSz){ + struct mmOnesz *pOnesz = (struct mmOnesz*)pMM; + if( pOld==0 ) return mmOneszMalloc(pMM, iSz); + if( iSz<=0 ){ + mmOneszFree(pMM, pOld); + return 0; + } + if( iSz>pOnesz->sz ) return 0; + return pOld; +} +static sqlite4_int64 mmOneszMsize(sqlite4_mm *pMM, void *pOld){ + struct mmOnesz *pOnesz = (struct mmOnesz*)pMM; + return pOld ? pOnesz->sz : 0; +} +static int mmOneszMember(sqlite4_mm *pMM, const void *pOld){ + struct mmOnesz *pOnesz = (struct mmOnesz*)pMM; + return pOld && pOld>=pOnesz->pSpace && pOld<=pOnesz->pLast; +} +static const sqlite4_mm_methods mmOneszMethods = { + /* iVersion */ 1, + /* xMalloc */ mmOneszMalloc, + /* xRealloc */ mmOneszRealloc, + /* xFree */ mmOneszFree, + /* xMsize */ mmOneszMsize, + /* xMember */ mmOneszMember, + /* xBenign */ 0, + /* xFinal */ 0 +}; +static sqlite4_mm *mmOneszNew(void *pSpace, int sz, int cnt){ + struct mmOnesz *pOnesz; + unsigned char *pMem; + if( szbase.pMethods = &mmOneszMethods; + pOnesz->pSpace = (const void*)pMem; + pOnesz->sz = sz; + pOnesz->pLast = (const void*)(pMem + sz*(cnt-2)); + pOnesz->pFree = 0; + while( cnt ){ + struct mmOneszBlock *pBlock = (struct mmOneszBlock*)pMem; + pBlock->pNext = pOnesz->pFree; + pOnesz->pFree = pBlock; + cnt--; + pMem += sz; + } + return &pOnesz->base; +} + +/************************************************************************* +** Main interfaces. +*/ +void *sqlite4_mm_malloc(sqlite4_mm *pMM, sqlite4_int64 iSize){ + if( pMM==0 ) pMM = &mmSystem; + return pMM->pMethods->xMalloc(pMM,iSize); +} +void *sqlite4_mm_realloc(sqlite4_mm *pMM, void *pOld, sqlite4_int64 iSize){ + if( pMM==0 ) pMM = &mmSystem; + return pMM->pMethods->xRealloc(pMM,pOld,iSize); +} +void sqlite4_mm_free(sqlite4_mm *pMM, void *pOld){ + if( pMM==0 ) pMM = &mmSystem; + pMM->pMethods->xFree(pMM,pOld); +} +sqlite4_int64 sqlite4_mm_msize(sqlite4_mm *pMM, void *pOld){ + if( pMM==0 ) pMM = &mmSystem; + return pMM->pMethods->xMsize(pMM,pOld); +} +int sqlite4_mm_member(sqlite4_mm *pMM, const void *pOld){ + return (pMM && pMM->pMethods->xMember!=0) ? + pMM->pMethods->xMember(pMM,pOld) : -1; +} +void sqlite4_mm_benign_failure(sqlite4_mm *pMM, int bEnable){ + if( pMM && pMM->pMethods->xBenign ){ + pMM->pMethods->xBenign(pMM, bEnable); + } +} +void sqlite4_mm_destroy(sqlite4_mm *pMM){ + if( pMM && pMM->pMethods->xFinal ) pMM->pMethods->xFinal(pMM); +} + +/* +** Create a new memory allocation object. eType determines the type of +** memory allocator and the arguments. +*/ +sqlite4_mm *sqlite4_mm_new(sqlite4_mm_type eType, ...){ + va_list ap; + sqlite4_mm *pMM; + + va_start(ap, eType); + switch( eType ){ + case SQLITE4_MM_SYSTEM: { + pMM = &mmSystem; + break; + } + case SQLITE4_MM_OVERFLOW: { + sqlite4_mm *pA = va_arg(ap, sqlite4_mm*); + sqlite4_mm *pB = va_arg(ap, sqlite4_mm*); + pMM = mmOvflNew(pA, pB); + break; + } + case SQLITE4_MM_ONESIZE: { + void *pSpace = va_arg(ap, void*); + int sz = va_arg(ap, int); + int cnt = va_arg(ap, int); + pMM = mmOneszNew(pSpace, sz, cnt); + break; + } + default: { + pMM = 0; + break; + } + } + va_end(ap); + return pMM; +} ADDED src/mem.h Index: src/mem.h ================================================================== --- /dev/null +++ src/mem.h @@ -0,0 +1,109 @@ +/* +** 2013-01-01 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface that the SQLite4 memory +** management logic. +** +** Some of this will eventually fold into sqliteInt.h. +*/ + + +/* +** object declarations +*/ +typedef struct sqlite4_mm sqlite4_mm; +typedef struct sqlite4_mm_methods sqlite4_mm_methods; + +/* +** Base class. Each implementation extends this with additional +** fields specific to its own needs. This needs to be public so that +** applications can supply their on customized memory allocators. +*/ +struct sqlite4_mm { + const sqlite4_mm_methods *pMethods; +}; + +/* +** An instance of the following object defines a BESPOKE memory alloator +*/ +struct sqlite4_mm_methods { + int iVersion; + void *(*xMalloc)(sqlite4_mm*, sqlite4_int64); + void *(*xRealloc)(sqlite4_mm*, void*, sqlite4_int64); + void (*xFree)(sqlite4_mm*, void*); + sqlite4_int64 (*xMsize)(sqlite4_mm*, void*); + int (*xMember)(sqlite4_mm*, const void*); + void (*xBenign)(sqlite4_mm*, int); + void (*xFinal)(sqlite4_mm*); +}; + +/* +** Available memory management types: +*/ +typedef enum { + SQLITE4_MM_SYSTEM, /* Use the system malloc() */ + SQLITE4_MM_ONESIZE, /* All allocations map to a fixed size */ + SQLITE4_MM_OVERFLOW, /* Two allocators. Use A first; failover to B */ + SQLITE4_MM_COMPACT, /* Like memsys3 from SQLite3 */ + SQLITE4_MM_ROBSON, /* Like memsys5 from SQLite3 */ + SQLITE4_MM_LINEAR, /* Allocate from a fixed buffer w/o free */ + SQLITE4_MM_DEBUG, /* Debugging memory allocator */ + SQLITE4_MM_STATS, /* Keep memory statistics */ + SQLITE4_MM_BESPOKE /* Caller-defined implementation */ +} sqlite4_mm_type; + +/* +** Allocate a new memory manager. Return NULL if unable. +*/ +sqlite4_mm *sqlite4_mm_new(sqlite4_mm_type, ...); + +/* +** Free the sqlite4_mm object. +** +** All outstanding memory for the allocator must be freed prior to +** invoking this interface, or else the behavior is undefined. +*/ +void sqlite4_mm_destroy(sqlite4_mm*); + +/* +** Core memory allocation routines: +*/ +void *sqlite4_mm_malloc(sqlite4_mm*, sqlite4_int64); +void *sqlite4_mm_realloc(sqlite4_mm*, void*, sqlite4_int64); +void sqlite4_mm_free(sqlite4_mm*, void*); + +/* +** Return the size of a memory allocation. +** +** All memory allocators in SQLite4 must be able to report their size. +** When using system malloc() on system that lack the malloc_usable_size() +** routine or its equivalent, then the sqlite4_mm object allocates 8 extra +** bytes for each memory allocation and stores the allocation size in those +** initial 8 bytes. +*/ +sqlite4_int64 sqlite4_mm_msize(sqlite4_mm*, void*); + +/* +** Check to see if pOld is a memory allocation from pMM. If it is, return +** 1. If not, return 0. If we cannot determine an answer, return -1. +** +** If pOld is not a valid memory allocation or is a memory allocation that +** has previously been freed, then the result of this routine is undefined. +*/ +int sqlite4_mm_member(sqlite4_mm *pMM, const void *pOld); + +/* +** Enable or disable benign failure mode. Benign failure mode can be +** nested. In benign failure mode, OOM errors do not necessarily propagate +** back out to the application but can be dealt with internally. Memory +** allocations that occur in benign failure mode are considered "optional". +*/ +void sqlite4_mm_benign_failures(sqlite4_mm*, int bEnable); Index: src/mutex_noop.c ================================================================== --- src/mutex_noop.c +++ src/mutex_noop.c @@ -35,12 +35,12 @@ ** ** This routines provide no mutual exclusion or error checking. */ static int noopMutexInit(void *p){ UNUSED_PARAMETER(p); return SQLITE4_OK; } static int noopMutexEnd(void *p){ UNUSED_PARAMETER(p); return SQLITE4_OK; } -static sqlite4_mutex *noopMutexAlloc(sqlite4_env *pEnv, int id){ - UNUSED_PARAMETER(pEnv); +static sqlite4_mutex *noopMutexAlloc(void *p, int id){ + UNUSED_PARAMETER(p); UNUSED_PARAMETER(id); return (sqlite4_mutex*)8; } static void noopMutexFree(sqlite4_mutex *p){ UNUSED_PARAMETER(p); return; } static void noopMutexEnter(sqlite4_mutex *p){ UNUSED_PARAMETER(p); return; } Index: src/parse.y ================================================================== --- src/parse.y +++ src/parse.y @@ -91,10 +91,18 @@ /* ** An instance of this structure holds the ATTACH key and the key type. */ struct AttachKey { int type; Token key; }; + +/* +** One or more VALUES claues +*/ +struct ValueList { + ExprList *pList; + Select *pSelect; +}; } // end %include // Input is a single SQL command input ::= cmdlist. @@ -677,30 +685,56 @@ sqlite4ExprListSetName(pParse, A, &X, 1); } ////////////////////////// The INSERT command ///////////////////////////////// // -cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) - VALUES LP itemlist(Y) RP. - {sqlite4Insert(pParse, X, Y, 0, F, R);} +cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) valuelist(Y). + {sqlite4Insert(pParse, X, Y.pList, Y.pSelect, F, R);} cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). {sqlite4Insert(pParse, X, 0, S, F, R);} cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES. {sqlite4Insert(pParse, X, 0, 0, F, R);} %type insert_cmd {u8} insert_cmd(A) ::= INSERT orconf(R). {A = R;} insert_cmd(A) ::= REPLACE. {A = OE_Replace;} - -%type itemlist {ExprList*} -%destructor itemlist {sqlite4ExprListDelete(pParse->db, $$);} - -itemlist(A) ::= itemlist(X) COMMA expr(Y). - {A = sqlite4ExprListAppend(pParse,X,Y.pExpr);} -itemlist(A) ::= expr(X). - {A = sqlite4ExprListAppend(pParse,0,X.pExpr);} +// A ValueList is either a single VALUES clause or a comma-separated list +// of VALUES clauses. If it is a single VALUES clause then the +// ValueList.pList field points to the expression list of that clause. +// If it is a list of VALUES clauses, then those clauses are transformed +// into a set of SELECT statements without FROM clauses and connected by +// UNION ALL and the ValueList.pSelect points to the right-most SELECT in +// that compound. +%type valuelist {struct ValueList} +%destructor valuelist { + sqlite4ExprListDelete(pParse->db, $$.pList); + sqlite4SelectDelete(pParse->db, $$.pSelect); +} +valuelist(A) ::= VALUES LP nexprlist(X) RP. { + A.pList = X; + A.pSelect = 0; +} +valuelist(A) ::= valuelist(X) COMMA LP exprlist(Y) RP. { + Select *pRight = sqlite4SelectNew(pParse, Y, 0, 0, 0, 0, 0, 0, 0, 0); + if( X.pList ){ + X.pSelect = sqlite4SelectNew(pParse, X.pList, 0, 0, 0, 0, 0, 0, 0, 0); + X.pList = 0; + } + A.pList = 0; + if( X.pSelect==0 || pRight==0 ){ + sqlite4SelectDelete(pParse->db, pRight); + sqlite4SelectDelete(pParse->db, X.pSelect); + A.pSelect = 0; + }else{ + pRight->op = TK_ALL; + pRight->pPrior = X.pSelect; + pRight->selFlags |= SF_Values; + pRight->pPrior->selFlags |= SF_Values; + A.pSelect = pRight; + } +} %type inscollist_opt {IdList*} %destructor inscollist_opt {sqlite4IdListDelete(pParse->db, $$);} %type inscollist {IdList*} %destructor inscollist {sqlite4IdListDelete(pParse->db, $$);} @@ -845,11 +879,11 @@ {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} expr(A) ::= expr(X) CONCAT(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} %type likeop {struct LikeOp} likeop(A) ::= LIKE_KW(X). {A.eOperator = X; A.not = 0;} likeop(A) ::= NOT LIKE_KW(X). {A.eOperator = X; A.not = 1;} -likeop(A) ::= MATCH(X). {A.eOperator = X; A.not = 0;} +/* likeop(A) ::= MATCH(X). {A.eOperator = X; A.not = 0;} */ likeop(A) ::= NOT MATCH(X). {A.eOperator = X; A.not = 1;} expr(A) ::= expr(X) likeop(OP) expr(Y). [LIKE_KW] { ExprList *pList; pList = sqlite4ExprListAppend(pParse,0, Y.pExpr); pList = sqlite4ExprListAppend(pParse,pList, X.pExpr); @@ -868,10 +902,14 @@ if( OP.not ) A.pExpr = sqlite4PExpr(pParse, TK_NOT, A.pExpr, 0, 0); A.zStart = X.zStart; A.zEnd = E.zEnd; if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc; } + +expr(A) ::= expr(L) MATCH expr(R). { + spanBinaryExpr(&A, pParse, TK_MATCH, &L, &R); +} %include { /* Construct an expression node for a unary postfix operator */ static void spanUnaryPostfix( @@ -1085,15 +1123,26 @@ {A = sqlite4ExprListAppend(pParse,0,Y.pExpr);} ///////////////////////////// The CREATE INDEX command /////////////////////// // -cmd ::= createkw(S) uniqueflag(U) INDEX ifnotexists(NE) nm(X) dbnm(D) - ON nm(Y) LP idxlist(Z) RP(E). { - sqlite4CreateIndex(pParse, &X, &D, - sqlite4SrcListAppend(pParse->db,0,&Y,0), Z, U, - &S, &E, SQLITE4_SO_ASC, NE, 0); +%type createindex {CreateIndex} +%destructor createindex {sqlite4SrcListDelete(pParse->db, $$.pTblName);} + +createindex(C) ::= createkw(S) uniqueflag(U) INDEX ifnotexists(NE) + nm(X) dbnm(D) ON nm(Y). { + C.bUnique = U; + C.bIfnotexist = NE; + C.tCreate = S; + C.tName1 = X; + C.tName2 = D; + C.pTblName = sqlite4SrcListAppend(pParse->db, 0, &Y, 0); +} + +cmd ::= createindex(C) LP idxlist(Z) RP(E). { + sqlite4CreateIndex(pParse, &C.tName1, &C.tName2, C.pTblName, Z, + C.bUnique, &C.tCreate, &E, SQLITE4_SO_ASC, C.bIfnotexist, 0); } %type uniqueflag {int} uniqueflag(A) ::= UNIQUE. {A = OE_Abort;} uniqueflag(A) ::= . {A = OE_None;} @@ -1129,10 +1178,37 @@ } %type collate {Token} collate(C) ::= . {C.z = 0; C.n = 0;} collate(C) ::= COLLATE ids(X). {C = X;} + +%type uidxlist_opt {ExprList*} +%destructor uidxlist_opt {sqlite4ExprListDelete(pParse->db, $$);} +%type uidxlist {ExprList*} +%destructor uidxlist {sqlite4ExprListDelete(pParse->db, $$);} + +cmd ::= createindex(C) USING nm(F) LP uidxlist_opt(U) RP(E). { + sqlite4CreateUsingIndex(pParse, &C, U, &F, &E); +} + +uidxlist_opt(A) ::= uidxlist(B). { A = B; } +uidxlist_opt(A) ::= . { A = 0; } + +uidxlist(A) ::= uidxlist(Z) COMMA ids(X) EQ ids(Y). { + A = sqlite4ExprListAppend(pParse, Z, sqlite4PExpr(pParse, @Y, 0, 0, &Y)); + sqlite4ExprListSetName(pParse, A, &X, 1); +} +uidxlist(A) ::= uidxlist(Z) COMMA ids(Y). { + A = sqlite4ExprListAppend(pParse, Z, sqlite4PExpr(pParse, @Y, 0, 0, &Y)); +} +uidxlist(A) ::= ids(X) EQ ids(Y). { + A = sqlite4ExprListAppend(pParse, 0, sqlite4PExpr(pParse, @Y, 0, 0, &Y)); + sqlite4ExprListSetName(pParse, A, &X, 1); +} +uidxlist(A) ::= ids(Y). { + A = sqlite4ExprListAppend(pParse, 0, sqlite4PExpr(pParse, @Y, 0, 0, &Y)); +} ///////////////////////////// The DROP INDEX command ///////////////////////// // cmd ::= DROP INDEX ifexists(E) fullname(X). {sqlite4DropIndex(pParse, X, E);} @@ -1250,12 +1326,12 @@ UPDATE orconf(R) trnm(X) tridxby SET setlist(Y) where_opt(Z). { A = sqlite4TriggerUpdateStep(pParse->db, &X, Y, Z, R); } // INSERT trigger_cmd(A) ::= - insert_cmd(R) INTO trnm(X) inscollist_opt(F) VALUES LP itemlist(Y) RP. - {A = sqlite4TriggerInsertStep(pParse->db, &X, F, Y, 0, R);} + insert_cmd(R) INTO trnm(X) inscollist_opt(F) valuelist(Y). + {A = sqlite4TriggerInsertStep(pParse->db, &X, F, Y.pList, Y.pSelect, R);} trigger_cmd(A) ::= insert_cmd(R) INTO trnm(X) inscollist_opt(F) select(S). {A = sqlite4TriggerInsertStep(pParse->db, &X, F, 0, S, R);} // DELETE Index: src/pragma.c ================================================================== --- src/pragma.c +++ src/pragma.c @@ -26,11 +26,11 @@ if( sqlite4Isdigit(*z) ){ return (u8)sqlite4Atoi(z); } n = sqlite4Strlen30(z); for(i=0; izName)==0 ){ + if( sqlite4_stricmp(zLeft, p->zName)==0 ){ sqlite4 *db = pParse->db; Vdbe *v; v = sqlite4GetVdbe(pParse); assert( v!=0 ); /* Already allocated by sqlite4Pragma() */ if( ALWAYS(v) ){ @@ -222,24 +223,87 @@ /* The flagPragma() subroutine also generates any necessary code ** there is nothing more to do here */ }else #endif /* SQLITE4_OMIT_FLAG_PRAGMAS */ - if( sqlite4StrICmp(zLeft, "lsm_flush")==0 ){ + if( sqlite4_stricmp(zLeft, "lsm_flush")==0 ){ sqlite4_kvstore_control(db, zDb, SQLITE4_KVCTRL_LSM_FLUSH, 0); }else - if( sqlite4StrICmp(zLeft, "lsm_checkpoint")==0 ){ + if( sqlite4_stricmp(zLeft, "lsm_checkpoint")==0 ){ sqlite4_kvstore_control(db, zDb, SQLITE4_KVCTRL_LSM_CHECKPOINT, 0); }else - if( sqlite4StrICmp(zLeft, "lsm_merge")==0 ){ + if( sqlite4_stricmp(zLeft, "lsm_merge")==0 ){ int nPage = zRight ? sqlite4Atoi(zRight) : 1000; sqlite4_kvstore_control(db, zDb, SQLITE4_KVCTRL_LSM_MERGE, &nPage); returnSingleInt(pParse, "nWrite", (sqlite4_int64)nPage); }else + /* + ** PRAGMA fts_check() + */ + if( sqlite4_stricmp(zLeft, "fts_check")==0 && zRight ){ + int iCksum1; + int iCksum2; + Index *pIdx; + Table *pTab; + Vdbe *v = sqlite4GetVdbe(pParse); + if( v==0 || sqlite4ReadSchema(pParse) ) goto pragma_out; + + iCksum1 = ++pParse->nMem; + iCksum2 = ++pParse->nMem; + sqlite4VdbeAddOp2(v, OP_Integer, 0, iCksum1); + sqlite4VdbeAddOp2(v, OP_Integer, 0, iCksum2); + + pIdx = sqlite4FindIndex(db, zRight, zDb); + if( pIdx && pIdx->eIndexType==SQLITE4_INDEX_FTS5 ){ + int iTab = pParse->nTab++; + int iAddr; + int iReg; + int i; + + pTab = pIdx->pTable; + sqlite4OpenPrimaryKey(pParse, iTab, iDb, pTab, OP_OpenRead); + iAddr = sqlite4VdbeAddOp2(v, OP_Rewind, iTab, 0); + + iReg = pParse->nMem+1; + pParse->nMem += (1 + pTab->nCol); + + sqlite4VdbeAddOp2(v, OP_RowKey, iTab, iReg); + for(i=0; inCol; i++){ + sqlite4VdbeAddOp3(v, OP_Column, iTab, i, iReg+1+i); + } + sqlite4Fts5CodeCksum(pParse, pIdx, iCksum1, iReg, 0); + + sqlite4VdbeAddOp2(v, OP_Next, iTab, iAddr+1); + sqlite4VdbeJumpHere(v, iAddr); + sqlite4VdbeAddOp1(v, OP_Close, iTab); + + sqlite4VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); + iAddr = sqlite4VdbeAddOp2(v, OP_Rewind, iTab, 0); + + iReg = pParse->nMem+1; + pParse->nMem += 2; + sqlite4VdbeAddOp2(v, OP_RowKey, iTab, iReg); + sqlite4VdbeAddOp2(v, OP_RowData, iTab, iReg+1); + sqlite4Fts5CodeCksum(pParse, pIdx, iCksum2, iReg, 1); + + sqlite4VdbeAddOp2(v, OP_Next, iTab, iAddr+1); + sqlite4VdbeJumpHere(v, iAddr); + sqlite4VdbeAddOp1(v, OP_Close, iTab); + + iReg = ++pParse->nMem; + sqlite4VdbeAddOp4(v, OP_String8, 0, iReg, 0, "ok", 0); + iAddr = sqlite4VdbeAddOp3(v, OP_Eq, iCksum1, 0, iCksum2); + sqlite4VdbeAddOp4(v, OP_String8, 0, iReg, 0, "error - cksum mismatch", 0); + sqlite4VdbeJumpHere(v, iAddr); + sqlite4VdbeAddOp2(v, OP_ResultRow, iReg, 1); + + sqlite4VdbeSetNumCols(v, 1); + } + } #ifndef SQLITE4_OMIT_SCHEMA_PRAGMAS /* @@ -252,11 +316,11 @@ ** name: Column name ** type: Column declaration type. ** notnull: True if 'NOT NULL' is part of column declaration ** dflt_value: The default value for the column, if any. */ - if( sqlite4StrICmp(zLeft, "table_info")==0 && zRight ){ + if( sqlite4_stricmp(zLeft, "table_info")==0 && zRight ){ Table *pTab; if( sqlite4ReadSchema(pParse) ) goto pragma_out; pTab = sqlite4FindTable(db, zRight, zDb); if( pTab ){ int i; @@ -290,11 +354,11 @@ sqlite4VdbeAddOp2(v, OP_ResultRow, 1, 6); } } }else - if( sqlite4StrICmp(zLeft, "index_info")==0 && zRight ){ + if( sqlite4_stricmp(zLeft, "index_info")==0 && zRight ){ Index *pIdx; Table *pTab; if( sqlite4ReadSchema(pParse) ) goto pragma_out; pIdx = sqlite4FindIndex(db, zRight, zDb); if( pIdx ){ @@ -314,11 +378,11 @@ sqlite4VdbeAddOp2(v, OP_ResultRow, 1, 3); } } }else - if( sqlite4StrICmp(zLeft, "index_list")==0 && zRight ){ + if( sqlite4_stricmp(zLeft, "index_list")==0 && zRight ){ Index *pIdx; Table *pTab; if( sqlite4ReadSchema(pParse) ) goto pragma_out; pTab = sqlite4FindTable(db, zRight, zDb); if( pTab ){ @@ -341,11 +405,11 @@ } } } }else - if( sqlite4StrICmp(zLeft, "database_list")==0 ){ + if( sqlite4_stricmp(zLeft, "database_list")==0 ){ int i; if( sqlite4ReadSchema(pParse) ) goto pragma_out; sqlite4VdbeSetNumCols(v, 3); pParse->nMem = 3; sqlite4VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE4_STATIC); @@ -360,11 +424,11 @@ "filename", 0); sqlite4VdbeAddOp2(v, OP_ResultRow, 1, 3); } }else - if( sqlite4StrICmp(zLeft, "collation_list")==0 ){ + if( sqlite4_stricmp(zLeft, "collation_list")==0 ){ int i = 0; HashElem *p; sqlite4VdbeSetNumCols(v, 2); pParse->nMem = 2; sqlite4VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE4_STATIC); @@ -377,11 +441,11 @@ } }else #endif /* SQLITE4_OMIT_SCHEMA_PRAGMAS */ #ifndef SQLITE4_OMIT_FOREIGN_KEY - if( sqlite4StrICmp(zLeft, "foreign_key_list")==0 && zRight ){ + if( sqlite4_stricmp(zLeft, "foreign_key_list")==0 && zRight ){ FKey *pFK; Table *pTab; if( sqlite4ReadSchema(pParse) ) goto pragma_out; pTab = sqlite4FindTable(db, zRight, zDb); if( pTab ){ @@ -423,11 +487,11 @@ } }else #endif /* !defined(SQLITE4_OMIT_FOREIGN_KEY) */ #ifndef NDEBUG - if( sqlite4StrICmp(zLeft, "parser_trace")==0 ){ + if( sqlite4_stricmp(zLeft, "parser_trace")==0 ){ if( zRight ){ if( sqlite4GetBoolean(zRight) ){ sqlite4ParserTrace(stderr, "parser: "); }else{ sqlite4ParserTrace(0, 0); @@ -437,11 +501,11 @@ #endif /* Reinstall the LIKE and GLOB functions. The variant of LIKE ** used will be case sensitive or not depending on the RHS. */ - if( sqlite4StrICmp(zLeft, "case_sensitive_like")==0 ){ + if( sqlite4_stricmp(zLeft, "case_sensitive_like")==0 ){ if( zRight ){ sqlite4RegisterLikeFunctions(db, sqlite4GetBoolean(zRight)); } }else @@ -471,11 +535,11 @@ ** ** In the second form this pragma sets the text encoding to be used in ** new database files created using this database handle. It is only ** useful if invoked immediately after the main database i */ - if( sqlite4StrICmp(zLeft, "encoding")==0 ){ + if( sqlite4_stricmp(zLeft, "encoding")==0 ){ static const struct EncName { char *zName; u8 enc; } encnames[] = { { "UTF8", SQLITE4_UTF8 }, @@ -508,11 +572,11 @@ if( !(DbHasProperty(db, 0, DB_SchemaLoaded)) || DbHasProperty(db, 0, DB_Empty) ){ for(pEnc=&encnames[0]; pEnc->zName; pEnc++){ - if( 0==sqlite4StrICmp(zRight, pEnc->zName) ){ + if( 0==sqlite4_stricmp(zRight, pEnc->zName) ){ ENC(pParse->db) = pEnc->enc ? pEnc->enc : SQLITE4_UTF16NATIVE; break; } } if( !pEnc->zName ){ @@ -529,11 +593,11 @@ ** PRAGMA compile_options ** ** Return the names of all compile-time options used in this build, ** one option per row. */ - if( sqlite4StrICmp(zLeft, "compile_options")==0 ){ + if( sqlite4_stricmp(zLeft, "compile_options")==0 ){ int i = 0; const char *zOpt; sqlite4VdbeSetNumCols(v, 1); pParse->nMem = 1; sqlite4VdbeSetColName(v, 0, COLNAME_NAME, "compile_option", SQLITE4_STATIC); @@ -548,21 +612,21 @@ /* ** PRAGMA kvdump ** ** Print an ascii rendering of the complete content of the database file. */ - if( sqlite4StrICmp(zLeft, "kvdump")==0 ){ + if( sqlite4_stricmp(zLeft, "kvdump")==0 ){ sqlite4KVStoreDump(db->aDb[0].pKV); }else #endif /* SQLITE4_OMIT_COMPILEOPTION_DIAGS */ /* ** PRAGMA integrity_check ** ** Check that for each table, the content of any auxilliary indexes are ** consistent with the primary key index. */ - if( sqlite4StrICmp(zLeft, "integrity_check")==0 ){ + if( sqlite4_stricmp(zLeft, "integrity_check")==0 ){ const int baseCsr = 1; /* Base cursor for OpenAllIndexes() call */ const int regErrcnt = 1; /* Register containing error count */ const int regErrstr = 2; /* Register containing error string */ const int regTmp = 3; /* Register for tmp use */ @@ -720,11 +784,11 @@ ** PRAGMA shrink_memory ** ** This pragma attempts to free as much memory as possible from the ** current database connection. */ - if( sqlite4StrICmp(zLeft, "shrink_memory")==0 ){ + if( sqlite4_stricmp(zLeft, "shrink_memory")==0 ){ sqlite4_db_release_memory(db); }else {/* Empty ELSE clause */} Index: src/resolve.c ================================================================== --- src/resolve.c +++ src/resolve.c @@ -107,11 +107,11 @@ */ static int nameInUsingClause(IdList *pUsing, const char *zCol){ if( pUsing ){ int k; for(k=0; knId; k++){ - if( sqlite4StrICmp(pUsing->a[k].zName, zCol)==0 ) return 1; + if( sqlite4_stricmp(pUsing->a[k].zName, zCol)==0 ) return 1; } } return 0; } @@ -119,11 +119,11 @@ ** Return true if table pTab has an implicit primary key, and zCol points ** to a column name that resolves to the implicit primary key (i.e. "rowid"). */ int isRowidReference(Table *pTab, const char *zCol){ int ret = 0; - if( 0==sqlite4StrICmp(zCol, "ROWID") ){ + if( 0==sqlite4_stricmp(zCol, "ROWID") ){ /* If the call to FindPrimaryKey() returns NULL, then pTab must be a ** sub-select or a view. Neither of these have an IPK. */ Index *pPk = sqlite4FindPrimaryKey(pTab, 0); if( pPk && pPk->aiColumn[0]==-1 ) ret = 1; } @@ -200,17 +200,17 @@ iDb = sqlite4SchemaToIndex(db, pTab->pSchema); assert( pTab->nCol>0 ); if( zTab ){ if( pItem->zAlias ){ char *zTabName = pItem->zAlias; - if( sqlite4StrICmp(zTabName, zTab)!=0 ) continue; + if( sqlite4_stricmp(zTabName, zTab)!=0 ) continue; }else{ char *zTabName = pTab->zName; - if( NEVER(zTabName==0) || sqlite4StrICmp(zTabName, zTab)!=0 ){ + if( NEVER(zTabName==0) || sqlite4_stricmp(zTabName, zTab)!=0 ){ continue; } - if( zDb!=0 && sqlite4StrICmp(db->aDb[iDb].zName, zDb)!=0 ){ + if( zDb!=0 && sqlite4_stricmp(db->aDb[iDb].zName, zDb)!=0 ){ continue; } } } if( 0==(cntTab++) ){ @@ -218,11 +218,11 @@ pExpr->pTab = pTab; pSchema = pTab->pSchema; pMatch = pItem; } for(j=0, pCol=pTab->aCol; jnCol; j++, pCol++){ - if( sqlite4StrICmp(pCol->zName, zCol)==0 ){ + if( sqlite4_stricmp(pCol->zName, zCol)==0 ){ /* If there has been exactly one prior match and this match ** is for the right-hand table of a NATURAL JOIN or is in a ** USING clause, then skip this match. */ if( cnt==1 ){ @@ -247,14 +247,14 @@ */ if( zDb==0 && zTab!=0 && cnt==0 && pParse->pTriggerTab!=0 ){ int op = pParse->eTriggerOp; Table *pTab = 0; assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT ); - if( op!=TK_DELETE && sqlite4StrICmp("new",zTab) == 0 ){ + if( op!=TK_DELETE && sqlite4_stricmp("new",zTab) == 0 ){ pExpr->iTable = 1; pTab = pParse->pTriggerTab; - }else if( op!=TK_INSERT && sqlite4StrICmp("old",zTab)==0 ){ + }else if( op!=TK_INSERT && sqlite4_stricmp("old",zTab)==0 ){ pExpr->iTable = 0; pTab = pParse->pTriggerTab; } if( pTab ){ @@ -261,11 +261,11 @@ int iCol; pSchema = pTab->pSchema; cntTab++; for(iCol=0; iColnCol; iCol++){ Column *pCol = &pTab->aCol[iCol]; - if( sqlite4StrICmp(pCol->zName, zCol)==0 ){ + if( sqlite4_stricmp(pCol->zName, zCol)==0 ){ break; } } if( iCol>=pTab->nCol && isRowidReference(pTab, zCol) ){ iCol = -1; /* IMP: R-44911-55124 */ @@ -313,11 +313,11 @@ ** resolved by the time the WHERE clause is resolved. */ if( cnt==0 && (pEList = pNC->pEList)!=0 && zTab==0 ){ for(j=0; jnExpr; j++){ char *zAs = pEList->a[j].zName; - if( zAs!=0 && sqlite4StrICmp(zAs, zCol)==0 ){ + if( zAs!=0 && sqlite4_stricmp(zAs, zCol)==0 ){ Expr *pOrig; assert( pExpr->pLeft==0 && pExpr->pRight==0 ); assert( pExpr->x.pList==0 ); assert( pExpr->x.pSelect==0 ); pOrig = pEList->a[j].pExpr; @@ -340,26 +340,10 @@ if( cnt==0 ){ pNC = pNC->pNext; } } - /* - ** If X and Y are NULL (in other words if only the column name Z is - ** supplied) and the value of Z is enclosed in double-quotes, then - ** Z is a string literal if it doesn't match any column names. In that - ** case, we need to return right away and not make any changes to - ** pExpr. - ** - ** Because no reference was made to outer contexts, the pNC->nRef - ** fields are not changed in any context. - */ - if( cnt==0 && zTab==0 && ExprHasProperty(pExpr,EP_DblQuoted) ){ - pExpr->op = TK_STRING; - pExpr->pTab = 0; - return WRC_Prune; - } - /* ** cnt==0 means there was not match. cnt>1 means there were two or ** more matches. Either way, we have an error. */ if( cnt!=1 ){ @@ -433,10 +417,76 @@ pItem->colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol); ExprSetProperty(p, EP_Resolved); } return p; } + +static void resolveMatchArg(Parse *pParse, NameContext *pNC, Expr *pExpr){ + SrcList *pSrc = pNC->pSrcList; + SrcListItem *pItem; + char *zLhs; + int i; + Index *pIdx; + + if( pExpr->op!=TK_ID || pSrc==0 || pExpr==0 ){ + sqlite4ErrorMsg(pParse, "first argument xxx must be a table name"); + return; + } + zLhs = pExpr->u.zToken; + + for(i=0; inSrc; i++){ + pItem = &pSrc->a[i]; + if( pItem->zAlias && sqlite4_stricmp(zLhs, pItem->zAlias)==0 ) break; + if( pItem->zAlias==0 && sqlite4_stricmp(zLhs, pItem->zName)==0 ) break; + } + if( i==pSrc->nSrc ){ + sqlite4ErrorMsg(pParse, "no such table: %s", zLhs); + return; + } + + pExpr->op = TK_NULL; + pExpr->iTable = pItem->iCursor; + ExprSetProperty(pExpr, EP_Resolved); +} + +static void resolveMatch(Parse *pParse, NameContext *pNC, Expr *pExpr){ + Expr *pLeft = pExpr->pLeft; + SrcList *pSrc = pNC->pSrcList; + SrcListItem *pItem; + char *zLhs; + int i; + Index *pIdx; + + if( pLeft->op!=TK_ID || pSrc==0 ){ + sqlite4ErrorMsg(pParse, "lhs of MATCH operator must be a table name"); + return; + } + zLhs = pLeft->u.zToken; + + for(i=0; inSrc; i++){ + pItem = &pSrc->a[i]; + if( pItem->zAlias && sqlite4_stricmp(zLhs, pItem->zAlias)==0 ) break; + if( pItem->zAlias==0 && sqlite4_stricmp(zLhs, pItem->zName)==0 ) break; + } + + if( i==pSrc->nSrc ){ + sqlite4ErrorMsg(pParse, "no such table: %s", zLhs); + return; + } + + for(pIdx=pItem->pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->eIndexType==SQLITE4_INDEX_FTS5 ) break; + } + if( !pIdx ){ + sqlite4ErrorMsg(pParse, "no index to process MATCH operator"); + return; + } + + pExpr->pLeft = 0; + pExpr->pIdx = pIdx; + sqlite4ExprDelete(pParse->db, pLeft); +} /* ** This routine is callback for sqlite4WalkExpr(). ** ** Resolve symbolic names into TK_COLUMN operators for the current @@ -576,13 +626,20 @@ } if( is_agg ){ pExpr->op = TK_AGG_FUNCTION; pNC->hasAgg = 1; } - if( is_agg ) pNC->allowAgg = 0; - sqlite4WalkExprList(pWalker, pList); - if( is_agg ) pNC->allowAgg = 1; + + if( pParse->nErr==0 ){ + if( pDef->bMatchinfo ){ + resolveMatchArg(pParse, pNC, n>0 ? pList->a[0].pExpr : 0); + } + if( is_agg ) pNC->allowAgg = 0; + sqlite4WalkExprList(pWalker, pList); + if( is_agg ) pNC->allowAgg = 1; + } + /* FIX ME: Compute pExpr->affinity based on the expected return ** type of the function */ return WRC_Prune; } @@ -613,10 +670,14 @@ sqlite4ErrorMsg(pParse,"parameters prohibited in CHECK constraints"); } break; } #endif + case TK_MATCH: { + resolveMatch(pParse, pNC, pExpr); + break; + } } return (pParse->nErr || pParse->db->mallocFailed) ? WRC_Abort : WRC_Continue; } /* @@ -642,11 +703,11 @@ if( pE->op==TK_ID ){ char *zCol = pE->u.zToken; for(i=0; inExpr; i++){ char *zAs = pEList->a[i].zName; - if( zAs!=0 && sqlite4StrICmp(zAs, zCol)==0 ){ + if( zAs!=0 && sqlite4_stricmp(zAs, zCol)==0 ){ return i+1; } } } return 0; Index: src/rowset.c ================================================================== --- src/rowset.c +++ src/rowset.c @@ -155,16 +155,19 @@ p->isSorted = 1; } static u8 *rowsetAllocateChunk(RowSet *p, int nByte){ - enum { rowChunkSize = ROUND8(sizeof(RowSetChunk)) }; - u8 *pNew; /* New RowSetChunk */ + int rowChunkSize = ROUND8(sizeof(RowSetChunk)); + RowSetChunk *pNew; /* New RowSetChunk */ int nAlloc; /* Bytes to request from malloc() */ nAlloc = rowChunkSize + nByte; - pNew = (u8 *)sqlite4DbMallocRaw(p->db, nAlloc); - return (pNew ? (pNew + rowChunkSize) : 0); + pNew = (RowSetChunk *)sqlite4DbMallocRaw(p->db, nAlloc); + if( !pNew ) return 0; + pNew->pNextChunk = p->pChunk; + p->pChunk = pNew; + return (u8 *)(&pNew[1]); } static int rowsetEntryKeyCmp(RowSetEntry *pLeft, const u8 *aKey, int nKey){ int nCmp = SQLITE4_MIN(pLeft->nKey, nKey); int res; Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -71,10 +71,11 @@ } if( pEList==0 ){ pEList = sqlite4ExprListAppend(pParse, 0, sqlite4Expr(db,TK_ALL,0)); } pNew->pEList = pEList; + if( pSrc==0 ) pSrc = sqlite4DbMallocZero(db, sizeof(*pSrc)); pNew->pSrc = pSrc; pNew->pWhere = pWhere; pNew->pGroupBy = pGroupBy; pNew->pHaving = pHaving; pNew->pOrderBy = pOrderBy; @@ -149,11 +150,11 @@ apAll[2] = pC; for(i=0; i<3 && apAll[i]; i++){ p = apAll[i]; for(j=0; jn==aKeyword[j].nChar - && sqlite4StrNICmp((char*)p->z, &zKeyText[aKeyword[j].i], p->n)==0 ){ + && sqlite4_strnicmp((char*)p->z, &zKeyText[aKeyword[j].i], p->n)==0 ){ jointype |= aKeyword[j].code; break; } } testcase( j==0 || j==1 || j==2 || j==3 || j==4 || j==5 || j==6 ); @@ -186,11 +187,11 @@ ** is not contained in the table. */ static int columnIndex(Table *pTab, const char *zCol){ int i; for(i=0; inCol; i++){ - if( sqlite4StrICmp(pTab->aCol[i].zName, zCol)==0 ) return i; + if( sqlite4_stricmp(pTab->aCol[i].zName, zCol)==0 ) return i; } return -1; } /* @@ -945,16 +946,16 @@ int ptab2 = pParse->nTab++; sqlite4VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, pOrderBy->nExpr+2); addr = 1 + sqlite4VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); codeOffset(v, p, addrContinue); sqlite4VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); - sqlite4VdbeAddOp3(v, OP_Column, ptab2, pOrderBy->nExpr+1, regRow); + sqlite4VdbeAddOp3(v, OP_Column, ptab2, 0, regRow); sqlite4VdbeChangeP5(v, OPFLAG_CLEARCACHE); }else{ addr = 1 + sqlite4VdbeAddOp2(v, OP_Sort, iTab, addrBreak); codeOffset(v, p, addrContinue); - /* sqlite4VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr+1, regRow); */ + sqlite4VdbeAddOp3(v, OP_Column, iTab, 0, regRow); } switch( eDest ){ case SRT_Table: case SRT_EphemTab: { testcase( eDest==SRT_Table ); @@ -1279,11 +1280,11 @@ int nCol; /* Number of columns in the result set */ Expr *p; /* Expression for a single result column */ char *zName; /* Column name */ int nName; /* Size of name in zName[] */ - *pnCol = nCol = pEList->nExpr; + *pnCol = nCol = pEList ? pEList->nExpr : 0; aCol = *paCol = sqlite4DbMallocZero(db, sizeof(aCol[0])*nCol); if( aCol==0 ) return SQLITE4_NOMEM; for(i=0, pCol=aCol; ipEList && pPrior->pEList ); if( p->pEList->nExpr!=pPrior->pEList->nExpr ){ - sqlite4ErrorMsg(pParse, "SELECTs to the left and right of %s" - " do not have the same number of result columns", selectOpName(p->op)); + if( p->selFlags & SF_Values ){ + sqlite4ErrorMsg(pParse, "all VALUES must have the same number of terms"); + }else{ + sqlite4ErrorMsg(pParse, "SELECTs to the left and right of %s" + " do not have the same number of result columns", selectOpName(p->op)); + } rc = 1; goto multi_select_end; } /* Compound SELECTs that have an ORDER BY clause are handled separately. @@ -2302,11 +2307,12 @@ if( op==TK_ALL ){ regPrev = 0; }else{ int nExpr = p->pEList->nExpr; assert( nOrderBy>=nExpr || db->mallocFailed ); - regPrev = sqlite4GetTempRange(pParse, nExpr+1); + regPrev = pParse->nMem + 1; + pParse->nMem += nExpr + 1; sqlite4VdbeAddOp2(v, OP_Integer, 0, regPrev); pKeyDup = sqlite4DbMallocZero(db, sizeof(*pKeyDup) + nExpr*(sizeof(CollSeq*)+1) ); if( pKeyDup ){ pKeyDup->aSortOrder = (u8*)&pKeyDup->aColl[nExpr]; @@ -2483,16 +2489,10 @@ sqlite4VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); sqlite4VdbeAddOp4(v, OP_Compare, destA.iMem, destB.iMem, nOrderBy, (char*)pKeyMerge, P4_KEYINFO_HANDOFF); sqlite4VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); - /* Release temporary registers - */ - if( regPrev ){ - sqlite4ReleaseTempRange(pParse, regPrev, nOrderBy+1); - } - /* Jump to the this point in order to terminate the query. */ sqlite4VdbeResolveLabel(v, labelEnd); /* Set the number of output columns @@ -3116,13 +3116,13 @@ if( NEVER(ExprHasProperty(pExpr, EP_xIsSelect)) ) return 0; pEList = pExpr->x.pList; if( pEList==0 || pEList->nExpr!=1 ) return 0; if( pEList->a[0].pExpr->op!=TK_AGG_COLUMN ) return WHERE_ORDERBY_NORMAL; assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( sqlite4StrICmp(pExpr->u.zToken,"min")==0 ){ + if( sqlite4_stricmp(pExpr->u.zToken,"min")==0 ){ return WHERE_ORDERBY_MIN; - }else if( sqlite4StrICmp(pExpr->u.zToken,"max")==0 ){ + }else if( sqlite4_stricmp(pExpr->u.zToken,"max")==0 ){ return WHERE_ORDERBY_MAX; } return WHERE_ORDERBY_NORMAL; } @@ -3137,11 +3137,11 @@ if( pFrom->pTab && pFrom->zIndex ){ Table *pTab = pFrom->pTab; char *zIndex = pFrom->zIndex; Index *pIdx; for(pIdx=pTab->pIndex; - pIdx && sqlite4StrICmp(pIdx->zName, zIndex); + pIdx && sqlite4_stricmp(pIdx->zName, zIndex); pIdx=pIdx->pNext ); if( !pIdx ){ sqlite4ErrorMsg(pParse, "no such index: %s", zIndex, 0); pParse->checkSchema = 1; @@ -3314,11 +3314,11 @@ char *zTabName = pFrom->zAlias; if( zTabName==0 ){ zTabName = pTab->zName; } if( db->mallocFailed ) break; - if( zTName && sqlite4StrICmp(zTName, zTabName)!=0 ){ + if( zTName && sqlite4_stricmp(zTName, zTabName)!=0 ){ continue; } tableSeen = 1; for(j=0; jnCol; j++){ Expr *pExpr, *pRight; @@ -4167,11 +4167,11 @@ ** to sort the rows, not to eliminate duplicates). */ sqlite4ExprCacheClear(pParse); regBase = sqlite4GetTempRange(pParse, nGroup); sqlite4ExprCodeExprList(pParse, pGroupBy, regBase, 0); sqlite4VdbeAddOp3(v, OP_MakeIdxKey, sAggInfo.sortingIdx,regBase,regKey); - sqlite4VdbeChangeP5(v, 1); + sqlite4VdbeChangeP5(v, OPFLAG_SEQCOUNT); sqlite4ReleaseTempRange(pParse, regBase, nGroup); /* Encode the record for the sorting index. The record contains all ** required column values from the elements of the FROM clause. ** If no column values are required, insert a NULL into the sorting Index: src/shell.c ================================================================== --- src/shell.c +++ src/shell.c @@ -34,23 +34,19 @@ #include #include "sqlite4.h" #include #include -#if !defined(_WIN32) && !defined(WIN32) && !defined(__OS2__) +#if !defined(_WIN32) && !defined(WIN32) # include # if !defined(__RTP__) && !defined(_WRS_KERNEL) # include # endif # include # include #endif -#ifdef __OS2__ -# include -#endif - #ifdef HAVE_EDITLINE # include #endif #if defined(HAVE_READLINE) && HAVE_READLINE==1 # include @@ -66,10 +62,14 @@ #if defined(_WIN32) || defined(WIN32) # include #define isatty(h) _isatty(h) #define access(f,m) _access((f),(m)) +#undef popen +#define popen(a,b) _popen((a),(b)) +#undef pclose +#define pclose(x) _pclose(x) #else /* Make sure isatty() has a prototype. */ extern int isatty(int); #endif @@ -88,11 +88,11 @@ /* ctype macros that work with signed characters */ #define IsSpace(X) isspace((unsigned char)X) #define IsDigit(X) isdigit((unsigned char)X) #define ToLower(X) (char)tolower((unsigned char)X) -#if !defined(_WIN32) && !defined(WIN32) && !defined(__OS2__) && !defined(__RTP__) && !defined(_WRS_KERNEL) +#if !defined(_WIN32) && !defined(WIN32) && !defined(_WRS_KERNEL) #include #include /* Saved resource information for the beginning of an operation */ static struct rusage sBegin; @@ -319,11 +319,11 @@ ){ assert( 0==argc ); assert( zShellStatic ); UNUSED_PARAMETER(argc); UNUSED_PARAMETER(argv); - sqlite4_result_text(context, zShellStatic, -1, SQLITE4_STATIC); + sqlite4_result_text(context, zShellStatic, -1, SQLITE4_STATIC, 0); } /* ** This routine reads a line of text from FILE in, stores @@ -417,10 +417,11 @@ sqlite4 *db; /* The database */ int echoOn; /* True to echo input commands */ int statsOn; /* True to display memory stats before each finalize */ int cnt; /* Number of records displayed so far */ FILE *out; /* Write results here */ + FILE *traceOut; /* Output for sqlite4_trace() */ int nErr; /* Number of errors seen */ int mode; /* An output mode setting */ int writableSchema; /* True if PRAGMA writable_schema=ON */ int showHeader; /* True to show column names in List or Column mode */ char *zDestTable; /* Name of destination table when MODE_Insert */ @@ -494,11 +495,11 @@ */ static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ int i; char *zBlob = (char *)pBlob; fprintf(out,"X'"); - for(i=0; icolWidth) ){ w = p->colWidth[i]; }else{ w = 0; } - if( w<=0 ){ + if( w==0 ){ w = strlen30(azCol[i] ? azCol[i] : ""); if( w<10 ) w = 10; n = strlen30(azArg && azArg[i] ? azArg[i] : p->nullvalue); if( wactualWidth) ){ p->actualWidth[i] = w; } if( p->showHeader ){ - fprintf(p->out,"%-*.*s%s",w,w,azCol[i], i==nArg-1 ? "\n": " "); + if( w<0 ){ + fprintf(p->out,"%*.*s%s",-w,-w,azCol[i], i==nArg-1 ? "\n": " "); + }else{ + fprintf(p->out,"%-*.*s%s",w,w,azCol[i], i==nArg-1 ? "\n": " "); + } } } if( p->showHeader ){ for(i=0; iactualWidth) ){ w = p->actualWidth[i]; + if( w<0 ) w = -w; }else{ w = 10; } fprintf(p->out,"%-*.*s%s",w,w,"-----------------------------------" "----------------------------------------------------------", @@ -732,12 +741,17 @@ } if( p->mode==MODE_Explain && azArg[i] && strlen30(azArg[i])>w ){ w = strlen30(azArg[i]); } - fprintf(p->out,"%-*.*s%s",w,w, - azArg[i] ? azArg[i] : p->nullvalue, i==nArg-1 ? "\n": " "); + if( w<0 ){ + fprintf(p->out,"%*.*s%s",-w,-w, + azArg[i] ? azArg[i] : p->nullvalue, i==nArg-1 ? "\n": " "); + }else{ + fprintf(p->out,"%-*.*s%s",w,w, + azArg[i] ? azArg[i] : p->nullvalue, i==nArg-1 ? "\n": " "); + } } break; } case MODE_Semi: case MODE_List: { @@ -783,18 +797,18 @@ } case MODE_Tcl: { if( p->cnt++==0 && p->showHeader ){ for(i=0; iout,azCol[i] ? azCol[i] : ""); - fprintf(p->out, "%s", p->separator); + if(iout, "%s", p->separator); } fprintf(p->out,"\n"); } if( azArg==0 ) break; for(i=0; iout, azArg[i] ? azArg[i] : p->nullvalue); - fprintf(p->out, "%s", p->separator); + if(iout, "%s", p->separator); } fprintf(p->out,"\n"); break; } case MODE_Csv: { @@ -935,36 +949,54 @@ return zIn; } /* -** Execute a query statement that has a single result column. Print -** that result column on a line by itself with a semicolon terminator. +** Execute a query statement that will generate SQL output. Print +** the result columns, comma-separated, on a line and then add a +** semicolon terminator to the end of that line. ** -** This is used, for example, to show the schema of the database by -** querying the SQLITE4_MASTER table. +** If the number of columns is 1 and that column contains text "--" +** then write the semicolon on a separate line. That way, if a +** "--" comment occurs at the end of the statement, the comment +** won't consume the semicolon terminator. */ static int run_table_dump_query( struct callback_data *p, /* Query context */ const char *zSelect, /* SELECT statement to extract content */ const char *zFirstRow /* Print before first row, if not NULL */ ){ sqlite4_stmt *pSelect; int rc; + int nResult; + int i; + const char *z; rc = sqlite4_prepare(p->db, zSelect, -1, &pSelect, 0); if( rc!=SQLITE4_OK || !pSelect ){ fprintf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, sqlite4_errmsg(p->db)); p->nErr++; return rc; } rc = sqlite4_step(pSelect); + nResult = sqlite4_column_count(pSelect); while( rc==SQLITE4_ROW ){ if( zFirstRow ){ fprintf(p->out, "%s", zFirstRow); zFirstRow = 0; } - fprintf(p->out, "%s;\n", sqlite4_column_text(pSelect, 0)); + z = (char const *)sqlite4_column_text(pSelect, 0); + fprintf(p->out, "%s", z); + for(i=1; iout, ",%s", sqlite4_column_text(pSelect, i)); + } + if( z==0 ) z = ""; + while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; + if( z[0] ){ + fprintf(p->out, "\n;\n"); + }else{ + fprintf(p->out, ";\n"); + } rc = sqlite4_step(pSelect); } rc = sqlite4_finalize(pSelect); if( rc!=SQLITE4_OK ){ fprintf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, sqlite4_errmsg(p->db)); @@ -1229,23 +1261,26 @@ if( rc!=SQLITE4_OK || !pTableInfo ){ return 1; } zSelect = appendText(zSelect, "SELECT 'INSERT INTO ' || ", 0); + /* Always quote the table name, even if it appears to be pure ascii, + ** in case it is a keyword. Ex: INSERT INTO "table" ... */ zTmp = appendText(zTmp, zTable, '"'); if( zTmp ){ zSelect = appendText(zSelect, zTmp, '\''); + free(zTmp); } zSelect = appendText(zSelect, " || ' VALUES(' || ", 0); rc = sqlite4_step(pTableInfo); while( rc==SQLITE4_ROW ){ const char *zText = (const char *)sqlite4_column_text(pTableInfo, 1); zSelect = appendText(zSelect, "quote(", 0); zSelect = appendText(zSelect, zText, '"'); rc = sqlite4_step(pTableInfo); if( rc==SQLITE4_ROW ){ - zSelect = appendText(zSelect, ") || ',' || ", 0); + zSelect = appendText(zSelect, "), ", 0); }else{ zSelect = appendText(zSelect, ") ", 0); } nRow++; } @@ -1260,11 +1295,11 @@ rc = run_table_dump_query(p, zSelect, zPrepStmt); if( rc==SQLITE4_CORRUPT ){ zSelect = appendText(zSelect, " ORDER BY rowid DESC", 0); run_table_dump_query(p, zSelect, 0); } - if( zSelect ) free(zSelect); + free(zSelect); } return 0; } /* @@ -1338,13 +1373,14 @@ " insert SQL insert statements for TABLE\n" " line One value per line\n" " list Values delimited by .separator string\n" " tabs Tab-separated values\n" " tcl TCL list elements\n" - ".nullvalue STRING Print STRING in place of NULL values\n" + ".nullvalue STRING Use STRING in place of NULL values\n" ".output FILENAME Send output to FILENAME\n" ".output stdout Send output to the screen\n" + ".print STRING... Print literal STRING\n" ".prompt MAIN CONTINUE Replace the standard prompts\n" ".quit Exit this program\n" ".read FILENAME Execute SQL in FILENAME\n" ".schema ?TABLE? Show the CREATE statements\n" " If TABLE specified, only show tables matching\n" @@ -1353,10 +1389,11 @@ ".show Show the current values for various settings\n" ".stats ON|OFF Turn stats on or off\n" ".tables ?TABLE? List names of tables\n" " If TABLE specified, only list tables matching\n" " LIKE pattern TABLE.\n" + ".trace FILE|off Output each SQL statement as it is run\n" ".width NUM1 NUM2 ... Set column widths for \"column\" mode\n" ; static char zTimerHelp[] = ".timer ON|OFF Turn the CPU timer measurement on or off\n" @@ -1440,10 +1477,56 @@ }else if( strcmp(zArg,"yes")==0 ){ val = 1; } return val; } + +/* +** Close an output file, assuming it is not stderr or stdout +*/ +static void output_file_close(FILE *f){ + if( f && f!=stdout && f!=stderr ) fclose(f); +} + +/* +** Try to open an output file. The names "stdout" and "stderr" are +** recognized and do the right thing. NULL is returned if the output +** filename is "off". +*/ +static FILE *output_file_open(const char *zFile){ + FILE *f; + if( strcmp(zFile,"stdout")==0 ){ + f = stdout; + }else if( strcmp(zFile, "stderr")==0 ){ + f = stderr; + }else if( strcmp(zFile, "off")==0 ){ + f = 0; + }else{ + f = fopen(zFile, "wb"); + if( f==0 ){ + fprintf(stderr, "Error: cannot open \"%s\"\n", zFile); + } + } + return f; +} + +/* +** A routine for handling output from sqlite4_trace(). +*/ +static void sql_trace_callback(void *pArg, const char *z){ + FILE *f = (FILE*)pArg; + if( f ) fprintf(f, "%s\n", z); +} + +/* +** A no-op routine that runs with the ".breakpoint" doc-command. This is +** a useful spot to set a debugger breakpoint. +*/ +static void test_breakpoint(void){ + static int nCall = 0; + nCall++; +} /* ** If an input line begins with "." then invoke this routine to ** process that line. ** @@ -1475,18 +1558,25 @@ if( zLine[i] ) zLine[i++] = 0; resolve_backslashes(azArg[nArg-1]); } } - if( nArg==0 ) return 0; /* no tokens, no error */ n = strlen30(azArg[0]); c = azArg[0][0]; if( c=='b' && n>=3 && strncmp(azArg[0], "bail", n)==0 && nArg>1 && nArg<3 ){ bail_on_error = booleanValue(azArg[1]); }else + + /* The undocumented ".breakpoint" command causes a call to the no-op + ** routine named test_breakpoint(). + */ + if( c=='b' && n>=3 && strncmp(azArg[0], "breakpoint", n)==0 ){ + test_breakpoint(); + }else + if( c=='d' && n>1 && strncmp(azArg[0], "databases", n)==0 && nArg==1 ){ struct callback_data data; char *zErrMsg = 0; open_db(p); memcpy(&data, p, sizeof(data)); @@ -1715,11 +1805,11 @@ if( z[j]=='"' ){ j++; if( z[j]==0 ) break; } z[k++] = z[j]; } z[k] = 0; } - sqlite4_bind_text(pStmt, i+1, azCol[i], -1, SQLITE4_STATIC); + sqlite4_bind_text(pStmt, i+1, azCol[i], -1, SQLITE4_STATIC, 0); } sqlite4_step(pStmt); rc = sqlite4_reset(pStmt); free(zLine); if( rc!=SQLITE4_OK ){ @@ -1814,26 +1904,12 @@ }else #endif if( c=='l' && strncmp(azArg[0], "log", n)==0 && nArg>=2 ){ const char *zFile = azArg[1]; - if( p->pLog && p->pLog!=stdout && p->pLog!=stderr ){ - fclose(p->pLog); - p->pLog = 0; - } - if( strcmp(zFile,"stdout")==0 ){ - p->pLog = stdout; - }else if( strcmp(zFile, "stderr")==0 ){ - p->pLog = stderr; - }else if( strcmp(zFile, "off")==0 ){ - p->pLog = 0; - }else{ - p->pLog = fopen(zFile, "w"); - if( p->pLog==0 ){ - fprintf(stderr, "Error: cannot open \"%s\"\n", zFile); - } - } + output_file_close(p->pLog); + p->pLog = output_file_open(zFile); }else if( c=='m' && strncmp(azArg[0], "mode", n)==0 && nArg==2 ){ int n2 = strlen30(azArg[1]); if( (n2==4 && strncmp(azArg[1],"line",n2)==0) @@ -1848,10 +1924,11 @@ p->mode = MODE_List; }else if( n2==4 && strncmp(azArg[1],"html",n2)==0 ){ p->mode = MODE_Html; }else if( n2==3 && strncmp(azArg[1],"tcl",n2)==0 ){ p->mode = MODE_Tcl; + sqlite4_snprintf(p->separator, sizeof(p->separator), " "); }else if( n2==3 && strncmp(azArg[1],"csv",n2)==0 ){ p->mode = MODE_Csv; sqlite4_snprintf(p->separator, sizeof(p->separator), ","); }else if( n2==4 && strncmp(azArg[1],"tabs",n2)==0 ){ p->mode = MODE_List; @@ -1882,27 +1959,47 @@ sqlite4_snprintf(p->nullvalue, sizeof(p->nullvalue), "%.*s", (int)ArraySize(p->nullvalue)-1, azArg[1]); }else if( c=='o' && strncmp(azArg[0], "output", n)==0 && nArg==2 ){ - if( p->out!=stdout ){ - fclose(p->out); + if( p->outfile[0]=='|' ){ + pclose(p->out); + }else{ + output_file_close(p->out); } - if( strcmp(azArg[1],"stdout")==0 ){ - p->out = stdout; - sqlite4_snprintf(p->outfile, sizeof(p->outfile), "stdout"); + p->outfile[0] = 0; + if( azArg[1][0]=='|' ){ + p->out = popen(&azArg[1][1], "w"); + if( p->out==0 ){ + fprintf(stderr,"Error: cannot open pipe \"%s\"\n", &azArg[1][1]); + p->out = stdout; + rc = 1; + }else{ + sqlite4_snprintf(p->outfile, sizeof(p->outfile), "%s", azArg[1]); + } }else{ - p->out = fopen(azArg[1], "wb"); + p->out = output_file_open(azArg[1]); if( p->out==0 ){ - fprintf(stderr,"Error: cannot write to \"%s\"\n", azArg[1]); + if( strcmp(azArg[1],"off")!=0 ){ + fprintf(stderr,"Error: cannot write to \"%s\"\n", azArg[1]); + } p->out = stdout; rc = 1; } else { - sqlite4_snprintf(p->outfile, sizeof(p->outfile), "%s", azArg[1]); + sqlite4_snprintf(p->outfile, sizeof(p->outfile), "%s", azArg[1]); } } }else + + if( c=='p' && n>=3 && strncmp(azArg[0], "print", n)==0 ){ + int i; + for(i=1; i1 ) fprintf(p->out, " "); + fprintf(p->out, "%s", azArg[i]); + } + fprintf(p->out, "\n"); + }else if( c=='p' && strncmp(azArg[0], "prompt", n)==0 && (nArg==2 || nArg==3)){ if( nArg >= 2) { strncpy(mainPrompt,azArg[1],(int)ArraySize(mainPrompt)-1); } @@ -1967,27 +2064,29 @@ rc = SQLITE4_OK; }else{ zShellStatic = azArg[1]; rc = sqlite4_exec(p->db, "SELECT sql FROM " - " (SELECT sql sql, type type, tbl_name tbl_name, name name" + " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" " FROM sqlite_master UNION ALL" - " SELECT sql, type, tbl_name, name FROM sqlite_temp_master) " + " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " "WHERE lower(tbl_name) LIKE shellstatic()" " AND type!='meta' AND sql NOTNULL " - "ORDER BY substr(type,2,1), name", + "ORDER BY substr(type,2,1)," + " CASE type WHEN 'view' THEN x ELSE name END", callback, &data, &zErrMsg); zShellStatic = 0; } }else{ rc = sqlite4_exec(p->db, "SELECT sql FROM " - " (SELECT sql sql, type type, tbl_name tbl_name, name name" + " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" " FROM sqlite_master UNION ALL" - " SELECT sql, type, tbl_name, name FROM sqlite_temp_master) " + " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%'" - "ORDER BY substr(type,2,1), name", + "ORDER BY substr(type,2,1)," + " CASE type WHEN 'view' THEN x ELSE name END", callback, &data, &zErrMsg ); } if( zErrMsg ){ fprintf(stderr,"Error: %s\n", zErrMsg); @@ -2071,13 +2170,13 @@ sqlite4_free(0, zSql); if( rc ) return rc; nRow = nAlloc = 0; azResult = 0; if( nArg>1 ){ - sqlite4_bind_text(pStmt, 1, azArg[1], -1, SQLITE4_TRANSIENT); + sqlite4_bind_text(pStmt, 1, azArg[1], -1, SQLITE4_TRANSIENT, 0); }else{ - sqlite4_bind_text(pStmt, 1, "%", -1, SQLITE4_STATIC); + sqlite4_bind_text(pStmt, 1, "%", -1, SQLITE4_STATIC, 0); } while( sqlite4_step(pStmt)==SQLITE4_ROW ){ if( nRow>=nAlloc ){ char **azNew; int n = nAlloc*2 + 10; @@ -2204,15 +2303,35 @@ if( HAS_TIMER && c=='t' && n>=5 && strncmp(azArg[0], "timer", n)==0 && nArg==2 ){ enableTimer = booleanValue(azArg[1]); }else + + if( c=='t' && strncmp(azArg[0], "trace", n)==0 && nArg>1 ){ + open_db(p); + output_file_close(p->traceOut); + p->traceOut = output_file_open(azArg[1]); +#if !defined(SQLITE4_OMIT_TRACE) && !defined(SQLITE4_OMIT_FLOATING_POINT) + if( p->traceOut==0 ){ + sqlite4_trace(p->db, 0, 0); + }else{ + sqlite4_trace(p->db, sql_trace_callback, p->traceOut); + } +#endif + }else if( c=='v' && strncmp(azArg[0], "version", n)==0 ){ printf("SQLite %s %s\n" /*extra-version-info*/, sqlite4_libversion(), sqlite4_sourceid()); }else + +#if defined(SQLITE4_DEBUG) && defined(SQLITE4_ENABLE_WHERETRACE) + if( c=='w' && strncmp(azArg[0], "wheretrace", n)==0 ){ + extern int sqlite4WhereTrace; + sqlite4WhereTrace = atoi(azArg[1]); + }else +#endif if( c=='w' && strncmp(azArg[0], "width", n)==0 && nArg>1 ){ int j; assert( nArg<=ArraySize(azArg) ); for(j=1; jcolWidth); j++){ @@ -2317,11 +2436,13 @@ while( errCnt==0 || !bail_on_error || (in==0 && stdin_is_interactive) ){ fflush(p->out); free(zLine); zLine = one_input_line(zSql, in); if( zLine==0 ){ - break; /* We have reached EOF */ + /* End of input */ + if( stdin_is_interactive ) printf("\n"); + break; } if( seenInterrupt ){ if( in!=0 ) break; seenInterrupt = 0; } @@ -2404,42 +2525,43 @@ return errCnt; } /* ** Return a pathname which is the user's home directory. A -** 0 return indicates an error of some kind. Space to hold the -** resulting string is obtained from malloc(). The calling -** function should free the result. +** 0 return indicates an error of some kind. */ static char *find_home_dir(void){ - char *home_dir = NULL; + static char *home_dir = NULL; + if( home_dir ) return home_dir; -#if !defined(_WIN32) && !defined(WIN32) && !defined(__OS2__) && !defined(_WIN32_WCE) && !defined(__RTP__) && !defined(_WRS_KERNEL) - struct passwd *pwent; - uid_t uid = getuid(); - if( (pwent=getpwuid(uid)) != NULL) { - home_dir = pwent->pw_dir; +#if !defined(_WIN32) && !defined(WIN32) && !defined(__RTP__) && !defined(_WRS_KERNEL) + { + struct passwd *pwent; + uid_t uid = getuid(); + if( (pwent=getpwuid(uid)) != NULL) { + home_dir = pwent->pw_dir; + } } #endif #if defined(_WIN32_WCE) /* Windows CE (arm-wince-mingw32ce-gcc) does not provide getenv() */ - home_dir = strdup("/"); + home_dir = "/"; #else -#if defined(_WIN32) || defined(WIN32) || defined(__OS2__) +#if defined(_WIN32) || defined(WIN32) if (!home_dir) { home_dir = getenv("USERPROFILE"); } #endif if (!home_dir) { home_dir = getenv("HOME"); } -#if defined(_WIN32) || defined(WIN32) || defined(__OS2__) +#if defined(_WIN32) || defined(WIN32) if (!home_dir) { char *zDrive, *zPath; int n; zDrive = getenv("HOMEDRIVE"); zPath = getenv("HOMEPATH"); @@ -2515,29 +2637,30 @@ /* ** Show available command line options */ static const char zOptions[] = - " -help show this message\n" - " -init filename read/process named file\n" - " -echo print commands before execution\n" - " -[no]header turn headers on or off\n" " -bail stop after hitting an error\n" - " -interactive force interactive I/O\n" " -batch force batch I/O\n" " -column set output mode to 'column'\n" + " -cmd COMMAND run \"COMMAND\" before reading stdin\n" " -csv set output mode to 'csv'\n" + " -echo print commands before execution\n" + " -init FILENAME read/process named file\n" + " -[no]header turn headers on or off\n" +#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) + " -heap SIZE Size of heap for memsys3 or memsys5\n" +#endif + " -help show this message\n" " -html set output mode to HTML\n" + " -interactive force interactive I/O\n" " -line set output mode to 'line'\n" " -list set output mode to 'list'\n" - " -separator 'x' set output field separator (|)\n" + " -nullvalue TEXT set text string for NULL values. Default ''\n" + " -separator SEP set output field separator. Default: '|'\n" " -stats print memory stats before each finalize\n" - " -nullvalue 'text' set text string for NULL values\n" " -version show SQLite version\n" -#ifdef SQLITE4_ENABLE_MULTIPLEX - " -multiplex enable the multiplexor VFS\n" -#endif ; static void usage(int showDetail){ fprintf(stderr, "Usage: %s [OPTIONS] FILENAME [SQL]\n" "FILENAME is the name of an SQLite database. A new database is created\n" @@ -2561,10 +2684,23 @@ sqlite4_env_config(0, SQLITE4_ENVCONFIG_LOG, shellLog, data); sqlite4_snprintf(mainPrompt,sizeof(mainPrompt), "sqlite> "); sqlite4_snprintf(continuePrompt,sizeof(continuePrompt)," ...> "); sqlite4_env_config(0, SQLITE4_ENVCONFIG_SINGLETHREAD); } + +/* +** Get the argument to an --option. Throw an error and die if no argument +** is available. +*/ +static char *cmdline_option_value(int argc, char **argv, int i){ + if( i==argc ){ + fprintf(stderr, "%s: Error: missing argument to %s\n", + argv[0], argv[argc-1]); + exit(1); + } + return argv[i]; +} int main(int argc, char **argv){ char *zErrMsg = 0; struct callback_data data; const char *zInitFile = 0; @@ -2591,78 +2727,67 @@ /* Do an initial pass through the command-line argument to locate ** the name of the database file, the name of the initialization file, ** the size of the alternative malloc heap, ** and the first command to execute. */ - for(i=1; i0x7fff0000 ) szHeap = 0x7fff0000; sqlite4_config(0,SQLITE4_CONFIG_HEAP, malloc((int)szHeap),(int)szHeap,64); #endif -#ifdef SQLITE4_ENABLE_MULTIPLEX - }else if( strcmp(argv[i],"-multiplex")==0 ){ - extern int sqlite4_multiple_initialize(const char*,int); - sqlite4_multiplex_initialize(0, 1); -#endif - } - } - if( i=argc){ - fprintf(stderr,"%s: Error: missing argument for option: %s\n", Argv0, z); - fprintf(stderr,"Use -help for a list of options.\n"); - return 1; - } sqlite4_snprintf(data.separator, sizeof(data.separator), - "%.*s",(int)sizeof(data.separator)-1,argv[i]); + "%s",cmdline_option_value(argc,argv,++i)); }else if( strcmp(z,"-nullvalue")==0 ){ - i++; - if(i>=argc){ - fprintf(stderr,"%s: Error: missing argument for option: %s\n", Argv0, z); - fprintf(stderr,"Use -help for a list of options.\n"); - return 1; - } sqlite4_snprintf(data.nullvalue, sizeof(data.nullvalue), - "%.*s",(int)sizeof(data.nullvalue)-1,argv[i]); + "%s",cmdline_option_value(argc,argv,++i)); }else if( strcmp(z,"-header")==0 ){ data.showHeader = 1; }else if( strcmp(z,"-noheader")==0 ){ data.showHeader = 0; }else if( strcmp(z,"-echo")==0 ){ @@ -2742,10 +2856,27 @@ }else if( strcmp(z,"-multiplex")==0 ){ i++; #endif }else if( strcmp(z,"-help")==0 || strcmp(z, "--help")==0 ){ usage(1); + }else if( strcmp(z,"-cmd")==0 ){ + if( i==argc-1 ) break; + z = cmdline_option_value(argc,argv,++i); + if( z[0]=='.' ){ + rc = do_meta_command(z, &data); + if( rc && bail_on_error ) return rc; + }else{ + open_db(&data); + rc = shell_exec(data.db, z, shell_callback, &data, &zErrMsg); + if( zErrMsg!=0 ){ + fprintf(stderr,"Error: %s\n", zErrMsg); + if( bail_on_error ) return rc!=0 ? rc : 1; + }else if( rc!=0 ){ + fprintf(stderr,"Error: unable to process SQL \"%s\"\n", z); + if( bail_on_error ) return rc; + } + } }else{ fprintf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); fprintf(stderr,"Use -help for a list of options.\n"); return 1; } @@ -2794,11 +2925,10 @@ if( zHistory ){ stifle_history(100); write_history(zHistory); free(zHistory); } - free(zHome); }else{ rc = process_input(&data, stdin); } } set_table_name(&data, 0); Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -123,11 +123,10 @@ #define SQLITE4_ENVCONFIG_LOG 11 /* xLog, pArg */ #define SQLITE4_ENVCONFIG_KVSTORE_PUSH 12 /* name, factory */ #define SQLITE4_ENVCONFIG_KVSTORE_POP 13 /* name */ #define SQLITE4_ENVCONFIG_KVSTORE_GET 14 /* name, *factor */ - /* ** CAPIREF: Compile-Time Library Version Numbers ** ** ^(The [SQLITE4_VERSION] C preprocessor macro in the sqlite4.h header ** evaluates to a string literal that is the SQLite version in the @@ -229,11 +228,12 @@ ** object is used by two or more threads at the same time. This ** corresponds to [SQLITE4_ENVCONFIG_MULTITHREAD]. ** ** ^The sqlite4_threadsafe(E) function returns two if the same ** [database connection] can be used at the same time from two or more -** separate threads. This setting corresponds to [SQLITE4_ENVCONFIG_SERIALIZED]. +** separate threads. This setting corresponds to +** [SQLITE4_ENVCONFIG_SERIALIZED]. ** ** Note that SQLite4 is always threadsafe in this sense: Two or more ** objects each associated with different [sqlite4_env] objects can ** always be used at the same time in separate threads. */ @@ -408,20 +408,19 @@ ** ** See also: [SQLITE4_IOERR_READ | extended result codes], ** [sqlite4_vtab_on_conflict()] [SQLITE4_ROLLBACK | result codes]. */ #define SQLITE4_OK 0 /* Successful result */ -/* beginning-of-error-codes */ #define SQLITE4_ERROR 1 /* SQL error or missing database */ #define SQLITE4_INTERNAL 2 /* Internal logic error in SQLite */ #define SQLITE4_PERM 3 /* Access permission denied */ #define SQLITE4_ABORT 4 /* Callback routine requested an abort */ #define SQLITE4_BUSY 5 /* The database file is locked */ #define SQLITE4_LOCKED 6 /* A table in the database is locked */ #define SQLITE4_NOMEM 7 /* A malloc() failed */ #define SQLITE4_READONLY 8 /* Attempt to write a readonly database */ -#define SQLITE4_INTERRUPT 9 /* Operation terminated by sqlite4_interrupt()*/ +#define SQLITE4_INTERRUPT 9 /* Stopped terminated by sqlite4_interrupt()*/ #define SQLITE4_IOERR 10 /* Some kind of disk I/O error occurred */ #define SQLITE4_CORRUPT 11 /* The database disk image is malformed */ #define SQLITE4_NOTFOUND 12 /* Unknown opcode in sqlite4_file_control() */ #define SQLITE4_FULL 13 /* Insertion failed because database is full */ #define SQLITE4_CANTOPEN 14 /* Unable to open the database file */ @@ -433,16 +432,15 @@ #define SQLITE4_MISMATCH 20 /* Data type mismatch */ #define SQLITE4_MISUSE 21 /* Library used incorrectly */ #define SQLITE4_NOLFS 22 /* Uses OS features not supported on host */ #define SQLITE4_AUTH 23 /* Authorization denied */ #define SQLITE4_FORMAT 24 /* Auxiliary database format error */ -#define SQLITE4_RANGE 25 /* 2nd parameter to sqlite4_bind out of range */ +#define SQLITE4_RANGE 25 /* 2nd param to sqlite4_bind out of range */ #define SQLITE4_NOTADB 26 /* File opened that is not a database file */ #define SQLITE4_ROW 100 /* sqlite4_step() has another row ready */ #define SQLITE4_DONE 101 /* sqlite4_step() has finished executing */ #define SQLITE4_INEXACT 102 /* xSeek method of storage finds nearby ans */ -/* end-of-error-codes */ /* ** CAPIREF: Extended Result Codes ** KEYWORDS: {extended error code} {extended error codes} ** KEYWORDS: {extended result code} {extended result codes} @@ -523,11 +521,11 @@ struct sqlite4_mutex_methods *pMutexMethods; /* Subclasses will typically add additional fields */ }; /* -** CAPIREF: Initialize The SQLite Library +** CAPIREF: Initialize An SQLite Environment ** ** ^The sqlite4_initialize(A) routine initializes an sqlite4_env object A. ** ^The sqlite4_shutdown(A) routine ** deallocates any resources that were allocated by sqlite4_initialize(A). ** A parameter value of NULL means to use the default environment. @@ -744,23 +742,17 @@ /* ** CAPIREF: Last Insert Rowid ** -** ^Each entry in an SQLite table has a unique 64-bit signed -** integer key called the [ROWID | "rowid"]. ^The rowid is always available -** as an undeclared column named ROWID, OID, or _ROWID_ as long as those -** names are not also used by explicitly declared columns. ^If -** the table has a column of type [INTEGER PRIMARY KEY] then that column -** is another alias for the rowid. -** -** ^This routine returns the [rowid] of the most recent -** successful [INSERT] into the database from the [database connection] -** in the first argument. ^As of SQLite version 3.7.7, this routines -** records the last insert rowid of both ordinary tables and [virtual tables]. -** ^If no successful [INSERT]s -** have ever occurred on that database connection, zero is returned. +** ^The sqlite4_last_insert_rowid(D) routine returns the primary key value +** of the the most recent successful [INSERT] from [database connection] D +** into a table where the primary key of the inserted row is a single +** integer. +** ^If there have been no successful [INSERT]s of rows with a single integer +** PRIMARY KEY value on database connection D, then this routine returns +** zero. ** ** ^(If an [INSERT] occurs within a trigger or within a [virtual table] ** method, then this routine will return the [rowid] of the inserted ** row as long as the trigger or virtual table method is running. ** But once the trigger or virtual table method ends, the value returned @@ -1839,17 +1831,20 @@ ** index is out of range. ^[SQLITE4_NOMEM] is returned if malloc() fails. ** ** See also: [sqlite4_bind_parameter_count()], ** [sqlite4_bind_parameter_name()], and [sqlite4_bind_parameter_index()]. */ -int sqlite4_bind_blob(sqlite4_stmt*, int, const void*, int n, void(*)(void*)); +int sqlite4_bind_blob(sqlite4_stmt*, int, const void*, int n, + void(*)(void*,void*),void*); int sqlite4_bind_double(sqlite4_stmt*, int, double); int sqlite4_bind_int(sqlite4_stmt*, int, int); int sqlite4_bind_int64(sqlite4_stmt*, int, sqlite4_int64); int sqlite4_bind_null(sqlite4_stmt*, int); -int sqlite4_bind_text(sqlite4_stmt*, int, const char*, int n, void(*)(void*)); -int sqlite4_bind_text16(sqlite4_stmt*, int, const void*, int, void(*)(void*)); +int sqlite4_bind_text(sqlite4_stmt*, int, const char*, int n, + void(*)(void*,void*),void*); +int sqlite4_bind_text16(sqlite4_stmt*, int, const void*, int, + void(*)(void*,void*),void*); int sqlite4_bind_value(sqlite4_stmt*, int, const sqlite4_value*); int sqlite4_bind_zeroblob(sqlite4_stmt*, int, int n); /* ** CAPIREF: Number Of SQL Parameters @@ -2670,11 +2665,12 @@ ** ** These routines must be called from the same thread in which ** the SQL function is running. */ void *sqlite4_get_auxdata(sqlite4_context*, int N); -void sqlite4_set_auxdata(sqlite4_context*, int N, void*, void (*)(void*)); +void sqlite4_set_auxdata(sqlite4_context*, int N, void*, + void (*)(void*,void*),void*); /* ** CAPIREF: Constants Defining Special Destructor Behavior ** @@ -2687,12 +2683,12 @@ ** the content before returning. ** ** The typedef is necessary to work around problems in certain ** C++ compilers. See ticket #2191. */ -typedef void (*sqlite4_destructor_type)(void*); -void sqlite4_dynamic(void*); +typedef void (*sqlite4_destructor_type)(void*,void*); +void sqlite4_dynamic(void*,void*); #define SQLITE4_STATIC ((sqlite4_destructor_type)0) #define SQLITE4_TRANSIENT ((sqlite4_destructor_type)-1) #define SQLITE4_DYNAMIC (sqlite4_dynamic) @@ -2804,24 +2800,29 @@ ** ** If these routines are called from within the different thread ** than the one containing the application-defined function that received ** the [sqlite4_context] pointer, the results are undefined. */ -void sqlite4_result_blob(sqlite4_context*, const void*, int, void(*)(void*)); +void sqlite4_result_blob(sqlite4_context*, const void*, int, + void(*)(void*,void*),void*); void sqlite4_result_double(sqlite4_context*, double); void sqlite4_result_error(sqlite4_context*, const char*, int); void sqlite4_result_error16(sqlite4_context*, const void*, int); void sqlite4_result_error_toobig(sqlite4_context*); void sqlite4_result_error_nomem(sqlite4_context*); void sqlite4_result_error_code(sqlite4_context*, int); void sqlite4_result_int(sqlite4_context*, int); void sqlite4_result_int64(sqlite4_context*, sqlite4_int64); void sqlite4_result_null(sqlite4_context*); -void sqlite4_result_text(sqlite4_context*, const char*, int, void(*)(void*)); -void sqlite4_result_text16(sqlite4_context*, const void*, int, void(*)(void*)); -void sqlite4_result_text16le(sqlite4_context*, const void*, int,void(*)(void*)); -void sqlite4_result_text16be(sqlite4_context*, const void*, int,void(*)(void*)); +void sqlite4_result_text(sqlite4_context*, const char*, int, + void(*)(void*,void*),void*); +void sqlite4_result_text16(sqlite4_context*, const void*, int, + void(*)(void*,void*),void*); +void sqlite4_result_text16le(sqlite4_context*, const void*, int, + void(*)(void*,void*),void*); +void sqlite4_result_text16be(sqlite4_context*, const void*, int, + void(*)(void*,void*),void*); void sqlite4_result_value(sqlite4_context*, sqlite4_value*); void sqlite4_result_zeroblob(sqlite4_context*, int n); /* ** CAPIREF: Define New Collating Sequences @@ -4124,10 +4125,11 @@ ** ^The [sqlite4_strnicmp()] API allows applications and extensions to ** compare the contents of two buffers containing UTF-8 strings in a ** case-independent fashion, using the same definition of case independence ** that SQLite uses internally when comparing identifiers. */ +int sqlite4_stricmp(const char*, const char*); int sqlite4_strnicmp(const char *, const char *, int); /* ** CAPIREF: Error Logging Interface ** @@ -4316,21 +4318,35 @@ typedef struct sqlite4_kv_methods sqlite4_kv_methods; /* ** CAPI4REF: Key-value storage engine open flags ** -** Allowed values to the flags parameter of an sqlite4_kvstore object -** factory. +** Allowed values to the flags parameter of an [sqlite4_kvfactory] object. ** -** The flags parameter to the sqlite4_kvstore factory (the fourth parameter) +** The flags parameter to the sqlite4_kvfactory fuction (the fourth parameter) ** is an OR-ed combination of these values and the ** [SQLITE4_OPEN_READONLY | SQLITE4_OPEN_xxxxx] flags that appear as ** arguments to [sqlite4_open()]. */ #define SQLITE4_KVOPEN_TEMPORARY 0x00010000 /* A temporary database */ #define SQLITE4_KVOPEN_NO_TRANSACTIONS 0x00020000 /* No transactions needed */ +/* +** CAPIREF: Key-value storage object factory +** +** New key/value storage engines can be added to SQLite4 at run-time. +** In order to create a new KV storage engine, the application must +** supply a "factory" function that creates an instance of the +** sqlite4_kvstore object. This is typedef defines the signature +** of that factory function. +*/ +typedef int (*sqlite4_kvfactory)( + sqlite4_env *pEnv, /* The environment to use */ + sqlite4_kvstore **ppKVStore, /* OUT: New KV store returned here */ + const char *zFilename, /* Name of database file to open */ + unsigned flags /* Bit flags */ +); /* ** CAPI4REF: Representation Of Numbers ** ** Every number in SQLite is represented in memory by an instance of @@ -4338,11 +4354,11 @@ */ typedef struct sqlite4_num sqlite4_num; struct sqlite4_num { unsigned char sign; /* Sign of the overall value */ unsigned char approx; /* True if the value is approximate */ - unsigned short e; /* The exponent. */ + short e; /* The exponent. */ sqlite4_uint64 m; /* The significant */ }; /* ** CAPI4REF: Operations On SQLite Number Objects @@ -4366,10 +4382,156 @@ /* ** CAPI4REF: Flags For Text-To-Numeric Conversion */ #define SQLITE4_PREFIX_ONLY 0x10 #define SQLITE4_IGNORE_WHITESPACE 0x20 + +typedef struct sqlite4_tokenizer sqlite4_tokenizer; + +/* +** CAPI4REF: Register an FTS tokenizer implementation +** +** xTokenize: +** This function does the actual tokenization of an input string. For +** each token in the input, the callback function is invoked once. The +** arguments passed to the callback by the tokenizer must be as follows: +** +** xCallback(ctx, iWeight, zToken, nToken, iSrc, nSrc); +** +** ctx - Copy of the first argument passed to xTokenize. +** iOff - Offset of token in document. +** iWeight - Weight assigned to the token by the tokenizer. Larger values +** indicate more important tokens. +** zToken - Pointer to buffer containing token text. +** nToken - Number of bytes in zToken. +** iSrc - Byte offset in source document of the start of this token. +** nSrc - Bytes of text in source document tokenized to this token. +*/ +int sqlite4_create_tokenizer( + sqlite4 *db, + const char *zName, + void *pCtx, + int (*xCreate)(void*, const char**, int, sqlite4_tokenizer**), + int (*xTokenize)(void*, sqlite4_tokenizer*, + const char*, int, + int(*x)(void *ctx, int iWeight, int iOff, + const char *zToken, int nToken, int iSrc, int nSrc) + ), + int (*xDestroy)(sqlite4_tokenizer *) +); + +/* +** CAPI4REF: Register a matchinfo function. +*/ +int sqlite4_create_mi_function( + sqlite4 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xFunc)(sqlite4_context*,int,sqlite4_value **), + void (*xDestroy)(void *) +); + +/* +** CAPIREF: Matchinfo APIs. +** +** Special functions that may be called from within matchinfo UDFs. All +** return an SQLite error code - SQLITE4_OK if successful, or some other +** error code otherwise. +** +** sqlite4_mi_column_count(): +** Set *pn to the number of columns in the queried table. +** +** sqlite4_mi_phrase_count(): +** Set *pn to the number of phrases in the query. +** +** sqlite4_mi_stream_count(): +** Set *pn to the number of streams in the FTS index. +** +** sqlite4_mi_phrase_token_count(): +** Set *pn to the number of tokens in phrase iP of the query. +** +** sqlite4_mi_size(): +** Set *pn to the number of tokens belonging to stream iS in the value +** stored in column iC of the current row. +** +** Either or both of iS and iC may be negative. If iC is negative, then the +** output value is the total number of tokens for the specified stream (or +** streams) across all table columns. Similarly, if iS is negative, the +** output value is the total number of tokens in the specified column or +** columns, regardless of stream. +** +** sqlite4_mi_total_size(): +** Similar to sqlite4_mi_size(), except the output parameter is set to +** the total number of tokens belonging to the specified column(s) +** and stream(s) in all rows of the table, not just the current row. +** +** sqlite4_mi_total_rows(): +** Set *pn to the total number of rows in the indexed table. +** +** sqlite4_mi_row_count(): +** Set the output parameter to the total number of rows in the table that +** contain at least one instance of the phrase identified by parameter +** iP in the column(s) and stream(s) identified by parameters iC and iS. +** +** sqlite4_mi_match_count(): +** Set the output parameter to the total number of occurences of phrase +** iP in the current row that belong to the column(s) and stream(s) +** identified by parameters iC and iS. +** +** Parameter iP may also be negative. In this case, the output value is +** set to the total number of occurrences of all query phrases in the +** current row, subject to the constraints imposed by iC and iS. +** +** sqlite4_mi_match_detail(): +** This function is used to access the details of the iMatch'th match +** (of any phrase) in the current row. Matches are sorted in order of +** occurrence. If parameter iMatch is equal to or greater than the number +** of matches in the current row, SQLITE_NOTFOUND is returned. Otherwise, +** unless an error occurs, SQLITE4_OK is returned and the *piOff, *piC, *piS, +** and *piP output parameters are set to the token offset, column number, +** stream number and phrase number respectively. +** +** It is anticipated that this function be used to iterate through matches +** in order of occurrence. It is optimized so that it is fastest when +** called with the iMatch parameter set to 0, P or P+1, where P is the +** iMatch value passed to the previous call. +** +** sqlite4_mi_column_value(): +** Set *ppVal to point to an sqlite4_value object containing the value +** read from column iCol of the current row. This object is valid until +** the function callback returns. +** +** sqlite4_mi_tokenize(): +** Use the tokenizer associated with the FTS index to tokenize the +** document in buffer zDoc (size nDoc bytes). Invoke callback function +** x() once for each token, in the same way as the xTokenizer() method +** of a tokenizer implementation. +*/ +int sqlite4_mi_column_count(sqlite4_context *, int *pn); +int sqlite4_mi_phrase_count(sqlite4_context *, int *pn); +int sqlite4_mi_stream_count(sqlite4_context *, int *pn); +int sqlite4_mi_phrase_token_count(sqlite4_context *, int iP, int *pn); + +int sqlite4_mi_total_size(sqlite4_context *, int iC, int iS, int *pn); +int sqlite4_mi_total_rows(sqlite4_context *, int *pn); + +int sqlite4_mi_row_count(sqlite4_context *, int iC, int iS, int iP, int *pn); + +int sqlite4_mi_size(sqlite4_context *, int iC, int iS, int *pn); +int sqlite4_mi_match_count(sqlite4_context *, int iC, int iS, int iP, int *pn); +int sqlite4_mi_match_detail( + sqlite4_context *, int iMatch, int *piOff, int *piC, int *piS, int *piP +); +int sqlite4_mi_column_value(sqlite4_context *, int iC, sqlite4_value **ppVal); + +int sqlite4_mi_tokenize(sqlite4_context *, const char *zDoc, int nDoc, + void *p, int(*x)(void *, int, int, const char *, int, int, int) +); + + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. */ Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -17,10 +17,11 @@ #define SQLITE4_OMIT_ANALYZE 1 #define SQLITE4_OMIT_PROGRESS_CALLBACK 1 #define SQLITE4_OMIT_VIRTUALTABLE 1 #define SQLITE4_OMIT_XFER_OPT 1 +#define SQLITE4_OMIT_LOCALTIME 1 /* ** These #defines should enable >2GB file support on POSIX if the ** underlying operating system supports it. If the OS lacks ** large file support, or if the OS is windows, these should be no-ops. @@ -461,11 +462,11 @@ ** These macros are designed to work correctly on both 32-bit and 64-bit ** compilers. */ #define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) #define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) -#define LARGEST_UINT64 (0xffffffff|(((i64)0xffffffff)<<32)) +#define LARGEST_UINT64 (0xffffffff|(((u64)0xffffffff)<<32)) /* ** Round up a number to the next larger multiple of 8. This is used ** to force 8-byte alignment on 64-bit architectures. */ @@ -549,10 +550,11 @@ typedef struct AggInfoFunc AggInfoFunc; typedef struct AuthContext AuthContext; typedef struct AutoincInfo AutoincInfo; typedef struct CollSeq CollSeq; typedef struct Column Column; +typedef struct CreateIndex CreateIndex; typedef struct Db Db; typedef struct Schema Schema; typedef struct Expr Expr; typedef struct ExprList ExprList; typedef struct ExprListItem ExprListItem; @@ -559,10 +561,14 @@ typedef struct ExprSpan ExprSpan; typedef struct FKey FKey; typedef struct FuncDestructor FuncDestructor; typedef struct FuncDef FuncDef; typedef struct FuncDefTable FuncDefTable; +typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct Fts5Index Fts5Index; +typedef struct Fts5Info Fts5Info; +typedef struct Fts5Cursor Fts5Cursor; typedef struct IdList IdList; typedef struct IdListItem IdListItem; typedef struct Index Index; typedef struct IndexSample IndexSample; typedef struct KeyClass KeyClass; @@ -632,10 +638,11 @@ void (*xStep)(sqlite4_context*,int,sqlite4_value**); /* Aggregate step */ void (*xFinalize)(sqlite4_context*); /* Aggregate finalizer */ char *zName; /* SQL name of the function. */ FuncDef *pNextName; /* Next function with a different name */ FuncDestructor *pDestructor; /* Reference counted destructor function */ + u8 bMatchinfo; /* True for matchinfo function */ }; /* ** A table of SQL functions. ** @@ -869,10 +876,14 @@ sqlite4 *pUnlockConnection; /* Connection to watch for unlock */ void *pUnlockArg; /* Argument to xUnlockNotify */ void (*xUnlockNotify)(void **, int); /* Unlock notify callback */ sqlite4 *pNextBlocked; /* Next in list of all blocked connections */ #endif + +#ifndef SQLITE_OMIT_FTS5 + Fts5Tokenizer *pTokenizer; /* First in list of tokenizers */ +#endif }; /* ** A macro to discover the encoding of a database. */ @@ -880,22 +891,18 @@ /* ** Possible values for the sqlite4.flags. */ #define SQLITE4_VdbeTrace 0x00000100 /* True to trace VDBE execution */ -#define SQLITE4_InternChanges 0x00000200 /* Uncommitted Hash table changes */ -#define SQLITE4_CountRows 0x00001000 /* Count rows changed by INSERT, */ - /* DELETE, or UPDATE and return */ - /* the count using a callback. */ -#define SQLITE4_SqlTrace 0x00004000 /* Debug print SQL as it executes */ -#define SQLITE4_VdbeListing 0x00008000 /* Debug listings of VDBE programs */ -#define SQLITE4_WriteSchema 0x00010000 /* OK to update SQLITE4_MASTER */ -#define SQLITE4_KvTrace 0x00020000 /* Trace Key/value storage calls */ -#define SQLITE4_IgnoreChecks 0x00040000 /* Do not enforce check constraints */ -#define SQLITE4_ReadUncommitted 0x0080000 /* For shared-cache mode */ -#define SQLITE4_LegacyFileFmt 0x00100000 /* Create new databases in format 1 */ -#define SQLITE4_RecoveryMode 0x00800000 /* Ignore schema errors */ +#define SQLITE4_SqlTrace 0x00000200 /* Debug print SQL as it executes */ +#define SQLITE4_VdbeListing 0x00000400 /* Debug listings of VDBE programs */ +#define SQLITE4_KvTrace 0x00000800 /* Trace Key/value storage calls */ +#define SQLITE4_VdbeAddopTrace 0x00001000 /* Trace sqlite4VdbeAddOp() calls */ +#define SQLITE4_InternChanges 0x00010000 /* Uncommitted Hash table changes */ +#define SQLITE4_WriteSchema 0x00020000 /* OK to update SQLITE4_MASTER */ +#define SQLITE4_IgnoreChecks 0x00040000 /* Dont enforce check constraints */ +#define SQLITE4_RecoveryMode 0x00080000 /* Ignore schema errors */ #define SQLITE4_ReverseOrder 0x01000000 /* Reverse unordered SELECTs */ #define SQLITE4_RecTriggers 0x02000000 /* Enable recursive triggers */ #define SQLITE4_ForeignKeys 0x04000000 /* Enforce foreign key constraints */ #define SQLITE4_AutoIndex 0x08000000 /* Enable automatic indexes */ #define SQLITE4_PreferBuiltin 0x10000000 /* Preference to built-in funcs */ @@ -1172,36 +1179,10 @@ }; /* ** Each SQL table is represented in memory by an instance of the ** following structure. -** -** Table.zName is the name of the table. The case of the original -** CREATE TABLE statement is stored, but case is not significant for -** comparisons. -** -** Table.nCol is the number of columns in this table. Table.aCol is a -** pointer to an array of Column structures, one for each column. -** -** If the table has an INTEGER PRIMARY KEY, then Table.iPKey is the index of -** the column that is that key. Otherwise Table.iPKey is negative. Note -** that the datatype of the PRIMARY KEY must be INTEGER for this field to -** be set. An INTEGER PRIMARY KEY is used as the rowid for each row of -** the table. If a table has no INTEGER PRIMARY KEY, then a random rowid -** is generated for each row of the table. TF_HasPrimaryKey is set if -** the table has any PRIMARY KEY, INTEGER or otherwise. -** -** Table.tnum is the page number for the root BTree page of the table in the -** database file. If Table.iDb is the index of the database table backend -** in sqlite.aDb[]. 0 is for the main database and 1 is for the file that -** holds temporary tables and indices. If TF_Ephemeral is set -** then the table is stored in a file that is automatically deleted -** when the VDBE cursor to the table is closed. In this case Table.tnum -** refers VDBE cursor number that holds the table open, not to the root -** page number. Transient tables are used to hold the results of a -** sub-query that appears instead of a real table name in the FROM clause -** of a SELECT statement. */ struct Table { char *zName; /* Name of the table or view */ int nCol; /* Number of columns in this table */ Column *aCol; /* Information about each column */ @@ -1413,11 +1394,11 @@ tRowcnt *aiRowEst; /* Result of ANALYZE: Est. rows selected by each column */ Table *pTable; /* The SQL table being indexed */ int tnum; /* Page containing root of this index in database file */ u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ u8 eIndexType; /* SQLITE4_INDEX_USER, UNIQUE or PRIMARYKEY */ - u8 bUnordered; /* Use this index for == or IN queries only */ + u8 fIndex; /* One or more of the IDX_* flags below */ char *zColAff; /* String defining the affinity of each column */ Index *pNext; /* The next index associated with the same table */ Schema *pSchema; /* Schema containing this index */ u8 *aSortOrder; /* Array of size Index.nColumn. True==DESC, False==ASC */ char **azColl; /* Array of collation sequence names for index */ @@ -1424,17 +1405,23 @@ #ifdef SQLITE4_ENABLE_STAT3 int nSample; /* Number of elements in aSample[] */ tRowcnt avgEq; /* Average nEq value for key values not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ #endif + Fts5Index *pFts; /* Fts5 data (or NULL if this is not an fts index) */ }; /* Index.eIndexType must be set to one of the following. */ #define SQLITE4_INDEX_USER 0 /* Index created by CREATE INDEX statement */ #define SQLITE4_INDEX_UNIQUE 1 /* Index created by UNIQUE constraint */ #define SQLITE4_INDEX_PRIMARYKEY 2 /* Index is the tables PRIMARY KEY */ -#define SQLITE4_INDEX_TEMP 3 /* Index is an automatic index */ +#define SQLITE4_INDEX_FTS5 3 /* Index is an FTS5 index */ +#define SQLITE4_INDEX_TEMP 4 /* Index is an automatic index */ + +/* Allowed values for Index.fIndex */ +#define IDX_IntPK 0x01 /* An INTEGER PRIMARY KEY index */ +#define IDX_Unordered 0x02 /* Implemented as a hashing index */ /* ** Each sample stored in the sqlite_stat3 table is represented in memory ** using a structure of this type. See documentation at the top of the ** analyze.c source file for additional information. @@ -1462,10 +1449,24 @@ */ struct Token { const char *z; /* Text of the token. Not NULL-terminated! */ unsigned int n; /* Number of characters in this token */ }; + +/* +** An instance of this structure holds the results of parsing the first +** part of a CREATE INDEX statement. Instances exist only transiently +** during parsing. +*/ +struct CreateIndex { + int bUnique; /* True if the UNIQUE keyword was present */ + int bIfnotexist; /* True if IF NOT EXISTS was present */ + Token tCreate; /* CREATE token */ + Token tName1; /* First part of two part name */ + Token tName2; /* Second part of two part name */ + SrcList *pTblName; /* Table index is created on */ +}; /* ** One for each column used in source tables. */ struct AggInfoCol { @@ -1634,10 +1635,11 @@ i16 iRightJoinTable; /* If EP_FromJoin, the right table of the join */ u8 flags2; /* Second set of flags. EP2_... */ u8 op2; /* If a TK_REGISTER, the original value of Expr.op */ AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ Table *pTab; /* Table for TK_COLUMN expressions. */ + Index *pIdx; /* Fts index used by MATCH expressions */ #if SQLITE4_MAX_EXPR_DEPTH>0 int nHeight; /* Height of the tree headed by this node */ #endif }; @@ -1648,11 +1650,10 @@ #define EP_Agg 0x0002 /* Contains one or more aggregate functions */ #define EP_Resolved 0x0004 /* IDs have been resolved to COLUMNs */ #define EP_Error 0x0008 /* Expression contains one or more errors */ #define EP_Distinct 0x0010 /* Aggregate function with DISTINCT keyword */ #define EP_VarSelect 0x0020 /* pSelect is correlated, not constant */ -#define EP_DblQuoted 0x0040 /* token.z was originally in "..." */ #define EP_InfixFunc 0x0080 /* True for an infix function: LIKE, GLOB, etc */ #define EP_ExpCollate 0x0100 /* Collating sequence specified explicitly */ #define EP_FixedDest 0x0200 /* Result needed in a specific register */ #define EP_IntValue 0x0400 /* Integer value contained in u.iValue */ #define EP_xIsSelect 0x0800 /* x.pSelect is valid (otherwise x.pList is) */ @@ -2039,10 +2040,11 @@ #define SF_Aggregate 0x04 /* Contains aggregate functions */ #define SF_UsesEphemeral 0x08 /* Uses the OpenEphemeral opcode */ #define SF_Expanded 0x10 /* sqlite4SelectExpand() called on this */ #define SF_HasTypeInfo 0x20 /* FROM subqueries have Table metadata */ #define SF_UseSorter 0x40 /* Sort using a sorter */ +#define SF_Values 0x80 /* Synthesized from VALUES clause */ /* ** The results of a select can be distributed in several ways. The ** "SRT" prefix means "SELECT Result Type". @@ -2254,11 +2256,11 @@ */ #define OPFLAG_NCHANGE 0x01 /* Set to update db->nChange */ #define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ #define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */ #define OPFLAG_APPEND 0x08 /* This is likely to be an append */ -#define OPFLAG_USESEEKRESULT 0x10 /* Try to avoid a seek on insert */ +#define OPFLAG_SEQCOUNT 0x10 /* Append sequence number to key */ #define OPFLAG_CLEARCACHE 0x20 /* Clear pseudo-table cache in OP_Column */ #define OPFLAG_APPENDBIAS 0x40 /* Bias inserts for appending */ /* * Each trigger present in the database schema is stored as an instance of @@ -2389,23 +2391,18 @@ int iDb; /* 0 for main database. 1 for TEMP, 2.. for ATTACHed */ char **pzErrMsg; /* Error message stored here */ int rc; /* Result code stored here */ } InitData; -/* -** FIXME: missing docs -*/ -typedef int (*KVFactory_factory_f)(sqlite4_env*,sqlite4_kvstore**,const char*,unsigned); - /* ** A pluggable storage engine */ typedef struct KVFactory { - struct KVFactory *pNext; /* Next in list of them all */ - const char *zName; /* Name of this factory */ - KVFactory_factory_f xFactory; - int isPerm; /* True if a built-in. Cannot be popped */ + struct KVFactory *pNext; /* Next in list of all storage engines */ + const char *zName; /* Name of this factory */ + sqlite4_kvfactory xFactory; /* Function to make an sqlite4_kvstore obj */ + int isPerm; /* True if a built-in. Cannot be popped */ } KVFactory; /* ** An instance of this structure defines the run-time environment. */ @@ -2453,11 +2450,24 @@ NameContext *pNC; /* Naming context */ int i; /* Integer value */ } u; }; -/* Forward declarations */ +/* +** An instance of this structure is used as the p4 argument to some fts5 +** related vdbe opcodes. +*/ +struct Fts5Info { + int iDb; /* Database containing this index */ + int iRoot; /* Root page number of index */ + int iTbl; /* Root page number of indexed table */ + int nCol; /* Number of columns in indexed table */ + char **azCol; /* Column names for table */ + Fts5Tokenizer *pTokenizer; /* Tokenizer module */ + sqlite4_tokenizer *p; /* Tokenizer instance */ +}; + int sqlite4WalkExpr(Walker*, Expr*); int sqlite4WalkExprList(Walker*, ExprList*); int sqlite4WalkSelect(Walker*, Select*); int sqlite4WalkSelectExpr(Walker*, Select*); int sqlite4WalkSelectFrom(Walker*, Select*); @@ -2537,13 +2547,11 @@ #endif /* ** Internal function prototypes */ -int sqlite4StrICmp(const char *, const char *); int sqlite4Strlen30(const char*); -#define sqlite4StrNICmp sqlite4_strnicmp int sqlite4MallocInit(sqlite4_env*); void sqlite4MallocEnd(sqlite4_env*); void *sqlite4Malloc(sqlite4_env*, int); void *sqlite4MallocZero(sqlite4_env*, int); @@ -2952,11 +2960,11 @@ u8 sqlite4GetBoolean(const char *z); const void *sqlite4ValueText(sqlite4_value*, u8); int sqlite4ValueBytes(sqlite4_value*, u8); void sqlite4ValueSetStr(sqlite4_value*, int, const void *,u8, - void(*)(void*)); + void(*)(void*,void*),void*); void sqlite4ValueFree(sqlite4_value*); sqlite4_value *sqlite4ValueNew(sqlite4 *); char *sqlite4Utf16to8(sqlite4 *, const void*, int, u8); #ifdef SQLITE4_ENABLE_STAT3 char *sqlite4Utf8to16(sqlite4 *, u8, char *, int, int *); @@ -3222,6 +3230,30 @@ #define MEMTYPE_HEAP 0x01 /* General heap allocations */ #define MEMTYPE_LOOKASIDE 0x02 /* Might have been lookaside memory */ #define MEMTYPE_SCRATCH 0x04 /* Scratch allocations */ #define MEMTYPE_DB 0x10 /* Uses sqlite4DbMalloc, not sqlite_malloc */ +int sqlite4InitFts5(sqlite4 *db); +int sqlite4InitFts5Func(sqlite4 *db); +void sqlite4ShutdownFts5(sqlite4 *db); +void sqlite4CreateUsingIndex(Parse*, CreateIndex*, ExprList*, Token*, Token*); + +int sqlite4Fts5IndexSz(void); +void sqlite4Fts5IndexInit(Parse *, Index *, ExprList *); +void sqlite4Fts5IndexFree(sqlite4 *, Index *); + +int sqlite4Fts5Update(sqlite4 *, Fts5Info *, int, Mem *, Mem *, int, char **); +void sqlite4Fts5FreeInfo(sqlite4 *db, Fts5Info *); +void sqlite4Fts5CodeUpdate(Parse *, Index *pIdx, int, int, int, int); +void sqlite4Fts5CodeCksum(Parse *, Index *, int, int, int); +void sqlite4Fts5CodeQuery(Parse *, Index *, int, int, int); + +int sqlite4Fts5Pk(Fts5Cursor *, int, KVByteArray **, KVSize *); +int sqlite4Fts5Next(Fts5Cursor *pCsr); + +int sqlite4Fts5EntryCksum(sqlite4 *, Fts5Info *, Mem *, Mem *, i64 *); +int sqlite4Fts5RowCksum(sqlite4 *, Fts5Info *, Mem *, Mem *, i64 *); +int sqlite4Fts5Open(sqlite4*, Fts5Info*, const char*, int, Fts5Cursor**,char**); +int sqlite4Fts5Valid(Fts5Cursor *); +void sqlite4Fts5Close(Fts5Cursor *); + #endif /* _SQLITEINT_H_ */ Index: src/tclsqlite.c ================================================================== --- src/tclsqlite.c +++ src/tclsqlite.c @@ -504,11 +504,11 @@ char c = zType[0]; if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){ /* Only return a BLOB type if the Tcl variable is a bytearray and ** has no string representation. */ data = Tcl_GetByteArrayFromObj(pVar, &n); - sqlite4_result_blob(context, data, n, SQLITE4_TRANSIENT); + sqlite4_result_blob(context, data, n, SQLITE4_TRANSIENT, 0); }else if( c=='b' && strcmp(zType,"boolean")==0 ){ Tcl_GetIntFromObj(0, pVar, &n); sqlite4_result_int(context, n); }else if( c=='d' && strcmp(zType,"double")==0 ){ double r; @@ -519,11 +519,11 @@ Tcl_WideInt v; Tcl_GetWideIntFromObj(0, pVar, &v); sqlite4_result_int64(context, v); }else{ data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n); - sqlite4_result_text(context, (char *)data, n, SQLITE4_TRANSIENT); + sqlite4_result_text(context, (char *)data, n, SQLITE4_TRANSIENT, 0); } } } #ifndef SQLITE4_OMIT_AUTHORIZATION @@ -846,11 +846,11 @@ (c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0) ){ /* Load a BLOB type if the Tcl variable is a bytearray and ** it has no string representation or the host ** parameter name begins with "@". */ data = Tcl_GetByteArrayFromObj(pVar, &n); - sqlite4_bind_blob(pStmt, i, data, n, SQLITE4_STATIC); + sqlite4_bind_blob(pStmt, i, data, n, SQLITE4_STATIC, 0); Tcl_IncrRefCount(pVar); pPreStmt->apParm[iParm++] = pVar; }else if( c=='b' && strcmp(zType,"boolean")==0 ){ Tcl_GetIntFromObj(interp, pVar, &n); sqlite4_bind_int(pStmt, i, n); @@ -863,11 +863,11 @@ Tcl_WideInt v; Tcl_GetWideIntFromObj(interp, pVar, &v); sqlite4_bind_int64(pStmt, i, v); }else{ data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n); - sqlite4_bind_text(pStmt, i, (char *)data, n, SQLITE4_STATIC); + sqlite4_bind_text(pStmt, i, (char *)data, n, SQLITE4_STATIC, 0); Tcl_IncrRefCount(pVar); pPreStmt->apParm[iParm++] = pVar; } }else{ sqlite4_bind_null(pStmt, i); @@ -1668,11 +1668,11 @@ if( (nNull>0 && strcmp(azCol[i], zNull)==0) || strlen30(azCol[i])==0 ){ sqlite4_bind_null(pStmt, i+1); }else{ - sqlite4_bind_text(pStmt, i+1, azCol[i], -1, SQLITE4_STATIC); + sqlite4_bind_text(pStmt, i+1, azCol[i], -1, SQLITE4_STATIC, 0); } } sqlite4_step(pStmt); rc = sqlite4_reset(pStmt); free(zLine); @@ -2801,11 +2801,11 @@ unsigned char digest[16]; char zBuf[33]; p = sqlite4_aggregate_context(context, sizeof(*p)); MD5Final(digest,p); MD5DigestToBase16(digest, zBuf); - sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT, 0); } int Md5_Register(sqlite4 *db){ int rc = sqlite4_create_function(db, "md5sum", -1, SQLITE4_UTF8, 0, 0, md5step, md5finalize); sqlite4_overload_function(db, "md5sum", -1); /* To exercise this API */ Index: src/tokenize.c ================================================================== --- src/tokenize.c +++ src/tokenize.c @@ -225,11 +225,10 @@ } case '~': { *tokenType = TK_BITNOT; return 1; } - case '`': case '\'': case '"': { int delim = z[0]; testcase( delim=='`' ); testcase( delim=='\'' ); Index: src/trigger.c ================================================================== --- src/trigger.c +++ src/trigger.c @@ -55,11 +55,11 @@ if( pTmpSchema!=pTab->pSchema ){ HashElem *p; for(p=sqliteHashFirst(&pTmpSchema->trigHash); p; p=sqliteHashNext(p)){ Trigger *pTrig = (Trigger *)sqliteHashData(p); if( pTrig->pTabSchema==pTab->pSchema - && 0==sqlite4StrICmp(pTrig->table, pTab->zName) + && 0==sqlite4_stricmp(pTrig->table, pTab->zName) ){ pTrig->pNext = (pList ? pList : pTab->pTrigger); pList = pTrig; } } @@ -188,11 +188,11 @@ } goto trigger_cleanup; } /* Do not create a trigger on a system table */ - if( sqlite4StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ + if( sqlite4_strnicmp(pTab->zName, "sqlite_", 7)==0 ){ sqlite4ErrorMsg(pParse, "cannot create trigger on system table"); pParse->nErr++; goto trigger_cleanup; } @@ -500,11 +500,11 @@ zDb = pName->a[0].zDatabase; zName = pName->a[0].zName; nName = sqlite4Strlen30(zName); for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ - if( zDb && sqlite4StrICmp(db->aDb[j].zName, zDb) ) continue; + if( zDb && sqlite4_stricmp(db->aDb[j].zName, zDb) ) continue; pTrigger = sqlite4HashFind(&(db->aDb[j].pSchema->trigHash), zName, nName); if( pTrigger ) break; } if( !pTrigger ){ if( !noErr ){ Index: src/update.c ================================================================== --- src/update.c +++ src/update.c @@ -210,11 +210,11 @@ goto update_cleanup; } /* Resolve the column name on the left of the assignment */ for(j=0; jnCol; j++){ - if( sqlite4StrICmp(pTab->aCol[j].zName, pChanges->a[i].zName)==0 ) break; + if( sqlite4_stricmp(pTab->aCol[j].zName, pChanges->a[i].zName)==0 ) break; } if( j==pTab->nCol ){ sqlite4ErrorMsg(pParse, "no such column: %s", pChanges->a[i].zName); pParse->checkSchema = 1; goto update_cleanup; @@ -506,11 +506,11 @@ j1 = sqlite4VdbeAddOp4(v, OP_NotFound, iCur+iPk, 0, regOldKey, 0, P4_INT32); sqlite4GenerateRowIndexDelete(pParse, pTab, 0, iCur, aRegIdx); /* Delete the old record */ if( hasFK || bChngPk ){ - sqlite4VdbeAddOp2(v, OP_Delete, iCur, 0); + sqlite4VdbeAddOp2(v, OP_Delete, iCur+iPk, 0); } sqlite4VdbeJumpHere(v, j1); if( hasFK ){ sqlite4FkCheck(pParse, pTab, 0, regNew); Index: src/utf.c ================================================================== --- src/utf.c +++ src/utf.c @@ -185,11 +185,44 @@ *pzNext = zIn; return c; } - +/* +** Versions of stricmp and strnicmp that work with (simple) unicode +** case mapping. +*/ +int sqlite4_stricmp(const char *zLeft, const char *zRight){ + unsigned char *a, *b; + unsigned int ac, bc; + int x; + a = (unsigned char *)zLeft; + b = (unsigned char *)zRight; + do{ + READ_UTF8(a, 0, ac); + ac = sqlite4_tolower(ac); + READ_UTF8(b, 0, bc); + bc = sqlite4_tolower(bc); + }while( ac==bc && ac!=0 ); + return ac - bc; +} +int sqlite4_strnicmp(const char *zLeft, const char *zRight, int N){ + unsigned char *a, *b, *aTerm, *bTerm; + unsigned int ac, bc; + int x; + a = (unsigned char *)zLeft; + b = (unsigned char *)zRight; + aTerm = a + N; + bTerm = b + N; + do{ + READ_UTF8(a, aTerm, ac); + ac = sqlite4_tolower(ac); + READ_UTF8(b, bTerm, bc); + bc = sqlite4_tolower(bc); + }while( ac==bc && ac!=0 && amallocFailed ){ sqlite4VdbeMemRelease(&m); m.z = 0; } @@ -467,11 +500,11 @@ #ifdef SQLITE4_ENABLE_STAT3 char *sqlite4Utf8to16(sqlite4 *db, u8 enc, char *z, int n, int *pnOut){ Mem m; memset(&m, 0, sizeof(m)); m.db = db; - sqlite4VdbeMemSetStr(&m, z, n, SQLITE4_UTF8, SQLITE4_STATIC); + sqlite4VdbeMemSetStr(&m, z, n, SQLITE4_UTF8, SQLITE4_STATIC, 0); if( sqlite4VdbeMemTranslate(&m, enc) ){ assert( db->mallocFailed ); return 0; } assert( m.z==m.zMalloc ); @@ -556,5 +589,278 @@ assert( (z-zBuf)==n ); } } #endif /* SQLITE4_TEST */ #endif /* SQLITE4_OMIT_UTF16 */ + + +/* +** Return true if the argument corresponds to a unicode codepoint +** classified as either a letter or a number. Otherwise false. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +int sqlite4_isalnum(int c){ + /* Each unsigned integer in the following array corresponds to a contiguous + ** range of unicode codepoints that are not either letters or numbers (i.e. + ** codepoints for which this function should return 0). + ** + ** The most significant 22 bits in each 32-bit value contain the first + ** codepoint in the range. The least significant 10 bits are used to store + ** the size of the range (always at least 1). In other words, the value + ** ((C<<22) + N) represents a range of N codepoints starting with codepoint + ** C. It is not possible to represent a range larger than 1023 codepoints + ** using this format. + */ + const static unsigned int aEntry[] = { + 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, + 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, + 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, + 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01, + 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01, + 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802, + 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F, + 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401, + 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804, + 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403, + 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812, + 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001, + 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802, + 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805, + 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401, + 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03, + 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807, + 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001, + 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01, + 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804, + 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001, + 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802, + 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01, + 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06, + 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007, + 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006, + 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417, + 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14, + 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07, + 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01, + 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001, + 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802, + 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F, + 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002, + 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802, + 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006, + 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D, + 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802, + 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027, + 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403, + 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805, + 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04, + 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401, + 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005, + 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B, + 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A, + 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001, + 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59, + 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807, + 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01, + 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E, + 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100, + 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10, + 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402, + 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804, + 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012, + 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004, + 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002, + 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, + 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, + 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, + 0x037FFC02, 0x03E3FC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, + 0x03F4F802, 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, + 0x03F95013, 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, + 0x03FCEC06, 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, + 0x04040003, 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, + 0x040E7C01, 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, + 0x04280403, 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, + 0x04294009, 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, + 0x04420003, 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, + 0x04460003, 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, + 0x05BD442E, 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, + 0x07480046, 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, + 0x075C5401, 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, + 0x075EA401, 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, + 0x07C2800F, 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, + 0x07C4C03C, 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, + 0x07C94002, 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, + 0x07CE8025, 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, + 0x07D108B6, 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, + 0x07D7EC46, 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, + 0x38008060, 0x380400F0, 0x3C000001, 0x3FFFF401, 0x40000001, + 0x43FFF401, + }; + static const unsigned int aAscii[4] = { + 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, + }; + + if( c<128 ){ + return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); + }else if( c<(1<<22) ){ + unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; + int iRes; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aEntry[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( aEntry[0]=aEntry[iRes] ); + return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); + } + return 1; +} + +/* +** Interpret the argument as a unicode codepoint. If the codepoint +** is an upper case character that has a lower case equivalent, +** return the codepoint corresponding to the lower case version. +** Otherwise, return a copy of the argument. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +int sqlite4_tolower(int c){ + /* Each entry in the following array defines a rule for folding a range + ** of codepoints to lower case. The rule applies to a range of nRange + ** codepoints starting at codepoint iCode. + ** + ** If the least significant bit in flags is clear, then the rule applies + ** to all nRange codepoints (i.e. all nRange codepoints are upper case and + ** need to be folded). Or, if it is set, then the rule only applies to + ** every second codepoint in the range, starting with codepoint C. + ** + ** The 7 most significant bits in flags are an index into the aiOff[] + ** array. If a specific codepoint C does require folding, then its lower + ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF). + ** + ** The contents of this array are generated by parsing the CaseFolding.txt + ** file distributed as part of the "Unicode Character Database". See + ** http://www.unicode.org for details. + */ + static const struct TableEntry { + unsigned short iCode; + unsigned char flags; + unsigned char nRange; + } aEntry[] = { + {65, 14, 26}, {181, 64, 1}, {192, 14, 23}, + {216, 14, 7}, {256, 1, 48}, {306, 1, 6}, + {313, 1, 16}, {330, 1, 46}, {376, 116, 1}, + {377, 1, 6}, {383, 104, 1}, {385, 50, 1}, + {386, 1, 4}, {390, 44, 1}, {391, 0, 1}, + {393, 42, 2}, {395, 0, 1}, {398, 32, 1}, + {399, 38, 1}, {400, 40, 1}, {401, 0, 1}, + {403, 42, 1}, {404, 46, 1}, {406, 52, 1}, + {407, 48, 1}, {408, 0, 1}, {412, 52, 1}, + {413, 54, 1}, {415, 56, 1}, {416, 1, 6}, + {422, 60, 1}, {423, 0, 1}, {425, 60, 1}, + {428, 0, 1}, {430, 60, 1}, {431, 0, 1}, + {433, 58, 2}, {435, 1, 4}, {439, 62, 1}, + {440, 0, 1}, {444, 0, 1}, {452, 2, 1}, + {453, 0, 1}, {455, 2, 1}, {456, 0, 1}, + {458, 2, 1}, {459, 1, 18}, {478, 1, 18}, + {497, 2, 1}, {498, 1, 4}, {502, 122, 1}, + {503, 134, 1}, {504, 1, 40}, {544, 110, 1}, + {546, 1, 18}, {570, 70, 1}, {571, 0, 1}, + {573, 108, 1}, {574, 68, 1}, {577, 0, 1}, + {579, 106, 1}, {580, 28, 1}, {581, 30, 1}, + {582, 1, 10}, {837, 36, 1}, {880, 1, 4}, + {886, 0, 1}, {902, 18, 1}, {904, 16, 3}, + {908, 26, 1}, {910, 24, 2}, {913, 14, 17}, + {931, 14, 9}, {962, 0, 1}, {975, 4, 1}, + {976, 140, 1}, {977, 142, 1}, {981, 146, 1}, + {982, 144, 1}, {984, 1, 24}, {1008, 136, 1}, + {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1}, + {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1}, + {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32}, + {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1}, + {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38}, + {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1}, + {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1}, + {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6}, + {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6}, + {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8}, + {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2}, + {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1}, + {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2}, + {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2}, + {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2}, + {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1}, + {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16}, + {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47}, + {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1}, + {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1}, + {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1}, + {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2}, + {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1}, + {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14}, + {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, + {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, + {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, + {65313, 14, 26}, + }; + static const unsigned short aiOff[] = { + 1, 2, 8, 15, 16, 26, 28, 32, + 37, 38, 40, 48, 63, 64, 69, 71, + 79, 80, 116, 202, 203, 205, 206, 207, + 209, 210, 211, 213, 214, 217, 218, 219, + 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, + 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, + 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, + 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, + 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, + 65514, 65521, 65527, 65528, 65529, + }; + + int ret = c; + + assert( c>=0 ); + assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); + + if( c<128 ){ + if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); + }else if( c<65536 ){ + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + int iRes = -1; + + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + int cmp = (c - aEntry[iTest].iCode); + if( cmp>=0 ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( iRes<0 || c>=aEntry[iRes].iCode ); + + if( iRes>=0 ){ + const struct TableEntry *p = &aEntry[iRes]; + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; + assert( ret>0 ); + } + } + } + + else if( c>=66560 && c<66600 ){ + ret = c + 40; + } + + return ret; +} Index: src/util.c ================================================================== --- src/util.c +++ src/util.c @@ -130,13 +130,13 @@ char *z; va_list ap; va_start(ap, zFormat); z = sqlite4VMPrintf(db, zFormat, ap); va_end(ap); - sqlite4ValueSetStr(db->pErr, -1, z, SQLITE4_UTF8, SQLITE4_DYNAMIC); + sqlite4ValueSetStr(db->pErr, -1, z, SQLITE4_UTF8, SQLITE4_DYNAMIC, 0); }else{ - sqlite4ValueSetStr(db->pErr, 0, 0, SQLITE4_UTF8, SQLITE4_STATIC); + sqlite4ValueSetStr(db->pErr, 0, 0, SQLITE4_UTF8, SQLITE4_STATIC, 0); } } } /* @@ -216,38 +216,10 @@ } z[j] = 0; return j; } -/* Convenient short-hand */ -#define UpperToLower sqlite4UpperToLower - -/* -** Some systems have stricmp(). Others have strcasecmp(). Because -** there is no consistency, we will define our own. -** -** IMPLEMENTATION-OF: R-20522-24639 The sqlite4_strnicmp() API allows -** applications and extensions to compare the contents of two buffers -** containing UTF-8 strings in a case-independent fashion, using the same -** definition of case independence that SQLite uses internally when -** comparing identifiers. -*/ -int sqlite4StrICmp(const char *zLeft, const char *zRight){ - register unsigned char *a, *b; - a = (unsigned char *)zLeft; - b = (unsigned char *)zRight; - while( *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } - return UpperToLower[*a] - UpperToLower[*b]; -} -int sqlite4_strnicmp(const char *zLeft, const char *zRight, int N){ - register unsigned char *a, *b; - a = (unsigned char *)zLeft; - b = (unsigned char *)zRight; - while( N-- > 0 && *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } - return N<0 ? 0 : UpperToLower[*a] - UpperToLower[*b]; -} - /* ** The string z[] is an text representation of a real number. ** Convert this string to a double and write it into *pResult. ** ** The string z[] is length bytes in length (bytes, not characters) and Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -462,10 +462,22 @@ p->zErrMsg = sqlite4DbStrDup(db, pVtab->zErrMsg); sqlite4_free(db->pEnv, pVtab->zErrMsg); pVtab->zErrMsg = 0; } +/* +** Return a pointer to a register in the root frame. +*/ +static Mem *sqlite4RegisterInRootFrame(Vdbe *p, int i){ + if( p->pFrame ){ + VdbeFrame *pFrame; + for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); + return &pFrame->aMem[i]; + }else{ + return &p->aMem[i]; + } +} /* ** Execute as much of a VDBE program as we can then return. ** ** sqlite4VdbeMakeReady() must be called before this routine in order to @@ -870,11 +882,12 @@ pOp->opcode = OP_String; pOp->p1 = sqlite4Strlen30(pOp->p4.z); #ifndef SQLITE4_OMIT_UTF16 if( encoding!=SQLITE4_UTF8 ){ - rc = sqlite4VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE4_UTF8, SQLITE4_STATIC); + rc = sqlite4VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE4_UTF8, + SQLITE4_STATIC, 0); if( rc==SQLITE4_TOOBIG ) goto too_big; if( SQLITE4_OK!=sqlite4VdbeChangeEncoding(pOut, encoding) ) goto no_mem; assert( pOut->zMalloc==pOut->z ); assert( pOut->flags & MEM_Dyn ); pOut->zMalloc = 0; @@ -936,11 +949,11 @@ ** P4 points to a blob of data P1 bytes long. Store this ** blob in register P2. */ case OP_Blob: { /* out2-prerelease */ assert( pOp->p1 <= SQLITE4_MAX_LENGTH ); - sqlite4VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0); + sqlite4VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0, 0); pOut->enc = encoding; UPDATE_MAX_BLOBSIZE(pOut); break; } @@ -1287,10 +1300,18 @@ */ case OP_CollSeq: { assert( pOp->p4type==P4_COLLSEQ ); break; } + +/* Opcode: Mifunction P1 +*/ +case OP_Mifunction: { + pc++; + pOp++; + /* fall through to OP_Function */ +}; /* Opcode: Function P1 P2 P3 P4 P5 ** ** Invoke a user function (P4 is a pointer to a Function structure that ** defines the function) with P5 arguments taken from register P2 and @@ -1342,10 +1363,17 @@ ctx.s.flags = MEM_Null; ctx.s.db = db; ctx.s.xDel = 0; ctx.s.zMalloc = 0; + if( pOp[-1].opcode==OP_Mifunction ){ + ctx.pFts = p->apCsr[pOp[-1].p1]->pFts; + apVal++; + n--; + }else{ + ctx.pFts = 0; + } /* The output cell may already have a buffer allocated. Move ** the pointer to ctx.s so in case the user-function can use ** the already allocated buffer instead of allocating a new one. */ @@ -2168,13 +2196,17 @@ ** ** This instruction encodes the N values into a database key and writes ** the result to register P3. No affinity transformations are applied to ** the input values before they are encoded. ** -** If P5 is non-zero, then a sequence number (unique within the cursor) -** is appended to the record. The sole purpose of this is to ensure that -** the key blob is unique within the cursors table. +** If the OPFLAG_SEQCOUNT bit of P5 is set, then a sequence number +** (unique within the cursor) is appended to the record. The sole purpose +** of this is to ensure that the key blob is unique within the cursors table. +** +** If the OPFLAG_LASTROWID bit of P5 is set and the value of the first and +** only field of the key is an integer, then set the lastRowid field to the +** value of that integer. */ case OP_MakeIdxKey: { VdbeCursor *pC; KeyInfo *pKeyInfo; Mem *pData0; /* First in array of input registers */ @@ -2194,18 +2226,21 @@ /* If pOp->p5 is non-zero, encode the sequence number blob to append to ** the end of the key. Variable nSeq is set to the number of bytes in ** the encoded key. */ nSeq = 0; - if( pOp->p5 ){ + if( pOp->p5 & OPFLAG_SEQCOUNT ){ iSeq = pC->seqCount++; do { nSeq++; aSeq[sizeof(aSeq)-nSeq] = (u8)(iSeq & 0x007F); iSeq = iSeq >> 7; }while( iSeq ); aSeq[sizeof(aSeq)-nSeq] |= 0x80; + } + if( (pOp->p5 & OPFLAG_LASTROWID)!=0 && (pData0->flags & MEM_Int)!=0 ){ + lastRowid = pData0->u.i; } memAboutToChange(p, pOut); nField = pKeyInfo->nField; @@ -2221,11 +2256,12 @@ sqlite4DbFree(db, aRec); }else{ if( nSeq ){ memcpy(&aRec[nRec], &aSeq[sizeof(aSeq)-nSeq], nSeq); } - rc = sqlite4VdbeMemSetStr(pOut, (char *)aRec, nRec+nSeq, 0, SQLITE4_DYNAMIC); + rc = sqlite4VdbeMemSetStr(pOut, (char *)aRec, nRec+nSeq, 0, + SQLITE4_DYNAMIC, 0); REGISTER_TRACE(pOp->p3, pOut); UPDATE_MAX_BLOBSIZE(pOut); } break; @@ -2311,11 +2347,12 @@ pData0, pC->pKeyInfo->nField, pC->iRoot, pC->pKeyInfo, &aRec, &nRec, 0 ); if( rc ){ sqlite4DbFree(db, aRec); }else{ - rc = sqlite4VdbeMemSetStr(pKeyOut, (char *)aRec, nRec, 0, SQLITE4_DYNAMIC); + rc = sqlite4VdbeMemSetStr(pKeyOut, (char *)aRec, nRec, 0, + SQLITE4_DYNAMIC, 0); REGISTER_TRACE(keyReg, pKeyOut); UPDATE_MAX_BLOBSIZE(pKeyOut); } } @@ -2327,11 +2364,11 @@ aRec = 0; rc = sqlite4VdbeEncodeData(db, pData0, nField, &aRec, &nRec); if( rc ){ sqlite4DbFree(db, aRec); }else{ - rc = sqlite4VdbeMemSetStr(pOut, (char *)aRec, nRec, 0, SQLITE4_DYNAMIC); + rc = sqlite4VdbeMemSetStr(pOut, (char *)aRec, nRec, 0, SQLITE4_DYNAMIC,0); REGISTER_TRACE(pOp->p3, pOut); UPDATE_MAX_BLOBSIZE(pOut); } } break; @@ -2414,11 +2451,11 @@ ** number. For example, to commit or rollback the top level transaction ** iSave==2. */ iSave = db->nSavepoint+1; for(pSave=db->pSavepoint; pSave; pSave=pSave->pNext){ if( zSave ){ - if( pSave->zName && 0==sqlite4StrICmp(zSave, pSave->zName) ) break; + if( pSave->zName && 0==sqlite4_stricmp(zSave, pSave->zName) ) break; }else{ if( pSave->pNext==0 ) break; } iSave--; } @@ -2546,11 +2583,11 @@ break; } /* Opcode: VerifyCookie P1 P2 P3 * * ** -** Check the value of global database parameter number 0 (the +** cHECK THe value of global database parameter number 0 (the ** schema version) and make sure it is equal to P2 and that the ** generation counter on the local schema parse equals P3. ** ** P1 is the database number which is 0 for the main database file ** and 1 for the file holding temporary tables and some higher number @@ -2844,31 +2881,40 @@ KVByteArray *aPkKey; KVSize nPkKey; pPk = p->apCsr[pOp->p1]; pIdx = p->apCsr[pOp->p3]; - assert( pIdx->pKeyInfo->nPK>0 ); - assert( pPk->pKeyInfo->nPK==0 ); - - rc = sqlite4KVCursorKey(pIdx->pKVCur, &aKey, &nKey); - if( rc==SQLITE4_OK ){ - nShort = sqlite4VdbeShortKey(aKey, nKey, - pIdx->pKeyInfo->nField - pIdx->pKeyInfo->nPK - ); - - nPkKey = sqlite4VarintLen(pPk->iRoot) + nKey - nShort; - aPkKey = sqlite4DbMallocRaw(db, nPkKey); - - if( aPkKey ){ - putVarint32(aPkKey, pPk->iRoot); - memcpy(&aPkKey[nPkKey - (nKey-nShort)], &aKey[nShort], nKey-nShort); + + if( pIdx->pFts ){ + rc = sqlite4Fts5Pk(pIdx->pFts, pPk->iRoot, &aPkKey, &nPkKey); + if( rc==SQLITE4_OK ){ rc = sqlite4KVCursorSeek(pPk->pKVCur, aPkKey, nPkKey, 0); - if( rc==SQLITE4_NOTFOUND ){ - rc = SQLITE4_CORRUPT_BKPT; - } + if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_CORRUPT_BKPT; pPk->nullRow = 0; - sqlite4DbFree(db, aPkKey); + } + }else{ + assert( pIdx->pKeyInfo->nPK>0 ); + assert( pPk->pKeyInfo->nPK==0 ); + rc = sqlite4KVCursorKey(pIdx->pKVCur, &aKey, &nKey); + if( rc==SQLITE4_OK ){ + nShort = sqlite4VdbeShortKey(aKey, nKey, + pIdx->pKeyInfo->nField - pIdx->pKeyInfo->nPK + ); + + nPkKey = sqlite4VarintLen(pPk->iRoot) + nKey - nShort; + aPkKey = sqlite4DbMallocRaw(db, nPkKey); + + if( aPkKey ){ + putVarint32(aPkKey, pPk->iRoot); + memcpy(&aPkKey[nPkKey - (nKey-nShort)], &aKey[nShort], nKey-nShort); + rc = sqlite4KVCursorSeek(pPk->pKVCur, aPkKey, nPkKey, 0); + if( rc==SQLITE4_NOTFOUND ){ + rc = SQLITE4_CORRUPT_BKPT; + } + pPk->nullRow = 0; + sqlite4DbFree(db, aPkKey); + } } } break; } @@ -3206,16 +3252,18 @@ pOut->u.i = p->apCsr[pOp->p1]->seqCount++; break; } -/* Opcode: NewRowid P1 P2 * * * +/* Opcode: NewRowid P1 P2 P3 * * ** -** Get a new integer record number (a.k.a "rowid") used as the key to a table. -** The record number is not previously used as a key in the database -** table that cursor P1 points to. The new record number is written -** to register P2. +** Get a new integer primary key (a.k.a "rowid") for table P1. The integer +** should not be currently in use as a primary key on that table. +** +** If P3 is not zero, then it is the number of a register in the top-level +** frame that holds a lower bound for the new rowid. In other words, the +** new rowid must be no less than reg[P3]+1. */ case OP_NewRowid: { /* out2-prerelease */ i64 v; /* The new rowid */ VdbeCursor *pC; /* Cursor of table to get the new rowid */ const KVByteArray *aKey; /* Key of an existing row */ @@ -3258,15 +3306,28 @@ if( n==0 ) rc = SQLITE4_CORRUPT_BKPT; if( v!=pC->iRoot ) rc = SQLITE4_CORRUPT_BKPT; } if( rc==SQLITE4_OK ){ n = sqlite4VdbeDecodeIntKey(&aKey[n], nKey-n, &v); - if( n==0 ) rc = SQLITE4_CORRUPT_BKPT; + if( n==0 || v==LARGEST_INT64 ) rc = SQLITE4_FULL; } }else{ break; } +#ifndef SQLITE_OMIT_AUTOINCREMENT + if( pOp->p3 && rc==SQLITE4_OK ){ + pIn3 = sqlite4RegisterInRootFrame(p, pOp->p3); + assert( memIsValid(pIn3) ); + REGISTER_TRACE(pOp->p3, pIn3); + sqlite4VdbeMemIntegerify(pIn3); + assert( (pIn3->flags & MEM_Int)!=0 ); /* mem(P3) holds an integer */ + if( pIn3->u.i==MAX_ROWID ){ + rc = SQLITE4_FULL; + } + if( vu.i ) v = pIn3->u.i; + } +#endif pOut->flags = MEM_Int; pOut->u.i = v+1; break; } @@ -3327,18 +3388,10 @@ ** If the OPFLAG_NCHANGE flag of P5 is set, then the row change count is ** incremented (otherwise not). If the OPFLAG_LASTROWID flag of P5 is set, ** then rowid is stored for subsequent return by the ** sqlite4_last_insert_rowid() function (otherwise it is unmodified). ** -** If the OPFLAG_USESEEKRESULT flag of P5 is set and if the result of -** the last seek operation (OP_NotExists) was a success, then this -** operation will not attempt to find the appropriate row before doing -** the insert but will instead overwrite the row that the cursor is -** currently pointing to. Presumably, the prior OP_NotExists opcode -** has already positioned the cursor correctly. This is an optimization -** that boosts performance by avoiding redundant seeks. -** ** If the OPFLAG_ISUPDATE flag is set, then this opcode is part of an ** UPDATE operation. Otherwise (if the flag is clear) then this opcode ** is part of an INSERT operation. The difference is only important to ** the update hook. ** @@ -3469,30 +3522,20 @@ if( (pIn3->flags & MEM_Blob) && pIn3->n==nKey && 0==memcmp(pIn3->z, aKey, nKey) ){ pc = pOp->p2-1; }else{ - sqlite4VdbeMemSetStr(pIn3, (const char*)aKey, nKey, 0, SQLITE4_TRANSIENT); + sqlite4VdbeMemSetStr(pIn3, (const char*)aKey, nKey, 0, SQLITE4_TRANSIENT,0); } break; }; /* Opcode: SorterData P1 P2 * * * ** ** Write into register P2 the current sorter data for sorter cursor P1. */ -case OP_SorterData: { - VdbeCursor *pC; - pOut = &aMem[pOp->p2]; - pC = p->apCsr[pOp->p1]; - assert( pC!=0 ); - pOp->opcode = OP_RowData; - pc--; - break; -} - /* Opcode: RowData P1 P2 * * * ** ** Write into register P2 the complete row data for cursor P1. ** There is no interpretation of the data. ** It is just copied onto the P2 register exactly as @@ -3509,10 +3552,11 @@ ** it is found in the database file. ** ** If the P1 cursor must be pointing to a valid row (not a NULL row) ** of a real table, not a pseudo-table. */ +case OP_SorterData: case OP_RowKey: case OP_RowData: { VdbeCursor *pC; KVCursor *pCrsr; const KVByteArray *pData; @@ -3536,11 +3580,11 @@ rc = sqlite4KVCursorData(pCrsr, 0, -1, &pData, &nData); } if( rc==SQLITE4_OK && nData>db->aLimit[SQLITE4_LIMIT_LENGTH] ){ goto too_big; } - sqlite4VdbeMemSetStr(pOut, (const char*)pData, nData, 0, SQLITE4_TRANSIENT); + sqlite4VdbeMemSetStr(pOut, (const char*)pData, nData, 0, SQLITE4_TRANSIENT,0); pOut->enc = SQLITE4_UTF8; /* In case the blob is ever cast to text */ UPDATE_MAX_BLOBSIZE(pOut); break; } @@ -4057,11 +4101,12 @@ pIn1 = &aMem[pOp->p1]; pOut = &aMem[pOp->p3]; if( (pIn1->flags & MEM_RowSet) && (aKey = sqlite4RowSetRead(pIn1->u.pRowSet, &nKey)) ){ - rc = sqlite4VdbeMemSetStr(pOut, (char const *)aKey, nKey, 0, SQLITE4_TRANSIENT); + rc = sqlite4VdbeMemSetStr(pOut, (char const *)aKey, nKey, 0, + SQLITE4_TRANSIENT, 0); sqlite4RowSetNext(pIn1->u.pRowSet); }else{ /* The RowSet is empty */ sqlite4VdbeMemSetNull(pIn1); pc = pOp->p2 - 1; @@ -4247,23 +4292,20 @@ ** an integer. */ case OP_MemMax: { /* in2 */ Mem *pIn1; VdbeFrame *pFrame; - if( p->pFrame ){ - for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); - pIn1 = &pFrame->aMem[pOp->p1]; - }else{ - pIn1 = &aMem[pOp->p1]; - } + pIn1 = sqlite4RegisterInRootFrame(p, pOp->p1); assert( memIsValid(pIn1) ); sqlite4VdbeMemIntegerify(pIn1); pIn2 = &aMem[pOp->p2]; + REGISTER_TRACE(pOp->p1, pIn1); sqlite4VdbeMemIntegerify(pIn2); if( pIn1->u.iu.i){ pIn1->u.i = pIn2->u.i; } + REGISTER_TRACE(pOp->p1, pIn1); break; } #endif /* SQLITE4_OMIT_AUTOINCREMENT */ /* Opcode: IfPos P1 P2 * * * @@ -4825,10 +4867,143 @@ #endif /* SQLITE4_DEBUG */ break; } #endif +/* Opcode: FtsUpdate P1 P2 P3 P4 P5 +** +** This opcode is used to write to an FTS index. P4 points to an Fts5Info +** object describing the index. +** +** If argument P5 is non-zero, then entries are removed from the FTS index. +** If it is zero, then entries are inserted. In other words, when a row +** is deleted from a table with an FTS index, this opcode is invoked with +** P5==1. When a row is inserted, it is invoked with P5==0. If an existing +** row is updated, this opcode is invoked twice - once with P5==1 and then +** again with P5==0. +** +** Register P1 contains the PK (a blob in key format) of the affected row. +** P3 is the first in an array of N registers, where N is the number of +** columns in the indexed table. Each register contains the value for the +** corresponding table column. +** +** If P2 is non-zero, then it is a register containing the root page number +** of the fts index to update. If it is zero, then the root page of the +** index is available as part of the Fts5Info structure. +*/ +case OP_FtsUpdate: { + Fts5Info *pInfo; /* Description of fts5 index to update */ + Mem *pKey; /* Primary key of indexed row */ + Mem *aArg; /* Pointer to array of N arguments */ + Mem *pRoot; /* Root page number */ + int iRoot; + + assert( pOp->p4type==P4_FTS5INFO ); + pInfo = pOp->p4.pFtsInfo; + aArg = &aMem[pOp->p3]; + pKey = &aMem[pOp->p1]; + + if( pOp->p2 ){ + iRoot = aMem[pOp->p2].u.i; + }else{ + iRoot = 0; + } + + rc = sqlite4Fts5Update(db, pInfo, iRoot, pKey, aArg, pOp->p5, &p->zErrMsg); + break; +} + +/* +** Opcode: FtsCksum P1 * P3 P4 P5 +** +** This opcode is used by the integrity-check procedure that verifies that +** the contents of an fts5 index and its corresponding table match. +*/ +case OP_FtsCksum: { + Fts5Info *pInfo; /* Description of fts5 index to update */ + Mem *pKey; /* Primary key of row */ + Mem *aArg; /* Pointer to array of N values */ + i64 cksum; /* Checksum for this row or index entry */ + + assert( pOp->p4type==P4_FTS5INFO ); + pInfo = pOp->p4.pFtsInfo; + + pOut = &aMem[pOp->p1]; + pKey = &aMem[pOp->p3]; + aArg = &aMem[pOp->p3+1]; + cksum = 0; + + if( pOp->p5 ){ + sqlite4Fts5EntryCksum(db, pInfo, pKey, aArg, &cksum); + pOut->u.i = pOut->u.i ^ cksum; + }else{ + sqlite4Fts5RowCksum(db, pInfo, pKey, aArg, &cksum); + pOut->u.i = pOut->u.i ^ cksum; + } + break; +} + +/* Opcode: FtsOpen P1 P2 P3 P4 P5 +** +** Open an FTS cursor named P1. P4 points to an Fts5Info object. +** +** Register P3 contains the MATCH expression that this cursor will iterate +** through the matches for. P5 is set to 0 to iterate through the results +** in ascending PK order, or 1 for descending PK order. +** +** If the expression matches zero rows, jump to instruction P2. Otherwise, +** leave the cursor pointing at the first match and fall through to the +** next instruction. +*/ +case OP_FtsOpen: { /* jump */ + Fts5Info *pInfo; /* Description of fts5 index to update */ + char *zErr; + VdbeCursor *pCur; + char *zMatch; + Mem *pMatch; + + pMatch = &aMem[pOp->p3]; + Stringify(pMatch, encoding); + zMatch = pMatch->z; + + assert( pOp->p4type==P4_FTS5INFO ); + pInfo = pOp->p4.pFtsInfo; + pCur = allocateCursor(p, pOp->p1, 0, pInfo->iDb, 0); + if( pCur ){ + rc = sqlite4Fts5Open(db, pInfo, zMatch, pOp->p5, &pCur->pFts, &p->zErrMsg); + } + if( rc==SQLITE4_OK && 0==sqlite4Fts5Valid(pCur->pFts) ){ + pc = pOp->p2-1; + } + break; +} + +/* Opcode: FtsNext P1 P2 * * * +** +** Advance FTS cursor P1 to the next entry and jump to instruction P2. Or, +** if there is no next entry, set the cursor to point to EOF and fall through +** to the next instruction. +*/ +case OP_FtsNext: { + VdbeCursor *pCsr; + + pCsr = p->apCsr[pOp->p1]; + rc = sqlite4Fts5Next(pCsr->pFts); + if( rc==SQLITE4_OK && sqlite4Fts5Valid(pCsr->pFts) ) pc = pOp->p2-1; + + break; +} + +/* Opcode: FtsPk P1 P2 * * * +** +** P1 is an FTS cursor that points to a valid entry (not EOF). Copy the PK +** blob for the current entry to register P2. +*/ +case OP_FtsPk: { + assert( 0 ); + break; +} /* Opcode: Noop * * * * * ** ** Do nothing. This instruction is often useful as a jump ** destination. @@ -4835,11 +5010,11 @@ */ /* ** The magic Explain opcode are only inserted when explain==2 (which ** is to say when the EXPLAIN QUERY PLAN syntax is used.) ** This opcode records information from the optimizer. It is the -** the same as a no-op. This opcodesnever appears in a real VM program. +** the same as a no-op. This opcode never appears in a real VM program. */ default: { /* This is really OP_Noop and OP_Explain */ assert( pOp->opcode==OP_Noop || pOp->opcode==OP_Explain ); break; } Index: src/vdbe.h ================================================================== --- src/vdbe.h +++ src/vdbe.h @@ -60,10 +60,11 @@ Mem *pMem; /* Used when p4type is P4_MEM */ VTable *pVtab; /* Used when p4type is P4_VTAB */ KeyInfo *pKeyInfo; /* Used when p4type is P4_KEYINFO */ int *ai; /* Used when p4type is P4_INTARRAY */ SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ + Fts5Info *pFtsInfo; /* Used when p4type is P4_FTS5INDEXINFO */ int (*xAdvance)(VdbeCursor*); } p4; #ifdef SQLITE4_DEBUG char *zComment; /* Comment to improve readability */ #endif @@ -118,10 +119,11 @@ #define P4_INT64 (-13) /* P4 is a 64-bit signed integer */ #define P4_INT32 (-14) /* P4 is a 32-bit signed integer */ #define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ #define P4_SUBPROGRAM (-18) /* P4 is a pointer to a SubProgram structure */ #define P4_ADVANCE (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */ +#define P4_FTS5INFO (-20) /* P4 points to an Fts5Info structure */ /* When adding a P4 argument using P4_KEYINFO, a copy of the KeyInfo structure ** is made. That copy is freed when the Vdbe is finalized. But if the ** argument is P4_KEYINFO_HANDOFF, the passed in pointer is used. It still ** gets freed when the Vdbe is finalized so it still should be obtained @@ -200,11 +202,11 @@ #endif void sqlite4VdbeResetStepResult(Vdbe*); void sqlite4VdbeRewind(Vdbe*); int sqlite4VdbeReset(Vdbe*); void sqlite4VdbeSetNumCols(Vdbe*,int); -int sqlite4VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*)); +int sqlite4VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*,void*)); void sqlite4VdbeCountChanges(Vdbe*); sqlite4 *sqlite4VdbeDb(Vdbe*); void sqlite4VdbeSetSql(Vdbe*, const char *z, int n); void sqlite4VdbeSwap(Vdbe*,Vdbe*); VdbeOp *sqlite4VdbeTakeOpArray(Vdbe*, int*, int*); Index: src/vdbeInt.h ================================================================== --- src/vdbeInt.h +++ src/vdbeInt.h @@ -68,10 +68,11 @@ const sqlite4_module *pModule; /* Module for cursor pVtabCursor */ i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred move-to */ i64 lastRowid; /* Last rowid from a Next or NextIdx operation */ VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ + Fts5Cursor *pFts; /* Fts5 cursor object (or NULL) */ /* Result of last sqlite4-Moveto() done by an OP_NotExists or ** OP_IsUnique opcode on this cursor. */ int seekResult; }; @@ -146,11 +147,12 @@ u8 enc; /* SQLITE4_UTF8, SQLITE4_UTF16BE, SQLITE4_UTF16LE */ #ifdef SQLITE4_DEBUG Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ void *pFiller; /* So that sizeof(Mem) is a multiple of 8 */ #endif - void (*xDel)(void *); /* If not null, call this function to delete Mem.z */ + void (*xDel)(void*,void*); /* Function to delete Mem.z */ + void *pDelArg; /* First argument to xDel() */ char *zMalloc; /* Dynamic buffer allocated by sqlite4_malloc() */ }; /* One or more of the following flags are set to indicate the validOK ** representations of the value stored in the Mem struct. @@ -214,11 +216,12 @@ struct VdbeFunc { FuncDef *pFunc; /* The definition of the function */ int nAux; /* Number of entries allocated for apAux[] */ struct AuxData { void *pAux; /* Aux data for the i-th argument */ - void (*xDelete)(void *); /* Destructor for the aux data */ + void (*xDelete)(void*,void*); /* Destructor for the aux data */ + void *pDeleteArg; /* First argument to xDelete */ } apAux[1]; /* One slot for each function argument */ }; /* ** The "context" argument for a installable function. A pointer to an @@ -238,10 +241,11 @@ VdbeFunc *pVdbeFunc; /* Auxilary data, if created. */ Mem s; /* The return value is stored here */ Mem *pMem; /* Memory cell used to store aggregate context */ int isError; /* Error code returned by the function. */ CollSeq *pColl; /* Collating sequence */ + Fts5Cursor *pFts; /* fts5 cursor for matchinfo functions */ }; /* ** An Explain object accumulates indented output which is helpful ** in describing recursive data structures. @@ -394,11 +398,12 @@ int sqlite4VdbeMemTooBig(Mem*); int sqlite4VdbeMemCopy(Mem*, const Mem*); void sqlite4VdbeMemShallowCopy(Mem*, const Mem*, int); void sqlite4VdbeMemMove(Mem*, Mem*); int sqlite4VdbeMemNulTerminate(Mem*); -int sqlite4VdbeMemSetStr(Mem*, const char*, int, u8, void(*)(void*)); +int sqlite4VdbeMemSetStr(Mem*, const char*, int, u8, + void(*)(void*,void*),void*); void sqlite4VdbeMemSetInt64(Mem*, i64); #ifdef SQLITE4_OMIT_FLOATING_POINT # define sqlite4VdbeMemSetDouble sqlite4VdbeMemSetInt64 #else void sqlite4VdbeMemSetDouble(Mem*, double); Index: src/vdbeapi.c ================================================================== --- src/vdbeapi.c +++ src/vdbeapi.c @@ -187,49 +187,51 @@ ** The setStrOrError() funtion calls sqlite4VdbeMemSetStr() to store the ** result as a string or blob but if the string or blob is too large, it ** then sets the error code to SQLITE4_TOOBIG */ static void setResultStrOrError( - sqlite4_context *pCtx, /* Function context */ - const char *z, /* String pointer */ - int n, /* Bytes in string, or negative */ - u8 enc, /* Encoding of z. 0 for BLOBs */ - void (*xDel)(void*) /* Destructor function */ + sqlite4_context *pCtx, /* Function context */ + const char *z, /* String pointer */ + int n, /* Bytes in string, or negative */ + u8 enc, /* Encoding of z. 0 for BLOBs */ + void (*xDel)(void*,void*), /* Destructor function */ + void *pDelArg /* First argument to xDel() */ ){ if( xDel==SQLITE4_DYNAMIC ){ assert( sqlite4MemdebugHasType(z, MEMTYPE_HEAP) ); assert( sqlite4MemdebugNoType(z, ~MEMTYPE_HEAP) ); sqlite4MemdebugSetType((char*)z, MEMTYPE_DB | MEMTYPE_HEAP); } - if( sqlite4VdbeMemSetStr(&pCtx->s, z, n, enc, xDel)==SQLITE4_TOOBIG ){ + if( sqlite4VdbeMemSetStr(&pCtx->s, z, n, enc, xDel,pDelArg)==SQLITE4_TOOBIG ){ sqlite4_result_error_toobig(pCtx); } } void sqlite4_result_blob( sqlite4_context *pCtx, const void *z, int n, - void (*xDel)(void *) + void (*xDel)(void*,void*), + void *pDelArg ){ assert( n>=0 ); assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, 0, xDel); + setResultStrOrError(pCtx, z, n, 0, xDel, pDelArg); } void sqlite4_result_double(sqlite4_context *pCtx, double rVal){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); sqlite4VdbeMemSetDouble(&pCtx->s, rVal); } void sqlite4_result_error(sqlite4_context *pCtx, const char *z, int n){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); pCtx->isError = SQLITE4_ERROR; - sqlite4VdbeMemSetStr(&pCtx->s, z, n, SQLITE4_UTF8, SQLITE4_TRANSIENT); + sqlite4VdbeMemSetStr(&pCtx->s, z, n, SQLITE4_UTF8, SQLITE4_TRANSIENT, 0); } #ifndef SQLITE4_OMIT_UTF16 void sqlite4_result_error16(sqlite4_context *pCtx, const void *z, int n){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); pCtx->isError = SQLITE4_ERROR; - sqlite4VdbeMemSetStr(&pCtx->s, z, n, SQLITE4_UTF16NATIVE, SQLITE4_TRANSIENT); + sqlite4VdbeMemSetStr(&pCtx->s, z, n, SQLITE4_UTF16NATIVE,SQLITE4_TRANSIENT,0); } #endif void sqlite4_result_int(sqlite4_context *pCtx, int iVal){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); sqlite4VdbeMemSetInt64(&pCtx->s, (i64)iVal); @@ -244,42 +246,46 @@ } void sqlite4_result_text( sqlite4_context *pCtx, const char *z, int n, - void (*xDel)(void *) + void (*xDel)(void*,void*), + void *pDelArg ){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE4_UTF8, xDel); + setResultStrOrError(pCtx, z, n, SQLITE4_UTF8, xDel, pDelArg); } #ifndef SQLITE4_OMIT_UTF16 void sqlite4_result_text16( sqlite4_context *pCtx, const void *z, int n, - void (*xDel)(void *) + void (*xDel)(void*,void*), + void *pDelArg ){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE4_UTF16NATIVE, xDel); + setResultStrOrError(pCtx, z, n, SQLITE4_UTF16NATIVE, xDel, pDelArg); } void sqlite4_result_text16be( sqlite4_context *pCtx, const void *z, int n, - void (*xDel)(void *) + void (*xDel)(void*,void*), + void *pDelArg ){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE4_UTF16BE, xDel); + setResultStrOrError(pCtx, z, n, SQLITE4_UTF16BE, xDel, pDelArg); } void sqlite4_result_text16le( sqlite4_context *pCtx, const void *z, int n, - void (*xDel)(void *) + void (*xDel)(void*,void*), + void *pDelArg ){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE4_UTF16LE, xDel); + setResultStrOrError(pCtx, z, n, SQLITE4_UTF16LE, xDel, pDelArg); } #endif /* SQLITE4_OMIT_UTF16 */ void sqlite4_result_value(sqlite4_context *pCtx, sqlite4_value *pValue){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); sqlite4VdbeMemCopy(&pCtx->s, pValue); @@ -290,20 +296,20 @@ } void sqlite4_result_error_code(sqlite4_context *pCtx, int errCode){ pCtx->isError = errCode; if( pCtx->s.flags & MEM_Null ){ sqlite4VdbeMemSetStr(&pCtx->s, sqlite4ErrStr(errCode), -1, - SQLITE4_UTF8, SQLITE4_STATIC); + SQLITE4_UTF8, SQLITE4_STATIC, 0); } } /* Force an SQLITE4_TOOBIG error. */ void sqlite4_result_error_toobig(sqlite4_context *pCtx){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); pCtx->isError = SQLITE4_TOOBIG; sqlite4VdbeMemSetStr(&pCtx->s, "string or blob too big", -1, - SQLITE4_UTF8, SQLITE4_STATIC); + SQLITE4_UTF8, SQLITE4_STATIC, 0); } /* An SQLITE4_NOMEM error. */ void sqlite4_result_error_nomem(sqlite4_context *pCtx){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); @@ -589,11 +595,12 @@ */ void sqlite4_set_auxdata( sqlite4_context *pCtx, int iArg, void *pAux, - void (*xDelete)(void*) + void (*xDelete)(void*,void*), + void *pDeleteArg ){ struct AuxData *pAuxData; VdbeFunc *pVdbeFunc; if( iArg<0 ) goto failed; @@ -612,19 +619,20 @@ pVdbeFunc->pFunc = pCtx->pFunc; } pAuxData = &pVdbeFunc->apAux[iArg]; if( pAuxData->pAux && pAuxData->xDelete ){ - pAuxData->xDelete(pAuxData->pAux); + pAuxData->xDelete(pAuxData->pDeleteArg, pAuxData->pAux); } pAuxData->pAux = pAux; pAuxData->xDelete = xDelete; + pAuxData->pDeleteArg = pDeleteArg; return; failed: if( xDelete ){ - xDelete(pAux); + xDelete(pDeleteArg, pAux); } } #ifndef SQLITE4_OMIT_DEPRECATED /* @@ -1006,35 +1014,36 @@ /* ** Bind a text or BLOB value. */ static int bindText( - sqlite4_stmt *pStmt, /* The statement to bind against */ - int i, /* Index of the parameter to bind */ - const void *zData, /* Pointer to the data to be bound */ - int nData, /* Number of bytes of data to be bound */ - void (*xDel)(void*), /* Destructor for the data */ - u8 encoding /* Encoding for the data */ + sqlite4_stmt *pStmt, /* The statement to bind against */ + int i, /* Index of the parameter to bind */ + const void *zData, /* Pointer to the data to be bound */ + int nData, /* Number of bytes of data to be bound */ + void (*xDel)(void*,void*), /* Destructor for the data */ + void *pDelArg, /* First argument to xDel() */ + u8 encoding /* Encoding for the data */ ){ Vdbe *p = (Vdbe *)pStmt; Mem *pVar; int rc; rc = vdbeUnbind(p, i); if( rc==SQLITE4_OK ){ if( zData!=0 ){ pVar = &p->aVar[i-1]; - rc = sqlite4VdbeMemSetStr(pVar, zData, nData, encoding, xDel); + rc = sqlite4VdbeMemSetStr(pVar, zData, nData, encoding, xDel, pDelArg); if( rc==SQLITE4_OK && encoding!=0 ){ rc = sqlite4VdbeChangeEncoding(pVar, ENC(p->db)); } sqlite4Error(p->db, rc, 0); rc = sqlite4ApiExit(p->db, rc); } sqlite4_mutex_leave(p->db->mutex); }else if( xDel!=SQLITE4_STATIC && xDel!=SQLITE4_TRANSIENT ){ - xDel((void*)zData); + xDel(pDelArg, (void*)zData); } return rc; } @@ -1044,13 +1053,14 @@ int sqlite4_bind_blob( sqlite4_stmt *pStmt, int i, const void *zData, int nData, - void (*xDel)(void*) + void (*xDel)(void*,void*), + void *pDelArg ){ - return bindText(pStmt, i, zData, nData, xDel, 0); + return bindText(pStmt, i, zData, nData, xDel, pDelArg, 0); } int sqlite4_bind_double(sqlite4_stmt *pStmt, int i, double rValue){ int rc; Vdbe *p = (Vdbe *)pStmt; rc = vdbeUnbind(p, i); @@ -1085,23 +1095,25 @@ int sqlite4_bind_text( sqlite4_stmt *pStmt, int i, const char *zData, int nData, - void (*xDel)(void*) + void (*xDel)(void*,void*), + void *pDelArg ){ - return bindText(pStmt, i, zData, nData, xDel, SQLITE4_UTF8); + return bindText(pStmt, i, zData, nData, xDel, pDelArg, SQLITE4_UTF8); } #ifndef SQLITE4_OMIT_UTF16 int sqlite4_bind_text16( sqlite4_stmt *pStmt, int i, const void *zData, int nData, - void (*xDel)(void*) + void (*xDel)(void*,void*), + void *pDelArg ){ - return bindText(pStmt, i, zData, nData, xDel, SQLITE4_UTF16NATIVE); + return bindText(pStmt, i, zData, nData, xDel, pDelArg, SQLITE4_UTF16NATIVE); } #endif /* SQLITE4_OMIT_UTF16 */ int sqlite4_bind_value(sqlite4_stmt *pStmt, int i, const sqlite4_value *pValue){ int rc; switch( pValue->type ){ @@ -1115,16 +1127,17 @@ } case SQLITE4_BLOB: { if( pValue->flags & MEM_Zero ){ rc = sqlite4_bind_zeroblob(pStmt, i, pValue->u.nZero); }else{ - rc = sqlite4_bind_blob(pStmt, i, pValue->z, pValue->n,SQLITE4_TRANSIENT); + rc = sqlite4_bind_blob(pStmt, i, pValue->z, pValue->n, + SQLITE4_TRANSIENT, 0); } break; } case SQLITE4_TEXT: { - rc = bindText(pStmt,i, pValue->z, pValue->n, SQLITE4_TRANSIENT, + rc = bindText(pStmt,i, pValue->z, pValue->n, SQLITE4_TRANSIENT, 0, pValue->enc); break; } default: { rc = sqlite4_bind_null(pStmt, i); Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -16,21 +16,10 @@ */ #include "sqliteInt.h" #include "vdbeInt.h" - -/* -** When debugging the code generator in a symbolic debugger, one can -** set the sqlite4VdbeAddopTrace to 1 and all opcodes will be printed -** as they are added to the instruction stream. -*/ -#ifdef SQLITE4_DEBUG -int sqlite4VdbeAddopTrace = 0; -#endif - - /* ** Create a new virtual database engine. */ Vdbe *sqlite4VdbeCreate(sqlite4 *db){ Vdbe *p; @@ -150,11 +139,13 @@ pOp->p3 = p3; pOp->p4.p = 0; pOp->p4type = P4_NOTUSED; #ifdef SQLITE4_DEBUG pOp->zComment = 0; - if( sqlite4VdbeAddopTrace ) sqlite4VdbePrintOp(0, i, &p->aOp[i]); + if( p->db->flags & SQLITE4_VdbeAddopTrace ){ + sqlite4VdbePrintOp(0, i, &p->aOp[i]); + } #endif #ifdef VDBE_PROFILE pOp->cycles = 0; pOp->cnt = 0; #endif @@ -503,11 +494,11 @@ pOut->p4type = P4_NOTUSED; pOut->p4.p = 0; pOut->p5 = 0; #ifdef SQLITE4_DEBUG pOut->zComment = 0; - if( sqlite4VdbeAddopTrace ){ + if( p->db->flags & SQLITE4_VdbeAddopTrace ){ sqlite4VdbePrintOp(0, i+addr, &p->aOp[i+addr]); } #endif } p->nOp += nOp; @@ -625,10 +616,14 @@ break; } case P4_VTAB : { if( db->pnBytesFreed==0 ) sqlite4VtabUnlock((VTable *)p4); break; + } + case P4_FTS5INFO : { + sqlite4Fts5FreeInfo(db, (Fts5Info *)p4); + break; } } } } @@ -1189,11 +1184,11 @@ return SQLITE4_ERROR; } pMem->flags = MEM_Dyn|MEM_Str|MEM_Term; z = displayP4(pOp, pMem->z, 32); if( z!=pMem->z ){ - sqlite4VdbeMemSetStr(pMem, z, -1, SQLITE4_UTF8, 0); + sqlite4VdbeMemSetStr(pMem, z, -1, SQLITE4_UTF8, 0, 0); }else{ assert( pMem->z!=0 ); pMem->n = sqlite4Strlen30(pMem->z); pMem->enc = SQLITE4_UTF8; } @@ -1494,10 +1489,11 @@ */ void sqlite4VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ if( pCx==0 ){ return; } + sqlite4Fts5Close(pCx->pFts); if( pCx->pKVCur ){ sqlite4KVCursorClose(pCx->pKVCur); } if( pCx->pTmpKV ){ sqlite4KVStoreClose(pCx->pTmpKV); @@ -1631,23 +1627,25 @@ int sqlite4VdbeSetColName( Vdbe *p, /* Vdbe being configured */ int idx, /* Index of column zName applies to */ int var, /* One of the COLNAME_* constants */ const char *zName, /* Pointer to buffer containing name */ - void (*xDel)(void*) /* Memory management strategy for zName */ + void (*xDel)(void*,void*) /* Memory management strategy for zName */ ){ int rc; Mem *pColName; assert( idxnResColumn ); assert( vardb->mallocFailed ){ assert( !zName || xDel!=SQLITE4_DYNAMIC ); return SQLITE4_NOMEM; } assert( p->aColName!=0 ); pColName = &(p->aColName[idx+var*p->nResColumn]); - rc = sqlite4VdbeMemSetStr(pColName, zName, -1, SQLITE4_UTF8, xDel); + rc = sqlite4VdbeMemSetStr(pColName, zName, -1, SQLITE4_UTF8, xDel, 0); assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 ); return rc; } /* @@ -1969,11 +1967,12 @@ sqlite4 *db = p->db; int rc = p->rc; if( p->zErrMsg ){ u8 mallocFailed = db->mallocFailed; sqlite4BeginBenignMalloc(db->pEnv); - sqlite4ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE4_UTF8, SQLITE4_TRANSIENT); + sqlite4ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE4_UTF8, + SQLITE4_TRANSIENT, 0); sqlite4EndBenignMalloc(db->pEnv); db->mallocFailed = mallocFailed; db->errCode = rc; }else{ sqlite4Error(db, rc, 0); @@ -2016,11 +2015,12 @@ /* The expired flag was set on the VDBE before the first call ** to sqlite4_step(). For consistency (since sqlite4_step() was ** called), set the database error in this case as well. */ sqlite4Error(db, p->rc, 0); - sqlite4ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE4_UTF8, SQLITE4_TRANSIENT); + sqlite4ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE4_UTF8, + SQLITE4_TRANSIENT, 0); sqlite4DbFree(db, p->zErrMsg); p->zErrMsg = 0; } /* Reclaim all memory used by the VDBE @@ -2078,11 +2078,11 @@ int i; for(i=0; inAux; i++){ struct AuxData *pAux = &pVdbeFunc->apAux[i]; if( (i>31 || !(mask&(((u32)1)<pAux ){ if( pAux->xDelete ){ - pAux->xDelete(pAux->pAux); + pAux->xDelete(pAux->pDeleteArg, pAux->pAux); } pAux->pAux = 0; } } } Index: src/vdbecodec.c ================================================================== --- src/vdbecodec.c +++ src/vdbecodec.c @@ -129,34 +129,41 @@ if( n!=size ) return SQLITE4_CORRUPT; r = (double)x; if( e&1 ) r = -r; if( e&2 ){ e = -(e>>2); - while( e<=-10 ){ r /= 1.0e10; e += 10; } - while( e<0 ){ r /= 10.0; e++; } + if( e==0 ){ + r *= 1e+300*1e+300; + }else{ + while( e<=-10 ){ r /= 1.0e10; e += 10; } + while( e<0 ){ r /= 10.0; e++; } + } }else{ e = e>>2; while( e>=10 ){ r *= 1.0e10; e -= 10; } while( e>0 ){ r *= 10.0; e--; } } sqlite4VdbeMemSetDouble(pOut, r); }else if( cclass==0 ){ if( size==0 ){ - sqlite4VdbeMemSetStr(pOut, "", 0, SQLITE4_UTF8, SQLITE4_TRANSIENT); + sqlite4VdbeMemSetStr(pOut, "", 0, SQLITE4_UTF8, SQLITE4_TRANSIENT, 0); }else if( p->a[ofst]>0x02 ){ sqlite4VdbeMemSetStr(pOut, (char*)(p->a+ofst), size, - SQLITE4_UTF8, SQLITE4_TRANSIENT); + SQLITE4_UTF8, SQLITE4_TRANSIENT, 0); }else{ - static const u8 enc[] = { SQLITE4_UTF8, SQLITE4_UTF16LE, SQLITE4_UTF16BE }; + static const u8 enc[] = {SQLITE4_UTF8,SQLITE4_UTF16LE,SQLITE4_UTF16BE }; sqlite4VdbeMemSetStr(pOut, (char*)(p->a+ofst+1), size-1, - enc[p->a[ofst]], SQLITE4_TRANSIENT); + enc[p->a[ofst]], SQLITE4_TRANSIENT, 0); } }else{ - sqlite4VdbeMemSetStr(pOut, (char*)(p->a+ofst), size, 0, SQLITE4_TRANSIENT); + sqlite4VdbeMemSetStr(pOut, (char*)(p->a+ofst), size, 0, + SQLITE4_TRANSIENT, 0); } } - if( inOut<=p->nAlloc ); if( p->nOut+needed>p->nAlloc ){ u8 *aNew; p->nAlloc = p->nAlloc + needed + 10; aNew = sqlite4DbRealloc(p->db, p->aOut, p->nAlloc); if( aNew==0 ){ @@ -594,12 +602,12 @@ memcpy(p->aOut+p->nOut, pEnc->z, pEnc->n); p->nOut += pEnc->n; }else{ int nSpc = p->nAlloc-p->nOut; n = pColl->xMkKey(pColl->pUser, pEnc->n, pEnc->z, nSpc, p->aOut+p->nOut); - if( n>nSpc ){ - if( enlargeEncoderAllocation(p, n) ) return SQLITE4_NOMEM; + if( n+1>nSpc ){ + if( enlargeEncoderAllocation(p, n+1) ) return SQLITE4_NOMEM; n = pColl->xMkKey(pColl->pUser, pEnc->n, pEnc->z, n, p->aOut+p->nOut); } p->nOut += n; } p->aOut[p->nOut++] = 0x00; @@ -634,10 +642,11 @@ p->aOut[p->nOut++] = 0x00; } if( sortOrder==SQLITE4_SO_DESC ){ for(i=iStart; inOut; i++) p->aOut[i] ^= 0xff; } + assert( p->nOut<=p->nAlloc ); return SQLITE4_OK; } /* ** Variables aKey/nKey contain an encoded index key. This function returns Index: src/vdbemem.c ================================================================== --- src/vdbemem.c +++ src/vdbemem.c @@ -91,11 +91,11 @@ if( pMem->z && preserve && pMem->zMalloc && pMem->z!=pMem->zMalloc ){ memcpy(pMem->zMalloc, pMem->z, pMem->n); } if( pMem->flags&MEM_Dyn && pMem->xDel ){ assert( pMem->xDel!=SQLITE4_DYNAMIC ); - pMem->xDel((void *)(pMem->z)); + pMem->xDel(pMem->pDelArg, (void *)(pMem->z)); } pMem->z = pMem->zMalloc; if( pMem->z==0 ){ pMem->flags = MEM_Null; @@ -242,11 +242,11 @@ assert( (p->flags & MEM_Agg)==0 ); sqlite4VdbeMemRelease(p); }else if( p->flags&MEM_Dyn && p->xDel ){ assert( (p->flags&MEM_RowSet)==0 ); assert( p->xDel!=SQLITE4_DYNAMIC ); - p->xDel((void *)p->z); + p->xDel(p->pDelArg, (void *)p->z); p->xDel = 0; }else if( p->flags&MEM_RowSet ){ sqlite4RowSetClear(p->u.pRowSet); }else if( p->flags&MEM_Frame ){ sqlite4VdbeMemSetNull(p); @@ -634,15 +634,16 @@ ** stored without allocating memory, then it is. If a memory allocation ** is required to store the string, then value of pMem is unchanged. In ** either case, SQLITE4_TOOBIG is returned. */ int sqlite4VdbeMemSetStr( - Mem *pMem, /* Memory cell to set to string value */ - const char *z, /* String pointer */ - int n, /* Bytes in string, or negative */ - u8 enc, /* Encoding of z. 0 for BLOBs */ - void (*xDel)(void*) /* Destructor function */ + Mem *pMem, /* Memory cell to set to string value */ + const char *z, /* String pointer */ + int n, /* Bytes in string, or negative */ + u8 enc, /* Encoding of z. 0 for BLOBs */ + void (*xDel)(void*,void*),/* Destructor function */ + void *pDelArg /* First argument to xDel() */ ){ int nByte = n; /* New value for pMem->n */ int iLimit; /* Maximum allowed string or blob size */ u16 flags = 0; /* New value for pMem->flags */ @@ -693,10 +694,11 @@ pMem->xDel = 0; }else{ sqlite4VdbeMemRelease(pMem); pMem->z = (char *)z; pMem->xDel = xDel; + pMem->pDelArg = pDelArg; flags |= ((xDel==SQLITE4_STATIC)?MEM_Static:MEM_Dyn); } pMem->n = nByte; pMem->flags = flags; @@ -949,11 +951,11 @@ if( ExprHasProperty(pExpr, EP_IntValue) ){ sqlite4VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); }else{ zVal = sqlite4MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); if( zVal==0 ) goto no_mem; - sqlite4ValueSetStr(pVal, -1, zVal, SQLITE4_UTF8, SQLITE4_DYNAMIC); + sqlite4ValueSetStr(pVal, -1, zVal, SQLITE4_UTF8, SQLITE4_DYNAMIC, 0); if( op==TK_FLOAT ) pVal->type = SQLITE4_FLOAT; } if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE4_AFF_NONE ){ sqlite4ValueApplyAffinity(pVal, SQLITE4_AFF_NUMERIC, SQLITE4_UTF8); }else{ @@ -990,11 +992,11 @@ if( !pVal ) goto no_mem; zVal = &pExpr->u.zToken[2]; nVal = sqlite4Strlen30(zVal)-1; assert( zVal[nVal]=='\'' ); sqlite4VdbeMemSetStr(pVal, sqlite4HexToBlob(db, zVal, nVal), nVal/2, - 0, SQLITE4_DYNAMIC); + 0, SQLITE4_DYNAMIC, 0); } #endif if( pVal ){ sqlite4VdbeMemStoreType(pVal); @@ -1012,17 +1014,18 @@ /* ** Change the string value of an sqlite4_value object */ void sqlite4ValueSetStr( - sqlite4_value *v, /* Value to be set */ - int n, /* Length of string z */ - const void *z, /* Text of the new string */ - u8 enc, /* Encoding to use */ - void (*xDel)(void*) /* Destructor for the string */ + sqlite4_value *v, /* Value to be set */ + int n, /* Length of string z */ + const void *z, /* Text of the new string */ + u8 enc, /* Encoding to use */ + void (*xDel)(void*,void*), /* Destructor for the string */ + void *pDelArg /* First argument to xDel() */ ){ - if( v ) sqlite4VdbeMemSetStr((Mem *)v, z, n, enc, xDel); + if( v ) sqlite4VdbeMemSetStr((Mem *)v, z, n, enc, xDel, pDelArg); } /* ** Free an sqlite4_value object */ Index: src/vdbetrace.c ================================================================== --- src/vdbetrace.c +++ src/vdbetrace.c @@ -128,11 +128,11 @@ u8 enc = ENC(db); if( enc!=SQLITE4_UTF8 ){ Mem utf8; memset(&utf8, 0, sizeof(utf8)); utf8.db = db; - sqlite4VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE4_STATIC); + sqlite4VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE4_STATIC, 0); sqlite4VdbeChangeEncoding(&utf8, SQLITE4_UTF8); sqlite4XPrintf(&out, "'%.*q'", utf8.n, utf8.z); sqlite4VdbeMemRelease(&utf8); }else #endif Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -770,11 +770,11 @@ ExprList *pList; if( pExpr->op!=TK_FUNCTION ){ return 0; } - if( sqlite4StrICmp(pExpr->u.zToken,"match")!=0 ){ + if( sqlite4_stricmp(pExpr->u.zToken,"match")!=0 ){ return 0; } pList = pExpr->x.pList; if( pList->nExpr!=2 ){ return 0; @@ -1456,11 +1456,11 @@ && p->iColumn==pIdx->aiColumn[iCol] && p->iTable==iBase ){ CollSeq *pColl = sqlite4ExprCollSeq(pParse, p); assert( pColl || p->iColumn==-1 ); - if( 0==pColl || 0==sqlite4StrICmp(pColl->zName, zColl) ){ + if( 0==pColl || 0==sqlite4_stricmp(pColl->zName, zColl) ){ return i; } } } @@ -1660,11 +1660,11 @@ Index *pPk; Table *pTab; if( !pOrderBy ) return 0; if( wsFlags & WHERE_COLUMN_IN ) return 0; - if( pIdx->bUnordered ) return 0; + if( pIdx->fIndex & IDX_Unordered ) return 0; pTab = pIdx->pTable; pPk = sqlite4FindPrimaryKey(pTab, 0); nTerm = pOrderBy->nExpr; nIdxCol = pIdx->nColumn + (pIdx==pPk ? 0 : pPk->nColumn); @@ -2861,10 +2861,58 @@ WHERETRACE(("IN row estimate: est=%g\n", nRowEst)); } return rc; } #endif /* defined(SQLITE4_ENABLE_STAT3) */ + +/* +** Try to find a MATCH expression that constrains the pTabItem table in the +** WHERE clause. If one exists, set *piTerm to the index in the pWC->a[] array +** and return non-zero. If no such expression exists, return 0. +*/ +static int findMatchExpr( + Parse *pParse, + WhereClause *pWC, + SrcListItem *pTabItem, + int *piTerm +){ + int i; + int iCsr = pTabItem->iCursor; + + for(i=0; inTerm; i++){ + Expr *pMatch = pWC->a[i].pExpr; + if( pMatch->iTable==iCsr && pMatch->op==TK_MATCH ) break; + } + if( i==pWC->nTerm ) return 0; + + *piTerm = i; + return 1; +} + +static int bestMatchIdx( + Parse *pParse, + WhereClause *pWC, + SrcListItem *pTabItem, + Bitmask notReady, + WhereCost *pCost +){ + int iTerm; + + if( 0==findMatchExpr(pParse, pWC, pTabItem, &iTerm) ) return 0; + + /* Check that the MATCH expression is not composed using values from any + ** tables that are not ready. If it does, return 0. */ + if( notReady & pWC->a[iTerm].prereqAll ) return 0; + + pCost->used = pWC->a[iTerm].prereqAll; + pCost->rCost = 1.0; + pCost->plan.wsFlags = WHERE_INDEXED; + pCost->plan.nEq = 0; + pCost->plan.nRow = 10; + pCost->plan.u.pIdx = pWC->a[iTerm].pExpr->pIdx; + return 1; +} /* ** Find the best query plan for accessing a particular table. Write the ** best query plan and its cost into the WhereCost object supplied as the ** last parameter. @@ -3022,10 +3070,12 @@ WhereTerm *pTerm; /* A single term of the WHERE clause */ #ifdef SQLITE4_ENABLE_STAT3 WhereTerm *pFirstTerm = 0; /* First term matching the index */ #endif int nCol = pProbe->nColumn; /* Total columns in index record */ + + if( pProbe->eIndexType==SQLITE4_INDEX_FTS5 ) continue; /* Unless pProbe is the primary key index, then the encoded PK column ** values are at the end of each record. Set variable nCol to the total ** number of columns encoded into each index record, including the PK ** columns. */ @@ -3072,11 +3122,11 @@ testcase( wsFlags & WHERE_COLUMN_IN ); testcase( wsFlags & WHERE_COLUMN_NULL ); if( (wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){ wsFlags |= WHERE_UNIQUE; } - }else if( pProbe->bUnordered==0 ){ + }else if( (pProbe->fIndex & IDX_Unordered)==0 ){ int j = idxColumnNumber(pProbe, pPk, nEq); if( findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE|WO_GT|WO_GE, pProbe) ){ WhereTerm *pTop = findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE, pProbe); WhereTerm *pBtm = findTerm(pWC, iCur, j, notReady, WO_GT|WO_GE, pProbe); whereRangeScanEst(pParse, pProbe, nEq, pBtm, pTop, &rangeDiv); @@ -3876,10 +3926,31 @@ pLevel->iLeftJoin = ++pParse->nMem; sqlite4VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); VdbeComment((v, "init LEFT JOIN no-match flag")); } + if( (pLevel->plan.wsFlags & WHERE_INDEXED) + && (pLevel->plan.u.pIdx->eIndexType==SQLITE4_INDEX_FTS5) + ){ + /* Case -1: An FTS query */ + int iTerm; + int rMatch; + int rFree; + findMatchExpr(pParse, pWC, pTabItem, &iTerm); + + rMatch = sqlite4ExprCodeTemp(pParse, pWC->a[iTerm].pExpr->pRight, &rFree); + pWC->a[iTerm].wtFlags |= TERM_CODED; + sqlite4Fts5CodeQuery(pParse, + pLevel->plan.u.pIdx, pLevel->iIdxCur, addrBrk, rMatch + ); + sqlite4ReleaseTempReg(pParse, rFree); + + pLevel->p2 = sqlite4VdbeCurrentAddr(v); + sqlite4VdbeAddOp3(v, OP_SeekPk, iCur, 0, pLevel->iIdxCur); + pLevel->op = OP_FtsNext; + pLevel->p1 = pLevel->iIdxCur; + }else #ifndef SQLITE4_OMIT_VIRTUALTABLE if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){ /* Case 0: The table is a virtual-table. Use the VFilter and VNext ** to access the data. */ @@ -4164,11 +4235,11 @@ ** If it is, jump to the next iteration of the loop. */ r1 = sqlite4GetTempReg(pParse); testcase( pLevel->plan.wsFlags & WHERE_BTM_LIMIT ); testcase( pLevel->plan.wsFlags & WHERE_TOP_LIMIT ); if( (pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){ - sqlite4VdbeAddOp3(v, OP_Column, iCur, iIneq, r1); + sqlite4ExprCodeGetColumnOfTable(v, pIdx->pTable, iCur, iIneq, r1); sqlite4VdbeAddOp2(v, OP_IsNull, r1, addrCont); } sqlite4ReleaseTempReg(pParse, r1); /* Record the instruction used to terminate the loop. Disable @@ -4783,10 +4854,14 @@ if( pTabItem->pIndex==0 ) nUnconstrained++; WHERETRACE(("=== trying table %d with isOptimal=%d ===\n", j, isOptimal)); assert( pTabItem->pTab ); + + if( bestMatchIdx(pParse, pWC, pTabItem, notReady, &sCost) ){ + /* no-op */ + }else #ifndef SQLITE4_OMIT_VIRTUALTABLE if( IsVirtual(pTabItem->pTab) ){ sqlite4_index_info **pp = &pWInfo->a[j].pIdxInfo; bestVirtualIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, &sCost, pp); @@ -4794,10 +4869,11 @@ #endif { bestKVIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, pDist, &sCost); } + assert( isOptimal || (sCost.used¬Ready)==0 ); /* If an INDEXED BY clause is present, then the plan must use that ** index if it uses any index at all */ assert( pTabItem->pIndex==0 @@ -4955,11 +5031,11 @@ #endif if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ Index *pIx = pLevel->plan.u.pIdx; if( pIx->eIndexType==SQLITE4_INDEX_PRIMARYKEY ){ pLevel->iIdxCur = pTabItem->iCursor; - }else{ + }else if( pIx->eIndexType!=SQLITE4_INDEX_FTS5 ){ KeyInfo *pKey = sqlite4IndexKeyinfo(pParse, pIx); int iIdxCur = pLevel->iIdxCur; assert( pIx->pSchema==pTab->pSchema ); assert( iIdxCur>=0 ); sqlite4VdbeAddOp4(v, OP_OpenRead, iIdxCur, pIx->tnum, iDb, @@ -5151,13 +5227,31 @@ pOp->p1 = pLevel->iIdxCur; } pOp++; } } + + if( (pLevel->plan.wsFlags & WHERE_INDEXED) + && (pLevel->plan.u.pIdx->eIndexType==SQLITE4_INDEX_FTS5) + ){ + VdbeOp *pOp; + VdbeOp *pEnd; + + assert( pLevel->iTabCur!=pLevel->iIdxCur ); + pOp = sqlite4VdbeGetOp(v, pWInfo->iTop); + pEnd = &pOp[sqlite4VdbeCurrentAddr(v) - pWInfo->iTop]; + + while( pOpp1==pLevel->iTabCur && pOp->opcode==OP_Mifunction ){ + pOp->p1 = pLevel->iIdxCur; + } + pOp++; + } + } } /* Final cleanup */ pParse->nQueryLoop = pWInfo->savedNQueryLoop; whereInfoFree(db, pWInfo); return; } Index: test/badutf.test ================================================================== --- test/badutf.test +++ test/badutf.test @@ -11,11 +11,10 @@ # This file implements regression tests for SQLite library. # # This file checks to make sure SQLite is able to gracefully # handle malformed UTF-8. # -# $Id: badutf.test,v 1.2 2007/09/12 17:01:45 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl do_test badutf-1.1 { @@ -25,23 +24,23 @@ do_test badutf-1.2 { sqlite4_exec db {SELECT hex('%81') AS x} } {0 {x 81}} do_test badutf-1.3 { sqlite4_exec db {SELECT hex('%bf') AS x} -} {0 {x BF}} +} {0 {x bf}} do_test badutf-1.4 { sqlite4_exec db {SELECT hex('%c0') AS x} -} {0 {x C0}} +} {0 {x c0}} do_test badutf-1.5 { sqlite4_exec db {SELECT hex('%e0') AS x} -} {0 {x E0}} +} {0 {x e0}} do_test badutf-1.6 { sqlite4_exec db {SELECT hex('%f0') AS x} -} {0 {x F0}} +} {0 {x f0}} do_test badutf-1.7 { sqlite4_exec db {SELECT hex('%ff') AS x} -} {0 {x FF}} +} {0 {x ff}} sqlite4 db2 {} ifcapable utf16 { do_test badutf-1.10 { db2 eval {PRAGMA encoding=UTF16be} @@ -50,35 +49,35 @@ do_test badutf-1.11 { sqlite4_exec db2 {SELECT hex('%81') AS x} } {0 {x 0081}} do_test badutf-1.12 { sqlite4_exec db2 {SELECT hex('%bf') AS x} - } {0 {x 00BF}} + } {0 {x 00bf}} do_test badutf-1.13 { sqlite4_exec db2 {SELECT hex('%c0') AS x} - } {0 {x FFFD}} + } {0 {x fffd}} do_test badutf-1.14 { sqlite4_exec db2 {SELECT hex('%c1') AS x} - } {0 {x FFFD}} + } {0 {x fffd}} do_test badutf-1.15 { sqlite4_exec db2 {SELECT hex('%c0%bf') AS x} - } {0 {x FFFD}} + } {0 {x fffd}} do_test badutf-1.16 { sqlite4_exec db2 {SELECT hex('%c1%bf') AS x} - } {0 {x FFFD}} + } {0 {x fffd}} do_test badutf-1.17 { sqlite4_exec db2 {SELECT hex('%c3%bf') AS x} - } {0 {x 00FF}} + } {0 {x 00ff}} do_test badutf-1.18 { sqlite4_exec db2 {SELECT hex('%e0') AS x} - } {0 {x FFFD}} + } {0 {x fffd}} do_test badutf-1.19 { sqlite4_exec db2 {SELECT hex('%f0') AS x} - } {0 {x FFFD}} + } {0 {x fffd}} do_test badutf-1.20 { sqlite4_exec db2 {SELECT hex('%ff') AS x} - } {0 {x FFFD}} + } {0 {x fffd}} } ifcapable bloblit { do_test badutf-2.1 { @@ -117,27 +116,27 @@ sqlite4_exec db {SELECT length('%80%80%80%80%80%f0%80%80%80%ff') AS x} } {0 {x 7}} do_test badutf-4.1 { sqlite4_exec db {SELECT hex(trim('%80%80%80%f0%80%80%80%ff','%80%ff')) AS x} -} {0 {x F0}} +} {0 {x f0}} do_test badutf-4.2 { sqlite4_exec db {SELECT hex(ltrim('%80%80%80%f0%80%80%80%ff','%80%ff')) AS x} -} {0 {x F0808080FF}} +} {0 {x f0808080ff}} do_test badutf-4.3 { sqlite4_exec db {SELECT hex(rtrim('%80%80%80%f0%80%80%80%ff','%80%ff')) AS x} -} {0 {x 808080F0}} +} {0 {x 808080f0}} do_test badutf-4.4 { sqlite4_exec db {SELECT hex(trim('%80%80%80%f0%80%80%80%ff','%ff%80')) AS x} -} {0 {x 808080F0808080FF}} +} {0 {x 808080f0808080ff}} do_test badutf-4.5 { sqlite4_exec db {SELECT hex(trim('%ff%80%80%f0%80%80%80%ff','%ff%80')) AS x} -} {0 {x 80F0808080FF}} +} {0 {x 80f0808080ff}} do_test badutf-4.6 { sqlite4_exec db {SELECT hex(trim('%ff%80%f0%80%80%80%ff','%ff%80')) AS x} -} {0 {x F0808080FF}} +} {0 {x f0808080ff}} do_test badutf-4.7 { sqlite4_exec db {SELECT hex(trim('%ff%80%f0%80%80%80%ff','%ff%80%80')) AS x} -} {0 {x FF80F0808080FF}} +} {0 {x ff80f0808080ff}} db2 close finish_test Index: test/bind.test ================================================================== --- test/bind.test +++ test/bind.test @@ -301,19 +301,19 @@ } {hello hello hello} set enc [db eval {PRAGMA encoding}] if {$enc=="UTF-8" || $enc==""} { do_test bind-6.5 { execsql {SELECT hex(a), hex(b), hex(c) FROM t1} - } {68656C6C6F00746865726500 68656C6C6F007468657265 68656C6C6F} + } {68656c6c6f00746865726500 68656c6c6f007468657265 68656c6c6f} } elseif {$enc=="UTF-16le"} { do_test bind-6.5 { execsql {SELECT hex(a), hex(b), hex(c) FROM t1} - } {680065006C006C006F000000740068006500720065000000 680065006C006C006F00000074006800650072006500 680065006C006C006F00} + } {680065006c006c006f000000740068006500720065000000 680065006c006c006f00000074006800650072006500 680065006c006c006f00} } elseif {$enc=="UTF-16be"} { do_test bind-6.5 { execsql {SELECT hex(a), hex(b), hex(c) FROM t1} - } {00680065006C006C006F0000007400680065007200650000 00680065006C006C006F000000740068006500720065 00680065006C006C006F} + } {00680065006c006c006f0000007400680065007200650000 00680065006c006c006f000000740068006500720065 00680065006c006c006f} } else { do_test bind-6.5 { set "Unknown database encoding: $::enc" } {} } @@ -350,19 +350,19 @@ execsql {SELECT * FROM t1} } {hi hi hi} if {$enc=="UTF-8"} { do_test bind-7.4 { execsql {SELECT hex(a), hex(b), hex(c) FROM t1} - } {68690079616C6C00 68690079616C6C 6869} + } {68690079616c6c00 68690079616c6c 6869} } elseif {$enc=="UTF-16le"} { do_test bind-7.4 { execsql {SELECT hex(a), hex(b), hex(c) FROM t1} - } {680069000000790061006C006C000000 680069000000790061006C006C00 68006900} + } {680069000000790061006c006c000000 680069000000790061006c006c00 68006900} } elseif {$enc=="UTF-16be"} { do_test bind-7.4 { execsql {SELECT hex(a), hex(b), hex(c) FROM t1} - } {00680069000000790061006C006C0000 00680069000000790061006C006C 00680069} + } {00680069000000790061006c006c0000 00680069000000790061006c006c 00680069} } do_test bind-7.5 { execsql {SELECT typeof(a), typeof(b), typeof(c) FROM t1} } {text text text} } @@ -624,17 +624,17 @@ sqlite4_finalize $VM execsql { SELECT typeof(x), length(x), quote(x), length(cast(x AS BLOB)), quote(cast(x AS BLOB)) FROM t3 } - } {text 3 'abc' 10 X'6162630078797A007071'} + } {text 3 'abc' 10 x'6162630078797a007071'} do_test bind-12.2 { sqlite4_create_function $DB execsql { SELECT quote(cast(x_coalesce(x) AS blob)) FROM t3 } - } {X'6162630078797A007071'} + } {x'6162630078797a007071'} } # Test the operation of sqlite4_clear_bindings # do_test bind-13.1 { Index: test/csr1.test ================================================================== --- test/csr1.test +++ test/csr1.test @@ -75,12 +75,12 @@ populate_db_2 do_execsql_test 3.1 { BEGIN; INSERT INTO t1 VALUES(10, randstr(910, 910)); } -do_test 3.2 { sqlite4_lsm_config db main write-buffer } [expr 2*1024*1024] -do_test 3.3 { sqlite4_lsm_config db main write-buffer 4096 } 4096 +do_test 3.2 { sqlite4_lsm_config db main autoflush } [expr 1*1024*1024] +do_test 3.3 { sqlite4_lsm_config db main autoflush 4096 } 4096 do_test 3.4 { set res [list] db eval { SELECT a, length(b) AS l FROM t1 } { lappend res $a $l Index: test/ctime.test ================================================================== --- test/ctime.test +++ test/ctime.test @@ -70,15 +70,10 @@ } {0 1} do_test ctime-1.4.2 { catchsql { SELECT sqlite_compileoption_used('THREADSAFE'); } -} {0 1} -do_test ctime-1.4.3 { - catchsql { - SELECT sqlite_compileoption_used("THREADSAFE"); - } } {0 1} do_test ctime-1.5 { set ans1 [ catchsql { SELECT sqlite_compileoption_used('THREADSAFE=0'); @@ -120,15 +115,10 @@ do_test ctime-2.1.2 { catchsql { SELECT sqlite_compileoption_used(NULL); } } {0 {{}}} -do_test ctime-2.1.3 { - catchsql { - SELECT sqlite_compileoption_used(""); - } -} {0 0} do_test ctime-2.1.4 { catchsql { SELECT sqlite_compileoption_used(''); } } {0 0} ADDED test/fts5create.test Index: test/fts5create.test ================================================================== --- /dev/null +++ test/fts5create.test @@ -0,0 +1,99 @@ +# 2012 December 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts5create + +do_execsql_test 1.1 { + CREATE TABLE t1(a, b, c); +} + +do_execsql_test 1.2 { + CREATE INDEX ft1 ON t1 USING fts5(); + SELECT * FROM sqlite_master; +} { + table t1 t1 2 "CREATE TABLE t1(a, b, c)" + index ft1 t1 3 "CREATE INDEX ft1 ON t1 USING fts5()" +} + +do_execsql_test 1.3 { + DROP INDEX ft1; + SELECT * FROM sqlite_master; +} { + table t1 t1 2 "CREATE TABLE t1(a, b, c)" +} + +do_execsql_test 1.4 { + CREATE INDEX ft1 ON t1 USING fts5(); + SELECT * FROM sqlite_master; +} { + table t1 t1 2 "CREATE TABLE t1(a, b, c)" + index ft1 t1 3 "CREATE INDEX ft1 ON t1 USING fts5()" +} + +do_execsql_test 1.5 { + DROP INDEX ft1; + SELECT * FROM sqlite_master; +} { + table t1 t1 2 "CREATE TABLE t1(a, b, c)" +} + +do_catchsql_test 1.6 { + CREATE UNIQUE INDEX ft2 ON t1 USING fts5(); +} {1 {USING fts5 index may not be UNIQUE}} + +#------------------------------------------------------------------------- +# +reset_db + +do_execsql_test 2.1 { + CREATE TABLE t2(x, y); + CREATE INDEX fulltext ON t2 USING fts5(tokenizer=simple); + SELECT * FROM sqlite_master; +} { + table t2 t2 2 {CREATE TABLE t2(x, y)} + index fulltext t2 3 {CREATE INDEX fulltext ON t2 USING fts5(tokenizer=simple)} +} + +do_catchsql_test 2.2 { + DROP INDEX fulltext; + CREATE INDEX ft ON t2 USING fts5(tukenizer=simple); +} {1 {unrecognized argument: "tukenizer"}} + +do_catchsql_test 2.3 { + CREATE INDEX ft ON t2 USING fts5("a b c"); +} {1 {unrecognized argument: "a b c"}} + +do_catchsql_test 2.4 { + CREATE INDEX ft ON t2 USING fts5(tokenizer="nosuch"); +} {1 {no such tokenizer: "nosuch"}} + +#------------------------------------------------------------------------- +# +reset_db + +do_execsql_test 3.1 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a)); + INSERT INTO t1 VALUES(1, 'a b c d', 'e f g h'); + INSERT INTO t1 VALUES(2, 'e f g h', 'a b c d'); +} + +do_execsql_test 3.2 { + CREATE INDEX ft ON t1 USING fts5(); + PRAGMA fts_check(ft); +} {ok} + + +finish_test + + ADDED test/fts5expr1.test Index: test/fts5expr1.test ================================================================== --- /dev/null +++ test/fts5expr1.test @@ -0,0 +1,74 @@ +# 2012 December 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts5expr1 + +foreach {tn expr res} { + 1 { abc } {"abc"} + 2 { abc AND def } {"abc" AND "def"} + 3 { (abc) } {"abc"} + 4 { (((abc))) } {"abc"} + 5 { one OR two AND three } {"one" OR ("two" AND "three")} + 6 { one AND two OR three } {("one" AND "two") OR "three"} + + 7 { "abc def" } {"abc"+"def"} + 8 { abc+def } {"abc"+"def"} + 9 { "abc def" + ghi } {"abc"+"def"+"ghi"} + + 10 { one AND two OR three } {("one" AND "two") OR "three"} + 11 { one AND (two OR three) } {"one" AND ("two" OR "three")} + + 12 { "abc""def" } {"abc"+"def"} + 13 { "abc" "def" } {"abc" AND "def"} + 14 { "abc" + "def" } {"abc"+"def"} + + 15 { a NOT b AND c OR d } {(("a" NOT "b") AND "c") OR "d"} + 16 { a OR b AND c NOT d } {"a" OR ("b" AND ("c" NOT "d"))} + 17 { (a OR b) AND c NOT d } {("a" OR "b") AND ("c" NOT "d")} + + 18 { a NEAR/10 b } {"a" NEAR/10 "b"} + 19 { a NEAR b } {"a" NEAR/10 "b"} + 20 { a AND b OR c + d NEAR/52 e } {("a" AND "b") OR ("c"+"d" NEAR/52 "e")} + +} { + do_execsql_test 1.$tn { + SELECT fts5_parse_expr('simple', $expr, '') + } [list [string trim $res]] +} + +do_execsql_test 2.0 { + CREATE TABLE t1(a, b, c); +} + +foreach {tn expr res} { + 1 { a : abc } {"a":"abc"} + 2 { b : abc + def} {"b":"abc"+"def"} +} { + do_execsql_test 2.$tn { + SELECT fts5_parse_expr('simple', $expr, 't1') + } [list [string trim $res]] +} + +breakpoint +foreach {tn expr res} { + 1 { abc* } {"abc"*} +} { + do_execsql_test 3.$tn { + SELECT fts5_parse_expr('simple', $expr, 't1') + } [list [string trim $res]] +} + + +finish_test ADDED test/fts5query1.test Index: test/fts5query1.test ================================================================== --- /dev/null +++ test/fts5query1.test @@ -0,0 +1,184 @@ +# 2012 December 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts5query1 + +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1 USING fts5(); +} + +foreach {tn stmt} { + 1 "INSERT INTO t1 VALUES(1, 'a b c', 'd e f')" + 2 "INSERT INTO t1 VALUES(2, 'b c e', 'A A a')" + 3 "INSERT INTO t1 VALUES(3, 'd A A', 'e c a')" + 4 "DELETE FROM t1 WHERE a=1" + 5 "DELETE FROM t1" + 6 "INSERT INTO t1 VALUES(1, 'May you do', 'good and not evil')" + 7 "INSERT INTO t1 VALUES(2, 'May you find', 'forgiveness for yourself')" + 8 "UPDATE t1 SET b = 'and forgive others' WHERE a = 2" + 9 "UPDATE t1 SET a = 4, c = 'a b c d' WHERE a = 2" +} { + do_execsql_test 1.$tn.1 $stmt + do_execsql_test 1.$tn.2 {PRAGMA fts_check(i1)} ok +} + +do_execsql_test 2.0 { + DROP TABLE t1; + CREATE TABLE t1(x PRIMARY KEY, y); + CREATE INDEX i1 ON t1 USING fts5(); + + INSERT INTO t1 VALUES(1, 'o n e'); + INSERT INTO t1 VALUES(2, 't w o'); + INSERT INTO t1 VALUES(3, 't h r e e'); + INSERT INTO t1 VALUES(4, 'f o u r'); +} + +foreach {tn stmt res} { + 1 {SELECT x FROM t1 WHERE t1 MATCH 't'} {2 3} + 2 {SELECT x FROM t1 WHERE t1 MATCH 'abc'} {} + 3 {SELECT x FROM t1 WHERE t1 MATCH 't+h'} {3} + 4 {SELECT x FROM t1 WHERE t1 MATCH 't+o'} {} +} { + do_execsql_test 2.$tn $stmt $res +} + +do_execsql_test 3.0 { + DROP TABLE t1; + CREATE TABLE t1(x PRIMARY KEY, y); + CREATE INDEX i1 ON t1 USING fts5(); + + INSERT INTO t1 VALUES(1, 'a b c d e f g h i j k l m n o p q r s t u'); + INSERT INTO t1 VALUES(2, 'a e i o u b c d f g h j k l m n p q r s t'); + INSERT INTO t1 VALUES(3, 'b c d f g h j k l m n p q r s t v w x y z'); + INSERT INTO t1 VALUES(4, 'a e i o u'); +} + +foreach {tn stmt res} { + 1 {SELECT x FROM t1 WHERE t1 MATCH 'a NEAR/5 i'} {2 4} + 2 {SELECT x FROM t1 WHERE t1 MATCH 'a NEAR/3 b'} {1} + 3 {SELECT x FROM t1 WHERE t1 MATCH 'a NEAR/2 d'} {1} + 4 {SELECT x FROM t1 WHERE t1 MATCH 'a NEAR/2 e'} {2 4} + 5 {SELECT x FROM t1 WHERE t1 MATCH 'a NEAR/3 e'} {1 2 4} + 6 {SELECT x FROM t1 WHERE t1 MATCH 'b+c NEAR/2 g+h'} {2 3} + 7 {SELECT x FROM t1 WHERE t1 MATCH 'b+c NEAR/3 g+h'} {1 2 3} + 8 {SELECT x FROM t1 WHERE t1 MATCH 'b+c NEAR/2 g+h+j'} {2 3} + 9 {SELECT x FROM t1 WHERE t1 MATCH 'b+c+d NEAR/1 g+h'} {2 3} + 10 {SELECT x FROM t1 WHERE t1 MATCH 'a AND d'} {1 2} + 11 {SELECT x FROM t1 WHERE t1 MATCH 'a OR d'} {1 2 3 4} + 12 {SELECT x FROM t1 WHERE t1 MATCH 'a NOT d'} {4} +} { + do_execsql_test 3.$tn $stmt $res +} + +do_execsql_test 4.0 { + CREATE TABLE t2(docid PRIMARY KEY, a, b, c); + CREATE INDEX i2 ON t2 USING fts5(); + INSERT INTO t2 VALUES(136895, 'qkfl my qkfl krag gw', NULL, NULL); +} + +do_execsql_test 4.1 { + SELECT docid FROM t2 WHERE t2 MATCH 'qkfl NEAR/2 gw'; +} {136895} + +do_execsql_test 6.0 { + CREATE TABLE t3(docid PRIMARY KEY, a, b, c); + CREATE INDEX i3 ON t3 USING fts5(); + INSERT INTO t3 VALUES(123, 'fix the hash table', NULL, NULL); +} +do_execsql_test 5.1 { + SELECT docid FROM t3 WHERE t3 MATCH 'h*'; +} {123} + +do_execsql_test 6.0 { + BEGIN TRANSACTION; + CREATE TABLE t4(docid PRIMARY KEY, a); + CREATE INDEX i4 ON t4 USING fts5(); + INSERT INTO "t4" VALUES(34, 'abc mnm xyz'); + INSERT INTO "t4" VALUES(50, 'abc mnm xyz'); + COMMIT; +} +do_execsql_test 6.1 { + SELECT docid FROM t4 WHERE t4 MATCH 'm*' +} {34 50} + +#------------------------------------------------------------------------- +# +do_execsql_test 7.0 { + BEGIN TRANSACTION; + CREATE TABLE t7(docid PRIMARY KEY, a, b, c); + CREATE INDEX i7 ON t7 USING fts5(); + INSERT INTO t7 VALUES(1, 'a b c', 'd e f', 'a b c'); + INSERT INTO t7 VALUES(2, 'x y z', 'a b c', 'a b c'); + COMMIT; +} + +foreach {tn expr res} { + 1 {a} {1 2} + 2 {a:a} {1} + 3 {b:a} {2} + 4 {c:a} {1 2} + 5 {a:a*} {1} +} { + do_execsql_test 7.$tn {SELECT docid FROM t7 WHERE t7 MATCH $expr} $res +} + +#------------------------------------------------------------------------- +# +do_execsql_test 8.0 { + CREATE TABLE t8(a PRIMARY KEY, b, c); + CREATE INDEX i8 ON t8 USING fts5(); + INSERT INTO t8 VALUES('one', 'a b c', 'a a a'); + INSERT INTO t8 VALUES('two', 'd e f', 'b b b'); +} + +#do_execsql_test 8.1 { +# SELECT rank(t8) FROM t8 WHERE t8 MATCH 'b a' +#} + +do_execsql_test 9.0 { + CREATE TABLE t9(a PRIMARY KEY, b); + CREATE INDEX i9 ON t9 USING fts5(); + INSERT INTO t9 VALUES('one', + 'a b c d e f g h i j k l m n o p q r s t u v w x y z ' || + 'a b c d e f g h i j k l m n o p q r s t u v w x y z' + ); +} + +#do_execsql_test 9.1 { +# SELECT snippet(t9) FROM t9 WHERE t9 MATCH 'b' +#} + +do_execsql_test 10.1 { + CREATE TABLE ft(content); + CREATE INDEX fti ON ft USING fts5(); +} +do_execsql_test 10.2 { + INSERT INTO ft VALUES('a b c d e'); + INSERT INTO ft VALUES('f g h i j'); +} +do_execsql_test 10.3 { SELECT rowid FROM ft WHERE ft MATCH 'c' } {1} +do_execsql_test 10.4 { SELECT rowid FROM ft WHERE ft MATCH 'f' } {2} + +do_execsql_test 10.5 { + DELETE FROM ft; + CREATE TABLE ft2(a, b, c); + CREATE INDEX fti2 ON ft2 USING fts5(); + INSERT INTO ft2 VALUES('1 2 3 4 5', '6 7 8 9 10', '11 12 13 14 15'); + SELECT snippet(ft2, '[', ']', '...', -1, 3) FROM ft2 WHERE ft2 MATCH '5'; +} {{...3 4 [5]}} + +finish_test + ADDED test/fts5rnd1.test Index: test/fts5rnd1.test ================================================================== --- /dev/null +++ test/fts5rnd1.test @@ -0,0 +1,452 @@ +# 2009 December 03 +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Brute force (random data) tests for fts5. +# + +#------------------------------------------------------------------------- +# +# The FTS3 tests implemented in this file focus on testing that FTS3 +# returns the correct set of documents for various types of full-text +# query. This is done using pseudo-randomly generated data and queries. +# The expected result of each query is calculated using Tcl code. +# +# 1. The database is initialized to contain a single table with three +# columns. 100 rows are inserted into the table. Each of the three +# values in each row is a document consisting of between 0 and 100 +# terms. Terms are selected from a vocabulary of $G(nVocab) terms. +# +# 2. The following is performed 100 times: +# +# a. A row is inserted into the database. The row contents are +# generated as in step 1. The docid is a pseudo-randomly selected +# value between 0 and 1000000. +# +# b. A psuedo-randomly selected row is updated. One of its columns is +# set to contain a new document generated in the same way as the +# documents in step 1. +# +# c. A psuedo-randomly selected row is deleted. +# +# d. For each of several types of fts3 queries, 10 SELECT queries +# of the form: +# +# SELECT docid FROM WHERE MATCH '' +# +# are evaluated. The results are compared to those calculated by +# Tcl code in this file. The patterns used for the different query +# types are: +# +# 1. query = +# 2. query = +# 3. query = " " +# 4. query = " " +# 5. query = " " +# 6. query = NEAR +# 7. query = NEAR/11 NEAR/11 +# 8. query = OR +# 9. query = NOT +# 10. query = AND +# 11. query = NEAR OR NEAR +# 12. query = NEAR NOT NEAR +# 13. query = NEAR AND NEAR +# +# where is a term psuedo-randomly selected from the vocabulary +# and prefix is the first 2 characters of such a term followed by +# a "*" character. +# +# Every second iteration, steps (a) through (d) above are performed +# within a single transaction. This forces the queries in (d) to +# read data from both the database and the in-memory hash table +# that caches the full-text index entries created by steps (a), (b) +# and (c) until the transaction is committed. +# +# The procedure above is run 5 times, using advisory fts3 node sizes of 50, +# 500, 1000 and 2000 bytes. +# +# After the test using an advisory node-size of 50, an OOM test is run using +# the database. This test is similar to step (d) above, except that it tests +# the effects of transient and persistent OOM conditions encountered while +# executing each query. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If this build does not include FTS3, skip the tests in this file. +# +#ifcapable !fts3 { finish_test ; return } +source $testdir/fts3_common.tcl +source $testdir/malloc_common.tcl + +set G(nVocab) 100 + +set nVocab 100 +set lVocab [list] + +expr srand(0) + +# Generate a vocabulary of nVocab words. Each word is between 2 and 4 +# characters long. +# +set lChar {a b c d e f g h i j k l m n o p q r s t u v w x y z} +for {set i 0} {$i < $nVocab} {incr i} { + set len [expr int(rand()*3)+2] + set word [lindex $lChar [expr int(rand()*26)]] + append word [lindex $lChar [expr int(rand()*26)]] + if {$len>2} { append word [lindex $lChar [expr int(rand()*26)]] } + if {$len>3} { append word [lindex $lChar [expr int(rand()*26)]] } + lappend lVocab $word +} + +proc random_term {} { + lindex $::lVocab [expr {int(rand()*$::nVocab)}] +} + +# Return a document consisting of $nWord arbitrarily selected terms +# from the $::lVocab list. +# +proc generate_doc {nWord} { + set doc [list] + for {set i 0} {$i < $nWord} {incr i} { + lappend doc [random_term] + } + return $doc +} + + + +# Primitives to update the table. +# +unset -nocomplain t1 +proc insert_row {docid} { + set a [generate_doc [expr int((rand()*100))]] + set b [generate_doc [expr int((rand()*100))]] + set c [generate_doc [expr int((rand()*100))]] + execsql { INSERT INTO t1(docid, a, b, c) VALUES($docid, $a, $b, $c) } + set ::t1($docid) [list $a $b $c] +} +proc delete_row {docid} { + execsql { DELETE FROM t1 WHERE docid = $docid } + catch {unset ::t1($docid)} +} +proc update_row {docid} { + set cols {a b c} + set iCol [expr int(rand()*3)] + set doc [generate_doc [expr int((rand()*100))]] + lset ::t1($docid) $iCol $doc + execsql "UPDATE t1 SET [lindex $cols $iCol] = \$doc WHERE docid = \$docid" +} + +proc simple_phrase {zPrefix} { + set ret [list] + + set reg [string map {* {[^ ]*}} $zPrefix] + set reg " $reg " + + foreach key [lsort -integer [array names ::t1]] { + set value $::t1($key) + set cnt [list] + foreach col $value { + if {[regexp $reg " $col "]} { lappend ret $key ; break } + } + } + + #lsort -uniq -integer $ret + set ret +} + +# This [proc] is used to test the FTS3 matchinfo() function. +# +proc simple_token_matchinfo {zToken bDesc} { + + set nDoc(0) 0 + set nDoc(1) 0 + set nDoc(2) 0 + set nHit(0) 0 + set nHit(1) 0 + set nHit(2) 0 + + set dir -inc + if {$bDesc} { set dir -dec } + + foreach key [array names ::t1] { + set value $::t1($key) + set a($key) [list] + foreach i {0 1 2} col $value { + set hit [llength [lsearch -all $col $zToken]] + lappend a($key) $hit + incr nHit($i) $hit + if {$hit>0} { incr nDoc($i) } + } + } + + set ret [list] + foreach docid [lsort -integer $dir [array names a]] { + if { [lindex [lsort -integer $a($docid)] end] } { + set matchinfo [list 1 3] + foreach i {0 1 2} hit $a($docid) { + lappend matchinfo $hit $nHit($i) $nDoc($i) + } + lappend ret $docid $matchinfo + } + } + + set ret +} + +proc simple_near {termlist nNear} { + set ret [list] + + foreach {key value} [array get ::t1] { + foreach v $value { + + set l [lsearch -exact -all $v [lindex $termlist 0]] + foreach T [lrange $termlist 1 end] { + set l2 [list] + foreach i $l { + set iStart [expr $i - $nNear - 1] + set iEnd [expr $i + $nNear + 1] + if {$iStart < 0} {set iStart 0} + foreach i2 [lsearch -exact -all [lrange $v $iStart $iEnd] $T] { + incr i2 $iStart + if {$i2 != $i} { lappend l2 $i2 } + } + } + set l [lsort -uniq -integer $l2] + } + + if {[llength $l]} { +#puts "MATCH($key): $v" + lappend ret $key + } + } + } + + lsort -unique -integer $ret +} + +# The following three procs: +# +# setup_not A B +# setup_or A B +# setup_and A B +# +# each take two arguments. Both arguments must be lists of integer values +# sorted by value. The return value is the list produced by evaluating +# the equivalent of "A op B", where op is the FTS3 operator NOT, OR or +# AND. +# +proc setop_not {A B} { + foreach b $B { set n($b) {} } + set ret [list] + foreach a $A { if {![info exists n($a)]} {lappend ret $a} } + return $ret +} +proc setop_or {A B} { + lsort -integer -uniq [concat $A $B] +} +proc setop_and {A B} { + foreach b $B { set n($b) {} } + set ret [list] + foreach a $A { if {[info exists n($a)]} {lappend ret $a} } + return $ret +} + +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit +set sqlite_fts3_enable_parentheses 1 + +proc do_orderbydocid_test {tn sql res} { + uplevel [list do_select_test $tn.asc "$sql ORDER BY docid ASC" $res] + uplevel [list do_select_test $tn.desc "$sql ORDER BY docid DESC" \ + [lsort -int -dec $res] + ] +} + +#------------------------------------------------------------------------- +# +set NUM_TRIALS 10 +catch { array unset ::t1 } + +# Create the fts5 table. Populate it (and the Tcl array) with 100 rows. +# +db transaction { + execsql { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(docid PRIMARY KEY, a, b, c); + CREATE INDEX i1 ON t1 USING fts5(); + } + for {set i 0} {$i < 100} {incr i} { insert_row $i } +} + +for {set iTest 1} {$iTest <= $NUM_TRIALS} {incr iTest} { + catchsql COMMIT + + set DO_MALLOC_TEST 0 + set nRep 10 + # if {$iTest==100 && $nodesize==50} { + # set DO_MALLOC_TEST 1 + # set nRep 2 + # } + + set ::testprefix fts5rnd-1.$iTest + + # Delete one row, update one row and insert one row. + # + set rows [array names ::t1] + set nRow [llength $rows] + set iUpdate [lindex $rows [expr {int(rand()*$nRow)}]] + set iDelete $iUpdate + while {$iDelete == $iUpdate} { + set iDelete [lindex $rows [expr {int(rand()*$nRow)}]] + } + set iInsert $iUpdate + while {[info exists ::t1($iInsert)]} { + set iInsert [expr {int(rand()*1000000)}] + } + execsql BEGIN + insert_row $iInsert + update_row $iUpdate + delete_row $iDelete + if {0==($iTest%2)} { execsql COMMIT } + + if {0==($iTest%2)} { + #do_test 0 { fts3_integrity_check t1 } ok + } + + # Pick 10 terms from the vocabulary. Check that the results of querying + # the database for the set of documents containing each of these terms + # is the same as the result obtained by scanning the contents of the Tcl + # array for each term. + # + for {set i 0} {$i < 10} {incr i} { + set term [random_term] + do_orderbydocid_test 1.$i.asc { + SELECT docid FROM t1 WHERE t1 MATCH $term + } [simple_phrase $term] + do_orderbydocid_test 1.$i.desc { + SELECT docid FROM t1 WHERE t1 MATCH $term + } [simple_phrase $term] + } + + # This time, use the first two characters of each term as a term prefix + # to query for. Test that querying the Tcl array produces the same results + # as querying the FTS3 table for the prefix. + # + for {set i 0} {$i < $nRep} {incr i} { + set prefix [string range [random_term] 0 end-1] + set match "${prefix}*" + do_orderbydocid_test 2.$i { + SELECT docid FROM t1 WHERE t1 MATCH $match + } [simple_phrase $match] + } + + # Similar to the above, except for phrase queries. + # + for {set i 0} {$i < $nRep} {incr i} { + set term [list [random_term] [random_term]] + set match "\"$term\"" + do_orderbydocid_test 3.$i { + SELECT docid FROM t1 WHERE t1 MATCH $match + } [simple_phrase $term] + } + + # Three word phrases. + # + for {set i 0} {$i < $nRep} {incr i} { + set term [list [random_term] [random_term] [random_term]] + set match "\"$term\"" + do_orderbydocid_test 4.$i { + SELECT docid FROM t1 WHERE t1 MATCH $match + } [simple_phrase $term] + } + + # Three word phrases made up of term-prefixes. + # + for {set i 0} {$i < $nRep} {incr i} { + set query "[string range [random_term] 0 end-1]* + " + append query "[string range [random_term] 0 end-1]* + " + append query "[string range [random_term] 0 end-1]*" + + do_orderbydocid_test 5.$i { + SELECT docid FROM t1 WHERE t1 MATCH $match + } [simple_phrase $query] + } + + # A NEAR query with terms as the arguments: + # + # ... MATCH '$term1 NEAR $term2' ... + # + for {set i 0} {$i < $nRep} {incr i} { + set terms [list [random_term] [random_term]] + set match [join $terms " NEAR "] + do_orderbydocid_test 6.$i { + SELECT docid FROM t1 WHERE t1 MATCH $match + } [simple_near $terms 10] + } + + # A 3-way NEAR query with terms as the arguments. + # + for {set i 0} {$i < $nRep} {incr i} { + set terms [list [random_term] [random_term] [random_term]] + set nNear 11 + set match [join $terms " NEAR/$nNear "] + do_orderbydocid_test 7.$i { + SELECT docid FROM t1 WHERE t1 MATCH $match + } [simple_near $terms $nNear] + } + + # Set operations on simple term queries. + # + foreach {tn op proc} { + 8 OR setop_or + 9 NOT setop_not + 10 AND setop_and + } { + for {set i 0} {$i < $nRep} {incr i} { + set term1 [random_term] + set term2 [random_term] + set match "$term1 $op $term2" + do_orderbydocid_test $tn.$i { + SELECT docid FROM t1 WHERE t1 MATCH $match + } [$proc [simple_phrase $term1] [simple_phrase $term2]] + } + } + + # Set operations on NEAR queries. + # + foreach {tn op proc} { + 11 OR setop_or + 12 NOT setop_not + 13 AND setop_and + } { + for {set i 0} {$i < $nRep} {incr i} { + set term1 [random_term] + set term2 [random_term] + set term3 [random_term] + set term4 [random_term] + set match "$term1 NEAR $term2 $op $term3 NEAR $term4" + do_orderbydocid_test $tn.$i { + SELECT docid FROM t1 WHERE t1 MATCH $match + } [$proc \ + [simple_near [list $term1 $term2] 10] \ + [simple_near [list $term3 $term4] 10] + ] + } + } + + catchsql COMMIT +} + +finish_test ADDED test/fts5snippet.test Index: test/fts5snippet.test ================================================================== --- /dev/null +++ test/fts5snippet.test @@ -0,0 +1,292 @@ +# 2013 January 10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# The tests in this file test the FTS5 snippet() function. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If SQLITE4_ENABLE_FTS3 is not defined, omit this file. +source $testdir/fts3_common.tcl + +set DO_MALLOC_TEST 0 + +# Transform the list $L to its "normal" form. So that it can be compared to +# another list with the same set of elements using [string compare]. +# +proc normalize {L} { + set ret [list] + foreach l $L {lappend ret $l} + return $ret +} + +# Document text used by a few tests. Contains the English names of all +# integers between 1 and 300. +# +set numbers [normalize { + one two three four five six seven eight nine ten eleven twelve thirteen + fourteen fifteen sixteen seventeen eighteen nineteen twenty twentyone + twentytwo twentythree twentyfour twentyfive twentysix twentyseven + twentyeight twentynine thirty thirtyone thirtytwo thirtythree thirtyfour + thirtyfive thirtysix thirtyseven thirtyeight thirtynine forty fortyone + fortytwo fortythree fortyfour fortyfive fortysix fortyseven fortyeight + fortynine fifty fiftyone fiftytwo fiftythree fiftyfour fiftyfive fiftysix + fiftyseven fiftyeight fiftynine sixty sixtyone sixtytwo sixtythree sixtyfour + sixtyfive sixtysix sixtyseven sixtyeight sixtynine seventy seventyone + seventytwo seventythree seventyfour seventyfive seventysix seventyseven + seventyeight seventynine eighty eightyone eightytwo eightythree eightyfour + eightyfive eightysix eightyseven eightyeight eightynine ninety ninetyone + ninetytwo ninetythree ninetyfour ninetyfive ninetysix ninetyseven + ninetyeight ninetynine onehundred onehundredone onehundredtwo + onehundredthree onehundredfour onehundredfive onehundredsix onehundredseven + onehundredeight onehundrednine onehundredten onehundredeleven + onehundredtwelve onehundredthirteen onehundredfourteen onehundredfifteen + onehundredsixteen onehundredseventeen onehundredeighteen onehundrednineteen + onehundredtwenty onehundredtwentyone onehundredtwentytwo + onehundredtwentythree onehundredtwentyfour onehundredtwentyfive + onehundredtwentysix onehundredtwentyseven onehundredtwentyeight + onehundredtwentynine onehundredthirty onehundredthirtyone + onehundredthirtytwo onehundredthirtythree onehundredthirtyfour + onehundredthirtyfive onehundredthirtysix onehundredthirtyseven + onehundredthirtyeight onehundredthirtynine onehundredforty + onehundredfortyone onehundredfortytwo onehundredfortythree + onehundredfortyfour onehundredfortyfive onehundredfortysix + onehundredfortyseven onehundredfortyeight onehundredfortynine + onehundredfifty onehundredfiftyone onehundredfiftytwo onehundredfiftythree + onehundredfiftyfour onehundredfiftyfive onehundredfiftysix + onehundredfiftyseven onehundredfiftyeight onehundredfiftynine + onehundredsixty onehundredsixtyone onehundredsixtytwo onehundredsixtythree + onehundredsixtyfour onehundredsixtyfive onehundredsixtysix + onehundredsixtyseven onehundredsixtyeight onehundredsixtynine + onehundredseventy onehundredseventyone onehundredseventytwo + onehundredseventythree onehundredseventyfour onehundredseventyfive + onehundredseventysix onehundredseventyseven onehundredseventyeight + onehundredseventynine onehundredeighty onehundredeightyone + onehundredeightytwo onehundredeightythree onehundredeightyfour + onehundredeightyfive onehundredeightysix onehundredeightyseven + onehundredeightyeight onehundredeightynine onehundredninety + onehundredninetyone onehundredninetytwo onehundredninetythree + onehundredninetyfour onehundredninetyfive onehundredninetysix + onehundredninetyseven onehundredninetyeight onehundredninetynine twohundred + twohundredone twohundredtwo twohundredthree twohundredfour twohundredfive + twohundredsix twohundredseven twohundredeight twohundrednine twohundredten + twohundredeleven twohundredtwelve twohundredthirteen twohundredfourteen + twohundredfifteen twohundredsixteen twohundredseventeen twohundredeighteen + twohundrednineteen twohundredtwenty twohundredtwentyone twohundredtwentytwo + twohundredtwentythree twohundredtwentyfour twohundredtwentyfive + twohundredtwentysix twohundredtwentyseven twohundredtwentyeight + twohundredtwentynine twohundredthirty twohundredthirtyone + twohundredthirtytwo twohundredthirtythree twohundredthirtyfour + twohundredthirtyfive twohundredthirtysix twohundredthirtyseven + twohundredthirtyeight twohundredthirtynine twohundredforty + twohundredfortyone twohundredfortytwo twohundredfortythree + twohundredfortyfour twohundredfortyfive twohundredfortysix + twohundredfortyseven twohundredfortyeight twohundredfortynine + twohundredfifty twohundredfiftyone twohundredfiftytwo twohundredfiftythree + twohundredfiftyfour twohundredfiftyfive twohundredfiftysix + twohundredfiftyseven twohundredfiftyeight twohundredfiftynine + twohundredsixty twohundredsixtyone twohundredsixtytwo twohundredsixtythree + twohundredsixtyfour twohundredsixtyfive twohundredsixtysix + twohundredsixtyseven twohundredsixtyeight twohundredsixtynine + twohundredseventy twohundredseventyone twohundredseventytwo + twohundredseventythree twohundredseventyfour twohundredseventyfive + twohundredseventysix twohundredseventyseven twohundredseventyeight + twohundredseventynine twohundredeighty twohundredeightyone + twohundredeightytwo twohundredeightythree twohundredeightyfour + twohundredeightyfive twohundredeightysix twohundredeightyseven + twohundredeightyeight twohundredeightynine twohundredninety + twohundredninetyone twohundredninetytwo twohundredninetythree + twohundredninetyfour twohundredninetyfive twohundredninetysix + twohundredninetyseven twohundredninetyeight twohundredninetynine + threehundred +}] + +foreach {DO_MALLOC_TEST enc} { + 0 utf8 + 1 utf8 + 1 utf16 +} { +if {$DO_MALLOC_TEST || $enc=="utf16"} continue + + db close + forcedelete test.db + sqlite4 db test.db + sqlite4_db_config_lookaside db 0 0 0 + db eval "PRAGMA encoding = \"$enc\"" + + # Set variable $T to the test name prefix for this iteration of the loop. + # + set T "fts5snippet-$enc" + + ########################################################################## + # Test the snippet function. + # + proc do_snippet_test {name expr iCol nTok args} { + set res [list] + foreach a $args { lappend res [string trim $a] } + do_select_test $name { + SELECT snippet(ft,'{','}','...',$iCol,$nTok) FROM ft WHERE ft MATCH $expr + } $res + } + do_test $T.3.1 { + execsql { + DROP TABLE IF EXISTS ft; + CREATE TABLE ft(content); + CREATE INDEX fti ON ft USING fts5(); + + INSERT INTO ft VALUES('one two three four five six seven eight nine ten'); + } + } {} + do_snippet_test $T.3.2 one 0 5 "{one} two three four five..." + do_snippet_test $T.3.3 two 0 5 "one {two} three four five..." + do_snippet_test $T.3.4 three 0 5 "one two {three} four five..." + do_snippet_test $T.3.5 four 0 5 "...two three {four} five six..." + do_snippet_test $T.3.6 five 0 5 "...three four {five} six seven..." + do_snippet_test $T.3.7 six 0 5 "...four five {six} seven eight..." + do_snippet_test $T.3.8 seven 0 5 "...five six {seven} eight nine..." + do_snippet_test $T.3.9 eight 0 5 "...six seven {eight} nine ten" + do_snippet_test $T.3.10 nine 0 5 "...six seven eight {nine} ten" + do_snippet_test $T.3.11 ten 0 5 "...six seven eight nine {ten}" + + do_test $T.4.1 { + execsql { + INSERT INTO ft VALUES( + 'one two three four five ' + || 'six seven eight nine ten ' + || 'eleven twelve thirteen fourteen fifteen ' + || 'sixteen seventeen eighteen nineteen twenty ' + || 'one two three four five ' + || 'six seven eight nine ten ' + || 'eleven twelve thirteen fourteen fifteen ' + || 'sixteen seventeen eighteen nineteen twenty' + ); + } + } {} + + do_snippet_test $T.4.2 {one nine} 0 5 { + {one} two three...eight {nine} ten + } { + {one} two three...eight {nine} ten... + } + + do_snippet_test $T.4.3 {one nine} 0 -5 { + {one} two three four five...six seven eight {nine} ten + } { + {one} two three four five...seven eight {nine} ten eleven... + } + do_snippet_test $T.4.3 {one nineteen} 0 -5 { + ...eighteen {nineteen} twenty {one} two... + } + do_snippet_test $T.4.4 {two nineteen} 0 -5 { + ...eighteen {nineteen} twenty one {two}... + } + do_snippet_test $T.4.5 {three nineteen} 0 -5 { + ...{nineteen} twenty one two {three}... + } + + do_snippet_test $T.4.6 {four nineteen} 0 -5 { + ...two three {four} five six...seventeen eighteen {nineteen} twenty one... + } + do_snippet_test $T.4.7 {four NEAR nineteen} 0 -5 { + ...seventeen eighteen {nineteen} twenty one...two three {four} five six... + } + + do_snippet_test $T.4.8 {four nineteen} 0 5 { + ...three {four} five...eighteen {nineteen} twenty... + } + do_snippet_test $T.4.9 {four NEAR nineteen} 0 5 { + ...eighteen {nineteen} twenty...three {four} five... + } + do_snippet_test $T.4.10 {four NEAR nineteen} 0 -5 { + ...seventeen eighteen {nineteen} twenty one...two three {four} five six... + } + do_snippet_test $T.4.11 {four NOT (nineteen+twentyone)} 0 5 { + ...two three {four} five six... + } { + ...two three {four} five six... + } + do_snippet_test $T.4.12 {four OR nineteen NEAR twentyone} 0 5 { + ...two three {four} five six... + } { + ...two three {four} five six... + } + + do_test $T.5.1 { + execsql { + DROP TABLE IF EXISTS ft; + CREATE TABLE ft(a, b, c); + CREATE INDEX fti ON ft USING fts5(); + INSERT INTO ft VALUES( + 'one two three four five', + 'four five six seven eight', + 'seven eight nine ten eleven' + ); + } + } {} + + do_snippet_test $T.5.2 {five} -1 3 {...three four {five}} + do_snippet_test $T.5.3 {five} 0 3 {...three four {five}} + do_snippet_test $T.5.4 {five} 1 3 {four {five} six...} + do_snippet_test $T.5.5 {five} 2 3 {seven eight nine...} + + do_test $T.5.6 { + execsql { UPDATE ft SET b = NULL } + } {} + + do_snippet_test $T.5.7 {five} -1 3 {...three four {five}} + do_snippet_test $T.5.8 {five} 0 3 {...three four {five}} + do_snippet_test $T.5.9 {five} 1 3 {} + do_snippet_test $T.5.10 {five} 2 3 {seven eight nine...} + + do_snippet_test $T.5.11 {one "seven eight nine"} -1 -3 { + {one} two three...{seven} {eight} {nine}... + } + + do_test $T.6.1 { + execsql { + DROP TABLE IF EXISTS ft; + CREATE TABLE ft(x); + CREATE INDEX fti ON ft USING fts5(); + INSERT INTO ft VALUES($numbers); + } + } {} + do_snippet_test $T.6.2 { + one fifty onehundred onehundredfifty twohundredfifty threehundred + } -1 4 { + {one}...{fifty}...{onehundred}...{onehundredfifty}... + } + do_snippet_test $T.6.3 { + one fifty onehundred onehundredfifty twohundredfifty threehundred + } -1 -4 { + {one} two three four...fortyeight fortynine {fifty} fiftyone...ninetyeight ninetynine {onehundred} onehundredone...onehundredfortyeight onehundredfortynine {onehundredfifty} onehundredfiftyone... + } + + do_test $T.7.1 { + execsql { + BEGIN; + DROP TABLE IF EXISTS ft; + CREATE TABLE ft(x); + CREATE INDEX fti ON ft USING fts5(); + } + set testresults [list] + for {set i 1} {$i < 150} {incr i} { + set commas [string repeat , $i] + execsql {INSERT INTO ft VALUES('one' || $commas || 'two')} + lappend testresults "{one}$commas{two}" + } + execsql COMMIT + } {} + eval [list do_snippet_test $T.7.2 {one two} -1 3] $testresults + +} + +finish_test Index: test/in.test ================================================================== --- test/in.test +++ test/in.test @@ -271,15 +271,10 @@ do_test in-8.1 { execsql { SELECT b FROM t1 WHERE a IN ('hello','there') } -} {world} -do_test in-8.2 { - execsql { - SELECT b FROM t1 WHERE a IN ("hello",'there') - } } {world} # Test constructs of the form: expr IN tablename # do_test in-9.1 { Index: test/insert.test ================================================================== --- test/insert.test +++ test/insert.test @@ -360,8 +360,23 @@ INSERT INTO t3 SELECT * FROM (SELECT * FROM t3 UNION ALL SELECT 1,2,3) } } {} } +# Multiple VALUES clauses +# +do_test insert-10.1 { + execsql { + CREATE TABLE t10(a,b,c); + INSERT INTO t10 VALUES(1,2,3), (4,5,6), (7,8,9); + SELECT * FROM t10; + } +} {1 2 3 4 5 6 7 8 9} +do_test insert-10.2 { + catchsql { + INSERT INTO t10 VALUES(11,12,13), (14,15); + } +} {1 {all VALUES must have the same number of terms}} + integrity_check insert-99.0 finish_test Index: test/join.test ================================================================== --- test/join.test +++ test/join.test @@ -429,25 +429,25 @@ # A test for ticket #247. # do_test join-7.1 { execsql { CREATE TABLE t7 (x, y); - INSERT INTO t7 VALUES ("pa1", 1); - INSERT INTO t7 VALUES ("pa2", NULL); - INSERT INTO t7 VALUES ("pa3", NULL); - INSERT INTO t7 VALUES ("pa4", 2); - INSERT INTO t7 VALUES ("pa30", 131); - INSERT INTO t7 VALUES ("pa31", 130); - INSERT INTO t7 VALUES ("pa28", NULL); + INSERT INTO t7 VALUES ('pa1', 1); + INSERT INTO t7 VALUES ('pa2', NULL); + INSERT INTO t7 VALUES ('pa3', NULL); + INSERT INTO t7 VALUES ('pa4', 2); + INSERT INTO t7 VALUES ('pa30', 131); + INSERT INTO t7 VALUES ('pa31', 130); + INSERT INTO t7 VALUES ('pa28', NULL); CREATE TABLE t8 (a integer primary key, b); - INSERT INTO t8 VALUES (1, "pa1"); - INSERT INTO t8 VALUES (2, "pa4"); + INSERT INTO t8 VALUES (1, 'pa1'); + INSERT INTO t8 VALUES (2, 'pa4'); INSERT INTO t8 VALUES (3, NULL); INSERT INTO t8 VALUES (4, NULL); - INSERT INTO t8 VALUES (130, "pa31"); - INSERT INTO t8 VALUES (131, "pa30"); + INSERT INTO t8 VALUES (130, 'pa31'); + INSERT INTO t8 VALUES (131, 'pa30'); SELECT coalesce(t8.a,999) from t7 LEFT JOIN t8 on y=a; } } {1 999 999 2 131 130 999} Index: test/lsm1.test ================================================================== --- test/lsm1.test +++ test/lsm1.test @@ -16,11 +16,11 @@ proc reopen {{bClear 0}} { catch {db close} if {$bClear} { forcedelete test.db } - lsm_open db test.db {mmap 0 nmerge 2 autowork 0} + lsm_open db test.db {mmap 0 automerge 2 autowork 0} } proc contents {} { db csr_open csr set res [list] @@ -162,11 +162,11 @@ contents } {{10 ten} {40 forty}} do_test 5.2 { reopen 1 - db config {nmerge 4} + db config {automerge 4} dbwrite { 10 ten } ; db flush dbwrite { 20 twenty } ; db flush dbwrite { 30 thirty } ; db flush dbwrite { 40 forty } ; db flush @@ -174,19 +174,19 @@ db delete_range 10 17 ; db flush dbwrite {17 seventeen} ; db flush db delete_range 10 17 ; db flush - db config {nmerge 3} + db config {automerge 3} db work 10 contents } {{10 ten} {17 seventeen} {20 twenty} {30 thirty} {40 forty}} do_test 5.3 { reopen 1 - db config {nmerge 4} + db config {automerge 4} dbwrite { 10 ten } ; db flush dbwrite { 20 twenty } ; db flush dbwrite { 30 thirty } ; db flush dbwrite { 40 forty } ; db flush @@ -194,12 +194,12 @@ db delete_range 10 17 ; db flush db delete_range 12 19 ; db flush dbwrite {17 seventeen} ; db flush - db config {nmerge 3} + db config {automerge 3} db work 10 contents } {{10 ten} {17 seventeen} {20 twenty} {30 thirty} {40 forty}} finish_test ADDED test/lsm3.test Index: test/lsm3.test ================================================================== --- /dev/null +++ test/lsm3.test @@ -0,0 +1,89 @@ +# 2012 November 02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file tests that the LSM_CONFIG_MULTIPLE_PROCESSES parameter seems +# to work as documented. +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set testprefix lsm3 +db close + +do_multiclient_test tn { + + # The [do_multiclient_test] command automatically opens a connection + # in each process (or three connections in this process). We don't want + # them in this case. + code1 { db close } + code2 { db2 close } + code3 { db3 close } + + if { $tn==1 } { + set locked {1 {database is locked}} + } else { + set locked {0 {}} + } + + # Open a single-process connection to the database from an external + # process (if $tn==1, otherwise open it from within the current + # process). + code2 { sqlite4 db2 file:test.db?lsm_multiple_processes=0 } + + # Try to open some other connections to the database file, both in multi + # and single process mode. If ($tn==1), then all such attempts fail. Or, + # if ($tn==2), they all succeed. + do_test $tn.1 { + catch { db close } + list [catch {sqlite4 db file:test.db?lsm_multiple_processes=0} msg] $msg + } $locked + do_test $tn.2 { + catch { db close } + list [catch {sqlite4 db file:test.db?lsm_multiple_processes=0} msg] $msg + } $locked + do_test $tn.3 { + catch { db close } + list [catch {sqlite4 db file:test.db?lsm_multiple_processes=1} msg] $msg + } $locked + do_test $tn.4 { + catch { db close } + list [catch {sqlite4 db file:test.db?lsm_multiple_processes=1} msg] $msg + } $locked + + # Now open a connection from an external process in multi-proc mode. + # Observe that further connections are allowed if they are from within + # the same process or if the LSM_CONFIG_MULTIPLE_PROCESSES parameter + # is set to true. + code2 { + db2 close + sqlite4 db2 file:test.db + } + + do_test $tn.5 { + catch { db close } + list [catch {sqlite4 db file:test.db?lsm_multiple_processes=0} msg] $msg + } $locked + do_test $tn.6 { + catch { db close } + list [catch {sqlite4 db file:test.db?lsm_multiple_processes=0} msg] $msg + } $locked + do_test $tn.7 { + catch { db close } + list [catch {sqlite4 db file:test.db?lsm_multiple_processes=1} msg] $msg + } {0 {}} + do_test $tn.8 { + catch { db close } + list [catch {sqlite4 db file:test.db?lsm_multiple_processes=1} msg] $msg + } {0 {}} +} + + +finish_test ADDED test/lsm4.test Index: test/lsm4.test ================================================================== --- /dev/null +++ test/lsm4.test @@ -0,0 +1,122 @@ +# 2013 February 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the LSM library. More specifically, +# it focuses on testing the compression, compression-id and +# compression-factory functionality. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix lsm4 +db close + +# Compression scheme ids (defined in test_lsm.c): +# +set compression_id(encrypt) 43 +set compression_id(rle) 44 +set compression_id(noop) 45 + +proc db_fetch {db key} { + db csr_open csr + csr seek $key eq + set ret [csr value] + csr close + set ret +} + +do_test 1.1 { + lsm_open db test.db + db config {set_compression noop} + db write 1 abc + db write 2 def + db close +} {} + +do_test 1.2 { + lsm_open db test.db + db config {set_compression noop} + list [db_fetch db 1] [db_fetch db 2] +} {abc def} + +do_test 1.3 { + db close + lsm_open db test.db + db config {set_compression rle} + list [catch {db_fetch db 1} msg] $msg +} {1 {error in lsm_csr_open() - 50}} + +do_test 1.4 { + db close + lsm_open db test.db + list [catch {db_fetch db 1} msg] $msg +} {1 {error in lsm_csr_open() - 50}} + +do_test 1.5 { + db config {set_compression_factory true} + list [db_fetch db 1] [db_fetch db 2] +} {abc def} + +do_test 1.6 { db info compression_id } $compression_id(noop) +db close + +#------------------------------------------------------------------------- +# +forcedelete test.db + +do_test 2.1 { + lsm_open db test.db + db info compression_id +} {0} + +do_test 2.2 { + db write 1 abc + db write 2 abc + db info compression_id +} {0} + +do_test 2.3 { + lsm_open db2 test.db + db2 info compression_id +} {0} + +do_test 2.4 { + db close + db2 info compression_id +} {0} + +do_test 2.5 { + db2 close + lsm_open db test.db + db info compression_id +} {1} + +db close +forcedelete test.db + +do_test 2.6 { + lsm_open db test.db + db config {set_compression rle} + db write 3 three + db write 4 four + db close + + lsm_open db test.db + db info compression_id +} $compression_id(rle) + +do_test 2.7 { + db config {set_compression rle} + list [db_fetch db 3] [db_fetch db 4] +} {three four} + +finish_test + ADDED test/lsm5.test Index: test/lsm5.test ================================================================== --- /dev/null +++ test/lsm5.test @@ -0,0 +1,131 @@ +# 2013 February 08 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the LSM library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix lsm5 +db close + +proc db_fetch {db key} { + db csr_open csr + csr seek $key eq + set ret [csr value] + csr close + set ret +} + +# Create a new database with file name $file. +# +proc create_abc_db {file} { + forcedelete $file + lsm_open db $file + db write a alpha + db write b bravo + db write c charlie + db close +} + +#------------------------------------------------------------------------- +# When the database system is shut down (i.e. when the last connection +# disconnects), an attempt is made to truncate the database file to the +# minimum number of blocks required. +# +# This test case checks that this process does not actually cause the +# database to grow. +# +do_test 1.1 { + lsm_open db test.db + db config {mmap 0} +} {0} +do_test 1.2 { + db write 1 one + db write 2 two + db close +} {} +do_test 1.3 { + expr [file size test.db] < (64*1024) +} 1 + +#------------------------------------------------------------------------- +# Test that if an attempt is made to open a read-write connection to a +# database that the client does not have permission to write to is attempted +# an error is reported. In order to open a read-write connection to a +# database, the client requires: +# +# * read-write access to the db file, +# * read-write access to the log file, +# * for multi-process mode, read-write access to the shm file. +# +# In the above, "read-write access" includes the ability to create the db, +# log or shm file if it does not exist. +# +# These tests verify that the lsm_open() command returns LSM_IOERR. At some +# point in the future this will be improved. Likely when sqlite4 level tests +# for opening read-only databases are added. +# + +foreach {tn filename setup} { + + 1 test.dir/test.db { + # Create a directory "test.dir". + forcedelete test.dir + file mkdir test.dir + + # Create a database within test.dir + create_abc_db test.dir/test.db + + # Now make the db and its directory read-only. + file attr test.dir/test.db -perm r--r--r-- + file attr test.dir -perm r-xr-xr-x + } + + 2 test.db { + # Create a database test.db and set its permissions to read-only + create_abc_db test.db + file attr test.db -perm r--r--r-- + } + + 3 test.dir/test.db { + # Create a directory "test.dir". + forcedelete test.dir + file mkdir test.dir + + # Create a database within test.dir + create_abc_db test.dir/test.db + + # Now make test.dir read-only. + file attr test.dir -perm r-xr-xr-x + } + +} { + + do_test 2.$tn.1 { + eval $setup + set rc [catch {lsm_open db $filename} msg] + list $rc $msg + } {1 {error in lsm_open() - 10}} + + do_test 2.$tn.2 { + eval $setup + lsm_open db $filename {readonly 1} + set res [list [db_fetch db a] [db_fetch db b] [db_fetch db c]] + db close + set res + } {alpha bravo charlie} + +} + + +finish_test + ADDED test/num.test Index: test/num.test ================================================================== --- /dev/null +++ test/num.test @@ -0,0 +1,91 @@ +# 2001 September 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the sqlite_*_printf() interface. +# +# $Id: printf.test,v 1.31 2009/02/01 00:21:10 drh Exp $ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_test num-1.1.1 { + sqlite4_num_compare 20 20 +} {equal} +do_test num-1.1.2 { + sqlite4_num_compare 20 2e1 +} {equal} +do_test num-1.1.3 { + sqlite4_num_compare -00034 -3.4e1 +} {equal} +do_test num-1.1.4 { + sqlite4_num_compare -inf +inf +} {lesser} +do_test num-1.1.5 { + sqlite4_num_compare -inf 0 +} {lesser} +do_test num-1.1.6 { + sqlite4_num_compare inf 4 +} {greater} +do_test num-1.1.7 { + sqlite4_num_compare nan 7 +} {incomparable} +# Is +0 > -0? +#do_test num-equal-1.1.4 { +# sqlite4_num_compare +0 -0 +#} {equal} + +do_test num-2.1.1 { + sqlite4_num_to_text [sqlite4_num_from_text 37] +} {37} +do_test num-2.1.2 { + sqlite4_num_to_text [sqlite4_num_from_text 37 2] +} {37} +do_test num-2.1.4 { + sqlite4_num_compare [sqlite4_num_from_text 2.9e2X 5] 290 +} {equal} +do_test num-2.1.5 { + sqlite4_num_isnan [sqlite4_num_from_text inf 2] +} {true} +do_test num-2.1.6 { + sqlite4_num_isinf [sqlite4_num_from_text inf 3] +} {true} + +do_test num-3.1.1 { + sqlite4_num_to_text [sqlite4_num_add 5 7] +} {12} + +do_test num-4.1.1 { + sqlite4_num_to_text [sqlite4_num_sub 9 3] +} {6} +do_test num-4.1.2 { + sqlite4_num_to_text [sqlite4_num_sub 5 12] +} {-7} +do_test num-4.2.1 { + sqlite4_num_compare [sqlite4_num_sub 1 1] [sqlite4_num_sub -1 -1] +} {equal} + +do_test num-5.1.1 { + sqlite4_num_to_text [sqlite4_num_mul 9 8] +} {72} + +do_test num-6.1.1 { + sqlite4_num_to_text [sqlite4_num_div 6 5] +} {1.2} +do_test num-6.1.2 { + sqlite4_num_compare 2 [sqlite4_num_div 2 1] +} {equal} +do_test num-6.1.3 { + sqlite4_num_to_text [sqlite4_num_div 2 1] +} {2} +do_test num-6.1.4 { + sqlite4_num_to_text [sqlite4_num_div 22 10] +} {2.2} +finish_test Index: test/permutations.test ================================================================== --- test/permutations.test +++ test/permutations.test @@ -137,10 +137,12 @@ log3.test lsm1.test lsm2.test csr1.test ckpt1.test mc1.test + fts5expr1.test fts5query1.test fts5rnd1.test fts5create.test + fts5snippet.test aggerror.test attach.test autoindex1.test badutf.test Index: test/quote.test ================================================================== --- test/quote.test +++ test/quote.test @@ -32,13 +32,10 @@ catchsql {SELECT * FROM '@abc'} } {0 {5 hello}} do_test quote-1.2.2 { catchsql {SELECT * FROM [@abc]} ;# SqlServer compatibility } {0 {5 hello}} -do_test quote-1.2.3 { - catchsql {SELECT * FROM `@abc`} ;# MySQL compatibility -} {0 {5 hello}} do_test quote-1.3 { catchsql { SELECT '@abc'.'!pqr', '@abc'.'#xyz'+5 FROM '@abc' } } {0 {hello 10}} @@ -49,15 +46,10 @@ } {0 {!pqr 5}} do_test quote-1.3.2 { catchsql { SELECT "!pqr", "#xyz"+5 FROM '@abc' } -} {0 {hello 10}} -do_test quote-1.3.3 { - catchsql { - SELECT [!pqr], `#xyz`+5 FROM '@abc' - } } {0 {hello 10}} do_test quote-1.3.4 { set r [catch { execsql {SELECT '@abc'.'!pqr', '@abc'.'#xyz'+5 FROM '@abc'} } msg ] Index: test/select6.test ================================================================== --- test/select6.test +++ test/select6.test @@ -163,15 +163,15 @@ } } {x 3 y 2} do_test select6-3.2 { execsql { SELECT * FROM - (SELECT a.q, a.p, b.r + (SELECT a.q AS x, a.p, b.r FROM (SELECT count(*) as p , b as q FROM t2 GROUP BY q) AS a, (SELECT max(a) as r, b as s FROM t2 GROUP BY s) as b WHERE a.q=b.s ORDER BY a.q) - ORDER BY "a.q" + ORDER BY x } } {1 1 1 2 2 3 3 4 7 4 8 15 5 5 20} do_test select6-3.3 { execsql { SELECT a,b,a+b FROM (SELECT avg(x) as 'a', avg(y) as 'b' FROM t1) ADDED test/shell1.test Index: test/shell1.test ================================================================== --- /dev/null +++ test/shell1.test @@ -0,0 +1,763 @@ +# 2009 Nov 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the CLI shell tool. +# +# + +# Test plan: +# +# shell1-1.*: Basic command line option handling. +# shell1-2.*: Basic "dot" command token parsing. +# shell1-3.*: Basic test that "dot" command can be called. +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl +if {$tcl_platform(platform)=="windows"} { + set CLI "sqlite4.exe" +} else { + set CLI "./sqlite4" +} +if {![file executable $CLI]} { + finish_test + return +} +db close +forcedelete test.db test.db-journal test.db-wal +sqlite4 db test.db + +#---------------------------------------------------------------------------- +# Test cases shell1-1.*: Basic command line option handling. +# + +# invalid option +do_test shell1-1.1.1 { + set res [catchcmd "-bad test.db" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: unknown option: -bad} $res] +} {1 1} +# error on extra options +do_test shell1-1.1.2 { + set res [catchcmd "-bad test.db \"select 3\" \"select 4\"" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: too many options: "select 4"} $res] +} {1 1} +# error on extra options +do_test shell1-1.1.3 { + set res [catchcmd "-bad FOO test.db BAD" ".quit"] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: too many options: "BAD"} $res] +} {1 1} + +# -help +do_test shell1-1.2.1 { + set res [catchcmd "-help test.db" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Usage} $res] \ + [regexp {\-init} $res] \ + [regexp {\-version} $res] +} {1 1 1 1} + +# -init filename read/process named file +do_test shell1-1.3.1 { + catchcmd "-init FOO test.db" "" +} {0 {}} +do_test shell1-1.3.2 { + set res [catchcmd "-init FOO test.db .quit BAD" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: too many options: "BAD"} $res] +} {1 1} + +# -echo print commands before execution +do_test shell1-1.4.1 { + catchcmd "-echo test.db" "" +} {0 {}} + +# -[no]header turn headers on or off +do_test shell1-1.5.1 { + catchcmd "-header test.db" "" +} {0 {}} +do_test shell1-1.5.2 { + catchcmd "-noheader test.db" "" +} {0 {}} + +# -bail stop after hitting an error +do_test shell1-1.6.1 { + catchcmd "-bail test.db" "" +} {0 {}} + +# -interactive force interactive I/O +do_test shell1-1.7.1 { + set res [catchcmd "-interactive test.db" ".quit"] + set rc [lindex $res 0] + list $rc \ + [regexp {SQLite version} $res] \ + [regexp {Enter SQL statements} $res] +} {0 1 1} + +# -batch force batch I/O +do_test shell1-1.8.1 { + catchcmd "-batch test.db" "" +} {0 {}} + +# -column set output mode to 'column' +do_test shell1-1.9.1 { + catchcmd "-column test.db" "" +} {0 {}} + +# -csv set output mode to 'csv' +do_test shell1-1.10.1 { + catchcmd "-csv test.db" "" +} {0 {}} + +# -html set output mode to HTML +do_test shell1-1.11.1 { + catchcmd "-html test.db" "" +} {0 {}} + +# -line set output mode to 'line' +do_test shell1-1.12.1 { + catchcmd "-line test.db" "" +} {0 {}} + +# -list set output mode to 'list' +do_test shell1-1.13.1 { + catchcmd "-list test.db" "" +} {0 {}} + +# -separator 'x' set output field separator (|) +do_test shell1-1.14.1 { + catchcmd "-separator 'x' test.db" "" +} {0 {}} +do_test shell1-1.14.2 { + catchcmd "-separator x test.db" "" +} {0 {}} +do_test shell1-1.14.3 { + set res [catchcmd "-separator" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: missing argument to -separator} $res] +} {1 1} + +# -stats print memory stats before each finalize +do_test shell1-1.14b.1 { + catchcmd "-stats test.db" "" +} {0 {}} + +# -nullvalue 'text' set text string for NULL values +do_test shell1-1.15.1 { + catchcmd "-nullvalue 'x' test.db" "" +} {0 {}} +do_test shell1-1.15.2 { + catchcmd "-nullvalue x test.db" "" +} {0 {}} +do_test shell1-1.15.3 { + set res [catchcmd "-nullvalue" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: missing argument to -nullvalue} $res] +} {1 1} + +# -version show SQLite version +do_test shell1-1.16.1 { + set x [catchcmd "-version test.db" ""] +} {/4.[0-9.]+ 20\d\d-[01]\d-\d\d \d\d:\d\d:\d\d [0-9a-f]+/} + +#---------------------------------------------------------------------------- +# Test cases shell1-2.*: Basic "dot" command token parsing. +# + +# check first token handling +do_test shell1-2.1.1 { + catchcmd "test.db" ".foo" +} {1 {Error: unknown command or invalid arguments: "foo". Enter ".help" for help}} +do_test shell1-2.1.2 { + catchcmd "test.db" ".\"foo OFF\"" +} {1 {Error: unknown command or invalid arguments: "foo OFF". Enter ".help" for help}} +do_test shell1-2.1.3 { + catchcmd "test.db" ".\'foo OFF\'" +} {1 {Error: unknown command or invalid arguments: "foo OFF". Enter ".help" for help}} + +# unbalanced quotes +do_test shell1-2.2.1 { + catchcmd "test.db" ".\"foo OFF" +} {1 {Error: unknown command or invalid arguments: "foo OFF". Enter ".help" for help}} +do_test shell1-2.2.2 { + catchcmd "test.db" ".\'foo OFF" +} {1 {Error: unknown command or invalid arguments: "foo OFF". Enter ".help" for help}} +do_test shell1-2.2.3 { + catchcmd "test.db" ".explain \"OFF" +} {0 {}} +do_test shell1-2.2.4 { + catchcmd "test.db" ".explain \'OFF" +} {0 {}} +do_test shell1-2.2.5 { + catchcmd "test.db" ".mode \"insert FOO" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} +do_test shell1-2.2.6 { + catchcmd "test.db" ".mode \'insert FOO" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} + +# check multiple tokens, and quoted tokens +do_test shell1-2.3.1 { + catchcmd "test.db" ".explain 1" +} {0 {}} +do_test shell1-2.3.2 { + catchcmd "test.db" ".explain on" +} {0 {}} +do_test shell1-2.3.3 { + catchcmd "test.db" ".explain \"1 2 3\"" +} {0 {}} +do_test shell1-2.3.4 { + catchcmd "test.db" ".explain \"OFF\"" +} {0 {}} +do_test shell1-2.3.5 { + catchcmd "test.db" ".\'explain\' \'OFF\'" +} {0 {}} +do_test shell1-2.3.6 { + catchcmd "test.db" ".explain \'OFF\'" +} {0 {}} +do_test shell1-2.3.7 { + catchcmd "test.db" ".\'explain\' \'OFF\'" +} {0 {}} + +# check quoted args are unquoted +do_test shell1-2.4.1 { + catchcmd "test.db" ".mode FOO" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} +do_test shell1-2.4.2 { + catchcmd "test.db" ".mode csv" +} {0 {}} +do_test shell1-2.4.2 { + catchcmd "test.db" ".mode \"csv\"" +} {0 {}} + + +#---------------------------------------------------------------------------- +# Test cases shell1-3.*: Basic test that "dot" command can be called. +# + + +# .bail ON|OFF Stop after hitting an error. Default OFF +do_test shell1-3.2.1 { + catchcmd "test.db" ".bail" +} {1 {Error: unknown command or invalid arguments: "bail". Enter ".help" for help}} +do_test shell1-3.2.2 { + catchcmd "test.db" ".bail ON" +} {0 {}} +do_test shell1-3.2.3 { + catchcmd "test.db" ".bail OFF" +} {0 {}} +do_test shell1-3.2.4 { + # too many arguments + catchcmd "test.db" ".bail OFF BAD" +} {1 {Error: unknown command or invalid arguments: "bail". Enter ".help" for help}} + +# .databases List names and files of attached databases +do_test shell1-3.3.1 { + catchcmd "-csv test.db" ".databases" +} "/0 +.*main +[string map {/ .} [string range [get_pwd] 0 10]].*/" +do_test shell1-3.3.2 { + # too many arguments + catchcmd "test.db" ".databases BAD" +} {1 {Error: unknown command or invalid arguments: "databases". Enter ".help" for help}} + +# .dump ?TABLE? ... Dump the database in an SQL text format +# If TABLE specified, only dump tables matching +# LIKE pattern TABLE. +do_test shell1-3.4.1 { + set res [catchcmd "test.db" ".dump"] + list [regexp {BEGIN TRANSACTION;} $res] \ + [regexp {COMMIT;} $res] +} {1 1} +do_test shell1-3.4.2 { + set res [catchcmd "test.db" ".dump FOO"] + list [regexp {BEGIN TRANSACTION;} $res] \ + [regexp {COMMIT;} $res] +} {1 1} +do_test shell1-3.4.3 { + # too many arguments + catchcmd "test.db" ".dump FOO BAD" +} {1 {Error: unknown command or invalid arguments: "dump". Enter ".help" for help}} + +# .echo ON|OFF Turn command echo on or off +do_test shell1-3.5.1 { + catchcmd "test.db" ".echo" +} {1 {Error: unknown command or invalid arguments: "echo". Enter ".help" for help}} +do_test shell1-3.5.2 { + catchcmd "test.db" ".echo ON" +} {0 {}} +do_test shell1-3.5.3 { + catchcmd "test.db" ".echo OFF" +} {0 {}} +do_test shell1-3.5.4 { + # too many arguments + catchcmd "test.db" ".echo OFF BAD" +} {1 {Error: unknown command or invalid arguments: "echo". Enter ".help" for help}} + +# .exit Exit this program +do_test shell1-3.6.1 { + catchcmd "test.db" ".exit" +} {0 {}} +do_test shell1-3.6.2 { + # too many arguments + catchcmd "test.db" ".exit BAD" +} {1 {Error: unknown command or invalid arguments: "exit". Enter ".help" for help}} + +# .explain ON|OFF Turn output mode suitable for EXPLAIN on or off. +do_test shell1-3.7.1 { + catchcmd "test.db" ".explain" + # explain is the exception to the booleans. without an option, it turns it on. +} {0 {}} +do_test shell1-3.7.2 { + catchcmd "test.db" ".explain ON" +} {0 {}} +do_test shell1-3.7.3 { + catchcmd "test.db" ".explain OFF" +} {0 {}} +do_test shell1-3.7.4 { + # too many arguments + catchcmd "test.db" ".explain OFF BAD" +} {1 {Error: unknown command or invalid arguments: "explain". Enter ".help" for help}} + + +# .header(s) ON|OFF Turn display of headers on or off +do_test shell1-3.9.1 { + catchcmd "test.db" ".header" +} {1 {Error: unknown command or invalid arguments: "header". Enter ".help" for help}} +do_test shell1-3.9.2 { + catchcmd "test.db" ".header ON" +} {0 {}} +do_test shell1-3.9.3 { + catchcmd "test.db" ".header OFF" +} {0 {}} +do_test shell1-3.9.4 { + # too many arguments + catchcmd "test.db" ".header OFF BAD" +} {1 {Error: unknown command or invalid arguments: "header". Enter ".help" for help}} + +do_test shell1-3.9.5 { + catchcmd "test.db" ".headers" +} {1 {Error: unknown command or invalid arguments: "headers". Enter ".help" for help}} +do_test shell1-3.9.6 { + catchcmd "test.db" ".headers ON" +} {0 {}} +do_test shell1-3.9.7 { + catchcmd "test.db" ".headers OFF" +} {0 {}} +do_test shell1-3.9.8 { + # too many arguments + catchcmd "test.db" ".headers OFF BAD" +} {1 {Error: unknown command or invalid arguments: "headers". Enter ".help" for help}} + +# .help Show this message +do_test shell1-3.10.1 { + set res [catchcmd "test.db" ".help"] + # look for a few of the possible help commands + list [regexp {.help} $res] \ + [regexp {.quit} $res] \ + [regexp {.show} $res] +} {1 1 1} +do_test shell1-3.10.2 { + # we allow .help to take extra args (it is help after all) + set res [catchcmd "test.db" ".help BAD"] + # look for a few of the possible help commands + list [regexp {.help} $res] \ + [regexp {.quit} $res] \ + [regexp {.show} $res] +} {1 1 1} + +# .import FILE TABLE Import data from FILE into TABLE +do_test shell1-3.11.1 { + catchcmd "test.db" ".import" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} +do_test shell1-3.11.2 { + catchcmd "test.db" ".import FOO" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} +do_test shell1-3.11.2 { + catchcmd "test.db" ".import FOO BAR" +} {1 {Error: no such table: BAR}} +do_test shell1-3.11.3 { + # too many arguments + catchcmd "test.db" ".import FOO BAR BAD" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} + +# .indices ?TABLE? Show names of all indices +# If TABLE specified, only show indices for tables +# matching LIKE pattern TABLE. +do_test shell1-3.12.1 { + catchcmd "test.db" ".indices" +} {0 {}} +do_test shell1-3.12.2 { + catchcmd "test.db" ".indices FOO" +} {0 {}} +do_test shell1-3.12.3 { + # too many arguments + catchcmd "test.db" ".indices FOO BAD" +} {1 {Error: unknown command or invalid arguments: "indices". Enter ".help" for help}} + +# .mode MODE ?TABLE? Set output mode where MODE is one of: +# csv Comma-separated values +# column Left-aligned columns. (See .width) +# html HTML code +# insert SQL insert statements for TABLE +# line One value per line +# list Values delimited by .separator string +# tabs Tab-separated values +# tcl TCL list elements +do_test shell1-3.13.1 { + catchcmd "test.db" ".mode" +} {1 {Error: unknown command or invalid arguments: "mode". Enter ".help" for help}} +do_test shell1-3.13.2 { + catchcmd "test.db" ".mode FOO" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} +do_test shell1-3.13.3 { + catchcmd "test.db" ".mode csv" +} {0 {}} +do_test shell1-3.13.4 { + catchcmd "test.db" ".mode column" +} {0 {}} +do_test shell1-3.13.5 { + catchcmd "test.db" ".mode html" +} {0 {}} +do_test shell1-3.13.6 { + catchcmd "test.db" ".mode insert" +} {0 {}} +do_test shell1-3.13.7 { + catchcmd "test.db" ".mode line" +} {0 {}} +do_test shell1-3.13.8 { + catchcmd "test.db" ".mode list" +} {0 {}} +do_test shell1-3.13.9 { + catchcmd "test.db" ".mode tabs" +} {0 {}} +do_test shell1-3.13.10 { + catchcmd "test.db" ".mode tcl" +} {0 {}} +do_test shell1-3.13.11 { + # too many arguments + catchcmd "test.db" ".mode tcl BAD" +} {1 {Error: invalid arguments: "BAD". Enter ".help" for help}} + +# don't allow partial mode type matches +do_test shell1-3.13.12 { + catchcmd "test.db" ".mode l" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} +do_test shell1-3.13.13 { + catchcmd "test.db" ".mode li" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} +do_test shell1-3.13.14 { + catchcmd "test.db" ".mode lin" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} + +# .nullvalue STRING Print STRING in place of NULL values +do_test shell1-3.14.1 { + catchcmd "test.db" ".nullvalue" +} {1 {Error: unknown command or invalid arguments: "nullvalue". Enter ".help" for help}} +do_test shell1-3.14.2 { + catchcmd "test.db" ".nullvalue FOO" +} {0 {}} +do_test shell1-3.14.3 { + # too many arguments + catchcmd "test.db" ".nullvalue FOO BAD" +} {1 {Error: unknown command or invalid arguments: "nullvalue". Enter ".help" for help}} + +# .output FILENAME Send output to FILENAME +do_test shell1-3.15.1 { + catchcmd "test.db" ".output" +} {1 {Error: unknown command or invalid arguments: "output". Enter ".help" for help}} +do_test shell1-3.15.2 { + catchcmd "test.db" ".output FOO" +} {0 {}} +do_test shell1-3.15.3 { + # too many arguments + catchcmd "test.db" ".output FOO BAD" +} {1 {Error: unknown command or invalid arguments: "output". Enter ".help" for help}} + +# .output stdout Send output to the screen +do_test shell1-3.16.1 { + catchcmd "test.db" ".output stdout" +} {0 {}} +do_test shell1-3.16.2 { + # too many arguments + catchcmd "test.db" ".output stdout BAD" +} {1 {Error: unknown command or invalid arguments: "output". Enter ".help" for help}} + +# .prompt MAIN CONTINUE Replace the standard prompts +do_test shell1-3.17.1 { + catchcmd "test.db" ".prompt" +} {1 {Error: unknown command or invalid arguments: "prompt". Enter ".help" for help}} +do_test shell1-3.17.2 { + catchcmd "test.db" ".prompt FOO" +} {0 {}} +do_test shell1-3.17.3 { + catchcmd "test.db" ".prompt FOO BAR" +} {0 {}} +do_test shell1-3.17.4 { + # too many arguments + catchcmd "test.db" ".prompt FOO BAR BAD" +} {1 {Error: unknown command or invalid arguments: "prompt". Enter ".help" for help}} + +# .quit Exit this program +do_test shell1-3.18.1 { + catchcmd "test.db" ".quit" +} {0 {}} +do_test shell1-3.18.2 { + # too many arguments + catchcmd "test.db" ".quit BAD" +} {1 {Error: unknown command or invalid arguments: "quit". Enter ".help" for help}} + +# .read FILENAME Execute SQL in FILENAME +do_test shell1-3.19.1 { + catchcmd "test.db" ".read" +} {1 {Error: unknown command or invalid arguments: "read". Enter ".help" for help}} +do_test shell1-3.19.2 { + file delete -force FOO + catchcmd "test.db" ".read FOO" +} {1 {Error: cannot open "FOO"}} +do_test shell1-3.19.3 { + # too many arguments + catchcmd "test.db" ".read FOO BAD" +} {1 {Error: unknown command or invalid arguments: "read". Enter ".help" for help}} + +# .schema ?TABLE? Show the CREATE statements +# If TABLE specified, only show tables matching +# LIKE pattern TABLE. +do_test shell1-3.21.1 { + catchcmd "test.db" ".schema" +} {0 {}} +do_test shell1-3.21.2 { + catchcmd "test.db" ".schema FOO" +} {0 {}} +do_test shell1-3.21.3 { + # too many arguments + catchcmd "test.db" ".schema FOO BAD" +} {1 {Error: unknown command or invalid arguments: "schema". Enter ".help" for help}} + +do_test shell1-3.21.4 { + catchcmd "test.db" { + CREATE TABLE t1(x); + CREATE VIEW v2 AS SELECT x+1 AS y FROM t1; + CREATE VIEW v1 AS SELECT y+1 FROM v2; + } + catchcmd "test.db" ".schema" +} {0 {CREATE TABLE t1(x); +CREATE VIEW v2 AS SELECT x+1 AS y FROM t1; +CREATE VIEW v1 AS SELECT y+1 FROM v2;}} +db eval {DROP VIEW v1; DROP VIEW v2; DROP TABLE t1;} + +# .separator STRING Change separator used by output mode and .import +do_test shell1-3.22.1 { + catchcmd "test.db" ".separator" +} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} +do_test shell1-3.22.2 { + catchcmd "test.db" ".separator FOO" +} {0 {}} +do_test shell1-3.22.3 { + # too many arguments + catchcmd "test.db" ".separator FOO BAD" +} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} + +# .show Show the current values for various settings +do_test shell1-3.23.1 { + set res [catchcmd "test.db" ".show"] + list [regexp {echo:} $res] \ + [regexp {explain:} $res] \ + [regexp {headers:} $res] \ + [regexp {mode:} $res] \ + [regexp {nullvalue:} $res] \ + [regexp {output:} $res] \ + [regexp {separator:} $res] \ + [regexp {stats:} $res] \ + [regexp {width:} $res] +} {1 1 1 1 1 1 1 1 1} +do_test shell1-3.23.2 { + # too many arguments + catchcmd "test.db" ".show BAD" +} {1 {Error: unknown command or invalid arguments: "show". Enter ".help" for help}} + +# .stats ON|OFF Turn stats on or off +do_test shell1-3.23b.1 { + catchcmd "test.db" ".stats" +} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} +do_test shell1-3.23b.2 { + catchcmd "test.db" ".stats ON" +} {0 {}} +do_test shell1-3.23b.3 { + catchcmd "test.db" ".stats OFF" +} {0 {}} +do_test shell1-3.23b.4 { + # too many arguments + catchcmd "test.db" ".stats OFF BAD" +} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} + +# .tables ?TABLE? List names of tables +# If TABLE specified, only list tables matching +# LIKE pattern TABLE. +do_test shell1-3.24.1 { + catchcmd "test.db" ".tables" +} {0 {}} +do_test shell1-3.24.2 { + catchcmd "test.db" ".tables FOO" +} {0 {}} +do_test shell1-3.24.3 { + # too many arguments + catchcmd "test.db" ".tables FOO BAD" +} {1 {Error: unknown command or invalid arguments: "tables". Enter ".help" for help}} + +# .width NUM NUM ... Set column widths for "column" mode +do_test shell1-3.26.1 { + catchcmd "test.db" ".width" +} {1 {Error: unknown command or invalid arguments: "width". Enter ".help" for help}} +do_test shell1-3.26.2 { + catchcmd "test.db" ".width xxx" + # this should be treated the same as a '0' width for col 1 +} {0 {}} +do_test shell1-3.26.3 { + catchcmd "test.db" ".width xxx yyy" + # this should be treated the same as a '0' width for col 1 and 2 +} {0 {}} +do_test shell1-3.26.4 { + catchcmd "test.db" ".width 1 1" + # this should be treated the same as a '1' width for col 1 and 2 +} {0 {}} +do_test shell1-3.26.5 { + catchcmd "test.db" ".mode column\n.width 10 -10\nSELECT 'abcdefg', 123456;" + # this should be treated the same as a '1' width for col 1 and 2 +} {0 {abcdefg 123456}} +do_test shell1-3.26.6 { + catchcmd "test.db" ".mode column\n.width -10 10\nSELECT 'abcdefg', 123456;" + # this should be treated the same as a '1' width for col 1 and 2 +} {0 { abcdefg 123456 }} + + +# .timer ON|OFF Turn the CPU timer measurement on or off +do_test shell1-3.27.1 { + catchcmd "test.db" ".timer" +} {1 {Error: unknown command or invalid arguments: "timer". Enter ".help" for help}} +do_test shell1-3.27.2 { + catchcmd "test.db" ".timer ON" +} {0 {}} +do_test shell1-3.27.3 { + catchcmd "test.db" ".timer OFF" +} {0 {}} +do_test shell1-3.27.4 { + # too many arguments + catchcmd "test.db" ".timer OFF BAD" +} {1 {Error: unknown command or invalid arguments: "timer". Enter ".help" for help}} + +do_test shell1-3-28.1 { + catchcmd test.db \ + ".log stdout\nSELECT coalesce(sqlite_log(123,'hello'),'456');" +} "0 {(123) hello\n456}" + +do_test shell1-3-29.1 { + catchcmd "test.db" ".print this is a test" +} {0 {this is a test}} + +# Test the output of the ".dump" command +# +do_test shell1-4.1 { + db eval { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(null); + INSERT INTO t1 VALUES(''); + INSERT INTO t1 VALUES(1); + INSERT INTO t1 VALUES(2.25); + INSERT INTO t1 VALUES('hello'); + INSERT INTO t1 VALUES(x'807f'); + } + catchcmd test.db {.dump} +} {0 {PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE t1(x); +INSERT INTO "t1" VALUES(NULL); +INSERT INTO "t1" VALUES(''); +INSERT INTO "t1" VALUES(1); +INSERT INTO "t1" VALUES(2.25); +INSERT INTO "t1" VALUES('hello'); +INSERT INTO "t1" VALUES(X'807F'); +COMMIT;}} + +# Test the output of ".mode insert" +# +do_test shell1-4.2 { + catchcmd test.db ".mode insert t1\nselect * from t1;" +} {0 {INSERT INTO t1 VALUES(NULL); +INSERT INTO t1 VALUES(''); +INSERT INTO t1 VALUES(1); +INSERT INTO t1 VALUES(2.25); +INSERT INTO t1 VALUES('hello'); +INSERT INTO t1 VALUES(X'807f');}} + +# Test the output of ".mode tcl" +# +do_test shell1-4.3 { + catchcmd test.db ".mode tcl\nselect * from t1;" +} {0 {"" +"" +"1" +"2.25" +"hello" +"\200\177"}} + +# Test the output of ".mode tcl" with multiple columns +# +do_test shell1-4.4 { + db eval { + CREATE TABLE t2(x,y); + INSERT INTO t2 VALUES(null, ''); + INSERT INTO t2 VALUES(1, 2.25); + INSERT INTO t2 VALUES('hello', x'807f'); + } + catchcmd test.db ".mode tcl\nselect * from t2;" +} {0 {"" "" +"1" "2.25" +"hello" "\200\177"}} + +# Test the output of ".mode tcl" with ".nullvalue" +# +do_test shell1-4.5 { + catchcmd test.db ".mode tcl\n.nullvalue NULL\nselect * from t2;" +} {0 {"NULL" "" +"1" "2.25" +"hello" "\200\177"}} + +# Test the output of ".mode tcl" with Tcl reserved characters +# +do_test shell1-4.6 { + db eval { + CREATE TABLE tcl1(x); + INSERT INTO tcl1 VALUES('"'); + INSERT INTO tcl1 VALUES('['); + INSERT INTO tcl1 VALUES(']'); + INSERT INTO tcl1 VALUES('\{'); + INSERT INTO tcl1 VALUES('\}'); + INSERT INTO tcl1 VALUES(';'); + INSERT INTO tcl1 VALUES('$'); + } + foreach {x y} [catchcmd test.db ".mode tcl\nselect * from tcl1;"] break + list $x $y [llength $y] +} {0 {"\"" +"[" +"]" +"\\{" +"\\}" +";" +"$"} 7} + +finish_test ADDED test/shell2.test Index: test/shell2.test ================================================================== --- /dev/null +++ test/shell2.test @@ -0,0 +1,197 @@ +# 2009 Nov 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the CLI shell tool. +# +# $Id: shell2.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ +# + +# Test plan: +# +# shell2-1.*: Misc. test of various tickets and reported errors. +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl +if {$tcl_platform(platform)=="windows"} { + set CLI "sqlite4.exe" +} else { + set CLI "./sqlite4" +} +if {![file executable $CLI]} { + finish_test + return +} +db close +forcedelete test.db test.db-journal test.db-wal +sqlite4 db test.db + + +#---------------------------------------------------------------------------- +# shell2-1.*: Misc. test of various tickets and reported errors. +# + +# Batch mode not creating databases. +# Reported on mailing list by Ken Zalewski. +# Ticket [aeff892c57]. +do_test shell2-1.1.1 { + file delete -force foo.db + set rc [ catchcmd "-batch foo.db" "CREATE TABLE t1(a);" ] + set fexist [file exist foo.db] + list $rc $fexist +} {{0 {}} 1} + +# Shell silently ignores extra parameters. +# Ticket [f5cb008a65]. +do_test shell2-1.2.1 { + set rc [catch { eval exec $CLI \":memory:\" \"select 3\" \"select 4\" } msg] + list $rc \ + [regexp {Error: too many options: "select 4"} $msg] +} {1 1} + +# Test a problem reported on the mailing list. The shell was at one point +# returning the generic SQLITE_ERROR message ("SQL error or missing database") +# instead of the "too many levels..." message in the test below. +# +do_test shell2-1.3 { + catchcmd "-batch test.db" { + PRAGMA recursive_triggers = ON; + CREATE TABLE t5(a PRIMARY KEY, b, c); + INSERT INTO t5 VALUES(1, 2, 3); + CREATE TRIGGER au_tble AFTER UPDATE ON t5 BEGIN + UPDATE OR IGNORE t5 SET a = new.a, c = 10; + END; + + UPDATE OR REPLACE t5 SET a = 4 WHERE a = 1; + } +} {1 {Error: near line 9: too many levels of trigger recursion}} + + + +# Shell not echoing all commands with echo on. +# Ticket [eb620916be]. + +# Test with echo off +# NB. whitespace is important +do_test shell2-1.4.1 { + file delete -force foo.db + catchcmd "foo.db" {CREATE TABLE foo(a); +INSERT INTO foo(a) VALUES(1); +SELECT * FROM foo;} +} {0 1} + +# Test with echo on using command line option +# NB. whitespace is important +do_test shell2-1.4.2 { + file delete -force foo.db + catchcmd "-echo foo.db" {CREATE TABLE foo(a); +INSERT INTO foo(a) VALUES(1); +SELECT * FROM foo;} +} {0 {CREATE TABLE foo(a); +INSERT INTO foo(a) VALUES(1); +SELECT * FROM foo; +1}} + +# Test with echo on using dot command +# NB. whitespace is important +do_test shell2-1.4.3 { + file delete -force foo.db + catchcmd "foo.db" {.echo ON +CREATE TABLE foo(a); +INSERT INTO foo(a) VALUES(1); +SELECT * FROM foo;} +} {0 {CREATE TABLE foo(a); +INSERT INTO foo(a) VALUES(1); +SELECT * FROM foo; +1}} + +# Test with echo on using dot command and +# turning off mid- processing. +# NB. whitespace is important +do_test shell2-1.4.4 { + file delete -force foo.db + catchcmd "foo.db" {.echo ON +CREATE TABLE foo(a); +.echo OFF +INSERT INTO foo(a) VALUES(1); +SELECT * FROM foo;} +} {0 {CREATE TABLE foo(a); +.echo OFF +1}} + +# Test with echo on using dot command and +# multiple commands per line. +# NB. whitespace is important +do_test shell2-1.4.5 { + file delete -force foo.db + catchcmd "foo.db" {.echo ON +CREATE TABLE foo1(a); +INSERT INTO foo1(a) VALUES(1); +CREATE TABLE foo2(b); +INSERT INTO foo2(b) VALUES(1); +SELECT * FROM foo1; SELECT * FROM foo2; +INSERT INTO foo1(a) VALUES(2); INSERT INTO foo2(b) VALUES(2); +SELECT * FROM foo1; SELECT * FROM foo2; +} +} {0 {CREATE TABLE foo1(a); +INSERT INTO foo1(a) VALUES(1); +CREATE TABLE foo2(b); +INSERT INTO foo2(b) VALUES(1); +SELECT * FROM foo1; +1 +SELECT * FROM foo2; +1 +INSERT INTO foo1(a) VALUES(2); +INSERT INTO foo2(b) VALUES(2); +SELECT * FROM foo1; +1 +2 +SELECT * FROM foo2; +1 +2}} + +# Test with echo on and headers on using dot command and +# multiple commands per line. +# NB. whitespace is important +do_test shell2-1.4.6 { + file delete -force foo.db + catchcmd "foo.db" {.echo ON +.headers ON +CREATE TABLE foo1(a); +INSERT INTO foo1(a) VALUES(1); +CREATE TABLE foo2(b); +INSERT INTO foo2(b) VALUES(1); +SELECT * FROM foo1; SELECT * FROM foo2; +INSERT INTO foo1(a) VALUES(2); INSERT INTO foo2(b) VALUES(2); +SELECT * FROM foo1; SELECT * FROM foo2; +} +} {0 {.headers ON +CREATE TABLE foo1(a); +INSERT INTO foo1(a) VALUES(1); +CREATE TABLE foo2(b); +INSERT INTO foo2(b) VALUES(1); +SELECT * FROM foo1; +a +1 +SELECT * FROM foo2; +b +1 +INSERT INTO foo1(a) VALUES(2); +INSERT INTO foo2(b) VALUES(2); +SELECT * FROM foo1; +a +1 +2 +SELECT * FROM foo2; +b +1 +2}} + +finish_test ADDED test/shell3.test Index: test/shell3.test ================================================================== --- /dev/null +++ test/shell3.test @@ -0,0 +1,97 @@ +# 2009 Dec 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the CLI shell tool. +# +# $Id: shell2.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ +# + +# Test plan: +# +# shell3-1.*: Basic tests for running SQL statments from command line. +# shell3-2.*: Basic tests for running SQL file from command line. +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl +if {$tcl_platform(platform)=="windows"} { + set CLI "sqlite4.exe" +} else { + set CLI "./sqlite4" +} +if {![file executable $CLI]} { + finish_test + return +} +db close +forcedelete test.db test.db-journal test.db-wal +sqlite4 db test.db + +#---------------------------------------------------------------------------- +# shell3-1.*: Basic tests for running SQL statments from command line. +# + +# Run SQL statement from command line +do_test shell3-1.1 { + file delete -force foo.db + set rc [ catchcmd "foo.db \"CREATE TABLE t1(a);\"" ] + set fexist [file exist foo.db] + list $rc $fexist +} {{0 {}} 1} +do_test shell3-1.2 { + catchcmd "foo.db" ".tables" +} {0 t1} +do_test shell3-1.3 { + catchcmd "foo.db \"DROP TABLE t1;\"" +} {0 {}} +do_test shell3-1.4 { + catchcmd "foo.db" ".tables" +} {0 {}} +do_test shell3-1.5 { + catchcmd "foo.db \"CREATE TABLE t1(a); DROP TABLE t1;\"" +} {0 {}} +do_test shell3-1.6 { + catchcmd "foo.db" ".tables" +} {0 {}} +do_test shell3-1.7 { + catchcmd "foo.db \"CREATE TABLE\"" +} {1 {Error: near "TABLE": syntax error}} + +#---------------------------------------------------------------------------- +# shell3-2.*: Basic tests for running SQL file from command line. +# + +# Run SQL file from command line +do_test shell3-2.1 { + file delete -force foo.db + set rc [ catchcmd "foo.db" "CREATE TABLE t1(a);" ] + set fexist [file exist foo.db] + list $rc $fexist +} {{0 {}} 1} +do_test shell3-2.2 { + catchcmd "foo.db" ".tables" +} {0 t1} +do_test shell3-2.3 { + catchcmd "foo.db" "DROP TABLE t1;" +} {0 {}} +do_test shell3-2.4 { + catchcmd "foo.db" ".tables" +} {0 {}} +do_test shell3-2.5 { + catchcmd "foo.db" "CREATE TABLE t1(a); DROP TABLE t1;" +} {0 {}} +do_test shell3-2.6 { + catchcmd "foo.db" ".tables" +} {0 {}} +do_test shell3-2.7 { + catchcmd "foo.db" "CREATE TABLE" +} {1 {Error: incomplete SQL: CREATE TABLE}} + +finish_test ADDED test/shell4.test Index: test/shell4.test ================================================================== --- /dev/null +++ test/shell4.test @@ -0,0 +1,116 @@ +# 2010 July 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the CLI shell tool. +# These tests are specific to the .stats command. +# +# $Id: shell4.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ +# + +# Test plan: +# +# shell4-1.*: Basic tests specific to the "stats" command. +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl +if {$tcl_platform(platform)=="windows"} { + set CLI "sqlite4.exe" +} else { + set CLI "./sqlite4" +} +if {![file executable $CLI]} { + finish_test + return +} +db close +forcedelete test.db test.db-journal test.db-wal +sqlite4 db test.db + +#---------------------------------------------------------------------------- +# Test cases shell4-1.*: Tests specific to the "stats" command. +# + +# should default to off +do_test shell4-1.1.1 { + set res [catchcmd "test.db" ".show"] + list [regexp {stats: off} $res] +} {1} + +do_test shell4-1.1.2 { + set res [catchcmd "test.db" ".show"] + list [regexp {stats: on} $res] +} {0} + +# -stats should turn it on +do_test shell4-1.2.1 { + set res [catchcmd "-stats test.db" ".show"] + list [regexp {stats: on} $res] +} {1} + +do_test shell4-1.2.2 { + set res [catchcmd "-stats test.db" ".show"] + list [regexp {stats: off} $res] +} {0} + +# .stats ON|OFF Turn stats on or off +do_test shell4-1.3.1 { + catchcmd "test.db" ".stats" +} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} +do_test shell4-1.3.2 { + catchcmd "test.db" ".stats ON" +} {0 {}} +do_test shell4-1.3.3 { + catchcmd "test.db" ".stats OFF" +} {0 {}} +do_test shell4-1.3.4 { + # too many arguments + catchcmd "test.db" ".stats OFF BAD" +} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} + +# NB. whitespace is important +do_test shell4-1.4.1 { + set res [catchcmd "test.db" {.show}] + list [regexp {stats: off} $res] +} {1} + +do_test shell4-1.4.2 { + set res [catchcmd "test.db" {.stats ON +.show +}] + list [regexp {stats: on} $res] +} {1} + +do_test shell4-1.4.3 { + set res [catchcmd "test.db" {.stats OFF +.show +}] + list [regexp {stats: off} $res] +} {1} + +# make sure stats not present when off +do_test shell4-1.5.1 { + set res [catchcmd "test.db" {SELECT 1;}] + list [regexp {Memory Used} $res] \ + [regexp {Heap Usage} $res] \ + [regexp {Autoindex Inserts} $res] +} {0 0 0} + +# make sure stats are present when on +do_test shell4-1.5.2 { + set res [catchcmd "test.db" {.stats ON +SELECT 1; +}] + list [regexp {Memory Used} $res] \ + [regexp {Heap Usage} $res] \ + [regexp {Autoindex Inserts} $res] +} {1 1 1} + +finish_test ADDED test/shell5.test Index: test/shell5.test ================================================================== --- /dev/null +++ test/shell5.test @@ -0,0 +1,229 @@ +# 2010 August 4 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the CLI shell tool. +# These tests are specific to the .import command. +# +# $Id: shell5.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ +# + +# Test plan: +# +# shell5-1.*: Basic tests specific to the ".import" command. +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl +if {$tcl_platform(platform)=="windows"} { + set CLI "sqlite4.exe" +} else { + set CLI "./sqlite4" +} +if {![file executable $CLI]} { + finish_test + return +} +db close +forcedelete test.db test.db-journal test.db-wal +sqlite4 db test.db + +#---------------------------------------------------------------------------- +# Test cases shell5-1.*: Basic handling of the .import and .separator commands. +# + +# .import FILE TABLE Import data from FILE into TABLE +do_test shell5-1.1.1 { + catchcmd "test.db" ".import" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} +do_test shell5-1.1.2 { + catchcmd "test.db" ".import FOO" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} +do_test shell5-1.1.2 { + catchcmd "test.db" ".import FOO BAR" +} {1 {Error: no such table: BAR}} +do_test shell5-1.1.3 { + # too many arguments + catchcmd "test.db" ".import FOO BAR BAD" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} + +# .separator STRING Change separator used by output mode and .import +do_test shell1-1.2.1 { + catchcmd "test.db" ".separator" +} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} +do_test shell1-1.2.2 { + catchcmd "test.db" ".separator FOO" +} {0 {}} +do_test shell1-1.2.3 { + # too many arguments + catchcmd "test.db" ".separator FOO BAD" +} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} + +# separator should default to "|" +do_test shell5-1.3.1 { + set res [catchcmd "test.db" ".show"] + list [regexp {separator: \"\|\"} $res] +} {1} + +# set separator to different value. +# check that .show reports new value +do_test shell5-1.3.2 { + set res [catchcmd "test.db" {.separator , +.show}] + list [regexp {separator: \",\"} $res] +} {1} + +# import file doesn't exist +do_test shell5-1.4.1 { + file delete -force FOO + set res [catchcmd "test.db" {CREATE TABLE t1(a, b); +.import FOO t1}] +} {1 {Error: cannot open "FOO"}} + +# empty import file +do_test shell5-1.4.2 { + file delete -force shell5.csv + set in [open shell5.csv w] + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 0} + +# import file with 1 row, 1 column (expecting 2 cols) +do_test shell5-1.4.3 { + set in [open shell5.csv w] + puts $in "1" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1}] +} {1 {Error: shell5.csv line 1: expected 2 columns of data but found 1}} + +# import file with 1 row, 3 columns (expecting 2 cols) +do_test shell5-1.4.4 { + set in [open shell5.csv w] + puts $in "1|2|3" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1}] +} {1 {Error: shell5.csv line 1: expected 2 columns of data but found 3}} + +# import file with 1 row, 2 columns +do_test shell5-1.4.5 { + set in [open shell5.csv w] + puts $in "1|2" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 1} + +# import file with 2 rows, 2 columns +# note we end up with 3 rows because of the 1 row +# imported above. +do_test shell5-1.4.6 { + set in [open shell5.csv w] + puts $in "2|3" + puts $in "3|4" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 3} + +# import file with 1 row, 2 columns, using a comma +do_test shell5-1.4.7 { + set in [open shell5.csv w] + puts $in "4,5" + close $in + set res [catchcmd "test.db" {.separator , +.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 4} + +# import file with 1 row, 2 columns, text data +do_test shell5-1.4.8.1 { + set in [open shell5.csv w] + puts $in "5|Now is the time for all good men to come to the aid of their country." + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 5} + +do_test shell5-1.4.8.2 { + catchcmd "test.db" {SELECT b FROM t1 WHERE a='5';} +} {0 {Now is the time for all good men to come to the aid of their country.}} + +# import file with 1 row, 2 columns, quoted text data +# note that currently sqlite doesn't support quoted fields, and +# imports the entire field, quotes and all. +do_test shell5-1.4.9.1 { + set in [open shell5.csv w] + puts $in "6|'Now is the time for all good men to come to the aid of their country.'" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 6} + +do_test shell5-1.4.9.2 { + catchcmd "test.db" {SELECT b FROM t1 WHERE a='6';} +} {0 {'Now is the time for all good men to come to the aid of their country.'}} + +# import file with 1 row, 2 columns, quoted text data +do_test shell5-1.4.10.1 { + set in [open shell5.csv w] + puts $in "7|\"Now is the time for all good men to come to the aid of their country.\"" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 7} + +do_test shell5-1.4.10.2 { + catchcmd "test.db" {SELECT b FROM t1 WHERE a='7';} +} {0 {Now is the time for all good men to come to the aid of their country.}} + +# check importing very long field +do_test shell5-1.5.1 { + set str [string repeat X 999] + set in [open shell5.csv w] + puts $in "8|$str" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT length(b) FROM t1 WHERE a='8';}] +} {0 999} + +# try importing into a table with a large number of columns. +# This is limited by SQLITE_MAX_VARIABLE_NUMBER, which defaults to 999. +set cols 999 +do_test shell5-1.6.1 { + set sql {CREATE TABLE t2(} + set data {} + for {set i 1} {$i<$cols} {incr i} { + append sql "c$i," + append data "$i|" + } + append sql "c$cols);" + append data "$cols" + catchcmd "test.db" $sql + set in [open shell5.csv w] + puts $in $data + close $in + set res [catchcmd "test.db" {.import shell5.csv t2 +SELECT COUNT(*) FROM t2;}] +} {0 1} + +# try importing a large number of rows +set rows 999999 +do_test shell5-1.7.1 { + set in [open shell5.csv w] + for {set i 1} {$i<=$rows} {incr i} { + puts $in $i + } + close $in + set res [catchcmd "test.db" {CREATE TABLE t3(a); +.import shell5.csv t3 +SELECT COUNT(*) FROM t3;}] +} [list 0 $rows] + +finish_test Index: test/simple.test ================================================================== --- test/simple.test +++ test/simple.test @@ -98,10 +98,11 @@ #------------------------------------------------------------------------- reset_db +breakpoint do_execsql_test 4.1 { CREATE TABLE t1(k PRIMARY KEY, v) } do_execsql_test 4.2 { CREATE INDEX i1 ON t1(v) } do_execsql_test 4.3 { SELECT * FROM sqlite_master Index: test/subquery.test ================================================================== --- test/subquery.test +++ test/subquery.test @@ -387,11 +387,11 @@ INSERT INTO t5 VALUES(1,11); INSERT INTO t5 VALUES(2,22); INSERT INTO t5 VALUES(3,33); INSERT INTO t5 VALUES(4,44); SELECT b FROM t5 WHERE a IN - (SELECT callcnt(y)+0 FROM t4 WHERE x="two") + (SELECT callcnt(y)+0 FROM t4 WHERE x='two') } } {22} do_test subquery-5.2 { # This is the key test. The subquery should have only run once. If # The double-quoted identifier "two" were causing the subquery to be Index: test/substr.test ================================================================== --- test/substr.test +++ test/substr.test @@ -126,15 +126,15 @@ # If these blobs were strings, then they would contain multi-byte # characters. But since they are blobs, the substr indices refer # to bytes. # subblob-test 4.1 61E188B462E28D8563E3919663 1 1 61 -subblob-test 4.2 61E188B462E28D8563E3919663 2 1 E1 -subblob-test 4.3 61E188B462E28D8563E3919663 1 2 61E1 +subblob-test 4.2 61E188B462E28D8563E3919663 2 1 e1 +subblob-test 4.3 61E188B462E28D8563E3919663 1 2 61e1 subblob-test 4.4 61E188B462E28D8563E3919663 -2 1 96 -subblob-test 4.5 61E188B462E28D8563E3919663 -5 4 63E39196 -subblob-test 4.6 61E188B462E28D8563E3919663 -100 98 61E188B462E28D8563E391 +subblob-test 4.5 61E188B462E28D8563E3919663 -5 4 63e39196 +subblob-test 4.6 61E188B462E28D8563E3919663 -100 98 61e188b462e28d8563e391 # Two-argument SUBSTR # proc substr-2-test {id string idx result} { db eval { Index: test/test_func.c ================================================================== --- test/test_func.c +++ test/test_func.c @@ -67,11 +67,11 @@ sqlite4_randomness(pEnv, n, zBuf); for(i=0; ipEnv, p); } /* @@ -258,11 +263,11 @@ sqlite4_result_error_nomem(pCtx); return; } pCounter->cnt = sqlite4_value_int(argv[0]); pCounter->pEnv = sqlite4_context_env(pCtx); - sqlite4_set_auxdata(pCtx, 0, pCounter, counterFree); + sqlite4_set_auxdata(pCtx, 0, pCounter, counterFree, 0); }else{ pCounter->cnt++; } sqlite4_result_int(pCtx, pCounter->cnt); } @@ -321,11 +326,11 @@ char *zErr; sqlite4_env *pEnv = sqlite4_context_env(pCtx); assert( pStmt==0 ); zErr = sqlite4_mprintf(pEnv, "sqlite4_prepare() error: %s", sqlite4_errmsg(db)); - sqlite4_result_text(pCtx, zErr, -1, SQLITE4_DYNAMIC); + sqlite4_result_text(pCtx, zErr, -1, SQLITE4_DYNAMIC, 0); sqlite4_result_error_code(pCtx, rc); } } @@ -374,11 +379,11 @@ zOut = sqlite4_malloc(sqlite4_context_env(pCtx), n/2 ); if( zOut==0 ){ sqlite4_result_error_nomem(pCtx); }else{ testHexToBin(zIn, zOut); - sqlite4_result_text16be(pCtx, zOut, n/2, SQLITE4_DYNAMIC); + sqlite4_result_text16be(pCtx, zOut, n/2, SQLITE4_DYNAMIC, 0); } } #endif /* @@ -401,11 +406,11 @@ zOut = sqlite4_malloc(sqlite4_context_env(pCtx), n/2 ); if( zOut==0 ){ sqlite4_result_error_nomem(pCtx); }else{ testHexToBin(zIn, zOut); - sqlite4_result_text(pCtx, zOut, n/2, SQLITE4_DYNAMIC); + sqlite4_result_text(pCtx, zOut, n/2, SQLITE4_DYNAMIC, 0); } } /* ** hex_to_utf16le(HEX) @@ -428,11 +433,11 @@ zOut = sqlite4_malloc(sqlite4_context_env(pCtx), n/2 ); if( zOut==0 ){ sqlite4_result_error_nomem(pCtx); }else{ testHexToBin(zIn, zOut); - sqlite4_result_text16le(pCtx, zOut, n/2, SQLITE4_DYNAMIC); + sqlite4_result_text16le(pCtx, zOut, n/2, SQLITE4_DYNAMIC, 0); } } #endif static int registerTestFunctions(sqlite4 *db){ Index: test/test_kv2.c ================================================================== --- test/test_kv2.c +++ test/test_kv2.c @@ -14,11 +14,11 @@ #include "sqliteInt.h" static struct KVWrapGlobal { - int (*xFactory)(sqlite4_env*, KVStore **, const char *, unsigned int); + sqlite4_kvfactory xFactory; int nStep; /* Total number of successful next/prev */ int nSeek; /* Total number of calls to xSeek */ } kvwg = {0}; typedef struct KVWrap KVWrap; @@ -289,12 +289,25 @@ Tcl_WrongNumArgs(interp, 2, objv, ""); return TCL_ERROR; } if( kvwg.xFactory==0 ){ - sqlite4_env_config(0, SQLITE4_ENVCONFIG_KVSTORE_GET, "main", &kvwg.xFactory); - sqlite4_env_config(0, SQLITE4_ENVCONFIG_KVSTORE_PUSH, "main",newFileStorage); + sqlite4_env_config(0, SQLITE4_ENVCONFIG_KVSTORE_GET,"main", &kvwg.xFactory); + sqlite4_env_config(0, SQLITE4_ENVCONFIG_KVSTORE_PUSH,"main",newFileStorage); + } + return TCL_OK; +} + +static int kvwrap_uninstall_cmd(Tcl_Interp *interp, int objc, Tcl_Obj **objv){ + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 2, objv, ""); + return TCL_ERROR; + } + + if( kvwg.xFactory ){ + sqlite4_env_config(0, SQLITE4_ENVCONFIG_KVSTORE_POP,"main", &kvwg.xFactory); + kvwg.xFactory = 0; } return TCL_OK; } static int kvwrap_seek_cmd(Tcl_Interp *interp, int objc, Tcl_Obj **objv){ @@ -342,14 +355,15 @@ ){ struct Subcmd { const char *zCmd; int (*xCmd)(Tcl_Interp *, int, Tcl_Obj **); } aSub[] = { - { "install", kvwrap_install_cmd }, - { "step", kvwrap_step_cmd }, - { "seek", kvwrap_seek_cmd }, - { "reset", kvwrap_reset_cmd }, + { "install", kvwrap_install_cmd }, + { "step", kvwrap_step_cmd }, + { "seek", kvwrap_seek_cmd }, + { "reset", kvwrap_reset_cmd }, + { "uninstall", kvwrap_uninstall_cmd }, }; int iSub; int rc; rc = Tcl_GetIndexFromObjStruct( Index: test/test_lsm.c ================================================================== --- test/test_lsm.c +++ test/test_lsm.c @@ -7,11 +7,10 @@ ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* -** */ #include #include "lsm.h" #include "sqlite4.h" @@ -18,10 +17,137 @@ #include #include extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite4 **ppDb); extern const char *sqlite4TestErrorName(int); + +/************************************************************************* +*/ +#define ENCRYPTION_XOR_MASK 0x23b2bbb6 +static int testCompressEncBound(void *pCtx, int nSrc){ + return nSrc; +} +static int testCompressEncCompress( + void *pCtx, + char *pOut, int *pnOut, + const char *pIn, int nIn +){ + int i; + unsigned int *aIn = (unsigned int *)pOut; + unsigned int *aOut = (unsigned int *)pIn; + + assert( (nIn%4)==0 ); + for(i=0; i<(nIn/4); i++){ + aOut[i] = (aIn[i] ^ ENCRYPTION_XOR_MASK); + } + *pnOut = nIn; + + return LSM_OK; +} +static int testCompressEncUncompress( + void *pCtx, + char *pOut, int *pnOut, + const char *pIn, int nIn +){ + return testCompressEncCompress(pCtx, pOut, pnOut, pIn, nIn); +} +static void testCompressEncFree(void *pCtx){ + /* no-op */ +} +/* +** End of compression routines "encrypt". +*************************************************************************/ + +/************************************************************************* +*/ +static int testCompressRleBound(void *pCtx, int nSrc){ + return nSrc*2; +} +static int testCompressRleCompress( + void *pCtx, + char *pOut, int *pnOut, + const char *pIn, int nIn +){ + int iOut = 0; + int i; + char c; + int n; + + c = pIn[0]; + n = 1; + for(i=1; idb); ckfree((char *)p); } } + +static int testInfoLsm(Tcl_Interp *interp, lsm_db *db, Tcl_Obj *pObj){ + struct Lsminfo { + const char *zOpt; + int eOpt; + } aInfo[] = { + { "compression_id", LSM_INFO_COMPRESSION_ID }, + { 0, 0 } + }; + int rc; + int iOpt; + + rc = Tcl_GetIndexFromObjStruct( + interp, pObj, aInfo, sizeof(aInfo[0]), "option", 0, &iOpt + ); + if( rc==LSM_OK ){ + switch( aInfo[iOpt].eOpt ){ + case LSM_INFO_COMPRESSION_ID: { + unsigned int iCmpId = 0; + rc = lsm_info(db, LSM_INFO_COMPRESSION_ID, &iCmpId); + if( rc==LSM_OK ){ + Tcl_SetObjResult(interp, Tcl_NewWideIntObj((Tcl_WideInt)iCmpId)); + }else{ + test_lsm_error(interp, "lsm_info", rc); + } + break; + } + } + } + + return rc; +} /* ** Usage: CSR sub-command ... */ static int test_lsm_cursor_cmd( @@ -519,10 +756,11 @@ /* 7 */ {"csr_open", 1, "CSR"}, /* 8 */ {"work", -1, "?NMERGE? NPAGE"}, /* 9 */ {"flush", 0, ""}, /* 10 */ {"config", 1, "LIST"}, /* 11 */ {"checkpoint", 0, ""}, + /* 12 */ {"info", 1, "OPTION"}, {0, 0, 0} }; int iCmd; int rc; TclLsm *p = (TclLsm *)clientData; @@ -621,11 +859,10 @@ case 8: assert( 0==strcmp(aCmd[8].zCmd, "work") ); { int nWork = 0; int nMerge = 1; int nWrite = 0; - int i; if( objc==3 ){ rc = Tcl_GetIntFromObj(interp, objv[2], &nWork); }else if( objc==4 ){ rc = Tcl_GetIntFromObj(interp, objv[2], &nMerge); @@ -654,10 +891,14 @@ case 11: assert( 0==strcmp(aCmd[11].zCmd, "checkpoint") ); { rc = lsm_checkpoint(p->db, 0); return test_lsm_error(interp, "lsm_checkpoint", rc); } + + case 12: assert( 0==strcmp(aCmd[12].zCmd, "info") ); { + return testInfoLsm(interp, p->db, objv[2]); + } default: assert( 0 ); } Index: test/test_main.c ================================================================== --- test/test_main.c +++ test/test_main.c @@ -684,11 +684,11 @@ int i; for(i=0; imutex); pVal = sqlite4ValueNew(db); - sqlite4ValueSetStr(pVal, -1, "x_sqlite_exec", SQLITE4_UTF8, SQLITE4_STATIC); + sqlite4ValueSetStr(pVal, -1, "x_sqlite_exec", SQLITE4_UTF8, + SQLITE4_STATIC, 0); zUtf16 = sqlite4ValueText(pVal, SQLITE4_UTF16NATIVE); if( db->mallocFailed ){ rc = SQLITE4_NOMEM; }else{ rc = sqlite4_create_function16(db, zUtf16, @@ -1706,22 +1707,22 @@ */ static void testFunc(sqlite4_context *context, int argc, sqlite4_value **argv){ while( argc>=2 ){ const char *zArg0 = (char*)sqlite4_value_text(argv[0]); if( zArg0 ){ - if( 0==sqlite4StrICmp(zArg0, "int") ){ + if( 0==sqlite4_stricmp(zArg0, "int") ){ sqlite4_result_int(context, sqlite4_value_int(argv[1])); - }else if( sqlite4StrICmp(zArg0,"int64")==0 ){ + }else if( sqlite4_stricmp(zArg0,"int64")==0 ){ sqlite4_result_int64(context, sqlite4_value_int64(argv[1])); - }else if( sqlite4StrICmp(zArg0,"string")==0 ){ + }else if( sqlite4_stricmp(zArg0,"string")==0 ){ sqlite4_result_text(context, (char*)sqlite4_value_text(argv[1]), -1, - SQLITE4_TRANSIENT); - }else if( sqlite4StrICmp(zArg0,"double")==0 ){ + SQLITE4_TRANSIENT, 0); + }else if( sqlite4_stricmp(zArg0,"double")==0 ){ sqlite4_result_double(context, sqlite4_value_double(argv[1])); - }else if( sqlite4StrICmp(zArg0,"null")==0 ){ + }else if( sqlite4_stricmp(zArg0,"null")==0 ){ sqlite4_result_null(context); - }else if( sqlite4StrICmp(zArg0,"value")==0 ){ + }else if( sqlite4_stricmp(zArg0,"value")==0 ){ sqlite4_result_value(context, argv[sqlite4_value_int(argv[1])]); }else{ goto error_out; } }else{ @@ -2098,18 +2099,18 @@ if( getStmtPointer(interp, argv[1], &pStmt) ) return TCL_ERROR; if( Tcl_GetInt(interp, argv[2], &idx) ) return TCL_ERROR; if( strcmp(argv[4],"null")==0 ){ rc = sqlite4_bind_null(pStmt, idx); }else if( strcmp(argv[4],"static")==0 ){ - rc = sqlite4_bind_text(pStmt, idx, sqlite_static_bind_value, -1, 0); + rc = sqlite4_bind_text(pStmt, idx, sqlite_static_bind_value, -1, 0, 0); }else if( strcmp(argv[4],"static-nbytes")==0 ){ rc = sqlite4_bind_text(pStmt, idx, sqlite_static_bind_value, - sqlite_static_bind_nbyte, 0); + sqlite_static_bind_nbyte, 0, 0); }else if( strcmp(argv[4],"normal")==0 ){ - rc = sqlite4_bind_text(pStmt, idx, argv[3], -1, SQLITE4_TRANSIENT); + rc = sqlite4_bind_text(pStmt, idx, argv[3], -1, SQLITE4_TRANSIENT, 0); }else if( strcmp(argv[4],"blob10")==0 ){ - rc = sqlite4_bind_text(pStmt, idx, "abc\000xyz\000pq", 10, SQLITE4_STATIC); + rc = sqlite4_bind_text(pStmt, idx, "abc\000xyz\000pq", 10,SQLITE4_STATIC,0); }else{ Tcl_AppendResult(interp, "4th argument should be " "\"null\" or \"static\" or \"normal\"", 0); return TCL_ERROR; } @@ -2185,15 +2186,15 @@ } sqlite4BeginBenignMalloc(pEnv); pVal = sqlite4ValueNew(0); if( pVal ){ - sqlite4ValueSetStr(pVal, nA, zA, encin, SQLITE4_STATIC); + sqlite4ValueSetStr(pVal, nA, zA, encin, SQLITE4_STATIC, 0); n = sqlite4_value_bytes(pVal); Tcl_ListObjAppendElement(i,pX, Tcl_NewStringObj((char*)sqlite4_value_text(pVal),n)); - sqlite4ValueSetStr(pVal, nB, zB, encin, SQLITE4_STATIC); + sqlite4ValueSetStr(pVal, nB, zB, encin, SQLITE4_STATIC, 0); n = sqlite4_value_bytes(pVal); Tcl_ListObjAppendElement(i,pX, Tcl_NewStringObj((char*)sqlite4_value_text(pVal),n)); sqlite4ValueFree(pVal); } @@ -2387,16 +2388,17 @@ Tcl_ListObjAppendElement(interp, pX, Tcl_NewStringObj("UTF-8", -1)); Tcl_ListObjAppendElement(interp, pX, Tcl_NewStringObj((char*)sqlite4_value_text(argv[0]), -1)); Tcl_EvalObjEx(interp, pX, 0); Tcl_DecrRefCount(pX); - sqlite4_result_text(pCtx, Tcl_GetStringResult(interp), -1, SQLITE4_TRANSIENT); + sqlite4_result_text(pCtx, Tcl_GetStringResult(interp), -1, + SQLITE4_TRANSIENT, 0); pVal = sqlite4ValueNew(0); sqlite4ValueSetStr(pVal, -1, Tcl_GetStringResult(interp), - SQLITE4_UTF8, SQLITE4_STATIC); + SQLITE4_UTF8, SQLITE4_STATIC, 0); sqlite4_result_text16be(pCtx, sqlite4_value_text16be(pVal), - -1, SQLITE4_TRANSIENT); + -1, SQLITE4_TRANSIENT, 0); sqlite4ValueFree(pVal); } static void test_function_utf16le( sqlite4_context *pCtx, int nArg, @@ -2413,12 +2415,13 @@ Tcl_NewStringObj((char*)sqlite4_value_text(argv[0]), -1)); Tcl_EvalObjEx(interp, pX, 0); Tcl_DecrRefCount(pX); pVal = sqlite4ValueNew(0); sqlite4ValueSetStr(pVal, -1, Tcl_GetStringResult(interp), - SQLITE4_UTF8, SQLITE4_STATIC); - sqlite4_result_text(pCtx,(char*)sqlite4_value_text(pVal),-1,SQLITE4_TRANSIENT); + SQLITE4_UTF8, SQLITE4_STATIC, 0); + sqlite4_result_text(pCtx, (char*)sqlite4_value_text(pVal), -1, + SQLITE4_TRANSIENT, 0); sqlite4ValueFree(pVal); } static void test_function_utf16be( sqlite4_context *pCtx, int nArg, @@ -2435,17 +2438,17 @@ Tcl_NewStringObj((char*)sqlite4_value_text(argv[0]), -1)); Tcl_EvalObjEx(interp, pX, 0); Tcl_DecrRefCount(pX); pVal = sqlite4ValueNew(0); sqlite4ValueSetStr(pVal, -1, Tcl_GetStringResult(interp), - SQLITE4_UTF8, SQLITE4_STATIC); + SQLITE4_UTF8, SQLITE4_STATIC, 0); sqlite4_result_text16(pCtx, sqlite4_value_text16le(pVal), - -1, SQLITE4_TRANSIENT); + -1, SQLITE4_TRANSIENT, 0); sqlite4_result_text16be(pCtx, sqlite4_value_text16le(pVal), - -1, SQLITE4_TRANSIENT); + -1, SQLITE4_TRANSIENT, 0); sqlite4_result_text16le(pCtx, sqlite4_value_text16le(pVal), - -1, SQLITE4_TRANSIENT); + -1, SQLITE4_TRANSIENT, 0); sqlite4ValueFree(pVal); } #endif /* SQLITE4_OMIT_UTF16 */ static int test_function( void * clientData, @@ -2786,11 +2789,11 @@ if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; if( Tcl_GetIntFromObj(interp, objv[2], &idx) ) return TCL_ERROR; value = (char*)Tcl_GetByteArrayFromObj(objv[3], &bytes); if( Tcl_GetIntFromObj(interp, objv[4], &bytes) ) return TCL_ERROR; - rc = sqlite4_bind_text(pStmt, idx, value, bytes, SQLITE4_TRANSIENT); + rc = sqlite4_bind_text(pStmt, idx, value, bytes, SQLITE4_TRANSIENT, 0); if( sqlite4TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; if( rc!=SQLITE4_OK ){ Tcl_AppendResult(interp, sqlite4TestErrorName(rc), 0); return TCL_ERROR; } @@ -2834,11 +2837,11 @@ if( getStmtPointer(interp, Tcl_GetString(oStmt), &pStmt) ) return TCL_ERROR; if( Tcl_GetIntFromObj(interp, oN, &idx) ) return TCL_ERROR; value = (char*)Tcl_GetByteArrayFromObj(oString, 0); if( Tcl_GetIntFromObj(interp, oBytes, &bytes) ) return TCL_ERROR; - rc = sqlite4_bind_text16(pStmt, idx, (void *)value, bytes, xDel); + rc = sqlite4_bind_text16(pStmt, idx, (void *)value, bytes, xDel, 0); if( sqlite4TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; if( rc!=SQLITE4_OK ){ Tcl_AppendResult(interp, sqlite4TestErrorName(rc), 0); return TCL_ERROR; } @@ -2881,11 +2884,11 @@ if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; if( Tcl_GetIntFromObj(interp, objv[2], &idx) ) return TCL_ERROR; value = Tcl_GetString(objv[3]); if( Tcl_GetIntFromObj(interp, objv[4], &bytes) ) return TCL_ERROR; - rc = sqlite4_bind_blob(pStmt, idx, value, bytes, xDestructor); + rc = sqlite4_bind_blob(pStmt, idx, value, bytes, xDestructor, 0); if( sqlite4TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; if( rc!=SQLITE4_OK ){ return TCL_ERROR; } @@ -4347,10 +4350,209 @@ return TCL_ERROR; } sqlite4_test_control(SQLITE4_TESTCTRL_OPTIMIZATIONS, db, mask); return TCL_OK; } + +#define NUM_FORMAT "sign:%d approx:%d e:%d m:%lld" + +/* Append a return value representing a sqlite4_num. +*/ +static void append_num_result( Tcl_Interp *interp, sqlite4_num A ){ + char buf[100]; + sprintf( buf, NUM_FORMAT, A.sign, A.approx, A.e, A.m ); + Tcl_AppendResult(interp, buf, 0); +} + +/* Convert a string either representing a sqlite4_num (listing its fields as +** returned by append_num_result) or that can be parsed as one. Invalid +** strings become NaN. +*/ +static sqlite4_num test_parse_num( char *arg ){ + sqlite4_num A; + int sign, approx, e; + if( sscanf( arg, NUM_FORMAT, &sign, &approx, &e, &A.m)==4 ){ + A.sign = sign; + A.approx = approx; + A.e = e; + return A; + } else { + return sqlite4_num_from_text(arg, -1, 0); + } +} + +/* Convert return values of sqlite4_num to strings that will be readable in +** the tests. +*/ +static char *describe_num_comparison( int code ){ + switch( code ){ + case 0: return "incomparable"; + case 1: return "lesser"; + case 2: return "equal"; + case 3: return "greater"; + default: return "error"; + } +} + +/* Compare two numbers A and B. Returns "incomparable", "lesser", "equal", +** "greater", or "error". +*/ +static int test_num_compare( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + sqlite4_num A, B; + int cmp; + if( argc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " NUM NUM\"", 0); + return TCL_ERROR; + } + + A = test_parse_num( argv[1] ); + B = test_parse_num( argv[2] ); + cmp = sqlite4_num_compare(A, B); + Tcl_AppendResult( interp, describe_num_comparison( cmp ), 0); + return TCL_OK; +} + +/* Create a sqlite4_num from a string. The optional second argument specifies +** how many bytes may be read. +*/ +static int test_num_from_text( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + sqlite4_num A; + int len; + if( argc!=2 && argc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " STRING\" or \"", argv[0], " STRING INTEGER\"", 0); + return TCL_ERROR; + } + + if( argc==3 ){ + if ( Tcl_GetInt(interp, argv[2], &len) ) return TCL_ERROR; + }else{ + len = -1; + } + + A = sqlite4_num_from_text( argv[1], len, 0 ); + append_num_result(interp, A); + return TCL_OK; +} + +static int test_num_to_text( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + char text[30]; + if( argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " NUM\"", 0); + return TCL_ERROR; + } + sqlite4_num_to_text( test_parse_num( argv[1] ), text ); + Tcl_AppendResult( interp, text, 0 ); + return TCL_OK; +} + +static int test_num_binary_op( + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv, /* Text of each argument */ + sqlite4_num (*op) (sqlite4_num, sqlite4_num) +){ + sqlite4_num A, B, R; + if( argc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " NUM NUM\"", 0); + return TCL_ERROR; + } + A = test_parse_num(argv[1]); + B = test_parse_num(argv[2]); + R = op(A, B); + append_num_result(interp, R); + return TCL_OK; +} + +static int test_num_add( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + return test_num_binary_op( interp, argc, argv, sqlite4_num_add ); +} + +static int test_num_sub( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + return test_num_binary_op( interp, argc, argv, sqlite4_num_sub ); +} + +static int test_num_mul( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + return test_num_binary_op( interp, argc, argv, sqlite4_num_mul ); +} + +static int test_num_div( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + return test_num_binary_op( interp, argc, argv, sqlite4_num_div ); +} + +static int test_num_predicate( + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv, /* Text of each argument */ + int (*pred) (sqlite4_num) +){ + sqlite4_num A; + if( argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " NUM\"", 0); + return TCL_ERROR; + } + A = test_parse_num(argv[1]); + Tcl_AppendResult(interp, pred(A) ? "true" : "false", 0); + return TCL_OK; +} + +static int test_num_isinf( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + return test_num_predicate( interp, argc, argv, sqlite4_num_isinf ); +} + +static int test_num_isnan( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + return test_num_predicate( interp, argc, argv, sqlite4_num_isnan ); +} /* ** Register commands with the TCL interpreter. */ int Sqlitetest1_Init(Tcl_Interp *interp){ @@ -4400,11 +4602,20 @@ { "sqlite_delete_function", (Tcl_CmdProc*)delete_function }, { "sqlite_delete_collation", (Tcl_CmdProc*)delete_collation }, { "sqlite4_get_autocommit", (Tcl_CmdProc*)get_autocommit }, { "sqlite4_stack_used", (Tcl_CmdProc*)test_stack_used }, { "printf", (Tcl_CmdProc*)test_printf }, - { "sqlite4IoTrace", (Tcl_CmdProc*)test_io_trace }, + { "sqlite4IoTrace", (Tcl_CmdProc*)test_io_trace }, + { "sqlite4_num_compare", (Tcl_CmdProc*)test_num_compare }, + { "sqlite4_num_from_text", (Tcl_CmdProc*)test_num_from_text }, + { "sqlite4_num_to_text", (Tcl_CmdProc*)test_num_to_text }, + { "sqlite4_num_add", (Tcl_CmdProc*)test_num_add }, + { "sqlite4_num_sub", (Tcl_CmdProc*)test_num_sub }, + { "sqlite4_num_mul", (Tcl_CmdProc*)test_num_mul }, + { "sqlite4_num_div", (Tcl_CmdProc*)test_num_div }, + { "sqlite4_num_isinf", (Tcl_CmdProc*)test_num_isinf }, + { "sqlite4_num_isnan", (Tcl_CmdProc*)test_num_isnan }, }; static struct { char *zName; Tcl_ObjCmdProc *xProc; void *clientData; @@ -4521,11 +4732,10 @@ extern int sqlite4_os_type; #endif #ifdef SQLITE4_DEBUG extern int sqlite4WhereTrace; extern int sqlite4OSTrace; - extern int sqlite4VdbeAddopTrace; #endif #ifdef SQLITE4_TEST extern char sqlite4_query_plan[]; static char *query_plan = sqlite4_query_plan; #ifdef SQLITE4_ENABLE_FTS3 @@ -4576,12 +4786,10 @@ #ifdef SQLITE4_TEST Tcl_LinkVar(interp, "sqlite_query_plan", (char*)&query_plan, TCL_LINK_STRING|TCL_LINK_READ_ONLY); #endif #ifdef SQLITE4_DEBUG - Tcl_LinkVar(interp, "sqlite_addop_trace", - (char*)&sqlite4VdbeAddopTrace, TCL_LINK_INT); Tcl_LinkVar(interp, "sqlite_where_trace", (char*)&sqlite4WhereTrace, TCL_LINK_INT); #endif Tcl_LinkVar(interp, "sqlite_static_bind_value", (char*)&sqlite_static_bind_value, TCL_LINK_STRING); Index: test/test_utf.c ================================================================== --- test/test_utf.c +++ test/test_utf.c @@ -101,11 +101,11 @@ { 0, 0 } }; struct EncName *pEnc; char *z = Tcl_GetString(pObj); for(pEnc=&encnames[0]; pEnc->zName; pEnc++){ - if( 0==sqlite4StrICmp(z, pEnc->zName) ){ + if( 0==sqlite4_stricmp(z, pEnc->zName) ){ break; } } if( !pEnc->enc ){ Tcl_AppendResult(interp, "No such encoding: ", z, 0); @@ -114,11 +114,13 @@ return SQLITE4_UTF16NATIVE; } return pEnc->enc; } -static void freeStr(void *pStr){ sqlite4_free(0, pStr); } +static void freeStr(void *pEnv, void *pStr){ + sqlite4_free((sqlite4_env*)pEnv, pStr); +} /* ** Usage: test_translate ?? ** */ @@ -132,11 +134,11 @@ u8 enc_to; sqlite4_value *pVal; char *z; int len; - void (*xDel)(void *p) = SQLITE4_STATIC; + void (*xDel)(void*,void*) = SQLITE4_STATIC; if( objc!=4 && objc!=5 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", Tcl_GetStringFromObj(objv[0], 0), " ", 0 @@ -157,19 +159,19 @@ if( enc_from==SQLITE4_UTF8 ){ z = Tcl_GetString(objv[1]); if( objc==5 ){ z = sqlite4_mprintf(0, "%s", z); } - sqlite4ValueSetStr(pVal, -1, z, enc_from, xDel); + sqlite4ValueSetStr(pVal, -1, z, enc_from, xDel, 0); }else{ z = (char*)Tcl_GetByteArrayFromObj(objv[1], &len); if( objc==5 ){ char *zTmp = z; z = sqlite4_malloc(0, len); memcpy(z, zTmp, len); } - sqlite4ValueSetStr(pVal, -1, z, enc_from, xDel); + sqlite4ValueSetStr(pVal, -1, z, enc_from, xDel, 0); } z = (char *)sqlite4ValueText(pVal, enc_to); len = sqlite4ValueBytes(pVal, enc_to) + (enc_to==SQLITE4_UTF8?1:2); Tcl_SetObjResult(interp, Tcl_NewByteArrayObj((u8*)z, len)); Index: test/tester.tcl ================================================================== --- test/tester.tcl +++ test/tester.tcl @@ -146,10 +146,28 @@ # return 100; # TODO: Good default? } return $::G(file-retry-delay) } + +# Return the string representing the name of the current directory. On +# Windows, the result is "normalized" to whatever our parent command shell +# is using to prevent case-mismatch issues. +# +proc get_pwd {} { + if {$::tcl_platform(platform) eq "windows"} { + # + # NOTE: Cannot use [file normalize] here because it would alter the + # case of the result to what Tcl considers canonical, which would + # defeat the purpose of this procedure. + # + return [string map [list \\ /] \ + [string trim [exec -- $::env(ComSpec) /c echo %CD%]]] + } else { + return [pwd] + } +} # Copy file $from into $to. This is used because some versions of # TCL for windows (notably the 8.4.1 binary package shipped with the # current mingw release) have a broken "file copy" command. # @@ -474,22 +492,48 @@ if {![info exists ::G(match)] || [string match $::G(match) $name]} { if {[catch {uplevel #0 "$cmd;\n"} result]} { puts "\nError: $result" fail_test $name - } elseif {[string compare $result $expected]} { - puts "\nExpected: \[$expected\]\n Got: \[$result\]" - fail_test $name } else { - puts " Ok" + if {[regexp {^~?/.*/$} $expected]} { + if {[string index $expected 0]=="~"} { + set re [string map {# {[-0-9.]+}} [string range $expected 2 end-1]] + set ok [expr {![regexp $re $result]}] + } else { + set re [string map {# {[-0-9.]+}} [string range $expected 1 end-1]] + set ok [regexp $re $result] + } + } else { + set ok [expr {[string compare $result $expected]==0}] + } + if {!$ok} { + # if {![info exists ::testprefix] || $::testprefix eq ""} { + # error "no test prefix" + # } + puts "\nExpected: \[$expected\]\n Got: \[$result\]" + fail_test $name + } else { + puts " Ok" + } } } else { puts " Omitted" omit_test $name "pattern mismatch" 0 } flush stdout } + +proc catchcmd {db {cmd ""}} { + global CLI + set out [open cmds.txt w] + puts $out $cmd + close $out + set line "exec $CLI $db < cmds.txt" + set rc [catch { eval $line } msg] + list $rc $msg +} proc filepath_normalize {p} { # test cases should be written to assume "unix"-like file paths if {$::tcl_platform(platform)!="unix"} { # lreverse*2 as a hack to remove any unneeded {} after the string map @@ -716,10 +760,11 @@ puts "******************************************************************" } if {$::cmdlinearg(binarylog)} { vfslog finalize binarylog } + kvwrap uninstall if {[lindex [sqlite4_env_status SQLITE4_ENVSTATUS_MALLOC_COUNT 0] 1]>0 || [sqlite4_memory_used]>0} { puts "Unfreed memory: [sqlite4_memory_used] bytes in\ [lindex [sqlite4_env_status SQLITE4_ENVSTATUS_MALLOC_COUNT 0] 1] allocations" incr nErr Index: test/tkt-3a77c9714e.test ================================================================== --- test/tkt-3a77c9714e.test +++ test/tkt-3a77c9714e.test @@ -37,11 +37,11 @@ ) } {ABCDEF ABCDEF} do_execsql_test 2.1 { CREATE TABLE [Beginnings] ( - [Id] INTEGER PRIMARY KEY AUTOINCREMENT,[Title] TEXT, [EndingId] INTEGER + [Id] INTEGER PRIMARY KEY /*AUTOINCREMENT*/,[Title] TEXT, [EndingId] INTEGER ); CREATE TABLE [Endings] (Id INT,Title TEXT,EndingId INT); INSERT INTO Beginnings (Id, Title, EndingId) VALUES (1, 'FACTOR', 18); INSERT INTO Beginnings (Id, Title, EndingId) VALUES (2, 'SWIMM', 18); INSERT INTO Endings (Id, Title, EndingId) VALUES (1, 'ING', 18); Index: test/tkt-3fe897352e.test ================================================================== --- test/tkt-3fe897352e.test +++ test/tkt-3fe897352e.test @@ -31,30 +31,30 @@ PRAGMA encoding=UTF8; CREATE TABLE t1(x); INSERT INTO t1 VALUES(hex_to_utf16be('D800')); SELECT hex(x) FROM t1; } -} {EDA080} +} {eda080} do_test tkt-3fe89-1.2 { db eval { DELETE FROM t1; INSERT INTO t1 VALUES(hex_to_utf16le('00D8')); SELECT hex(x) FROM t1; } -} {EDA080} +} {eda080} do_test tkt-3fe89-1.3 { db eval { DELETE FROM t1; INSERT INTO t1 VALUES(hex_to_utf16be('DFFF')); SELECT hex(x) FROM t1; } -} {EDBFBF} +} {edbfbf} do_test tkt-3fe89-1.4 { db eval { DELETE FROM t1; INSERT INTO t1 VALUES(hex_to_utf16le('FFDF')); SELECT hex(x) FROM t1; } -} {EDBFBF} +} {edbfbf} finish_test Index: test/tkt3442.test ================================================================== --- test/tkt3442.test +++ test/tkt3442.test @@ -47,13 +47,10 @@ # and verify that the query plan is the same. # ifcapable explain { do_test tkt3442-1.2 { EQP { SELECT node FROM listhash WHERE id='5000' LIMIT 1; } - } {0 0 0 {SEARCH TABLE listhash USING INDEX ididx (id=?) (~1 rows)}} - do_test tkt3442-1.3 { - EQP { SELECT node FROM listhash WHERE id="5000" LIMIT 1; } } {0 0 0 {SEARCH TABLE listhash USING INDEX ididx (id=?) (~1 rows)}} } # Some extra tests testing other permutations of 5000. Index: test/tkt3493.test ================================================================== --- test/tkt3493.test +++ test/tkt3493.test @@ -18,14 +18,14 @@ source $testdir/tester.tcl do_test tkt3493-1.1 { execsql { BEGIN; - CREATE TABLE A (id INTEGER PRIMARY KEY AUTOINCREMENT, val TEXT); + CREATE TABLE A (id INTEGER PRIMARY KEY /*AUTOINCREMENT*/, val TEXT); INSERT INTO A VALUES(1,'123'); INSERT INTO A VALUES(2,'456'); - CREATE TABLE B (id INTEGER PRIMARY KEY AUTOINCREMENT, val TEXT); + CREATE TABLE B (id INTEGER PRIMARY KEY /*AUTOINCREMENT*/, val TEXT); INSERT INTO B VALUES(1,1); INSERT INTO B VALUES(2,2); CREATE TABLE A_B (B_id INTEGER NOT NULL, A_id INTEGER); INSERT INTO A_B VALUES(1,1); INSERT INTO A_B VALUES(2,2); Index: test/tkt3841.test ================================================================== --- test/tkt3841.test +++ test/tkt3841.test @@ -25,18 +25,18 @@ do_test tkt3841.1 { execsql { CREATE TABLE table2 (key TEXT, x TEXT); CREATE TABLE list (key TEXT, value TEXT); - INSERT INTO table2 VALUES ("a", "alist"); - INSERT INTO table2 VALUES ("b", "blist"); - INSERT INTO list VALUES ("a", 1); - INSERT INTO list VALUES ("a", 2); - INSERT INTO list VALUES ("a", 3); - INSERT INTO list VALUES ("b", 4); - INSERT INTO list VALUES ("b", 5); - INSERT INTO list VALUES ("b", 6); + INSERT INTO table2 VALUES ('a', 'alist'); + INSERT INTO table2 VALUES ('b', 'blist'); + INSERT INTO list VALUES ('a', 1); + INSERT INTO list VALUES ('a', 2); + INSERT INTO list VALUES ('a', 3); + INSERT INTO list VALUES ('b', 4); + INSERT INTO list VALUES ('b', 5); + INSERT INTO list VALUES ('b', 6); SELECT table2.x, (SELECT group_concat(list.value) FROM list Index: test/trigger1.test ================================================================== --- test/trigger1.test +++ test/trigger1.test @@ -288,13 +288,25 @@ CREATE TEMP TABLE t2(x,y); INSERT INTO t1 VALUES(1,2); SELECT * FROM t2; } } {1 {no such table: main.t2}} - do_test trigger-3.6 { + do_test trigger-3.6.1 { + catchsql { + DROP TRIGGER r1; + CREATE TEMP TRIGGER r1 AFTER INSERT ON t1 BEGIN + INSERT INTO t2 VALUES(NEW.a,NEW.b), (NEW.b*100, NEW.a*100); + END; + INSERT INTO t1 VALUES(1,2); + SELECT * FROM t2; + } + } {0 {1 2 200 100}} + do_test trigger-3.6.2 { catchsql { DROP TRIGGER r1; + DELETE FROM t1; + DELETE FROM t2; CREATE TEMP TRIGGER r1 AFTER INSERT ON t1 BEGIN INSERT INTO t2 VALUES(NEW.a,NEW.b); END; INSERT INTO t1 VALUES(1,2); SELECT * FROM t2; Index: tool/lsmperf.tcl ================================================================== --- tool/lsmperf.tcl +++ tool/lsmperf.tcl @@ -1,10 +1,14 @@ #!/bin/sh # \ exec tclsh "$0" "$@" +package require sqlite3 +########################################################################## +# Procedures used when running tests (collecting data). +# proc exec_lsmtest_speed {nSec spec} { set fd [open [list |./lsmtest speed2 {*}$spec]] set res [list] puts -nonewline "./lsmtest speed2" @@ -30,180 +34,368 @@ } catch { close $fd } set res } -proc write_to_file {zFile zScript} { - set fd [open $zFile w] - puts $fd $zScript - close $fd -} - -proc exec_gnuplot_script {script png} { - write_to_file out " - $script - pause -1 - " - - set script " - set terminal pngcairo size 1200,400 - $script - " - exec gnuplot << $script > $png 2>/dev/null -} - -proc make_totalset {res nOp bFetch} { - set ret "" - set nMs 0 - set nIns 0 - foreach row $res { - foreach {i msInsert msFetch} $row {} - incr nIns $nOp - if {$bFetch==0} { - incr nMs $msInsert - } else { - incr nMs $msFetch - } - append ret "$nIns [expr $nIns*1000.0/$nMs]\n" - } - append ret "end\n" - set ret -} - -proc make_dataset {res iRes nWrite nShift nOp} { - set ret "" - foreach row $res { - set i [lindex $row 0] - set j [lindex $row [expr $iRes+1]] - set x [expr $i*$nWrite + $nShift] - append ret "$x [expr int($nOp * 1000.0 / $j)]\n" - } - append ret "end\n" - set ret -} - -proc do_write_test {zPng nSec nWrite nFetch nRepeat lSys} { - - if {[llength $lSys]!=2 && [llength $lSys]!=4} { - error "lSys must be a list of 2 or 4 elements" - } - - set lRes [list] - foreach {name sys} $lSys { - set wt [list -w $nWrite -r $nRepeat -f $nFetch -system $sys] - lappend lRes [exec_lsmtest_speed $nSec $wt] - if {$sys != [lindex $lSys end]} { - puts "Sleeping 20 seconds..." - after 20000 - } - } - - # Set up the header part of the gnuplot script. - # - set xmax 0 - foreach res $lRes { - set xthis [expr [lindex $res end 0]*$nWrite + 5*$nWrite/4] - if {$xthis>$xmax} {set xmax $xthis} - } - - append labeltext "Test parameters:\\n" - append labeltext " $nWrite writes per iteration\\n" - append labeltext " $nFetch fetches per iteration\\n" - append labeltext " key size is 12 bytes\\n" - append labeltext " value size is 100 bytes\\n" - set labelx [expr int($xmax * 1.2)] - - set nWrite2 [expr $nWrite/2] - set y2setup "" - if {$nFetch>0} { - set y2setup { - set ytics nomirror - set y2tics nomirror - set y2range [0:*] - } - } - set script [subst -nocommands { - set boxwidth $nWrite2 - set xlabel "Database Size" - set y2label "Queries per second" - set ylabel "Writes per second" - set yrange [0:*] - set xrange [0:$xmax] - set key outside bottom - $y2setup - set label 1 "$labeltext" at screen 0.95,graph 1.0 right +proc run_speed_test {zDb nTimeout nWrite nFetch nPause nRepeat zSystem zName} { + + set spec [list -w $nWrite -f $nFetch -r $nRepeat -p $nPause -system $zSystem] + set res [exec_lsmtest_speed $nTimeout $spec] + set cmd "lsmtest speed2 -w $nWrite -f $nFetch -r $nRepeat -p $nPause" + append cmd " -system \"$zSystem\"" + + sqlite3 db $zDb + + db eval { + PRAGMA synchronous = OFF; + CREATE TABLE IF NOT EXISTS run( + runid INTEGER PRIMARY KEY, nWrite INT, nFetch INT, nPause INT, + cmd TEXT, name TEXT + ); + CREATE TABLE IF NOT EXISTS sample( + runid INT, sample INT, writems INT, fetchms INT, + PRIMARY KEY(runid, sample) + ); + INSERT INTO run VALUES(NULL, $nWrite, $nFetch, $nPause, $cmd, $zName); + } + + set id [db last_insert_rowid] + foreach sample $res { + foreach {a b c} $sample {} + db eval { INSERT INTO sample VALUES($id, $a, $b, $c) } + } + + db close +} +# +# End of procs used while gathering data. +########################################################################## + +########################################################################## + +proc chart_y_to_canvas {y} { + foreach v [uplevel {info vars c_*}] { upvar $v $v } + set ytotal [expr ($c_height - $c_top_margin - $c_bottom_margin)] + expr $c_height - $c_bottom_margin - int($ytotal * $y / $c_ymax) +} +proc chart_sample_to_canvas {nSample x} { + foreach v [uplevel {info vars c_*}] { upvar $v $v } + set xtotal [expr ($c_width - $c_left_margin - $c_right_margin)] + expr {$c_left_margin + ($x*$xtotal/$nSample)} +} + +proc draw_line_series {nSample lSeries tag} { + foreach v [uplevel {info vars c_*}] { upvar $v $v } + set xtotal [expr ($c_width - $c_left_margin - $c_right_margin)] + set ytotal [expr ($c_height - $c_top_margin - $c_bottom_margin)] + + set x 0 + for {set i 1} {$i < $nSample} {incr i} { + set x2 [expr $x + int( double($xtotal - $x) / ($nSample-$i) )] + if {[lindex $lSeries $i-1] < $c_ymax && [lindex $lSeries $i] < $c_ymax} { + + set y1 [chart_y_to_canvas [lindex $lSeries $i-1]] + set y2 [chart_y_to_canvas [lindex $lSeries $i]] + .c create line \ + [expr ($c_left_margin + $x)] $y1 \ + [expr ($c_left_margin + $x2)] $y2 \ + -tag $tag + } + + set x $x2 + } +} + +proc draw_points_series {nSample lSeries tag} { + foreach v [uplevel {info vars c_*}] { upvar $v $v } + set xtotal [expr ($c_width - $c_left_margin - $c_right_margin)] + set ytotal [expr ($c_height - $c_top_margin - $c_bottom_margin)] + + for {set i 0} {$i < $nSample} {incr i} { + set x [chart_sample_to_canvas $nSample $i] + set y [chart_y_to_canvas [lindex $lSeries $i]] + .c create rectangle $x $y [expr $x+2] [expr $y+2] -tag $tag + } +} + +proc draw_bars_series {nSample nShift lSeries tag} { + foreach v [uplevel {info vars c_*}] { upvar $v $v } + set xtotal [expr ($c_width - $c_left_margin - $c_right_margin)] + set ytotal [expr ($c_height - $c_top_margin - $c_bottom_margin)] + + set b [expr $c_height - $c_bottom_margin] + for {set i 0} {$i < $nSample} {incr i} { + set x [expr [chart_sample_to_canvas $nSample $i] + $nShift] + set y [chart_y_to_canvas [lindex $lSeries $i]] + .c create rectangle $x $y [expr $x+$c_bar_width-1] $b -tag $tag + } +} + +proc draw_text {iRun iRun2} { + foreach v [uplevel {info vars c_*}] { upvar $v $v } + + set y $c_height + if {$iRun2!=""} { + array set metrics [font metrics {-size 8}] + set cmd [db one {SELECT cmd FROM run WHERE runid=$iRun2}] + .c create text 10 $y -anchor sw -text $cmd -fill grey70 -font {-size 8} + set y [expr $y-$metrics(-ascent)] + } + set cmd [db one {SELECT cmd FROM run WHERE runid=$iRun}] + .c create text 10 $y -anchor sw -text $cmd -fill grey70 -font {-size 8} +} + +proc format_integer {n} { + if { ($n % 1000000)==0 } { return "[expr $n/1000000]M" } + if { $n>1000000 && ($n % 100000)==0 } { + return "[expr double($n)/1000000]M" + } + if { ($n % 1000)==0 } { return "[expr $n/1000]K" } + return $n +} + +proc populate_chart {nSample nShift iRun colors} { + foreach v [uplevel {info vars c_*}] { upvar $v $v } + upvar nWrite nWrite + upvar nFetch nFetch + set name [db one "SELECT name FROM run WHERE runid=$iRun"] + + set lWrite [list] + set lFetch [list] + for {set i 0} {$i < $nSample} {incr i} { + set q "SELECT writems, fetchms FROM sample WHERE runid=$iRun AND sample=$i" + db eval $q break + lappend lWrite [expr {(1000.0*$nWrite) / $writems}] + lappend lFetch [expr {(1000.0*$nFetch) / $fetchms}] + } + + draw_bars_series $nSample $nShift $lWrite writes_p_$iRun + draw_points_series $nSample $lFetch fetches_p_$iRun + + set lWrite [list] + set lFetch [list] + for {set i 0} {$i < $nSample} {incr i} { + set q "SELECT 1000.0 * ($i+1) * $nWrite / sum(writems) AS r1, " + append q " 1000.0 * ($i+1) * $nFetch / sum(fetchms) AS r2 " + append q "FROM sample WHERE runid=$iRun AND sample<=$i" + + db eval $q break + lappend lWrite $r1 + lappend lFetch $r2 + } + + draw_line_series $nSample $lWrite writes_$iRun + draw_line_series $nSample $lFetch fetches_$iRun + + # Create the legend for the data drawn above. + # + array set metrics [font metrics default] + set y $c_legend_ypadding + set x [expr $c_width - $c_legend_padding] + set xsym [expr { $c_width + - [font measure default "$name cumulative fetches/sec"] }] - - set cols [list {#B0C4DE #00008B #000000} {#F08080 #8B0000 #FFA500}] - set cols [lrange $cols 0 [expr ([llength $lSys]/2)-1]] - - set nShift [expr ($nWrite/2)] - set plot1 "" - set plot2 "" - set plot3 "" - set plot4 "" - set data1 "" - set data2 "" - set data3 "" - set data4 "" - - foreach {name sys} $lSys res $lRes col $cols { - foreach {c1 c2 c3} $col {} - - # First plot. Writes per second (bar chart). - # - if {$plot1 != ""} { set plot1 ", $plot1" } - set plot1 "\"-\" ti \"$name writes/sec\" with boxes fs solid lc rgb \"$c1\"$plot1" - set data1 "[make_dataset $res 0 $nWrite $nShift $nWrite] $data1" - - # Third plot. Cumulative writes per second (line chart). - # - set plot3 ",\"-\" ti \"$name cumulative writes/sec\" with lines lc rgb \"$c2\" lw 2 $plot3" - set data3 "[make_totalset $res $nWrite 0] $data3" - - if {$nFetch>0} { - set new ", \"-\" ti \"$name fetches/sec\" axis x1y2 " - append new "with points lw 1 lc rgb \"$c2\"" - set plot2 "$new $plot2" - set data2 "[make_dataset $res 1 $nWrite $nWrite $nFetch] $data2" - - set new ",\"-\" ti \"$name cumulative fetches/sec\" " - append new "with lines lc rgb \"$c2\" lw 1 " - set plot4 "$new $plot4" - set data4 "[make_totalset $res $nFetch 1] $data4" - } - - incr nShift [expr $nWrite/4] - } - append script "plot " - append script $plot1 - append script $plot2 - append script $plot3 - append script $plot4 - append script "\n" - append script $data1 - append script $data2 - append script $data3 - append script $data4 - - append script "pause -1\n" - exec_gnuplot_script $script $zPng -} - -do_write_test x.png 600 50000 50000 20 { - lsm-mt-1 "mmap=1 multi_proc=0 safety=0 threads=3 autowork=0 block_size=1M" -} - -# lsm-mt "mmap=1 multi_proc=0 threads=2 autowork=0 autocheckpoint=8192000" -# lsm-mt "mmap=1 multi_proc=0 safety=1 threads=3 autowork=0" -# lsm-st "mmap=1 multi_proc=0 safety=1 threads=1 autowork=1" -# lsm-mt "mmap=1 multi_proc=0 safety=1 threads=3 autowork=0" -# lsm-mt "mmap=1 multi_proc=0 safety=1 threads=3 autowork=0" -# LevelDB leveldb -# lsm-st "mmap=1 multi_proc=0 safety=1 threads=1 autowork=1" -# LevelDB leveldb -# SQLite sqlite3 - - + foreach {t g} { + "$name fetches/sec" { + rectangle 0 0 -2 2 -tag fetches_p_$iRun + } + "$name writes/sec" { + rectangle 0 0 $c_bar_width $metrics(-ascent) -tag writes_p_$iRun + } + "$name cumulative fetches/sec" { + line 0 0 -30 0 -tag fetches_$iRun + } + "$name cumulative writes/sec" { + line 0 0 -30 0 -tag writes_$iRun + } + } { + .c create text $x $y -tag legend_$iRun -text [subst $t] -anchor e + + set id [eval [concat {.c create} [subst $g]]] + .c addtag legend_$iRun withtag $id + + set box [.c bbox $id] + set xmove [expr {$xsym - ([lindex $box 0]/2 + [lindex $box 2]/2)}] + set ymove [expr $y - ([lindex $box 1]/2 + [lindex $box 3]/2)] + + .c move $id $xmove $ymove + incr y $metrics(-linespace) + } + + .c itemconfigure writes_$iRun -fill [lindex $colors 0] -width 2 + .c itemconfigure fetches_$iRun -fill [lindex $colors 2] + .c itemconfigure writes_p_$iRun -fill [lindex $colors 1] + .c itemconfigure fetches_p_$iRun -fill [lindex $colors 3] + catch { .c itemconfigure fetches_p_$iRun -outline [lindex $colors 3] } + catch { .c itemconfigure writes_p_$iRun -outline [lindex $colors 1] } +} + +proc generate_chart {png db iRun {iRun2 {}}} { + sqlite3 db $db + db eval { SELECT nWrite, nFetch FROM run WHERE runid=$iRun } {} + if {0==[info exists nWrite]} { + error "No such run in db $db: $iRun" + } + + set c_left_margin 50 + set c_bottom_margin 60 + set c_top_margin 20 + set c_right_margin 400 + set c_width 1250 + set c_height 350 + + set c_ymax 300000 + set c_ytick 50000 + set c_ticksize 5 + set c_nxtick 10 + set c_dbsize_padding 20 + set c_legend_padding 20 + set c_legend_ypadding 100 + set c_bar_width 2 + + package require Tk + canvas .c + .c configure -width $c_width -height $c_height + pack .c -fill both -expand 1 + + # Make the background white + .c configure -background white + draw_text $iRun $iRun2 + + # Draw the box for the chart + # + set y [expr $c_height - $c_bottom_margin] + set x [expr $c_width - $c_right_margin] + .c create rectangle $c_left_margin $c_top_margin $x $y + + # Draw the vertical ticks + # + set ytotal [expr ($c_height - $c_top_margin - $c_bottom_margin)] + for {set y $c_ytick} {$y <= $c_ymax} {incr y $c_ytick} { + + # Calculate the canvas y coord at which to draw the tick + set ypix [expr {$c_height - $c_bottom_margin - ($y * $ytotal) / $c_ymax}] + + set left_tick_x $c_left_margin + set right_tick_x [expr $c_width - $c_right_margin - $c_ticksize] + foreach x [list $left_tick_x $right_tick_x] { + .c create line $x $ypix [expr $x+$c_ticksize] $ypix + } + + .c create text $c_left_margin $ypix -anchor e -text "[format_integer $y] " + set x [expr $c_width - $c_right_margin] + .c create text $x $ypix -anchor w -text " [format_integer $y]" + } + + # Figure out the total number of samples for this chart + # + set nSample [db one {SELECT count(*) FROM sample where runid=$iRun}] + if {$nSample==0} { error "No such run: $iRun" } + #set nSample 100 + + # Draw the horizontal ticks + # + set w [expr {$c_width - $c_left_margin - $c_right_margin}] + set xincr [expr ($nSample * $nWrite) / $c_nxtick] + for {set i 1} {$i <= $c_nxtick} {incr i} { + set x [expr $i * $xincr] + set xpix [expr {$c_left_margin + ($w / $c_nxtick) * $i}] + + set b [expr $c_height-$c_bottom_margin] + .c create line $xpix $b $xpix [expr $b-$c_ticksize] + .c create text $xpix $b -anchor n -text [format_integer $x] + } + + set x [expr (($c_width-$c_right_margin-$c_left_margin) / 2) + $c_left_margin] + set y [expr $c_height - $c_bottom_margin + $c_dbsize_padding] + .c create text $x $y -anchor n -text "Database Size (number of entries)" + + populate_chart $nSample 0 $iRun {black grey55 black black} + if {$iRun2 != ""} { + #populate_chart $nSample 3 $iRun2 {royalblue skyblue royalblue royalblue} + + set s $c_bar_width + populate_chart $nSample $s $iRun2 {royalblue lightsteelblue royalblue royalblue} + set box [.c bbox legend_$iRun] + set shift [expr $c_legend_padding + [lindex $box 3]-[lindex $box 1]] + .c move legend_$iRun2 0 $shift + } + + .c lower fetches_p_$iRun + .c lower fetches_p_$iRun2 + .c lower writes_p_$iRun + .c lower writes_p_$iRun2 + + bind .c exit + bind . exit + db close +} + +proc capture_photo {z} { + package require Img + set img [image create photo -format window -data .c] + $img write $z -format GIF +} + +# "autoflush=1M multi_proc=0" +# "autoflush=1M multi_proc=0 mt_mode=4 mt_min_ckpt=2M mt_max_ckpt=3M" +# "autoflush=4M multi_proc=0 autocheckpoint=8M" +# "autoflush=4M multi_proc=0 mt_mode=4" +proc run_all_tests {} { + set nInsert 50000 + set nSelect 50000 + set nSleep 20000 + set nPause 2500 + + #set nInsert 5000 + #set nSelect 5000 + #set nSleep 2000 + #set nPause 250 + + set bFirst 0 + + foreach {name config} { + single-threaded "autoflush=1M multi_proc=0" + multi-threaded + "autoflush=1M multi_proc=0 mt_mode=4 mt_min_ckpt=2M mt_max_ckpt=3M" + single-threaded "autoflush=4M multi_proc=0 autocheckpoint=8M" + multi-threaded "autoflush=4M multi_proc=0 mt_mode=4" + } { + if {$bFirst!=0} { + puts "sleeping 20 seconds..." ; after $nSleep + } + run_speed_test res.db 900 $nInsert $nSelect 0 200 $config $name + set bFirst 1 + } + + # Tests with a 2.5 second delay. + # + foreach {name config} { + single-threaded "autoflush=4M multi_proc=0 autocheckpoint=8M" + multi-threaded "autoflush=4M multi_proc=0 mt_mode=4" + } { + puts "sleeping 20 seconds..." ; after $nSleep + run_speed_test res.db 900 $nInsert $nSelect $nPause 200 $config $name + } +} + +#run_all_tests + +generate_chart png res.db 1 2 +update +capture_photo lsmperf1.gif +destroy .c + +generate_chart png res.db 3 4 +update +capture_photo lsmperf2.gif +destroy .c + +generate_chart png res.db 5 6 +update +capture_photo lsmperf3.gif +destroy .c + +exit Index: tool/lsmview.tcl ================================================================== --- tool/lsmview.tcl +++ tool/lsmview.tcl @@ -41,11 +41,11 @@ } namespace import ::autoscroll::* ############################################################################# proc exec_lsmtest_show {args} { - set fd [open [list |lsmtest show {*}$args]] + set fd [open [list |lsmtest show {*}$args 2>/dev/null]] set res "" while {![eof $fd]} { set line [gets $fd] if {[regexp {^\#.*} $line]} continue if {[regexp {^Leaked*} $line]} continue @@ -166,20 +166,20 @@ eval $myScript [list $segment] } proc draw_level {C level tags} { - set lhs [lindex $level 0] + set lhs [lindex $level 1] set l [lindex $tags end] draw_segment $C $lhs [concat $tags "$l.s0"] foreach {x1 y1 x2 y2} [$C bbox "$l.s0"] {} set i 0 set y 0 set x [expr $x2+10] - foreach seg [lrange $level 1 end] { + foreach seg [lrange $level 2 end] { set tag "$l.s[incr i]" draw_segment $C $seg [concat $tags $tag] $C move $tag $x $y foreach {x1 y1 x2 y2} [$C bbox $tag] {} @@ -196,15 +196,15 @@ # Figure out the scale to use. # set nMaxWidth 0.0 foreach level $structure { set nRight 0 - foreach seg [lrange $level 1 end] { + foreach seg [lrange $level 2 end] { set sz [lindex $seg 3] if {$sz > $nRight} { set nRight $sz } } - set nLeft [lindex $level 0 3] + set nLeft [lindex $level 1 3] set nTotal [log2 [expr max($nLeft, 2)]] if {$nRight} {set nTotal [expr $nTotal + [log2 [expr max($nRight, 2)]]]} if {$nTotal > $nMaxWidth} { set nMaxWidth $nTotal } } @@ -219,12 +219,13 @@ draw_level $C $level $tag foreach {x1 y1 x2 y2} [$C bbox $tag] {} $C move $tag [expr ($W-$x2)/2] $y incr y [expr $y2 + $::G(vertical_padding)] + set age [lindex $level 0] foreach {x1 y1 x2 y2} [$C bbox $tag.text] {} - $C create text 10 $y1 -anchor nw -text [string toupper "${tag}:"] + $C create text 5 $y1 -anchor nw -text [string toupper "${tag} ($age):"] } if {[info exists myVertShift]} { set H [winfo height $C] set region [$C bbox all] @@ -246,12 +247,15 @@ set line [$C create line [expr ($xf1+$xf2)/2] $yf2 [expr ($xt1+$xt2)/2] $yt1] $C itemconfigure $line -arrow last } proc draw_internal_pointers {C iLevel level} { - for {set j 2} {$j < [llength $level]} {incr j} { - if {[lindex $level $j 2]==0} { + set iAge [lindex $level 0] ;# Age of this level + set lSeg [lrange $level 1 end] ;# List of segments that make up this level + + for {set j 2} {$j < [llength $lSeg]} {incr j} { + if {[lindex $lSeg $j 2]==0} { draw_one_pointer $C "l$iLevel.s[expr $j-1]" "l$iLevel.s$j" } } } @@ -264,28 +268,32 @@ set i2 [expr $i+1] set l1 [lindex $structure $i] set l2 [lindex $structure $i2] - set bMerge1 [expr [llength $l1]>1] - set bMerge2 [expr [llength $l2]>1] + # Set to true if levels $i and $i2 are currently undergoing a merge + # (have one or more rhs segments), respectively. + # + set bMerge1 [expr [llength $l1]>2] + set bMerge2 [expr [llength $l2]>2] if {$bMerge2} { - if {[lindex $l2 1 2]==0} { + if {[lindex $l2 2 2]==0} { draw_one_pointer $C "l$i.s0" "l$i2.s1" if {$bMerge1} { - draw_one_pointer $C "l$i.s[expr [llength $l1]-1]" "l$i2.s1" + draw_one_pointer $C "l$i.s[expr [llength $l1]-2]" "l$i2.s1" } } } else { - set bBtree [expr [lindex $l2 0 2]!=0] + + set bBtree [expr [lindex $l2 1 2]!=0] if {$bBtree==0 || $bMerge1} { draw_one_pointer $C "l$i.s0" "l$i2.s0" } if {$bBtree==0} { - draw_one_pointer $C "l$i.s[expr [llength $l1]-1]" "l$i2.s0" + draw_one_pointer $C "l$i.s[expr [llength $l1]-2]" "l$i2.s0" } } } } @@ -363,10 +371,27 @@ proc static_redraw {C} { link_varset $C myText myDb myData $C delete all draw_canvas_content $C $myData [list static_select_callback $C] } + +# The first parameter is the number of pages stored on each block of the +# database file (i.e. if the page size is 4K and the block size 1M, 256). +# The second parameter is a list of pages in a segment. +# +# The return value is a list of blocks occupied by the segment. +# +proc pagelist_to_blocklist {nBlkPg pglist} { + set blklist [list] + foreach pg $pglist { + set blk [expr 1 + (($pg-1) / $nBlkPg)] + if {[lindex $blklist end]!=$blk} { + lappend blklist $blk + } + } + set blklist +} proc static_select_callback {C segment} { link_varset $C myText myDb myData myTree myCfg foreach {iFirst iLast iRoot nSize $segment} $segment {} @@ -374,11 +399,11 @@ $myText delete 0.0 end # Delete the existing tree entries. $myTree delete [$myTree children {}] - set nBlksz [exec_lsmtest_show -c $myCfg $myDb blocksize] + set nBlksz [expr [exec_lsmtest_show -c $myCfg $myDb blocksize] * 1024] set nPgsz [exec_lsmtest_show -c $myCfg $myDb pagesize] if {[regexp {c=1} $myCfg] || [regexp {co=1} $myCfg] || [regexp {com=1} $myCfg] || [regexp {comp=1} $myCfg] || [regexp {compr=1} $myCfg] || [regexp {compres=1} $myCfg] @@ -389,23 +414,23 @@ } else { set nBlkPg [expr ($nBlksz / $nPgsz)] } foreach pg $data { - set blk [expr 1 + ($pg / $nBlkPg)] + set blk [expr 1 + (($pg-1) / $nBlkPg)] if {[info exists block($blk)]} { incr block($blk) 1 } else { set block($blk) 1 } } - foreach blk [lsort -integer [array names block]] { + foreach blk [pagelist_to_blocklist $nBlkPg $data] { set nPg $block($blk) set blkid($blk) [$myTree insert {} end -text "block $blk ($nPg pages)"] } foreach pg $data { - set blk [expr 1 + ($pg / $nBlkPg)] + set blk [expr 1 + (($pg-1) / $nBlkPg)] set id $blkid($blk) set item [$myTree insert $id end -text $pg] if {$pg == $iRoot} { set rootitem $item set rootparent $id @@ -513,11 +538,11 @@ set mySelected "l0.s0.main" static_redraw $C bind $C [list static_redraw $C] bind $myTree <> [list static_treeview_select $C] - static_select_callback $C [lindex $myData 0 0] + static_select_callback $C [lindex $myData 0 1] } if {[llength $argv] > 2} { puts stderr "Usage: $argv0 ?CONFIG? ?DATABASE?" exit -1 Index: tool/mkkeywordhash.c ================================================================== --- tool/mkkeywordhash.c +++ tool/mkkeywordhash.c @@ -574,11 +574,11 @@ printf(" h = ((charMap(z[0])*4) ^\n" " (charMap(z[n-1])*3) ^\n" " n) %% %d;\n", bestSize); printf(" for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){\n"); printf(" if( aLen[i]==n &&" - " sqlite4StrNICmp(&zText[aOffset[i]],z,n)==0 ){\n"); + " sqlite4_strnicmp(&zText[aOffset[i]],z,n)==0 ){\n"); for(i=0; i code -# insert SQL insert statements for TABLE -# line One value per line -# list Values delimited by .separator string -# tabs Tab-separated values -# tcl TCL list elements -do_test shell1-3.13.1 { - catchcmd "test.db" ".mode" -} {1 {Error: unknown command or invalid arguments: "mode". Enter ".help" for help}} -do_test shell1-3.13.2 { - catchcmd "test.db" ".mode FOO" -} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} -do_test shell1-3.13.3 { - catchcmd "test.db" ".mode csv" -} {0 {}} -do_test shell1-3.13.4 { - catchcmd "test.db" ".mode column" -} {0 {}} -do_test shell1-3.13.5 { - catchcmd "test.db" ".mode html" -} {0 {}} -do_test shell1-3.13.6 { - catchcmd "test.db" ".mode insert" -} {0 {}} -do_test shell1-3.13.7 { - catchcmd "test.db" ".mode line" -} {0 {}} -do_test shell1-3.13.8 { - catchcmd "test.db" ".mode list" -} {0 {}} -do_test shell1-3.13.9 { - catchcmd "test.db" ".mode tabs" -} {0 {}} -do_test shell1-3.13.10 { - catchcmd "test.db" ".mode tcl" -} {0 {}} -do_test shell1-3.13.11 { - # too many arguments - catchcmd "test.db" ".mode tcl BAD" -} {1 {Error: invalid arguments: "BAD". Enter ".help" for help}} - -# don't allow partial mode type matches -do_test shell1-3.13.12 { - catchcmd "test.db" ".mode l" -} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} -do_test shell1-3.13.13 { - catchcmd "test.db" ".mode li" -} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} -do_test shell1-3.13.14 { - catchcmd "test.db" ".mode lin" -} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} - -# .nullvalue STRING Print STRING in place of NULL values -do_test shell1-3.14.1 { - catchcmd "test.db" ".nullvalue" -} {1 {Error: unknown command or invalid arguments: "nullvalue". Enter ".help" for help}} -do_test shell1-3.14.2 { - catchcmd "test.db" ".nullvalue FOO" -} {0 {}} -do_test shell1-3.14.3 { - # too many arguments - catchcmd "test.db" ".nullvalue FOO BAD" -} {1 {Error: unknown command or invalid arguments: "nullvalue". Enter ".help" for help}} - -# .output FILENAME Send output to FILENAME -do_test shell1-3.15.1 { - catchcmd "test.db" ".output" -} {1 {Error: unknown command or invalid arguments: "output". Enter ".help" for help}} -do_test shell1-3.15.2 { - catchcmd "test.db" ".output FOO" -} {0 {}} -do_test shell1-3.15.3 { - # too many arguments - catchcmd "test.db" ".output FOO BAD" -} {1 {Error: unknown command or invalid arguments: "output". Enter ".help" for help}} - -# .output stdout Send output to the screen -do_test shell1-3.16.1 { - catchcmd "test.db" ".output stdout" -} {0 {}} -do_test shell1-3.16.2 { - # too many arguments - catchcmd "test.db" ".output stdout BAD" -} {1 {Error: unknown command or invalid arguments: "output". Enter ".help" for help}} - -# .prompt MAIN CONTINUE Replace the standard prompts -do_test shell1-3.17.1 { - catchcmd "test.db" ".prompt" -} {1 {Error: unknown command or invalid arguments: "prompt". Enter ".help" for help}} -do_test shell1-3.17.2 { - catchcmd "test.db" ".prompt FOO" -} {0 {}} -do_test shell1-3.17.3 { - catchcmd "test.db" ".prompt FOO BAR" -} {0 {}} -do_test shell1-3.17.4 { - # too many arguments - catchcmd "test.db" ".prompt FOO BAR BAD" -} {1 {Error: unknown command or invalid arguments: "prompt". Enter ".help" for help}} - -# .quit Exit this program -do_test shell1-3.18.1 { - catchcmd "test.db" ".quit" -} {0 {}} -do_test shell1-3.18.2 { - # too many arguments - catchcmd "test.db" ".quit BAD" -} {1 {Error: unknown command or invalid arguments: "quit". Enter ".help" for help}} - -# .read FILENAME Execute SQL in FILENAME -do_test shell1-3.19.1 { - catchcmd "test.db" ".read" -} {1 {Error: unknown command or invalid arguments: "read". Enter ".help" for help}} -do_test shell1-3.19.2 { - file delete -force FOO - catchcmd "test.db" ".read FOO" -} {1 {Error: cannot open "FOO"}} -do_test shell1-3.19.3 { - # too many arguments - catchcmd "test.db" ".read FOO BAD" -} {1 {Error: unknown command or invalid arguments: "read". Enter ".help" for help}} - -# .restore ?DB? FILE Restore content of DB (default "main") from FILE -do_test shell1-3.20.1 { - catchcmd "test.db" ".restore" -} {1 {Error: unknown command or invalid arguments: "restore". Enter ".help" for help}} -do_test shell1-3.20.2 { - catchcmd "test.db" ".restore FOO" -} {0 {}} -do_test shell1-3.20.3 { - catchcmd "test.db" ".restore FOO BAR" -} {1 {Error: unknown database FOO}} -do_test shell1-3.20.4 { - # too many arguments - catchcmd "test.db" ".restore FOO BAR BAD" -} {1 {Error: unknown command or invalid arguments: "restore". Enter ".help" for help}} - -# .schema ?TABLE? Show the CREATE statements -# If TABLE specified, only show tables matching -# LIKE pattern TABLE. -do_test shell1-3.21.1 { - catchcmd "test.db" ".schema" -} {0 {}} -do_test shell1-3.21.2 { - catchcmd "test.db" ".schema FOO" -} {0 {}} -do_test shell1-3.21.3 { - # too many arguments - catchcmd "test.db" ".schema FOO BAD" -} {1 {Error: unknown command or invalid arguments: "schema". Enter ".help" for help}} - -# .separator STRING Change separator used by output mode and .import -do_test shell1-3.22.1 { - catchcmd "test.db" ".separator" -} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} -do_test shell1-3.22.2 { - catchcmd "test.db" ".separator FOO" -} {0 {}} -do_test shell1-3.22.3 { - # too many arguments - catchcmd "test.db" ".separator FOO BAD" -} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} - -# .show Show the current values for various settings -do_test shell1-3.23.1 { - set res [catchcmd "test.db" ".show"] - list [regexp {echo:} $res] \ - [regexp {explain:} $res] \ - [regexp {headers:} $res] \ - [regexp {mode:} $res] \ - [regexp {nullvalue:} $res] \ - [regexp {output:} $res] \ - [regexp {separator:} $res] \ - [regexp {stats:} $res] \ - [regexp {width:} $res] -} {1 1 1 1 1 1 1 1 1} -do_test shell1-3.23.2 { - # too many arguments - catchcmd "test.db" ".show BAD" -} {1 {Error: unknown command or invalid arguments: "show". Enter ".help" for help}} - -# .stats ON|OFF Turn stats on or off -do_test shell1-3.23b.1 { - catchcmd "test.db" ".stats" -} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} -do_test shell1-3.23b.2 { - catchcmd "test.db" ".stats ON" -} {0 {}} -do_test shell1-3.23b.3 { - catchcmd "test.db" ".stats OFF" -} {0 {}} -do_test shell1-3.23b.4 { - # too many arguments - catchcmd "test.db" ".stats OFF BAD" -} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} - -# .tables ?TABLE? List names of tables -# If TABLE specified, only list tables matching -# LIKE pattern TABLE. -do_test shell1-3.24.1 { - catchcmd "test.db" ".tables" -} {0 {}} -do_test shell1-3.24.2 { - catchcmd "test.db" ".tables FOO" -} {0 {}} -do_test shell1-3.24.3 { - # too many arguments - catchcmd "test.db" ".tables FOO BAD" -} {1 {Error: unknown command or invalid arguments: "tables". Enter ".help" for help}} - -# .timeout MS Try opening locked tables for MS milliseconds -do_test shell1-3.25.1 { - catchcmd "test.db" ".timeout" -} {1 {Error: unknown command or invalid arguments: "timeout". Enter ".help" for help}} -do_test shell1-3.25.2 { - catchcmd "test.db" ".timeout zzz" - # this should be treated the same as a '0' timeout -} {0 {}} -do_test shell1-3.25.3 { - catchcmd "test.db" ".timeout 1" -} {0 {}} -do_test shell1-3.25.4 { - # too many arguments - catchcmd "test.db" ".timeout 1 BAD" -} {1 {Error: unknown command or invalid arguments: "timeout". Enter ".help" for help}} - -# .width NUM NUM ... Set column widths for "column" mode -do_test shell1-3.26.1 { - catchcmd "test.db" ".width" -} {1 {Error: unknown command or invalid arguments: "width". Enter ".help" for help}} -do_test shell1-3.26.2 { - catchcmd "test.db" ".width xxx" - # this should be treated the same as a '0' width for col 1 -} {0 {}} -do_test shell1-3.26.3 { - catchcmd "test.db" ".width xxx yyy" - # this should be treated the same as a '0' width for col 1 and 2 -} {0 {}} -do_test shell1-3.26.4 { - catchcmd "test.db" ".width 1 1" - # this should be treated the same as a '1' width for col 1 and 2 -} {0 {}} - -# .timer ON|OFF Turn the CPU timer measurement on or off -do_test shell1-3.27.1 { - catchcmd "test.db" ".timer" -} {1 {Error: unknown command or invalid arguments: "timer". Enter ".help" for help}} -do_test shell1-3.27.2 { - catchcmd "test.db" ".timer ON" -} {0 {}} -do_test shell1-3.27.3 { - catchcmd "test.db" ".timer OFF" -} {0 {}} -do_test shell1-3.27.4 { - # too many arguments - catchcmd "test.db" ".timer OFF BAD" -} {1 {Error: unknown command or invalid arguments: "timer". Enter ".help" for help}} - -do_test shell1-3-28.1 { - catchcmd test.db \ - ".log stdout\nSELECT coalesce(sqlite_log(123,'hello'),'456');" -} "0 {(123) hello\n456}" - -puts "CLI tests completed successfully" DELETED tool/shell2.test Index: tool/shell2.test ================================================================== --- tool/shell2.test +++ /dev/null @@ -1,222 +0,0 @@ -# 2009 Nov 11 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# The focus of this file is testing the CLI shell tool. -# -# $Id: shell2.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ -# - -# Test plan: -# -# shell2-1.*: Misc. test of various tickets and reported errors. -# - -package require sqlite4 - -set CLI "./sqlite4" - -proc do_test {name cmd expected} { - puts -nonewline "$name ..." - set res [uplevel $cmd] - if {$res eq $expected} { - puts Ok - } else { - puts Error - puts " Got: $res" - puts " Expected: $expected" - exit - } -} - -proc execsql {sql} { - uplevel [list db eval $sql] -} - -proc catchsql {sql} { - set rc [catch {uplevel [list db eval $sql]} msg] - list $rc $msg -} - -proc catchcmd {db {cmd ""}} { - global CLI - set out [open cmds.txt w] - puts $out $cmd - close $out - set line "exec $CLI $db < cmds.txt" - set rc [catch { eval $line } msg] - list $rc $msg -} - -file delete -force test.db test.db.journal -sqlite4 db test.db - - -#---------------------------------------------------------------------------- -# shell2-1.*: Misc. test of various tickets and reported errors. -# - -# Batch mode not creating databases. -# Reported on mailing list by Ken Zalewski. -# Ticket [aeff892c57]. -do_test shell2-1.1.1 { - file delete -force foo.db - set rc [ catchcmd "-batch foo.db" "CREATE TABLE t1(a);" ] - set fexist [file exist foo.db] - list $rc $fexist -} {{0 {}} 1} - -# Shell silently ignores extra parameters. -# Ticket [f5cb008a65]. -do_test shell2-1.2.1 { - set rc [catch { eval exec $CLI \":memory:\" \"select 3\" \"select 4\" } msg] - list $rc \ - [regexp {Error: too many options: "select 4"} $msg] -} {1 1} - -# Test a problem reported on the mailing list. The shell was at one point -# returning the generic SQLITE_ERROR message ("SQL error or missing database") -# instead of the "too many levels..." message in the test below. -# -do_test shell2-1.3 { - catchcmd "-batch test.db" { - PRAGMA recursive_triggers = ON; - CREATE TABLE t5(a PRIMARY KEY, b, c); - INSERT INTO t5 VALUES(1, 2, 3); - CREATE TRIGGER au_tble AFTER UPDATE ON t5 BEGIN - UPDATE OR IGNORE t5 SET a = new.a, c = 10; - END; - - UPDATE OR REPLACE t5 SET a = 4 WHERE a = 1; - } -} {1 {Error: near line 9: too many levels of trigger recursion}} - - - -# Shell not echoing all commands with echo on. -# Ticket [eb620916be]. - -# Test with echo off -# NB. whitespace is important -do_test shell2-1.4.1 { - file delete -force foo.db - catchcmd "foo.db" {CREATE TABLE foo(a); -INSERT INTO foo(a) VALUES(1); -SELECT * FROM foo;} -} {0 1} - -# Test with echo on using command line option -# NB. whitespace is important -do_test shell2-1.4.2 { - file delete -force foo.db - catchcmd "-echo foo.db" {CREATE TABLE foo(a); -INSERT INTO foo(a) VALUES(1); -SELECT * FROM foo;} -} {0 {CREATE TABLE foo(a); -INSERT INTO foo(a) VALUES(1); -SELECT * FROM foo; -1}} - -# Test with echo on using dot command -# NB. whitespace is important -do_test shell2-1.4.3 { - file delete -force foo.db - catchcmd "foo.db" {.echo ON -CREATE TABLE foo(a); -INSERT INTO foo(a) VALUES(1); -SELECT * FROM foo;} -} {0 {CREATE TABLE foo(a); -INSERT INTO foo(a) VALUES(1); -SELECT * FROM foo; -1}} - -# Test with echo on using dot command and -# turning off mid- processing. -# NB. whitespace is important -do_test shell2-1.4.4 { - file delete -force foo.db - catchcmd "foo.db" {.echo ON -CREATE TABLE foo(a); -.echo OFF -INSERT INTO foo(a) VALUES(1); -SELECT * FROM foo;} -} {0 {CREATE TABLE foo(a); -.echo OFF -1}} - -# Test with echo on using dot command and -# multiple commands per line. -# NB. whitespace is important -do_test shell2-1.4.5 { - file delete -force foo.db - catchcmd "foo.db" {.echo ON -CREATE TABLE foo1(a); -INSERT INTO foo1(a) VALUES(1); -CREATE TABLE foo2(b); -INSERT INTO foo2(b) VALUES(1); -SELECT * FROM foo1; SELECT * FROM foo2; -INSERT INTO foo1(a) VALUES(2); INSERT INTO foo2(b) VALUES(2); -SELECT * FROM foo1; SELECT * FROM foo2; -} -} {0 {CREATE TABLE foo1(a); -INSERT INTO foo1(a) VALUES(1); -CREATE TABLE foo2(b); -INSERT INTO foo2(b) VALUES(1); -SELECT * FROM foo1; -1 -SELECT * FROM foo2; -1 -INSERT INTO foo1(a) VALUES(2); -INSERT INTO foo2(b) VALUES(2); -SELECT * FROM foo1; -1 -2 -SELECT * FROM foo2; -1 -2}} - -# Test with echo on and headers on using dot command and -# multiple commands per line. -# NB. whitespace is important -do_test shell2-1.4.6 { - file delete -force foo.db - catchcmd "foo.db" {.echo ON -.headers ON -CREATE TABLE foo1(a); -INSERT INTO foo1(a) VALUES(1); -CREATE TABLE foo2(b); -INSERT INTO foo2(b) VALUES(1); -SELECT * FROM foo1; SELECT * FROM foo2; -INSERT INTO foo1(a) VALUES(2); INSERT INTO foo2(b) VALUES(2); -SELECT * FROM foo1; SELECT * FROM foo2; -} -} {0 {.headers ON -CREATE TABLE foo1(a); -INSERT INTO foo1(a) VALUES(1); -CREATE TABLE foo2(b); -INSERT INTO foo2(b) VALUES(1); -SELECT * FROM foo1; -a -1 -SELECT * FROM foo2; -b -1 -INSERT INTO foo1(a) VALUES(2); -INSERT INTO foo2(b) VALUES(2); -SELECT * FROM foo1; -a -1 -2 -SELECT * FROM foo2; -b -1 -2}} - -puts "CLI tests completed successfully" DELETED tool/shell3.test Index: tool/shell3.test ================================================================== --- tool/shell3.test +++ /dev/null @@ -1,124 +0,0 @@ -# 2009 Dec 16 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# The focus of this file is testing the CLI shell tool. -# -# $Id: shell2.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ -# - -# Test plan: -# -# shell3-1.*: Basic tests for running SQL statments from command line. -# shell3-2.*: Basic tests for running SQL file from command line. -# - -package require sqlite4 - -set CLI "./sqlite4" - -proc do_test {name cmd expected} { - puts -nonewline "$name ..." - set res [uplevel $cmd] - if {$res eq $expected} { - puts Ok - } else { - puts Error - puts " Got: $res" - puts " Expected: $expected" - exit - } -} - -proc execsql {sql} { - uplevel [list db eval $sql] -} - -proc catchsql {sql} { - set rc [catch {uplevel [list db eval $sql]} msg] - list $rc $msg -} - -proc catchcmd {db {cmd ""}} { - global CLI - set out [open cmds.txt w] - puts $out $cmd - close $out - set line "exec $CLI $db < cmds.txt" - set rc [catch { eval $line } msg] - list $rc $msg -} - -file delete -force test.db test.db.journal -sqlite4 db test.db - - -#---------------------------------------------------------------------------- -# shell3-1.*: Basic tests for running SQL statments from command line. -# - -# Run SQL statement from command line -do_test shell3-1.1 { - file delete -force foo.db - set rc [ catchcmd "foo.db \"CREATE TABLE t1(a);\"" ] - set fexist [file exist foo.db] - list $rc $fexist -} {{0 {}} 1} -do_test shell3-1.2 { - catchcmd "foo.db" ".tables" -} {0 t1} -do_test shell3-1.3 { - catchcmd "foo.db \"DROP TABLE t1;\"" -} {0 {}} -do_test shell3-1.4 { - catchcmd "foo.db" ".tables" -} {0 {}} -do_test shell3-1.5 { - catchcmd "foo.db \"CREATE TABLE t1(a); DROP TABLE t1;\"" -} {0 {}} -do_test shell3-1.6 { - catchcmd "foo.db" ".tables" -} {0 {}} -do_test shell3-1.7 { - catchcmd "foo.db \"CREATE TABLE\"" -} {1 {Error: near "TABLE": syntax error}} - -#---------------------------------------------------------------------------- -# shell3-2.*: Basic tests for running SQL file from command line. -# - -# Run SQL file from command line -do_test shell3-2.1 { - file delete -force foo.db - set rc [ catchcmd "foo.db" "CREATE TABLE t1(a);" ] - set fexist [file exist foo.db] - list $rc $fexist -} {{0 {}} 1} -do_test shell3-2.2 { - catchcmd "foo.db" ".tables" -} {0 t1} -do_test shell3-2.3 { - catchcmd "foo.db" "DROP TABLE t1;" -} {0 {}} -do_test shell3-2.4 { - catchcmd "foo.db" ".tables" -} {0 {}} -do_test shell3-2.5 { - catchcmd "foo.db" "CREATE TABLE t1(a); DROP TABLE t1;" -} {0 {}} -do_test shell3-2.6 { - catchcmd "foo.db" ".tables" -} {0 {}} -do_test shell3-2.7 { - catchcmd "foo.db" "CREATE TABLE" -} {1 {Error: incomplete SQL: CREATE TABLE}} - - -puts "CLI tests completed successfully" DELETED tool/shell4.test Index: tool/shell4.test ================================================================== --- tool/shell4.test +++ /dev/null @@ -1,129 +0,0 @@ -# 2010 July 28 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# The focus of this file is testing the CLI shell tool. -# These tests are specific to the .stats command. -# -# $Id: shell4.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ -# - -# Test plan: -# -# shell4-1.*: Basic tests specific to the "stats" command. -# - -set CLI "./sqlite4" - -proc do_test {name cmd expected} { - puts -nonewline "$name ..." - set res [uplevel $cmd] - if {$res eq $expected} { - puts Ok - } else { - puts Error - puts " Got: $res" - puts " Expected: $expected" - exit - } -} - -proc catchcmd {db {cmd ""}} { - global CLI - set out [open cmds.txt w] - puts $out $cmd - close $out - set line "exec $CLI $db < cmds.txt" - set rc [catch { eval $line } msg] - list $rc $msg -} - -file delete -force test.db test.db.journal - -#---------------------------------------------------------------------------- -# Test cases shell4-1.*: Tests specific to the "stats" command. -# - -# should default to off -do_test shell4-1.1.1 { - set res [catchcmd "test.db" ".show"] - list [regexp {stats: off} $res] -} {1} - -do_test shell4-1.1.2 { - set res [catchcmd "test.db" ".show"] - list [regexp {stats: on} $res] -} {0} - -# -stats should turn it on -do_test shell4-1.2.1 { - set res [catchcmd "-stats test.db" ".show"] - list [regexp {stats: on} $res] -} {1} - -do_test shell4-1.2.2 { - set res [catchcmd "-stats test.db" ".show"] - list [regexp {stats: off} $res] -} {0} - -# .stats ON|OFF Turn stats on or off -do_test shell4-1.3.1 { - catchcmd "test.db" ".stats" -} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} -do_test shell4-1.3.2 { - catchcmd "test.db" ".stats ON" -} {0 {}} -do_test shell4-1.3.3 { - catchcmd "test.db" ".stats OFF" -} {0 {}} -do_test shell4-1.3.4 { - # too many arguments - catchcmd "test.db" ".stats OFF BAD" -} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} - -# NB. whitespace is important -do_test shell4-1.4.1 { - set res [catchcmd "test.db" {.show}] - list [regexp {stats: off} $res] -} {1} - -do_test shell4-1.4.2 { - set res [catchcmd "test.db" {.stats ON -.show -}] - list [regexp {stats: on} $res] -} {1} - -do_test shell4-1.4.3 { - set res [catchcmd "test.db" {.stats OFF -.show -}] - list [regexp {stats: off} $res] -} {1} - -# make sure stats not present when off -do_test shell4-1.5.1 { - set res [catchcmd "test.db" {SELECT 1;}] - list [regexp {Memory Used} $res] \ - [regexp {Heap Usage} $res] \ - [regexp {Autoindex Inserts} $res] -} {0 0 0} - -# make sure stats are present when on -do_test shell4-1.5.2 { - set res [catchcmd "test.db" {.stats ON -SELECT 1; -}] - list [regexp {Memory Used} $res] \ - [regexp {Heap Usage} $res] \ - [regexp {Autoindex Inserts} $res] -} {1 1 1} - -puts "CLI tests completed successfully" DELETED tool/shell5.test Index: tool/shell5.test ================================================================== --- tool/shell5.test +++ /dev/null @@ -1,243 +0,0 @@ -# 2010 August 4 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# The focus of this file is testing the CLI shell tool. -# These tests are specific to the .import command. -# -# $Id: shell5.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ -# - -# Test plan: -# -# shell5-1.*: Basic tests specific to the ".import" command. -# - -set CLI "./sqlite4" - -proc do_test {name cmd expected} { - puts -nonewline "$name ..." - set res [uplevel $cmd] - if {$res eq $expected} { - puts Ok - } else { - puts Error - puts " Got: $res" - puts " Expected: $expected" - exit - } -} - -proc catchcmd {db {cmd ""}} { - global CLI - set out [open cmds.txt w] - puts $out $cmd - close $out - set line "exec $CLI $db < cmds.txt" - set rc [catch { eval $line } msg] - list $rc $msg -} - -file delete -force test.db test.db.journal - -#---------------------------------------------------------------------------- -# Test cases shell5-1.*: Basic handling of the .import and .separator commands. -# - -# .import FILE TABLE Import data from FILE into TABLE -do_test shell5-1.1.1 { - catchcmd "test.db" ".import" -} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} -do_test shell5-1.1.2 { - catchcmd "test.db" ".import FOO" -} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} -do_test shell5-1.1.2 { - catchcmd "test.db" ".import FOO BAR" -} {1 {Error: no such table: BAR}} -do_test shell5-1.1.3 { - # too many arguments - catchcmd "test.db" ".import FOO BAR BAD" -} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} - -# .separator STRING Change separator used by output mode and .import -do_test shell1-1.2.1 { - catchcmd "test.db" ".separator" -} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} -do_test shell1-1.2.2 { - catchcmd "test.db" ".separator FOO" -} {0 {}} -do_test shell1-1.2.3 { - # too many arguments - catchcmd "test.db" ".separator FOO BAD" -} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} - -# separator should default to "|" -do_test shell5-1.3.1 { - set res [catchcmd "test.db" ".show"] - list [regexp {separator: \"\|\"} $res] -} {1} - -# set separator to different value. -# check that .show reports new value -do_test shell5-1.3.2 { - set res [catchcmd "test.db" {.separator , -.show}] - list [regexp {separator: \",\"} $res] -} {1} - -# import file doesn't exist -do_test shell5-1.4.1 { - file delete -force FOO - set res [catchcmd "test.db" {CREATE TABLE t1(a, b); -.import FOO t1}] -} {1 {Error: cannot open "FOO"}} - -# empty import file -do_test shell5-1.4.2 { - file delete -force shell5.csv - set in [open shell5.csv w] - close $in - set res [catchcmd "test.db" {.import shell5.csv t1 -SELECT COUNT(*) FROM t1;}] -} {0 0} - -# import file with 1 row, 1 column (expecting 2 cols) -do_test shell5-1.4.3 { - set in [open shell5.csv w] - puts $in "1" - close $in - set res [catchcmd "test.db" {.import shell5.csv t1}] -} {1 {Error: shell5.csv line 1: expected 2 columns of data but found 1}} - -# import file with 1 row, 3 columns (expecting 2 cols) -do_test shell5-1.4.4 { - set in [open shell5.csv w] - puts $in "1|2|3" - close $in - set res [catchcmd "test.db" {.import shell5.csv t1}] -} {1 {Error: shell5.csv line 1: expected 2 columns of data but found 3}} - -# import file with 1 row, 2 columns -do_test shell5-1.4.5 { - set in [open shell5.csv w] - puts $in "1|2" - close $in - set res [catchcmd "test.db" {.import shell5.csv t1 -SELECT COUNT(*) FROM t1;}] -} {0 1} - -# import file with 2 rows, 2 columns -# note we end up with 3 rows because of the 1 row -# imported above. -do_test shell5-1.4.6 { - set in [open shell5.csv w] - puts $in "2|3" - puts $in "3|4" - close $in - set res [catchcmd "test.db" {.import shell5.csv t1 -SELECT COUNT(*) FROM t1;}] -} {0 3} - -# import file with 1 row, 2 columns, using a comma -do_test shell5-1.4.7 { - set in [open shell5.csv w] - puts $in "4,5" - close $in - set res [catchcmd "test.db" {.separator , -.import shell5.csv t1 -SELECT COUNT(*) FROM t1;}] -} {0 4} - -# import file with 1 row, 2 columns, text data -do_test shell5-1.4.8.1 { - set in [open shell5.csv w] - puts $in "5|Now is the time for all good men to come to the aid of their country." - close $in - set res [catchcmd "test.db" {.import shell5.csv t1 -SELECT COUNT(*) FROM t1;}] -} {0 5} - -do_test shell5-1.4.8.2 { - catchcmd "test.db" {SELECT b FROM t1 WHERE a='5';} -} {0 {Now is the time for all good men to come to the aid of their country.}} - -# import file with 1 row, 2 columns, quoted text data -# note that currently sqlite doesn't support quoted fields, and -# imports the entire field, quotes and all. -do_test shell5-1.4.9.1 { - set in [open shell5.csv w] - puts $in "6|'Now is the time for all good men to come to the aid of their country.'" - close $in - set res [catchcmd "test.db" {.import shell5.csv t1 -SELECT COUNT(*) FROM t1;}] -} {0 6} - -do_test shell5-1.4.9.2 { - catchcmd "test.db" {SELECT b FROM t1 WHERE a='6';} -} {0 {'Now is the time for all good men to come to the aid of their country.'}} - -# import file with 1 row, 2 columns, quoted text data -do_test shell5-1.4.10.1 { - set in [open shell5.csv w] - puts $in "7|\"Now is the time for all good men to come to the aid of their country.\"" - close $in - set res [catchcmd "test.db" {.import shell5.csv t1 -SELECT COUNT(*) FROM t1;}] -} {0 7} - -do_test shell5-1.4.10.2 { - catchcmd "test.db" {SELECT b FROM t1 WHERE a='7';} -} {0 {Now is the time for all good men to come to the aid of their country.}} - -# check importing very long field -do_test shell5-1.5.1 { - set str [string repeat X 999] - set in [open shell5.csv w] - puts $in "8|$str" - close $in - set res [catchcmd "test.db" {.import shell5.csv t1 -SELECT length(b) FROM t1 WHERE a='8';}] -} {0 999} - -# try importing into a table with a large number of columns. -# This is limited by SQLITE_MAX_VARIABLE_NUMBER, which defaults to 999. -set cols 999 -do_test shell5-1.6.1 { - set sql {CREATE TABLE t2(} - set data {} - for {set i 1} {$i<$cols} {incr i} { - append sql "c$i," - append data "$i|" - } - append sql "c$cols);" - append data "$cols" - catchcmd "test.db" $sql - set in [open shell5.csv w] - puts $in $data - close $in - set res [catchcmd "test.db" {.import shell5.csv t2 -SELECT COUNT(*) FROM t2;}] -} {0 1} - -# try importing a large number of rows -set rows 999999 -do_test shell5-1.7.1 { - set in [open shell5.csv w] - for {set i 1} {$i<=$rows} {incr i} { - puts $in $i - } - close $in - set res [catchcmd "test.db" {CREATE TABLE t3(a); -.import shell5.csv t3 -SELECT COUNT(*) FROM t3;}] -} [list 0 $rows] - - -puts "CLI tests completed successfully" Index: www/index.wiki ================================================================== --- www/index.wiki +++ www/index.wiki @@ -6,5 +6,8 @@ * [./data_encoding.wiki | The Data Encoding] * [./key_encoding.wiki | The Key Encoding] * [./decimal.wiki | Internal representation of numeric values] * [./porting.wiki | Porting an app from SQLite3 to SQLite4] * [./storage.wiki | How to create a new storage engine] + * [./lsmusr.wiki | LSM User Manual] + * [./lsmperf.wiki | LSM Benchmarks] + * [./lsmapi.wiki | LSM API Reference] Index: www/key_encoding.wiki ================================================================== --- www/key_encoding.wiki +++ www/key_encoding.wiki @@ -90,11 +90,11 @@ byte of 0x23. Every other numeric value encoding begins with a smaller byte, ensuring that positive infinity always sorts last among numeric values. 0x0d is also smaller than 0x0e, the initial byte of a text value, ensuring that every numeric value sorts before every text value. -If the numeric value is exactly zero then then is encoded is a single +If the numeric value is exactly zero then it is encoded as a single byte of 0x15. Finite negative values will have initial bytes of 0x08 through 0x14 and finite positive values will have initial bytes of 0x16 through 0x22. For all values, we compute a mantissa M and an exponent E. The mantissa Index: www/lsm.wiki ================================================================== --- www/lsm.wiki +++ www/lsm.wiki @@ -1,10 +1,39 @@ LSM Design Overview -

1. Summary

+ +
+      1. Summary
+      2. Data Structures
+            2.1. Locks
+            2.2. Database file
+                  2.2.1. Sorted Runs
+                  2.2.2. Levels
+                  2.2.3. Snapshots
+            2.3. In-Memory Tree
+                  2.3.1. Memory Allocation
+                  2.3.2. Header Fields
+            2.4. Other Shared-Memory Fields
+            2.5. Log file
+      3. Database Recovery and Shutdown
+            3.1. Read-write clients
+      4. Database Operations
+            4.1. Reading
+            4.2. Writing
+                  4.2.1. Flushing the in-memory tree to disk
+                  4.2.2. Shared-memory management
+                  4.2.3. Log file management
+            4.3. Working
+                  4.3.1. Free-block list management
+            4.4. Checkpoint Operations
+      5. Scheduling Policies
+ +
+ +

1. Summary

The LSM embedded database software stores data in three distinct data structures:
    @@ -44,19 +73,19 @@ in-memory tree. Once the in-memory tree has grown large enough, its contents are written into the database file as a new sorted run. To reduce the number of sorted runs in the database file, chronologically adjacent sorted runs may be merged together into a single run, either automatically or on demand. -

    2. Data Structures

    +

    2. Data Structures

    -

    Locks

    +

    2.1. Locks

    Read/write (shared/exclusive) file locks are used to control concurrent access. LSM uses the following file-locks:

      -

    • The DMS1 and DMS2 locking regions. These are used to +

    • The DMS1, DMS2 locking regions. These are used to implement the "dead-man-switch" mechanism copied from SQLite's WAL mode for safely connecting to and disconnecting from a database. See "Database Recovery and Shutdown" below.

    • Several (say 3) READER locking regions. Database clients @@ -90,11 +119,11 @@ on the WRITER locking region. For example "holding the WRITER lock" is equivalent to "holding an exclusive lock on the WRITER locking region". Similar interpretations apply to "the WORKER lock" and "the CHECKPOINTER lock". -

      Database file

      +

      2.2. Database file

      This section summarizes the contents of the database file informally. A detailed description is found in the header comments for source code files lsm_file.c (blocks, pages etc.), @@ -124,11 +153,11 @@ As with an SQLite database file, each page in the database may be addressed by its 32-bit page number. This means the maximum database size is roughly (pgsz * 2^32) bytes. The first and last pages in each block are 4 bytes smaller than the others. This is to make room for a single page-number. -

      Sorted Runs

      +

      2.2.1. Sorted Runs

      A single sorted run is spread across one or more database pages (each page is a part of at most one sorted run). Given the page number of a page in a sorted run the following statements are true: @@ -166,11 +195,11 @@ it is possible to traverse the entire run in either direction or query for arbitrary values.

      TODO: Embedded pointers. -

      Levels

      +

      2.2.2. Levels

      Each sorted run is assigned to a "level". Normally, a level consists of a single sorted run. However, a level may also consist of a set of sorted runs being incrementally merged into a single run. @@ -227,11 +256,11 @@ -

      Snapshots

      +

      2.2.3. Snapshots

      Each meta page may contain a database snapshot. A snapshot contains all the information required to interpret the remainder of the database file (the sorted runs and free space). Specifically, it contains: @@ -252,11 +281,11 @@

      A more detailed description is available in the header comments in source code file lsm_ckpt.c -

      In-Memory Tree

      +

      2.3. In-Memory Tree

      The in-memory tree is an append-only b-tree of order 4 (each node may have up to 4 children), which is more or less equivalent to a red-black tree. An append-only tree is convenient, as it naturally supports the @@ -266,11 +295,11 @@ The implementation includes some optimizations to reduce the number of interior nodes that are updated when a leaf node is written that are not described here. See header comments in source code file lsm_tree.c for details. -

      Memory Allocation

      +

      2.3.1. Memory Allocation

      More than one in-memory tree may exist in shared-memory at any time. For example in the following scenario: @@ -332,11 +361,11 @@ reconstruct the linked list. Any sequence ids assigned by the failed writer are reverted (perhaps not to their original values, but to values that put them at the start of the linked list - before those chunks that may still be in use by existing readers). -

      Header Fields

      +

      2.3.2. Header Fields

      As well as the in-memory tree data, the following fixed-size fields stored in well-known locations in shared-memory are part of the in-memory tree. Like the in-memory tree data, outside of recovery these fields are only ever written to by clients holding the WRITER lock. @@ -353,29 +382,31 @@ occur mid-transaction. It is only ever read (or written) by clients that hold the WRITER lock.

    -

    Other Shared-Memory Fields

    +

    2.4. Other Shared-Memory Fields

    • Snapshot 1.
    • Snapshot 2.
    • The meta-page pointer. This value is either 1 or 2. It indicates which of the two meta-pages contains the most recent database snapshot.
    • READER lock values.
    -

    Log file

    +

    2.5. Log file

    lsm_log.c. -

    3. Database Recovery and Shutdown

    +

    3. Database Recovery and Shutdown

    + +

    3.1. Read-write clients

    Exclusive locks on locking region DMS1 are used to serialize all connect and -disconnect operations. +disconnect operations performed by read-write clients.

    When an LSM database connection is opened (i.e. lsm_open() is called): 

       lock(DMS1, EXCLUSIVE)           # Block until successful
@@ -417,13 +448,21 @@
     }
     unlock(DMS2)
   unlock(DMS1)
    -

    4. Database Operations

    +

    3.1. Read-only clients

    -

    Reading

    +

    It is assumed that read-only clients may take SHARED locks only. And +that a read-only client may not run database recovery when a db is opened +in multi-process mode. + +

    + +

    4. Database Operations

    + +

    4.1. Reading

    Opening a read transaction:

      @@ -430,12 +469,12 @@

    1. Load the current tree-header from shared-memory.

    2. Load the current snapshot from shared-memory.

      Steps 1 and 2 are similar. In both cases, there are two copies of the data structure being read in shared memory. No lock is held to prevent -another client updating them while the read is taking place using the following -pattern: +another client updating them while the read is taking place. Updaters use the +following pattern:

      1. Update copy 2.
      2. Invoke xShmBarrier().
      3. Update copy 1. @@ -514,11 +553,11 @@ lock is held.

        To close a read transaction all that is required is to drop the SHARED lock held on the READER slot. -

        Writing

        +

        4.2. Writing

        To open a write transaction:

          @@ -562,11 +601,11 @@ that it is consistent with the current tree-header.
        1. Clear the writer flag.
        -

        Flushing the in-memory tree to disk

        +

        4.2.1. Flushing the in-memory tree to disk

        For the purposes of writing, the database file and the in-memory tree are largely independent. Processes holding the WRITER lock write to the in-memory tree, and processes holding the WORKER lock write to the database file. @@ -584,11 +623,11 @@

      4. Commit the write transaction, writing the new, empty tree to shared-memory.
      -

      Shared-memory management

      +

      4.2.2. Shared-memory management

      A writer client may have to allocate new shared-memory chunks. This can be done either by extending the shared-memory region or by recycling the first chunk in the linked-list. To check if the first chunk in the linked-list may @@ -603,11 +642,11 @@ reader. A writer checks this by scanning (and possibly updating) the values associated with the READER locks - similar to the way SQLite does in WAL mode.

-

Log file management

+

4.2.3. Log file management

A writer client also writes to the log file. All information required to write to the log file (the offset to write to and the initial checksum values) is embedded in the tree-header. Except, in order to reuse log file space (wrap @@ -623,11 +662,11 @@ access to information stored in the newest snapshot written into the database header. Their exists a shared-memory variable indicating which of the two meta-pages contain this snapshot, but the writer process still has to read the snapshot data and verify its checksum from disk. -

Working

+

4.3. Working

Working is similar to writing. The difference is that a "writer" modifies the in-memory tree. A "worker" modifies the contents of the database file. @@ -647,11 +686,11 @@

  • Update snapshot-1 in shared-memory.

  • Release the WORKER lock. -

    Free-block list management

    +

    4.3.1. Free-block list management

    Worker clients occasionally need to allocate new database blocks or move existing blocks to the free-block list. Along with the block number of each free block, the free-block list contains the snapshot-id of the first @@ -677,11 +716,11 @@ shared-memory variable. -

    Checkpoint Operations

    +

    4.4. Checkpoint Operations

    1. Take CHECKPOINTER lock.
    2. Load snapshot-1 from shared-memory. If the checksum does not match @@ -703,11 +742,11 @@ step 5.
    3. Drop the CHECKPOINTER lock.
    -

    5. Scheduling Policies

    +

    5. Scheduling Policies

    When a client writes to a database, the in-memory tree and log file are updated by the client itself before the lsm_write() call returns. Eventually, once sufficient writes have accumulated in memory, the client marks the @@ -746,9 +785,11 @@

    If the WORKER lock cannot be obtained immediately, block until it can be

    Auto work + + Index: www/lsmapi.wiki ================================================================== --- www/lsmapi.wiki +++ www/lsmapi.wiki @@ -19,11 +19,11 @@

    1. Database Runtime Environment
    2. LSM Error Codes
    3. Creating and Destroying Database Connection Handles
    4. Connecting to a Database -
    5. Obtaining pointers to databases environments +
    6. Obtaining pointers to database environments
    7. Configuring a database connection.
    8. Compression and/or Encryption Hooks
    9. Allocating and Freeing Memory
    10. Querying a Connection For Operational Data
    11. Opening and Closing Write Transactions @@ -36,11 +36,10 @@

    All LSM API Functions

    lsm_begin lsm_checkpoint -lsm_ckpt_size lsm_close lsm_commit lsm_config lsm_config_log lsm_config_work_hook @@ -62,46 +61,49 @@ lsm_info lsm_insert lsm_new lsm_open lsm_rollback -lsm_tree_size lsm_work

    All LSM API Types

    +lsm_compress lsm_compress lsm_env

    All LSM API Constants

    LSM_BUSY LSM_CANTOPEN LSM_CONFIG_AUTOCHECKPOINT +LSM_CONFIG_AUTOFLUSH +LSM_CONFIG_AUTOMERGE LSM_CONFIG_AUTOWORK LSM_CONFIG_BLOCK_SIZE LSM_CONFIG_GET_COMPRESSION -LSM_CONFIG_LOG_SIZE LSM_CONFIG_MAX_FREELIST LSM_CONFIG_MMAP LSM_CONFIG_MULTIPLE_PROCESSES -LSM_CONFIG_NMERGE LSM_CONFIG_PAGE_SIZE LSM_CONFIG_SAFETY LSM_CONFIG_SET_COMPRESSION +LSM_CONFIG_SET_COMPRESSION_FACTORY LSM_CONFIG_USE_LOG -LSM_CONFIG_WRITE_BUFFER LSM_CORRUPT LSM_ERROR LSM_FULL LSM_INFO_ARRAY_PAGES LSM_INFO_ARRAY_STRUCTURE +LSM_INFO_CHECKPOINT_SIZE LSM_INFO_DB_STRUCTURE LSM_INFO_FREELIST +LSM_INFO_FREELIST_SIZE LSM_INFO_LOG_STRUCTURE LSM_INFO_NREAD LSM_INFO_NWRITE LSM_INFO_PAGE_ASCII_DUMP LSM_INFO_PAGE_HEX_DUMP +LSM_INFO_TREE_SIZE LSM_IOERR LSM_MISUSE LSM_MUTEX_GLOBAL LSM_MUTEX_HEAP LSM_NOMEM @@ -180,49 +182,63 @@

    Open and close a database connection handle.

    Connecting to a Database

    int lsm_open(lsm_db *pDb, const char *zFilename); -

    Obtaining pointers to databases environments

    +

    Obtaining pointers to database environments

    lsm_env *lsm_get_env(lsm_db *pDb); lsm_env *lsm_default_env(void);

    Return a pointer to the environment used by the database connection passed as the first argument. Assuming the argument is valid, this function always returns a valid environment pointer - it cannot fail. The lsm_default_env() function returns a pointer to the default LSM environment for the current platform. -

    Configuring a database connection.

    +

    Configuring a database connection.

    int lsm_config(lsm_db *, int, ...); -#define LSM_CONFIG_WRITE_BUFFER 1 -#define LSM_CONFIG_PAGE_SIZE 2 -#define LSM_CONFIG_SAFETY 3 -#define LSM_CONFIG_BLOCK_SIZE 4 -#define LSM_CONFIG_AUTOWORK 5 -#define LSM_CONFIG_LOG_SIZE 6 -#define LSM_CONFIG_MMAP 7 -#define LSM_CONFIG_USE_LOG 8 -#define LSM_CONFIG_NMERGE 9 -#define LSM_CONFIG_MAX_FREELIST 10 -#define LSM_CONFIG_MULTIPLE_PROCESSES 11 -#define LSM_CONFIG_AUTOCHECKPOINT 12 -#define LSM_CONFIG_SET_COMPRESSION 13 -#define LSM_CONFIG_GET_COMPRESSION 14 +#define LSM_CONFIG_AUTOFLUSH 1 +#define LSM_CONFIG_PAGE_SIZE 2 +#define LSM_CONFIG_SAFETY 3 +#define LSM_CONFIG_BLOCK_SIZE 4 +#define LSM_CONFIG_AUTOWORK 5 +#define LSM_CONFIG_MMAP 7 +#define LSM_CONFIG_USE_LOG 8 +#define LSM_CONFIG_AUTOMERGE 9 +#define LSM_CONFIG_MAX_FREELIST 10 +#define LSM_CONFIG_MULTIPLE_PROCESSES 11 +#define LSM_CONFIG_AUTOCHECKPOINT 12 +#define LSM_CONFIG_SET_COMPRESSION 13 +#define LSM_CONFIG_GET_COMPRESSION 14 +#define LSM_CONFIG_SET_COMPRESSION_FACTORY 15 #define LSM_SAFETY_OFF 0 #define LSM_SAFETY_NORMAL 1 #define LSM_SAFETY_FULL 2

    The lsm_config() function is used to configure a database connection. The following values may be passed as the second argument to lsm_config(). -

    LSM_CONFIG_WRITE_BUFFER
    A read/write integer parameter. This value determines the maximum amount -of space (in bytes) used to accumulate writes in main-memory before -they are flushed to a level 0 segment. +

    LSM_CONFIG_AUTOFLUSH
    A read/write integer parameter. +

    This value determines the amount of data allowed to accumulate in a +live in-memory tree before it is marked as old. After committing a +transaction, a connection checks if the size of the live in-memory tree, +including data structure overhead, is greater than the value of this +option in KB. If it is, and there is not already an old in-memory tree, +the live in-memory tree is marked as old. +

    The maximum allowable value is 1048576 (1GB). There is no minimum +value. If this parameter is set to zero, then an attempt is made to +mark the live in-memory tree as old after each transaction is committed. +

    The default value is 1024 (1MB).

    LSM_CONFIG_PAGE_SIZE
    A read/write integer parameter. This parameter may only be set before lsm_open() has been called. -

    LSM_CONFIG_BLOCK_SIZE
    A read/write integer parameter. This parameter may only be set before -lsm_open() has been called. -

    LSM_CONFIG_LOG_SIZE
    A read/write integer parameter. +

    LSM_CONFIG_BLOCK_SIZE
    A read/write integer parameter. +

    This parameter may only be set before lsm_open() has been called. It +must be set to a power of two between 64 and 65536, inclusive (block +sizes between 64KB and 64MB). +

    If the connection creates a new database, the block size of the new +database is set to the value of this option in KB. After lsm_open() +has been called, querying this parameter returns the actual block +size of the opened database. +

    The default value is 1024 (1MB blocks).

    LSM_CONFIG_SAFETY
    A read/write integer parameter. Valid values are 0, 1 (the default) and 2. This parameter determines how robust the database is in the face of a system crash (e.g. a power failure or operating system crash). As follows:

    0 (off): No robustness. A system crash may corrupt the database. @@ -232,15 +248,26 @@

    2 (full): Full robustness. A system crash may not corrupt the database file. Following recovery the database file contains all successfully committed transactions.

    LSM_CONFIG_AUTOWORK
    A read/write integer parameter.

    LSM_CONFIG_AUTOCHECKPOINT
    A read/write integer parameter. +

    If this option is set to non-zero value N, then a checkpoint is +automatically attempted after each N KB of data have been written to +the database file. +

    The amount of uncheckpointed data already written to the database file +is a global parameter. After performing database work (writing to the +database file), the process checks if the total amount of uncheckpointed +data exceeds the value of this paramter. If so, a checkpoint is performed. +This means that this option may cause the connection to perform a +checkpoint even if the current connection has itself written very little +data into the database file. +

    The default value is 2048 (checkpoint every 2MB).

    LSM_CONFIG_MMAP
    A read/write integer parameter. True to use mmap() to access the database file. False otherwise.

    LSM_CONFIG_USE_LOG
    A read/write boolean parameter. True (the default) to use the log file normally. False otherwise. -

    LSM_CONFIG_NMERGE
    A read/write integer parameter. The minimum number of segments to +

    LSM_CONFIG_AUTOMERGE
    A read/write integer parameter. The minimum number of segments to merge together at a time. Default value 4.

    LSM_CONFIG_MAX_FREELIST
    A read/write integer parameter. The maximum number of free-list entries that are stored in a database checkpoint (the others are stored elsewhere in the database).

    There is no reason for an application to configure or query this @@ -257,17 +284,25 @@ structures contents.

    This option may only be used before lsm_open() is called. Invoking it after lsm_open() has been called results in an LSM_MISUSE error.

    LSM_CONFIG_GET_COMPRESSION
    Query the compression methods used to compress and decompress database content. -

    Compression and/or Encryption Hooks

    +

    LSM_CONFIG_SET_COMPRESSION_FACTORY
    Configure a factory method to be invoked in case of an LSM_MISMATCH +error. +

    Compression and/or Encryption Hooks

    struct lsm_compress { void *pCtx; unsigned int iId; int (*xBound)(void *, int nSrc); int (*xCompress)(void *, char *, int *, const char *, int); int (*xUncompress)(void *, char *, int *, const char *, int); + void (*xFree)(void *pCtx); +}; +struct lsm_compress_factory { + void *pCtx; + int (*xFactory)(void *, lsm_db *, u32); + void (*xFree)(void *pCtx); };

    Allocating and Freeing Memory

    void *lsm_malloc(lsm_env*, size_t); void *lsm_realloc(lsm_env*, void *, size_t); @@ -274,11 +309,11 @@ void lsm_free(lsm_env*, void *);

    Invoke the memory allocation functions that belong to environment pEnv. Or the system defaults if no memory allocation functions have been registered. -

    Querying a Connection For Operational Data

    +

    Querying a Connection For Operational Data

    int lsm_info(lsm_db *, int, ...); #define LSM_INFO_NWRITE 1 #define LSM_INFO_NREAD 2 #define LSM_INFO_DB_STRUCTURE 3 #define LSM_INFO_LOG_STRUCTURE 4 @@ -285,10 +320,13 @@ #define LSM_INFO_ARRAY_STRUCTURE 5 #define LSM_INFO_PAGE_ASCII_DUMP 6 #define LSM_INFO_PAGE_HEX_DUMP 7 #define LSM_INFO_FREELIST 8 #define LSM_INFO_ARRAY_PAGES 9 +#define LSM_INFO_CHECKPOINT_SIZE 10 +#define LSM_INFO_TREE_SIZE 11 +#define LSM_INFO_FREELIST_SIZE 12

    Query a database connection for operational statistics or data. The following values may be passed as the second argument to lsm_info().

    LSM_INFO_NWRITE
    The third parameter should be of type (int *). The location pointed to by the third parameter is set to the number of 4KB pages written to @@ -349,10 +387,26 @@ the string representation of a Tcl data-structure. The returned string should be eventually freed by the caller using lsm_free().

    The Tcl structure returned is a list containing one element for each free block in the database. The element itself consists of two integers - the block number and the id of the snapshot that freed it. +

    LSM_INFO_CHECKPOINT_SIZE
    The third argument should be of type (int *). The location pointed to +by this argument is populated with the number of KB written to the +database file since the most recent checkpoint. +

    LSM_INFO_TREE_SIZE
    If this value is passed as the second argument to an lsm_info() call, it +should be followed by two arguments of type (int *) (for a total of four +arguments). +

    At any time, there are either one or two tree structures held in shared +memory that new database clients will access (there may also be additional +tree structures being used by older clients - this API does not provide +information on them). One tree structure - the current tree - is used to +accumulate new data written to the database. The other tree structure - +the old tree - is a read-only tree holding older data and may be flushed +to disk at any time. +

    Assuming no error occurs, the location pointed to by the first of the two +(int *) arguments is set to the size of the old in-memory tree in KB. +The second is set to the size of the current, or live in-memory tree.

    Opening and Closing Write Transactions

    int lsm_begin(lsm_db *pDb, int iLevel); int lsm_commit(lsm_db *pDb, int iLevel); int lsm_rollback(lsm_db *pDb, int iLevel); @@ -389,23 +443,23 @@ Delete all database entries with keys that are greater than (pKey1/nKey1) and smaller than (pKey2/nKey2). Note that keys (pKey1/nKey1) and (pKey2/nKey2) themselves, if they exist in the database, are not deleted.

    Return LSM_OK if successful, or an LSM error code otherwise.

    Explicit Database Work and Checkpointing

    -int lsm_work(lsm_db *pDb, int nMerge, int nPage, int *pnWrite); +int lsm_work(lsm_db *pDb, int nMerge, int nKB, int *pnWrite); int lsm_flush(lsm_db *pDb); -int lsm_checkpoint(lsm_db *pDb, int *pnByte); +int lsm_checkpoint(lsm_db *pDb, int *pnKB);

    This function is called by a thread to work on the database structure. Attempt to checkpoint the current database snapshot. Return an LSM error code if an error occurs or LSM_OK otherwise.

    If the current snapshot has already been checkpointed, calling this -function is a no-op. In this case if pnByte is not NULL, *pnByte is +function is a no-op. In this case if pnKB is not NULL, *pnKB is set to 0. Or, if the current snapshot is successfully checkpointed -by this function and pbCkpt is not NULL, *pnByte is set to the number +by this function and pbKB is not NULL, *pnKB is set to the number of bytes written to the database file since the previous checkpoint -(the same measure as returned by lsm_ckpt_size()). +(the same measure as returned by the LSM_INFO_CHECKPOINT_SIZE query).

    Opening and Closing Database Cursors

    int lsm_csr_open(lsm_db *pDb, lsm_cursor **ppCsr); int lsm_csr_close(lsm_cursor *pCsr);

    Open and close a database cursor. @@ -479,39 +533,14 @@ or greater than pKey/nKey, *piRes is set to less than, equal to or greater than zero before returning. LSM_OK is returned in this case.

    Or, if an error occurs, an LSM error code is returned and the final value of *piRes is undefined. If the cursor does not point to a valid key when this function is called, LSM_MISUSE is returned. -

    Change these!!

    +

    Change these!!

    void lsm_config_log(lsm_db *, void (*)(void *, int, const char *), void *); void lsm_config_work_hook(lsm_db *, void (*)(lsm_db *, void *), void *); -int lsm_tree_size(lsm_db *, int *pbOld, int *pnNew); -int lsm_ckpt_size(lsm_db *, int *pnByte);

    Configure a callback to which debugging and other messages should be directed. Only useful for debugging lsm. Configure a callback that is invoked if the database connection ever writes to the database file. -The lsm_tree_size() function reports on the current state of the -in-memory tree data structure. -

    At any time, there are either one or two tree structures held in shared -memory that new database clients will access (there may also be additional -tree structures being used by older clients - this API does not provide -information on them). One tree structure - the current tree - is used to -accumulate new data written to the database. The other tree structure - the -old tree - is a read-only tree holding older data and may be flushed to disk -at any time. -

    If successful, this function sets *pnNew to the number of bytes of shared -memory space used by the current tree. *pbOld is set to true if the old -tree exists, or false if it does not. -

    If no error occurs, LSM_OK is returned. Otherwise an LSM error code. -

    RACE CONDITION: -

    Describe the race condition this function is subject to.
    -This function is used to query the amount of data that has been written -to the database file but not checkpointed (synced). If successful, *pnByte -is set to the number of bytes before returning. -

    LSM_OK is returned if successful. Or if an error occurs, an LSM error -code is returned. -

    RACE CONDITION: -

    Describe the race condition this function is subject to. Or remove
    -
    it somehow.
    ADDED www/lsmperf.wiki Index: www/lsmperf.wiki ================================================================== --- /dev/null +++ www/lsmperf.wiki @@ -0,0 +1,106 @@ + +LSM Benchmarks + + +

    +This page contains the results of a benchmark test run on the LSM library +in several different configurations. + +

    +The test uses a single client database connection. It begins with an empty +database. The test runs for 200 iterations. Each iteration, the client: + +

      +
    • Inserts 50,000 new key-value pairs into the database. Each key is + a pseudo-randomly generated 12-byte blob. Each value is a + pseudo-randomly generated 100 byte blob. Each insert operation is + run in a separate implicit transaction. +
    • Runs 50,000 fetch queries to retrieve arbitrarily selected entries + from the database (all searches are hits - the queried keys are + selected from the set of keys that are present in the db). +
    + +

    +The time taken to insert or retrieve each batch of 50,000 keys is recorded, +and transformed to an operations-per-second rate (e.g. if it takes 0.5 +seconds to insert 50,000 keys, this is reported as 100,000 +operations-per-second). These values are the values reported below. +The final database size is roughly 1.2GB. + +

    +All tested configurations set the +LSM_CONFIG_SAFETY parameter to "normal" +and the +LSM_CONFIG_MULTIPLE_PROCESSES parameter to zero (single process mode). + +

    +Tests were run on a 3.3GHz dual-core PC with 4GB of RAM. The file-system is +ext4 on a 7200rpm HDD. + +

    Small Write Buffers Test

    + +

    +The following plot shows the performance of LSM in its default, single-threaded +configuration against a multi-threaded deployment. + +

    +In the single-threaded mode the in-memory tree is allowed to use up to 1MB of +memory before it is flushed to disk, and the database is checkpointed after +each 2MB of data is written into the database file. + +

    +The multi-threaded configuration launches two background threads as well as +the main client thread used to run the test case. One background thread is +dedicated to calling lsm_work() (writing data into the database file) and +the other to calling lsm_checkpoint() (writing to the database header and +syncing the database file). The in-memory tree is declared eligible to +be flushed to disk when it uses 1MB of memory. If it grows to 1.5MB, then +the client thread is blocked from continuing until the background thread +has flushed it to disk. After 2MB of data has been written into the database +file it is eligible for checkpointing. Once the database file contains 3MB of +uncheckpointed data, the worker thread is prevented from writing any further +data to the file until a checkpoint has been completed. + +

    +More detail regarding lsm_work() and lsm_checkpoint() is +available here. + +

    + + +

    Large Write Buffers Test

    +

    +This test is similar to the one above, except with larger write buffers. +Specifically, in the single-threaded mode: +

      +
    • The in-memory tree is allowed to use up to 4MB of memory before it + is flushed to disk, and +
    • The database is checkpointed after each 8MB of data is written to + the database file. +
    + +

    +And in multi-threaded mode: +

      +
    • The in-memory tree is eligible to be flushed to disk once it uses + 4MB of memory. The client thread is blocked if it reaches 6MB in + size. +
    • The database is still eligible for checkpointing after each 2MB of + data is written to it, but up to 8MB of uncheckpointed data is allowed + to accumulate in the database file before the worker thread is + blocked. +
    + +

    + + +

    Large Write Buffers Test With Pauses

    + +

    +This test is the same as the previous test, except that each iteration the +client thread pauses (sleeps) for 2.5 seconds after performing the database +inserts and queries. + +

    + + ADDED www/lsmperf1.gif Index: www/lsmperf1.gif ================================================================== --- /dev/null +++ www/lsmperf1.gif cannot compute difference between binary files ADDED www/lsmperf2.gif Index: www/lsmperf2.gif ================================================================== --- /dev/null +++ www/lsmperf2.gif cannot compute difference between binary files ADDED www/lsmperf3.gif Index: www/lsmperf3.gif ================================================================== --- /dev/null +++ www/lsmperf3.gif cannot compute difference between binary files Index: www/lsmusr.wiki ================================================================== --- www/lsmusr.wiki +++ www/lsmusr.wiki @@ -5,10 +5,15 @@

    Table of Contents

    + + + + +
          1. Introduction to LSM
          2. Using LSM in Applications
          3. Basic Usage
    @@ -17,25 +22,25 @@             3.3. Reading from a Database
                3.4. Database Transactions and MVCC
          4. Data Durability
          5. Compressed and Encrypted Databases
          6. Performance Tuning
    -            6.1. Performance Related Configuration Options
    -            6.2. Using Worker Threads or Processes
    -                  6.2.1. Architectural Overview
    -                  6.2.2. Automatic Work and Checkpoint Scheduling
    -                  6.2.3. Explicit Work and Checkpoint Scheduling
    -                  6.2.4. Compulsary Work and Checkpoint Scheduling
    -            6.3. Database File Optimization
    +            6.1. Overview of LSM Architecture
    +            6.2. Performance Related Configuration Options
    +            6.3. Work and Checkpoint Scheduling
    +                  6.3.1. Automatic Scheduling
    +                  6.3.2. Explicit Scheduling
    +                  6.3.3. Compulsary Work and Checkpoints
    +            6.4. Database File Optimization

    Overview

    -

    This document describes the LSM embedded database library and use thereof. -It is intended to be part user-manual and part tutorial. It is intended to -complement the LSM API reference manual. +

    This document describes the LSM embedded database library and use thereof. +It is part user-manual and part tutorial. It is intended to complement the +LSM API reference manual.

    The first section of this document contains a description of the LSM library and its features. Section 2 describes how to use LSM from within a C or C++ application (how to compile and link LSM, what to #include @@ -47,13 +52,14 @@ create applications that use LSM. The remaining sections discuss more specialized topics. Section 4 discusses the configuration parameter that influences transaction durability (the guarantees offered with respect to recently committed transactions if a power failure occurs). Section 5 explains -the interface provided by LSM that allow external data compression and/or +the interface provided by LSM that allows external data compression and/or encryption functions to be used to create compressed and/or encrypted -databases. Todo: Clarify exactly what section 6 is and link it to here. +databases. Section 6 deals with performance +tuning.

    1. Introduction to LSM

    LSM is an embedded database library for key-value data, roughly similar in scope to @@ -75,25 +81,25 @@

    Other salient features are:

      -

    • LSM supports a single-writer/multiple-reader MVCC based - transactional concurrency model. SQL style nested sub-transactions are - supported. Clients may concurrently access a single LSM database from - within a single or multiple application processes. +

    • A single-writer/multiple-reader MVCC based transactional + concurrency model. SQL style nested sub-transactions are supported. + Clients may concurrently access a single LSM database from within a + single process or multiple application processes. -

    • An entire LSM database is stored in a single file on disk. +

    • An entire database is stored in a single file on disk.

    • Data durability in the face of application or power failure. LSM may optionally use a write-ahead log file when writing to the database to ensure committed transactions are not lost if an application or power failure occurs. -

    • LSM may be configured to use external data compression and/or - encryption routines to create and access compressed and/or encrypted - databases. +
    • An API that allows external data compression and/or encryption + routines to be used to create and access compressed and/or + encrypted databases.

    Many database systems that support range queries, including SQLite 3, Berkeley DB and Kyoto Cabinet, are @@ -102,43 +108,40 @@ B-trees are attractive because a b-tree structure minimizes the number of disk sectors that must be read from disk when searching the database for a specific key. However, b-tree implementations usually suffer from poor write localization - updating the contents of a b-tree often involves modifying the contents of nodes scattered throughout the database file. If the database is -stored on a spinning disk (HDD), then the disk heads must be moved before -writing non-contiguous sector, which is extremely slow. If the database is +stored on a spinning disk (HDD), then the disk heads must be moved between +writing non-contiguous sectors, which is extremely slow. If the database is stored on solid state storage (SDD) a similar phenomena is encountered due to the large erase-block sizes. In general, writing to a series of contiguous disk sectors is orders of magnitude faster than updating to the same number of disk sectors scattered randomly throughout a large file. Additionally, b-tree structures are prone to fragmentation, reducing the speed of range queries. -

    Todo: Should have references for the claims above. - -

    Also, fix the link in the next paragraph to point to a description +

    TODO: fix the link in the next paragraph to point to a description of the log-structured-merge tree within lsm.wiki (or its successor).

    LSM uses a different data structure that makes the following performance tradeoffs relative to a b-tree:

    • A very large percentage of the disk sectors modified when writing to the database are contiguous. Additionally, in many cases the total number of sectors written - to disk is reduced. This makes writing to an LSM database much - faster than the equivalent b-tree. + to disk is reduced. This makes writing to an LSM database faster + than the equivalent b-tree.
    • LSM databases do not suffer from fragmentation to the same degree as b-trees. This means that the performance of large range queries does not degrade as the database is updated as it may with a b-tree. -
    • It is accepted that under some circumstances searching an LSM - database for a given key will involve examining more disk sectors - than it would with a b-tree. In terms of disk sectors accessed when - searching a database of size N, both b-trees and LSM provide O(log(N)) - efficiency, but the base of the logarithm is generally larger for a - b-tree than for LSM. +
    • Under some circumstances searching an LSM database for a given key will + involve examining more disk sectors than it would with a b-tree. In + terms of disk sectors accessed when searching a database of size N, + both b-trees and LSM provide O(log(N)) efficiency, but the base of + the logarithm is generally larger for a b-tree than for LSM.

    In other words, writing to an LSM database should be very fast and scanning through large ranges of keys should also perform well, but searching the database for specific keys may be slightly slower than when using a b-tree @@ -147,22 +150,20 @@ flash memory devices.

    Although it has quite different features to LSM in other respects, LevelDB makes similar performance tradeoffs. -

    Benchmark test results for LSM are available here. Todo: -Fix this link to point to a page with performance graphs. - +

    Benchmark test results for LSM are available here.

    2. Using LSM in Applications

    LSM is not currently built or distributed independently. Instead, it is part of the SQLite4 library. To use LSM in an application, the application links against libsqlite4 and includes the header file "lsm.h" in any files that access the LSM API. -

    Pointer to build instructions for sqlite4 +

    Pointer to build instructions for sqlite4

    3. Basic Usage

    3.1. Opening and Closing Database Connections

    @@ -191,14 +192,13 @@ object allows the application to supply custom implementations of the various operating system calls that LSM uses to read and write files, allocate heap memory, and coordinate between multiple application threads and processes. This is normally only required if LSM is being used on a platform that is not supported by default. Passing NULL instructs the library to use the default -implementations of all these things, which is usually the right thing to do. -The second argument to lsm_new() is an output variable. Assuming the call -is successful, *pDb is set to point to the new database handle before -returning. +implementations of all these things. The second argument to lsm_new() is an +output variable. Assuming the call is successful, *pDb is set to point to the +new database handle before returning.

    The first argument passed to lsm_open() must be an existing database handle. The second is the name of the database file to connect to. Once lsm_open() has been successfully called on a database handle, it can not be called again on the same handle. Attempting to do so is an LSM_MISUSE error. @@ -228,15 +228,15 @@ rc = lsm_close(db);

    It is important that lsm_close() is called to close all database handles created with lsm_new(), particularly if the connection has written to the -database. If an application that has written to a database exits without +database. If an application writes to the database and then exits without closing its database connection, then subsequent clients may have to run -"database recovery" when they open the database, making the lsm_open() call -less responsive. Additionally, not matching each successful lsm_new() call -with a call to lsm_close() is a resource leak. +"database recovery" when they open the database, slowing down the lsm_open() +call. Additionally, not matching each successful lsm_new() call with a call +to lsm_close() is a resource leak.

    Counter-intuitively, an lsm_close() call may fail. In this case the database handle is not closed, so if the application exits it invites the "database recovery" performance problem mentioned above. The usual reason for an lsm_close() call failing is that the database handle has been used to create @@ -372,17 +372,17 @@ lsm_csr_valid() returns 0, and the loop is finished. API function lsm_csr_key() is used to retrieve the key associated with each database entry visited. - for(rc = lsm_csr_first(csr); lsm_csr_valid(csr); rc = lsm_csr_next(csr)){ + for(rc=lsm_csr_first(csr); rc==LSM_OK && lsm_csr_valid(csr); rc=lsm_csr_next(csr)){ const void *pKey; int nKey; const void *pVal; int nVal; rc = lsm_csr_key(csr, &pKey, &nKey); if( rc==LSM_OK ) rc = lsm_csr_value(csr, &pVal, &nVal); - if( rc!=LSM_OK ) break; + if( rc==LSM_OK ) break; /* At this point pKey points to the current key (size nKey bytes) and ** pVal points to the corresponding value (size nVal bytes). */ } @@ -520,11 +520,11 @@ write-transaction.

    Write-transactions may be opened either implicitly or explicitly. If any of the following functions are called to write to the database when there is no write-transaction open, then an implicit write-transaction is opened and -close (committed) within the function: +closed (committed) within the call:

    • lsm_insert()
    • lsm_delete()
    • lsm_delete_range() @@ -725,76 +725,224 @@ int (*xUncompress)(void *pCtx, void *pOut, int *pnOut, const void *pIn, int nIn); void (*xFree)(void *pCtx); }; -

      Explain how the hooks work here (same as zipvfs) +

      Explain how the hooks work here (same as zipvfs) + -

      Example code? Using zlib? Or something simple like an RLE -implementation? +

      Example code? Using zlib? Or something simple like an +RLE implementation?

      The database file header of any LSM database contains a 32-bit unsigned "compression id" field. If the database is not a compressed database, this field is set to 1. Otherwise, it is set to an application supplied value identifying the compression and/or encryption scheme in use. Application compression scheme ids must be greater than or equal to 10000. Values smaller than 10000 are reserved for internal use. -

      The lsm_compression_id() API may be used to read the compression id from -a database connection. Because the compression id is stored in the database +

      The lsm_info() API may be used to read the compression id from a database +connection as follows: + + + unsigned int iCompressionId; + rc = lsm_info(db, LSM_INFO_COMPRESSION_ID, &iCompressionId); + if( rc==LSM_OK ){ + /* Variable iCompressionId now contains the db compression id */ + } + + +Because the compression id is stored in the database header, it may be read before any required compression or encryption hooks are configured. #define LSM_COMPRESSION_EMPTY 0 #define LSM_COMPRESSION_NONE 1 - int lsm_compression_id(lsm_db *db, u32 *piId);

      When a database is opened for the first time, before it is first written, -the compression id field is set to LSM_COMPRESSION_EMPTY (0). The first time -a transaction is committed, the database compression id is set to a copy of -the lsm_compress.iId field of the compression hooks for the database handle -committing the transaction, or to LSM_COMPRESSION_NONE (1) if no compression -hooks are configured. +the compression id field is set to LSM_COMPRESSION_EMPTY (0). After data is +written into the database file, the database compression id is set to a copy +of the lsm_compress.iId field of the compression hooks for the database handle +doing the writing, or to LSM_COMPRESSION_NONE (1) if no compression hooks +are configured.

      Once the compression id is set to something other than -LSM_COMPRESSION_EMPTY, when a database handle opens a read or write -transaction on the database, the compression id is compared against the -lsm_compress.iId field of the configured compression hooks, or against LSM_COMPRESSION_NONE if no compression hooks are configured. If the compression id -does not match, then an LSM_MISMATCH error is returned and the operation -fails (no transaction or database cursor is opened). - -

      Maybe there should be a way to register a mismatch-handler callback. -Otherwise, applications have to handle LSM_MISMATCH everywhere... - - +LSM_COMPRESSION_EMPTY, when a database handle attempts to read or write the +database file, the compression id is compared against the lsm_compress.iId +field of the configured compression hooks, or against LSM_COMPRESSION_NONE if +no compression hooks are configured. If the compression id does not match, then +an LSM_MISMATCH error is returned and the operation fails (no transaction or +database cursor is opened). + +

      It is also possible to register a compression factory callback with a +database handle. If one is registered, the compression factory callback is +invoked instead of returning LSM_MISMATCH if the configured compression hooks +do not match the compression id of a database. If the callback registers +compatible compression hooks with the database handle (using the normal +lsm_config() interface), then the database read or write operation resumes +after it returns. Otherwise, if the compression factory callback does not +register new, compatible, compression hooks with the database handle, +LSM_MISMATCH is returned to the user. + +

      A compression factory callback is registered with a database handle +by calling lsm_config() with the second argument set to +LSM_CONFIG_SET_COMPRESSION_FACTORY, and the third argument set to point to +an instance of structure lsm_compress_factory. The lsm_config() copies the +contents of the structure - it does not retain a pointer to it. + + + typedef struct lsm_compress_factory lsm_compress_factory; + struct lsm_compress_factory { + void *pCtx; + int (*xFactory)(void *pCtx, lsm_db *db, unsigned int iCompressionId); + void (*xFree)(void *pCtx); + }; + + +

      Explain how the xFactory hook works here.

      6. Performance Tuning

      This section describes the various measures that can be taken in order to fine-tune LSM in order to improve performance in specific circumstances. -Sub-section 6.1 identifies the - configuration -parameters that can be used to influence database performance. -Sub-section 6.2 discusses methods for shifting the time-consuming processes of -actually writing and syncing the database file to -background threads or processes -in order to make writing to the database more responsive. Finally, 6. -3 introduces "database optimization" +Sub-section 6.1 contains a high-level overview of the +system architecture +intended to help in understanding the various performance tradeoffs and +optimizations available to LSM applications. Sub-section 6.2 identifies the + configuration parameters +that can be used to influence database performance. +Sub-section 6.3 discusses options and methods for scheduling the time-consuming +processes of actually +writing and syncing the database file +to disk. Finally, 6.4 introduces "database optimization" - the process of reorganizing a database file internally so that it is as small as possible and optimized for search queries. -

      +

      6.1. Overview of LSM Architecture

      -

      The options in this section all take integer values. They may be both +

      The following steps describe the journey taken by data written to the + database from the application to the database file on disk: + +

        +

      1. + When an application writes to an LSM database, the new data is first + written to a log file and stored in an in-memory tree structure. The + log file is used for database recovery - if an application crash or + power failure occurs and the contents of the in-memory tree are lost, + data is recovered by reading the log file. + +

      2. + Once sufficient data has been accumulated in an in-memory tree (by + default "sufficient data" means 1MB, including data structure + overhead), it is marked as "old" and a new "live" in-memory tree + created. An old in-memory tree is immutable - new data is always + inserted into the live tree. There may be at most one old tree + in memory at a time. + +

      3. + The contents of an old in-memory tree may be written into the + database file at any point. Once its contents have been written (or + "flushed") to the database file, the in-memory tree may be discarded. + Flushing an in-memory tree to the database file creates a new database + "segment". A database segment is an immutable b-tree structure stored + within the database file. A single database file may contain up to 64 + segments. + +

      4. + At any point, two or more existing segments within the database file + may be merged together into a single segment. Once their contents has + been merged into the new segment, the original segments may be + discarded. + +

      5. + After the set of segments in a database file has been modified (either + by flushing an in-memory tree to disk or by merging existing segments + together), the changes may be made persistent by "checkpointing" the + database. Checkpointing involves updating the database file header and + and (usually) syncing the contents of the database file to disk. +

      + +

      Steps 3 and 4 above are known as "working" on the database. Step 5 is +refered to as "checkpointing". By default, database connections perform work +and checkpoint operations periodically from within calls to API functions +lsm_insert, lsm_delete, lsm_delete_range +and lsm_commit (i.e. functions that write to the database). +Alternatively, work and checkpoint operations may be performed on demand using +the lsm_work and lsm_checkpoint APIs. By opening a +second database connection, these operations may be moved to a background +thread or process. + +

      Optimizing database write throughput and responsiveness is done by +configuring and scheduling work and checkpoint operations, and by configuring +a few other parameters, as described in the following two sections. + +

      The speed of database read operations is largely determined by the number +of segments in the database file. So optimizing read operations is also linked +to the configuring and scheduling of database write operations, as these +policies determine the number of segments that are present in the database +file at any time. + +

      Any data written to the database file since the last checkpoint may be +lost if a power or application failure occurs. As a result of this, regular +database checkpoints are required to ensure that unused space within the log +file and database file can be reused in a timely fashion. Specifically: + +

        +

      • Space within the log file cannot be recycled until the corresponding + data has been written into a database segment and a checkpoint + performed. + +

      • When two or more existing segments are merged into a new segment + within the database file, the space occupied by the original segments + may not be recycled until after a checkpoint has been performed. +

      + +

      +In other words, without checkpoints the system will function, but both the log and database files will grow indefinitely as the database is modified (even if the size of the dataset remains constant). Additionally, if a crash or power failure occurs, the next client to open the database file has to process all data written to the log file since the most recent checkpoint. If checkpoints are performed infrequently, this can be a time consuming exercise. + +

      If a connection attempts to open a write transaction on the database when +another connection already has an open write transaction, the attempt fails +and LSM_BUSY is returned to the caller. This is because to write to the +database, the connection must first obtain the WRITER lock - and at most +one connection may simultaneously hold the WRITER lock. As well as the WRITER +lock, there are two other exclusive locks that may be obtained on the database +- the WORKER and CHECKPOINTER locks. These are used, not surprisingly, to +ensure that at most one connection attempts to work on or checkpoint the +database at a time. More specifically, the roles of the three locks are: + +

    • +
    WRITER +The WRITER lock is required to modify the contents of the in-memory tree. +Including marking an in-memory tree as "old" and starting a new live tree. +It is also required to write to the log file. + +
    WORKER +The WORKER lock is required to write to segments within the database file. +Either when merging two or more existing segments within the database, or +when flushing an in-memory tree to disk to create a new segment. + +
    CHECKPOINTER +The CHECKPOINTER lock is required to write to the database file header. +
    + +

    The three locks are independent. It is possible to simultaneously have one +client writing to the database, another working on the database file and a +third performing a checkpoint operation. + + +

    + +

    The options in this section are all set to integer values. They may be set and queried using the lsm_config() function. To set an option to a value, lsm_config() is used as follows: - /* Set the LSM_CONFIG_AUTOFLUSH option to 1MB */ - int iVal = 1 * 1024 * 1024; + /* Set the LSM_CONFIG_AUTOFLUSH option to 1MB (1024 KB) */ + int iVal = 1024; rc = lsm_config(db, LSM_CONFIG_AUTOFLUSH, &iVal);

    In order to query the current value of an option, the initial value of the parameter (iVal in the example code above) should be set to a negative @@ -807,364 +955,237 @@ int iVal = -1; rc = lsm_config(db, LSM_CONFIG_AUTOFLUSH, &iVal);

    -
    LSM_CONFIG_MMAP -

    - This option may be set to either 1 (true) or 0 (false). If it is set to - true and LSM is running on a system with a 64-bit address space, the - entire database file is memory mapped. Or, if it is false or LSM is - running in a 32-bit address space, data is accessed using ordinary - OS file read and write primitives. Memory mapping the database file - can significantly improve the performance of read operations, as database - pages do not have to be copied from operating system buffers into user - space buffers before they can be examined. - -

    This option can only be set before lsm_open() is called on the database - connection. - -

    The default value is 1 (true). - -

    LSM_CONFIG_MULTIPLE_PROCESSES -

    - This option may also be set to either 1 (true) or 0 (false). If it is - set to 0, then the library assumes that all database clients are located - within the same process (have access to the same memory space). Assuming - this means the library can avoid using OS file locking primitives to lock - the database file, which speeds up opening and closing read and write - transactions. - -

    This option can only be set before lsm_open() is called on the database - connection. - -

    The default value is 1 (true). - -

    LSM_CONFIG_USE_LOG -

    - This is another option may also be set to either 1 (true) or 0 (false). - If it is set to false, then the library does not write data into the - database log file. This makes writing faster, but also means that if - an application crash or power failure occurs, it is very likely that - any recently committed transactions will be lost. +

    LSM_CONFIG_AUTOCHECKPOINT +

    + This option determines how often the database is checkpointed (synced to + disk). A checkpoint is performed after N KB (approximately) have been + written to the database file, where N is the value of this option. + Increasing this value (say to 4MB or even 8MB) may improve overall write + throughput. + +

    The default value is 2048 (2MB). + +

    LSM_CONFIG_AUTOFLUSH +

    + This option determines how much data in KB is allowed to accumulate in a + live in-memory tree before it is marked as "old" (and made eligible to be + flushed through to the database file). Increasing this value may improve + overall write throughput. Decreasing it reduces memory usage. + +

    The default value is 1024 (1MB). + +

    LSM_CONFIG_AUTOMERGE +

    +

    If auto-work (the LSM_CONFIG_AUTOWORK option below) is enabled, then + this option is set to the number of segments that the library attempts + to merge simultaneously. Increasing this value may reduce the total + amount of data written to the database file. Decreasing it increases + the amount of data written to the file, but also decreases the average + number of segments present in the file, which can improve the performance + of database read operations. + +

    Additionally, whether or not auto-work is enabled, this option is used + to determine the maximum number of segments of a given age that are + allowed to accumulate in the database file. This is described in the + compulsary work and checkpoints + section below. + +

    The default value is 4. This option must be set to a value between + 2 and 8, inclusive. + +

    LSM_CONFIG_AUTOWORK +

    +

    This option may be set to either 1 (true) or 0 (false). If it is set + to true, then work and checkpoint operations are automatically scheduled + within calls to lsm_insert(), lsm_delete(), lsm_delete_range() and + lsm_commit(). Otherwise, if it is set to false, these operations must + be explicitly scheduled by the + application. + +

    The default value is 1. + +

    LSM_CONFIG_MMAP +

    + If LSM is running on a system with a 64-bit address space, this option + may be set to either 1 (true) or 0 (false). On a 32-bit platform, it is + always set to 0. +

    If it is set to true, the entire database file is memory mapped. Or, if + it is false, data is accessed using ordinary OS file read and write + primitives. Memory mapping the database file can significantly improve the + performance of read operations, as database pages do not have to be copied + from operating system buffers into user space buffers before they can be + examined. +

    This option may not be set if there is a read or write transaction + open on the database. +

    The default value is 1 (true) on a 64-bit platform, and 0 otherwise. + +

    LSM_CONFIG_MULTIPLE_PROCESSES +

    + This option may also be set to either 1 (true) or 0 (false). The default + value is 1 (true). If it is set to false, then the library assumes that all + database clients are located within the same process (have access to the + same memory space). Assuming this means the library can avoid using OS file + locking primitives to lock the database file, which speeds up opening and + closing read and write transactions. + +

    This option can only be set before lsm_open() is called on the database + connection. + +

    If this option is set to false and there is already a connection to the + database from another process when lsm_open() is called, the lsm_open() + call fails with error code LSM_BUSY. + +

    LSM_CONFIG_SAFETY +

    + The effect of this option on data durability + is described above. + +

    From a performance point of view, this option determines how often the + library pauses to wait for data written to the file-system to be stored + on the persistent media (e.g. hard disk or solid-state memory). This is + also known as "syncing" data to disk. Since this is orders of magnitude + slower than simply copying data into operating system buffers, the value + of this option has a large effect on write performance. + +

    If LSM_CONFIG_SAFETY is set to 2 (FULL), then the library syncs the + data written to the log file to disk whenever a transaction is committed. + Or, if LSM_CONFIG_SAFETY is set to 1 (NORMAL), then data is only synced + to disk when a checkpoint is performed (see above). Finally, if it is set + to 0 (OFF), then no data is ever synced to disk. + +

    The default value is 1 (NORMAL). + +

    LSM_CONFIG_USE_LOG +

    + This is another option that may be set to either 1 (true) or 0 (false). + The default value is 1 (true). If it is set to false, then the library + does not write data into the database log file. This makes writing faster, + but also means that if an application crash or power failure occurs, it is + very likely that any recently committed transactions will be lost.

    If this option is set to true, then an application crash cannot cause data loss. Whether or not data loss may occur in the event of a power failure depends on the value of the LSM_CONFIG_SAFETY parameter.

    This option can only be set if the connection does not currently have an open write transaction. -

    The default value is 1 (true). - -

    LSM_CONFIG_AUTOFLUSH -

    - -

    LSM_CONFIG_AUTOCHECKPOINT -

    -

    -

    6.2. Using Worker Threads or Processes

    - -

    Todo: Fix the following

    - -

    The section above describes the three stages of transfering data written -to the database from the application to persistent storage. A "writer" -client writes the data into the in-memory tree and log file. Later on a -"worker" client flushes the data from the in-memory tree to a new segment -in the the database file. Additionally, a worker client must periodically -merge existing database segments together to prevent them from growing too -numerous. - -

    6.2.1. Architectural Overview

    - -

    The LSM library implements two separate data structures that are used -together to store user data. When the database is queried, the library -actually runs parallel queries on both of these data stores and merges the -results together to return to the user. The data structures are: - -

      -
    • The in-memory tree. The in-memory tree is an append-only b-tree - variant designed to be stored entirely in main-memory (i.e. not - written out to disk). The library strives to limit the total size of - the in-memory tree (by default to 1MB in total). - -

      At any one time, there may actually be two in-memory tree structures - present in memory. One immutable tree marked as "old" waiting to be - written into the database file (see below) and one "live" tree to - which new data may be appended. - -

    • The log-structured-merge tree structure for which LSM is - named. This data structure is used to store data within the database - file on disk. - -

      The log-structured-merge tree is made up of a series of "segments". - Each segment is an immutable tree structure stored (more or less) - contiguously within the database file. When the database is queried, the - library runs parallel queries on each of the segments in the database - and merges the results to return to the user. - -

      The only way to insert new data to the database is to add a new - segment. In order to prevent the number of segments from growing too - large, two or more existing segments may be merged together into a - single larger segment at any point. Deleting existing key-value pairs - is accomplished by inserting "delete-keys" into the new segment. - -

      The log-structured-merge tree structure is described in more detail - link to lsm.wiki section here. -

    - -

    When a database client writes a transaction to the database, the new -data is inserted into the "live" in-memory tree structure. At the same time, -the new data is also appended to the log file on disk. The purpose of the log -file is to provide a persistent backup of any data that the system does not -consider to have been safely stored (see below) in the database file. If a -crash or power failure occurs, this data is recovered by reading the log -file. - -

    Once sufficient data has accumulated within the "live" in-memory tree, -it is marked as "old" and a new live tree created. At any point thereafter, -the contents of the old in-memory tree may be used to populate a new segment -within the database file and then discarded. When this happens, the old -in-memory tree is said to have been "flushed to disk". If there is already an -old in-memory tree when the live tree is deemed to have accumulated enough data -to itself become an old tree, the existing old tree must first be flushed to -disk. - -

    The set of segments that make up the log-structured-merge tree at any time -and the order in which they should be queried is termed a "snapshot". The -header of the database file contains a snapshot. A snapshot is also stored in -main memory. When set of segments that make up the log-structured-merge tree -is modified, either by flushing an old in-memory tree to disk or by merging -two or more existing segments, the in-memory snapshot is updated immediately. -This is the snapshot that database clients use when querying or otherwise -operating on the database. - -

    At any point after the in-memory snapshot has been updated, the in-memory -snapshot may be written into the database file header. This is known as -"checkpointing" the database. Depending on the value of the -LSM_CONFIG_SAFETY parameter, it may -be necessary to ensure that all segments referenced by the snapshot have been -synced to disk (safely stored on the persistent media such that they will not -be lost if a power failure occurs) before doing so. It is not necessary for -every version of the in-memory snapshot to be checkpointed. The in-memory -snapshot may be modified multiple times between checkpoints. - -

    -Because a checkpointer process is often required to sync the database file -before updating the database header, "checkpointing" often appears to be the -costliest part of transfering data to the database file, at least in terms of -wall-clock time. - -

    Regular database checkpoints are required to ensure that unused space -within the log file and database file can be reused in a timely fashion. -Specifically: - -

      -

    • Space within the log file cannot be recycled until the corresponding - data has been written into a database segment and a checkpoint - performed. - -

    • When two or more existing segments are merged within the database - file, database clients may start using the new, larger, segment - immediately. However the space occupied by the original segments may - not be reused until after a snapshot that refers to the new segment, and - not the old ones, has been checkpointed. -

    - -

    In other words, without checkpoints the system will function, but both the -log and database files will grow indefinitely as the database is modified -(even if the size of the dataset remains constant). Additionally, if a crash -or power failure occurs, the next client to open the database file has to -process all data written to the log file since the most recent checkpoint. If -checkpoints are performed infrequently, this can be a time consuming exercise. - -

    In order to safely write data into the in-memory tree (by calling -lsm_insert, lsm_delete or lsm_delete_range), the database client must hold -the database WRITER lock. At most one client may concurrently hold the WRITER -lock. Holding the WRITER lock is synonymous with having an open write -transaction - the client obtains the WRITER lock when the transaction is -opened and relinquishes it when the transaction is closed. - -

    As well as the WRITER lock, there are two other locks that may be held by -at most one client at any time - the WORKER and CHECKPOINTER locks. The roles -of the three locks are roughly as follows: - - -
    WRITER -The WRITER lock is required to modify the contents of the in-memory tree. -Including marking an in-memory tree as "old" and starting a new live tree. -It is also required to write to the log file. - -
    WORKER -The WORKER lock is required to write to segments within the database file. -Either when merging two or more existing segments within the database, or -when flushing an in-memory tree to disk to create a new segment. -The WORKER lock is also required to update the database snapshot stored in -main memory (updated so that new clients will use the new segments the worker -creates). - -
    CHECKPOINTER -The CHECKPOINTER lock is required to update the snapshot stored in the -database file header (to checkpoint the database). -
    - -

    The tasks associated with each of the locks above may be performed -concurrently by multiple database connections, located either in the same -application process or different processes. - -

    6.2.2. Automatic Work and Checkpoint Scheduling

    - -

    By default, database "work" (the flushing and merging of segments, performed -by clients holding the WORKER lock) and checkpointing are scheduled and -performed automatically from within calls to "write" API functions. The -"write" functions are: +

    6.3. Work and Checkpoint Scheduling

    + +

    6.3.1. Automatic Scheduling

    + +

    This section describes how work and checkpoint operations are scheduled +if the boolean LSM_CONFIG_AUTOWORK parameter is set to true. Automatic +work operations may occur within calls to any of the following API functions:

    • lsm_insert()
    • lsm_delete()
    • lsm_delete_range()
    • lsm_commit()
    -

    It is expected that automatic work and checkpoint scheduling will be -suitable for most applications. The advantage of this model is that it is -simple to use. However, any call to one of the functions listed above may be -co-opted by the system to perform database work or a checkpoint, causing it to -return more slowly than it otherwise would. In some situations, for example -when a writer thread is also responsible for handling user-interface events, -this may be undesirable. - -

    Automatic work and checkpoint scheduling is controlled by four integer -parameters set or queried using the lsm_config() interface. The parameters -are: - -

    -
    LSM_CONFIG_AUTOWORK -

    - This is a boolean parameter (default 1). If set, auto-work mode is - enabled for the database connection. Otherwise, it is not. - -

    LSM_CONFIG_AUTOFLUSH -

    - An integer parameter (default 1048576). If this parameter is set - to a non-zero value, then after each transaction is committed, the - library checks the total size of the live in-memory tree. If it is - equal to or greater than the configured value of this parameter in - bytes and there is no existing old in-memory tree, then the current - live tree is marked as old. - -

    Additionally, if the LSM_CONFIG_AUTOWORK parameter is set, the - contents of the in-memory tree are immediately flushed to disk. - -

    LSM_CONFIG_AUTOMERGE -

    - This parameter must be set to an integer value between 2 and 8, - inclusive. It controls the number of existing segments that - auto-work attempts to merge together at a time. The default value is 4. - -

    LSM_CONFIG_AUTOCHECKPOINT -

    - If this parameter is set to an integer value greater than 0, then - a checkpoint is automatically attempted after this many bytes are - written into the database file. The default value is 2097152. -

    - -

    Each segment in the database file is assigned an "age" - an integer zero -or greater indicating how many times the data in the segment has been merged. -A segment created by flushing the in-memory tree to disk is assigned an age -of 1. When two or more segments with age=1 are merged together to create a -larger segment, it is assigned an age of 2. And so on. - -

    If auto-work is enabled, the library periodically checks the state of the -database file to see if there exist N or more segments with the same age -value A, where N is the value assigned to the LSM_CONFIG_AUTOMERGE parameter. -If so, work is done to merge all such segments with age=A into a new, larger -segment assigned age=A+1. At present, "periodically" as used above means -roughly once for every 32KB of data (including overhead) written to the -in-memory tree. The merge operation is not necessarily completed within -a single call to a write API (this would result in blocking the writer thread -for too long in many cases - in large databases segments may grow to be many GB -in size). Currently, the amount of data written by a single auto-work -operation is roughly 32KB multiplied by the number of segments in the database -file. This formula may change - the point is that the library attempts to limit -the amount of data written in order to avoid blocking the writer thread for too -long within a single API call. -

    Each time a transaction is committed in auto-work mode, the library checks to see if there exists an "old" in-memory tree (see the LSM_CONFIG_AUTOFLUSH option above). If so, it attempts to flush it to disk immediately. Unlike -merges of existing segments, the entire in-memory tree must be flushed to -disk before control is returned to the user. It is not possible to -incrementally flush an in-memory tree in the same ways as it is possible to -incrementally merge existing database segments together. - -

    In order to perform auto-work on the database file, either to flush the -contents of an old in-memory tree to disk or to merge existing segments within -the database file together, the client must obtain the WORKER lock. If some -other client is already holding the WORKER lock, this will not be possible. -This is not an error. If this occurs, the writer thread simply returns -immediately, without performing any work on the database file. - -

    Assuming the LSM_CONFIG_AUTOCHECKPOINT parameter is set to a value greater -than zero, after performing database work, the library automatically checks -how many bytes of raw data have been written to the database file since the -last checkpoint (by any client, not just by the current client). If this -value is greater than the value of the LSM_CONFIG_AUTOCHECKPOINT parameter, -a checkpoint is attempted. It is not an error if the attempt fails because the -CHECKPOINTER lock cannot be obtained. - -

    6.2.3. Explicit Work and Checkpoint Scheduling

    +merges of existing segments, the entire in-memory tree must be flushed to disk +before control is returned to the user. It is not possible to incrementally +flush an in-memory tree in the same ways as it is possible to incrementally +merge existing database segments together. + +

    Each segment in the database file is assigned an "age" - an integer zero +or greater indicating how many times the data in the segment has been merged. +A segment created by flushing the in-memory tree to disk is assigned an age +of 1. When two or more segments with age=1 are merged together to create a +larger segment, it is assigned an age of 2. And so on. + +

    Assuming auto-work is enabled, the library periodically checks the state of +the database file to see if there exist N or more segments with the same age +value A, where N is the value assigned to the LSM_CONFIG_AUTOMERGE parameter. +If so, work is done to merge all such segments with age=A into a new, larger +segment assigned age=A+1. At present, "periodically" as used above means +roughly once for every 32KB of data (including overhead) written to the +in-memory tree. The merge operation is not necessarily completed within a +single call to a write API (this would result in blocking the writer thread for +too long in many cases - in large databases segments may grow to be many GB in +size). Currently, the amount of data written by a single auto-work operation is +roughly 32KB multiplied by the number of segments in the database file. This +formula may change - the point is that the library attempts to limit the amount +of data written in order to avoid blocking the writer thread for too long +within a single API call. + +

    Checkpoint operations are scheduled based on the value assigned to the +LSM_CONFIG_AUTOCHECKPOINT configuration parameter. + +

    In order to automatically perform work and checkpoint operations, the +client must obtain the WORKER and CHECKPOINTER locks, respectively. If an +attempt to obtain either of these locks fails (because some other client is +already holding them), it is not an error, the scheduled work or checkpoint +is simply not performed. + + + +

    6.3.2. Explicit Scheduling

    The alternative to automatic scheduling of work and checkpoint operations -is to explicitly schedule them. Possibly in a background thread or dedicated +is to explicitly schedule them - possibly in a background thread or dedicated application process. In order to disable automatic work, a client must set the LSM_CONFIG_AUTOWORK parameter to zero. This parameter is a property of a database connection, not of a database itself, so it must be cleared separately by all processes that may write to the database. Otherwise, they may attempt automatic database work or checkpoints. + + /* Disable auto-work on connection db */ + int iVal = 0; + lsm_config(db, LSM_CONFIG_AUTOWORK, &iVal); + +

    The lsm_work() function is used to explicitly perform work on the database: - int lsm_work(lsm_db *db, int nMerge, int nByte, int *pnWrite); + int lsm_work(lsm_db *db, int nMerge, int nKB, int *pnWrite); -

    Parameter nByte is passed a limit on the number of bytes of data that +

    Parameter nKB is passed a limit on the number of KB of data that should be written to the database file before the call returns. It is a hint only, the library does not honor this limit strictly.

    If the database has an old in-memory tree when lsm_work() is called, it is -flushed to disk. If this means that more than nByte bytes of data is written +flushed to disk. If this means that more than nKB KB of data is written to the database file, no further work is performed. Otherwise, the number -of bytes written is subtracted from nByte before proceeding. +of KB written is subtracted from nKB before proceeding.

    If parameter nMerge is greater than 1, then the library searches for nMerge or more segments of the same age within the database file and performs -up to nByte bytes of work to merge them together. If the merge is completed -before the nByte limit is exceeded, the library searches for another set of +up to nKB KB of work to merge them together. If the merge is completed +before the nKB limit is exceeded, the library searches for another set of nMerge or more segments to work on, and so on. If at any point no such set of nMerge segments can be found, the call returns without performing any further work.

    Calling lsm_work() with the nMerge argument set to 1 is used to "optimize" the database (see below). Passing a value of zero or less for the nMerge parameter is an error.

    In any case, before returning the value of *pnWrite is set to the actual -number of bytes written to the database file. +number of KB written to the database file.

    The example code below might be executed in a background thread or process in order to perform database work and checkpointing. In this case all other clients should set the LSM_CONFIG_AUTOWORK parameter to zero. int rc; lsm_db *db; - int nCkpt = 4*1024*1024; + int nCkpt = 4*1024; /* 4096KB == 4MB */ /* Open a database connection to database "test.db". ** ** Configure the connection to automatically checkpoint the database after ** writing each 4MB of data to it (instead of the default 2MB). As well @@ -1178,11 +1199,11 @@ while( 1 ){ int nWrite; /* Attempt up to 512KB of work. Set nWrite to the number of bytes ** actually written to disk. */ - rc = lsm_work(db, 2, 512*1024, &nWrite); + rc = lsm_work(db, 2, 512, &nWrite); if( rc!=LSM_OK && rc!=LSM_BUSY ){ /* Anything other than LSM_OK or LSM_BUSY is a problem. LSM_BUSY ** indicates that some other client has taken the WORKER lock. Any ** other error indicates something has gone quite wrong. */ lsm_close(db); @@ -1203,127 +1224,128 @@ */ if( nWrite==0 ) sleep(1); } -

    Checkpoints can also be requested explicitly, using the lsm_checkpoint() -API: +

    The mechanism associated with the LSM_CONFIG_AUTOCHECKPOINT configuration +parameter applies to data written by both automatically scheduled work and +work performed by calls to the lsm_work() function. The amount of +uncheckpointed data that has been written into the database file is a property +of the database file, not a single connection, so checkpoints occur at the +configured interval even if multiple connections are used to work on the +database. + +

    Alternatively, checkpoint operations may be scheduled separately. If the +LSM_CONFIG_AUTOCHECKPOINT parameter is set to zero, then a connection never +performs a database checkpoint, regardless of how much data it or any other +connection writes into the database file. As with LSM_CONFIG_AUTOWORK, this +parameter must be zeroed for all connections that may perform work on the +database. Otherwise, they may perform a checkpoint operation. + +

    The lsm_checkpoint() API is used +to expicitly request a checkpoint. - int lsm_checkpoint(lsm_db *db, int *pnCkpt); + int lsm_checkpoint(lsm_db *db, int *pnKB); -

    If no work has been performed on the database since the most recent -checkpoint (implying that the snapshot has not changed and there is no need -to write it into the database file), lsm_checkpoint() sets *pnCkpt to zero -and returns immediately. Otherwise, it checkpoints the database and sets -*pnCkpt to the number of bytes written to the database file since the -previous checkpoint. - -

    The number of bytes written to the database since the most recent checkpoint -can also be using the lsm_info() API function. As follows: +

    If no work has been performed on the database since the previous +checkpoint, lsm_checkpoint() sets *pnKB to zero and returns immediately. +Otherwise, it checkpoints the database and sets *pnKB to the number of KB of +data written to the database file since the previous checkpoint. + +

    A database may be queried for the number of KB written to the database +since the most recent checkpoint using the +lsm_info() API function. As follows: int nCkpt; rc = lsm_info(db, LSM_INFO_CHECKPOINT_SIZE, &nCkpt); +

    It may also be queried for the size of the in-memory tree or trees. The +following block of code sets variable nLive to the size of the current +live in-memory tree in KB, and nOld to the size of the old in-memory tree in +KB (or 0 if there is no old in-memory tree). + int nOld, nLive; rc = lsm_info(db, LSM_INFO_TREE_SIZE, &nOld, &nLive); - - int lsm_flush(lsm_db *db); - - -

    6.2.4. Compulsary Work and Checkpoint Scheduling

    - -

    Apart from the scenarios described above, there are two there are two -scenarios where database work or checkpointing may be performed automatically, -regardless of the value of the LSM_CONFIG_AUTOWORK parameter. +

    6.3.3. Compulsary Work and Checkpoints

    + +

    There are three scenarios where database work or checkpointing may be +performed automatically, regardless of the value of the LSM_CONFIG_AUTOWORK +parameter.

    • When closing a database connection, and -
    • When the number of segments with a common age in the database file grows +
    • When the number of segments with a common age in the database + file grows unacceptably high. +
    • When the total number of segments in the database file grows unacceptably high.

    Whenever an lsm_close() call would mean that the total number of connections to a database drops to zero, the connection checks if the in-memory tree is empty. If not, it is flushed to disk. Both the live and old in-memory trees are flushed to disk in this case. It also checks if the database file has been modified since the most recent checkpoint was -performed. If so, it also performs a checkpoint. And, assuming no error -has occurred, deletes the log file. +performed. If so, it also performs a checkpoint. Finally, assuming no error +has occurred, it deletes the log file.

    Additionally, whenever a worker wishes to flush an in-memory tree to a new -age=1 segment, it must first ensure that there are less than N existing age=1 -segments, where N is the value that the LSM_CONFIG_AUTOMERGE parameter is -set to. If there are already N or more age=1 segments, they must be merged +age=1 segment, it must first ensure that there are less than (N+1) existing +age=1 segments, where N is the value that the LSM_CONFIG_AUTOMERGE parameter is +set to. If there are already (N+1) or more age=1 segments, they must be merged into an age=2 segment before a new age=1 segment can be created within the database file. Similar rules apply to segments of other ages - it is not -possible to create a new age=I segment if there are already N segments with -age=I in the database file. This has two implications: - -

      -
    • The database is prevented from accumulating too many segments, - regardless of whether or not auto-work is enabled or how infrequently - lsm_work() is called, and - -
    • If auto-work is disabled and lsm_work() is not called frequently enough, - it is possible that flushing an in-memory tree may required writing a - tremendous amount of data to disk (possibly even rewriting the entire - database file). -
    +possible to create a new age=I segment if there are already (N+1) segments +with age=I in the database file. This has two implications: + +

    This scenario should never come about if all connections that write to the +database have auto-work enabled. It only occurs if auto-work is disabled and +the lsm_work() function is called too infrequently. In this case it is possible +that flushing an in-memory tree may require writing a tremendous amount of +data to disk (possibly even rewriting the entire database file).

    Finally, regardless of age, a database is limited to a maximum of 64 segments in total. If an attempt is made to flush an in-memory tree to disk when the database already contains 64 segments, two or more existing segments must be merged together before the new segment can be created. -

    6.3. Database File Optimization

    +

    6.4. Database File Optimization

    Database optimization transforms the contents of database file so that the following are true:

    • All database content is stored in a single segment. This makes the - database effectively equivalent to an optimally packed b-tree stucture + data structure equivalent to an optimally packed b-tree stucture for search operations - minimizing the number of disk sectors that need to be visted when searching the database.

    • The database file contains no (or as little as possible) free space. In other words, it is no larger than required to contain the single segment.

    -

    Should we add a convenience function lsm_optimize() that does not -return until the database is completely optimized? One that more or less does -the same as the example code below and deals with the AUTOCHECKPOINT issue? -This would help with this user manual if nothing else, as it means a method -for database optimization can be presented without depending on the previous -section. - - - -

    In order to optimize the database, lsm_work() should be called repeatedly -with the nMerge argument set to 1 until it returns without writing any data -to the database file. For example: +

    In order to optimize the database, lsm_work() should be called with +the nMerge argument set to 1 and the third parameter set to a negative value +(interpreted as "keep working until there is no more work to do"). For +example: - int nWrite; - int rc; - do { - rc = lsm_work(db, 1, 2*1024*1024, &nWrite); - }while( rc==LSM_OK && nWrite>0 ); + rc = lsm_work(db, 1, -1, 0);

    When optimizing the database as above, either the LSM_CONFIG_AUTOCHECKPOINT parameter should be set to a non-zero value or lsm_checkpoint() should be called periodically. Otherwise, no checkpoints will be performed, preventing the library from reusing any space occupied by old segments even after their content has been merged into the new segment. The result - a database file that is optimized, except that it is up to twice as large as it otherwise would be. + Index: www/varint.wiki ================================================================== --- www/varint.wiki +++ www/varint.wiki @@ -1,19 +1,19 @@ Variable-Length Integers A variable length integer is an encoding of 64-bit unsigned integers into between 1 and 9 bytes. The encoding has the following properties: - 1. Smaller (and common) values use fewer bytes and take up less space + 1. Smaller (and more common) values use fewer bytes and take up less space than larger (and less common) values. 2. The length of any varint can be determined by looking at just the first byte of the encoding. 3. Lexicographical and numeric ordering for varints are the same. Hence if a group of varints are order lexicographically (that is to say, if - they are order by memcmp() with shorted varints coming first) then + they are order by memcmp() with shorter varints coming first) then those varints will also be in numeric order. This property means that varints can be used as keys in the key/value backend storage and the records will occur in numerical order of the keys. The encoding is described by algorithms to decode (convert from