Index: ext/fts5/fts5_vocab.c ================================================================== --- ext/fts5/fts5_vocab.c +++ ext/fts5/fts5_vocab.c @@ -27,10 +27,15 @@ ** ** One row for each term in the database. The value of $doc is set to ** the number of fts5 rows that contain at least one instance of term ** $term. Field $cnt is set to the total number of instances of term ** $term in the database. +** +** instance: +** CREATE TABLE vocab(term, doc, col, offset, PRIMARY KEY()); +** +** One row for each term instance in the database. */ #include "fts5Int.h" @@ -42,11 +47,11 @@ sqlite3_vtab base; char *zFts5Tbl; /* Name of fts5 table */ char *zFts5Db; /* Db containing fts5 table */ sqlite3 *db; /* Database handle */ Fts5Global *pGlobal; /* FTS5 global object for this database */ - int eType; /* FTS5_VOCAB_COL or ROW */ + int eType; /* FTS5_VOCAB_COL, ROW or INSTANCE */ }; struct Fts5VocabCursor { sqlite3_vtab_cursor base; sqlite3_stmt *pStmt; /* Statement holding lock on pIndex */ @@ -62,20 +67,26 @@ Fts5Config *pConfig; /* Fts5 table configuration */ int iCol; i64 *aCnt; i64 *aDoc; - /* Output values used by 'row' and 'col' tables */ + /* Output values used by all tables. */ i64 rowid; /* This table's current rowid value */ Fts5Buffer term; /* Current value of 'term' column */ + + /* Output values Used by 'instance' tables only */ + i64 iInstPos; + int iInstOff; }; -#define FTS5_VOCAB_COL 0 -#define FTS5_VOCAB_ROW 1 +#define FTS5_VOCAB_COL 0 +#define FTS5_VOCAB_ROW 1 +#define FTS5_VOCAB_INSTANCE 2 #define FTS5_VOCAB_COL_SCHEMA "term, col, doc, cnt" #define FTS5_VOCAB_ROW_SCHEMA "term, doc, cnt" +#define FTS5_VOCAB_INST_SCHEMA "term, doc, col, offset" /* ** Bits for the mask used as the idxNum value by xBestIndex/xFilter. */ #define FTS5_VOCAB_TERM_EQ 0x01 @@ -98,10 +109,13 @@ *peType = FTS5_VOCAB_COL; }else if( sqlite3_stricmp(zCopy, "row")==0 ){ *peType = FTS5_VOCAB_ROW; + }else + if( sqlite3_stricmp(zCopy, "instance")==0 ){ + *peType = FTS5_VOCAB_INSTANCE; }else { *pzErr = sqlite3_mprintf("fts5vocab: unknown table type: %Q", zCopy); rc = SQLITE_ERROR; } @@ -159,11 +173,12 @@ sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ char **pzErr /* Write any error message here */ ){ const char *azSchema[] = { "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA ")", - "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA ")" + "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA ")", + "CREATE TABlE vocab(" FTS5_VOCAB_INST_SCHEMA ")" }; Fts5VocabTable *pRet = 0; int rc = SQLITE_OK; /* Return code */ int bDb; @@ -233,10 +248,19 @@ return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr); } /* ** Implementation of the xBestIndex method. +** +** Only constraints of the form: +** +** term <= ? +** term == ? +** term >= ? +** +** are interpreted. Less-than and less-than-or-equal are treated +** identically, as are greater-than and greater-than-or-equal. */ static int fts5VocabBestIndexMethod( sqlite3_vtab *pUnused, sqlite3_index_info *pInfo ){ @@ -376,10 +400,58 @@ sqlite3_finalize(pCsr->pStmt); sqlite3_free(pCsr); return SQLITE_OK; } +static int fts5VocabInstanceNewTerm(Fts5VocabCursor *pCsr){ + int rc = SQLITE_OK; + + if( sqlite3Fts5IterEof(pCsr->pIter) ){ + pCsr->bEof = 1; + }else{ + const char *zTerm; + int nTerm; + zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); + if( pCsr->nLeTerm>=0 ){ + int nCmp = MIN(nTerm, pCsr->nLeTerm); + int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp); + if( bCmp<0 || (bCmp==0 && pCsr->nLeTermbEof = 1; + } + } + + sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm); + } + return rc; +} + +static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){ + int eDetail = pCsr->pConfig->eDetail; + int rc = SQLITE_OK; + Fts5IndexIter *pIter = pCsr->pIter; + i64 *pp = &pCsr->iInstPos; + int *po = &pCsr->iInstOff; + + while( eDetail==FTS5_DETAIL_NONE + || sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp) + ){ + pCsr->iInstPos = 0; + pCsr->iInstOff = 0; + + rc = sqlite3Fts5IterNextScan(pCsr->pIter); + if( rc==SQLITE_OK ){ + rc = fts5VocabInstanceNewTerm(pCsr); + if( eDetail==FTS5_DETAIL_NONE ) break; + } + if( rc ){ + pCsr->bEof = 1; + break; + } + } + + return rc; +} /* ** Advance the cursor to the next row in the table. */ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ @@ -387,18 +459,22 @@ Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab; int rc = SQLITE_OK; int nCol = pCsr->pConfig->nCol; pCsr->rowid++; + + if( pTab->eType==FTS5_VOCAB_INSTANCE ){ + return fts5VocabInstanceNext(pCsr); + } if( pTab->eType==FTS5_VOCAB_COL ){ for(pCsr->iCol++; pCsr->iColiCol++){ if( pCsr->aDoc[pCsr->iCol] ) break; } } - if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=nCol ){ + if( pTab->eType!=FTS5_VOCAB_COL || pCsr->iCol>=nCol ){ if( sqlite3Fts5IterEof(pCsr->pIter) ){ pCsr->bEof = 1; }else{ const char *zTerm; int nTerm; @@ -418,26 +494,30 @@ memset(pCsr->aDoc, 0, nCol * sizeof(i64)); pCsr->iCol = 0; assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW ); while( rc==SQLITE_OK ){ + int eDetail = pCsr->pConfig->eDetail; const u8 *pPos; int nPos; /* Position list */ i64 iPos = 0; /* 64-bit position read from poslist */ int iOff = 0; /* Current offset within position list */ pPos = pCsr->pIter->pData; nPos = pCsr->pIter->nData; - switch( pCsr->pConfig->eDetail ){ - case FTS5_DETAIL_FULL: - pPos = pCsr->pIter->pData; - nPos = pCsr->pIter->nData; - if( pTab->eType==FTS5_VOCAB_ROW ){ + + switch( pTab->eType ){ + case FTS5_VOCAB_ROW: + if( eDetail==FTS5_DETAIL_FULL ){ while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ pCsr->aCnt[0]++; } - pCsr->aDoc[0]++; - }else{ + } + pCsr->aDoc[0]++; + break; + + case FTS5_VOCAB_COL: + if( eDetail==FTS5_DETAIL_FULL ){ int iCol = -1; while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ int ii = FTS5_POS2COLUMN(iPos); pCsr->aCnt[ii]++; if( iCol!=ii ){ @@ -447,37 +527,34 @@ } pCsr->aDoc[ii]++; iCol = ii; } } - } - break; - - case FTS5_DETAIL_COLUMNS: - if( pTab->eType==FTS5_VOCAB_ROW ){ - pCsr->aDoc[0]++; - }else{ + }else if( eDetail==FTS5_DETAIL_COLUMNS ){ while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){ assert_nc( iPos>=0 && iPos=nCol ){ rc = FTS5_CORRUPT; break; } pCsr->aDoc[iPos]++; } + }else{ + assert( eDetail==FTS5_DETAIL_NONE ); + pCsr->aDoc[0]++; } break; - default: - assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE ); - pCsr->aDoc[0]++; + default: + assert( pTab->eType==FTS5_VOCAB_INSTANCE ); break; } if( rc==SQLITE_OK ){ rc = sqlite3Fts5IterNextScan(pCsr->pIter); } + if( pTab->eType==FTS5_VOCAB_INSTANCE ) break; if( rc==SQLITE_OK ){ zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ){ break; @@ -503,11 +580,13 @@ int idxNum, /* Strategy index */ const char *zUnused, /* Unused */ int nUnused, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ + Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab; Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + int eType = pTab->eType; int rc = SQLITE_OK; int iVal = 0; int f = FTS5INDEX_QUERY_SCAN; const char *zTerm = 0; @@ -543,15 +622,20 @@ memcpy(pCsr->zLeTerm, zCopy, pCsr->nLeTerm+1); } } } - if( rc==SQLITE_OK ){ rc = sqlite3Fts5IndexQuery(pCsr->pIndex, zTerm, nTerm, f, 0, &pCsr->pIter); } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && eType==FTS5_VOCAB_INSTANCE ){ + rc = fts5VocabInstanceNewTerm(pCsr); + } + if( rc==SQLITE_OK + && !pCsr->bEof + && (eType!=FTS5_VOCAB_INSTANCE || pCsr->pConfig->eDetail!=FTS5_DETAIL_NONE) + ){ rc = fts5VocabNextMethod(pCursor); } return rc; } @@ -589,16 +673,45 @@ }else if( iCol==2 ){ iVal = pCsr->aDoc[pCsr->iCol]; }else{ iVal = pCsr->aCnt[pCsr->iCol]; } - }else{ + }else if( eType==FTS5_VOCAB_ROW ){ assert( iCol==1 || iCol==2 ); if( iCol==1 ){ iVal = pCsr->aDoc[0]; }else{ iVal = pCsr->aCnt[0]; + } + }else{ + int eDetail = pCsr->pConfig->eDetail; + assert( eType==FTS5_VOCAB_INSTANCE ); + switch( iCol ){ + case 1: + sqlite3_result_int64(pCtx, pCsr->pIter->iRowid); + break; + case 2: { + int ii = -1; + if( eDetail==FTS5_DETAIL_FULL ){ + ii = FTS5_POS2COLUMN(pCsr->iInstPos); + }else if( eDetail==FTS5_DETAIL_COLUMNS ){ + ii = pCsr->iInstPos; + } + if( ii>=0 && iipConfig->nCol ){ + const char *z = pCsr->pConfig->azCol[ii]; + sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); + } + break; + } + default: { + assert( iCol==3 ); + if( eDetail==FTS5_DETAIL_FULL ){ + int ii = FTS5_POS2OFFSET(pCsr->iInstPos); + sqlite3_result_int(pCtx, ii); + } + break; + } } } if( iVal>0 ) sqlite3_result_int64(pCtx, iVal); return SQLITE_OK; ADDED ext/fts5/test/fts5vocab2.test Index: ext/fts5/test/fts5vocab2.test ================================================================== --- ext/fts5/test/fts5vocab2.test +++ ext/fts5/test/fts5vocab2.test @@ -0,0 +1,209 @@ +# 2017 August 10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The tests in this file focus on testing the fts5vocab module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5vocab + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b); + CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance); + + INSERT INTO t1 VALUES('one two', 'two three'); + INSERT INTO t1 VALUES('three four', 'four five five five'); +} + +do_execsql_test 1.1 { + SELECT * FROM v1; +} { + five 2 b 1 + five 2 b 2 + five 2 b 3 + four 2 a 1 + four 2 b 0 + one 1 a 0 + three 1 b 1 + three 2 a 0 + two 1 a 1 + two 1 b 0 +} + +do_execsql_test 1.2 { + SELECT * FROM v1 WHERE term='three'; +} { + three 1 b 1 + three 2 a 0 +} + +do_execsql_test 1.3 { + BEGIN; + DELETE FROM t1 WHERE rowid=2; + SELECT * FROM v1; + ROLLBACK; +} { + one 1 a 0 + three 1 b 1 + two 1 a 1 + two 1 b 0 +} + +do_execsql_test 1.4 { + BEGIN; + DELETE FROM t1 WHERE rowid=1; + SELECT * FROM v1; + ROLLBACK; +} { + five 2 b 1 + five 2 b 2 + five 2 b 3 + four 2 a 1 + four 2 b 0 + three 2 a 0 +} + +do_execsql_test 1.5 { + DELETE FROM t1; + SELECT * FROM v1; +} { +} + +#------------------------------------------------------------------------- +# +do_execsql_test 2.0 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS v1; + + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=column); + CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance); + + INSERT INTO t1 VALUES('one two', 'two three'); + INSERT INTO t1 VALUES('three four', 'four five five five'); +} + +do_execsql_test 2.1 { + SELECT * FROM v1; +} { + five 2 b {} + four 2 a {} + four 2 b {} + one 1 a {} + three 1 b {} + three 2 a {} + two 1 a {} + two 1 b {} +} + +do_execsql_test 2.2 { + SELECT * FROM v1 WHERE term='three'; +} { + three 1 b {} + three 2 a {} +} + +do_execsql_test 2.3 { + BEGIN; + DELETE FROM t1 WHERE rowid=2; + SELECT * FROM v1; + ROLLBACK; +} { + one 1 a {} + three 1 b {} + two 1 a {} + two 1 b {} +} + +do_execsql_test 2.4 { + BEGIN; + DELETE FROM t1 WHERE rowid=1; + SELECT * FROM v1; + ROLLBACK; +} { + five 2 b {} + four 2 a {} + four 2 b {} + three 2 a {} +} + +do_execsql_test 2.5 { + DELETE FROM t1; + SELECT * FROM v1; +} { +} + +#------------------------------------------------------------------------- +# +do_execsql_test 3.0 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS v1; + + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=none); + CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance); + + INSERT INTO t1 VALUES('one two', 'two three'); + INSERT INTO t1 VALUES('three four', 'four five five five'); +} + +do_execsql_test 3.1 { + SELECT * FROM v1; +} { + five 2 {} {} + four 2 {} {} + one 1 {} {} + three 1 {} {} + three 2 {} {} + two 1 {} {} +} + +do_execsql_test 3.2 { + SELECT * FROM v1 WHERE term='three'; +} { + three 1 {} {} + three 2 {} {} +} + +do_execsql_test 3.3 { + BEGIN; + DELETE FROM t1 WHERE rowid=2; + SELECT * FROM v1; + ROLLBACK; +} { + one 1 {} {} + three 1 {} {} + two 1 {} {} +} + +do_execsql_test 3.4 { + BEGIN; + DELETE FROM t1 WHERE rowid=1; + SELECT * FROM v1; + ROLLBACK; +} { + five 2 {} {} + four 2 {} {} + three 2 {} {} +} + +do_execsql_test 3.5 { + DELETE FROM t1; + SELECT * FROM v1; +} { +} + +finish_test + Index: ext/lsm1/lsm_vtab.c ================================================================== --- ext/lsm1/lsm_vtab.c +++ ext/lsm1/lsm_vtab.c @@ -8,12 +8,86 @@ ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* ** -** This file implements a simple virtual table wrapper around the LSM +** This file implements a virtual table for SQLite3 around the LSM ** storage engine from SQLite4. +** +** USAGE +** +** CREATE VIRTUAL TABLE demo USING lsm1(filename,key,keytype,value1,...); +** +** The filename parameter is the name of the LSM database file, which is +** separate and distinct from the SQLite3 database file. +** +** The keytype must be one of: UINT, TEXT, BLOB. All keys must be of that +** one type. "UINT" means unsigned integer. The values may be of any +** SQLite datatype: BLOB, TEXT, INTEGER, FLOAT, or NULL. +** +** The virtual table contains read-only hidden columns: +** +** lsm1_key A BLOB which is the raw LSM key. If the "keytype" +** is BLOB or TEXT then this column is exactly the +** same as the key. For the UINT keytype, this column +** will be a variable-length integer encoding of the key. +** +** lsm1_value A BLOB which is the raw LSM value. All of the value +** columns are packed into this BLOB using the encoding +** described below. +** +** Attempts to write values into the lsm1_key and lsm1_value columns are +** silently ignored. +** +** EXAMPLE +** +** The virtual table declared this way: +** +** CREATE VIRTUAL TABLE demo2 USING lsm1('x.lsm',id,UINT,a,b,c,d); +** +** Results in a new virtual table named "demo2" that acts as if it has +** the following schema: +** +** CREATE TABLE demo2( +** id UINT PRIMARY KEY ON CONFLICT REPLACE, +** a ANY, +** b ANY, +** c ANY, +** d ANY, +** lsm1_key BLOB HIDDEN, +** lsm1_value BLOB HIDDEN +** ) WITHOUT ROWID; +** +** +** +** INTERNALS +** +** The key encoding for BLOB and TEXT is just a copy of the blob or text. +** UTF-8 is used for text. The key encoding for UINT is the variable-length +** integer format at https://sqlite.org/src4/doc/trunk/www/varint.wiki. +** +** The values are encoded as a single blob (since that is what lsm stores as +** its content). There is a "type integer" followed by "content" for each +** value, alternating back and forth. The content might be empty. +** +** TYPE1 CONTENT1 TYPE2 CONTENT2 TYPE3 CONTENT3 .... +** +** Each "type integer" is encoded as a variable-length integer in the +** format of the link above. Let the type integer be T. The actual +** datatype is an integer 0-5 equal to T%6. Values 1 through 5 correspond +** to SQLITE_INTEGER through SQLITE_NULL. The size of the content in bytes +** is T/6. Type value 0 means that the value is an integer whose actual +** values is T/6 and there is no content. The type-value-0 integer format +** only works for integers in the range of 0 through 40. +** +** There is no content for NULL or type-0 integers. For BLOB and TEXT +** values, the content is the blob data or the UTF-8 text data. For +** non-negative integers X, the content is a variable-length integer X*2. +** For negative integers Y, the content is varaible-length integer (1-Y)*2+1. +** For FLOAT values, the content is the IEEE754 floating point value in +** native byte-order. This means that FLOAT values will be corrupted when +** database file is moved between big-endian and little-endian machines. */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 #include "lsm.h" #include @@ -20,18 +94,23 @@ #include /* Forward declaration of subclasses of virtual table objects */ typedef struct lsm1_vtab lsm1_vtab; typedef struct lsm1_cursor lsm1_cursor; +typedef struct lsm1_vblob lsm1_vblob; /* Primitive types */ typedef unsigned char u8; +typedef unsigned int u32; +typedef sqlite3_uint64 u64; /* An open connection to an LSM table */ struct lsm1_vtab { sqlite3_vtab base; /* Base class - must be first */ lsm_db *pDb; /* Open connection to the LSM table */ + u8 keyType; /* SQLITE_BLOB, _TEXT, or _INTEGER */ + u32 nVal; /* Number of value columns */ }; /* lsm1_cursor is a subclass of sqlite3_vtab_cursor which will ** serve as the underlying representation of a cursor that scans @@ -41,11 +120,85 @@ sqlite3_vtab_cursor base; /* Base class - must be first */ lsm_cursor *pLsmCur; /* The LSM cursor */ u8 isDesc; /* 0: scan forward. 1: scan reverse */ u8 atEof; /* True if the scan is complete */ u8 bUnique; /* True if no more than one row of output */ + u8 *zData; /* Content of the current row */ + u32 nData; /* Number of bytes in the current row */ + u8 *aeType; /* Types for all column values */ + u32 *aiOfst; /* Offsets to the various fields */ + u32 *aiLen; /* Length of each field */ + u8 *pKey2; /* Loop termination key, or NULL */ + u32 nKey2; /* Length of the loop termination key */ +}; + +/* An extensible buffer object. +** +** Content can be appended. Space to hold new content is automatically +** allocated. +*/ +struct lsm1_vblob { + u8 *a; /* Space to hold content, from sqlite3_malloc64() */ + u64 n; /* Bytes of space used */ + u64 nAlloc; /* Bytes of space allocated */ + u8 errNoMem; /* True if a memory allocation error has been seen */ }; + +#if defined(__GNUC__) +# define LSM1_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) && _MSC_VER>=1310 +# define LSM1_NOINLINE __declspec(noinline) +#else +# define LSM1_NOINLINE +#endif + + +/* Increase the available space in the vblob object so that it can hold +** at least N more bytes. Return the number of errors. +*/ +static int lsm1VblobEnlarge(lsm1_vblob *p, u32 N){ + if( p->n+N>p->nAlloc ){ + if( p->errNoMem ) return 1; + p->nAlloc += N + (p->nAlloc ? p->nAlloc : N); + p->a = sqlite3_realloc64(p->a, p->nAlloc); + if( p->a==0 ){ + p->n = 0; + p->nAlloc = 0; + p->errNoMem = 1; + return 1; + } + p->nAlloc = sqlite3_msize(p->a); + } + return 0; +} + +/* Append N bytes to a vblob after first enlarging it */ +static LSM1_NOINLINE void lsm1VblobEnlargeAndAppend( + lsm1_vblob *p, + const u8 *pData, + u32 N +){ + if( p->n+N>p->nAlloc && lsm1VblobEnlarge(p, N) ) return; + memcpy(p->a+p->n, pData, N); + p->n += N; +} + +/* Append N bytes to a vblob */ +static void lsm1VblobAppend(lsm1_vblob *p, const u8 *pData, u32 N){ + sqlite3_int64 n = p->n; + if( n+N>p->nAlloc ){ + lsm1VblobEnlargeAndAppend(p, pData, N); + }else{ + p->n += N; + memcpy(p->a+n, pData, N); + } +} + +/* append text to a vblob */ +static void lsm1VblobAppendText(lsm1_vblob *p, const char *z){ + lsm1VblobAppend(p, (u8*)z, (u32)strlen(z)); +} /* Dequote the string */ static void lsm1Dequote(char *z){ int j; char cQuote = z[0]; @@ -74,21 +227,41 @@ char **pzErr ){ lsm1_vtab *pNew; int rc; char *zFilename; + u8 keyType = 0; + int i; + lsm1_vblob sql; + static const char *azTypes[] = { "UINT", "TEXT", "BLOB" }; + static const u8 aeTypes[] = { SQLITE_INTEGER, SQLITE_TEXT, SQLITE_BLOB }; + static const char *azArgName[] = {"filename", "key", "key type", "value1" }; - if( argc!=4 || argv[3]==0 || argv[3][0]==0 ){ - *pzErr = sqlite3_mprintf("filename argument missing"); + for(i=0; ikeyType = keyType; rc = lsm_new(0, &pNew->pDb); if( rc ){ *pzErr = sqlite3_mprintf("lsm_new failed with error code %d", rc); rc = SQLITE_ERROR; goto connect_failed; @@ -101,26 +274,33 @@ *pzErr = sqlite3_mprintf("lsm_open failed with %d", rc); rc = SQLITE_ERROR; goto connect_failed; } -/* Column numbers */ -#define LSM1_COLUMN_KEY 0 -#define LSM1_COLUMN_BLOBKEY 1 -#define LSM1_COLUMN_VALUE 2 -#define LSM1_COLUMN_BLOBVALUE 3 -#define LSM1_COLUMN_COMMAND 4 + memset(&sql, 0, sizeof(sql)); + lsm1VblobAppendText(&sql, "CREATE TABLE x("); + lsm1VblobAppendText(&sql, argv[4]); + lsm1VblobAppendText(&sql, " "); + lsm1VblobAppendText(&sql, argv[5]); + lsm1VblobAppendText(&sql, " PRIMARY KEY"); + for(i=6; inVal++; + } + lsm1VblobAppendText(&sql, + ", lsm1_command HIDDEN" + ", lsm1_key HIDDEN" + ", lsm1_value HIDDEN) WITHOUT ROWID"); + lsm1VblobAppend(&sql, (u8*)"", 1); + if( sql.errNoMem ){ + rc = SQLITE_NOMEM; + goto connect_failed; + } + rc = sqlite3_declare_vtab(db, (const char*)sql.a); + sqlite3_free(sql.a); - rc = sqlite3_declare_vtab(db, - "CREATE TABLE x(" - " key," /* The primary key. Any non-NULL */ - " blobkey," /* Pure BLOB primary key */ - " value," /* The value associated with key. Any non-NULL */ - " blobvalue," /* Pure BLOB value */ - " command hidden" /* Insert here for control operations */ - ");" - ); connect_failed: if( rc!=SQLITE_OK ){ if( pNew ){ if( pNew->pDb ) lsm_close(pNew->pDb); sqlite3_free(pNew); @@ -145,13 +325,17 @@ */ static int lsm1Open(sqlite3_vtab *pVtab, sqlite3_vtab_cursor **ppCursor){ lsm1_vtab *p = (lsm1_vtab*)pVtab; lsm1_cursor *pCur; int rc; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) + + p->nVal*(sizeof(pCur->aiOfst)+sizeof(pCur->aiLen)+1) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); + pCur->aiOfst = (u32*)&pCur[1]; + pCur->aiLen = &pCur->aiOfst[p->nVal]; + pCur->aeType = (u8*)&pCur->aiLen[p->nVal]; *ppCursor = &pCur->base; rc = lsm_csr_open(p->pDb, &pCur->pLsmCur); if( rc==LSM_OK ){ rc = SQLITE_OK; }else{ @@ -165,10 +349,11 @@ /* ** Destructor for a lsm1_cursor. */ static int lsm1Close(sqlite3_vtab_cursor *cur){ lsm1_cursor *pCur = (lsm1_cursor*)cur; + sqlite3_free(pCur->pKey2); lsm_csr_close(pCur->pLsmCur); sqlite3_free(pCur); return SQLITE_OK; } @@ -188,10 +373,25 @@ rc = lsm_csr_next(pCur->pLsmCur); } if( rc==LSM_OK && lsm_csr_valid(pCur->pLsmCur)==0 ){ pCur->atEof = 1; } + if( pCur->pKey2 && pCur->atEof==0 ){ + const u8 *pVal; + u32 nVal; + assert( pCur->isDesc==0 ); + rc = lsm_csr_key(pCur->pLsmCur, (const void**)&pVal, (int*)&nVal); + if( rc==LSM_OK ){ + u32 len = pCur->nKey2; + int c; + if( len>nVal ) len = nVal; + c = memcmp(pVal, pCur->pKey2, len); + if( c==0 ) c = nVal - pCur->nKey2; + if( c>0 ) pCur->atEof = 1; + } + } + pCur->zData = 0; } return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; } /* @@ -292,10 +492,18 @@ z[0] = 255; varintWrite32(z+1, w); varintWrite32(z+5, y); return 9; } + +/* Append non-negative integer x as a variable-length integer. +*/ +static void lsm1VblobAppendVarint(lsm1_vblob *p, sqlite3_uint64 x){ + sqlite3_int64 n = p->n; + if( n+9>p->nAlloc && lsm1VblobEnlarge(p, 9) ) return; + p->n += lsm1PutVarint64(p->a+p->n, x); +} /* ** Decode the varint in the first n bytes z[]. Write the integer value ** into *pResult and return the number of bytes in the varint. ** @@ -347,73 +555,78 @@ *pResult = (((sqlite3_uint64)x)<<32) + (0xffffffff & ((z[5]<<24) + (z[6]<<16) + (z[7]<<8) + z[8])); return 9; } -/* -** Generate a key encoding for pValue such that all keys compare in -** lexicographical order. Return an SQLite error code or SQLITE_OK. +/* Encoded a signed integer as a varint. Numbers close to zero uses fewer +** bytes than numbers far away from zero. However, the result is not in +** lexicographical order. ** -** The key encoding is *pnKey bytes in length written into *ppKey. -** Space to hold the key is taken from pSpace if sufficient, or else -** from sqlite3_malloc(). The caller is responsible for freeing malloced -** space. +** Encoding: Non-negative integer X is encoding as an unsigned +** varint X*2. Negative integer Y is encoding as an unsigned +** varint (1-Y)*2 + 1. */ -static int lsm1EncodeKey( - sqlite3_value *pValue, /* Value to be encoded */ - unsigned char **ppKey, /* Write the encoding here */ - int *pnKey, /* Write the size of the encoding here */ - unsigned char *pSpace, /* Use this space if it is large enough */ - int nSpace /* Size of pSpace[] */ +static int lsm1PutSignedVarint64(u8 *z, sqlite3_int64 v){ + sqlite3_uint64 u; + if( v>=0 ){ + u = (sqlite3_uint64)v; + return lsm1PutVarint64(z, u*2); + }else{ + u = (sqlite3_uint64)(-1-v); + return lsm1PutVarint64(z, u*2+1); + } +} + +/* Decoded a signed varint. */ +static int lsm1GetSignedVarint64( + const unsigned char *z, + int n, + sqlite3_int64 *pResult ){ - int eType = sqlite3_value_type(pValue); - *ppKey = 0; - *pnKey = 0; - assert( nSpace>=32 ); - switch( eType ){ - default: { - return SQLITE_ERROR; /* We cannot handle NULL keys */ - } - case SQLITE_BLOB: - case SQLITE_TEXT: { - int nVal = sqlite3_value_bytes(pValue); - const void *pVal; - if( eType==SQLITE_BLOB ){ - eType = LSM1_TYPE_BLOB; - pVal = sqlite3_value_blob(pValue); - }else{ - eType = LSM1_TYPE_TEXT; - pVal = (const void*)sqlite3_value_text(pValue); - if( pVal==0 ) return SQLITE_NOMEM; - } - if( nVal+1>nSpace ){ - pSpace = sqlite3_malloc( nVal+1 ); - if( pSpace==0 ) return SQLITE_NOMEM; - } - pSpace[0] = (unsigned char)eType; - memcpy(&pSpace[1], pVal, nVal); - *ppKey = pSpace; - *pnKey = nVal+1; - break; + sqlite3_uint64 u = 0; + n = lsm1GetVarint64(z, n, &u); + if( u&1 ){ + *pResult = -1 - (sqlite3_int64)(u>>1); + }else{ + *pResult = (sqlite3_int64)(u>>1); + } + return n; +} + + +/* +** Read the value part of the key-value pair and decode it into columns. +*/ +static int lsm1DecodeValues(lsm1_cursor *pCur){ + lsm1_vtab *pTab = (lsm1_vtab*)(pCur->base.pVtab); + int i, n; + int rc; + u8 eType; + sqlite3_uint64 v; + + if( pCur->zData ) return 1; + rc = lsm_csr_value(pCur->pLsmCur, (const void**)&pCur->zData, + (int*)&pCur->nData); + if( rc ) return 0; + for(i=n=0; inVal; i++){ + v = 0; + n += lsm1GetVarint64(pCur->zData+n, pCur->nData-n, &v); + pCur->aeType[i] = eType = (u8)(v%6); + if( eType==0 ){ + pCur->aiOfst[i] = (u32)(v/6); + pCur->aiLen[i] = 0; + }else{ + pCur->aiOfst[i] = n; + n += (pCur->aiLen[i] = (u32)(v/6)); } - case SQLITE_INTEGER: { - sqlite3_int64 iVal = sqlite3_value_int64(pValue); - sqlite3_uint64 uVal; - if( iVal<0 ){ - if( iVal==0xffffffffffffffffLL ) return SQLITE_ERROR; - uVal = *(sqlite3_uint64*)&iVal; - eType = LSM1_TYPE_NEGATIVE; - }else{ - uVal = iVal; - eType = LSM1_TYPE_POSITIVE; - } - pSpace[0] = (unsigned char)eType; - *ppKey = pSpace; - *pnKey = 1 + lsm1PutVarint64(&pSpace[1], uVal); - } + if( n>pCur->nData ) break; } - return SQLITE_OK; + if( inVal ){ + pCur->zData = 0; + return 0; + } + return 1; } /* ** Return values of columns for the row at which the lsm1_cursor ** is currently pointing. @@ -422,115 +635,184 @@ sqlite3_vtab_cursor *cur, /* The cursor */ sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ int i /* Which column to return */ ){ lsm1_cursor *pCur = (lsm1_cursor*)cur; - switch( i ){ - case LSM1_COLUMN_BLOBKEY: { + lsm1_vtab *pTab = (lsm1_vtab*)(cur->pVtab); + if( i==0 ){ + /* The key column */ + const void *pVal; + int nVal; + if( lsm_csr_key(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ + if( pTab->keyType==SQLITE_BLOB ){ + sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); + }else if( pTab->keyType==SQLITE_TEXT ){ + sqlite3_result_text(ctx,(const char*)pVal, nVal, SQLITE_TRANSIENT); + }else{ + const unsigned char *z = (const unsigned char*)pVal; + sqlite3_uint64 v1; + lsm1GetVarint64(z, nVal, &v1); + sqlite3_result_int64(ctx, (sqlite3_int64)v1); + } + } + }else if( i>pTab->nVal ){ + if( i==pTab->nVal+2 ){ /* lsm1_key */ const void *pVal; int nVal; if( lsm_csr_key(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); } - break; - } - case LSM1_COLUMN_KEY: { - const unsigned char *pVal; - int nVal; - if( lsm_csr_key(pCur->pLsmCur, (const void**)&pVal, &nVal)==LSM_OK - && nVal>=1 - ){ - if( pVal[0]==LSM1_TYPE_BLOB ){ - sqlite3_result_blob(ctx, (const void*)&pVal[1],nVal-1, - SQLITE_TRANSIENT); - }else if( pVal[0]==LSM1_TYPE_TEXT ){ - sqlite3_result_text(ctx, (const char*)&pVal[1],nVal-1, - SQLITE_TRANSIENT); - }else if( nVal>=2 && nVal<=10 && - (pVal[0]==LSM1_TYPE_POSITIVE || pVal[0]==LSM1_TYPE_NEGATIVE) - ){ - sqlite3_int64 iVal; - lsm1GetVarint64(pVal+1, nVal-1, (sqlite3_uint64*)&iVal); - sqlite3_result_int64(ctx, iVal); - } - } - break; - } - case LSM1_COLUMN_BLOBVALUE: { + }else if( i==pTab->nVal+3 ){ /* lsm1_value */ const void *pVal; int nVal; - if( lsm_csr_value(pCur->pLsmCur, (const void**)&pVal, &nVal)==LSM_OK ){ + if( lsm_csr_value(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); } - break; } - case LSM1_COLUMN_VALUE: { - const unsigned char *aVal; - int nVal; - if( lsm_csr_value(pCur->pLsmCur, (const void**)&aVal, &nVal)==LSM_OK - && nVal>=1 - ){ - switch( aVal[0] ){ - case SQLITE_FLOAT: - case SQLITE_INTEGER: { - sqlite3_uint64 x = 0; - int j; - for(j=1; jzData + pCur->aiOfst[i]; + nData = pCur->aiLen[i]; + switch( pCur->aeType[i] ){ + case 0: { /* in-line integer */ + sqlite3_result_int(ctx, pCur->aiOfst[i]); + break; + } + case SQLITE_INTEGER: { + sqlite3_int64 v; + lsm1GetSignedVarint64(zData, nData, &v); + sqlite3_result_int64(ctx, v); + break; + } + case SQLITE_FLOAT: { + double v; + if( nData==sizeof(v) ){ + memcpy(&v, zData, sizeof(v)); + sqlite3_result_double(ctx, v); } + break; + } + case SQLITE_TEXT: { + sqlite3_result_text(ctx, (const char*)zData, nData, SQLITE_TRANSIENT); + break; + } + case SQLITE_BLOB: { + sqlite3_result_blob(ctx, zData, nData, SQLITE_TRANSIENT); + break; } - break; - } - default: { - break; + default: { + /* A NULL. Do nothing */ + } } } return SQLITE_OK; } + +/* Parameter "pValue" contains an SQL value that is to be used as +** a key in an LSM table. The type of the key is determined by +** "keyType". Extract the raw bytes used for the key in LSM1. +*/ +static void lsm1KeyFromValue( + int keyType, /* The key type */ + sqlite3_value *pValue, /* The key value */ + u8 *pBuf, /* Storage space for a generated key */ + const u8 **ppKey, /* OUT: the bytes of the key */ + int *pnKey /* OUT: size of the key */ +){ + if( keyType==SQLITE_BLOB ){ + *ppKey = (const u8*)sqlite3_value_blob(pValue); + *pnKey = sqlite3_value_bytes(pValue); + }else if( keyType==SQLITE_TEXT ){ + *ppKey = (const u8*)sqlite3_value_text(pValue); + *pnKey = sqlite3_value_bytes(pValue); + }else{ + sqlite3_int64 v = sqlite3_value_int64(pValue); + if( v<0 ) v = 0; + *pnKey = lsm1PutVarint64(pBuf, v); + *ppKey = pBuf; + } +} /* Move to the first row to return. */ static int lsm1Filter( sqlite3_vtab_cursor *pVtabCursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv ){ lsm1_cursor *pCur = (lsm1_cursor *)pVtabCursor; + lsm1_vtab *pTab = (lsm1_vtab*)(pCur->base.pVtab); int rc = LSM_OK; + int seekType = -1; + const u8 *pVal = 0; + int nVal; + u8 keyType = pTab->keyType; + u8 aKey1[16]; + pCur->atEof = 1; - if( idxNum==1 ){ - assert( argc==1 ); - pCur->isDesc = 0; - pCur->bUnique = 1; - if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ - const void *pVal = sqlite3_value_blob(argv[0]); - int nVal = sqlite3_value_bytes(argv[0]); - rc = lsm_csr_seek(pCur->pLsmCur, pVal, nVal, LSM_SEEK_EQ); + sqlite3_free(pCur->pKey2); + pCur->pKey2 = 0; + if( idxNum<99 ){ + lsm1KeyFromValue(keyType, argv[0], aKey1, &pVal, &nVal); + } + switch( idxNum ){ + case 0: { /* key==argv[0] */ + assert( argc==1 ); + seekType = LSM_SEEK_EQ; + pCur->isDesc = 0; + pCur->bUnique = 1; + break; + } + case 1: { /* key>=argv[0] AND key<=argv[1] */ + u8 aKey[12]; + seekType = LSM_SEEK_GE; + pCur->isDesc = 0; + pCur->bUnique = 0; + if( keyType==SQLITE_INTEGER ){ + sqlite3_int64 v = sqlite3_value_int64(argv[1]); + if( v<0 ) v = 0; + pCur->nKey2 = lsm1PutVarint64(aKey, (sqlite3_uint64)v); + pCur->pKey2 = sqlite3_malloc( pCur->nKey2 ); + if( pCur->pKey2==0 ) return SQLITE_NOMEM; + memcpy(pCur->pKey2, aKey, pCur->nKey2); + }else{ + pCur->nKey2 = sqlite3_value_bytes(argv[1]); + pCur->pKey2 = sqlite3_malloc( pCur->nKey2 ); + if( pCur->pKey2==0 ) return SQLITE_NOMEM; + if( keyType==SQLITE_BLOB ){ + memcpy(pCur->pKey2, sqlite3_value_blob(argv[1]), pCur->nKey2); + }else{ + memcpy(pCur->pKey2, sqlite3_value_text(argv[1]), pCur->nKey2); + } + } + break; + } + case 2: { /* key>=argv[0] */ + seekType = LSM_SEEK_GE; + pCur->isDesc = 0; + pCur->bUnique = 0; + break; + } + case 3: { /* key<=argv[0] */ + seekType = LSM_SEEK_LE; + pCur->isDesc = 1; + pCur->bUnique = 0; + break; + } + default: { /* full table scan */ + pCur->isDesc = 0; + pCur->bUnique = 0; + break; } + } + if( pVal ){ + rc = lsm_csr_seek(pCur->pLsmCur, pVal, nVal, seekType); }else{ rc = lsm_csr_first(pCur->pLsmCur); - pCur->isDesc = 0; - pCur->bUnique = 0; } if( rc==LSM_OK && lsm_csr_valid(pCur->pLsmCur)!=0 ){ pCur->atEof = 0; } return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; @@ -538,63 +820,91 @@ /* ** Only comparisons against the key are allowed. The idxNum defines ** which comparisons are available: ** -** 0 Full table scan only -** bit 1 key==?1 single argument for ?1 -** bit 2 key>?1 -** bit 3 key>=?1 -** bit 4 key?1 -** 4 key>=?1 -** 8 key?1 AND key=?1 AND key?1 AND key<=?2 -** 20 key>=?1 AND key<=?2 -** 33..52 Use blobkey in place of key... +** 0 key==?1 +** 1 key>=?1 AND key<=?2 +** 2 key>?1 or key>=?1 +** 3 keyaConstraint; for(i=0; inConstraint && idxNum<16; i++, pConstraint++){ if( pConstraint->usable==0 ) continue; - if( pConstraint->iColumn!=LSM1_COLUMN_KEY ) continue; - if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pConstraint->iColumn!=0 ) continue; switch( pConstraint->op ){ case SQLITE_INDEX_CONSTRAINT_EQ: { - eqIdx = i; - idxNum = 1; + if( idxNum>0 ){ + argIdx = i; + iIdx2 = -1; + idxNum = 0; + omit1 = 1; + } + break; + } + case SQLITE_INDEX_CONSTRAINT_GE: + case SQLITE_INDEX_CONSTRAINT_GT: { + if( idxNum==99 ){ + argIdx = i; + idxNum = 2; + omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_GE; + }else if( idxNum==3 ){ + iIdx2 = idxNum; + omit2 = omit1; + argIdx = i; + idxNum = 1; + omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_GE; + } + break; + } + case SQLITE_INDEX_CONSTRAINT_LE: + case SQLITE_INDEX_CONSTRAINT_LT: { + if( idxNum==99 ){ + argIdx = i; + idxNum = 3; + omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE; + }else if( idxNum==2 ){ + iIdx2 = i; + idxNum = 1; + omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE; + } break; } } } - if( eqIdx>=0 ){ - pIdxInfo->aConstraintUsage[eqIdx].argvIndex = ++nArg; - pIdxInfo->aConstraintUsage[eqIdx].omit = 1; + if( argIdx>=0 ){ + pIdxInfo->aConstraintUsage[argIdx].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[argIdx].omit = omit1; + } + if( iIdx2>=0 ){ + pIdxInfo->aConstraintUsage[iIdx2].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[iIdx2].omit = omit2; } - if( idxNum==1 ){ + if( idxNum==0 ){ pIdxInfo->estimatedCost = (double)1; pIdxInfo->estimatedRows = 1; pIdxInfo->orderByConsumed = 1; + }else if( idxNum==1 ){ + pIdxInfo->estimatedCost = (double)100; + pIdxInfo->estimatedRows = 100; + }else if( idxNum<99 ){ + pIdxInfo->estimatedCost = (double)5000; + pIdxInfo->estimatedRows = 5000; }else{ /* Full table scan */ pIdxInfo->estimatedCost = (double)2147483647; pIdxInfo->estimatedRows = 2147483647; } @@ -613,102 +923,88 @@ int argc, sqlite3_value **argv, sqlite_int64 *pRowid ){ lsm1_vtab *p = (lsm1_vtab*)pVTab; - const void *pKey; - void *pFree = 0; - int nKey; - int eType; + int nKey, nKey2; + int i; int rc = LSM_OK; - sqlite3_value *pValue; - const unsigned char *pVal; - unsigned char *pData; - int nVal; - unsigned char pSpace[100]; + const u8 *pKey, *pKey2; + unsigned char aKey[16]; + unsigned char pSpace[16]; + lsm1_vblob val; if( argc==1 ){ - pVTab->zErrMsg = sqlite3_mprintf("cannot DELETE"); - return SQLITE_ERROR; + /* DELETE the record whose key is argv[0] */ + lsm1KeyFromValue(p->keyType, argv[0], aKey, &pKey, &nKey); + lsm_delete(p->pDb, pKey, nKey); + return SQLITE_OK; } + if( sqlite3_value_type(argv[0])!=SQLITE_NULL ){ - pVTab->zErrMsg = sqlite3_mprintf("cannot UPDATE"); - return SQLITE_ERROR; + /* An UPDATE */ + lsm1KeyFromValue(p->keyType, argv[0], aKey, &pKey, &nKey); + lsm1KeyFromValue(p->keyType, argv[1], pSpace, &pKey2, &nKey2); + if( nKey!=nKey2 || memcmp(pKey, pKey2, nKey)!=0 ){ + /* The UPDATE changes the PRIMARY KEY value. DELETE the old key */ + lsm_delete(p->pDb, pKey, nKey); + } + /* Fall through into the INSERT case to complete the UPDATE */ } - /* "INSERT INTO tab(command) VALUES('....')" is used to implement + /* "INSERT INTO tab(lsm1_command) VALUES('....')" is used to implement ** special commands. */ - if( sqlite3_value_type(argv[2+LSM1_COLUMN_COMMAND])!=SQLITE_NULL ){ + if( sqlite3_value_type(argv[3+p->nVal])!=SQLITE_NULL ){ return SQLITE_OK; } - if( sqlite3_value_type(argv[2+LSM1_COLUMN_BLOBKEY])==SQLITE_BLOB ){ - /* Use the blob key exactly as supplied */ - pKey = sqlite3_value_blob(argv[2+LSM1_COLUMN_BLOBKEY]); - nKey = sqlite3_value_bytes(argv[2+LSM1_COLUMN_BLOBKEY]); - }else{ - /* Use a key encoding that sorts in lexicographical order */ - rc = lsm1EncodeKey(argv[2+LSM1_COLUMN_KEY], - (unsigned char**)&pKey,&nKey, - pSpace,sizeof(pSpace)); - if( rc ) return rc; - if( pKey!=(const void*)pSpace ) pFree = (void*)pKey; - } - if( sqlite3_value_type(argv[2+LSM1_COLUMN_BLOBVALUE])==SQLITE_BLOB ){ - pVal = sqlite3_value_blob(argv[2+LSM1_COLUMN_BLOBVALUE]); - nVal = sqlite3_value_bytes(argv[2+LSM1_COLUMN_BLOBVALUE]); - rc = lsm_insert(p->pDb, pKey, nKey, pVal, nVal); - }else{ - pValue = argv[2+LSM1_COLUMN_VALUE]; - eType = sqlite3_value_type(pValue); + lsm1KeyFromValue(p->keyType, argv[2], aKey, &pKey, &nKey); + memset(&val, 0, sizeof(val)); + for(i=0; inVal; i++){ + sqlite3_value *pArg = argv[3+i]; + u8 eType = sqlite3_value_type(pArg); switch( eType ){ case SQLITE_NULL: { - rc = lsm_delete(p->pDb, pKey, nKey); + lsm1VblobAppendVarint(&val, SQLITE_NULL); break; } - case SQLITE_BLOB: - case SQLITE_TEXT: { - if( eType==SQLITE_TEXT ){ - pVal = sqlite3_value_text(pValue); + case SQLITE_INTEGER: { + sqlite3_int64 v = sqlite3_value_int64(pArg); + if( v>=0 && v<=240/6 ){ + lsm1VblobAppendVarint(&val, v*6); }else{ - pVal = (unsigned char*)sqlite3_value_blob(pValue); - } - nVal = sqlite3_value_bytes(pValue); - pData = sqlite3_malloc( nVal+1 ); - if( pData==0 ){ - rc = SQLITE_NOMEM; - }else{ - pData[0] = (unsigned char)eType; - memcpy(&pData[1], pVal, nVal); - rc = lsm_insert(p->pDb, pKey, nKey, pData, nVal+1); - sqlite3_free(pData); + int n = lsm1PutSignedVarint64(pSpace, v); + lsm1VblobAppendVarint(&val, SQLITE_INTEGER + n*6); + lsm1VblobAppend(&val, pSpace, n); } break; } - case SQLITE_INTEGER: case SQLITE_FLOAT: { - sqlite3_uint64 x; - unsigned char aVal[9]; - int i; - if( eType==SQLITE_INTEGER ){ - *(sqlite3_int64*)&x = sqlite3_value_int64(pValue); - }else{ - double r = sqlite3_value_double(pValue); - assert( sizeof(r)==sizeof(x) ); - memcpy(&x, &r, sizeof(r)); - } - for(i=8; x>0 && i>=1; i--){ - aVal[i] = x & 0xff; - x >>= 8; - } - aVal[i] = (unsigned char)eType; - rc = lsm_insert(p->pDb, pKey, nKey, &aVal[i], 9-i); + double r = sqlite3_value_double(pArg); + lsm1VblobAppendVarint(&val, SQLITE_FLOAT + 8*6); + lsm1VblobAppend(&val, (u8*)&r, sizeof(r)); + break; + } + case SQLITE_BLOB: { + int n = sqlite3_value_bytes(pArg); + lsm1VblobAppendVarint(&val, n*6 + SQLITE_BLOB); + lsm1VblobAppend(&val, sqlite3_value_blob(pArg), n); + break; + } + case SQLITE_TEXT: { + int n = sqlite3_value_bytes(pArg); + lsm1VblobAppendVarint(&val, n*6 + SQLITE_TEXT); + lsm1VblobAppend(&val, sqlite3_value_text(pArg), n); break; } } } - sqlite3_free(pFree); + if( val.errNoMem ){ + return SQLITE_NOMEM; + } + rc = lsm_insert(p->pDb, pKey, nKey, val.a, val.n); + sqlite3_free(val.a); return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; } /* Begin a transaction */ Index: ext/lsm1/test/lsm1_simple.test ================================================================== --- ext/lsm1/test/lsm1_simple.test +++ ext/lsm1/test/lsm1_simple.test @@ -17,40 +17,77 @@ return_if_no_lsm1 load_lsm1_vtab db forcedelete testlsm.db -do_execsql_test 1.0 { - CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db); +do_execsql_test 100 { + CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,UINT,b,c,d); PRAGMA table_info(x1); } { - 0 key {} 0 {} 0 - 1 blobkey {} 0 {} 0 - 2 value {} 0 {} 0 - 3 blobvalue {} 0 {} 0 + 0 a UINT 1 {} 1 + 1 b {} 0 {} 0 + 2 c {} 0 {} 0 + 3 d {} 0 {} 0 } -do_execsql_test 1.1 { - INSERT INTO x1(blobkey, blobvalue) VALUES(x'abcd', x'1234'); - SELECT quote(blobkey), quote(blobvalue) FROM x1; -} {X'ABCD' X'1234'} +do_execsql_test 110 { + INSERT INTO x1(a,b,c,d) VALUES(15, 11, 22, 33),(8,'banjo',x'333231',NULL), + (12,NULL,3.25,-559281390); + SELECT a, quote(b), quote(c), quote(d) FROM x1; +} {8 'banjo' X'333231' NULL 12 NULL 3.25 -559281390 15 11 22 33} +do_execsql_test 111 { + SELECT a, quote(lsm1_key), quote(lsm1_value) FROM x1; +} {8 X'08' X'2162616E6A6F1633323105' 12 X'0C' X'05320000000000000A401FFB42ABE9DB' 15 X'0F' X'4284C6'} + +do_execsql_test 120 { + UPDATE x1 SET d = d+1.0 WHERE a=15; + SELECT a, quote(b), quote(c), quote(d) FROM x1; +} {8 'banjo' X'333231' NULL 12 NULL 3.25 -559281390 15 11 22 34.0} + +do_execsql_test 130 { + UPDATE x1 SET a=123456789 WHERE a=12; + SELECT a, quote(b), quote(c), quote(d) FROM x1; +} {8 'banjo' X'333231' NULL 15 11 22 34.0 123456789 NULL 3.25 -559281390} +do_execsql_test 131 { + SELECT quote(lsm1_key), printf('0x%x',a) FROM x1 WHERE a > 100000000; +} {X'FB075BCD15' 0x75bcd15} -do_catchsql_test 1.2 { - UPDATE x1 SET blobvalue = x'7890' WHERE blobkey = x'abcd'; -} {1 {cannot UPDATE}} +do_execsql_test 140 { + DELETE FROM x1 WHERE a=15; + SELECT a, quote(b), quote(c), quote(d) FROM x1; +} {8 'banjo' X'333231' NULL 123456789 NULL 3.25 -559281390} -do_catchsql_test 1.3 { - DELETE FROM x1 WHERE blobkey = x'abcd' -} {1 {cannot DELETE}} - -do_test 1.4 { +do_test 150 { lsort [glob testlsm.db*] } {testlsm.db testlsm.db-log testlsm.db-shm} db close -do_test 1.5 { +do_test 160 { lsort [glob testlsm.db*] } {testlsm.db} -finish_test +forcedelete testlsm.db +forcedelete test.db +sqlite3 db test.db +load_lsm1_vtab db + + +do_execsql_test 200 { + CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,TEXT,b,c,d); + PRAGMA table_info(x1); +} { + 0 a TEXT 1 {} 1 + 1 b {} 0 {} 0 + 2 c {} 0 {} 0 + 3 d {} 0 {} 0 +} +do_execsql_test 210 { + INSERT INTO x1(a,b,c,d) VALUES(15, 11, 22, 33),(8,'banjo',x'333231',NULL), + (12,NULL,3.25,-559281390); + SELECT quote(a), quote(b), quote(c), quote(d), '|' FROM x1; +} {'12' NULL 3.25 -559281390 | '15' 11 22 33 | '8' 'banjo' X'333231' NULL |} +do_execsql_test 211 { + SELECT quote(a), quote(lsm1_key), quote(lsm1_value), '|' FROM x1; +} {'12' X'3132' X'05320000000000000A401FFB42ABE9DB' | '15' X'3135' X'4284C6' | '8' X'38' X'2162616E6A6F1633323105' |} +finish_test Index: ext/misc/csv.c ================================================================== --- ext/misc/csv.c +++ ext/misc/csv.c @@ -678,20 +678,20 @@ } memcpy(pCur->azVal[i], z, pCur->rdr.n+1); i++; } }while( pCur->rdr.cTerm==',' ); - while( inCol ){ - sqlite3_free(pCur->azVal[i]); - pCur->azVal[i] = 0; - pCur->aLen[i] = 0; - i++; - } - if( z==0 || pCur->rdr.cTerm==EOF ){ + if( z==0 || (pCur->rdr.cTerm==EOF && inCol) ){ pCur->iRowid = -1; }else{ pCur->iRowid++; + while( inCol ){ + sqlite3_free(pCur->azVal[i]); + pCur->azVal[i] = 0; + pCur->aLen[i] = 0; + i++; + } } return SQLITE_OK; } /* ADDED ext/misc/vtablog.c Index: ext/misc/vtablog.c ================================================================== --- ext/misc/vtablog.c +++ ext/misc/vtablog.c @@ -0,0 +1,509 @@ +/* +** 2017-08-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a virtual table that prints diagnostic information +** on stdout when its key interfaces are called. This is intended for +** interactive analysis and debugging of virtual table interfaces. +** +** Usage example: +** +** .load ./vtablog +** CREATE VIRTUAL TABLE temp.log USING vtablog( +** schema='CREATE TABLE x(a,b,c)', +** rows=25 +** ); +** SELECT * FROM log; +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include +#include +#include + + +/* vtablog_vtab is a subclass of sqlite3_vtab which will +** serve as the underlying representation of a vtablog virtual table +*/ +typedef struct vtablog_vtab vtablog_vtab; +struct vtablog_vtab { + sqlite3_vtab base; /* Base class - must be first */ + int nRow; /* Number of rows in the table */ + int iInst; /* Instance number for this vtablog table */ + int nCursor; /* Number of cursors created */ +}; + +/* vtablog_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct vtablog_cursor vtablog_cursor; +struct vtablog_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + int iCursor; /* Cursor number */ + sqlite3_int64 iRowid; /* The rowid */ +}; + +/* Skip leading whitespace. Return a pointer to the first non-whitespace +** character, or to the zero terminator if the string has only whitespace */ +static const char *vtablog_skip_whitespace(const char *z){ + while( isspace((unsigned char)z[0]) ) z++; + return z; +} + +/* Remove trailing whitespace from the end of string z[] */ +static void vtablog_trim_whitespace(char *z){ + size_t n = strlen(z); + while( n>0 && isspace((unsigned char)z[n]) ) n--; + z[n] = 0; +} + +/* Dequote the string */ +static void vtablog_dequote(char *z){ + int j; + char cQuote = z[0]; + size_t i, n; + + if( cQuote!='\'' && cQuote!='"' ) return; + n = strlen(z); + if( n<2 || z[n-1]!=z[0] ) return; + for(i=1, j=0; inRow = 10; + if( zNRow ) pNew->nRow = atoi(zNRow); + pNew->iInst = iInst; + } + return rc; +} +static int vtablogCreate( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return vtablogConnectCreate(db,pAux,argc,argv,ppVtab,pzErr,1); +} +static int vtablogConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return vtablogConnectCreate(db,pAux,argc,argv,ppVtab,pzErr,0); +} + + +/* +** This method is the destructor for vtablog_cursor objects. +*/ +static int vtablogDisconnect(sqlite3_vtab *pVtab){ + vtablog_vtab *pTab = (vtablog_vtab*)pVtab; + printf("vtablogDisconnect(%d)\n", pTab->iInst); + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** This method is the destructor for vtablog_cursor objects. +*/ +static int vtablogDestroy(sqlite3_vtab *pVtab){ + vtablog_vtab *pTab = (vtablog_vtab*)pVtab; + printf("vtablogDestroy(%d)\n", pTab->iInst); + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new vtablog_cursor object. +*/ +static int vtablogOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + vtablog_vtab *pTab = (vtablog_vtab*)p; + vtablog_cursor *pCur; + printf("vtablogOpen(tab=%d, cursor=%d)\n", pTab->iInst, ++pTab->nCursor); + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->iCursor = pTab->nCursor; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a vtablog_cursor. +*/ +static int vtablogClose(sqlite3_vtab_cursor *cur){ + vtablog_cursor *pCur = (vtablog_cursor*)cur; + vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab; + printf("vtablogClose(tab=%d, cursor=%d)\n", pTab->iInst, pCur->iCursor); + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a vtablog_cursor to its next row of output. +*/ +static int vtablogNext(sqlite3_vtab_cursor *cur){ + vtablog_cursor *pCur = (vtablog_cursor*)cur; + vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab; + printf("vtablogNext(tab=%d, cursor=%d) rowid %d -> %d\n", + pTab->iInst, pCur->iCursor, (int)pCur->iRowid, (int)pCur->iRowid+1); + pCur->iRowid++; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the vtablog_cursor +** is currently pointing. +*/ +static int vtablogColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + vtablog_cursor *pCur = (vtablog_cursor*)cur; + vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab; + char zVal[50]; + + if( i<26 ){ + sqlite3_snprintf(sizeof(zVal),zVal,"%c%d", + "abcdefghijklmnopqrstuvwyz"[i], pCur->iRowid); + }else{ + sqlite3_snprintf(sizeof(zVal),zVal,"{%d}%d", i, pCur->iRowid); + } + printf("vtablogColumn(tab=%d, cursor=%d, i=%d): [%s]\n", + pTab->iInst, pCur->iCursor, i, zVal); + sqlite3_result_text(ctx, zVal, -1, SQLITE_TRANSIENT); + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int vtablogRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + vtablog_cursor *pCur = (vtablog_cursor*)cur; + vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab; + printf("vtablogRowid(tab=%d, cursor=%d): %d\n", + pTab->iInst, pCur->iCursor, (int)pCur->iRowid); + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int vtablogEof(sqlite3_vtab_cursor *cur){ + vtablog_cursor *pCur = (vtablog_cursor*)cur; + vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab; + int rc = pCur->iRowid >= pTab->nRow; + printf("vtablogEof(tab=%d, cursor=%d): %d\n", + pTab->iInst, pCur->iCursor, rc); + return rc; +} + +/* +** Output an sqlite3_value object's value as an SQL literal. +*/ +static void vtablogQuote(sqlite3_value *p){ + char z[50]; + switch( sqlite3_value_type(p) ){ + case SQLITE_NULL: { + printf("NULL"); + break; + } + case SQLITE_INTEGER: { + sqlite3_snprintf(50,z,"%lld", sqlite3_value_int64(p)); + printf("%s", z); + break; + } + case SQLITE_FLOAT: { + sqlite3_snprintf(50,z,"%!.20g", sqlite3_value_double(p)); + printf("%s", z); + break; + } + case SQLITE_BLOB: { + int n = sqlite3_value_bytes(p); + const unsigned char *z = (const unsigned char*)sqlite3_value_blob(p); + int i; + printf("x'"); + for(i=0; ipVtab; + printf("vtablogFilter(tab=%d, cursor=%d):\n", pTab->iInst, pCur->iCursor); + pCur->iRowid = 0; + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the vtablog virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +*/ +static int vtablogBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + vtablog_vtab *pTab = (vtablog_vtab*)tab; + printf("vtablogBestIndex(tab=%d):\n", pTab->iInst); + pIdxInfo->estimatedCost = (double)500; + pIdxInfo->estimatedRows = 500; + return SQLITE_OK; +} + +/* +** SQLite invokes this method to INSERT, UPDATE, or DELETE content from +** the table. +** +** This implementation does not actually make any changes to the table +** content. It merely logs the fact that the method was invoked +*/ +static int vtablogUpdate( + sqlite3_vtab *tab, + int argc, + sqlite3_value **argv, + sqlite_int64 *pRowid +){ + vtablog_vtab *pTab = (vtablog_vtab*)tab; + int i; + printf("vtablogUpdate(tab=%d):\n", pTab->iInst); + printf(" argc=%d\n", argc); + for(i=0; i0 AND x1<100 AND x2>0 AND x2<100; } {1 {undersize RTree blobs in "t1_node"}} +do_test rtreeA-7.120 { + sqlite3_extended_errcode db +} {SQLITE_CORRUPT} + finish_test Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -3881,22 +3881,24 @@ ** Add an INDEXED BY or NOT INDEXED clause to the most recently added ** element of the source-list passed as the second argument. */ void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){ assert( pIndexedBy!=0 ); - if( p && ALWAYS(p->nSrc>0) ){ - struct SrcList_item *pItem = &p->a[p->nSrc-1]; + if( p && pIndexedBy->n>0 ){ + struct SrcList_item *pItem; + assert( p->nSrc>0 ); + pItem = &p->a[p->nSrc-1]; assert( pItem->fg.notIndexed==0 ); assert( pItem->fg.isIndexedBy==0 ); assert( pItem->fg.isTabFunc==0 ); if( pIndexedBy->n==1 && !pIndexedBy->z ){ /* A "NOT INDEXED" clause was supplied. See parse.y ** construct "indexed_opt" for details. */ pItem->fg.notIndexed = 1; }else{ pItem->u1.zIndexedBy = sqlite3NameFromToken(pParse->db, pIndexedBy); - pItem->fg.isIndexedBy = (pItem->u1.zIndexedBy!=0); + pItem->fg.isIndexedBy = 1; } } } /* Index: src/delete.c ================================================================== --- src/delete.c +++ src/delete.c @@ -500,11 +500,15 @@ sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); VdbeCoverage(v); } }else if( pPk ){ addrLoop = sqlite3VdbeAddOp1(v, OP_Rewind, iEphCur); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_RowData, iEphCur, iKey); + if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp3(v, OP_Column, iEphCur, 0, iKey); + }else{ + sqlite3VdbeAddOp2(v, OP_RowData, iEphCur, iKey); + } assert( nKey==0 ); /* OP_Found will use a composite key */ }else{ addrLoop = sqlite3VdbeAddOp3(v, OP_RowSetRead, iRowSet, 0, iKey); VdbeCoverage(v); assert( nKey==1 ); Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -3882,10 +3882,18 @@ i64 newLimit = *(i64*)pArg; int rc = SQLITE_OK; if( newLimit>sqlite3GlobalConfig.mxMmap ){ newLimit = sqlite3GlobalConfig.mxMmap; } + + /* The value of newLimit may be eventually cast to (size_t) and passed + ** to mmap(). Restrict its value to 2GB if (size_t) is not at least a + ** 64-bit type. */ + if( newLimit>0 && sizeof(size_t)<8 ){ + newLimit = (newLimit & 0x7FFFFFFF); + } + *(i64*)pArg = pFile->mmapSizeMax; if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){ pFile->mmapSizeMax = newLimit; if( pFile->mmapSize>0 ){ unixUnmapfile(pFile); Index: src/os_win.c ================================================================== --- src/os_win.c +++ src/os_win.c @@ -3557,10 +3557,18 @@ i64 newLimit = *(i64*)pArg; int rc = SQLITE_OK; if( newLimit>sqlite3GlobalConfig.mxMmap ){ newLimit = sqlite3GlobalConfig.mxMmap; } + + /* The value of newLimit may be eventually cast to (SIZE_T) and passed + ** to MapViewOfFile(). Restrict its value to 2GB if (SIZE_T) is not at + ** least a 64-bit type. */ + if( newLimit>0 && sizeof(SIZE_T)<8 ){ + newLimit = (newLimit & 0x7FFFFFFF); + } + *(i64*)pArg = pFile->mmapSizeMax; if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){ pFile->mmapSizeMax = newLimit; if( pFile->mmapSize>0 ){ winUnmapfile(pFile); Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -126,12 +126,12 @@ ** ** PAGERID() takes a pointer to a Pager struct as its argument. The ** associated file-descriptor is returned. FILEHANDLEID() takes an sqlite3_file ** struct as its argument. */ -#define PAGERID(p) ((int)(p->fd)) -#define FILEHANDLEID(fd) ((int)fd) +#define PAGERID(p) (SQLITE_PTR_TO_INT(p->fd)) +#define FILEHANDLEID(fd) (SQLITE_PTR_TO_INT(fd)) /* ** The Pager.eState variable stores the current 'state' of a pager. A ** pager may be in any one of the seven states shown in the following ** state diagram. Index: src/printf.c ================================================================== --- src/printf.c +++ src/printf.c @@ -780,11 +780,10 @@ setStrAccumError(p, STRACCUM_TOOBIG); return N; }else{ char *zOld = isMalloced(p) ? p->zText : 0; i64 szNew = p->nChar; - assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); szNew += N + 1; if( szNew+p->nChar<=p->mxAlloc ){ /* Force exponential buffer size growth as long as it does not overflow, ** to avoid having to call this routine too often */ szNew += p->nChar; @@ -822,11 +821,10 @@ void sqlite3AppendChar(StrAccum *p, int N, char c){ testcase( p->nChar + (i64)N > 0x7fffffff ); if( p->nChar+(i64)N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ){ return; } - assert( (p->zText==p->zBase)==!isMalloced(p) ); while( (N--)>0 ) p->zText[p->nChar++] = c; } /* ** The StrAccum "p" is not large enough to accept N new bytes of z[]. @@ -840,11 +838,10 @@ N = sqlite3StrAccumEnlarge(p, N); if( N>0 ){ memcpy(&p->zText[p->nChar], z, N); p->nChar += N; } - assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); } /* ** Append N bytes of text from z to the StrAccum object. Increase the ** size of the memory allocation for StrAccum if necessary. @@ -875,23 +872,24 @@ ** Finish off a string by making sure it is zero-terminated. ** Return a pointer to the resulting string. Return a NULL ** pointer if any kind of error was encountered. */ static SQLITE_NOINLINE char *strAccumFinishRealloc(StrAccum *p){ + char *zText; assert( p->mxAlloc>0 && !isMalloced(p) ); - p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); - if( p->zText ){ - memcpy(p->zText, p->zBase, p->nChar+1); + zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); + if( zText ){ + memcpy(zText, p->zText, p->nChar+1); p->printfFlags |= SQLITE_PRINTF_MALLOCED; }else{ setStrAccumError(p, STRACCUM_NOMEM); } - return p->zText; + p->zText = zText; + return zText; } char *sqlite3StrAccumFinish(StrAccum *p){ if( p->zText ){ - assert( (p->zText==p->zBase)==!isMalloced(p) ); p->zText[p->nChar] = 0; if( p->mxAlloc>0 && !isMalloced(p) ){ return strAccumFinishRealloc(p); } } @@ -900,11 +898,10 @@ /* ** Reset an StrAccum string. Reclaim all malloced memory. */ void sqlite3StrAccumReset(StrAccum *p){ - assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); if( isMalloced(p) ){ sqlite3DbFree(p->db, p->zText); p->printfFlags &= ~SQLITE_PRINTF_MALLOCED; } p->zText = 0; @@ -923,15 +920,15 @@ ** n then no memory allocations ever occur. ** mx: Maximum number of bytes to accumulate. If mx==0 then no memory ** allocations will ever occur. */ void sqlite3StrAccumInit(StrAccum *p, sqlite3 *db, char *zBase, int n, int mx){ - p->zText = p->zBase = zBase; + p->zText = zBase; p->db = db; - p->nChar = 0; p->nAlloc = n; p->mxAlloc = mx; + p->nChar = 0; p->accError = 0; p->printfFlags = 0; } /* Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -1708,11 +1708,13 @@ Expr *pColExpr = sqlite3ExprSkipCollate(pEList->a[i].pExpr); while( pColExpr->op==TK_DOT ){ pColExpr = pColExpr->pRight; assert( pColExpr!=0 ); } - if( pColExpr->op==TK_COLUMN && pColExpr->pTab!=0 ){ + if( (pColExpr->op==TK_COLUMN || pColExpr->op==TK_AGG_COLUMN) + && pColExpr->pTab!=0 + ){ /* For columns use the column name name */ int iCol = pColExpr->iColumn; Table *pTab = pColExpr->pTab; if( iCol<0 ) iCol = pTab->iPKey; zName = iCol>=0 ? pTab->aCol[iCol].zName : "rowid"; Index: src/sqlite3ext.h ================================================================== --- src/sqlite3ext.h +++ src/sqlite3ext.h @@ -244,11 +244,11 @@ int (*stmt_readonly)(sqlite3_stmt*); int (*stricmp)(const char*,const char*); int (*uri_boolean)(const char*,const char*,int); sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64); const char *(*uri_parameter)(const char*,const char*); - char *(*vsnprintf)(int,char*,const char*,va_list); + char *(*xvsnprintf)(int,char*,const char*,va_list); int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*); /* Version 3.8.7 and later */ int (*auto_extension)(void(*)(void)); int (*bind_blob64)(sqlite3_stmt*,int,const void*,sqlite3_uint64, void(*)(void*)); @@ -440,11 +440,11 @@ #define sqlite3_value_text16 sqlite3_api->value_text16 #define sqlite3_value_text16be sqlite3_api->value_text16be #define sqlite3_value_text16le sqlite3_api->value_text16le #define sqlite3_value_type sqlite3_api->value_type #define sqlite3_vmprintf sqlite3_api->vmprintf -#define sqlite3_vsnprintf sqlite3_api->vsnprintf +#define sqlite3_vsnprintf sqlite3_api->xvsnprintf #define sqlite3_overload_function sqlite3_api->overload_function #define sqlite3_prepare_v2 sqlite3_api->prepare_v2 #define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 #define sqlite3_clear_bindings sqlite3_api->clear_bindings #define sqlite3_bind_zeroblob sqlite3_api->bind_zeroblob @@ -516,11 +516,11 @@ #define sqlite3_stmt_readonly sqlite3_api->stmt_readonly #define sqlite3_stricmp sqlite3_api->stricmp #define sqlite3_uri_boolean sqlite3_api->uri_boolean #define sqlite3_uri_int64 sqlite3_api->uri_int64 #define sqlite3_uri_parameter sqlite3_api->uri_parameter -#define sqlite3_uri_vsnprintf sqlite3_api->vsnprintf +#define sqlite3_uri_vsnprintf sqlite3_api->xvsnprintf #define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 /* Version 3.8.7 and later */ #define sqlite3_auto_extension sqlite3_api->auto_extension #define sqlite3_bind_blob64 sqlite3_api->bind_blob64 #define sqlite3_bind_text64 sqlite3_api->bind_text64 Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -3227,15 +3227,14 @@ ** An objected used to accumulate the text of a string where we ** do not necessarily know how big the string will be in the end. */ struct StrAccum { sqlite3 *db; /* Optional database for lookaside. Can be NULL */ - char *zBase; /* A base allocation. Not from malloc. */ char *zText; /* The string collected so far */ - u32 nChar; /* Length of the string so far */ u32 nAlloc; /* Amount of space allocated in zText */ u32 mxAlloc; /* Maximum allowed allocation. 0 for no malloc usage */ + u32 nChar; /* Length of the string so far */ u8 accError; /* STRACCUM_NOMEM or STRACCUM_TOOBIG */ u8 printfFlags; /* SQLITE_PRINTF flags below */ }; #define STRACCUM_NOMEM 1 #define STRACCUM_TOOBIG 2 Index: src/test_tclvar.c ================================================================== --- src/test_tclvar.c +++ src/test_tclvar.c @@ -13,10 +13,29 @@ ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. ** ** The emphasis of this file is a virtual table that provides ** access to TCL variables. +** +** The TCLVAR eponymous virtual table has a schema like this: +** +** CREATE TABLE tclvar( +** name TEXT, -- base name of the variable: "x" in "$x(y)" +** arrayname TEXT, -- array index name: "y" in "$x(y)" +** value TEXT, -- the value of the variable +** fullname TEXT, -- the full name of the variable +** PRIMARY KEY(fullname) +** ) WITHOUT ROWID; +** +** DELETE, INSERT, and UPDATE operations use the "fullname" field to +** determine the variable to be modified. Changing "value" to NULL +** deletes the variable. +** +** For SELECT operations, the "name" and "arrayname" fields will always +** match the "fullname" field. For DELETE, INSERT, and UPDATE, the +** "name" and "arrayname" fields are ignored and the variable is modified +** according to "fullname" and "value" only. */ #include "sqliteInt.h" #if defined(INCLUDE_SQLITE_TCL_H) # include "sqlite_tcl.h" #else @@ -65,11 +84,16 @@ sqlite3_vtab **ppVtab, char **pzErr ){ tclvar_vtab *pVtab; static const char zSchema[] = - "CREATE TABLE whatever(name TEXT, arrayname TEXT, value TEXT)"; + "CREATE TABLE x(" + " name TEXT," /* Base name */ + " arrayname TEXT," /* Array index */ + " value TEXT," /* Value */ + " fullname TEXT PRIMARY KEY" /* base(index) name */ + ") WITHOUT ROWID"; pVtab = sqlite3MallocZero( sizeof(*pVtab) ); if( pVtab==0 ) return SQLITE_NOMEM; *ppVtab = &pVtab->base; pVtab->interp = (Tcl_Interp *)pAux; sqlite3_declare_vtab(db, zSchema); @@ -249,10 +273,20 @@ case 2: { Tcl_Obj *pVal = Tcl_GetVar2Ex(interp, z1, *z2?z2:0, TCL_GLOBAL_ONLY); sqlite3_result_text(ctx, Tcl_GetString(pVal), -1, SQLITE_TRANSIENT); break; } + case 3: { + char *z3; + if( p2 ){ + z3 = sqlite3_mprintf("%s(%s)", z1, z2); + sqlite3_result_text(ctx, z3, -1, sqlite3_free); + }else{ + sqlite3_result_text(ctx, z1, -1, SQLITE_TRANSIENT); + } + break; + } } return SQLITE_OK; } static int tclvarRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ @@ -373,10 +407,62 @@ pIdxInfo->idxStr = zStr; pIdxInfo->needToFreeIdxStr = 1; return SQLITE_OK; } + +/* +** Invoked for any UPDATE, INSERT, or DELETE against a tclvar table +*/ +static int tclvarUpdate( + sqlite3_vtab *tab, + int argc, + sqlite3_value **argv, + sqlite_int64 *pRowid +){ + tclvar_vtab *pTab = (tclvar_vtab*)tab; + if( argc==1 ){ + /* A DELETE operation. The variable to be deleted is stored in argv[0] */ + const char *zVar = (const char*)sqlite3_value_text(argv[0]); + Tcl_UnsetVar(pTab->interp, zVar, TCL_GLOBAL_ONLY); + return SQLITE_OK; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + /* An INSERT operation */ + const char *zValue = (const char*)sqlite3_value_text(argv[4]); + const char *zName; + if( sqlite3_value_type(argv[5])!=SQLITE_TEXT ){ + tab->zErrMsg = sqlite3_mprintf("the 'fullname' column must be TEXT"); + return SQLITE_ERROR; + } + zName = (const char*)sqlite3_value_text(argv[5]); + if( zValue ){ + Tcl_SetVar(pTab->interp, zName, zValue, TCL_GLOBAL_ONLY); + }else{ + Tcl_UnsetVar(pTab->interp, zName, TCL_GLOBAL_ONLY); + } + return SQLITE_OK; + } + if( sqlite3_value_type(argv[0])==SQLITE_TEXT + && sqlite3_value_type(argv[1])==SQLITE_TEXT + ){ + /* An UPDATE operation */ + const char *zOldName = (const char*)sqlite3_value_text(argv[0]); + const char *zNewName = (const char*)sqlite3_value_text(argv[1]); + const char *zValue = (const char*)sqlite3_value_text(argv[4]); + + if( strcmp(zOldName, zNewName)!=0 || zValue==0 ){ + Tcl_UnsetVar(pTab->interp, zOldName, TCL_GLOBAL_ONLY); + } + if( zValue!=0 ){ + Tcl_SetVar(pTab->interp, zNewName, zValue, TCL_GLOBAL_ONLY); + } + return SQLITE_OK; + } + tab->zErrMsg = sqlite3_mprintf("prohibited TCL variable change"); + return SQLITE_ERROR; +} /* ** A virtual table module that provides read-only access to a ** Tcl global variable namespace. */ @@ -392,11 +478,11 @@ tclvarFilter, /* xFilter - configure scan constraints */ tclvarNext, /* xNext - advance a cursor */ tclvarEof, /* xEof - check for end of scan */ tclvarColumn, /* xColumn - read data */ tclvarRowid, /* xRowid - read data */ - 0, /* xUpdate */ + tclvarUpdate, /* xUpdate */ 0, /* xBegin */ 0, /* xSync */ 0, /* xCommit */ 0, /* xRollback */ 0, /* xFindMethod */ Index: src/update.c ================================================================== --- src/update.c +++ src/update.c @@ -801,23 +801,34 @@ /* Start scanning the virtual table */ pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0,0,WHERE_ONEPASS_DESIRED,0); if( pWInfo==0 ) return; /* Populate the argument registers. */ - sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg); - if( pRowid ){ - sqlite3ExprCode(pParse, pRowid, regArg+1); - }else{ - sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg+1); - } for(i=0; inCol; i++){ if( aXRef[i]>=0 ){ sqlite3ExprCode(pParse, pChanges->a[aXRef[i]].pExpr, regArg+2+i); }else{ sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, i, regArg+2+i); } } + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg); + if( pRowid ){ + sqlite3ExprCode(pParse, pRowid, regArg+1); + }else{ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg+1); + } + }else{ + Index *pPk; /* PRIMARY KEY index */ + i16 iPk; /* PRIMARY KEY column */ + pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + assert( pPk->nKeyCol==1 ); + iPk = pPk->aiColumn[0]; + sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, iPk, regArg); + sqlite3VdbeAddOp2(v, OP_SCopy, regArg+2+iPk, regArg+1); + } bOnePass = sqlite3WhereOkOnePass(pWInfo, aDummy); if( bOnePass ){ /* If using the onepass strategy, no-op out the OP_OpenEphemeral coded Index: src/vtab.c ================================================================== --- src/vtab.c +++ src/vtab.c @@ -642,10 +642,11 @@ }else{ char *zErr = 0; rc = vtabCallConstructor(db, pTab, pMod, pMod->pModule->xConnect, &zErr); if( rc!=SQLITE_OK ){ sqlite3ErrorMsg(pParse, "%s", zErr); + pParse->rc = rc; } sqlite3DbFree(db, zErr); } return rc; @@ -770,11 +771,17 @@ pTab->nCol = pNew->nCol; pTab->tabFlags |= pNew->tabFlags & (TF_WithoutRowid|TF_NoVisibleRowid); pNew->nCol = 0; pNew->aCol = 0; assert( pTab->pIndex==0 ); - if( !HasRowid(pNew) && pCtx->pVTable->pMod->pModule->xUpdate!=0 ){ + assert( HasRowid(pNew) || sqlite3PrimaryKeyIndex(pNew)!=0 ); + if( !HasRowid(pNew) + && pCtx->pVTable->pMod->pModule->xUpdate!=0 + && sqlite3PrimaryKeyIndex(pNew)->nKeyCol!=1 + ){ + /* WITHOUT ROWID virtual tables must either be read-only (xUpdate==0) + ** or else must have a single-column PRIMARY KEY */ rc = SQLITE_ERROR; } pIdx = pNew->pIndex; if( pIdx ){ assert( pIdx->pNext==0 ); Index: src/whereexpr.c ================================================================== --- src/whereexpr.c +++ src/whereexpr.c @@ -192,11 +192,11 @@ Expr *pExpr, /* Test this expression */ Expr **ppPrefix, /* Pointer to TK_STRING expression with pattern prefix */ int *pisComplete, /* True if the only wildcard is % in the last character */ int *pnoCase /* True if uppercase is equivalent to lowercase */ ){ - const char *z = 0; /* String on RHS of LIKE operator */ + const u8 *z = 0; /* String on RHS of LIKE operator */ Expr *pRight, *pLeft; /* Right and left size of LIKE operator */ ExprList *pList; /* List of operands to the LIKE operator */ int c; /* One character in z[] */ int cnt; /* Number of non-wildcard prefix characters */ char wc[4]; /* Wildcard characters */ @@ -219,16 +219,16 @@ if( op==TK_VARIABLE && (db->flags & SQLITE_EnableQPSG)==0 ){ Vdbe *pReprepare = pParse->pReprepare; int iCol = pRight->iColumn; pVal = sqlite3VdbeGetBoundValue(pReprepare, iCol, SQLITE_AFF_BLOB); if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ - z = (char *)sqlite3_value_text(pVal); + z = sqlite3_value_text(pVal); } sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); }else if( op==TK_STRING ){ - z = pRight->u.zToken; + z = (u8*)pRight->u.zToken; } if( z ){ /* If the RHS begins with a digit or a minus sign, then the LHS must ** be an ordinary column (not a virtual table column) with TEXT affinity. @@ -249,13 +249,11 @@ /* Count the number of prefix characters prior to the first wildcard */ cnt = 0; while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ cnt++; - if( c==wc[3] && z[cnt]!=0 ){ - if( z[cnt++]>0xc0 ) while( (z[cnt]&0xc0)==0x80 ){ cnt++; } - } + if( c==wc[3] && z[cnt]!=0 ) cnt++; } /* The optimization is possible only if (1) the pattern does not begin ** with a wildcard and if (2) the non-wildcard prefix does not end with ** an (illegal 0xff) character. The second condition is necessary so @@ -267,11 +265,11 @@ /* A "complete" match if the pattern ends with "*" or "%" */ *pisComplete = c==wc[0] && z[cnt+1]==0; /* Get the pattern prefix. Remove all escapes from the prefix. */ - pPrefix = sqlite3Expr(db, TK_STRING, z); + pPrefix = sqlite3Expr(db, TK_STRING, (char*)z); if( pPrefix ){ int iFrom, iTo; char *zNew = pPrefix->u.zToken; zNew[cnt] = 0; for(iFrom=iTo=0; iFrom mmap_limit +} +if {$mmap_limit < [expr 8 * 1<<30]} { + puts "Skipping bigmmap.test - requires SQLITE_MAX_MMAP_SIZE >= 8G" + finish_test + return +} + + +#------------------------------------------------------------------------- +# Create the database file roughly 8GiB in size. Most pages are unused, +# except that there is a table and index clustered around each 1GiB +# boundary. +# +do_execsql_test 1.0 { + PRAGMA page_size = 4096; + CREATE TABLE t0(a INTEGER PRIMARY KEY, b, c, UNIQUE(b, c)); + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s LIMIT 100 ) + INSERT INTO t0 SELECT i, 't0', randomblob(800) FROM s; +} + +for {set i 1} {$i < 8} {incr i} { + fake_big_file [expr $i*1024] [get_pwd]/test.db + hexio_write test.db 28 [format %.8x [expr ($i*1024*1024*1024/4096) - 5]] + + do_execsql_test 1.$i " + CREATE TABLE t$i (a INTEGER PRIMARY KEY, b, c, UNIQUE(b, c)); + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s LIMIT 100 ) + INSERT INTO t$i SELECT i, 't$i', randomblob(800) FROM s; + " +} + +#------------------------------------------------------------------------- +# Check that data can be retrieved from the db with a variety of +# configured mmap size limits. +# +for {set i 0} {$i < 9} {incr i} { + + # Configure a memory mapping $i GB in size. + # + set val [expr $i*1024*1024*1024] + execsql "PRAGMA main.mmap_size = $val" + do_execsql_test 2.$i.0 { + PRAGMA main.mmap_size + } $val + + for {set t 0} {$t < 8} {incr t} { + do_execsql_test 2.$i.$t.1 " + SELECT count(*) FROM t$t; + SELECT count(b || c) FROM t$t GROUP BY b; + " {100 100} + + do_execsql_test 2.$i.$t.2 " + SELECT * FROM t$t AS o WHERE + NOT EXISTS( SELECT * FROM t$t AS i WHERE a=o.a AND +b=o.b AND +c=o.c ) + ORDER BY b, c; + " {} + + do_eqp_test 2.$i.$t.3 " + SELECT * FROM t$t AS o WHERE + NOT EXISTS( SELECT * FROM t$t AS i WHERE a=o.a AND +b=o.b AND +c=o.c ) + ORDER BY b, c; + " " + 0 0 0 {SCAN TABLE t$t AS o USING COVERING INDEX sqlite_autoindex_t${t}_1} + 0 0 0 {EXECUTE CORRELATED SCALAR SUBQUERY 1} + 1 0 0 {SEARCH TABLE t$t AS i USING INTEGER PRIMARY KEY (rowid=?)} + " + } +} + +finish_test + Index: test/colname.test ================================================================== --- test/colname.test +++ test/colname.test @@ -375,8 +375,25 @@ execsql2 {SELECT t1.a AS n, v3.a FROM t1 JOIN v3} } {n 1 a 3} do_test colname-9.210 { execsql2 {SELECT t1.a, v3.a AS n FROM t1 JOIN v3} } {a 1 n 3} + +# Make sure the quotation marks get removed from the column names +# when constructing a new table from an aggregate SELECT. +# Email from Juergen Palm on 2017-07-11. +# +do_execsql_test colname-10.100 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1("with space" TEXT); + DROP TABLE IF EXISTS t2; + CREATE TABLE t2 AS SELECT "with space" FROM t1; + PRAGMA table_info(t2); +} {0 {with space} TEXT 0 {} 0} +do_execsql_test colname-10.110 { + DROP TABLE IF EXISTS t3; + CREATE TABLE t3 AS SELECT "with space" FROM t1 GROUP BY 1; + PRAGMA table_info(t3); +} {0 {with space} TEXT 0 {} 0} finish_test Index: test/csv01.test ================================================================== --- test/csv01.test +++ test/csv01.test @@ -91,22 +91,54 @@ # The rowid column is not visible on a WITHOUT ROWID virtual table do_catchsql_test 3.2 { SELECT rowid, a FROM t3; } {1 {no such column: rowid}} +# Multi-column WITHOUT ROWID virtual tables may not be writable. do_catchsql_test 4.0 { DROP TABLE t3; CREATE VIRTUAL TABLE temp.t4 USING csv_wr( data= '1,2,3,4 5,6,7,8 9,10,11,12 -13,14,15,16 -', +13,14,15,16', columns=4, schema= - 'CREATE TABLE t3(a PRIMARY KEY,b TEXT,c TEXT,d TEXT) WITHOUT ROWID', + 'CREATE TABLE t3(a,b,c,d,PRIMARY KEY(a,b)) WITHOUT ROWID', testflags=1 ); } {1 {vtable constructor failed: t4}} + +# WITHOUT ROWID tables with a single-column PRIMARY KEY may be writable. +do_catchsql_test 4.1 { + DROP TABLE IF EXISTS t4; + CREATE VIRTUAL TABLE temp.t4 USING csv_wr( + data= +'1,2,3,4 +5,6,7,8 +9,10,11,12 +13,14,15,16', + columns=4, + schema= + 'CREATE TABLE t3(a,b,c,d,PRIMARY KEY(b)) WITHOUT ROWID', + testflags=1 + ); +} {0 {}} + +do_catchsql_test 4.2 { + DROP TABLE IF EXISTS t5; + CREATE VIRTUAL TABLE temp.t5 USING csv_wr( + data= + '1,2,3,4 + 5,6,7,8 + 9,10,11,12 + 13,14,15,16', + columns=4, + schema= + 'CREATE TABLE t3(a,b,c,d) WITHOUT ROWID', + testflags=1 + ); +} {1 {vtable constructor failed: t5}} + finish_test Index: test/swarmvtab.test ================================================================== --- test/swarmvtab.test +++ test/swarmvtab.test @@ -236,11 +236,11 @@ ("test.db2", "t1", 11, 20) ', 'fetch_db' ); } {} -do_catchsql_test 3.3 { SELECT * FROM xyz } {1 {fetch_db error!}} +do_catchsql_test 3.3.2 { SELECT * FROM xyz } {1 {fetch_db error!}} finish_test Index: test/vtab2.test ================================================================== --- test/vtab2.test +++ test/vtab2.test @@ -58,19 +58,19 @@ register_tclvar_module [sqlite3_connection_pointer db] do_test vtab2-2.1 { set ::abc 123 execsql { CREATE VIRTUAL TABLE vars USING tclvar; - SELECT * FROM vars WHERE name='abc'; + SELECT name, arrayname, value FROM vars WHERE name='abc'; } } [list abc "" 123] do_test vtab2-2.2 { set A(1) 1 set A(2) 4 set A(3) 9 execsql { - SELECT * FROM vars WHERE name='A'; + SELECT name, arrayname, value FROM vars WHERE name='A'; } } [list A 1 1 A 2 4 A 3 9] unset -nocomplain result unset -nocomplain var set result {} Index: test/vtabE.test ================================================================== --- test/vtabE.test +++ test/vtabE.test @@ -37,14 +37,16 @@ do_test vtabE-1 { db eval { CREATE VIRTUAL TABLE t1 USING tclvar; CREATE VIRTUAL TABLE t2 USING tclvar; CREATE TABLE t3(a INTEGER PRIMARY KEY, b); - SELECT t1.*, t2.*, abs(t3.b + abs(t2.value + abs(t1.value))) + SELECT t1.name, t1.arrayname, t1.value, + t2.name, t2.arrayname, t2.value, + abs(t3.b + abs(t2.value + abs(t1.value))) FROM t1 LEFT JOIN t2 ON t2.name = t1.arrayname LEFT JOIN t3 ON t3.a=t2.value WHERE t1.name = 'vtabE' ORDER BY t1.value, t2.value; } } {vtabE vtabE1 11 vtabE1 w x {} vtabE vtabE1 11 vtabE1 y z {} vtabE vtabE2 22 vtabE2 a b {} vtabE vtabE2 22 vtabE2 c d {}} finish_test Index: test/vtabH.test ================================================================== --- test/vtabH.test +++ test/vtabH.test @@ -53,11 +53,11 @@ register_tclvar_module db set ::xyz 10 do_execsql_test 2.0 { CREATE VIRTUAL TABLE vars USING tclvar; - SELECT * FROM vars WHERE name = 'xyz'; + SELECT name, arrayname, value FROM vars WHERE name = 'xyz'; } {xyz {} 10} set x1 aback set x2 abaft set x3 abandon ADDED test/vtabJ.test Index: test/vtabJ.test ================================================================== --- test/vtabJ.test +++ test/vtabJ.test @@ -0,0 +1,126 @@ +# 2017-08-10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements tests of writing to WITHOUT ROWID virtual tables +# using the tclvar eponymous virtual table. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix vtabJ + +ifcapable !vtab { + finish_test + return +} + +register_tclvar_module db + +unset -nocomplain vtabJ +do_test 100 { + set vtabJ(1) this + set vtabJ(two) is + set vtabJ(3) {a test} + db eval { + SELECT fullname, value FROM tclvar WHERE name='vtabJ' ORDER BY fullname; + } +} {vtabJ(1) this vtabJ(3) {a test} vtabJ(two) is} + +do_execsql_test 110 { + INSERT INTO tclvar(fullname, value) + VALUES('vtabJ(4)',4),('vtabJ(five)',555); + SELECT fullname, value FROM tclvar WHERE name='vtabJ' ORDER BY fullname; +} {vtabJ(1) this vtabJ(3) {a test} vtabJ(4) 4 vtabJ(five) 555 vtabJ(two) is} +do_test 111 { + set res {} + foreach vname [lsort [array names vtabJ]] { + lappend res vtabJ($vname) $vtabJ($vname) + } + set res +} {vtabJ(1) this vtabJ(3) {a test} vtabJ(4) 4 vtabJ(five) 555 vtabJ(two) is} + +do_test 120 { + db eval { + INSERT INTO tclvar(fullname, value) VALUES('vtabJ(4)',444); + } + set vtabJ(4) +} {444} + +do_test 130 { + db eval { + INSERT INTO tclvar(fullname, value) VALUES('vtabJ(4)',NULL); + } + info exists vtabJ(4) +} {0} + +do_test 140 { + db eval { + UPDATE tclvar SET value=55 WHERE fullname='vtabJ(five)'; + } + set vtabJ(five) +} {55} + +do_test 150 { + db eval { + UPDATE tclvar SET fullname='vtabJ(5)' WHERE fullname='vtabJ(five)'; + } + set vtabJ(5) +} {55} +do_test 151 { + info exists vtabJ(five) +} {0} +do_test 152 { + set res {} + foreach vname [lsort [array names vtabJ]] { + lappend res vtabJ($vname) $vtabJ($vname) + } + set res +} {vtabJ(1) this vtabJ(3) {a test} vtabJ(5) 55 vtabJ(two) is} + +do_execsql_test 160 { + SELECT fullname FROM tclvar WHERE arrayname='two' +} {vtabJ(two)} +do_execsql_test 161 { + DELETE FROM tclvar WHERE arrayname='two'; + SELECT fullname, value FROM tclvar WHERE name='vtabJ' ORDER BY fullname; +} {vtabJ(1) this vtabJ(3) {a test} vtabJ(5) 55} +do_test 162 { + set res {} + foreach vname [lsort [array names vtabJ]] { + lappend res vtabJ($vname) $vtabJ($vname) + } + set res +} {vtabJ(1) this vtabJ(3) {a test} vtabJ(5) 55} + +# Try to trick the module into updating the same variable twice for a +# single UPDATE statement. +# +do_execsql_test 171 { + INSERT INTO tclvar(fullname, value) VALUES('xx', 'a'); + SELECT name, value FROM tclvar where name = 'xx'; +} {xx a} +do_execsql_test 172 { + UPDATE tclvar SET value = value || 't' + WHERE name = 'xx' OR name = 'x'||'x'; + SELECT name, value FROM tclvar where name = 'xx'; +} {xx at} +do_execsql_test 173 { + UPDATE tclvar SET value = value || 't' + WHERE name = 'xx' OR name BETWEEN 'xx' AND 'xx'; + SELECT name, value FROM tclvar where name = 'xx'; +} {xx att} + +do_execsql_test 181 { + DELETE FROM tclvar WHERE name BETWEEN 'xx' AND 'xx' OR name='xx'; + SELECT name, value FROM tclvar where name = 'xx'; +} {} + + +finish_test Index: tool/speed-check.sh ================================================================== --- tool/speed-check.sh +++ tool/speed-check.sh @@ -145,12 +145,14 @@ fi size sqlite3.o | tee -a summary-$NAME.txt wc sqlite3.c if test $doCachegrind -eq 1; then cg_anno.tcl cachegrind.out.* >cout-$NAME.txt + echo '*****************************************************' >>cout-$NAME.txt + sed 's/^[0-9=-]\{9\}/==00000==/' summary-$NAME.txt >>cout-$NAME.txt fi if test $doExplain -eq 1; then ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt fi if test "$NAME" != "trunk"; then fossil test-diff --tk -c 20 cout-trunk.txt cout-$NAME.txt fi