Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -418,10 +418,11 @@ $(TOP)/ext/misc/eval.c \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/fts5/fts5_tcl.c \ $(TOP)/ext/fts5/fts5_test_mi.c \ + $(TOP)/ext/fts5/fts5_test_tok.c \ $(TOP)/ext/misc/ieee754.c \ $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/series.c \ Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -1085,10 +1085,11 @@ $(TOP)\ext\misc\eval.c \ $(TOP)\ext\misc\fileio.c \ $(TOP)\ext\misc\fuzzer.c \ $(TOP)\ext\fts5\fts5_tcl.c \ $(TOP)\ext\fts5\fts5_test_mi.c \ + $(TOP)\ext\fts5\fts5_test_tok.c \ $(TOP)\ext\misc\ieee754.c \ $(TOP)\ext\misc\nextchar.c \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\series.c \ @@ -1712,12 +1713,12 @@ $(LTCOMPILE) $(NO_WARN) -c fts5.c fts5.dll: fts5_ext.lo $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ fts5_ext.lo -sqlite3rbu.lo: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c +sqlite3rbu.lo: $(TOP)\ext\rbu\sqlite3rbu.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\rbu\sqlite3rbu.c # Rules to build the 'testfixture' application. # # If using the amalgamation, use sqlite3.c directly to build the test # fixture. Otherwise link against libsqlite3.lib. (This distinction is @@ -1838,12 +1839,12 @@ speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) - $(LTLINK) $(NO_WARN) -I. -DSQLITE_ENABLE_RBU -Fe$@ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) \ - $(LDFLAGS) $(LTLINKOPTS) + $(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) \ + /link $(LDFLAGS) $(LTLINKOPTS) clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL del /Q *.bsc *.cod *.da *.bb *.bbg gmon.out 2>NUL del /Q sqlite3.h opcodes.c opcodes.h 2>NUL Index: ext/fts5/fts5Int.h ================================================================== --- ext/fts5/fts5Int.h +++ ext/fts5/fts5Int.h @@ -223,17 +223,17 @@ ** Buffer object for the incremental building of string data. */ typedef struct Fts5Buffer Fts5Buffer; struct Fts5Buffer { u8 *p; - int n; - int nSpace; + u32 n; + u32 nSpace; }; -int sqlite3Fts5BufferSize(int*, Fts5Buffer*, int); +int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32); void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64); -void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*); +void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*); void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*); void sqlite3Fts5BufferFree(Fts5Buffer*); void sqlite3Fts5BufferZero(Fts5Buffer*); void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*); void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...); @@ -584,11 +584,11 @@ int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName); int sqlite3Fts5DropAll(Fts5Config*); int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **); -int sqlite3Fts5StorageDelete(Fts5Storage *p, i64); +int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**); int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*); int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64); int sqlite3Fts5StorageIntegrity(Fts5Storage *p); @@ -604,12 +604,10 @@ int sqlite3Fts5StorageConfigValue( Fts5Storage *p, const char*, sqlite3_value*, int ); -int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**); - int sqlite3Fts5StorageDeleteAll(Fts5Storage *p); int sqlite3Fts5StorageRebuild(Fts5Storage *p); int sqlite3Fts5StorageOptimize(Fts5Storage *p); int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge); Index: ext/fts5/fts5_buffer.c ================================================================== --- ext/fts5/fts5_buffer.c +++ ext/fts5/fts5_buffer.c @@ -13,12 +13,12 @@ #include "fts5Int.h" -int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, int nByte){ - int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64; +int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){ + u32 nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64; u8 *pNew; while( nNewp, nNew); @@ -59,14 +59,14 @@ ** is called, it is a no-op. */ void sqlite3Fts5BufferAppendBlob( int *pRc, Fts5Buffer *pBuf, - int nData, + u32 nData, const u8 *pData ){ - assert( *pRc || nData>=0 ); + assert_nc( *pRc || nData>=0 ); if( fts5BufferGrow(pRc, pBuf, nData) ) return; memcpy(&pBuf->p[pBuf->n], pData, nData); pBuf->n += nData; } @@ -320,18 +320,21 @@ ){ int rc = SQLITE_OK; *pbPresent = 0; if( p ){ int i; - int hash; + int hash = 13; Fts5TermsetEntry *pEntry; - /* Calculate a hash value for this term */ - hash = 104 + iIdx; - for(i=0; i=0; i--){ + hash = (hash << 3) ^ hash ^ pTerm[i]; } + hash = (hash << 3) ^ hash ^ iIdx; hash = hash % ArraySize(p->apHash); for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){ if( pEntry->iIdx==iIdx && pEntry->nTerm==nTerm Index: ext/fts5/fts5_config.c ================================================================== --- ext/fts5/fts5_config.c +++ ext/fts5/fts5_config.c @@ -276,11 +276,11 @@ while( p[0]>='0' && p[0]<='9' && nPre<1000 ){ nPre = nPre*10 + (p[0] - '0'); p++; } - if( rc==SQLITE_OK && (nPre<=0 || nPre>=1000) ){ + if( nPre<=0 || nPre>=1000 ){ *pzErr = sqlite3_mprintf("prefix length out of range (max 999)"); rc = SQLITE_ERROR; break; } Index: ext/fts5/fts5_expr.c ================================================================== --- ext/fts5/fts5_expr.c +++ ext/fts5/fts5_expr.c @@ -832,11 +832,11 @@ assert( pNode->eType==FTS5_TERM ); assert( pNear->nPhrase==1 && pPhrase->nTerm==1 ); assert( pPhrase->aTerm[0].pSynonym==0 ); rc = sqlite3Fts5IterPoslist(pIter, pColset, - (const u8**)&pPhrase->poslist.p, &pPhrase->poslist.n, &pNode->iRowid + (const u8**)&pPhrase->poslist.p, (int*)&pPhrase->poslist.n, &pNode->iRowid ); pNode->bNomatch = (pPhrase->poslist.n==0); return rc; } @@ -2394,11 +2394,11 @@ aPopulator[i].bOk = 1; } } return sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb + FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb ); } static void fts5ExprClearPoslists(Fts5ExprNode *pNode){ if( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING ){ @@ -2410,53 +2410,48 @@ } } } static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ - if( pNode ){ - pNode->iRowid = iRowid; - pNode->bEof = 0; - switch( pNode->eType ){ - case FTS5_TERM: - case FTS5_STRING: - return (pNode->pNear->apPhrase[0]->poslist.n>0); - - case FTS5_AND: { - int i; - for(i=0; inChild; i++){ - if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){ - fts5ExprClearPoslists(pNode); - return 0; - } - } - break; - } - - case FTS5_OR: { - int i; - int bRet = 0; - for(i=0; inChild; i++){ - if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){ - bRet = 1; - } - } - if( bRet==0 ){ - fts5ExprClearPoslists(pNode); - } - return bRet; - } - - default: { - assert( pNode->eType==FTS5_NOT ); - if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid) - || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid) - ){ - fts5ExprClearPoslists(pNode); - return 0; - } - break; - } + pNode->iRowid = iRowid; + pNode->bEof = 0; + switch( pNode->eType ){ + case FTS5_TERM: + case FTS5_STRING: + return (pNode->pNear->apPhrase[0]->poslist.n>0); + + case FTS5_AND: { + int i; + for(i=0; inChild; i++){ + if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){ + fts5ExprClearPoslists(pNode); + return 0; + } + } + break; + } + + case FTS5_OR: { + int i; + int bRet = 0; + for(i=0; inChild; i++){ + if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){ + bRet = 1; + } + } + return bRet; + } + + default: { + assert( pNode->eType==FTS5_NOT ); + if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid) + || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid) + ){ + fts5ExprClearPoslists(pNode); + return 0; + } + break; } } return 1; } Index: ext/fts5/fts5_index.c ================================================================== --- ext/fts5/fts5_index.c +++ ext/fts5/fts5_index.c @@ -1873,17 +1873,19 @@ ){ Fts5Data *pLeaf = pIter->pLeaf; int iOff; int bNewTerm = 0; int nKeep = 0; + u8 *a; + int n; assert( pbNewTerm==0 || *pbNewTerm==0 ); assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); /* Search for the end of the position list within the current page. */ - u8 *a = pLeaf->p; - int n = pLeaf->szLeaf; + a = pLeaf->p; + n = pLeaf->szLeaf; ASSERT_SZLEAF_OK(pLeaf); iOff = pIter->iLeafOffset + pIter->nPos; if( iOffdb, zSql, -1, &pRet, 0); - if( rc!=SQLITE_OK ){ - *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db)); - } - sqlite3_free(zSql); - } - *pRc = rc; + zSql = sqlite3_vmprintf(zFmt, ap); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0); + if( rc!=SQLITE_OK ){ + *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db)); + } + sqlite3_free(zSql); } va_end(ap); - return pRet; + *ppStmt = pRet; + return rc; } static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){ Fts5Config *pConfig = pTab->pConfig; Fts5Sorter *pSorter; int nPhrase; int nByte; - int rc = SQLITE_OK; + int rc; const char *zRank = pCsr->zRank; const char *zRankArgs = pCsr->zRankArgs; nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1); @@ -894,11 +893,11 @@ ** is not possible as SQLite reference counts the virtual table objects. ** And since the statement required here reads from this very virtual ** table, saving it creates a circular reference. ** ** If SQLite a built-in statement cache, this wouldn't be a problem. */ - pSorter->pStmt = fts5PrepareStatement(&rc, pConfig, + rc = fts5PrepareStatement(&pSorter->pStmt, pConfig, "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s", pConfig->zDb, pConfig->zName, zRank, pConfig->zName, (zRankArgs ? ", " : ""), (zRankArgs ? zRankArgs : ""), bDesc ? "DESC" : "ASC" @@ -1403,11 +1402,11 @@ ){ int rc = SQLITE_OK; int eType1 = sqlite3_value_type(apVal[1]); if( eType1==SQLITE_INTEGER ){ sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]); - rc = sqlite3Fts5StorageSpecialDelete(pTab->pStorage, iDel, &apVal[2]); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2]); } return rc; } static void fts5StorageInsert( @@ -1510,21 +1509,21 @@ } /* Case 1: DELETE */ else if( nArg==1 ){ i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0); } /* Case 2: INSERT */ else if( eType0!=SQLITE_INTEGER ){ /* If this is a REPLACE, first remove the current entry (if any) */ if( eConflict==SQLITE_REPLACE && sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); } fts5StorageInsert(&rc, pTab, apVal, pRowid); } /* Case 2: UPDATE */ @@ -1531,26 +1530,26 @@ else{ i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */ i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */ if( iOld!=iNew ){ if( eConflict==SQLITE_REPLACE ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); } fts5StorageInsert(&rc, pTab, apVal, pRowid); }else{ rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *pRowid); } } }else{ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); fts5StorageInsert(&rc, pTab, apVal, pRowid); } } } @@ -1745,11 +1744,13 @@ /* Initialize all iterators */ for(i=0; ibase.pVtab))->pConfig; if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + Fts5Sorter *pSorter = pCsr->pSorter; int n; - rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n); + if( pSorter ){ + int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); + n = pSorter->aIdx[iPhrase] - i1; + pIter->a = &pSorter->aPoslist[i1]; + }else{ + rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n); + } if( rc==SQLITE_OK ){ pIter->b = &pIter->a[n]; *piCol = 0; fts5ApiPhraseNextColumn(pCtx, pIter, piCol); } Index: ext/fts5/fts5_storage.c ================================================================== --- ext/fts5/fts5_storage.c +++ ext/fts5/fts5_storage.c @@ -376,43 +376,56 @@ /* ** If a row with rowid iDel is present in the %_content table, add the ** delete-markers to the FTS index necessary to delete it. Do not actually ** remove the %_content row at this time though. */ -static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){ +static int fts5StorageDeleteFromIndex( + Fts5Storage *p, + i64 iDel, + sqlite3_value **apVal +){ Fts5Config *pConfig = p->pConfig; - sqlite3_stmt *pSeek; /* SELECT to read row iDel from %_data */ + sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */ int rc; /* Return code */ + int rc2; /* sqlite3_reset() return code */ + int iCol; + Fts5InsertCtx ctx; - rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0); - if( rc==SQLITE_OK ){ - int rc2; + if( apVal==0 ){ + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0); + if( rc!=SQLITE_OK ) return rc; sqlite3_bind_int64(pSeek, 1, iDel); - if( sqlite3_step(pSeek)==SQLITE_ROW ){ - int iCol; - Fts5InsertCtx ctx; - ctx.pStorage = p; - ctx.iCol = -1; - rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); - for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ - if( pConfig->abUnindexed[iCol-1] ) continue; - ctx.szCol = 0; - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - (const char*)sqlite3_column_text(pSeek, iCol), - sqlite3_column_bytes(pSeek, iCol), - (void*)&ctx, - fts5StorageInsertCallback - ); - p->aTotalSize[iCol-1] -= (i64)ctx.szCol; - } - p->nTotalRow--; - } - rc2 = sqlite3_reset(pSeek); - if( rc==SQLITE_OK ) rc = rc2; - } - + if( sqlite3_step(pSeek)!=SQLITE_ROW ){ + return sqlite3_reset(pSeek); + } + } + + ctx.pStorage = p; + ctx.iCol = -1; + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); + for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ + if( pConfig->abUnindexed[iCol-1]==0 ){ + const char *zText; + int nText; + if( pSeek ){ + zText = (const char*)sqlite3_column_text(pSeek, iCol); + nText = sqlite3_column_bytes(pSeek, iCol); + }else{ + zText = (const char*)sqlite3_value_text(apVal[iCol-1]); + nText = sqlite3_value_bytes(apVal[iCol-1]); + } + ctx.szCol = 0; + rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, + zText, nText, (void*)&ctx, fts5StorageInsertCallback + ); + p->aTotalSize[iCol-1] -= (i64)ctx.szCol; + } + } + p->nTotalRow--; + + rc2 = sqlite3_reset(pSeek); + if( rc==SQLITE_OK ) rc = rc2; return rc; } /* @@ -488,20 +501,21 @@ } /* ** Remove a row from the FTS table. */ -int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){ +int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ Fts5Config *pConfig = p->pConfig; int rc; sqlite3_stmt *pDel = 0; + assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 ); rc = fts5StorageLoadTotals(p, 1); /* Delete the index records */ if( rc==SQLITE_OK ){ - rc = fts5StorageDeleteFromIndex(p, iDel); + rc = fts5StorageDeleteFromIndex(p, iDel, apVal); } /* Delete the %_docsize record */ if( rc==SQLITE_OK && pConfig->bColumnsize ){ rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0); @@ -515,65 +529,10 @@ /* Delete the %_content record */ if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ if( rc==SQLITE_OK ){ rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0); } - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pDel, 1, iDel); - sqlite3_step(pDel); - rc = sqlite3_reset(pDel); - } - } - - /* Write the averages record */ - if( rc==SQLITE_OK ){ - rc = fts5StorageSaveTotals(p); - } - - return rc; -} - -int sqlite3Fts5StorageSpecialDelete( - Fts5Storage *p, - i64 iDel, - sqlite3_value **apVal -){ - Fts5Config *pConfig = p->pConfig; - int rc; - sqlite3_stmt *pDel = 0; - - assert( pConfig->eContent!=FTS5_CONTENT_NORMAL ); - rc = fts5StorageLoadTotals(p, 1); - - /* Delete the index records */ - if( rc==SQLITE_OK ){ - int iCol; - Fts5InsertCtx ctx; - ctx.pStorage = p; - ctx.iCol = -1; - - rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); - for(iCol=0; rc==SQLITE_OK && iColnCol; iCol++){ - if( pConfig->abUnindexed[iCol] ) continue; - ctx.szCol = 0; - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - (const char*)sqlite3_value_text(apVal[iCol]), - sqlite3_value_bytes(apVal[iCol]), - (void*)&ctx, - fts5StorageInsertCallback - ); - p->aTotalSize[iCol] -= (i64)ctx.szCol; - } - p->nTotalRow--; - } - - /* Delete the %_docsize record */ - if( pConfig->bColumnsize ){ - if( rc==SQLITE_OK ){ - rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0); - } if( rc==SQLITE_OK ){ sqlite3_bind_int64(pDel, 1, iDel); sqlite3_step(pDel); rc = sqlite3_reset(pDel); } Index: ext/fts5/fts5_tcl.c ================================================================== --- ext/fts5/fts5_tcl.c +++ ext/fts5/fts5_tcl.c @@ -21,11 +21,12 @@ #include "fts5.h" #include #include extern int sqlite3_fts5_may_be_corrupt; -extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *); +extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*); +extern int sqlite3Fts5TestRegisterTok(sqlite3*, fts5_api*); /************************************************************************* ** This is a copy of the first part of the SqliteDb structure in ** tclsqlite.c. We need it here so that the get_sqlite_pointer routine ** can extract the sqlite3* pointer from an existing Tcl SQLite @@ -444,14 +445,16 @@ if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; zColvar = Tcl_GetString(objv[3]); zOffvar = Tcl_GetString(objv[4]); - for(p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff); - iCol>=0; - p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) - ){ + rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + return TCL_ERROR; + } + for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){ Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0); Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0); rc = Tcl_EvalObjEx(interp, pScript, 0); if( rc==TCL_CONTINUE ) rc = TCL_OK; if( rc!=TCL_OK ){ @@ -471,14 +474,16 @@ Fts5PhraseIter iter; if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; zColvar = Tcl_GetString(objv[3]); - for(p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol); - iCol>=0; - p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol) - ){ + rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + return TCL_ERROR; + } + for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){ Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0); rc = Tcl_EvalObjEx(interp, pScript, 0); if( rc==TCL_CONTINUE ) rc = TCL_OK; if( rc!=TCL_OK ){ if( rc==TCL_BREAK ) rc = TCL_OK; @@ -1075,10 +1080,36 @@ Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_ERROR; } return TCL_OK; } + +static int f5tRegisterTok( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; + sqlite3 *db = 0; + fts5_api *pApi = 0; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){ + return TCL_ERROR; + } + + rc = sqlite3Fts5TestRegisterTok(db, pApi); + if( rc!=SQLITE_OK ){ + Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); + return TCL_ERROR; + } + return TCL_OK; +} /* ** Entry point. */ int Fts5tcl_Init(Tcl_Interp *interp){ @@ -1091,11 +1122,12 @@ { "sqlite3_fts5_token", f5tTokenizerReturn, 1 }, { "sqlite3_fts5_tokenize", f5tTokenize, 0 }, { "sqlite3_fts5_create_function", f5tCreateFunction, 0 }, { "sqlite3_fts5_may_be_corrupt", f5tMayBeCorrupt, 0 }, { "sqlite3_fts5_token_hash", f5tTokenHash, 0 }, - { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 } + { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 }, + { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 } }; int i; F5tTokenizerContext *pContext; pContext = (F5tTokenizerContext*)ckalloc(sizeof(F5tTokenizerContext)); ADDED ext/fts5/fts5_test_tok.c Index: ext/fts5/fts5_test_tok.c ================================================================== --- /dev/null +++ ext/fts5/fts5_test_tok.c @@ -0,0 +1,482 @@ +/* +** 2013 Apr 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code for the "fts5tokenize" virtual table module. +** An fts5tokenize virtual table is created as follows: +** +** CREATE VIRTUAL TABLE USING fts5tokenize( +** , , ... +** ); +** +** The table created has the following schema: +** +** CREATE TABLE (input HIDDEN, token, start, end, position) +** +** When queried, the query must include a WHERE clause of type: +** +** input = +** +** The virtual table module tokenizes this , using the FTS3 +** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE +** statement and returns one row for each token in the result. With +** fields set as follows: +** +** input: Always set to a copy of +** token: A token from the input. +** start: Byte offset of the token within the input . +** end: Byte offset of the byte immediately following the end of the +** token within the input string. +** pos: Token offset of token within input. +** +*/ +#if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5) + +#include +#include +#include + +typedef struct Fts5tokTable Fts5tokTable; +typedef struct Fts5tokCursor Fts5tokCursor; +typedef struct Fts5tokRow Fts5tokRow; + +/* +** Virtual table structure. +*/ +struct Fts5tokTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + fts5_tokenizer tok; /* Tokenizer functions */ + Fts5Tokenizer *pTok; /* Tokenizer instance */ +}; + +/* +** A container for a rows values. +*/ +struct Fts5tokRow { + char *zToken; + int iStart; + int iEnd; + int iPos; +}; + +/* +** Virtual table cursor structure. +*/ +struct Fts5tokCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + int iRowid; /* Current 'rowid' value */ + char *zInput; /* Input string */ + int nRow; /* Number of entries in aRow[] */ + Fts5tokRow *aRow; /* Array of rows to return */ +}; + +static void fts5tokDequote(char *z){ + char q = z[0]; + + if( q=='[' || q=='\'' || q=='"' || q=='`' ){ + int iIn = 1; + int iOut = 0; + if( q=='[' ) q = ']'; + + while( z[iIn] ){ + if( z[iIn]==q ){ + if( z[iIn+1]!=q ){ + /* Character iIn was the close quote. */ + iIn++; + break; + }else{ + /* Character iIn and iIn+1 form an escaped quote character. Skip + ** the input cursor past both and copy a single quote character + ** to the output buffer. */ + iIn += 2; + z[iOut++] = q; + } + }else{ + z[iOut++] = z[iIn++]; + } + } + + z[iOut] = '\0'; + } +} + +/* +** The second argument, argv[], is an array of pointers to nul-terminated +** strings. This function makes a copy of the array and strings into a +** single block of memory. It then dequotes any of the strings that appear +** to be quoted. +** +** If successful, output parameter *pazDequote is set to point at the +** array of dequoted strings and SQLITE_OK is returned. The caller is +** responsible for eventually calling sqlite3_free() to free the array +** in this case. Or, if an error occurs, an SQLite error code is returned. +** The final value of *pazDequote is undefined in this case. +*/ +static int fts5tokDequoteArray( + int argc, /* Number of elements in argv[] */ + const char * const *argv, /* Input array */ + char ***pazDequote /* Output array */ +){ + int rc = SQLITE_OK; /* Return code */ + if( argc==0 ){ + *pazDequote = 0; + }else{ + int i; + int nByte = 0; + char **azDequote; + + for(i=0; i0 ){ + zModule = azDequote[0]; + } + + rc = pApi->xFindTokenizer(pApi, zModule, &pTokCtx, &pTab->tok); + if( rc==SQLITE_OK ){ + const char **azArg = (const char **)&azDequote[1]; + int nArg = nDequote>0 ? nDequote-1 : 0; + rc = pTab->tok.xCreate(pTokCtx, azArg, nArg, &pTab->pTok); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(pTab); + pTab = 0; + } + + *ppVtab = (sqlite3_vtab*)pTab; + sqlite3_free(azDequote); + return rc; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts5tokDisconnectMethod(sqlite3_vtab *pVtab){ + Fts5tokTable *pTab = (Fts5tokTable *)pVtab; + if( pTab->pTok ){ + pTab->tok.xDelete(pTab->pTok); + } + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts5tokBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable + && pInfo->aConstraint[i].iColumn==0 + && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + pInfo->idxNum = 1; + pInfo->aConstraintUsage[i].argvIndex = 1; + pInfo->aConstraintUsage[i].omit = 1; + pInfo->estimatedCost = 1; + return SQLITE_OK; + } + } + + pInfo->idxNum = 0; + assert( pInfo->estimatedCost>1000000.0 ); + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts5tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts5tokCursor *pCsr; + + pCsr = (Fts5tokCursor *)sqlite3_malloc(sizeof(Fts5tokCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts5tokCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Reset the tokenizer cursor passed as the only argument. As if it had +** just been returned by fts5tokOpenMethod(). +*/ +static void fts5tokResetCursor(Fts5tokCursor *pCsr){ + int i; + for(i=0; inRow; i++){ + sqlite3_free(pCsr->aRow[i].zToken); + } + sqlite3_free(pCsr->zInput); + sqlite3_free(pCsr->aRow); + pCsr->zInput = 0; + pCsr->aRow = 0; + pCsr->nRow = 0; + pCsr->iRowid = 0; +} + +/* +** xClose - Close a cursor. +*/ +static int fts5tokCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor; + fts5tokResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts5tokNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor; + pCsr->iRowid++; + return SQLITE_OK; +} + +static int fts5tokCb( + void *pCtx, /* Pointer to Fts5tokCursor */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ +){ + Fts5tokCursor *pCsr = (Fts5tokCursor*)pCtx; + Fts5tokRow *pRow; + + if( (pCsr->nRow & (pCsr->nRow-1))==0 ){ + int nNew = pCsr->nRow ? pCsr->nRow*2 : 32; + Fts5tokRow *aNew; + aNew = (Fts5tokRow*)sqlite3_realloc(pCsr->aRow, nNew*sizeof(Fts5tokRow)); + if( aNew==0 ) return SQLITE_NOMEM; + memset(&aNew[pCsr->nRow], 0, sizeof(Fts5tokRow)*(nNew-pCsr->nRow)); + pCsr->aRow = aNew; + } + + pRow = &pCsr->aRow[pCsr->nRow]; + pRow->iStart = iStart; + pRow->iEnd = iEnd; + if( pCsr->nRow ){ + pRow->iPos = pRow[-1].iPos + ((tflags & FTS5_TOKEN_COLOCATED) ? 0 : 1); + } + pRow->zToken = sqlite3_malloc(nToken+1); + if( pRow->zToken==0 ) return SQLITE_NOMEM; + memcpy(pRow->zToken, pToken, nToken); + pRow->zToken[nToken] = 0; + pCsr->nRow++; + + return SQLITE_OK; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts5tokFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_ERROR; + Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor; + Fts5tokTable *pTab = (Fts5tokTable *)(pCursor->pVtab); + + fts5tokResetCursor(pCsr); + if( idxNum==1 ){ + const char *zByte = (const char *)sqlite3_value_text(apVal[0]); + int nByte = sqlite3_value_bytes(apVal[0]); + pCsr->zInput = sqlite3_malloc(nByte+1); + if( pCsr->zInput==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCsr->zInput, zByte, nByte); + pCsr->zInput[nByte] = 0; + rc = pTab->tok.xTokenize( + pTab->pTok, (void*)pCsr, 0, zByte, nByte, fts5tokCb + ); + } + } + + if( rc!=SQLITE_OK ) return rc; + return fts5tokNextMethod(pCursor); +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts5tokEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor; + return (pCsr->iRowid>pCsr->nRow); +} + +/* +** xColumn - Return a column value. +*/ +static int fts5tokColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor; + Fts5tokRow *pRow = &pCsr->aRow[pCsr->iRowid-1]; + + /* CREATE TABLE x(input, token, start, end, position) */ + switch( iCol ){ + case 0: + sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_text(pCtx, pRow->zToken, -1, SQLITE_TRANSIENT); + break; + case 2: + sqlite3_result_int(pCtx, pRow->iStart); + break; + case 3: + sqlite3_result_int(pCtx, pRow->iEnd); + break; + default: + assert( iCol==4 ); + sqlite3_result_int(pCtx, pRow->iPos); + break; + } + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts5tokRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor; + *pRowid = (sqlite3_int64)pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts5tok module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){ + static const sqlite3_module fts5tok_module = { + 0, /* iVersion */ + fts5tokConnectMethod, /* xCreate */ + fts5tokConnectMethod, /* xConnect */ + fts5tokBestIndexMethod, /* xBestIndex */ + fts5tokDisconnectMethod, /* xDisconnect */ + fts5tokDisconnectMethod, /* xDestroy */ + fts5tokOpenMethod, /* xOpen */ + fts5tokCloseMethod, /* xClose */ + fts5tokFilterMethod, /* xFilter */ + fts5tokNextMethod, /* xNext */ + fts5tokEofMethod, /* xEof */ + fts5tokColumnMethod, /* xColumn */ + fts5tokRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts5tokenize", &fts5tok_module, (void*)pApi); + return rc; +} + +#endif /* defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5) */ Index: ext/fts5/fts5_vocab.c ================================================================== --- ext/fts5/fts5_vocab.c +++ ext/fts5/fts5_vocab.c @@ -425,10 +425,14 @@ int iCol = -1; while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ int ii = FTS5_POS2COLUMN(iPos); pCsr->aCnt[ii]++; if( iCol!=ii ){ + if( ii>=nCol ){ + rc = FTS5_CORRUPT; + break; + } pCsr->aDoc[ii]++; iCol = ii; } } } @@ -442,11 +446,15 @@ Fts5Buffer buf = {0, 0, 0}; rc = sqlite3Fts5IterPoslistBuffer(pCsr->pIter, &buf); if( rc==SQLITE_OK ){ while( 0==sqlite3Fts5PoslistNext64(buf.p, buf.n, &iOff,&iPos) ){ assert_nc( iPos>=0 && iPosaDoc[iPos]++; + if( iPos>=nCol ){ + rc = FTS5_CORRUPT; + break; + } + pCsr->aDoc[iPos]++; } } sqlite3Fts5BufferFree(&buf); } break; @@ -470,11 +478,11 @@ } } } } - if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){ + if( rc==SQLITE_OK && pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){ while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++; assert( pCsr->iColpConfig->nCol ); } return rc; } Index: ext/fts5/test/fts5_common.tcl ================================================================== --- ext/fts5/test/fts5_common.tcl +++ ext/fts5/test/fts5_common.tcl @@ -12,15 +12,26 @@ if {![info exists testdir]} { set testdir [file join [file dirname [info script]] .. .. .. test] } source $testdir/tester.tcl + +ifcapable !fts5 { + finish_test + return +} catch { sqlite3_fts5_may_be_corrupt 0 reset_db } + +# If SQLITE_ENABLE_FTS5 is not defined, skip this test. +ifcapable !fts5 { + finish_test + return +} proc fts5_test_poslist {cmd} { set res [list] for {set i 0} {$i < [$cmd xInstCount]} {incr i} { lappend res [string map {{ } .} [$cmd xInst $i]] @@ -507,10 +518,26 @@ foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] { lappend res $rowid $poslist } set res } + +proc fts5_collist_data {expr tbl {order ASC} {aDictVar ""}} { + set res [list] + + if {$aDictVar!=""} { + upvar $aDictVar aDict + set dict aDict + } else { + set dict "" + } + + foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] { + lappend res $rowid $collist + } + set res +} #------------------------------------------------------------------------- # # This command will only work inside a [foreach_detail_mode] block. It tests @@ -559,5 +586,56 @@ set ::expr_not_ok 1 } list } + +#------------------------------------------------------------------------- +# Code for a simple Tcl tokenizer that supports synonyms at query time. +# +proc tclnum_tokenize {mode tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags == $mode && [info exists ::tclnum_syn($w)]} { + foreach s $::tclnum_syn($w) { sqlite3_fts5_token -colo $s $iStart $iEnd } + } + } +} + +proc tclnum_create {args} { + set mode query + if {[llength $args]} { + set mode [lindex $args 0] + } + if {$mode != "query" && $mode != "document"} { error "bad mode: $mode" } + return [list tclnum_tokenize $mode] +} + +proc fts5_tclnum_register {db} { + foreach SYNDICT { + {zero 0} + {one 1 i} + {two 2 ii} + {three 3 iii} + {four 4 iv} + {five 5 v} + {six 6 vi} + {seven 7 vii} + {eight 8 viii} + {nine 9 ix} + + {a1 a2 a3 a4 a5 a6 a7 a8 a9} + {b1 b2 b3 b4 b5 b6 b7 b8 b9} + {c1 c2 c3 c4 c5 c6 c7 c8 c9} + } { + foreach s $SYNDICT { + set o [list] + foreach x $SYNDICT {if {$x!=$s} {lappend o $x}} + set ::tclnum_syn($s) $o + } + } + sqlite3_fts5_create_tokenizer db tclnum tclnum_create +} +# +# End of tokenizer code. +#------------------------------------------------------------------------- + ADDED ext/fts5/test/fts5bigtok.test Index: ext/fts5/test/fts5bigtok.test ================================================================== --- /dev/null +++ ext/fts5/test/fts5bigtok.test @@ -0,0 +1,67 @@ +# 2016 Jan 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5bigtok + +proc rndterm {} { + set L [list a b c d e f g h i j k l m n o p q r s t u v w x y z] + set l [lindex $L [expr int(rand() * [llength $L])]] + string repeat $l [expr int(rand() * 5) + 60] +} + +proc rnddoc {n} { + set res [list] + for {set i 0} {$i < $n} {incr i} { + lappend res [rndterm] + } + set res +} + +foreach_detail_mode $::testprefix { + db func rnddoc rnddoc + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + CREATE VIRTUAL TABLE t1vocab USING fts5vocab(t1, row); + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 ) + INSERT INTO t1 SELECT rnddoc(3) FROM s; + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 ) + INSERT INTO t1 SELECT rnddoc(3) FROM s; + } + + foreach v [db eval {SELECT term FROM t1vocab}] { + set res [db eval {SELECT rowid FROM t1($v)}] + do_execsql_test 1.[string range $v 0 0] { + SELECT rowid FROM t1($v) ORDER BY rowid DESC + } [lsort -integer -decr $res] + } + + do_execsql_test 2.0 { + INSERT INTO t1(t1) VALUES('optimize'); + } + + foreach v [db eval {SELECT term FROM t1vocab}] { + set res [db eval {SELECT rowid FROM t1($v)}] + do_execsql_test 2.[string range $v 0 0] { + SELECT rowid FROM t1($v) ORDER BY rowid DESC + } [lsort -integer -decr $res] + } +} + +finish_test + + Index: ext/fts5/test/fts5config.test ================================================================== --- ext/fts5/test/fts5config.test +++ ext/fts5/test/fts5config.test @@ -42,10 +42,12 @@ 1 {prefix=x} 2 {prefix='x'} 3 {prefix='$'} 4 {prefix='1,2,'} 5 {prefix=',1'} + 6 {prefix='1,2,3...'} + 7 {prefix='1,2,3xyz'} } { set res [list 1 {malformed prefix=... directive}] do_catchsql_test 2.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res } @@ -157,10 +159,12 @@ # Errors in: # # 9.1.* 'pgsz' options. # 9.2.* 'automerge' options. # 9.3.* 'crisismerge' options. +# 9.4.* a non-existant option. +# 9.5.* 'hashsize' options. # do_execsql_test 9.0 { CREATE VIRTUAL TABLE abc USING fts5(a, b); } {} do_catchsql_test 9.1.1 { @@ -200,10 +204,20 @@ } {} do_catchsql_test 9.4.1 { INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1); } {1 {SQL logic error or missing database}} + +do_catchsql_test 9.5.1 { + INSERT INTO abc(abc, rank) VALUES('hashsize', 'not an integer'); +} {1 {SQL logic error or missing database}} +do_catchsql_test 9.5.2 { + INSERT INTO abc(abc, rank) VALUES('hashsize', -500000); +} {1 {SQL logic error or missing database}} +do_catchsql_test 9.5.3 { + INSERT INTO abc(abc, rank) VALUES('hashsize', 500000); +} {0 {}} #------------------------------------------------------------------------- # Too many prefix indexes. Maximum allowed is 31. # foreach {tn spec} { @@ -211,8 +225,23 @@ 2 {prefix="1 2 3 4", prefix="5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32"} } { set sql "CREATE VIRTUAL TABLE xyz USING fts5(x, $spec)" do_catchsql_test 10.$tn $sql {1 {too many prefix indexes (max 31)}} } + +#------------------------------------------------------------------------- +# errors in the detail= option. +# +foreach {tn opt} { + 1 {detail=x} + 2 {detail='x'} + 3 {detail='$'} + 4 {detail='1,2,'} + 5 {detail=',1'} + 6 {detail=''} +} { + set res [list 1 {malformed detail=... directive}] + do_catchsql_test 11.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res +} finish_test Index: ext/fts5/test/fts5fault4.test ================================================================== --- ext/fts5/test/fts5fault4.test +++ ext/fts5/test/fts5fault4.test @@ -38,31 +38,10 @@ execsql { DROP TABLE xx } } -test { faultsim_test_result [list 0 {}] } -#------------------------------------------------------------------------- -# An OOM within an "ORDER BY rank" query. -# -db func rnddoc fts5_rnddoc -do_execsql_test 2.0 { - CREATE VIRTUAL TABLE xx USING fts5(x); - INSERT INTO xx VALUES ('abc ' || rnddoc(10)); - INSERT INTO xx VALUES ('abc abc' || rnddoc(9)); - INSERT INTO xx VALUES ('abc abc abc' || rnddoc(8)); -} {} -faultsim_save_and_close - -do_faultsim_test 2 -faults oom-* -prep { - faultsim_restore_and_reopen - execsql { SELECT * FROM xx } -} -body { - execsql { SELECT rowid FROM xx WHERE xx MATCH 'abc' ORDER BY rank } -} -test { - faultsim_test_result [list 0 {3 2 1}] -} - #------------------------------------------------------------------------- # An OOM while "reseeking" an FTS cursor. # do_execsql_test 3.0 { CREATE VIRTUAL TABLE jj USING fts5(j); Index: ext/fts5/test/fts5fault5.test ================================================================== --- ext/fts5/test/fts5fault5.test +++ ext/fts5/test/fts5fault5.test @@ -63,23 +63,30 @@ } -test { faultsim_test_result {0 {}} } #------------------------------------------------------------------------- -# OOM while scanning an fts5vocab table. +# OOM while scanning fts5vocab tables. # reset_db do_test 3.0 { execsql { CREATE VIRTUAL TABLE tt USING fts5(x); CREATE VIRTUAL TABLE tv USING fts5vocab(tt, 'row'); + + CREATE VIRTUAL TABLE tt2 USING fts5(x, detail=col); + CREATE VIRTUAL TABLE tv2 USING fts5vocab(tt2, 'col'); + INSERT INTO tt(tt, rank) VALUES('pgsz', 32); + INSERT INTO tt2(tt2, rank) VALUES('pgsz', 32); BEGIN; } + for {set i 0} {$i < 20} {incr i} { set str [string repeat "$i " 50] execsql { INSERT INTO tt VALUES($str) } + execsql { INSERT INTO tt2 VALUES($str) } } execsql COMMIT } {} do_faultsim_test 3.1 -faults oom-t* -body { @@ -95,10 +102,32 @@ SELECT term FROM tv WHERE term BETWEEN '1' AND '2'; } } -test { faultsim_test_result {0 {1 10 11 12 13 14 15 16 17 18 19 2}} } + +breakpoint +do_execsql_test 3.3.0 { + SELECT * FROM tv2; +} { + 0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {} + 14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19 x 1 {} + 2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {} + 9 x 1 {} +} +do_faultsim_test 3.3 -faults oom-t* -body { + db eval { + SELECT * FROM tv2; + } +} -test { + faultsim_test_result [list 0 [list \ + 0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {} \ + 14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19 x 1 {} \ + 2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {} \ + 9 x 1 {} + ]] +} finish_test Index: ext/fts5/test/fts5fault8.test ================================================================== --- ext/fts5/test/fts5fault8.test +++ ext/fts5/test/fts5fault8.test @@ -22,28 +22,34 @@ return } foreach_detail_mode $testprefix { -if {[detail_is_none]==0} continue - fts5_aux_test_functions db do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1 VALUES('a b c d', '1 2 3 4'); INSERT INTO t1 VALUES('a b a b', NULL); INSERT INTO t1 VALUES(NULL, '1 2 1 2'); } -do_faultsim_test 1 -faults oom-t* -body { +do_faultsim_test 1 -faults oom-* -body { execsql { SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2' } } -test { faultsim_test_result {0 {1 {0.0.1 1.1.1} 2 {0.0.1 0.0.3} 3 {1.1.1 1.1.3}}} \ {1 SQLITE_NOMEM} } + +do_faultsim_test 2 -faults oom-* -body { + execsql { + INSERT INTO t1(t1) VALUES('integrity-check'); + } +} -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} +} } finish_test ADDED ext/fts5/test/fts5fault9.test Index: ext/fts5/test/fts5fault9.test ================================================================== --- /dev/null +++ ext/fts5/test/fts5fault9.test @@ -0,0 +1,156 @@ +# 2015 September 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5fault9 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + +fts5_aux_test_functions db + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + WITH seq(s) AS ( SELECT 1 UNION ALL SELECT s+1 FROM seq WHERE s<50) + INSERT INTO t1 SELECT 'x x x y y y', 'a b c d e f' FROM seq; +} + +do_faultsim_test 1 -faults oom-* -body { + execsql { SELECT count(*) FROM t1('x AND y') } +} -test { + faultsim_test_result {0 50} +} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t2 USING fts5(a, b, detail=%DETAIL%); + INSERT INTO t2(t2, rank) VALUES('pgsz', 32); + INSERT INTO t2 VALUES('abc cba', 'cba abc'); + INSERT INTO t2 VALUES('abc cba', 'cba abc'); + INSERT INTO t2 VALUES('abc cba', 'cba abc'); + + INSERT INTO t2 VALUES('axy cyx', 'cyx axy'); + INSERT INTO t2 VALUES('axy cyx', 'cyx axy'); + INSERT INTO t2 VALUES('axy cyx', 'cyx axy'); +} + +do_faultsim_test 2 -faults oom-* -body { + execsql { SELECT count(*) FROM t2('a* AND c*') } +} -test { + faultsim_test_result {0 6} +} + + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t3 USING fts5(a, detail=%DETAIL%); + INSERT INTO t3 VALUES('a x x a x a a a'); + INSERT INTO t3 VALUES('x a a x a x x x'); +} + +do_faultsim_test 3.1 -faults oom-* -body { + execsql { SELECT highlight(t3, 0, '[', ']') FROM t3('a') } +} -test { + faultsim_test_result {0 {{[a] x x [a] x [a] [a] [a]} {x [a] [a] x [a] x x x}}} +} + +do_faultsim_test 3.2 -faults oom-t* -body { + execsql { SELECT fts5_test_poslist2(t3) FROM t3('x') } +} -test { + faultsim_test_result \ + {0 {{0.0.1 0.0.2 0.0.4} {0.0.0 0.0.3 0.0.5 0.0.6 0.0.7}}} \ + {1 SQLITE_NOMEM} +} + +#------------------------------------------------------------------------- +# Test OOM injection with the xPhraseFirstColumn() API and a tokenizer +# uses query synonyms. +# +fts5_tclnum_register db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t4 USING fts5(x, y, z, detail=%DETAIL%, tokenize=tclnum); + INSERT INTO t4 VALUES('one two three', '1 2 3', 'i ii iii'); + INSERT INTO t4 VALUES('1 2 3', 'i ii iii', 'one two three'); + INSERT INTO t4 VALUES('i ii iii', 'one two three', 'i ii iii'); + + INSERT INTO t4 VALUES('a1 a2 a3', 'a4 a5 a6', 'a7 a8 a9'); + INSERT INTO t4 VALUES('b1 b2 b3', 'b4 b5 b6', 'b7 b8 b9'); + INSERT INTO t4 VALUES('c1 c2 c3', 'c4 c5 c6', 'c7 c8 c9'); +} + +do_faultsim_test 4.1 -faults oom-t* -body { + execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('2') } +} -test { + faultsim_test_result \ + {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} {1 SQLITE_NOMEM} +} + +do_faultsim_test 4.2 -faults oom-t* -body { + execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('a5 OR b5 OR c5') } +} -test { + faultsim_test_result \ + {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} {1 SQLITE_NOMEM} +} + + +#------------------------------------------------------------------------- +# An OOM within an "ORDER BY rank" query. +# +db func rnddoc fts5_rnddoc +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=%DETAIL%); + INSERT INTO xx VALUES ('def', 'abc ' || rnddoc(10)); + INSERT INTO xx VALUES ('def', 'abc abc' || rnddoc(9)); + INSERT INTO xx VALUES ('def', 'abc abc abc' || rnddoc(8)); +} {} +faultsim_save_and_close + +do_faultsim_test 5 -faults oom-* -prep { + faultsim_restore_and_reopen + execsql { SELECT * FROM xx } +} -body { + execsql { SELECT rowid FROM xx('abc AND def') ORDER BY rank } +} -test { + faultsim_test_result [list 0 {3 2 1}] +} + +set doc [string repeat "xyz " 500] +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE yy USING fts5(y, detail=%DETAIL%); + INSERT INTO yy(yy, rank) VALUES('pgsz', 64); + INSERT INTO yy VALUES ($doc); + INSERT INTO yy VALUES ('1 2 3'); + INSERT INTO yy VALUES ('xyz'); + UPDATE yy SET y = y WHERE rowid = 1; + UPDATE yy SET y = y WHERE rowid = 1; + UPDATE yy SET y = y WHERE rowid = 1; + UPDATE yy SET y = y WHERE rowid = 1; +} {} + +do_faultsim_test 6 -faults oom-* -body { + execsql { SELECT rowid FROM yy('xyz') } +} -test { + faultsim_test_result [list 0 {1 3}] +} + + +} ;# foreach_detail_mode... + +finish_test + Index: ext/fts5/test/fts5hash.test ================================================================== --- ext/fts5/test/fts5hash.test +++ ext/fts5/test/fts5hash.test @@ -62,15 +62,17 @@ lappend doc [lindex $vocab $j] } return $doc } +foreach_detail_mode $testprefix { + set vocab [build_vocab1] db func r random_doc do_execsql_test 1.0 { - CREATE VIRTUAL TABLE eee USING fts5(e, ee); + CREATE VIRTUAL TABLE eee USING fts5(e, ee, detail=%DETAIL%); BEGIN; WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100) INSERT INTO eee SELECT r($vocab, 5), r($vocab, 7) FROM ii; INSERT INTO eee(eee) VALUES('integrity-check'); COMMIT; @@ -89,22 +91,24 @@ for {set i 1} {$i <= 100} {incr i} { execsql { INSERT INTO eee VALUES( r($vocab, 5), r($vocab, 7) ) } } } {} -do_test 1.2 { +do_test 1.3 { db eval { SELECT term, doc FROM vocab } { set nRow [db one {SELECT count(*) FROM eee WHERE eee MATCH $term}] if {$nRow != $doc} { error "term=$term fts5vocab=$doc cnt=$nRow" } } set {} {} } {} -do_execsql_test 1.3 { +do_execsql_test 1.4 { COMMIT; INSERT INTO eee(eee) VALUES('integrity-check'); } + +} ;# foreach_detail_mode finish_test Index: ext/fts5/test/fts5integrity.test ================================================================== --- ext/fts5/test/fts5integrity.test +++ ext/fts5/test/fts5integrity.test @@ -143,12 +143,71 @@ do_execsql_test 5.2 { INSERT INTO gg(gg) VALUES('optimize'); } -breakpoint do_execsql_test 5.3 { INSERT INTO gg(gg) VALUES('integrity-check'); } + +do_test 5.4.1 { + set ok 0 + for {set i 0} {$i < 10000} {incr i} { + set T [format %.5d $i] + set res [db eval { SELECT rowid FROM gg($T) ORDER BY rowid ASC }] + set res2 [db eval { SELECT rowid FROM gg($T) ORDER BY rowid DESC }] + if {$res == [lsort -integer $res2]} { incr ok } + } + set ok +} {10000} + +do_test 5.4.2 { + set ok 0 + for {set i 0} {$i < 100} {incr i} { + set T "[format %.3d $i]*" + set res [db eval { SELECT rowid FROM gg($T) ORDER BY rowid ASC }] + set res2 [db eval { SELECT rowid FROM gg($T) ORDER BY rowid DESC }] + if {$res == [lsort -integer $res2]} { incr ok } + } + set ok +} {100} + +#------------------------------------------------------------------------- +# Similar to 5.*. +# +foreach {tn pgsz} { + 1 32 + 2 36 + 3 40 + 4 44 + 5 48 +} { + do_execsql_test 6.$tn.1 { + DROP TABLE IF EXISTS hh; + CREATE VIRTUAL TABLE hh USING fts5(y); + INSERT INTO hh(hh, rank) VALUES('pgsz', $pgsz); + + WITH s(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<999) + INSERT INTO hh SELECT printf("%.3d%.3d%.3d %.3d%.3d%.3d",i,i,i,i+1,i+1,i+1) + FROM s; + + WITH s(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<999) + INSERT INTO hh SELECT printf("%.3d%.3d%.3d %.3d%.3d%.3d",i,i,i,i+1,i+1,i+1) + FROM s; + + INSERT INTO hh(hh) VALUES('optimize'); + } + + do_test 6.$tn.2 { + set ok 0 + for {set i 0} {$i < 1000} {incr i} { + set T [format %.3d%.3d%.3d $i $i $i] + set res [db eval { SELECT rowid FROM hh($T) ORDER BY rowid ASC }] + set res2 [db eval { SELECT rowid FROM hh($T) ORDER BY rowid DESC }] + if {$res == [lsort -integer $res2]} { incr ok } + } + set ok + } {1000} +} finish_test ADDED ext/fts5/test/fts5merge2.test Index: ext/fts5/test/fts5merge2.test ================================================================== --- /dev/null +++ ext/fts5/test/fts5merge2.test @@ -0,0 +1,59 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test that focus on incremental merges of segments. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5merge + +proc dump_structure {} { + db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} { + foreach lvl [lrange $t 1 end] { + set seg [string repeat . [expr [llength $lvl]-2]] + puts "[lrange $lvl 0 1] $seg" + } + } +} + +foreach_detail_mode $testprefix { + +if {[detail_is_none]==0} continue + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1(t1, rank) VALUES('crisismerge', 2); + INSERT INTO t1 VALUES('1 2 3 4'); +} + +expr srand(0) +db func rnddoc fts5_rnddoc +do_test 1.1 { + for {set i 0} {$i < 100} {incr i} { + execsql { + BEGIN; + DELETE FROM t1 WHERE rowid = 1; + INSERT INTO t1(rowid, x) VALUES(1, '1 2 3 4'); + INSERT INTO t1 VALUES(rnddoc(10)); + COMMIT; + } + } +} {} + +do_execsql_test 1.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +} + +finish_test + Index: ext/fts5/test/fts5rowid.test ================================================================== --- ext/fts5/test/fts5rowid.test +++ ext/fts5/test/fts5rowid.test @@ -61,20 +61,20 @@ WITH r(a, b) AS ( SELECT rnddoc(6), rnddoc(6) UNION ALL SELECT rnddoc(6), rnddoc(6) FROM r ) INSERT INTO x1 SELECT * FROM r LIMIT 10000; + DELETE FROM x1 WHERE (rowid%2); } set res [db one {SELECT count(*) FROM x1_data}] do_execsql_test 2.3 { SELECT count(fts5_decode(rowid, block)) FROM x1_data; } $res do_execsql_test 2.4 { UPDATE x1_data SET block = X''; - -- SELECT count(fts5_decode(rowid, block)) FROM x1_data; - SELECT count(*) FROM x1_data; + SELECT count(fts5_decode(rowid, block)) FROM x1_data; } $res do_execsql_test 2.5 { INSERT INTO x1(x1, rank) VALUES('pgsz', 1024); INSERT INTO x1(x1) VALUES('rebuild'); @@ -181,8 +181,39 @@ set res [db one {SELECT count(*) FROM x4_data}] do_execsql_test 5.2 { SELECT count(fts5_decode(rowid, block)) FROM x4_data; } $res + +#------------------------------------------------------------------------- +# + +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE x5 USING fts5(x, detail=none); + INSERT INTO x5(x5, rank) VALUES('pgsz', 32); + INSERT INTO x5 VALUES('a b c d e f'); + INSERT INTO x5 VALUES('a b c d e f'); + INSERT INTO x5 VALUES('a b c d e f'); + BEGIN; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) INSERT INTO x5 SELECT 'a b c d e f' FROM s; + COMMIT; + SELECT count(fts5_decode_none(rowid, block)) FROM x5_data; +} {32} + +do_execsql_test 6.1 { + DELETE FROM x5 WHERE rowid <= 2; + SELECT count(fts5_decode_none(rowid, block)) FROM x5_data; +} {34} + +do_execsql_test 6.2 { + UPDATE x5 SET x='a b c d e f' WHERE rowid=3; + SELECT count(fts5_decode_none(rowid, block)) FROM x5_data; +} {36} + +#db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM x5_data} {puts $r} + + finish_test Index: ext/fts5/test/fts5simple2.test ================================================================== --- ext/fts5/test/fts5simple2.test +++ ext/fts5/test/fts5simple2.test @@ -17,12 +17,10 @@ ifcapable !fts5 { finish_test return } -if 1 { - do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); INSERT INTO t1 VALUES('a b c'); } do_execsql_test 1.1 { @@ -265,12 +263,10 @@ do_execsql_test 14.1 { SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank; } {0.0.1} -} - #------------------------------------------------------------------------- # reset_db do_execsql_test 15.1 { CREATE VIRTUAL TABLE t1 USING fts5(x, detail=none); @@ -297,8 +293,46 @@ } {} do_test 15.4 { execsql { INSERT INTO t1(t1) VALUES('integrity-check') } } {} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 16.0 { + CREATE VIRTUAL TABLE t2 USING fts5(x, detail=none); + BEGIN; + INSERT INTO t2(rowid, x) VALUES(1, 'a b c'); + INSERT INTO t2(rowid, x) VALUES(456, 'a b c'); + INSERT INTO t2(rowid, x) VALUES(1000, 'a b c'); + COMMIT; + UPDATE t2 SET x=x; +} + +do_execsql_test 16.1 { + INSERT INTO t2(t2) VALUES('integrity-check'); +} {} + +do_execsql_test 16.2 { + SELECT rowid FROM t2('b') ORDER BY rowid DESC +} {1000 456 1} + + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 16.0 { + CREATE VIRTUAL TABLE t2 USING fts5(x, detail=none); + BEGIN; + INSERT INTO t2(rowid, x) VALUES(1, 'a b c'); + INSERT INTO t2(rowid, x) VALUES(456, 'a b c'); + INSERT INTO t2(rowid, x) VALUES(1000, 'a b c'); + COMMIT; + UPDATE t2 SET x=x; + DELETE FROM t2; +} + +#db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM t2_data} {puts $r} finish_test Index: ext/fts5/test/fts5synonym.test ================================================================== --- ext/fts5/test/fts5synonym.test +++ ext/fts5/test/fts5synonym.test @@ -19,46 +19,20 @@ ifcapable !fts5 { finish_test return } -foreach S { - {zero 0} - {one 1 i} - {two 2 ii} - {three 3 iii} - {four 4 iv} - {five 5 v} - {six 6 vi} - {seven 7 vii} - {eight 8 viii} - {nine 9 ix} -} { - foreach s $S { - set o [list] - foreach x $S {if {$x!=$s} {lappend o $x}} - set ::syn($s) $o - } -} - -proc tcl_tokenize {tflags text} { - foreach {w iStart iEnd} [fts5_tokenize_split $text] { - sqlite3_fts5_token $w $iStart $iEnd - } -} - -proc tcl_create {args} { - return "tcl_tokenize" -} - -sqlite3_fts5_create_tokenizer db tcl tcl_create +proc tcl_create {args} { return "tcl_tokenize" } + +foreach_detail_mode $testprefix { #------------------------------------------------------------------------- # Warm body test for the code in fts5_tcl.c. # +fts5_tclnum_register db do_execsql_test 1.0 { - CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl); + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = "tclnum document", detail=%DETAIL%); INSERT INTO ft VALUES('abc def ghi'); INSERT INTO ft VALUES('jkl mno pqr'); SELECT rowid, x FROM ft WHERE ft MATCH 'def'; SELECT x, rowid FROM ft WHERE ft MATCH 'pqr'; } {1 {abc def ghi} {jkl mno pqr} 2} @@ -65,26 +39,17 @@ #------------------------------------------------------------------------- # Test a tokenizer that supports synonyms by adding extra entries to the # FTS index. # - -proc tcl_tokenize {tflags text} { - foreach {w iStart iEnd} [fts5_tokenize_split $text] { - sqlite3_fts5_token $w $iStart $iEnd - if {$tflags=="document" && [info exists ::syn($w)]} { - foreach s $::syn($w) { - sqlite3_fts5_token -colo $s $iStart $iEnd - } - } - } -} reset_db -sqlite3_fts5_create_tokenizer db tcl tcl_create +fts5_tclnum_register db do_execsql_test 2.0 { - CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl); + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize = "tclnum document", detail=%DETAIL% + ); INSERT INTO ft VALUES('one two three'); INSERT INTO ft VALUES('four five six'); INSERT INTO ft VALUES('eight nine ten'); } {} @@ -93,10 +58,11 @@ 2 "eight OR 8 OR 5" {2 3} 3 "10" {} 4 "1*" {1} 5 "1 + 2" {1} } { + if {![fts5_expr_ok $expr ft]} continue do_execsql_test 2.1.$tn { SELECT rowid FROM ft WHERE ft MATCH $expr } $res } @@ -178,33 +144,25 @@ #------------------------------------------------------------------------- # Check that expressions with synonyms can be parsed and executed. # reset_db -sqlite3_fts5_create_tokenizer db tcl tcl_create -proc tcl_tokenize {tflags text} { - foreach {w iStart iEnd} [fts5_tokenize_split $text] { - sqlite3_fts5_token $w $iStart $iEnd - if {$tflags=="query" && [info exists ::syn($w)]} { - foreach s $::syn($w) { - sqlite3_fts5_token -colo $s $iStart $iEnd - } - } - } -} +fts5_tclnum_register db foreach {tn expr res} { 1 {abc} {"abc"} 2 {one} {"one"|"i"|"1"} 3 {3} {"3"|"iii"|"three"} 4 {3*} {"3"|"iii"|"three" *} } { - do_execsql_test 4.1.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res] + do_execsql_test 4.1.$tn { + SELECT fts5_expr($expr, 'tokenize=tclnum') + } [list $res] } do_execsql_test 4.2.1 { - CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tcl); + CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tclnum, detail=%DETAIL%); INSERT INTO xx VALUES('one two'); INSERT INTO xx VALUES('three four'); } do_execsql_test 4.2.2 { @@ -215,11 +173,11 @@ SELECT rowid FROM xx WHERE xx MATCH '3' } {2} do_test 5.0 { execsql { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tcl) + CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tclnum, detail=%DETAIL%) } foreach {rowid a b} { 1 {four v 4 i three} {1 3 five five 4 one} 2 {5 1 3 4 i} {2 2 v two 4} 3 {5 i 5 2 four 4 1} {iii ii five two 1} @@ -283,10 +241,11 @@ 6 {"v v"} { 1 {four v 4 i three} {1 3 [five five] 4 one} 5 {three i v i four 4 1} {ii [five five five] iii} } } { + if {![fts5_expr_ok $q t1]} continue do_execsql_test 5.1.$tn { SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']') FROM t1 WHERE t1 MATCH $q } $res } @@ -314,11 +273,10 @@ do_execsql_test 5.2.$tn { SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH $q } $res } - #------------------------------------------------------------------------- # Test terms with more than 4 synonyms. # reset_db sqlite3_fts5_create_tokenizer db tcl tcl_create @@ -332,21 +290,23 @@ } } } do_execsql_test 6.0.1 { - CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl); + CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl, detail=%DETAIL%); INSERT INTO t1 VALUES('yy xx qq'); INSERT INTO t1 VALUES('yy xx xx'); } -do_execsql_test 6.0.2 { - SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)'; -} {{yy xx qq}} +if {[fts5_expr_ok "NEAR(y q)" t1]} { + do_execsql_test 6.0.2 { + SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)'; + } {{yy xx qq}} +} do_test 6.0.3 { execsql { - CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl) + CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl, detail=%DETAIL%) } foreach {rowid a b} { 1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq aaaa} 2 {ww oooooo bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq} 3 {zzzz llll gggggg cccc uu} {hhhhhh aaaa ppppp rr ee jjjj} @@ -385,10 +345,12 @@ 4 {NEAR(q y, 20)} { 1 {[yyyy] vvvvv [qq] oo [yyyyyy] vvvv eee} {ffff uu r qq aaaa} 2 {ww oooooo bbbbb ssssss mm} {ffffff [yy] iiii rr s ccc [qqqqq]} } } { + if {![fts5_expr_ok $q t2]} continue + do_execsql_test 6.1.$tn.asc { SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']') FROM t2 WHERE t2 MATCH $q } $res @@ -433,11 +395,11 @@ } } } do_execsql_test 7.0.1 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl, detail=%DETAIL%); INSERT INTO t1 VALUES('0 2 3', '4 5 6 7'); INSERT INTO t1 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0'); SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH '000 AND 00 AND 0'; } {{3 4} {2 10}} @@ -444,17 +406,19 @@ do_execsql_test 7.0.2 { INSERT INTO t1(t1) VALUES('integrity-check'); } do_execsql_test 7.1.1 { - CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl); + CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl, detail=%DETAIL%); INSERT INTO t2 VALUES('0 2 3', '4 5 6 7'); INSERT INTO t2 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0'); SELECT fts5_test_columnsize(t2) FROM t2 WHERE t2 MATCH '000 AND 00 AND 0'; } {{3 4} {2 10}} do_execsql_test 7.1.2 { INSERT INTO t2(t2) VALUES('integrity-check'); } + +} ;# foreach_detail_mode finish_test Index: ext/fts5/test/fts5synonym2.test ================================================================== --- ext/fts5/test/fts5synonym2.test +++ ext/fts5/test/fts5synonym2.test @@ -19,56 +19,26 @@ ifcapable !fts5 { finish_test return } -#------------------------------------------------------------------------- -# Code for a simple Tcl tokenizer that supports synonyms at query time. -# -foreach SYNDICT { - {zero 0} - {one 1 i} - {two 2 ii} - {three 3 iii} - {four 4 iv} - {five 5 v} - {six 6 vi} - {seven 7 vii} - {eight 8 viii} - {nine 9 ix} -} { - foreach s $SYNDICT { - set o [list] - foreach x $SYNDICT {if {$x!=$s} {lappend o $x}} - set ::syn($s) $o - } -} - -proc tcl_tokenize {tflags text} { - foreach {w iStart iEnd} [fts5_tokenize_split $text] { - sqlite3_fts5_token $w $iStart $iEnd - if {$tflags == "query"} { - foreach s $::syn($w) { sqlite3_fts5_token -colo $s $iStart $iEnd } - } - } -} - -proc tcl_create {args} { - return "tcl_tokenize" -} - -# -# End of tokenizer code. -#------------------------------------------------------------------------- - +foreach tok {query document} { foreach_detail_mode $testprefix { -sqlite3_fts5_create_tokenizer db tcl tcl_create +fts5_tclnum_register db fts5_aux_test_functions db -do_execsql_test 1.0 { - CREATE VIRTUAL TABLE ss USING fts5(a, b, tokenize=tcl, detail=%DETAIL%); +proc fts5_rowid {cmd} { expr [$cmd xColumnText -1] } +sqlite3_fts5_create_function db fts5_rowid fts5_rowid + +do_execsql_test 1.$tok.0.1 " + CREATE VIRTUAL TABLE ss USING fts5(a, b, + tokenize='tclnum $tok', detail=%DETAIL%); + INSERT INTO ss(ss, rank) VALUES('rank', 'fts5_rowid()'); +" + +do_execsql_test 1.$tok.0.2 { INSERT INTO ss VALUES('5 5 five seven 3 seven i', '2 1 5 0 two 1 i'); INSERT INTO ss VALUES('six ix iii 7 i vii iii', 'one seven nine 4 9 1 vi'); INSERT INTO ss VALUES('6 viii i five six zero seven', '5 v iii iv iv 3'); INSERT INTO ss VALUES('9 ii six 8 1 6', 'six 4 iv iv 7'); INSERT INTO ss VALUES('1 5 4 eight ii iv iii', 'nine 2 eight ix v vii'); @@ -135,20 +105,25 @@ 4.1 "NEAR(one two, 2)" 4.2 "NEAR(one two three, 2)" 4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)" } { if {[fts5_expr_ok $expr ss]==0} { - do_test 1.$tn.OMITTED { list } [list] + do_test 1.$tok.$tn.OMITTED { list } [list] continue } - set res [fts5_query_data $expr ss ASC ::syn] - breakpoint - do_execsql_test 1.$tn.[llength $res].asc { + set res [fts5_query_data $expr ss ASC ::tclnum_syn] + do_execsql_test 1.$tok.$tn.[llength $res].asc.1 { + SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr) + } $res + + do_execsql_test 1.$tok.$tn.[llength $res].asc.2 { SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr) + ORDER BY rank ASC } $res } +} } finish_test ADDED ext/fts5/test/fts5tok1.test Index: ext/fts5/test/fts5tok1.test ================================================================== --- /dev/null +++ ext/fts5/test/fts5tok1.test @@ -0,0 +1,115 @@ +# 2016 Jan 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5tok1 + + +sqlite3_fts5_register_fts5tokenize db + +#------------------------------------------------------------------------- +# Simple test cases. Using the default (ascii) tokenizer. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5tokenize(ascii); + CREATE VIRTUAL TABLE t2 USING fts5tokenize(); + CREATE VIRTUAL TABLE t3 USING fts5tokenize( + ascii, 'separators', 'xyz', tokenchars, '''' + ); +} + +foreach {tn tbl} {1 t1 2 t2 3 t3} { + do_execsql_test 1.$tn.1 "SELECT input, * FROM $tbl ('one two three')" { + {one two three} one 0 3 0 + {one two three} two 4 7 1 + {one two three} three 8 13 2 + } + + do_execsql_test 1.$tn.2 " + SELECT token FROM $tbl WHERE input = 'OnE tWo tHrEe' + " { + one two three + } +} + +do_execsql_test 1.4 { + SELECT token FROM t3 WHERE input = '1x2x3x' +} {1 2 3} + +do_execsql_test 1.5 { + SELECT token FROM t1 WHERE input = '1x2x3x' +} {1x2x3x} + +do_execsql_test 1.6 { + SELECT token FROM t3 WHERE input = '1''2x3x' +} {1'2 3} + +do_execsql_test 1.7 { + SELECT token FROM t3 WHERE input = '' +} {} + +do_execsql_test 1.8 { + SELECT token FROM t3 WHERE input = NULL +} {} + +do_execsql_test 1.9 { + SELECT input, * FROM t3 WHERE input = 123 +} {123 123 0 3 0} + +do_execsql_test 1.10 { + SELECT input, * FROM t1 WHERE input = 'a b c' AND token = 'b'; +} { + {a b c} b 2 3 1 +} + +do_execsql_test 1.11 { + SELECT input, * FROM t1 WHERE token = 'b' AND input = 'a b c'; +} { + {a b c} b 2 3 1 +} + +do_execsql_test 1.12 { + SELECT input, * FROM t1 WHERE input < 'b' AND input = 'a b c'; +} { + {a b c} a 0 1 0 + {a b c} b 2 3 1 + {a b c} c 4 5 2 +} + +do_execsql_test 1.13.1 { + CREATE TABLE c1(x); + INSERT INTO c1(x) VALUES('a b c'); + INSERT INTO c1(x) VALUES('d e f'); +} +do_execsql_test 1.13.2 { + SELECT c1.*, input, t1.* FROM c1, t1 WHERE input = x AND c1.rowid=t1.rowid; +} { + {a b c} {a b c} a 0 1 0 + {d e f} {d e f} e 2 3 1 +} + + +#------------------------------------------------------------------------- +# Error cases. +# +do_catchsql_test 2.0 { + CREATE VIRTUAL TABLE tX USING fts5tokenize(nosuchtokenizer); +} {1 {vtable constructor failed: tX}} + +do_catchsql_test 2.1 { + CREATE VIRTUAL TABLE t4 USING fts5tokenize; + SELECT * FROM t4; +} {1 {SQL logic error or missing database}} + + +finish_test ADDED ext/fts5/test/fts5tok2.test Index: ext/fts5/test/fts5tok2.test ================================================================== --- /dev/null +++ ext/fts5/test/fts5tok2.test @@ -0,0 +1,47 @@ +# 2016 Jan 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5||!fts3 { finish_test ; return } +set ::testprefix fts5tok2 + +sqlite3_fts5_register_fts5tokenize db + +#------------------------------------------------------------------------- +# Simple test cases. Using the default (ascii) tokenizer. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t5 USING fts5tokenize(unicode61); + CREATE VIRTUAL TABLE t3 USING fts3tokenize(unicode61); +} + +do_test 1.1 { + array unset -nocomplain A + + for {set i 1} {$i < 65536} {incr i} { + set input [format "abc%cxyz" $i] + set expect [execsql { + SELECT input, token, start, end FROM t3 WHERE input=$input + }] + + incr A([llength $expect]) + + set res [execsql { + SELECT input, token, start, end FROM t5($input) + }] + if {$res != $expect} {error "failed at i=$i"} + } +} {} + +do_test 1.1.nTokenChars=$A(4).nSeparators=$A(8) {} {} + +finish_test ADDED ext/fts5/test/fts5update.test Index: ext/fts5/test/fts5update.test ================================================================== --- /dev/null +++ ext/fts5/test/fts5update.test @@ -0,0 +1,121 @@ +# 2016 Jan 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5update + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set docs { + "eight zero iv eight 7" "ix one 8 one three ii one" + "1 9 9 three viii" "5 zero ii 6 nine ix 3" + "3 zero 5 2 seven nine" "two eight viii eight 1" + "4 six two 5 9 vii" "viii ii four 8 i i iv" + "vii 0 iv seven 7 viii" "five 1 nine vi seven" + "1 zero zero iii 1" "one one six 6 nine seven" + "one v 4 zero 4 iii ii" "2 3 eight six ix" + "six iv 7 three 5" "ix zero 0 8 ii 7 3" + "four six nine 2 vii 3" "five viii 5 8 0 7" +} + +foreach_detail_mode $::testprefix { + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); +} {} + +do_test 1.1 { + foreach {a b} $docs { + execsql {INSERT INTO t1 VALUES($a, $b)} + } +} {} + +proc update {iRowid iA iB} { + set a [lindex $::docs $iA] + set b [lindex $::docs $iB] + execsql { UPDATE t1 SET a=$a, b=$b WHERE rowid=$iRowid } +} + +set nDoc [llength $::docs] +foreach n {1 5 10 50 100} { + do_test 1.2.$n { + execsql BEGIN + for {set i 1} {$i <= 1000} {incr i} { + set iRowid [expr {int(rand() * ($nDoc/2)) + 1}] + set iA [expr {int(rand() * $nDoc)}] + set iB [expr {int(rand() * $nDoc)}] + update $iRowid $iA $iB + + if {($i % $n)==0} { + execsql { COMMIT; BEGIN } + } + + if {($i % $n)==100} { + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } + } + } + execsql COMMIT + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } + } {} +} + +do_execsql_test 1.3 { + UPDATE t1 SET a=a AND b=b; + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_test 1.4 { + execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 32) } + for {set i 0} {$i < 50} {incr i} { + execsql { UPDATE t1 SET a=a AND b=b } + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } + } +} {} + +#------------------------------------------------------------------------- +# Lots of deletes/inserts of the same document with the same rowid. +# +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE x2 USING fts5(x, detail=%DETAIL%); + INSERT INTO x2(x2, rank) VALUES('crisismerge', 2); + INSERT INTO x2 VALUES('a b c'); + INSERT INTO x2 VALUES('a b c'); +} +do_test 2.1 { + for {set i 0} {$i < 1000} {incr i} { + execsql { DELETE FROM x2 WHERE rowid = 2 } + execsql { INSERT INTO x2(rowid, x) VALUES(2, 'a b c') } + } +} {} +do_execsql_test 2.1.integrity { + INSERT INTO x2(x2) VALUES('integrity-check'); +} + +do_test 2.2 { + for {set i 0} {$i < 1000} {incr i} { + execsql { UPDATE x2 SET x=x WHERE rowid=2 } + } +} {} +do_execsql_test 2.2.integrity { + INSERT INTO x2(x2) VALUES('integrity-check'); +} + +} +finish_test + + Index: ext/fts5/test/fts5vocab.test ================================================================== --- ext/fts5/test/fts5vocab.test +++ ext/fts5/test/fts5vocab.test @@ -54,10 +54,12 @@ foreach {a b c} $L { lappend ret $a {} $b {} } set ret } + +if 1 { do_execsql_test 1.1.1 { CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1, detail=%DETAIL%); CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row'); PRAGMA table_info = v1; @@ -387,9 +389,61 @@ SELECT count(*) FROM txc, txc_c WHERE txc.term = txc_c.term AND txc.col=txc_c.col; } {57} } +} + +#------------------------------------------------------------------------- +# Test the fts5vocab tables response to a specific types of corruption: +# where the fts5 index contains hits for columns that do not exist. +# +do_execsql_test 8.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a, b, c, detail=%DETAIL%); + INSERT INTO x1 VALUES('a b c', 'd e f', 'g h i'); + INSERT INTO x1 VALUES('g h i', 'a b c', 'd e f'); + INSERT INTO x1 VALUES('d e f', 'g h i', 'a b c'); + CREATE VIRTUAL TABLE x1_r USING fts5vocab(x1, row); + CREATE VIRTUAL TABLE x1_c USING fts5vocab(x1, col); +} + +set resr [star_from_row {a 3 3 b 3 3 c 3 3 d 3 3 e 3 3 f 3 3 g 3 3 h 3 3 i 3 3}] +set resc [star_from_col { + a a 1 1 a b 1 1 a c 1 1 b a 1 1 + b b 1 1 b c 1 1 c a 1 1 c b 1 1 + c c 1 1 d a 1 1 d b 1 1 d c 1 1 + e a 1 1 e b 1 1 e c 1 1 f a 1 1 + f b 1 1 f c 1 1 g a 1 1 g b 1 1 + g c 1 1 h a 1 1 h b 1 1 h c 1 1 + i a 1 1 i b 1 1 i c 1 1 +}] +if {[detail_is_none]} { set resc [row_to_col $resr] } + +do_execsql_test 8.1.1 { SELECT * FROM x1_r; } $resr +do_execsql_test 8.1.2 { SELECT * FROM x1_c } $resc + +do_execsql_test 8.2 { + PRAGMA writable_schema = 1; + UPDATE sqlite_master + SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(a, detail=%DETAIL%)' + WHERE name = 'x1'; +} +db close +sqlite3 db test.db +sqlite3_fts5_may_be_corrupt 1 + +do_execsql_test 8.2.1 { SELECT * FROM x1_r } $resr + +if {[detail_is_none]} { + do_execsql_test 8.2.2 { SELECT * FROM x1_c } $resc +} else { + do_catchsql_test 8.2.2 { + SELECT * FROM x1_c + } {1 {database disk image is malformed}} +} + +sqlite3_fts5_may_be_corrupt 0 + } finish_test Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -333,11 +333,12 @@ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/totype.c \ $(TOP)/ext/misc/wholenumber.c \ $(TOP)/ext/misc/vfslog.c \ $(TOP)/ext/fts5/fts5_tcl.c \ - $(TOP)/ext/fts5/fts5_test_mi.c + $(TOP)/ext/fts5/fts5_test_mi.c \ + $(TOP)/ext/fts5/fts5_test_tok.c #TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c #TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c Index: src/analyze.c ================================================================== --- src/analyze.c +++ src/analyze.c @@ -476,12 +476,11 @@ static const FuncDef statInitFuncdef = { 2+IsStat34, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - statInit, /* xFunc */ - 0, /* xStep */ + statInit, /* xSFunc */ 0, /* xFinalize */ "stat_init", /* zName */ 0, /* pHash */ 0 /* pDestructor */ }; @@ -777,12 +776,11 @@ static const FuncDef statPushFuncdef = { 2+IsStat34, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - statPush, /* xFunc */ - 0, /* xStep */ + statPush, /* xSFunc */ 0, /* xFinalize */ "stat_push", /* zName */ 0, /* pHash */ 0 /* pDestructor */ }; @@ -924,12 +922,11 @@ static const FuncDef statGetFuncdef = { 1+IsStat34, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - statGet, /* xFunc */ - 0, /* xStep */ + statGet, /* xSFunc */ 0, /* xFinalize */ "stat_get", /* zName */ 0, /* pHash */ 0 /* pDestructor */ }; @@ -941,12 +938,12 @@ #elif SQLITE_DEBUG assert( iParam==STAT_GET_STAT1 ); #else UNUSED_PARAMETER( iParam ); #endif - sqlite3VdbeAddOp3(v, OP_Function0, 0, regStat4, regOut); - sqlite3VdbeChangeP4(v, -1, (char*)&statGetFuncdef, P4_FUNCDEF); + sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4, regOut, + (char*)&statGetFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 1 + IsStat34); } /* ** Generate code to do an analysis of all indices associated with @@ -1096,12 +1093,12 @@ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3); #endif sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1); sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2); - sqlite3VdbeAddOp3(v, OP_Function0, 0, regStat4+1, regStat4); - sqlite3VdbeChangeP4(v, -1, (char*)&statInitFuncdef, P4_FUNCDEF); + sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4+1, regStat4, + (char*)&statInitFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 2+IsStat34); /* Implementation of the following: ** ** Rewind csr @@ -1193,12 +1190,12 @@ sqlite3VdbeAddOp3(v, OP_MakeRecord, regKey, pPk->nKeyCol, regRowid); sqlite3ReleaseTempRange(pParse, regKey, pPk->nKeyCol); } #endif assert( regChng==(regStat4+1) ); - sqlite3VdbeAddOp3(v, OP_Function0, 1, regStat4, regTemp); - sqlite3VdbeChangeP4(v, -1, (char*)&statPushFuncdef, P4_FUNCDEF); + sqlite3VdbeAddOp4(v, OP_Function0, 1, regStat4, regTemp, + (char*)&statPushFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 2+IsStat34); sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); /* Add the entry to the stat1 table. */ callStatGet(v, regStat4, STAT_GET_STAT1, regStat1); Index: src/attach.c ================================================================== --- src/attach.c +++ src/attach.c @@ -357,15 +357,15 @@ sqlite3ExprCode(pParse, pDbname, regArgs+1); sqlite3ExprCode(pParse, pKey, regArgs+2); assert( v || db->mallocFailed ); if( v ){ - sqlite3VdbeAddOp3(v, OP_Function0, 0, regArgs+3-pFunc->nArg, regArgs+3); + sqlite3VdbeAddOp4(v, OP_Function0, 0, regArgs+3-pFunc->nArg, regArgs+3, + (char *)pFunc, P4_FUNCDEF); assert( pFunc->nArg==-1 || (pFunc->nArg&0xff)==pFunc->nArg ); sqlite3VdbeChangeP5(v, (u8)(pFunc->nArg)); - sqlite3VdbeChangeP4(v, -1, (char *)pFunc, P4_FUNCDEF); - + /* Code an OP_Expire. For an ATTACH statement, set P1 to true (expire this ** statement only). For DETACH, set it to false (expire all existing ** statements). */ sqlite3VdbeAddOp1(v, OP_Expire, (type==SQLITE_ATTACH)); @@ -386,12 +386,11 @@ static const FuncDef detach_func = { 1, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - detachFunc, /* xFunc */ - 0, /* xStep */ + detachFunc, /* xSFunc */ 0, /* xFinalize */ "sqlite_detach", /* zName */ 0, /* pHash */ 0 /* pDestructor */ }; @@ -407,12 +406,11 @@ static const FuncDef attach_func = { 3, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - attachFunc, /* xFunc */ - 0, /* xStep */ + attachFunc, /* xSFunc */ 0, /* xFinalize */ "sqlite_attach", /* zName */ 0, /* pHash */ 0 /* pDestructor */ }; Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -7211,13 +7211,12 @@ ** This must be done in advance. Once the balance starts, the cell ** offset section of the btree page will be overwritten and we will no ** long be able to find the cells if a pointer to each cell is not saved ** first. */ - memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*limit); + memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); if( pOld->nOverflow>0 ){ - memset(&b.szCell[b.nCell+limit], 0, sizeof(b.szCell[0])*pOld->nOverflow); limit = pOld->aiOvfl[0]; for(j=0; jpCursor) ){ sqlite3ConnectionBlocked(p->db, pBt->pCursor->pBtree->db); return SQLITE_LOCKED_SHAREDCACHE; } + + /* + ** It is illegal to drop the sqlite_master table on page 1. But again, + ** this error is caught long before reaching this point. + */ + if( NEVER(iTable<2) ){ + return SQLITE_CORRUPT_BKPT; + } rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); if( rc ) return rc; rc = sqlite3BtreeClearTable(p, iTable, 0); if( rc ){ @@ -8576,80 +8583,71 @@ return rc; } *piMoved = 0; - if( iTable>1 ){ -#ifdef SQLITE_OMIT_AUTOVACUUM - freePage(pPage, &rc); - releasePage(pPage); -#else - if( pBt->autoVacuum ){ - Pgno maxRootPgno; - sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &maxRootPgno); - - if( iTable==maxRootPgno ){ - /* If the table being dropped is the table with the largest root-page - ** number in the database, put the root page on the free list. - */ - freePage(pPage, &rc); - releasePage(pPage); - if( rc!=SQLITE_OK ){ - return rc; - } - }else{ - /* The table being dropped does not have the largest root-page - ** number in the database. So move the page that does into the - ** gap left by the deleted root-page. - */ - MemPage *pMove; - releasePage(pPage); - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0); - releasePage(pMove); - if( rc!=SQLITE_OK ){ - return rc; - } - pMove = 0; - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); - freePage(pMove, &rc); - releasePage(pMove); - if( rc!=SQLITE_OK ){ - return rc; - } - *piMoved = maxRootPgno; - } - - /* Set the new 'max-root-page' value in the database header. This - ** is the old value less one, less one more if that happens to - ** be a root-page number, less one again if that is the - ** PENDING_BYTE_PAGE. - */ - maxRootPgno--; - while( maxRootPgno==PENDING_BYTE_PAGE(pBt) - || PTRMAP_ISPAGE(pBt, maxRootPgno) ){ - maxRootPgno--; - } - assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) ); - - rc = sqlite3BtreeUpdateMeta(p, 4, maxRootPgno); - }else{ - freePage(pPage, &rc); - releasePage(pPage); - } -#endif - }else{ - /* If sqlite3BtreeDropTable was called on page 1. - ** This really never should happen except in a corrupt - ** database. - */ - zeroPage(pPage, PTF_INTKEY|PTF_LEAF ); - releasePage(pPage); - } +#ifdef SQLITE_OMIT_AUTOVACUUM + freePage(pPage, &rc); + releasePage(pPage); +#else + if( pBt->autoVacuum ){ + Pgno maxRootPgno; + sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &maxRootPgno); + + if( iTable==maxRootPgno ){ + /* If the table being dropped is the table with the largest root-page + ** number in the database, put the root page on the free list. + */ + freePage(pPage, &rc); + releasePage(pPage); + if( rc!=SQLITE_OK ){ + return rc; + } + }else{ + /* The table being dropped does not have the largest root-page + ** number in the database. So move the page that does into the + ** gap left by the deleted root-page. + */ + MemPage *pMove; + releasePage(pPage); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0); + releasePage(pMove); + if( rc!=SQLITE_OK ){ + return rc; + } + pMove = 0; + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + freePage(pMove, &rc); + releasePage(pMove); + if( rc!=SQLITE_OK ){ + return rc; + } + *piMoved = maxRootPgno; + } + + /* Set the new 'max-root-page' value in the database header. This + ** is the old value less one, less one more if that happens to + ** be a root-page number, less one again if that is the + ** PENDING_BYTE_PAGE. + */ + maxRootPgno--; + while( maxRootPgno==PENDING_BYTE_PAGE(pBt) + || PTRMAP_ISPAGE(pBt, maxRootPgno) ){ + maxRootPgno--; + } + assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) ); + + rc = sqlite3BtreeUpdateMeta(p, 4, maxRootPgno); + }else{ + freePage(pPage, &rc); + releasePage(pPage); + } +#endif return rc; } int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ int rc; sqlite3BtreeEnter(p); Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -226,19 +226,23 @@ /* A minimum of one cursor is required if autoincrement is used * See ticket [a696379c1f08866] */ if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1; sqlite3VdbeMakeReady(v, pParse); pParse->rc = SQLITE_DONE; - pParse->colNamesSet = 0; }else{ pParse->rc = SQLITE_ERROR; } + + /* We are done with this Parse object. There is no need to de-initialize it */ +#if 0 + pParse->colNamesSet = 0; pParse->nTab = 0; pParse->nMem = 0; pParse->nSet = 0; pParse->nVar = 0; DbMaskZero(pParse->cookieMask); +#endif } /* ** Run the parser and code generator recursively in order to generate ** code for the SQL statement given onto the end of the pParse context @@ -493,11 +497,10 @@ if( jaDb[j] = db->aDb[i]; } j++; } - memset(&db->aDb[j], 0, (db->nDb-j)*sizeof(db->aDb[j])); db->nDb = j; if( db->nDb<=2 && db->aDb!=db->aDbStatic ){ memcpy(db->aDbStatic, db->aDb, 2*sizeof(db->aDb[0])); sqlite3DbFree(db, db->aDb); db->aDb = db->aDbStatic; @@ -756,11 +759,12 @@ Token **pUnqual /* Write the unqualified object name here */ ){ int iDb; /* Database holding the object */ sqlite3 *db = pParse->db; - if( ALWAYS(pName2!=0) && pName2->n>0 ){ + assert( pName2!=0 ); + if( pName2->n>0 ){ if( db->init.busy ) { sqlite3ErrorMsg(pParse, "corrupt database"); return -1; } *pUnqual = pName2; @@ -845,66 +849,50 @@ sqlite3 *db = pParse->db; Vdbe *v; int iDb; /* Database number to create the table in */ Token *pName; /* Unqualified name of the table to create */ - /* The table or view name to create is passed to this routine via tokens - ** pName1 and pName2. If the table name was fully qualified, for example: - ** - ** CREATE TABLE xxx.yyy (...); - ** - ** Then pName1 is set to "xxx" and pName2 "yyy". On the other hand if - ** the table name is not fully qualified, i.e.: - ** - ** CREATE TABLE yyy(...); - ** - ** Then pName1 is set to "yyy" and pName2 is "". - ** - ** The call below sets the pName pointer to point at the token (pName1 or - ** pName2) that stores the unqualified table name. The variable iDb is - ** set to the index of the database that the table or view is to be - ** created in. - */ - iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); - if( iDb<0 ) return; - if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ - /* If creating a temp table, the name may not be qualified. Unless - ** the database name is "temp" anyway. */ - sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); - return; - } - if( !OMIT_TEMPDB && isTemp ) iDb = 1; - - pParse->sNameToken = *pName; - zName = sqlite3NameFromToken(db, pName); + if( db->init.busy && db->init.newTnum==1 ){ + /* Special case: Parsing the sqlite_master or sqlite_temp_master schema */ + iDb = db->init.iDb; + zName = sqlite3DbStrDup(db, SCHEMA_TABLE(iDb)); + pName = pName1; + }else{ + /* The common case */ + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); + if( iDb<0 ) return; + if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ + /* If creating a temp table, the name may not be qualified. Unless + ** the database name is "temp" anyway. */ + sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); + return; + } + if( !OMIT_TEMPDB && isTemp ) iDb = 1; + zName = sqlite3NameFromToken(db, pName); + } + pParse->sNameToken = *pName; if( zName==0 ) return; if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ goto begin_table_error; } if( db->init.iDb==1 ) isTemp = 1; #ifndef SQLITE_OMIT_AUTHORIZATION - assert( (isTemp & 1)==isTemp ); + assert( isTemp==0 || isTemp==1 ); + assert( isView==0 || isView==1 ); { - int code; + static const u8 aCode[] = { + SQLITE_CREATE_TABLE, + SQLITE_CREATE_TEMP_TABLE, + SQLITE_CREATE_VIEW, + SQLITE_CREATE_TEMP_VIEW + }; char *zDb = db->aDb[iDb].zName; if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(isTemp), 0, zDb) ){ goto begin_table_error; } - if( isView ){ - if( !OMIT_TEMPDB && isTemp ){ - code = SQLITE_CREATE_TEMP_VIEW; - }else{ - code = SQLITE_CREATE_VIEW; - } - }else{ - if( !OMIT_TEMPDB && isTemp ){ - code = SQLITE_CREATE_TEMP_TABLE; - }else{ - code = SQLITE_CREATE_TABLE; - } - } - if( !isVirtual && sqlite3AuthCheck(pParse, code, zName, 0, zDb) ){ + if( !isVirtual && sqlite3AuthCheck(pParse, (int)aCode[isTemp+2*isView], + zName, 0, zDb) ){ goto begin_table_error; } } #endif @@ -1862,13 +1850,17 @@ /* If the db->init.busy is 1 it means we are reading the SQL off the ** "sqlite_master" or "sqlite_temp_master" table on the disk. ** So do not write to the disk again. Extract the root page number ** for the table from the db->init.newTnum field. (The page number ** should have been put there by the sqliteOpenCb routine.) + ** + ** If the root page number is 1, that means this is the sqlite_master + ** table itself. So mark it read-only. */ if( db->init.busy ){ p->tnum = db->init.newTnum; + if( p->tnum==1 ) p->tabFlags |= TF_Readonly; } /* Special processing for WITHOUT ROWID Tables */ if( tabOpts & TF_WithoutRowid ){ if( (p->tabFlags & TF_Autoincrement) ){ @@ -2317,10 +2309,11 @@ ** erasing iTable (this can happen with an auto-vacuum database). */ static void destroyRootPage(Parse *pParse, int iTable, int iDb){ Vdbe *v = sqlite3GetVdbe(pParse); int r1 = sqlite3GetTempReg(pParse); + assert( iTable>1 ); sqlite3VdbeAddOp3(v, OP_Destroy, iTable, r1, iDb); sqlite3MayAbort(pParse); #ifndef SQLITE_OMIT_AUTOVACUUM /* OP_Destroy stores an in integer r1. If this integer ** is non-zero, then it is the root page number of a table moved to @@ -3715,13 +3708,14 @@ Token *pDatabase /* Database of the table */ ){ struct SrcList_item *pItem; assert( pDatabase==0 || pTable!=0 ); /* Cannot have C without B */ if( pList==0 ){ - pList = sqlite3DbMallocZero(db, sizeof(SrcList) ); + pList = sqlite3DbMallocRaw(db, sizeof(SrcList) ); if( pList==0 ) return 0; pList->nAlloc = 1; + pList->nSrc = 0; } pList = sqlite3SrcListEnlarge(db, pList, 1, pList->nSrc); if( db->mallocFailed ){ sqlite3SrcListDelete(db, pList); return 0; @@ -4120,11 +4114,11 @@ assert( (errCode&0xff)==SQLITE_CONSTRAINT ); if( onError==OE_Abort ){ sqlite3MayAbort(pParse); } sqlite3VdbeAddOp4(v, OP_Halt, errCode, onError, 0, p4, p4type); - if( p5Errmsg ) sqlite3VdbeChangeP5(v, p5Errmsg); + sqlite3VdbeChangeP5(v, p5Errmsg); } /* ** Code an OP_Halt due to UNIQUE or PRIMARY KEY constraint violation. */ Index: src/callback.c ================================================================== --- src/callback.c +++ src/callback.c @@ -241,12 +241,12 @@ ** 3: encoding matches and function takes any number of arguments ** 4: UTF8/16 conversion required - argument count matches exactly ** 5: UTF16 byte order conversion required - argument count matches exactly ** 6: Perfect match: encoding and argument count match exactly. ** -** If nArg==(-2) then any function with a non-null xStep or xFunc is -** a perfect match and any function with both xStep and xFunc NULL is +** If nArg==(-2) then any function with a non-null xSFunc is +** a perfect match and any function with xSFunc NULL is ** a non-match. */ #define FUNC_PERFECT_MATCH 6 /* The score for a perfect match */ static int matchQuality( FuncDef *p, /* The function we are evaluating for match quality */ @@ -254,11 +254,11 @@ u8 enc /* Desired text encoding */ ){ int match; /* nArg of -2 is a special case */ - if( nArg==(-2) ) return (p->xFunc==0 && p->xStep==0) ? 0 : FUNC_PERFECT_MATCH; + if( nArg==(-2) ) return (p->xSFunc==0) ? 0 : FUNC_PERFECT_MATCH; /* Wrong number of arguments means "no match" */ if( p->nArg!=nArg && p->nArg>=0 ) return 0; /* Give a better score to a function with a specific number of arguments @@ -332,11 +332,11 @@ ** If the createFlag argument is true, then a new (blank) FuncDef ** structure is created and liked into the "db" structure if a ** no matching function previously existed. ** ** If nArg is -2, then the first valid function found is returned. A -** function is valid if either xFunc or xStep is non-zero. The nArg==(-2) +** function is valid if xSFunc is non-zero. The nArg==(-2) ** case is used to see if zName is a valid function name for some number ** of arguments. If nArg is -2, then createFlag must be 0. ** ** If createFlag is false, then a function with the required name and ** number of arguments may be returned even if the eTextRep flag does not @@ -409,11 +409,11 @@ memcpy(pBest->zName, zName, nName); pBest->zName[nName] = 0; sqlite3FuncDefInsert(&db->aFunc, pBest); } - if( pBest && (pBest->xStep || pBest->xFunc || createFlag) ){ + if( pBest && (pBest->xSFunc || createFlag) ){ return pBest; } return 0; } Index: src/date.c ================================================================== --- src/date.c +++ src/date.c @@ -68,52 +68,68 @@ char tzSet; /* Timezone was set explicitly */ }; /* -** Convert zDate into one or more integers. Additional arguments -** come in groups of 5 as follows: -** -** N number of digits in the integer -** min minimum allowed value of the integer -** max maximum allowed value of the integer -** nextC first character after the integer -** pVal where to write the integers value. -** -** Conversions continue until one with nextC==0 is encountered. +** Convert zDate into one or more integers according to the conversion +** specifier zFormat. +** +** zFormat[] contains 4 characters for each integer converted, except for +** the last integer which is specified by three characters. The meaning +** of a four-character format specifiers ABCD is: +** +** A: number of digits to convert. Always "2" or "4". +** B: minimum value. Always "0" or "1". +** C: maximum value, decoded as: +** a: 12 +** b: 14 +** c: 24 +** d: 31 +** e: 59 +** f: 9999 +** D: the separator character, or \000 to indicate this is the +** last number to convert. +** +** Example: To translate an ISO-8601 date YYYY-MM-DD, the format would +** be "40f-21a-20c". The "40f-" indicates the 4-digit year followed by "-". +** The "21a-" indicates the 2-digit month followed by "-". The "20c" indicates +** the 2-digit day which is the last integer in the set. +** ** The function returns the number of successful conversions. */ -static int getDigits(const char *zDate, ...){ +static int getDigits(const char *zDate, const char *zFormat, ...){ + /* The aMx[] array translates the 3rd character of each format + ** spec into a max size: a b c d e f */ + static const u16 aMx[] = { 12, 14, 24, 31, 59, 9999 }; va_list ap; - int val; - int N; - int min; - int max; - int nextC; - int *pVal; int cnt = 0; - va_start(ap, zDate); + char nextC; + va_start(ap, zFormat); do{ - N = va_arg(ap, int); - min = va_arg(ap, int); - max = va_arg(ap, int); - nextC = va_arg(ap, int); - pVal = va_arg(ap, int*); + char N = zFormat[0] - '0'; + char min = zFormat[1] - '0'; + int val = 0; + u16 max; + + assert( zFormat[2]>='a' && zFormat[2]<='f' ); + max = aMx[zFormat[2] - 'a']; + nextC = zFormat[3]; val = 0; while( N-- ){ if( !sqlite3Isdigit(*zDate) ){ goto end_getDigits; } val = val*10 + *zDate - '0'; zDate++; } - if( valmax || (nextC!=0 && nextC!=*zDate) ){ + if( val<(int)min || val>(int)max || (nextC!=0 && nextC!=*zDate) ){ goto end_getDigits; } - *pVal = val; + *va_arg(ap,int*) = val; zDate++; cnt++; + zFormat += 4; }while( nextC ); end_getDigits: va_end(ap); return cnt; } @@ -150,11 +166,11 @@ goto zulu_time; }else{ return c!=0; } zDate++; - if( getDigits(zDate, 2, 0, 14, ':', &nHr, 2, 0, 59, 0, &nMn)!=2 ){ + if( getDigits(zDate, "20b:20e", &nHr, &nMn)!=2 ){ return 1; } zDate += 5; p->tz = sgn*(nMn + nHr*60); zulu_time: @@ -171,17 +187,17 @@ ** Return 1 if there is a parsing error and 0 on success. */ static int parseHhMmSs(const char *zDate, DateTime *p){ int h, m, s; double ms = 0.0; - if( getDigits(zDate, 2, 0, 24, ':', &h, 2, 0, 59, 0, &m)!=2 ){ + if( getDigits(zDate, "20c:20e", &h, &m)!=2 ){ return 1; } zDate += 5; if( *zDate==':' ){ zDate++; - if( getDigits(zDate, 2, 0, 59, 0, &s)!=1 ){ + if( getDigits(zDate, "20e", &s)!=1 ){ return 1; } zDate += 2; if( *zDate=='.' && sqlite3Isdigit(zDate[1]) ){ double rScale = 1.0; @@ -265,11 +281,11 @@ zDate++; neg = 1; }else{ neg = 0; } - if( getDigits(zDate,4,0,9999,'-',&Y,2,1,12,'-',&M,2,1,31,0,&D)!=3 ){ + if( getDigits(zDate, "40f-21a-21d", &Y, &M, &D)!=3 ){ return 1; } zDate += 10; while( sqlite3Isspace(*zDate) || 'T'==*(u8*)zDate ){ zDate++; } if( parseHhMmSs(zDate, p)==0 ){ Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -1133,14 +1133,15 @@ ExprList *pList, /* List to which to append. Might be NULL */ Expr *pExpr /* Expression to be appended. Might be NULL */ ){ sqlite3 *db = pParse->db; if( pList==0 ){ - pList = sqlite3DbMallocZero(db, sizeof(ExprList) ); + pList = sqlite3DbMallocRaw(db, sizeof(ExprList) ); if( pList==0 ){ goto no_mem; } + pList->nExpr = 0; pList->a = sqlite3DbMallocRaw(db, sizeof(pList->a[0])); if( pList->a==0 ) goto no_mem; }else if( (pList->nExpr & (pList->nExpr-1))==0 ){ struct ExprList_item *a; assert( pList->nExpr>0 ); @@ -2894,11 +2895,11 @@ nFarg = pFarg ? pFarg->nExpr : 0; assert( !ExprHasProperty(pExpr, EP_IntValue) ); zId = pExpr->u.zToken; nId = sqlite3Strlen30(zId); pDef = sqlite3FindFunction(db, zId, nId, nFarg, enc, 0); - if( pDef==0 || pDef->xFunc==0 ){ + if( pDef==0 || pDef->xFinalize!=0 ){ sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId); break; } /* Attempt a direct implementation of the built-in COALESCE() and Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -1593,11 +1593,11 @@ if( useSeekResult ) pik_flags = OPFLAG_USESEEKRESULT; if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ assert( pParse->nested==0 ); pik_flags |= OPFLAG_NCHANGE; } - if( pik_flags ) sqlite3VdbeChangeP5(v, pik_flags); + sqlite3VdbeChangeP5(v, pik_flags); } if( !HasRowid(pTab) ) return; regData = regNewData + 1; regRec = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec); @@ -2009,13 +2009,13 @@ }else{ addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); assert( (pDest->tabFlags & TF_Autoincrement)==0 ); } sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData); - sqlite3VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid); + sqlite3VdbeAddOp4(v, OP_Insert, iDest, regData, regRowid, + pDest->zName, 0); sqlite3VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND); - sqlite3VdbeChangeP4(v, -1, pDest->zName, 0); sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); }else{ sqlite3TableLock(pParse, iDbDest, pDest->tnum, 1, pDest->zName); Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -1580,11 +1580,11 @@ sqlite3 *db, const char *zFunctionName, int nArg, int enc, void *pUserData, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), void (*xStep)(sqlite3_context*,int,sqlite3_value **), void (*xFinal)(sqlite3_context*), FuncDestructor *pDestructor ){ FuncDef *p; @@ -1591,13 +1591,13 @@ int nName; int extraFlags; assert( sqlite3_mutex_held(db->mutex) ); if( zFunctionName==0 || - (xFunc && (xFinal || xStep)) || - (!xFunc && (xFinal && !xStep)) || - (!xFunc && (!xFinal && xStep)) || + (xSFunc && (xFinal || xStep)) || + (!xSFunc && (xFinal && !xStep)) || + (!xSFunc && (!xFinal && xStep)) || (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) || (255<(nName = sqlite3Strlen30( zFunctionName))) ){ return SQLITE_MISUSE_BKPT; } @@ -1616,14 +1616,14 @@ if( enc==SQLITE_UTF16 ){ enc = SQLITE_UTF16NATIVE; }else if( enc==SQLITE_ANY ){ int rc; rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8|extraFlags, - pUserData, xFunc, xStep, xFinal, pDestructor); + pUserData, xSFunc, xStep, xFinal, pDestructor); if( rc==SQLITE_OK ){ rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE|extraFlags, - pUserData, xFunc, xStep, xFinal, pDestructor); + pUserData, xSFunc, xStep, xFinal, pDestructor); } if( rc!=SQLITE_OK ){ return rc; } enc = SQLITE_UTF16BE; @@ -1663,12 +1663,11 @@ pDestructor->nRef++; } p->pDestructor = pDestructor; p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; testcase( p->funcFlags & SQLITE_DETERMINISTIC ); - p->xFunc = xFunc; - p->xStep = xStep; + p->xSFunc = xSFunc ? xSFunc : xStep; p->xFinalize = xFinal; p->pUserData = pUserData; p->nArg = (u16)nArg; return SQLITE_OK; } @@ -1680,25 +1679,25 @@ sqlite3 *db, const char *zFunc, int nArg, int enc, void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), void (*xStep)(sqlite3_context*,int,sqlite3_value **), void (*xFinal)(sqlite3_context*) ){ - return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xFunc, xStep, + return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xSFunc, xStep, xFinal, 0); } int sqlite3_create_function_v2( sqlite3 *db, const char *zFunc, int nArg, int enc, void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), void (*xStep)(sqlite3_context*,int,sqlite3_value **), void (*xFinal)(sqlite3_context*), void (*xDestroy)(void *) ){ int rc = SQLITE_ERROR; @@ -1717,11 +1716,11 @@ goto out; } pArg->xDestroy = xDestroy; pArg->pUserData = p; } - rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xFunc, xStep, xFinal, pArg); + rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xSFunc, xStep, xFinal, pArg); if( pArg && pArg->nRef==0 ){ assert( rc!=SQLITE_OK ); xDestroy(p); sqlite3DbFree(db, pArg); } @@ -1737,11 +1736,11 @@ sqlite3 *db, const void *zFunctionName, int nArg, int eTextRep, void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**), void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ){ int rc; char *zFunc8; @@ -1750,11 +1749,11 @@ if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT; #endif sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); - rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0); + rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xSFunc,xStep,xFinal,0); sqlite3DbFree(db, zFunc8); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; } @@ -3064,11 +3063,10 @@ sqlite3GlobalConfig.nLookaside); sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); opendb_out: - sqlite3_free(zOpen); if( db ){ assert( db->mutex!=0 || isThreadsafe==0 || sqlite3GlobalConfig.bFullMutex==0 ); sqlite3_mutex_leave(db->mutex); } @@ -3124,10 +3122,11 @@ } sqlite3_key_v2(db, 0, zKey, i/2); } } #endif + sqlite3_free(zOpen); return rc & 0xff; } /* ** Open a new database handle. Index: src/malloc.c ================================================================== --- src/malloc.c +++ src/malloc.c @@ -581,12 +581,13 @@ } return p; } /* -** Allocate and zero memory. If the allocation fails, make -** the mallocFailed flag in the connection pointer. +** Allocate memory, either lookaside (if possible) or heap. +** If the allocation fails, set the mallocFailed flag in +** the connection pointer. ** ** If db!=0 and db->mallocFailed is true (indicating a prior malloc ** failure on the same database connection) then always return 0. ** Hence for a particular database connection, once malloc starts ** failing, it fails consistently until mallocFailed is reset. @@ -598,12 +599,12 @@ ** if( b ) a[10] = 9; ** ** In other words, if a subsequent malloc (ex: "b") worked, it is assumed ** that all prior mallocs (ex: "a") worked too. */ +static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n); void *sqlite3DbMallocRaw(sqlite3 *db, u64 n){ - void *p; assert( db==0 || sqlite3_mutex_held(db->mutex) ); assert( db==0 || db->pnBytesFreed==0 ); #ifndef SQLITE_OMIT_LOOKASIDE if( db ){ LookasideSlot *pBuf; @@ -629,11 +630,14 @@ #else if( db && db->mallocFailed ){ return 0; } #endif - p = sqlite3Malloc(n); + return dbMallocRawFinish(db, n); +} +static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n){ + void *p = sqlite3Malloc(n); if( !p && db ){ db->mallocFailed = 1; } sqlite3MemdebugSetType(p, (db && db->lookaside.bEnabled) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP); Index: src/pragma.c ================================================================== --- src/pragma.c +++ src/pragma.c @@ -428,19 +428,21 @@ { OP_IfPos, 1, 8, 0}, { OP_Integer, 0, 1, 0}, /* 6 */ { OP_Noop, 0, 0, 0}, { OP_ResultRow, 1, 1, 0}, }; - int addr; + VdbeOp *aOp; sqlite3VdbeUsesBtree(v, iDb); if( !zRight ){ setOneColumnName(v, "cache_size"); pParse->nMem += 2; - addr = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize,iLn); - sqlite3VdbeChangeP1(v, addr, iDb); - sqlite3VdbeChangeP1(v, addr+1, iDb); - sqlite3VdbeChangeP1(v, addr+6, SQLITE_DEFAULT_CACHE_SIZE); + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(getCacheSize)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize, iLn); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[6].p1 = SQLITE_DEFAULT_CACHE_SIZE; }else{ int size = sqlite3AbsInt32(sqlite3Atoi(zRight)); sqlite3BeginWriteOperation(pParse, 0, iDb); sqlite3VdbeAddOp2(v, OP_Integer, size, 1); sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_DEFAULT_CACHE_SIZE, 1); @@ -696,17 +698,20 @@ { OP_If, 1, 0, 0}, /* 2 */ { OP_Halt, SQLITE_OK, OE_Abort, 0}, /* 3 */ { OP_Integer, 0, 1, 0}, /* 4 */ { OP_SetCookie, 0, BTREE_INCR_VACUUM, 1}, /* 5 */ }; - int iAddr; - iAddr = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6, iLn); - sqlite3VdbeChangeP1(v, iAddr, iDb); - sqlite3VdbeChangeP1(v, iAddr+1, iDb); - sqlite3VdbeChangeP2(v, iAddr+2, iAddr+4); - sqlite3VdbeChangeP1(v, iAddr+4, eAuto-1); - sqlite3VdbeChangeP1(v, iAddr+5, iDb); + VdbeOp *aOp; + int iAddr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(setMeta6)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6, iLn); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[2].p2 = iAddr+4; + aOp[4].p1 = eAuto - 1; + aOp[5].p1 = iDb; sqlite3VdbeUsesBtree(v, iDb); } } break; } @@ -1411,22 +1416,10 @@ ** without most of the overhead of a full integrity-check. */ case PragTyp_INTEGRITY_CHECK: { int i, j, addr, mxErr; - /* Code that appears at the end of the integrity check. If no error - ** messages have been generated, output OK. Otherwise output the - ** error message - */ - static const int iLn = VDBE_OFFSET_LINENO(2); - static const VdbeOpList endCode[] = { - { OP_AddImm, 1, 0, 0}, /* 0 */ - { OP_If, 1, 0, 0}, /* 1 */ - { OP_String8, 0, 3, 0}, /* 2 */ - { OP_ResultRow, 3, 1, 0}, - }; - int isQuick = (sqlite3Tolower(zLeft[0])=='q'); /* If the PRAGMA command was of the form "PRAGMA .integrity_check", ** then iDb is set to the index of the database identified by . ** In this case, the integrity of database iDb only is verified by @@ -1619,14 +1612,28 @@ sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1); } #endif /* SQLITE_OMIT_BTREECOUNT */ } } - addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); - sqlite3VdbeChangeP2(v, addr, -mxErr); - sqlite3VdbeJumpHere(v, addr+1); - sqlite3VdbeChangeP4(v, addr+2, "ok", P4_STATIC); + { + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList endCode[] = { + { OP_AddImm, 1, 0, 0}, /* 0 */ + { OP_If, 1, 0, 0}, /* 1 */ + { OP_String8, 0, 3, 0}, /* 2 */ + { OP_ResultRow, 3, 1, 0}, + }; + VdbeOp *aOp; + + aOp = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); + if( aOp ){ + aOp[0].p2 = -mxErr; + aOp[1].p2 = sqlite3VdbeCurrentAddr(v); + aOp[2].p4type = P4_STATIC; + aOp[2].p4.z = "ok"; + } + } } break; #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ #ifndef SQLITE_OMIT_UTF16 @@ -1739,26 +1746,32 @@ static const VdbeOpList setCookie[] = { { OP_Transaction, 0, 1, 0}, /* 0 */ { OP_Integer, 0, 1, 0}, /* 1 */ { OP_SetCookie, 0, 0, 1}, /* 2 */ }; - int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie, 0); - sqlite3VdbeChangeP1(v, addr, iDb); - sqlite3VdbeChangeP1(v, addr+1, sqlite3Atoi(zRight)); - sqlite3VdbeChangeP1(v, addr+2, iDb); - sqlite3VdbeChangeP2(v, addr+2, iCookie); + VdbeOp *aOp; + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(setCookie)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie, 0); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = sqlite3Atoi(zRight); + aOp[2].p1 = iDb; + aOp[2].p2 = iCookie; }else{ /* Read the specified cookie value */ static const VdbeOpList readCookie[] = { { OP_Transaction, 0, 0, 0}, /* 0 */ { OP_ReadCookie, 0, 1, 0}, /* 1 */ { OP_ResultRow, 1, 1, 0} }; - int addr = sqlite3VdbeAddOpList(v, ArraySize(readCookie), readCookie, 0); - sqlite3VdbeChangeP1(v, addr, iDb); - sqlite3VdbeChangeP1(v, addr+1, iDb); - sqlite3VdbeChangeP3(v, addr+1, iCookie); + VdbeOp *aOp; + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(readCookie)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(readCookie),readCookie,0); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[1].p3 = iCookie; sqlite3VdbeSetNumCols(v, 1); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); } } break; Index: src/prepare.c ================================================================== --- src/prepare.c +++ src/prepare.c @@ -138,65 +138,31 @@ int rc; int i; #ifndef SQLITE_OMIT_DEPRECATED int size; #endif - Table *pTab; Db *pDb; char const *azArg[4]; int meta[5]; InitData initData; - char const *zMasterSchema; - char const *zMasterName; + const char *zMasterName; int openedTransaction = 0; - /* - ** The master database table has a structure like this - */ - static const char master_schema[] = - "CREATE TABLE sqlite_master(\n" - " type text,\n" - " name text,\n" - " tbl_name text,\n" - " rootpage integer,\n" - " sql text\n" - ")" - ; -#ifndef SQLITE_OMIT_TEMPDB - static const char temp_master_schema[] = - "CREATE TEMP TABLE sqlite_temp_master(\n" - " type text,\n" - " name text,\n" - " tbl_name text,\n" - " rootpage integer,\n" - " sql text\n" - ")" - ; -#else - #define temp_master_schema 0 -#endif - assert( iDb>=0 && iDbnDb ); assert( db->aDb[iDb].pSchema ); assert( sqlite3_mutex_held(db->mutex) ); assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) ); - /* zMasterSchema and zInitScript are set to point at the master schema - ** and initialisation script appropriate for the database being - ** initialized. zMasterName is the name of the master table. - */ - if( !OMIT_TEMPDB && iDb==1 ){ - zMasterSchema = temp_master_schema; - }else{ - zMasterSchema = master_schema; - } - zMasterName = SCHEMA_TABLE(iDb); - - /* Construct the schema tables. */ - azArg[0] = zMasterName; + /* Construct the in-memory representation schema tables (sqlite_master or + ** sqlite_temp_master) by invoking the parser directly. The appropriate + ** table name will be inserted automatically by the parser so we can just + ** use the abbreviation "x" here. The parser will also automatically tag + ** the schema table as read-only. */ + azArg[0] = zMasterName = SCHEMA_TABLE(iDb); azArg[1] = "1"; - azArg[2] = zMasterSchema; + azArg[2] = "CREATE TABLE x(type text,name text,tbl_name text," + "rootpage integer,sql text)"; azArg[3] = 0; initData.db = db; initData.iDb = iDb; initData.rc = SQLITE_OK; initData.pzErrMsg = pzErrMsg; @@ -203,14 +169,10 @@ sqlite3InitCallback(&initData, 3, (char **)azArg, 0); if( initData.rc ){ rc = initData.rc; goto error_out; } - pTab = sqlite3FindTable(db, zMasterName, db->aDb[iDb].zName); - if( ALWAYS(pTab) ){ - pTab->tabFlags |= TF_Readonly; - } /* Create a cursor to hold the database open */ pDb = &db->aDb[iDb]; if( pDb->pBt==0 ){ @@ -325,11 +287,11 @@ */ assert( db->init.busy ); { char *zSql; zSql = sqlite3MPrintf(db, - "SELECT name, rootpage, sql FROM '%q'.%s ORDER BY rowid", + "SELECT name, rootpage, sql FROM \"%w\".%s ORDER BY rowid", db->aDb[iDb].zName, zMasterName); #ifndef SQLITE_OMIT_AUTHORIZATION { sqlite3_xauth xAuth; xAuth = db->xAuth; Index: src/resolve.c ================================================================== --- src/resolve.c +++ src/resolve.c @@ -663,11 +663,11 @@ no_such_func = 1; }else{ wrong_num_args = 1; } }else{ - is_agg = pDef->xFunc==0; + is_agg = pDef->xFinalize!=0; if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ ExprSetProperty(pExpr, EP_Unlikely|EP_Skip); if( n==2 ){ pExpr->iTable = exprProbability(pList->a[1].pExpr); if( pExpr->iTable<0 ){ @@ -1391,14 +1391,16 @@ pParse->nHeight += pExpr->nHeight; } #endif savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg); pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg); - memset(&w, 0, sizeof(w)); + w.pParse = pNC->pParse; w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; - w.pParse = pNC->pParse; + w.xSelectCallback2 = 0; + w.walkerDepth = 0; + w.eCode = 0; w.u.pNC = pNC; sqlite3WalkExpr(&w, pExpr); #if SQLITE_MAX_EXPR_DEPTH>0 pNC->pParse->nHeight -= pExpr->nHeight; #endif Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -1003,19 +1003,20 @@ /* ** Allocate a KeyInfo object sufficient for an index of N key columns and ** X extra columns. */ KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ - KeyInfo *p = sqlite3DbMallocZero(0, - sizeof(KeyInfo) + (N+X)*(sizeof(CollSeq*)+1)); + int nExtra = (N+X)*(sizeof(CollSeq*)+1); + KeyInfo *p = sqlite3Malloc(sizeof(KeyInfo) + nExtra); if( p ){ p->aSortOrder = (u8*)&p->aColl[N+X]; p->nField = (u16)N; p->nXField = (u16)X; p->enc = ENC(db); p->db = db; p->nRef = 1; + memset(&p[1], 0, nExtra); }else{ db->mallocFailed = 1; } return p; } Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -400,10 +400,25 @@ #else # define ALWAYS(X) (X) # define NEVER(X) (X) #endif +/* +** Some malloc failures are only possible if SQLITE_TEST_REALLOC_STRESS is +** defined. We need to defend against those failures when testing with +** SQLITE_TEST_REALLOC_STRESS, but we don't want the unreachable branches +** during a normal build. The following macro can be used to disable tests +** that are always false except when SQLITE_TEST_REALLOC_STRESS is set. +*/ +#if defined(SQLITE_TEST_REALLOC_STRESS) +# define ONLY_IF_REALLOC_STRESS(X) (X) +#elif !defined(NDEBUG) +# define ONLY_IF_REALLOC_STRESS(X) ((X)?(assert(0),1):0) +#else +# define ONLY_IF_REALLOC_STRESS(X) (0) +#endif + /* ** Declarations used for tracing the operating system interfaces. */ #if defined(SQLITE_FORCE_OS_TRACE) || defined(SQLITE_TEST) || \ (defined(SQLITE_DEBUG) && SQLITE_OS_WIN) @@ -1371,13 +1386,12 @@ struct FuncDef { i16 nArg; /* Number of arguments. -1 means unlimited */ u16 funcFlags; /* Some combination of SQLITE_FUNC_* */ void *pUserData; /* User data parameter */ FuncDef *pNext; /* Next function with same name */ - void (*xFunc)(sqlite3_context*,int,sqlite3_value**); /* Regular function */ - void (*xStep)(sqlite3_context*,int,sqlite3_value**); /* Aggregate step */ - void (*xFinalize)(sqlite3_context*); /* Aggregate finalizer */ + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**); /* func or agg-step */ + void (*xFinalize)(sqlite3_context*); /* Agg finalizer */ char *zName; /* SQL name of the function. */ FuncDef *pHash; /* Next with a different name but the same hash */ FuncDestructor *pDestructor; /* Reference counted destructor function */ }; @@ -1456,32 +1470,32 @@ ** FuncDef.flags variable is set to the value passed as the flags ** parameter. */ #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, 0, 0} #define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, 0, 0} #define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, 0, 0} #define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, 0, 0} #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - pArg, 0, xFunc, 0, 0, #zName, 0, 0} + pArg, 0, xFunc, 0, #zName, 0, 0} #define LIKEFUNC(zName, nArg, arg, flags) \ {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ - (void *)arg, 0, likeFunc, 0, 0, #zName, 0, 0} + (void *)arg, 0, likeFunc, 0, #zName, 0, 0} #define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \ {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0} + SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName,0,0} #define AGGREGATE2(zName, nArg, arg, nc, xStep, xFinal, extraFlags) \ {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|extraFlags, \ - SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0} + SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName,0,0} /* ** All current savepoints are stored in a linked list starting at ** sqlite3.pSavepoint. The first element in the list is the most recently ** opened savepoint. Savepoints are added to the list by the vdbe @@ -2788,11 +2802,11 @@ ** each recursion. The boundary between these two regions is determined ** using offsetof(Parse,nVar) so the nVar field must be the first field ** in the recursive region. ************************************************************************/ - int nVar; /* Number of '?' variables seen in the SQL so far */ + ynVar nVar; /* Number of '?' variables seen in the SQL so far */ int nzVar; /* Number of available slots in azVar[] */ u8 iPkSortOrder; /* ASC or DESC for INTEGER PRIMARY KEY */ u8 explain; /* True if the EXPLAIN flag is found on the query */ #ifndef SQLITE_OMIT_VIRTUALTABLE u8 declareVtab; /* True if inside sqlite3_declare_vtab() */ @@ -3070,14 +3084,14 @@ /* ** Context pointer passed down through the tree-walk. */ struct Walker { + Parse *pParse; /* Parser context. */ int (*xExprCallback)(Walker*, Expr*); /* Callback for expressions */ int (*xSelectCallback)(Walker*,Select*); /* Callback for SELECTs */ void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ - Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ u8 eCode; /* A small processing code */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ int n; /* A counter */ Index: src/trigger.c ================================================================== --- src/trigger.c +++ src/trigger.c @@ -950,12 +950,12 @@ /* Code the OP_Program opcode in the parent VDBE. P4 of the OP_Program ** is a pointer to the sub-vdbe containing the trigger program. */ if( pPrg ){ int bRecursive = (p->zName && 0==(pParse->db->flags&SQLITE_RecTriggers)); - sqlite3VdbeAddOp3(v, OP_Program, reg, ignoreJump, ++pParse->nMem); - sqlite3VdbeChangeP4(v, -1, (const char *)pPrg->pProgram, P4_SUBPROGRAM); + sqlite3VdbeAddOp4(v, OP_Program, reg, ignoreJump, ++pParse->nMem, + (const char *)pPrg->pProgram, P4_SUBPROGRAM); VdbeComment( (v, "Call: %s.%s", (p->zName?p->zName:"fkey"), onErrorText(orconf))); /* Set the P5 operand of the OP_Program instruction to non-zero if ** recursive invocation of this trigger program is disallowed. Recursive Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -1661,12 +1661,12 @@ } #endif MemSetTypeFlag(pCtx->pOut, MEM_Null); pCtx->fErrorOrAux = 0; db->lastRowid = lastRowid; - (*pCtx->pFunc->xFunc)(pCtx, pCtx->argc, pCtx->argv); /* IMP: R-24505-23230 */ - lastRowid = db->lastRowid; /* Remember rowid changes made by xFunc */ + (*pCtx->pFunc->xSFunc)(pCtx, pCtx->argc, pCtx->argv);/* IMP: R-24505-23230 */ + lastRowid = db->lastRowid; /* Remember rowid changes made by xSFunc */ /* If the function returned an error, throw an exception */ if( pCtx->fErrorOrAux ){ if( pCtx->isError ){ sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut)); @@ -5098,10 +5098,11 @@ case OP_Destroy: { /* out2 */ int iMoved; int iDb; assert( p->readOnly==0 ); + assert( pOp->p1>1 ); pOut = out2Prerelease(p, pOp); pOut->flags = MEM_Null; if( db->nVdbeRead > db->nVDestroy+1 ){ rc = SQLITE_LOCKED; p->errorAction = OE_Abort; @@ -5902,11 +5903,11 @@ pMem->n++; sqlite3VdbeMemInit(&t, db, MEM_Null); pCtx->pOut = &t; pCtx->fErrorOrAux = 0; pCtx->skipFlag = 0; - (pCtx->pFunc->xStep)(pCtx,pCtx->argc,pCtx->argv); /* IMP: R-24505-23230 */ + (pCtx->pFunc->xSFunc)(pCtx,pCtx->argc,pCtx->argv); /* IMP: R-24505-23230 */ if( pCtx->fErrorOrAux ){ if( pCtx->isError ){ sqlite3VdbeError(p, "%s", sqlite3_value_text(&t)); rc = pCtx->isError; } Index: src/vdbe.h ================================================================== --- src/vdbe.h +++ src/vdbe.h @@ -178,19 +178,24 @@ void sqlite3VdbeMultiLoad(Vdbe*,int,const char*,...); int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); int sqlite3VdbeAddOp4Dup8(Vdbe*,int,int,int,int,const u8*,int); int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); -int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno); +#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) + void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N); +#else +# define sqlite3VdbeVerifyNoMallocRequired(A,B) +#endif +VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno); void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8); void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1); void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2); void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); void sqlite3VdbeChangeP5(Vdbe*, u8 P5); void sqlite3VdbeJumpHere(Vdbe*, int addr); -void sqlite3VdbeChangeToNoop(Vdbe*, int addr); +int sqlite3VdbeChangeToNoop(Vdbe*, int addr); int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op); void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N); void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); void sqlite3VdbeUsesBtree(Vdbe*, int); VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); Index: src/vdbeapi.c ================================================================== --- src/vdbeapi.c +++ src/vdbeapi.c @@ -793,11 +793,11 @@ ** Allocate or return the aggregate context for a user function. A new ** context is allocated on the first call. Subsequent calls return the ** same context that was returned on prior calls. */ void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ - assert( p && p->pFunc && p->pFunc->xStep ); + assert( p && p->pFunc && p->pFunc->xFinalize ); assert( sqlite3_mutex_held(p->pOut->db->mutex) ); testcase( nByte<0 ); if( (p->pMem->flags & MEM_Agg)==0 ){ return createAggContext(p, nByte); }else{ @@ -884,11 +884,11 @@ ** provide only to avoid breaking legacy code. New aggregate function ** implementations should keep their own counts within their aggregate ** context. */ int sqlite3_aggregate_count(sqlite3_context *p){ - assert( p && p->pMem && p->pFunc && p->pFunc->xStep ); + assert( p && p->pMem && p->pFunc && p->pFunc->xFinalize ); return p->pMem->n; } #endif /* Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -248,12 +248,11 @@ char c; va_start(ap, zTypes); for(i=0; (c = zTypes[i])!=0; i++){ if( c=='s' ){ const char *z = va_arg(ap, const char*); - int addr = sqlite3VdbeAddOp2(p, z==0 ? OP_Null : OP_String8, 0, iDest++); - if( z ) sqlite3VdbeChangeP4(p, addr, z, 0); + sqlite3VdbeAddOp4(p, z==0 ? OP_Null : OP_String8, 0, iDest++, 0, z, 0); }else{ assert( c=='i' ); sqlite3VdbeAddOp2(p, OP_Integer, va_arg(ap, int), iDest++); } } @@ -604,10 +603,24 @@ int sqlite3VdbeCurrentAddr(Vdbe *p){ assert( p->magic==VDBE_MAGIC_INIT ); return p->nOp; } +/* +** Verify that at least N opcode slots are available in p without +** having to malloc for more space (except when compiled using +** SQLITE_TEST_REALLOC_STRESS). This interface is used during testing +** to verify that certain calls to sqlite3VdbeAddOpList() can never +** fail due to a OOM fault and hence that the return value from +** sqlite3VdbeAddOpList() will always be non-NULL. +*/ +#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) +void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N){ + assert( p->nOp + N <= p->pParse->nOpAlloc ); +} +#endif + /* ** This function returns a pointer to the array of opcodes associated with ** the Vdbe passed as the first argument. It is the callers responsibility ** to arrange for the returned array to be eventually freed using the ** vdbeFreeOpArray() function. @@ -629,23 +642,27 @@ p->aOp = 0; return aOp; } /* -** Add a whole list of operations to the operation stack. Return the -** address of the first operation added. +** Add a whole list of operations to the operation stack. Return a +** pointer to the first operation inserted. */ -int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){ - int addr, i; - VdbeOp *pOut; +VdbeOp *sqlite3VdbeAddOpList( + Vdbe *p, /* Add opcodes to the prepared statement */ + int nOp, /* Number of opcodes to add */ + VdbeOpList const *aOp, /* The opcodes to be added */ + int iLineno /* Source-file line number of first opcode */ +){ + int i; + VdbeOp *pOut, *pFirst; assert( nOp>0 ); assert( p->magic==VDBE_MAGIC_INIT ); if( p->nOp + nOp > p->pParse->nOpAlloc && growOpArray(p, nOp) ){ return 0; } - addr = p->nOp; - pOut = &p->aOp[addr]; + pFirst = pOut = &p->aOp[p->nOp]; for(i=0; iopcode = aOp->opcode; pOut->p1 = aOp->p1; pOut->p2 = aOp->p2; assert( aOp->p2>=0 ); @@ -661,16 +678,16 @@ #else (void)iLineno; #endif #ifdef SQLITE_DEBUG if( p->db->flags & SQLITE_VdbeAddopTrace ){ - sqlite3VdbePrintOp(0, i+addr, &p->aOp[i+addr]); + sqlite3VdbePrintOp(0, i+p->nOp, &p->aOp[i+p->nOp]); } #endif } p->nOp += nOp; - return addr; + return pFirst; } #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) /* ** Add an entry to the array of counters managed by sqlite3_stmt_scanstatus(). @@ -714,11 +731,11 @@ } void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){ sqlite3VdbeGetOp(p,addr)->p3 = val; } void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){ - sqlite3VdbeGetOp(p,-1)->p5 = p5; + if( !p->db->mallocFailed ) p->aOp[p->nOp-1].p5 = p5; } /* ** Change the P2 operand of instruction addr so that it points to ** the address of the next instruction to be coded. @@ -824,28 +841,29 @@ } /* ** Change the opcode at addr into OP_Noop */ -void sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ - if( addrnOp ){ - VdbeOp *pOp = &p->aOp[addr]; - sqlite3 *db = p->db; - freeP4(db, pOp->p4type, pOp->p4.p); - memset(pOp, 0, sizeof(pOp[0])); - pOp->opcode = OP_Noop; - } +int sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ + VdbeOp *pOp; + if( p->db->mallocFailed ) return 0; + assert( addr>=0 && addrnOp ); + pOp = &p->aOp[addr]; + freeP4(p->db, pOp->p4type, pOp->p4.p); + pOp->p4type = P4_NOTUSED; + pOp->p4.z = 0; + pOp->opcode = OP_Noop; + return 1; } /* ** If the last opcode is "op" and it is not a jump destination, ** then remove it. Return true if and only if an opcode was removed. */ int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ if( (p->nOp-1)>(p->pParse->iFixedOp) && p->aOp[p->nOp-1].opcode==op ){ - sqlite3VdbeChangeToNoop(p, p->nOp-1); - return 1; + return sqlite3VdbeChangeToNoop(p, p->nOp-1); }else{ return 0; } } @@ -1880,11 +1898,10 @@ do { nByte = 0; p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), zCsr, &nFree, &nByte); p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), zCsr, &nFree, &nByte); p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), zCsr, &nFree, &nByte); - p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), zCsr, &nFree, &nByte); p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), zCsr, &nFree, &nByte); p->aOnceFlag = allocSpace(p->aOnceFlag, nOnce, zCsr, &nFree, &nByte); #ifdef SQLITE_ENABLE_STMT_SCANSTATUS p->anExec = allocSpace(p->anExec, p->nOp*sizeof(i64), zCsr, &nFree, &nByte); @@ -1903,15 +1920,14 @@ for(n=0; naVar[n].flags = MEM_Null; p->aVar[n].db = db; } } - if( p->azVar && pParse->nzVar>0 ){ - p->nzVar = pParse->nzVar; - memcpy(p->azVar, pParse->azVar, p->nzVar*sizeof(p->azVar[0])); - memset(pParse->azVar, 0, pParse->nzVar*sizeof(pParse->azVar[0])); - } + p->nzVar = pParse->nzVar; + p->azVar = pParse->azVar; + pParse->nzVar = 0; + pParse->azVar = 0; if( p->aMem ){ p->aMem--; /* aMem[] goes from 1..nMem */ p->nMem = nMem; /* not from 0..nMem-1 */ for(n=1; n<=nMem; n++){ p->aMem[n].flags = MEM_Undefined; @@ -2894,10 +2910,11 @@ pNext = pSub->pNext; vdbeFreeOpArray(db, pSub->aOp, pSub->nOp); sqlite3DbFree(db, pSub); } for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]); + sqlite3DbFree(db, p->azVar); vdbeFreeOpArray(db, p->aOp, p->nOp); sqlite3DbFree(db, p->aColName); sqlite3DbFree(db, p->zSql); sqlite3DbFree(db, p->pFree); #ifdef SQLITE_ENABLE_STMT_SCANSTATUS @@ -3638,13 +3655,13 @@ v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); n1 = v1==0 ? 0 : c1.n; v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); n2 = v2==0 ? 0 : c2.n; rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2); + if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM; sqlite3VdbeMemRelease(&c1); sqlite3VdbeMemRelease(&c2); - if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM; return rc; } } /* @@ -4428,12 +4445,14 @@ ** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored ** in memory obtained from sqlite3_malloc) into a Vdbe.zErrMsg (text stored ** in memory obtained from sqlite3DbMalloc). */ void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){ - sqlite3 *db = p->db; - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg); - sqlite3_free(pVtab->zErrMsg); - pVtab->zErrMsg = 0; + if( pVtab->zErrMsg ){ + sqlite3 *db = p->db; + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg); + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = 0; + } } #endif /* SQLITE_OMIT_VIRTUALTABLE */ Index: src/vdbeblob.c ================================================================== --- src/vdbeblob.c +++ src/vdbeblob.c @@ -113,42 +113,10 @@ int flags, /* True -> read/write access, false -> read-only */ sqlite3_blob **ppBlob /* Handle for accessing the blob returned here */ ){ int nAttempt = 0; int iCol; /* Index of zColumn in row-record */ - - /* This VDBE program seeks a btree cursor to the identified - ** db/table/row entry. The reason for using a vdbe program instead - ** of writing code to use the b-tree layer directly is that the - ** vdbe program will take advantage of the various transaction, - ** locking and error handling infrastructure built into the vdbe. - ** - ** After seeking the cursor, the vdbe executes an OP_ResultRow. - ** Code external to the Vdbe then "borrows" the b-tree cursor and - ** uses it to implement the blob_read(), blob_write() and - ** blob_bytes() functions. - ** - ** The sqlite3_blob_close() function finalizes the vdbe program, - ** which closes the b-tree cursor and (possibly) commits the - ** transaction. - */ - static const int iLn = VDBE_OFFSET_LINENO(4); - static const VdbeOpList openBlob[] = { - /* {OP_Transaction, 0, 0, 0}, // 0: Inserted separately */ - {OP_TableLock, 0, 0, 0}, /* 1: Acquire a read or write lock */ - /* One of the following two instructions is replaced by an OP_Noop. */ - {OP_OpenRead, 0, 0, 0}, /* 2: Open cursor 0 for reading */ - {OP_OpenWrite, 0, 0, 0}, /* 3: Open cursor 0 for read/write */ - {OP_Variable, 1, 1, 1}, /* 4: Push the rowid to the stack */ - {OP_NotExists, 0, 10, 1}, /* 5: Seek the cursor */ - {OP_Column, 0, 0, 1}, /* 6 */ - {OP_ResultRow, 1, 0, 0}, /* 7 */ - {OP_Goto, 0, 4, 0}, /* 8 */ - {OP_Close, 0, 0, 0}, /* 9 */ - {OP_Halt, 0, 0, 0}, /* 10 */ - }; - int rc = SQLITE_OK; char *zErr = 0; Table *pTab; Parse *pParse = 0; Incrblob *pBlob = 0; @@ -263,49 +231,84 @@ } pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(pParse); assert( pBlob->pStmt || db->mallocFailed ); if( pBlob->pStmt ){ + + /* This VDBE program seeks a btree cursor to the identified + ** db/table/row entry. The reason for using a vdbe program instead + ** of writing code to use the b-tree layer directly is that the + ** vdbe program will take advantage of the various transaction, + ** locking and error handling infrastructure built into the vdbe. + ** + ** After seeking the cursor, the vdbe executes an OP_ResultRow. + ** Code external to the Vdbe then "borrows" the b-tree cursor and + ** uses it to implement the blob_read(), blob_write() and + ** blob_bytes() functions. + ** + ** The sqlite3_blob_close() function finalizes the vdbe program, + ** which closes the b-tree cursor and (possibly) commits the + ** transaction. + */ + static const int iLn = VDBE_OFFSET_LINENO(4); + static const VdbeOpList openBlob[] = { + /* addr/ofst */ + /* {OP_Transaction, 0, 0, 0}, // 0/ inserted separately */ + {OP_TableLock, 0, 0, 0}, /* 1/0: Acquire a read or write lock */ + {OP_OpenRead, 0, 0, 0}, /* 2/1: Open a cursor */ + {OP_Variable, 1, 1, 0}, /* 3/2: Move ?1 into reg[1] */ + {OP_NotExists, 0, 8, 1}, /* 4/3: Seek the cursor */ + {OP_Column, 0, 0, 1}, /* 5/4 */ + {OP_ResultRow, 1, 0, 0}, /* 6/5 */ + {OP_Goto, 0, 3, 0}, /* 7/6 */ + {OP_Close, 0, 0, 0}, /* 8/7 */ + {OP_Halt, 0, 0, 0}, /* 9/8 */ + }; Vdbe *v = (Vdbe *)pBlob->pStmt; int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - + VdbeOp *aOp; sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, flags, pTab->pSchema->schema_cookie, pTab->pSchema->iGeneration); sqlite3VdbeChangeP5(v, 1); - sqlite3VdbeAddOpList(v, ArraySize(openBlob), openBlob, iLn); + aOp = sqlite3VdbeAddOpList(v, ArraySize(openBlob), openBlob, iLn); /* Make sure a mutex is held on the table to be accessed */ sqlite3VdbeUsesBtree(v, iDb); - /* Configure the OP_TableLock instruction */ + if( db->mallocFailed==0 ){ + assert( aOp!=0 ); + /* Configure the OP_TableLock instruction */ #ifdef SQLITE_OMIT_SHARED_CACHE - sqlite3VdbeChangeToNoop(v, 1); + aOp[0].opcode = OP_Noop; #else - sqlite3VdbeChangeP1(v, 1, iDb); - sqlite3VdbeChangeP2(v, 1, pTab->tnum); - sqlite3VdbeChangeP3(v, 1, flags); - sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT); + aOp[0].p1 = iDb; + aOp[0].p2 = pTab->tnum; + aOp[0].p3 = flags; + sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT); + } + if( db->mallocFailed==0 ){ #endif - /* Remove either the OP_OpenWrite or OpenRead. Set the P2 - ** parameter of the other to pTab->tnum. */ - sqlite3VdbeChangeToNoop(v, 3 - flags); - sqlite3VdbeChangeP2(v, 2 + flags, pTab->tnum); - sqlite3VdbeChangeP3(v, 2 + flags, iDb); - - /* Configure the number of columns. Configure the cursor to - ** think that the table has one more column than it really - ** does. An OP_Column to retrieve this imaginary column will - ** always return an SQL NULL. This is useful because it means - ** we can invoke OP_Column to fill in the vdbe cursors type - ** and offset cache without causing any IO. - */ - sqlite3VdbeChangeP4(v, 2+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32); - sqlite3VdbeChangeP2(v, 6, pTab->nCol); - if( !db->mallocFailed ){ + /* Remove either the OP_OpenWrite or OpenRead. Set the P2 + ** parameter of the other to pTab->tnum. */ + if( flags ) aOp[1].opcode = OP_OpenWrite; + aOp[1].p2 = pTab->tnum; + aOp[1].p3 = iDb; + + /* Configure the number of columns. Configure the cursor to + ** think that the table has one more column than it really + ** does. An OP_Column to retrieve this imaginary column will + ** always return an SQL NULL. This is useful because it means + ** we can invoke OP_Column to fill in the vdbe cursors type + ** and offset cache without causing any IO. + */ + aOp[1].p4type = P4_INT32; + aOp[1].p4.i = pTab->nCol+1; + aOp[4].p2 = pTab->nCol; + pParse->nVar = 1; pParse->nMem = 1; pParse->nTab = 1; sqlite3VdbeMakeReady(v, pParse); } Index: src/vdbemem.c ================================================================== --- src/vdbemem.c +++ src/vdbemem.c @@ -1222,11 +1222,11 @@ assert( pCtx->pParse->rc==SQLITE_OK ); memset(&ctx, 0, sizeof(ctx)); ctx.pOut = pVal; ctx.pFunc = pFunc; - pFunc->xFunc(&ctx, nVal, apVal); + pFunc->xSFunc(&ctx, nVal, apVal); if( ctx.isError ){ rc = ctx.isError; sqlite3ErrorMsg(pCtx->pParse, "%s", sqlite3_value_text(pVal)); }else{ sqlite3ValueApplyAffinity(pVal, aff, SQLITE_UTF8); Index: src/vtab.c ================================================================== --- src/vtab.c +++ src/vtab.c @@ -1014,11 +1014,11 @@ Expr *pExpr /* First argument to the function */ ){ Table *pTab; sqlite3_vtab *pVtab; sqlite3_module *pMod; - void (*xFunc)(sqlite3_context*,int,sqlite3_value**) = 0; + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**) = 0; void *pArg = 0; FuncDef *pNew; int rc = 0; char *zLowerName; unsigned char *z; @@ -1042,11 +1042,11 @@ zLowerName = sqlite3DbStrDup(db, pDef->zName); if( zLowerName ){ for(z=(unsigned char*)zLowerName; *z; z++){ *z = sqlite3UpperToLower[*z]; } - rc = pMod->xFindFunction(pVtab, nArg, zLowerName, &xFunc, &pArg); + rc = pMod->xFindFunction(pVtab, nArg, zLowerName, &xSFunc, &pArg); sqlite3DbFree(db, zLowerName); } if( rc==0 ){ return pDef; } @@ -1059,11 +1059,11 @@ return pDef; } *pNew = *pDef; pNew->zName = (char *)&pNew[1]; memcpy(pNew->zName, pDef->zName, sqlite3Strlen30(pDef->zName)+1); - pNew->xFunc = xFunc; + pNew->xSFunc = xSFunc; pNew->pUserData = pArg; pNew->funcFlags |= SQLITE_FUNC_EPHEM; return pNew; } Index: src/wherecode.c ================================================================== --- src/wherecode.c +++ src/wherecode.c @@ -325,12 +325,11 @@ n--; } /* Code the OP_Affinity opcode if there is anything left to do. */ if( n>0 ){ - sqlite3VdbeAddOp2(v, OP_Affinity, base, n); - sqlite3VdbeChangeP4(v, -1, zAff, n); + sqlite3VdbeAddOp4(v, OP_Affinity, base, n, 0, zAff, n); sqlite3ExprCacheAffinityChange(pParse, base, n); } } Index: src/whereexpr.c ================================================================== --- src/whereexpr.c +++ src/whereexpr.c @@ -200,10 +200,11 @@ int cnt; /* Number of non-wildcard prefix characters */ char wc[3]; /* Wildcard characters */ sqlite3 *db = pParse->db; /* Database connection */ sqlite3_value *pVal = 0; int op; /* Opcode of pRight */ + int rc; /* Result code to return */ if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){ return 0; } #ifdef SQLITE_EBCDIC @@ -265,12 +266,13 @@ }else{ z = 0; } } + rc = (z!=0); sqlite3ValueFree(pVal); - return (z!=0); + return rc; } #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ #ifndef SQLITE_OMIT_VIRTUALTABLE Index: test/threadtest3.c ================================================================== --- test/threadtest3.c +++ test/threadtest3.c @@ -878,11 +878,11 @@ static double currentTime(void){ double t; static sqlite3_vfs *pTimelimitVfs = 0; if( pTimelimitVfs==0 ) pTimelimitVfs = sqlite3_vfs_find(0); - if( pTimelimitVfs->iVersion>=1 && pTimelimitVfs->xCurrentTimeInt64!=0 ){ + if( pTimelimitVfs->iVersion>=2 && pTimelimitVfs->xCurrentTimeInt64!=0 ){ sqlite3_int64 tm; pTimelimitVfs->xCurrentTimeInt64(pTimelimitVfs, &tm); t = tm/86400000.0; }else{ pTimelimitVfs->xCurrentTime(pTimelimitVfs, &t);