Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -416,10 +416,38 @@ static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ sqlite3_int64 iVal; *pp += sqlite3Fts3GetVarint(*pp, &iVal); *pVal += iVal; } + +/* +** When this function is called, *pp points to the first byte following a +** varint that is part of a doclist (or position-list, or any other list +** of varints). This function moves *pp to point to the start of that varint, +** and decrements the value stored in *pVal by the varint value. +** +** Argument pStart points to the first byte of the doclist that the +** varint is part of. +*/ +static void fts3GetReverseDeltaVarint( + char **pp, + char *pStart, + sqlite3_int64 *pVal +){ + sqlite3_int64 iVal; + char *p = *pp; + + /* Pointer p now points at the first byte past the varint we are + ** interested in. So, unless the doclist is corrupt, the 0x80 bit is + ** clear on character p[-1]. */ + for(p = (*pp)-2; p>=pStart && *p&0x80; p--); + p++; + *pp = p; + + sqlite3Fts3GetVarint(p, &iVal); + *pVal -= iVal; +} /* ** As long as *pp has not reached its end (pEnd), then do the same ** as fts3GetDeltaVarint(): read a single varint and add it to *pVal. ** But if we have reached the end of the varint, just set *pp=0 and @@ -521,10 +549,12 @@ if( *pRc==SQLITE_OK ){ int i; /* Iterator variable */ int rc; /* Return code */ char *zSql; /* SQL statement passed to declare_vtab() */ char *zCols; /* List of user defined columns */ + + sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); /* Create a list of user columns for the virtual table */ zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); for(i=1; zCols && inColumn; i++){ zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); @@ -1090,10 +1120,26 @@ if( iCons>=0 ){ pInfo->aConstraintUsage[iCons].argvIndex = 1; pInfo->aConstraintUsage[iCons].omit = 1; } + + /* Regardless of the strategy selected, FTS can deliver rows in rowid (or + ** docid) order. Both ascending and descending are possible. + */ + if( pInfo->nOrderBy==1 ){ + struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; + if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ + if( pOrder->desc ){ + pInfo->idxStr = "DESC"; + }else{ + pInfo->idxStr = "ASC"; + } + } + pInfo->orderByConsumed = 1; + } + return SQLITE_OK; } /* ** Implementation of xOpen method. @@ -2994,16 +3040,24 @@ rc = sqlite3_reset(pCsr->pStmt); break; } pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); }else{ - if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ - pCsr->isEof = 1; - break; + if( pCsr->desc==0 ){ + if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ + pCsr->isEof = 1; + break; + } + fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); + }else{ + fts3GetReverseDeltaVarint(&pCsr->pNextId,pCsr->aDoclist,&pCsr->iPrevId); + if( pCsr->pNextId<=pCsr->aDoclist ){ + pCsr->isEof = 1; + break; + } } sqlite3_reset(pCsr->pStmt); - fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; } }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 ); @@ -3032,12 +3086,12 @@ const char *idxStr, /* Unused */ int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ const char *azSql[] = { - "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?", /* non-full-scan */ - "SELECT %s FROM %Q.'%q_content' AS x ", /* full-scan */ + "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?", /* non-full-scan */ + "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s", /* full-scan */ }; int rc; /* Return code */ char *zSql; /* SQL statement used to access %_content */ Fts3Table *p = (Fts3Table *)pCursor->pVtab; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; @@ -3089,11 +3143,13 @@ ** statement loops through all rows of the %_content table. For a ** full-text query or docid lookup, the statement retrieves a single ** row by docid. */ zSql = (char *)azSql[idxNum==FTS3_FULLSCAN_SEARCH]; - zSql = sqlite3_mprintf(zSql, p->zReadExprlist, p->zDb, p->zName); + zSql = sqlite3_mprintf( + zSql, p->zReadExprlist, p->zDb, p->zName, (idxStr ? idxStr : "ASC") + ); if( !zSql ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); sqlite3_free(zSql); @@ -3101,11 +3157,26 @@ if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){ rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); } pCsr->eSearch = (i16)idxNum; + assert( pCsr->desc==0 ); if( rc!=SQLITE_OK ) return rc; + if( rc==SQLITE_OK && pCsr->nDoclist>0 && idxStr && idxStr[0]=='D' ){ + sqlite3_int64 iDocid = 0; + char *csr = pCsr->aDoclist; + while( csr<&pCsr->aDoclist[pCsr->nDoclist] ){ + fts3GetDeltaVarint(&csr, &iDocid); + } + pCsr->pNextId = csr; + pCsr->iPrevId = iDocid; + pCsr->desc = 1; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->eEvalmode = FTS3_EVAL_NEXT; + return SQLITE_OK; + } return fts3NextMethod(pCursor); } /* ** This is the xEof method of the virtual table. SQLite calls this @@ -3253,17 +3324,37 @@ pCsr->eEvalmode = FTS3_EVAL_MATCHINFO; rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1); pCsr->eEvalmode = FTS3_EVAL_NEXT; return rc; } + + +/* +** When called, *ppPoslist must point to the byte immediately following the +** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function +** moves *ppPoslist so that it instead points to the first byte of the +** same position list. +*/ +static void fts3ReversePoslist(char *pStart, char **ppPoslist){ + char *p = &(*ppPoslist)[-3]; + char c = p[1]; + while( p>=pStart && (*p & 0x80) | c ){ + c = *p--; + } + if( p>pStart ){ p = &p[2]; } + while( *p++&0x80 ); + *ppPoslist = p; +} + /* ** After ExprLoadDoclist() (see above) has been called, this function is ** used to iterate/search through the position lists that make up the doclist ** stored in pExpr->aDoclist. */ char *sqlite3Fts3FindPositions( + Fts3Cursor *pCursor, /* Associate FTS3 cursor */ Fts3Expr *pExpr, /* Access this expressions doclist */ sqlite3_int64 iDocid, /* Docid associated with requested pos-list */ int iCol /* Column of requested pos-list */ ){ assert( pExpr->isLoaded ); @@ -3270,23 +3361,39 @@ if( pExpr->aDoclist ){ char *pEnd = &pExpr->aDoclist[pExpr->nDoclist]; char *pCsr; if( pExpr->pCurrent==0 ){ - pExpr->pCurrent = pExpr->aDoclist; - pExpr->iCurrent = 0; - pExpr->pCurrent += sqlite3Fts3GetVarint(pExpr->pCurrent,&pExpr->iCurrent); + if( pCursor->desc==0 ){ + pExpr->pCurrent = pExpr->aDoclist; + pExpr->iCurrent = 0; + fts3GetDeltaVarint(&pExpr->pCurrent, &pExpr->iCurrent); + }else{ + pCsr = pExpr->aDoclist; + while( pCsriCurrent); + fts3PoslistCopy(0, &pCsr); + } + fts3ReversePoslist(pExpr->aDoclist, &pCsr); + pExpr->pCurrent = pCsr; + } } pCsr = pExpr->pCurrent; assert( pCsr ); - while( pCsriCurrentdesc==0 && pCsrdesc && pCsr>pExpr->aDoclist) + ){ + if( pCursor->desc==0 && pExpr->iCurrentiCurrent); } + pExpr->pCurrent = pCsr; + }else if( pCursor->desc && pExpr->iCurrent>iDocid ){ + fts3GetReverseDeltaVarint(&pCsr, pExpr->aDoclist, &pExpr->iCurrent); + fts3ReversePoslist(pExpr->aDoclist, &pCsr); pExpr->pCurrent = pCsr; }else{ if( pExpr->iCurrent==iDocid ){ int iThis = 0; if( iCol<0 ){ @@ -3539,13 +3646,24 @@ "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", p->zDb, p->zName, zName ); return rc; } + +static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); +} +static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ + return SQLITE_OK; +} +static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ + sqlite3Fts3PendingTermsClear((Fts3Table *)pVtab); + return SQLITE_OK; +} static const sqlite3_module fts3Module = { - /* iVersion */ 0, + /* iVersion */ 1, /* xCreate */ fts3CreateMethod, /* xConnect */ fts3ConnectMethod, /* xBestIndex */ fts3BestIndexMethod, /* xDisconnect */ fts3DisconnectMethod, /* xDestroy */ fts3DestroyMethod, @@ -3561,10 +3679,13 @@ /* xSync */ fts3SyncMethod, /* xCommit */ fts3CommitMethod, /* xRollback */ fts3RollbackMethod, /* xFindFunction */ fts3FindFunctionMethod, /* xRename */ fts3RenameMethod, + /* xSavepoint */ fts3SavepointMethod, + /* xRelease */ fts3ReleaseMethod, + /* xRollbackTo */ fts3RollbackToMethod, }; /* ** This function is registered as the module destructor (called when an ** FTS3 enabled database connection is closed). It frees the memory @@ -3606,10 +3727,15 @@ #ifdef SQLITE_ENABLE_ICU const sqlite3_tokenizer_module *pIcu = 0; sqlite3Fts3IcuTokenizerModule(&pIcu); #endif + +#ifdef SQLITE_TEST + rc = sqlite3Fts3InitTerm(db); + if( rc!=SQLITE_OK ) return rc; +#endif rc = sqlite3Fts3InitAux(db); if( rc!=SQLITE_OK ) return rc; sqlite3Fts3SimpleTokenizerModule(&pSimple); Index: ext/fts3/fts3Int.h ================================================================== --- ext/fts3/fts3Int.h +++ ext/fts3/fts3Int.h @@ -169,10 +169,11 @@ Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ char *pNextId; /* Pointer into the body of aDoclist */ char *aDoclist; /* List of docids for full-text queries */ int nDoclist; /* Size of buffer at aDoclist */ + int desc; /* True to sort in descending order */ int eEvalmode; /* An FTS3_EVAL_XX constant */ int nRowAvg; /* Average size of database rows, in pages */ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ u32 *aMatchinfo; /* Information about most recent match */ @@ -351,11 +352,11 @@ int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); -char *sqlite3Fts3FindPositions(Fts3Expr *, sqlite3_int64, int); +char *sqlite3Fts3FindPositions(Fts3Cursor *, Fts3Expr *, sqlite3_int64, int); int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *); int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); /* fts3_tokenizer.c */ Index: ext/fts3/fts3_snippet.c ================================================================== --- ext/fts3/fts3_snippet.c +++ ext/fts3/fts3_snippet.c @@ -413,11 +413,11 @@ SnippetPhrase *pPhrase = &p->aPhrase[iPhrase]; char *pCsr; pPhrase->nToken = pExpr->pPhrase->nToken; - pCsr = sqlite3Fts3FindPositions(pExpr, p->pCsr->iPrevId, p->iCol); + pCsr = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->pCsr->iPrevId, p->iCol); if( pCsr ){ int iFirst = 0; pPhrase->pList = pCsr; fts3GetDeltaPosition(&pCsr, &iFirst); pPhrase->pHead = pCsr; @@ -886,11 +886,11 @@ for(i=0; inCol; i++) p->aMatchinfo[iStart+i*3] = 0; if( pExpr->aDoclist ){ char *pCsr; - pCsr = sqlite3Fts3FindPositions(pExpr, p->pCursor->iPrevId, -1); + pCsr = sqlite3Fts3FindPositions(p->pCursor, pExpr, p->pCursor->iPrevId, -1); if( pCsr ){ fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0); } } @@ -1053,11 +1053,11 @@ (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); for(i=0; inPhrase; i++){ LcsIterator *pIter = &aIter[i]; nToken -= pIter->pExpr->pPhrase->nToken; pIter->iPosOffset = nToken; - pIter->pRead = sqlite3Fts3FindPositions(pIter->pExpr, pCsr->iPrevId, -1); + pIter->pRead = sqlite3Fts3FindPositions(pCsr,pIter->pExpr,pCsr->iPrevId,-1); if( pIter->pRead ){ pIter->iPos = pIter->iPosOffset; fts3LcsIteratorAdvance(&aIter[i]); }else{ pIter->iCol = LCS_ITERATOR_FINISHED; @@ -1406,10 +1406,11 @@ int iPos; /* Position just read from pList */ int iOff; /* Offset of this term from read positions */ }; struct TermOffsetCtx { + Fts3Cursor *pCsr; int iCol; /* Column of table to populate aTerm for */ int iTerm; sqlite3_int64 iDocid; TermOffset *aTerm; }; @@ -1423,11 +1424,11 @@ int iTerm; /* For looping through nTerm phrase terms */ char *pList; /* Pointer to position list for phrase */ int iPos = 0; /* First position in position-list */ UNUSED_PARAMETER(iPhrase); - pList = sqlite3Fts3FindPositions(pExpr, p->iDocid, p->iCol); + pList = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->iDocid, p->iCol); nTerm = pExpr->pPhrase->nToken; if( pList ){ fts3GetDeltaPosition(&pList, &iPos); assert( iPos>=0 ); } @@ -1476,10 +1477,11 @@ if( 0==sCtx.aTerm ){ rc = SQLITE_NOMEM; goto offsets_out; } sCtx.iDocid = pCsr->iPrevId; + sCtx.pCsr = pCsr; /* Loop through the table columns, appending offset information to ** string-buffer res for each column. */ for(iCol=0; iColnColumn; iCol++){ ADDED ext/fts3/fts3_term.c Index: ext/fts3/fts3_term.c ================================================================== --- /dev/null +++ ext/fts3/fts3_term.c @@ -0,0 +1,360 @@ +/* +** 2011 Jan 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file is not part of the production FTS code. It is only used for +** testing. It contains a virtual table implementation that provides direct +** access to the full-text index of an FTS table. +*/ + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +#ifdef SQLITE_TEST + +#include "fts3Int.h" +#include +#include + +typedef struct Fts3termTable Fts3termTable; +typedef struct Fts3termCursor Fts3termCursor; + +struct Fts3termTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + Fts3Table *pFts3Tab; +}; + +struct Fts3termCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + Fts3SegReaderCursor csr; /* Must be right after "base" */ + Fts3SegFilter filter; + + int isEof; /* True if cursor is at EOF */ + char *pNext; + + sqlite3_int64 iRowid; /* Current 'rowid' value */ + sqlite3_int64 iDocid; /* Current 'docid' value */ + int iCol; /* Current 'col' value */ + int iPos; /* Current 'pos' value */ +}; + +/* +** Schema of the terms table. +*/ +#define FTS3_TERMS_SCHEMA "CREATE TABLE x(term, docid, col, pos)" + +/* +** This function does all the work for both the xConnect and xCreate methods. +** These tables have no persistent representation of their own, so xConnect +** and xCreate are identical operations. +*/ +static int fts3termConnectMethod( + sqlite3 *db, /* Database connection */ + void *pUnused, /* Unused */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + char const *zDb; /* Name of database (e.g. "main") */ + char const *zFts3; /* Name of fts3 table */ + int nDb; /* Result of strlen(zDb) */ + int nFts3; /* Result of strlen(zFts3) */ + int nByte; /* Bytes of space to allocate here */ + int rc; /* value returned by declare_vtab() */ + Fts3termTable *p; /* Virtual table object to return */ + + UNUSED_PARAMETER(pUnused); + + /* The user should specify a single argument - the name of an fts3 table. */ + if( argc!=4 ){ + *pzErr = sqlite3_mprintf( + "wrong number of arguments to fts4term constructor" + ); + return SQLITE_ERROR; + } + + zDb = argv[1]; + nDb = strlen(zDb); + zFts3 = argv[3]; + nFts3 = strlen(zFts3); + + rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA); + if( rc!=SQLITE_OK ) return rc; + + nByte = sizeof(Fts3termTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; + p = (Fts3termTable *)sqlite3_malloc(nByte); + if( !p ) return SQLITE_NOMEM; + memset(p, 0, nByte); + + p->pFts3Tab = (Fts3Table *)&p[1]; + p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; + p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; + p->pFts3Tab->db = db; + + memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); + memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); + sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); + + *ppVtab = (sqlite3_vtab *)p; + return SQLITE_OK; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3termDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3termTable *p = (Fts3termTable *)pVtab; + Fts3Table *pFts3 = p->pFts3Tab; + int i; + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(pFts3->aStmt[i]); + } + sqlite3_free(pFts3->zSegmentsTbl); + sqlite3_free(p); + return SQLITE_OK; +} + +#define FTS4AUX_EQ_CONSTRAINT 1 +#define FTS4AUX_GE_CONSTRAINT 2 +#define FTS4AUX_LE_CONSTRAINT 4 + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3termBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + UNUSED_PARAMETER(pVTab); + + /* This vtab naturally does "ORDER BY term, docid, col, pos". */ + if( pInfo->nOrderBy ){ + int i; + for(i=0; inOrderBy; i++){ + if( pInfo->aOrderBy[i].iColumn!=i || pInfo->aOrderBy[i].desc ) break; + } + if( i==pInfo->nOrderBy ){ + pInfo->orderByConsumed = 1; + } + } + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3termOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3termCursor *pCsr; /* Pointer to cursor object to return */ + + UNUSED_PARAMETER(pVTab); + + pCsr = (Fts3termCursor *)sqlite3_malloc(sizeof(Fts3termCursor)); + if( !pCsr ) return SQLITE_NOMEM; + memset(pCsr, 0, sizeof(Fts3termCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3termCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab; + Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; + + sqlite3Fts3SegmentsClose(pFts3); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3termNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab; + int rc; + sqlite3_int64 v; + + /* Increment our pretend rowid value. */ + pCsr->iRowid++; + + /* Advance to the next term in the full-text index. */ + if( pCsr->csr.aDoclist==0 + || pCsr->pNext>=&pCsr->csr.aDoclist[pCsr->csr.nDoclist-1] + ){ + rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr); + if( rc!=SQLITE_ROW ){ + pCsr->isEof = 1; + return rc; + } + + pCsr->iCol = 0; + pCsr->iPos = 0; + pCsr->iDocid = 0; + pCsr->pNext = pCsr->csr.aDoclist; + + /* Read docid */ + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &pCsr->iDocid); + } + + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); + if( v==0 ){ + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); + pCsr->iDocid += v; + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); + pCsr->iCol = 0; + pCsr->iPos = 0; + } + + if( v==1 ){ + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); + pCsr->iCol += v; + pCsr->iPos = 0; + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); + } + + pCsr->iPos += (v - 2); + + return SQLITE_OK; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3termFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab; + int rc; + + UNUSED_PARAMETER(nVal); + UNUSED_PARAMETER(idxNum); + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(apVal); + + assert( idxStr==0 && idxNum==0 ); + + /* In case this cursor is being reused, close and zero it. */ + testcase(pCsr->filter.zTerm); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); + + pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + pCsr->filter.flags |= FTS3_SEGMENT_SCAN; + + rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL, + pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); + } + if( rc==SQLITE_OK ){ + rc = fts3termNextMethod(pCursor); + } + return rc; +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3termEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; + return pCsr->isEof; +} + +/* +** xColumn - Return a column value. +*/ +static int fts3termColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3termCursor *p = (Fts3termCursor *)pCursor; + + assert( iCol>=0 && iCol<=3 ); + switch( iCol ){ + case 0: + sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_int64(pCtx, p->iDocid); + break; + case 2: + sqlite3_result_int64(pCtx, p->iCol); + break; + default: + sqlite3_result_int64(pCtx, p->iPos); + break; + } + + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3termRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3term module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +int sqlite3Fts3InitTerm(sqlite3 *db){ + static const sqlite3_module fts3term_module = { + 0, /* iVersion */ + fts3termConnectMethod, /* xCreate */ + fts3termConnectMethod, /* xConnect */ + fts3termBestIndexMethod, /* xBestIndex */ + fts3termDisconnectMethod, /* xDisconnect */ + fts3termDisconnectMethod, /* xDestroy */ + fts3termOpenMethod, /* xOpen */ + fts3termCloseMethod, /* xClose */ + fts3termFilterMethod, /* xFilter */ + fts3termNextMethod, /* xNext */ + fts3termEofMethod, /* xEof */ + fts3termColumnMethod, /* xColumn */ + fts3termRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0 /* xRename */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts4term", &fts3term_module, 0); + return rc; +} + +#endif +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ Index: ext/fts3/fts3_write.c ================================================================== --- ext/fts3/fts3_write.c +++ ext/fts3/fts3_write.c @@ -745,18 +745,18 @@ ** full-text index. */ static void fts3DeleteTerms( int *pRC, /* Result code */ Fts3Table *p, /* The FTS table to delete from */ - sqlite3_value **apVal, /* apVal[] contains the docid to be deleted */ + sqlite3_value *pRowid, /* The docid to be deleted */ u32 *aSz /* Sizes of deleted document written here */ ){ int rc; sqlite3_stmt *pSelect; if( *pRC ) return; - rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, apVal); + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pSelect) ){ int i; for(i=1; i<=p->nColumn; i++){ const char *zText = (const char *)sqlite3_column_text(pSelect, i); @@ -1892,20 +1892,20 @@ /* ** The first value in the apVal[] array is assumed to contain an integer. ** This function tests if there exist any documents with docid values that ** are different from that integer. i.e. if deleting the document with docid -** apVal[0] would mean the FTS3 table were empty. +** pRowid would mean the FTS3 table were empty. ** ** If successful, *pisEmpty is set to true if the table is empty except for -** document apVal[0], or false otherwise, and SQLITE_OK is returned. If an +** document pRowid, or false otherwise, and SQLITE_OK is returned. If an ** error occurs, an SQLite error code is returned. */ -static int fts3IsEmpty(Fts3Table *p, sqlite3_value **apVal, int *pisEmpty){ +static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ sqlite3_stmt *pStmt; int rc; - rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, apVal); + rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pisEmpty = sqlite3_column_int(pStmt, 0); } rc = sqlite3_reset(pStmt); @@ -2625,10 +2625,44 @@ pToken->pDeferred = pDeferred; return SQLITE_OK; } +/* +** SQLite value pRowid contains the rowid of a row that may or may not be +** present in the FTS3 table. If it is, delete it and adjust the contents +** of subsiduary data structures accordingly. +*/ +static int fts3DeleteByRowid( + Fts3Table *p, + sqlite3_value *pRowid, + int *pnDoc, + u32 *aSzDel +){ + int isEmpty = 0; + int rc = fts3IsEmpty(p, pRowid, &isEmpty); + if( rc==SQLITE_OK ){ + if( isEmpty ){ + /* Deleting this row means the whole table is empty. In this case + ** delete the contents of all three tables and throw away any + ** data in the pendingTerms hash table. */ + rc = fts3DeleteAll(p); + *pnDoc = *pnDoc - 1; + }else{ + sqlite3_int64 iRemove = sqlite3_value_int64(pRowid); + rc = fts3PendingTermsDocid(p, iRemove); + fts3DeleteTerms(&rc, p, pRowid, aSzDel); + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1; + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + } + } + } + + return rc; +} /* ** This function does the work for the xUpdate method of FTS3 virtual ** tables. */ @@ -2643,50 +2677,95 @@ int isRemove = 0; /* True for an UPDATE or DELETE */ sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ u32 *aSzIns; /* Sizes of inserted documents */ u32 *aSzDel; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ + int bInsertDone = 0; assert( p->pSegments==0 ); + + /* Check for a "special" INSERT operation. One of the form: + ** + ** INSERT INTO xyz(xyz) VALUES('command'); + */ + if( nArg>1 + && sqlite3_value_type(apVal[0])==SQLITE_NULL + && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL + ){ + return fts3SpecialInsert(p, apVal[p->nColumn+2]); + } /* Allocate space to hold the change in document sizes */ aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); if( aSzIns==0 ) return SQLITE_NOMEM; aSzDel = &aSzIns[p->nColumn+1]; memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); - /* If this is a DELETE or UPDATE operation, remove the old record. */ - if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ - int isEmpty = 0; - rc = fts3IsEmpty(p, apVal, &isEmpty); - if( rc==SQLITE_OK ){ - if( isEmpty ){ - /* Deleting this row means the whole table is empty. In this case - ** delete the contents of all three tables and throw away any - ** data in the pendingTerms hash table. - */ - rc = fts3DeleteAll(p); - }else{ - isRemove = 1; - iRemove = sqlite3_value_int64(apVal[0]); - rc = fts3PendingTermsDocid(p, iRemove); - fts3DeleteTerms(&rc, p, apVal, aSzDel); - fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, apVal); - if( p->bHasDocsize ){ - fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, apVal); - } - nChng--; - } - } - }else if( sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){ - sqlite3_free(aSzIns); - return fts3SpecialInsert(p, apVal[p->nColumn+2]); + /* If this is an INSERT operation, or an UPDATE that modifies the rowid + ** value, then this operation requires constraint handling. + ** + ** If the on-conflict mode is REPLACE, this means that the existing row + ** should be deleted from the database before inserting the new row. Or, + ** if the on-conflict mode is other than REPLACE, then this method must + ** detect the conflict and return SQLITE_CONSTRAINT before beginning to + ** modify the database file. + */ + if( nArg>1 ){ + /* Find the value object that holds the new rowid value. */ + sqlite3_value *pNewRowid = apVal[3+p->nColumn]; + if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ + pNewRowid = apVal[1]; + } + + if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( + sqlite3_value_type(apVal[0])==SQLITE_NULL + || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid) + )){ + /* The new rowid is not NULL (in this case the rowid will be + ** automatically assigned and there is no chance of a conflict), and + ** the statement is either an INSERT or an UPDATE that modifies the + ** rowid column. So if the conflict mode is REPLACE, then delete any + ** existing row with rowid=pNewRowid. + ** + ** Or, if the conflict mode is not REPLACE, insert the new record into + ** the %_content table. If we hit the duplicate rowid constraint (or any + ** other error) while doing so, return immediately. + ** + ** This branch may also run if pNewRowid contains a value that cannot + ** be losslessly converted to an integer. In this case, the eventual + ** call to fts3InsertData() (either just below or further on in this + ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is + ** invoked, it will delete zero rows (since no row will have + ** docid=$pNewRowid if $pNewRowid is not an integer value). + */ + if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){ + rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel); + }else{ + rc = fts3InsertData(p, apVal, pRowid); + bInsertDone = 1; + } + } + } + if( rc!=SQLITE_OK ){ + sqlite3_free(aSzIns); + return rc; + } + + /* If this is a DELETE or UPDATE operation, remove the old record. */ + if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); + rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); + isRemove = 1; + iRemove = sqlite3_value_int64(apVal[0]); } /* If this is an INSERT or UPDATE operation, insert the new record. */ if( nArg>1 && rc==SQLITE_OK ){ - rc = fts3InsertData(p, apVal, pRowid); + if( bInsertDone==0 ){ + rc = fts3InsertData(p, apVal, pRowid); + if( rc==SQLITE_CONSTRAINT ) rc = SQLITE_CORRUPT; + } if( rc==SQLITE_OK && (!isRemove || *pRowid!=iRemove) ){ rc = fts3PendingTermsDocid(p, *pRowid); } if( rc==SQLITE_OK ){ rc = fts3InsertTerms(p, apVal, aSzIns); Index: ext/rtree/rtree.c ================================================================== --- ext/rtree/rtree.c +++ ext/rtree/rtree.c @@ -2622,10 +2622,94 @@ sqlite3_step(pRtree->pWriteRowid); rc = sqlite3_reset(pRtree->pWriteRowid); *piRowid = sqlite3_last_insert_rowid(pRtree->db); return rc; } + +/* +** Remove the entry with rowid=iDelete from the r-tree structure. +*/ +static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ + int rc; /* Return code */ + RtreeNode *pLeaf; /* Leaf node containing record iDelete */ + int iCell; /* Index of iDelete cell in pLeaf */ + RtreeNode *pRoot; /* Root node of rtree structure */ + + + /* Obtain a reference to the root node to initialise Rtree.iDepth */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + + /* Obtain a reference to the leaf node that contains the entry + ** about to be deleted. + */ + if( rc==SQLITE_OK ){ + rc = findLeafNode(pRtree, iDelete, &pLeaf); + } + + /* Delete the cell in question from the leaf node. */ + if( rc==SQLITE_OK ){ + int rc2; + rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); + if( rc==SQLITE_OK ){ + rc = deleteCell(pRtree, pLeaf, iCell, 0); + } + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + + /* Delete the corresponding entry in the _rowid table. */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pRtree->pDeleteRowid, 1, iDelete); + sqlite3_step(pRtree->pDeleteRowid); + rc = sqlite3_reset(pRtree->pDeleteRowid); + } + + /* Check if the root node now has exactly one child. If so, remove + ** it, schedule the contents of the child for reinsertion and + ** reduce the tree height by one. + ** + ** This is equivalent to copying the contents of the child into + ** the root node (the operation that Gutman's paper says to perform + ** in this scenario). + */ + if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ + int rc2; + RtreeNode *pChild; + i64 iChild = nodeGetRowid(pRtree, pRoot, 0); + rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); + if( rc==SQLITE_OK ){ + rc = removeNode(pRtree, pChild, pRtree->iDepth-1); + } + rc2 = nodeRelease(pRtree, pChild); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK ){ + pRtree->iDepth--; + writeInt16(pRoot->zData, pRtree->iDepth); + pRoot->isDirty = 1; + } + } + + /* Re-insert the contents of any underfull nodes removed from the tree. */ + for(pLeaf=pRtree->pDeleted; pLeaf; pLeaf=pRtree->pDeleted){ + if( rc==SQLITE_OK ){ + rc = reinsertNodeContent(pRtree, pLeaf); + } + pRtree->pDeleted = pLeaf->pNext; + sqlite3_free(pLeaf); + } + + /* Release the reference to the root node. */ + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pRoot); + }else{ + nodeRelease(pRtree, pRoot); + } + + return rc; +} /* ** The xUpdate method for rtree module virtual tables. */ static int rtreeUpdate( @@ -2634,107 +2718,29 @@ sqlite3_value **azData, sqlite_int64 *pRowid ){ Rtree *pRtree = (Rtree *)pVtab; int rc = SQLITE_OK; + RtreeCell cell; /* New cell to insert if nData>1 */ + int bHaveRowid = 0; /* Set to 1 after new rowid is determined */ rtreeReference(pRtree); - assert(nData>=1); - /* If azData[0] is not an SQL NULL value, it is the rowid of a - ** record to delete from the r-tree table. The following block does - ** just that. - */ - if( sqlite3_value_type(azData[0])!=SQLITE_NULL ){ - i64 iDelete; /* The rowid to delete */ - RtreeNode *pLeaf; /* Leaf node containing record iDelete */ - int iCell; /* Index of iDelete cell in pLeaf */ - RtreeNode *pRoot; - - /* Obtain a reference to the root node to initialise Rtree.iDepth */ - rc = nodeAcquire(pRtree, 1, 0, &pRoot); - - /* Obtain a reference to the leaf node that contains the entry - ** about to be deleted. - */ - if( rc==SQLITE_OK ){ - iDelete = sqlite3_value_int64(azData[0]); - rc = findLeafNode(pRtree, iDelete, &pLeaf); - } - - /* Delete the cell in question from the leaf node. */ - if( rc==SQLITE_OK ){ - int rc2; - rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); - if( rc==SQLITE_OK ){ - rc = deleteCell(pRtree, pLeaf, iCell, 0); - } - rc2 = nodeRelease(pRtree, pLeaf); - if( rc==SQLITE_OK ){ - rc = rc2; - } - } - - /* Delete the corresponding entry in the _rowid table. */ - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pRtree->pDeleteRowid, 1, iDelete); - sqlite3_step(pRtree->pDeleteRowid); - rc = sqlite3_reset(pRtree->pDeleteRowid); - } - - /* Check if the root node now has exactly one child. If so, remove - ** it, schedule the contents of the child for reinsertion and - ** reduce the tree height by one. - ** - ** This is equivalent to copying the contents of the child into - ** the root node (the operation that Gutman's paper says to perform - ** in this scenario). - */ - if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ - int rc2; - RtreeNode *pChild; - i64 iChild = nodeGetRowid(pRtree, pRoot, 0); - rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); - if( rc==SQLITE_OK ){ - rc = removeNode(pRtree, pChild, pRtree->iDepth-1); - } - rc2 = nodeRelease(pRtree, pChild); - if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK ){ - pRtree->iDepth--; - writeInt16(pRoot->zData, pRtree->iDepth); - pRoot->isDirty = 1; - } - } - - /* Re-insert the contents of any underfull nodes removed from the tree. */ - for(pLeaf=pRtree->pDeleted; pLeaf; pLeaf=pRtree->pDeleted){ - if( rc==SQLITE_OK ){ - rc = reinsertNodeContent(pRtree, pLeaf); - } - pRtree->pDeleted = pLeaf->pNext; - sqlite3_free(pLeaf); - } - - /* Release the reference to the root node. */ - if( rc==SQLITE_OK ){ - rc = nodeRelease(pRtree, pRoot); - }else{ - nodeRelease(pRtree, pRoot); - } - } - - /* If the azData[] array contains more than one element, elements - ** (azData[2]..azData[argc-1]) contain a new record to insert into - ** the r-tree structure. - */ - if( rc==SQLITE_OK && nData>1 ){ - /* Insert a new record into the r-tree */ - RtreeCell cell; - int ii; - RtreeNode *pLeaf; + /* Constraint handling. A write operation on an r-tree table may return + ** SQLITE_CONSTRAINT for two reasons: + ** + ** 1. A duplicate rowid value, or + ** 2. The supplied data violates the "x2>=x1" constraint. + ** + ** In the first case, if the conflict-handling mode is REPLACE, then + ** the conflicting row can be removed before proceeding. In the second + ** case, SQLITE_CONSTRAINT must be returned regardless of the + ** conflict-handling mode specified by the user. + */ + if( nData>1 ){ + int ii; /* Populate the cell.aCoord[] array. The first coordinate is azData[3]. */ assert( nData==(pRtree->nDim*2 + 3) ); if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ for(ii=0; ii<(pRtree->nDim*2); ii+=2){ @@ -2754,22 +2760,53 @@ goto constraint; } } } - /* Figure out the rowid of the new row. */ - if( sqlite3_value_type(azData[2])==SQLITE_NULL ){ - rc = newRowid(pRtree, &cell.iRowid); - }else{ + /* If a rowid value was supplied, check if it is already present in + ** the table. If so, the constraint has failed. */ + if( sqlite3_value_type(azData[2])!=SQLITE_NULL ){ cell.iRowid = sqlite3_value_int64(azData[2]); - sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); - if( SQLITE_ROW==sqlite3_step(pRtree->pReadRowid) ){ - sqlite3_reset(pRtree->pReadRowid); - rc = SQLITE_CONSTRAINT; - goto constraint; + if( sqlite3_value_type(azData[0])==SQLITE_NULL + || sqlite3_value_int64(azData[0])!=cell.iRowid + ){ + int steprc; + sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); + steprc = sqlite3_step(pRtree->pReadRowid); + rc = sqlite3_reset(pRtree->pReadRowid); + if( SQLITE_ROW==steprc ){ + if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ + rc = rtreeDeleteRowid(pRtree, cell.iRowid); + }else{ + rc = SQLITE_CONSTRAINT; + goto constraint; + } + } } - rc = sqlite3_reset(pRtree->pReadRowid); + bHaveRowid = 1; + } + } + + /* If azData[0] is not an SQL NULL value, it is the rowid of a + ** record to delete from the r-tree table. The following block does + ** just that. + */ + if( sqlite3_value_type(azData[0])!=SQLITE_NULL ){ + rc = rtreeDeleteRowid(pRtree, sqlite3_value_int64(azData[0])); + } + + /* If the azData[] array contains more than one element, elements + ** (azData[2]..azData[argc-1]) contain a new record to insert into + ** the r-tree structure. + */ + if( rc==SQLITE_OK && nData>1 ){ + /* Insert the new record into the r-tree */ + RtreeNode *pLeaf; + + /* Figure out the rowid of the new row. */ + if( bHaveRowid==0 ){ + rc = newRowid(pRtree, &cell.iRowid); } *pRowid = cell.iRowid; if( rc==SQLITE_OK ){ rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf); @@ -3005,10 +3042,12 @@ int iErr = (argc<6) ? 2 : argc>(RTREE_MAX_DIMENSIONS*2+4) ? 3 : argc%2; if( aErrMsg[iErr] ){ *pzErr = sqlite3_mprintf("%s", aErrMsg[iErr]); return SQLITE_ERROR; } + + sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); /* Allocate the sqlite3_vtab structure */ nDb = strlen(argv[1]); nName = strlen(argv[2]); pRtree = (Rtree *)sqlite3_malloc(sizeof(Rtree)+nDb+nName+2); Index: ext/rtree/rtree1.test ================================================================== --- ext/rtree/rtree1.test +++ ext/rtree/rtree1.test @@ -29,10 +29,12 @@ # rtree-5.*: Test DELETE # rtree-6.*: Test UPDATE # rtree-7.*: Test renaming an r-tree table. # rtree-8.*: Test constrained scans of r-tree data. # +# rtree-12.*: Test that on-conflict clauses are supported. +# ifcapable !rtree { finish_test return } @@ -414,6 +416,85 @@ INSERT INTO t8 VALUES(NULL, 1.0, 1.0, 2.0, 2.0); SELECT last_insert_rowid(); } } {2} +#------------------------------------------------------------------------- +# Test on-conflict clause handling. +# +db_delete_and_reopen +do_execsql_test 12.0 { + CREATE VIRTUAL TABLE t1 USING rtree_i32(idx, x1, x2, y1, y2); + INSERT INTO t1 VALUES(1, 1, 2, 3, 4); + INSERT INTO t1 VALUES(2, 2, 3, 4, 5); + INSERT INTO t1 VALUES(3, 3, 4, 5, 6); + + CREATE TABLE source(idx, x1, x2, y1, y2); + INSERT INTO source VALUES(5, 8, 8, 8, 8); + INSERT INTO source VALUES(2, 7, 7, 7, 7); + +} +db_save_and_close +foreach {tn sql_template testdata} { + 1 "INSERT %CONF% INTO t1 VALUES(2, 7, 7, 7, 7)" { + ROLLBACK 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6} + ABORT 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + IGNORE 0 0 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + FAIL 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + REPLACE 0 0 {1 1 2 3 4 2 7 7 7 7 3 3 4 5 6 4 4 5 6 7} + } + + 2 "INSERT %CONF% INTO t1 SELECT * FROM source" { + ROLLBACK 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6} + ABORT 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + IGNORE 1 0 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7 5 8 8 8 8} + FAIL 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7 5 8 8 8 8} + REPLACE 1 0 {1 1 2 3 4 2 7 7 7 7 3 3 4 5 6 4 4 5 6 7 5 8 8 8 8} + } + + 3 "UPDATE %CONF% t1 SET idx = 2 WHERE idx = 4" { + ROLLBACK 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6} + ABORT 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + IGNORE 1 0 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + FAIL 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + REPLACE 1 0 {1 1 2 3 4 2 4 5 6 7 3 3 4 5 6} + } + + 3 "UPDATE %CONF% t1 SET idx = ((idx+1)%5)+1 WHERE idx > 2" { + ROLLBACK 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6} + ABORT 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + IGNORE 1 0 {1 1 2 3 4 2 2 3 4 5 4 4 5 6 7 5 3 4 5 6} + FAIL 1 1 {1 1 2 3 4 2 2 3 4 5 4 4 5 6 7 5 3 4 5 6} + REPLACE 1 0 {1 4 5 6 7 2 2 3 4 5 5 3 4 5 6} + } + + 4 "INSERT %CONF% INTO t1 VALUES(2, 7, 6, 7, 7)" { + ROLLBACK 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6} + ABORT 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + IGNORE 0 0 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + FAIL 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + REPLACE 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + } + +} { + foreach {mode uses error data} $testdata { + db_restore_and_reopen + + set sql [string map [list %CONF% "OR $mode"] $sql_template] + set testname "12.$tn.[string tolower $mode]" + + execsql { + BEGIN; + INSERT INTO t1 VALUES(4, 4, 5, 6, 7); + } + + set res(0) {0 {}} + set res(1) {1 {constraint failed}} + do_catchsql_test $testname.1 $sql $res($error) + do_test $testname.2 [list sql_uses_stmt db $sql] $uses + do_execsql_test $testname.3 { SELECT * FROM t1 ORDER BY idx } $data + + do_test $testname.4 { rtree_check db t1 } 0 + db close + } +} finish_test Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -297,10 +297,11 @@ $(TOP)/src/where.c \ parse.c \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3_aux.c \ $(TOP)/ext/fts3/fts3_expr.c \ + $(TOP)/ext/fts3/fts3_term.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ $(TOP)/ext/async/sqlite3async.c # Header files used by all library source files. Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -967,10 +967,11 @@ #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pTab) ){ const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); sqlite3VtabMakeWritable(pParse, pTab); sqlite3VdbeAddOp4(v, OP_VUpdate, 1, pTab->nCol+2, regIns, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); sqlite3MayAbort(pParse); }else #endif { int isReplace; /* Set to true if constraints may cause a replace */ Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -4605,10 +4605,15 @@ int (*xRollback)(sqlite3_vtab *pVTab); int (*xFindFunction)(sqlite3_vtab *pVtab, int nArg, const char *zName, void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), void **ppArg); int (*xRename)(sqlite3_vtab *pVtab, const char *zNew); + /* The methods above are in version 0 of the sqlite_module object. Those + ** below are for version 1 and greater. */ + int (*xSavepoint)(sqlite3_vtab *pVTab, int); + int (*xRelease)(sqlite3_vtab *pVTab, int); + int (*xRollbackTo)(sqlite3_vtab *pVTab, int); }; /* ** CAPI3REF: Virtual Table Indexing Information ** KEYWORDS: sqlite3_index_info @@ -6382,14 +6387,78 @@ ** ** These constants can be used as the 3rd parameter to ** [sqlite3_wal_checkpoint_v2()]. See the [sqlite3_wal_checkpoint_v2()] ** documentation for additional information about the meaning and use of ** each of these values. +** +**
SQLITE_CONFIG_GETMUTEX
+**
^(This option takes a single argument which is a pointer to an */ #define SQLITE_CHECKPOINT_PASSIVE 0 #define SQLITE_CHECKPOINT_FULL 1 #define SQLITE_CHECKPOINT_RESTART 2 + +/* +** CAPI3REF: Virtual Table Interface Configuration +** +** This function is called by a virtual table implementation to configure +** various facets of the virtual table interface. At present, there is only +** one option that may be configured using this function. Further options +** may be added in the future. +** +**
+**
SQLITE_VTAB_CONSTRAINT_SUPPORT +**
If the second argument to sqlite3_vtab_config() is +** SQLITE_VTAB_CONSTRAINT_SUPPORT, then SQLite expects this function to +** have been called with three arguments, the third of which being of +** type 'int'. If the third argument is zero, then the virtual table +** is indicating that it does not support constraints. In this case if +** a call to the xUpdate method returns SQLITE_CONSTRAINT, the entire +** statement is rolled back as if [ON CONFLICT | OR ABORT] had been +** specified as part of the users SQL statement, regardless of the actual +** ON CONFLICT mode specified. +** +** If the third argument passed is non-zero, then the virtual table +** implementation must guarantee that if xUpdate returns +** SQLITE_CONSTRAINT, it does so before any modifications to internal +** or persistent data structures have been made. If the [ON CONFLICT] +** mode is ABORT, FAIL, IGNORE or ROLLBACK, SQLite is able to roll back +** a statement or database transaction, and abandon or continue processing +** the current SQL statement as appropriate. If the ON CONFLICT mode is +** REPLACE and the xUpdate method returns SQLITE_CONSTRAINT, SQLite +** handles this as if the ON CONFLICT mode had been ABORT. +** +** Virtual table implementations that are required to handle OR REPLACE +** must do so within the xUpdate method. If a call to the +** [sqlite3_vtab_on_conflict()] function indicates that the current ON +** CONFLICT policy is REPLACE, the virtual table implementation should +** silently replace the appropriate rows within the xUpdate callback and +** return SQLITE_OK. Or, if this is not possible, it may return +** SQLITE_CONSTRAINT, in which case SQLite falls back to OR ABORT +** constraint handling. +**
+** +*/ +#define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 +int sqlite3_vtab_config(sqlite3*, int op, ...); + +/* +** CAPI3REF: Determine The Virtual Table Conflict Policy +** +** This function may only be called from within a call to the xUpdate method +** of a virtual table implementation for an INSERT or UPDATE operation. The +** value returned is one of SQLITE_ROLLBACK, SQLITE_IGNORE, SQLITE_FAIL, +** SQLITE_ABORT or SQLITE_REPLACE, according to the [ON CONFLICT] mode of the +** SQL statement that triggered the callback. +*/ +#define SQLITE_ROLLBACK 1 +/* #define SQLITE_IGNORE 2 */ +#define SQLITE_FAIL 3 +/* #define SQLITE_ABORT 4 */ +#define SQLITE_REPLACE 5 +int sqlite3_vtab_on_conflict(sqlite3 *); + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -630,10 +630,11 @@ typedef struct Trigger Trigger; typedef struct TriggerPrg TriggerPrg; typedef struct TriggerStep TriggerStep; typedef struct UnpackedRecord UnpackedRecord; typedef struct VTable VTable; +typedef struct VtabCtx VtabCtx; typedef struct Walker Walker; typedef struct WherePlan WherePlan; typedef struct WhereInfo WhereInfo; typedef struct WhereLevel WhereLevel; @@ -809,10 +810,11 @@ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ u8 dfltLockMode; /* Default locking-mode for attached dbs */ signed char nextAutovac; /* Autovac setting after VACUUM if >=0 */ u8 suppressErr; /* Do not issue error messages if true */ + u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ int nextPagesize; /* Pagesize after VACUUM if >0 */ int nTable; /* Number of tables in the database */ CollSeq *pDfltColl; /* The default collating sequence (BINARY) */ i64 lastRowid; /* ROWID of most recent insert (see above) */ u32 magic; /* Magic number for detect library misuse */ @@ -867,11 +869,11 @@ void *pProgressArg; /* Argument to the progress callback */ int nProgressOps; /* Number of opcodes for progress callback */ #endif #ifndef SQLITE_OMIT_VIRTUALTABLE Hash aModule; /* populated by sqlite3_create_module() */ - Table *pVTab; /* vtab with active Connect/Create method */ + VtabCtx *pVtabCtx; /* Context for active vtab connect/create */ VTable **aVTrans; /* Virtual tables with open transactions */ int nVTrans; /* Allocated size of aVTrans */ VTable *pDisconnect; /* Disconnect these in next sqlite3_prepare() */ #endif FuncDefHash aFunc; /* Hash table of connection functions */ @@ -1230,10 +1232,11 @@ struct VTable { sqlite3 *db; /* Database connection associated with this table */ Module *pMod; /* Pointer to module implementation */ sqlite3_vtab *pVtab; /* Pointer to vtab instance */ int nRef; /* Number of pointers to this structure */ + u8 bConstraint; /* True if constraints are supported */ VTable *pNext; /* Next in linked list (see above) */ }; /* ** Each SQL table is represented in memory by an instance of the @@ -3041,18 +3044,20 @@ # define sqlite3VtabCommit(X) # define sqlite3VtabInSync(db) 0 # define sqlite3VtabLock(X) # define sqlite3VtabUnlock(X) # define sqlite3VtabUnlockList(X) +# define sqlite3VtabSavepoint(X, Y, Z) SQLITE_OK #else void sqlite3VtabClear(sqlite3 *db, Table*); int sqlite3VtabSync(sqlite3 *db, char **); int sqlite3VtabRollback(sqlite3 *db); int sqlite3VtabCommit(sqlite3 *db); void sqlite3VtabLock(VTable *); void sqlite3VtabUnlock(VTable *); void sqlite3VtabUnlockList(sqlite3*); + int sqlite3VtabSavepoint(sqlite3 *, int, int); # define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0) #endif void sqlite3VtabMakeWritable(Parse*,Table*); void sqlite3VtabBeginParse(Parse*, Token*, Token*, Token*); void sqlite3VtabFinishParse(Parse*, Token*); Index: src/test1.c ================================================================== --- src/test1.c +++ src/test1.c @@ -12,10 +12,11 @@ ** Code for testing all sorts of SQLite interfaces. This code ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. */ #include "sqliteInt.h" +#include "vdbeInt.h" #include "tcl.h" #include #include /* @@ -2323,10 +2324,36 @@ if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; rc = sqlite3_stmt_readonly(pStmt); Tcl_SetObjResult(interp, Tcl_NewBooleanObj(rc)); return TCL_OK; } + +/* +** Usage: uses_stmt_journal STMT +** +** Return true if STMT uses a statement journal. +*/ +static int uses_stmt_journal( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3_stmt *pStmt; + int rc; + + if( objc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " STMT", 0); + return TCL_ERROR; + } + + if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; + rc = sqlite3_stmt_readonly(pStmt); + Tcl_SetObjResult(interp, Tcl_NewBooleanObj(((Vdbe *)pStmt)->usesStmtJournal)); + return TCL_OK; +} /* ** Usage: sqlite3_reset STMT ** @@ -5581,10 +5608,11 @@ { "sqlite3_changes", test_changes ,0 }, { "sqlite3_step", test_step ,0 }, { "sqlite3_sql", test_sql ,0 }, { "sqlite3_next_stmt", test_next_stmt ,0 }, { "sqlite3_stmt_readonly", test_stmt_readonly ,0 }, + { "uses_stmt_journal", uses_stmt_journal ,0 }, { "sqlite3_release_memory", test_release_memory, 0}, { "sqlite3_soft_heap_limit", test_soft_heap_limit, 0}, { "sqlite3_thread_cleanup", test_thread_cleanup, 0}, { "sqlite3_pager_refcounts", test_pager_refcounts, 0}, Index: src/update.c ================================================================== --- src/update.c +++ src/update.c @@ -21,11 +21,12 @@ SrcList *pSrc, /* The virtual table to be modified */ Table *pTab, /* The virtual table */ ExprList *pChanges, /* The columns to change in the UPDATE statement */ Expr *pRowidExpr, /* Expression used to recompute the rowid */ int *aXRef, /* Mapping from columns of pTab to entries in pChanges */ - Expr *pWhere /* WHERE clause of the UPDATE statement */ + Expr *pWhere, /* WHERE clause of the UPDATE statement */ + int onError /* ON CONFLICT strategy */ ); #endif /* SQLITE_OMIT_VIRTUALTABLE */ /* ** The most recently coded instruction was an OP_Column to retrieve the @@ -265,11 +266,11 @@ #ifndef SQLITE_OMIT_VIRTUALTABLE /* Virtual tables must be handled separately */ if( IsVirtual(pTab) ){ updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, - pWhere); + pWhere, onError); pWhere = 0; pTabList = 0; goto update_cleanup; } #endif @@ -595,11 +596,12 @@ SrcList *pSrc, /* The virtual table to be modified */ Table *pTab, /* The virtual table */ ExprList *pChanges, /* The columns to change in the UPDATE statement */ Expr *pRowid, /* Expression used to recompute the rowid */ int *aXRef, /* Mapping from columns of pTab to entries in pChanges */ - Expr *pWhere /* WHERE clause of the UPDATE statement */ + Expr *pWhere, /* WHERE clause of the UPDATE statement */ + int onError /* ON CONFLICT strategy */ ){ Vdbe *v = pParse->pVdbe; /* Virtual machine under construction */ ExprList *pEList = 0; /* The result set of the SELECT statement */ Select *pSelect = 0; /* The SELECT statement */ Expr *pExpr; /* Temporary expression */ @@ -652,14 +654,15 @@ for(i=0; inCol; i++){ sqlite3VdbeAddOp3(v, OP_Column, ephemTab, i+1+(pRowid!=0), iReg+2+i); } sqlite3VtabMakeWritable(pParse, pTab); sqlite3VdbeAddOp4(v, OP_VUpdate, 0, pTab->nCol+2, iReg, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); sqlite3MayAbort(pParse); sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); sqlite3VdbeJumpHere(v, addr); sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); /* Cleanup */ sqlite3SelectDelete(db, pSelect); } #endif /* SQLITE_OMIT_VIRTUALTABLE */ Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -2578,10 +2578,18 @@ "SQL statements in progress"); rc = SQLITE_BUSY; }else{ nName = sqlite3Strlen30(zName); + /* This call is Ok even if this savepoint is actually a transaction + ** savepoint (and therefore should not prompt xSavepoint()) callbacks. + ** If this is a transaction savepoint being opened, it is guaranteed + ** that the db->aVTrans[] array is empty. */ + assert( db->autoCommit==0 || db->nVTrans==0 ); + rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, p->iStatement); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + /* Create a new savepoint structure. */ pNew = sqlite3DbMallocRaw(db, sizeof(Savepoint)+nName+1); if( pNew ){ pNew->zName = (char *)&pNew[1]; memcpy(pNew->zName, zName, nName+1); @@ -2684,10 +2692,15 @@ db->nSavepoint--; } }else{ db->nDeferredCons = pSavepoint->nDeferredCons; } + + if( !isTransaction ){ + rc = sqlite3VtabSavepoint(db, p1, iSavepoint); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + } } } break; } @@ -2819,11 +2832,15 @@ if( p->iStatement==0 ){ assert( db->nStatement>=0 && db->nSavepoint>=0 ); db->nStatement++; p->iStatement = db->nSavepoint + db->nStatement; } - rc = sqlite3BtreeBeginStmt(pBt, p->iStatement); + + rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, p->iStatement); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginStmt(pBt, p->iStatement); + } /* Store the current value of the database handles deferred constraint ** counter. If the statement transaction needs to be rolled back, ** the value of this counter needs to be restored too. */ p->nStmtDefCons = db->nDeferredCons; @@ -5771,31 +5788,45 @@ int i; sqlite_int64 rowid; Mem **apArg; Mem *pX; + assert( pOp->p2==1 || pOp->p5==OE_Fail || pOp->p5==OE_Rollback + || pOp->p5==OE_Abort || pOp->p5==OE_Ignore || pOp->p5==OE_Replace + ); pVtab = pOp->p4.pVtab->pVtab; pModule = (sqlite3_module *)pVtab->pModule; nArg = pOp->p2; assert( pOp->p4type==P4_VTAB ); if( ALWAYS(pModule->xUpdate) ){ + u8 vtabOnConflict = db->vtabOnConflict; apArg = p->apArg; pX = &aMem[pOp->p3]; for(i=0; ivtabOnConflict = pOp->p5; rc = pModule->xUpdate(pVtab, nArg, apArg, &rowid); + db->vtabOnConflict = vtabOnConflict; importVtabErrMsg(p, pVtab); if( rc==SQLITE_OK && pOp->p1 ){ assert( nArg>1 && apArg[0] && (apArg[0]->flags&MEM_Null) ); db->lastRowid = rowid; } - p->nChange++; + if( rc==SQLITE_CONSTRAINT && pOp->p4.pVtab->bConstraint ){ + if( pOp->p5==OE_Ignore ){ + rc = SQLITE_OK; + }else{ + p->errorAction = ((pOp->p5==OE_Replace) ? OE_Abort : pOp->p5); + } + }else{ + p->nChange++; + } } break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -2010,10 +2010,19 @@ } } } db->nStatement--; p->iStatement = 0; + + if( rc==SQLITE_OK ){ + if( eOp==SAVEPOINT_ROLLBACK ){ + rc = sqlite3VtabSavepoint(db, SAVEPOINT_ROLLBACK, iSavepoint); + } + if( rc==SQLITE_OK ){ + rc = sqlite3VtabSavepoint(db, SAVEPOINT_RELEASE, iSavepoint); + } + } /* If the statement transaction is being rolled back, also restore the ** database handles deferred constraint counter to the value it had when ** the statement transaction was opened. */ if( eOp==SAVEPOINT_ROLLBACK ){ Index: src/vtab.c ================================================================== --- src/vtab.c +++ src/vtab.c @@ -12,10 +12,22 @@ ** This file contains code used to help implement virtual tables. */ #ifndef SQLITE_OMIT_VIRTUALTABLE #include "sqliteInt.h" +/* +** Before a virtual table xCreate() or xConnect() method is invoked, the +** sqlite3.pVtabCtx member variable is set to point to an instance of +** this struct allocated on the stack. It is used by the implementation of +** the sqlite3_declare_vtab() and sqlite3_vtab_config() APIs, both of which +** are invoked only from within xCreate and xConnect methods. +*/ +struct VtabCtx { + Table *pTab; + VTable *pVTable; +}; + /* ** The actual function that does the work of creating a new module. ** This function implements the sqlite3_create_module() and ** sqlite3_create_module_v2() interfaces. */ @@ -432,10 +444,11 @@ Table *pTab, Module *pMod, int (*xConstruct)(sqlite3*,void*,int,const char*const*,sqlite3_vtab**,char**), char **pzErr ){ + VtabCtx sCtx; VTable *pVTable; int rc; const char *const*azArg = (const char *const*)pTab->azModuleArg; int nArg = pTab->nModuleArg; char *zErr = 0; @@ -451,16 +464,18 @@ return SQLITE_NOMEM; } pVTable->db = db; pVTable->pMod = pMod; - assert( !db->pVTab ); + /* Invoke the virtual table constructor */ + assert( &db->pVtabCtx ); assert( xConstruct ); - db->pVTab = pTab; - - /* Invoke the virtual table constructor */ + sCtx.pTab = pTab; + sCtx.pVTable = pVTable; + db->pVtabCtx = &sCtx; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); + db->pVtabCtx = 0; if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; if( SQLITE_OK!=rc ){ if( zErr==0 ){ *pzErr = sqlite3MPrintf(db, "vtable constructor failed: %s", zModuleName); @@ -472,11 +487,11 @@ }else if( ALWAYS(pVTable->pVtab) ){ /* Justification of ALWAYS(): A correct vtab constructor must allocate ** the sqlite3_vtab object if successful. */ pVTable->pVtab->pModule = pMod->pModule; pVTable->nRef = 1; - if( db->pVTab ){ + if( sCtx.pTab ){ const char *zFormat = "vtable constructor did not declare schema: %s"; *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName); sqlite3VtabUnlock(pVTable); rc = SQLITE_ERROR; }else{ @@ -520,11 +535,10 @@ } } } sqlite3DbFree(db, zModuleName); - db->pVTab = 0; return rc; } /* ** This function is invoked by the parser to call the xConnect() method @@ -640,12 +654,11 @@ int rc = SQLITE_OK; Table *pTab; char *zErr = 0; sqlite3_mutex_enter(db->mutex); - pTab = db->pVTab; - if( !pTab ){ + if( !db->pVtabCtx || !(pTab = db->pVtabCtx->pTab) ){ sqlite3Error(db, SQLITE_MISUSE, 0); sqlite3_mutex_leave(db->mutex); return SQLITE_MISUSE_BKPT; } assert( (pTab->tabFlags & TF_Virtual)!=0 ); @@ -668,11 +681,11 @@ pTab->aCol = pParse->pNewTable->aCol; pTab->nCol = pParse->pNewTable->nCol; pParse->pNewTable->nCol = 0; pParse->pNewTable->aCol = 0; } - db->pVTab = 0; + db->pVtabCtx->pTab = 0; }else{ sqlite3Error(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); rc = SQLITE_ERROR; } @@ -820,11 +833,10 @@ pModule = pVTab->pVtab->pModule; if( pModule->xBegin ){ int i; - /* If pVtab is already in the aVTrans array, return early */ for(i=0; inVTrans; i++){ if( db->aVTrans[i]==pVTab ){ return SQLITE_OK; } @@ -833,10 +845,53 @@ /* Invoke the xBegin method */ rc = pModule->xBegin(pVTab->pVtab); if( rc==SQLITE_OK ){ rc = addToVTrans(db, pVTab); } + } + return rc; +} + +/* +** Invoke either the xSavepoint, xRollbackTo or xRelease method of all +** virtual tables that currently have an open transaction. Pass iSavepoint +** as the second argument to the virtual table method invoked. +** +** If op is SAVEPOINT_BEGIN, the xSavepoint method is invoked. If it is +** SAVEPOINT_ROLLBACK, the xRollbackTo method. Otherwise, if op is +** SAVEPOINT_RELEASE, then the xRelease method of each virtual table with +** an open transaction is invoked. +** +** If any virtual table method returns an error code other than SQLITE_OK, +** processing is abandoned and the error returned to the caller of this +** function immediately. If all calls to virtual table methods are successful, +** SQLITE_OK is returned. +*/ +int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ + int rc = SQLITE_OK; + + assert( op==SAVEPOINT_RELEASE||op==SAVEPOINT_ROLLBACK||op==SAVEPOINT_BEGIN ); + if( db->aVTrans ){ + int i; + for(i=0; rc==SQLITE_OK && inVTrans; i++){ + const sqlite3_module *pMod = db->aVTrans[i]->pMod->pModule; + if( pMod->iVersion>=1 ){ + int (*xMethod)(sqlite3_vtab *, int); + switch( op ){ + case SAVEPOINT_BEGIN: + xMethod = pMod->xSavepoint; + break; + case SAVEPOINT_ROLLBACK: + xMethod = pMod->xRollbackTo; + break; + default: + xMethod = pMod->xRelease; + break; + } + if( xMethod ) rc = xMethod(db->aVTrans[i]->pVtab, iSavepoint); + } + } } return rc; } /* @@ -934,7 +989,47 @@ pToplevel->apVtabLock[pToplevel->nVtabLock++] = pTab; }else{ pToplevel->db->mallocFailed = 1; } } + +int sqlite3_vtab_on_conflict(sqlite3 *db){ + int aMap[] = { + SQLITE_ROLLBACK, SQLITE_IGNORE, SQLITE_ABORT, SQLITE_FAIL, SQLITE_REPLACE + }; + assert( OE_Rollback==1 && OE_Abort==2 && OE_Fail==3 ); + assert( OE_Ignore==4 && OE_Replace==5 ); + assert( db->vtabOnConflict>=1 && db->vtabOnConflict<=5 ); + return aMap[db->vtabOnConflict-1]; +} + + +int sqlite3_vtab_config(sqlite3 *db, int op, ...){ + va_list ap; + int rc = SQLITE_OK; + + sqlite3_mutex_enter(db->mutex); + + va_start(ap, op); + switch( op ){ + case SQLITE_VTAB_CONSTRAINT_SUPPORT: { + VtabCtx *p = db->pVtabCtx; + if( !p ){ + rc = SQLITE_MISUSE_BKPT; + }else{ + assert( (p->pTab->tabFlags & TF_Virtual)!=0 ); + p->pVTable->bConstraint = (u8)va_arg(ap, int); + } + break; + } + default: + rc = SQLITE_MISUSE_BKPT; + break; + } + va_end(ap); + + if( rc!=SQLITE_OK ) sqlite3Error(db, rc, 0); + sqlite3_mutex_leave(db->mutex); + return rc; +} #endif /* SQLITE_OMIT_VIRTUALTABLE */ Index: test/fts3atoken.test ================================================================== --- test/fts3atoken.test +++ test/fts3atoken.test @@ -165,18 +165,19 @@ append output "3 [string tolower $longtoken] $longtoken" do_icu_test fts3token-4.6 MiddleOfTheOcean $input $output do_icu_test fts3token-4.7 th_TH $input $output do_icu_test fts3token-4.8 en_US $input $output + + do_execsql_test 5.1 { + CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US); + insert into x1 (name) values (NULL); + insert into x1 (name) values (NULL); + delete from x1; + } } -do_execsql_test 5.1 { - CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US); - insert into x1 (name) values (NULL); - insert into x1 (name) values (NULL); - delete from x1; -} do_test fts3token-internal { execsql { SELECT fts3_tokenizer_internal_test() } } {ok} Index: test/fts3aux1.test ================================================================== --- test/fts3aux1.test +++ test/fts3aux1.test @@ -36,14 +36,14 @@ } { five 2 2 four 2 2 one 3 5 seven 1 1 six 1 1 three 4 6 two 1 1 } -do_execsql_test 1.3 { - DELETE FROM t1; +do_execsql_test 1.3.1 { DELETE FROM t1; } +do_execsql_test 1.3.2 { SELECT term, documents, occurrences FROM terms WHERE col = '*'; -} {} +} do_execsql_test 1.4 { INSERT INTO t1 VALUES('a b a b a b a'); INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; ADDED test/fts3conf.test Index: test/fts3conf.test ================================================================== --- /dev/null +++ test/fts3conf.test @@ -0,0 +1,139 @@ +# 2011 April 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS3 module. + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts3conf + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + + +proc fts3_integrity {tn db tbl} { + + if {[sqlite3_get_autocommit $db]==0} { + error "fts3_integrity does not work with an open transaction" + } + + set sql [db one {SELECT sql FROM sqlite_master WHERE name = $tbl}] + regexp -nocase {[^(]* using (.*)} $sql -> tail + set cols [list] + $db eval "PRAGMA table_info($tbl)" { + lappend cols $name + } + set cols [join [concat docid $cols] ,] + + $db eval [subst { + CREATE VIRTUAL TABLE fts3check USING fts4term($tbl); + CREATE VIRTUAL TABLE temp.fts3check2 USING $tail; + INSERT INTO temp.fts3check2($cols) SELECT docid, * FROM $tbl; + CREATE VIRTUAL TABLE temp.fts3check3 USING fts4term(fts3check2); + }] + + set m1 [$db one {SELECT md5sum(term, docid, col, pos) FROM fts3check}] + set m2 [$db one {SELECT md5sum(term, docid, col, pos) FROM fts3check3}] + + $db eval { + DROP TABLE fts3check; + DROP TABLE temp.fts3check2; + DROP TABLE temp.fts3check3; + } + + uplevel [list do_test $tn [list set {} $m1] $m2] +} + +do_execsql_test 1.0.1 { + CREATE VIRTUAL TABLE t1 USING fts3(x); + INSERT INTO t1(rowid, x) VALUES(1, 'a b c d'); + INSERT INTO t1(rowid, x) VALUES(2, 'e f g h'); + + CREATE TABLE source(a, b); + INSERT INTO source VALUES(4, 'z'); + INSERT INTO source VALUES(2, 'y'); +} +db_save_and_close + +set T1 "INTO t1(rowid, x) VALUES(1, 'x')" +set T2 "INTO t1(rowid, x) SELECT * FROM source" + +set T3 "t1 SET docid = 2 WHERE docid = 1" +set T4 "t1 SET docid = CASE WHEN docid = 1 THEN 4 ELSE 3 END WHERE docid <=2" + +foreach {tn sql uses constraint data} [subst { + 1 "INSERT OR ROLLBACK $T1" 0 1 {{a b c d} {e f g h}} + 2 "INSERT OR ABORT $T1" 0 1 {{a b c d} {e f g h} {i j k l}} + 3 "INSERT OR FAIL $T1" 0 1 {{a b c d} {e f g h} {i j k l}} + 4 "INSERT OR IGNORE $T1" 0 0 {{a b c d} {e f g h} {i j k l}} + 5 "INSERT OR REPLACE $T1" 0 0 {x {e f g h} {i j k l}} + + 6 "INSERT OR ROLLBACK $T2" 1 1 {{a b c d} {e f g h}} + 7 "INSERT OR ABORT $T2" 1 1 {{a b c d} {e f g h} {i j k l}} + 8 "INSERT OR FAIL $T2" 1 1 {{a b c d} {e f g h} {i j k l} z} + 9 "INSERT OR IGNORE $T2" 1 0 {{a b c d} {e f g h} {i j k l} z} + 10 "INSERT OR REPLACE $T2" 1 0 {{a b c d} y {i j k l} z} + + 11 "UPDATE OR ROLLBACK $T3" 1 1 {{a b c d} {e f g h}} + 12 "UPDATE OR ABORT $T3" 1 1 {{a b c d} {e f g h} {i j k l}} + 13 "UPDATE OR FAIL $T3" 1 1 {{a b c d} {e f g h} {i j k l}} + 14 "UPDATE OR IGNORE $T3" 1 0 {{a b c d} {e f g h} {i j k l}} + 15 "UPDATE OR REPLACE $T3" 1 0 {{a b c d} {i j k l}} + + 16 "UPDATE OR ROLLBACK $T4" 1 1 {{a b c d} {e f g h}} + 17 "UPDATE OR ABORT $T4" 1 1 {{a b c d} {e f g h} {i j k l}} + 18 "UPDATE OR FAIL $T4" 1 1 {{e f g h} {i j k l} {a b c d}} + 19 "UPDATE OR IGNORE $T4" 1 0 {{e f g h} {i j k l} {a b c d}} + 20 "UPDATE OR REPLACE $T4" 1 0 {{e f g h} {a b c d}} +}] { + db_restore_and_reopen + execsql { + BEGIN; + INSERT INTO t1(rowid, x) VALUES(3, 'i j k l'); + } + set R(0) {0 {}} + set R(1) {1 {constraint failed}} + do_catchsql_test 1.$tn.1 $sql $R($constraint) + do_catchsql_test 1.$tn.2 { SELECT * FROM t1 } [list 0 $data] + catchsql COMMIT + + fts3_integrity 1.$tn.3 db t1 + + do_test 1.$tn.4 [list sql_uses_stmt db $sql] $uses +} + +do_execsql_test 2.1.1 { + DELETE FROM t1; + BEGIN; + INSERT INTO t1 VALUES('a b c'); + SAVEPOINT a; + INSERT INTO t1 VALUES('x y z'); + ROLLBACK TO a; + COMMIT; +} +fts3_integrity 2.1.2 db t1 + +do_catchsql_test 2.2.1 { + DELETE FROM t1; + BEGIN; + INSERT INTO t1(docid, x) VALUES(0, 'a b c'); + INSERT INTO t1(docid, x) VALUES(1, 'a b c'); + REPLACE INTO t1(docid, x) VALUES('zero', 'd e f'); +} {1 {datatype mismatch}} +do_execsql_test 2.2.2 { COMMIT } +do_execsql_test 2.2.3 { SELECT * FROM t1 } {{a b c} {a b c}} +fts3_integrity 2.2.4 db t1 + +finish_test ADDED test/fts3sort.test Index: test/fts3sort.test ================================================================== --- /dev/null +++ test/fts3sort.test @@ -0,0 +1,108 @@ +# 2011 May 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS3 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +set testprefix fts3sort + +proc build_database {nRow} { + db close + forcedelete test.db + sqlite3 db test.db + + set vocab [list aa ab ac ba bb bc ca cb cc da] + expr srand(0) + + execsql { CREATE VIRTUAL TABLE t1 USING fts4 } + for {set i 0} {$i < $nRow} {incr i} { + set v [expr int(rand()*1000000)] + set doc [list] + for {set div 1} {$div < 1000000} {set div [expr $div*10]} { + lappend doc [lindex $vocab [expr ($v/$div) % 10]] + } + execsql { INSERT INTO t1 VALUES($doc) } + } +} + +set nRow 1000 +do_test 1.0 { + build_database $nRow + execsql { SELECT count(*) FROM t1 } +} $nRow + +foreach {tn query} { + 1 "SELECT docid, * FROM t1" + 2 "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa'" + 3 "SELECT docid, * FROM t1 WHERE t1 MATCH 'a*'" + 4 "SELECT docid, quote(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'a*'" + 5 "SELECT docid, quote(matchinfo(t1,'pcnxals')) FROM t1 WHERE t1 MATCH 'b*'" + 6 "SELECT docid, * FROM t1 WHERE t1 MATCH 'a* b* c*'" + 7 "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa OR da'" + 8 "SELECT docid, * FROM t1 WHERE t1 MATCH 'nosuchtoken'" + 9 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR da'" + 10 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR nosuchtoken'" +} { + + unset -nocomplain A B C D + set A_list [list] + set B_list [list] + set C_list [list] + set D_list [list] + + unset -nocomplain X + db eval "$query ORDER BY rowid ASC" X { + set A($X(docid)) [array get X] + lappend A_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY rowid DESC" X { + set B($X(docid)) [array get X] + lappend B_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY docid ASC" X { + set C($X(docid)) [array get X] + lappend C_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY docid DESC" X { + set D($X(docid)) [array get X] + lappend D_list $X(docid) + } + + do_test 1.$tn.1 { set A_list } [lsort -integer -increasing $A_list] + do_test 1.$tn.2 { set B_list } [lsort -integer -decreasing $B_list] + do_test 1.$tn.3 { set C_list } [lsort -integer -increasing $C_list] + do_test 1.$tn.4 { set D_list } [lsort -integer -decreasing $D_list] + + unset -nocomplain DATA + unset -nocomplain X + db eval "$query" X { + set DATA($X(docid)) [array get X] + } + + do_test 1.$tn.5 { lsort [array get A] } [lsort [array get DATA]] + do_test 1.$tn.6 { lsort [array get B] } [lsort [array get DATA]] + do_test 1.$tn.7 { lsort [array get C] } [lsort [array get DATA]] + do_test 1.$tn.8 { lsort [array get D] } [lsort [array get DATA]] +} + +finish_test Index: test/tester.tcl ================================================================== --- test/tester.tcl +++ test/tester.tcl @@ -745,10 +745,21 @@ ifcapable integrityck { do_test $name [list execsql {PRAGMA integrity_check} $db] {ok} } } + +# Return true if the SQL statement passed as the second argument uses a +# statement transaction. +# +proc sql_uses_stmt {db sql} { + set stmt [sqlite3_prepare $db $sql -1 dummy] + set uses [uses_stmt_journal $stmt] + sqlite3_finalize $stmt + return $uses +} + proc fix_ifcapable_expr {expr} { set ret "" set state 0 for {set i 0} {$i < [string length $expr]} {incr i} { set char [string range $expr $i $i]