Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -421,16 +421,16 @@ /* ** When this function is called, *pp points to the first byte following a ** varint that is part of a doclist (or position-list, or any other list ** of varints). This function moves *pp to point to the start of that varint, -** and decrements the value stored in *pVal by the varint value. +** and sets *pVal by the varint value. ** ** Argument pStart points to the first byte of the doclist that the ** varint is part of. */ -static void fts3GetReverseDeltaVarint( +static void fts3GetReverseVarint( char **pp, char *pStart, sqlite3_int64 *pVal ){ sqlite3_int64 iVal; @@ -442,11 +442,11 @@ for(p = (*pp)-2; p>=pStart && *p&0x80; p--); p++; *pp = p; sqlite3Fts3GetVarint(p, &iVal); - *pVal -= iVal; + *pVal = iVal; } /* ** As long as *pp has not reached its end (pEnd), then do the same ** as fts3GetDeltaVarint(): read a single varint and add it to *pVal. @@ -914,21 +914,41 @@ int nCol = 0; /* Number of columns in the FTS table */ char *zCsr; /* Space for holding column names */ int nDb; /* Bytes required to hold database name */ int nName; /* Bytes required to hold table name */ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ - int bNoDocsize = 0; /* True to omit %_docsize table */ const char **aCol; /* Array of column names */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ - char *zPrefix = 0; /* Prefix parameter value (or NULL) */ int nIndex; /* Size of aIndex[] array */ struct Fts3Index *aIndex; /* Array of indexes for this table */ struct Fts3Index *aFree = 0; /* Free this before returning */ - char *zCompress = 0; - char *zUncompress = 0; + int bNoDocsize = 0; /* True to omit %_docsize table */ + int bDescIdx = 0; /* True to store descending indexes */ + + char *zMatchinfo = 0; /* Prefix parameter value (or NULL) */ + char *zPrefix = 0; /* Prefix parameter value (or NULL) */ + char *zCompress = 0; /* compress=? parameter (or NULL) */ + char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ + char *zOrder = 0; /* order=? parameter (or NULL) */ + struct Fts4Option { + const char *zOpt; + int nOpt; + char **pzVar; + } aFts4Opt[] = { + { "matchinfo", 9, 0 }, + { "prefix", 6, 0 }, + { "compress", 8, 0 }, + { "uncompress", 10, 0 }, + { "order", 5, 0 } + }; + aFts4Opt[0].pzVar = &zMatchinfo; + aFts4Opt[1].pzVar = &zPrefix; + aFts4Opt[2].pzVar = &zCompress; + aFts4Opt[3].pzVar = &zUncompress; + aFts4Opt[4].pzVar = &zOrder; assert( strlen(argv[0])==4 ); assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) ); @@ -965,36 +985,31 @@ rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); } /* Check if it is an FTS4 special argument. */ else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ + int iOpt; if( !zVal ){ rc = SQLITE_NOMEM; - goto fts3_init_out; - } - if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){ - if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){ - bNoDocsize = 1; - }else{ - *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); + }else{ + for(iOpt=0; iOptpTokenizer = pTokenizer; p->nNodeSize = 1000; p->nMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; + p->bDescIdx = bDescIdx; TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); @@ -1106,10 +1142,12 @@ fts3_init_out: sqlite3_free(zPrefix); sqlite3_free(aFree); sqlite3_free(zCompress); sqlite3_free(zUncompress); + sqlite3_free(zOrder); + sqlite3_free(zMatchinfo); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ fts3DisconnectMethod((sqlite3_vtab *)p); }else if( pTokenizer ){ @@ -1878,14 +1916,15 @@ ** Values that may be used as the first parameter to fts3DoclistMerge(). */ #define MERGE_NOT 2 /* D + D -> D */ #define MERGE_AND 3 /* D + D -> D */ #define MERGE_OR 4 /* D + D -> D */ -#define MERGE_POS_OR 5 /* P + P -> P */ #define MERGE_PHRASE 6 /* P + P -> D */ -#define MERGE_POS_PHRASE 7 /* P + P -> P */ #define MERGE_NEAR 8 /* P + P -> D */ + +#define MERGE_POS_OR 5 /* P + P -> P */ +#define MERGE_POS_PHRASE 7 /* P + P -> P */ #define MERGE_POS_NEAR 9 /* P + P -> P */ /* ** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2 ** (size n2 bytes). The output is written to pre-allocated buffer aBuffer, @@ -1922,10 +1961,11 @@ assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR || mergetype==MERGE_AND || mergetype==MERGE_NOT || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR ); + assert( mergetype==MERGE_POS_PHRASE || mergetype==MERGE_POS_NEAR ); if( !aBuffer ){ *pnBuffer = 0; return SQLITE_NOMEM; } @@ -2062,10 +2102,231 @@ struct TermSelect { int isReqPos; char *aaOutput[16]; /* Malloc'd output buffer */ int anOutput[16]; /* Size of output in bytes */ }; + + +static void fts3GetDeltaVarint3( + char **pp, + char *pEnd, + int bDescIdx, + sqlite3_int64 *pVal +){ + if( *pp>=pEnd ){ + *pp = 0; + }else{ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + if( bDescIdx ){ + *pVal -= iVal; + }else{ + *pVal += iVal; + } + } +} + +static void fts3PutDeltaVarint3( + char **pp, /* IN/OUT: Output pointer */ + int bDescIdx, /* True for descending docids */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + int *pbFirst, /* IN/OUT: True after first int written */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + sqlite3_int64 iWrite; + if( bDescIdx==0 || *pbFirst==0 ){ + iWrite = iVal - *piPrev; + }else{ + iWrite = *piPrev - iVal; + } + assert( *pbFirst || *piPrev==0 ); + assert( *pbFirst==0 || iWrite>0 ); + *pp += sqlite3Fts3PutVarint(*pp, iWrite); + *piPrev = iVal; + *pbFirst = 1; +} + +#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2)) + +static int fts3DoclistOrMerge( + int bDescIdx, /* True if arguments are desc */ + u8 *a1, int n1, /* First doclist */ + u8 *a2, int n2, /* Second doclist */ + u8 **paOut, int *pnOut /* OUT: Malloc'd doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &a1[n1]; + char *pEnd2 = &a2[n2]; + char *p1 = a1; + char *p2 = a2; + char *p; + int nOut; + char *aOut; + int bFirstOut = 0; + + *paOut = 0; + *pnOut = 0; + aOut = sqlite3_malloc(n1+n2); + if( !aOut ) return SQLITE_NOMEM; + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + while( p1 || p2 ){ + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + + if( p2 && p1 && iDiff==0 ){ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PoslistMerge(&p, &p1, &p2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + }else if( !p2 || (p1 && iDiff<0) ){ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PoslistCopy(&p, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + }else{ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2); + fts3PoslistCopy(&p, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + } + } + + *paOut = aOut; + *pnOut = (p-aOut); + return SQLITE_OK; +} + +static void fts3DoclistPhraseMerge( + int bDescIdx, /* True if arguments are desc */ + int nDist, /* Distance from left to right (1=adjacent) */ + u8 *aLeft, int nLeft, /* Left doclist */ + u8 *aRight, int *pnRight /* IN/OUT: Right/output doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &aLeft[nLeft]; + char *pEnd2 = &aRight[*pnRight]; + char *p1 = aLeft; + char *p2 = aRight; + char *p; + int bFirstOut = 0; + char *aOut = aRight; + + assert( nDist>0 ); + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + + while( p1 && p2 ){ + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + if( iDiff==0 ){ + char *pSave = p; + sqlite3_int64 iPrevSave = iPrev; + int bFirstOutSave = bFirstOut; + + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ + p = pSave; + iPrev = iPrevSave; + bFirstOut = bFirstOutSave; + } + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + }else if( iDiff<0 ){ + fts3PoslistCopy(0, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + }else{ + fts3PoslistCopy(0, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + } + } + + *pnRight = p - aOut; +} + +/* +** This function merges two doclists according to the requirements of a +** NEAR operator. +*/ +static int fts3DoclistNearMerge( + int bDescIdx, + int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */ + int nNear, /* Parameter to NEAR operator */ + int nTokenLeft, /* Number of tokens in LHS phrase arg */ + char *aLeft, /* Doclist for LHS (incl. positions) */ + int nLeft, /* Size of LHS doclist in bytes */ + int nTokenRight, /* As nTokenLeft */ + char *aRight, /* As aLeft */ + int nRight, /* As nRight */ + char **paOut, /* OUT: Results of merge (malloced) */ + int *pnOut /* OUT: Sized of output buffer */ +){ + char *aOut; /* Buffer to write output doclist to */ + char *aTmp; /* Temp buffer used by PoslistNearMerge() */ + + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + int bFirstOut = 0; + + char *pEnd1 = &aLeft[nLeft]; + char *pEnd2 = &aRight[nRight]; + char *p1 = aLeft; + char *p2 = aRight; + char *p; + + int nParam1 = nNear+nTokenRight; + int nParam2 = nNear+nTokenLeft; + + p = aOut = sqlite3_malloc(nLeft+nRight+1); + aTmp = sqlite3_malloc(2*(nLeft+nRight+1)); + if( !aOut || !aTmp ){ + sqlite3_free(aOut); + sqlite3_free(aTmp); + *paOut = 0; + *pnOut = 0; + return SQLITE_NOMEM; + } + + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + + while( p1 && p2 ){ + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + if( iDiff==0 ){ + char *pSave = p; + sqlite3_int64 iPrevSave = iPrev; + int bFirstOutSave = bFirstOut; + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + if( !fts3PoslistNearMerge(&p, aTmp, nParam1, nParam2, &p1, &p2) ){ + p = pSave; + iPrev = iPrevSave; + bFirstOut = bFirstOutSave; + } + + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + }else if( iDiff<0 ){ + fts3PoslistCopy(0, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + }else{ + fts3PoslistCopy(0, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + } + } + + sqlite3_free(aTmp); + *paOut = aOut; + *pnOut = p - aOut; + return SQLITE_OK; +} + + /* ** Merge all doclists in the TermSelect.aaOutput[] array into a single ** doclist stored in TermSelect.aaOutput[0]. If successful, delete all ** other doclists (except the aaOutput[0] one) and return SQLITE_OK. @@ -2072,11 +2333,11 @@ ** ** If an OOM error occurs, return SQLITE_NOMEM. In this case it is ** the responsibility of the caller to free any doclists left in the ** TermSelect.aaOutput[] array. */ -static int fts3TermSelectMerge(TermSelect *pTS){ +static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){ int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); char *aOut = 0; int nOut = 0; int i; @@ -2088,19 +2349,21 @@ if( !aOut ){ aOut = pTS->aaOutput[i]; nOut = pTS->anOutput[i]; pTS->aaOutput[i] = 0; }else{ - int nNew = nOut + pTS->anOutput[i]; - char *aNew = sqlite3_malloc(nNew); - if( !aNew ){ + int nNew; + u8 *aNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, + pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ sqlite3_free(aOut); - return SQLITE_NOMEM; + return rc; } - fts3DoclistMerge(mergetype, 0, 0, - aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0 - ); + sqlite3_free(pTS->aaOutput[i]); sqlite3_free(aOut); pTS->aaOutput[i] = 0; aOut = aNew; nOut = nNew; @@ -2132,13 +2395,11 @@ UNUSED_PARAMETER(zTerm); UNUSED_PARAMETER(nTerm); if( pTS->aaOutput[0]==0 ){ /* If this is the first term selected, copy the doclist to the output - ** buffer using memcpy(). TODO: Add a way to transfer control of the - ** aDoclist buffer from the caller so as to avoid the memcpy(). - */ + ** buffer using memcpy(). */ pTS->aaOutput[0] = sqlite3_malloc(nDoclist); pTS->anOutput[0] = nDoclist; if( pTS->aaOutput[0] ){ memcpy(pTS->aaOutput[0], aDoclist, nDoclist); }else{ @@ -2149,40 +2410,37 @@ char *aMerge = aDoclist; int nMerge = nDoclist; int iOut; for(iOut=0; iOutaaOutput); iOut++){ - char *aNew; - int nNew; if( pTS->aaOutput[iOut]==0 ){ assert( iOut>0 ); pTS->aaOutput[iOut] = aMerge; pTS->anOutput[iOut] = nMerge; break; - } - - nNew = nMerge + pTS->anOutput[iOut]; - aNew = sqlite3_malloc(nNew); - if( !aNew ){ - if( aMerge!=aDoclist ){ - sqlite3_free(aMerge); - } - return SQLITE_NOMEM; - } - fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, - pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0 - ); - - if( iOut>0 ) sqlite3_free(aMerge); - sqlite3_free(pTS->aaOutput[iOut]); - pTS->aaOutput[iOut] = 0; - - aMerge = aNew; - nMerge = nNew; - if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ - pTS->aaOutput[iOut] = aMerge; - pTS->anOutput[iOut] = nMerge; + }else{ + u8 *aNew; + int nNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, + pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + return rc; + } + + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + sqlite3_free(pTS->aaOutput[iOut]); + pTS->aaOutput[iOut] = 0; + + aMerge = aNew; + nMerge = nNew; + if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + } } } } return SQLITE_OK; } @@ -2424,11 +2682,11 @@ pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist ); } if( rc==SQLITE_OK ){ - rc = fts3TermSelectMerge(&tsc); + rc = fts3TermSelectMerge(p, &tsc); } if( rc==SQLITE_OK ){ *ppOut = tsc.aaOutput[0]; *pnOut = tsc.anOutput[0]; }else{ @@ -2474,52 +2732,10 @@ } return nDoc; } -/* -** This function merges two doclists according to the requirements of a -** NEAR operator. -** -** Both input doclists must include position information. The output doclist -** includes position information if the first argument to this function -** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR. -*/ -static int fts3NearMerge( - int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */ - int nNear, /* Parameter to NEAR operator */ - int nTokenLeft, /* Number of tokens in LHS phrase arg */ - char *aLeft, /* Doclist for LHS (incl. positions) */ - int nLeft, /* Size of LHS doclist in bytes */ - int nTokenRight, /* As nTokenLeft */ - char *aRight, /* As aLeft */ - int nRight, /* As nRight */ - char **paOut, /* OUT: Results of merge (malloced) */ - int *pnOut /* OUT: Sized of output buffer */ -){ - char *aOut; /* Buffer to write output doclist to */ - int rc; /* Return code */ - - assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR ); - - aOut = sqlite3_malloc(nLeft+nRight+1); - if( aOut==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, - aOut, pnOut, aLeft, nLeft, aRight, nRight, 0 - ); - if( rc!=SQLITE_OK ){ - sqlite3_free(aOut); - aOut = 0; - } - } - - *paOut = aOut; - return rc; -} - /* ** Advance the cursor to the next row in the %_content table that ** matches the search criteria. For a MATCH search, this will be ** the next row that matches. For a full-table scan, this will be ** simply the next row in the %_content table. For a docid lookup, @@ -2586,11 +2802,15 @@ sqlite3_finalize(pCsr->pStmt); sqlite3_free(pCsr->aDoclist); sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); - pCsr->bDesc = (idxStr && idxStr[0]=='D'); + if( idxStr ){ + pCsr->bDesc = (idxStr[0]=='D'); + }else{ + pCsr->bDesc = p->bDescIdx; + } pCsr->eSearch = (i16)idxNum; if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){ int iCol = idxNum-FTS3_FULLTEXT_SEARCH; const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); @@ -2625,11 +2845,11 @@ ** statement loops through all rows of the %_content table. For a ** full-text query or docid lookup, the statement retrieves a single ** row by docid. */ if( idxNum==FTS3_FULLSCAN_SEARCH ){ - const char *zSort = (idxStr ? idxStr : "ASC"); + const char *zSort = (pCsr->bDesc ? "DESC" : "ASC"); const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s"; zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort); }else{ const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?"; zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName); @@ -3263,12 +3483,12 @@ }else if( aDoclist==0 ){ aDoclist = pThis; nDoclist = nThis; }else{ assert( iPrev>=0 ); - fts3DoclistMerge(MERGE_POS_PHRASE, iToken-iPrev, - 0, pThis, &nThis, aDoclist, nDoclist, pThis, nThis, 0 + fts3DoclistPhraseMerge(pTab->bDescIdx, + iToken-iPrev, aDoclist, nDoclist, pThis, &nThis ); sqlite3_free(aDoclist); aDoclist = pThis; nDoclist = nThis; } @@ -3409,18 +3629,18 @@ */ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ int rc; Fts3Doclist *pList = &p->doclist; Fts3PhraseToken *pFirst = &p->aToken[0]; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; assert( pList->aAll==0 ); - if( pCsr->bDesc==0 && bOptOk==1 && p->nToken==1 + if( pCsr->bDesc==pTab->bDescIdx && bOptOk==1 && p->nToken==1 && pFirst->pSegcsr && pFirst->pSegcsr->bLookup ){ /* Use the incremental approach. */ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); rc = sqlite3Fts3MsrIncrStart( pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n); p->bIncr = 1; @@ -3431,10 +3651,62 @@ } assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); return rc; } + +void sqlite3Fts3DoclistPrev( + int bDescIdx, /* True if the doclist is desc */ + char *aDoclist, /* Pointer to entire doclist */ + int nDoclist, /* Length of aDoclist in bytes */ + char **ppIter, /* IN/OUT: Iterator pointer */ + sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ + int *pnList, /* IN/OUT: List length pointer */ + u8 *pbEof /* OUT: End-of-file flag */ +){ + char *p = *ppIter; + int iMul = (bDescIdx ? -1 : 1); + + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); + + if( p==0 ){ + sqlite3_int64 iDocid = 0; + char *pNext = 0; + char *pDocid = aDoclist; + char *pEnd = &aDoclist[nDoclist]; + + pDocid += sqlite3Fts3GetVarint(pDocid, &iDocid); + pNext = pDocid; + fts3PoslistCopy(0, &pDocid); + while( pDociddoclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; if( p->bIncr ){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; assert( p->nToken==1 ); rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, &pDL->iDocid, &pDL->pList, &pDL->nList ); if( rc==SQLITE_OK && !pDL->pList ){ *pbEof = 1; } - }else if( pCsr->bDesc && pDL->aAll ){ - - if( pDL->pNextDocid==0 ){ - sqlite3_int64 iDocid = 0; - char *pNext; - char *pDocid = pDL->aAll; - char *pEnd = &pDocid[pDL->nAll]; - - while( pDocidpNextDocid = pDocid; - pDL->pList = pDocid; - fts3PoslistCopy(0, &pDocid); - } - pDL->nList = (pEnd - pDL->pList); - pDL->iDocid = iDocid; - }else{ - - assert( *pbEof==0 ); - assert( pDL->pNextDocid>pDL->aAll ); - - fts3GetReverseDeltaVarint( - &pDL->pNextDocid, pDL->aAll, &pDL->iDocid - ); - if( pDL->pNextDocid==pDL->aAll ){ - *pbEof = 1; - }else{ - char *pSave = pDL->pNextDocid; - fts3ReversePoslist(pDL->aAll, &pDL->pNextDocid); - pDL->pList = pDL->pNextDocid; - pDL->nList = pSave - pDL->pNextDocid; - } - } - + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->aAll ){ + sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, + &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof + ); + pDL->pList = pDL->pNextDocid; }else{ - char *pIter; if( pDL->pNextDocid ){ pIter = pDL->pNextDocid; }else{ pIter = pDL->aAll; @@ -3505,11 +3747,17 @@ if( pIter>=&pDL->aAll[pDL->nAll] ){ /* We have already reached the end of this doclist. EOF. */ *pbEof = 1; }else{ - fts3GetDeltaVarint(&pIter, &pDL->iDocid); + sqlite3_int64 iDelta; + pIter += sqlite3Fts3GetVarint(pIter, &iDelta); + if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ + pDL->iDocid += iDelta; + }else{ + pDL->iDocid -= iDelta; + } pDL->pList = pIter; fts3PoslistCopy(0, &pIter); pDL->nList = (pIter - pDL->pList); pDL->pNextDocid = pIter; *pbEof = 0; @@ -3544,10 +3792,11 @@ } } } static void fts3EvalNearMerge( + int bDescIdx, Fts3Expr *p1, Fts3Expr *p2, int nNear, int *pRc ){ @@ -3565,11 +3814,11 @@ pRight->doclist.nAll = 0; }else if( pRight->doclist.aAll ){ char *aOut; /* Buffer in which to assemble new doclist */ int nOut; /* Size of buffer aOut in bytes */ - *pRc = fts3NearMerge(MERGE_POS_NEAR, nNear, + *pRc = fts3DoclistNearMerge(bDescIdx, MERGE_POS_NEAR, nNear, pLeft->nToken, pLeft->doclist.aAll, pLeft->doclist.nAll, pRight->nToken, pRight->doclist.aAll, pRight->doclist.nAll, &aOut, &nOut ); sqlite3_free(pRight->doclist.aAll); @@ -3600,24 +3849,25 @@ aPhrase = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nPhrase); if( !aPhrase ){ *pRc = SQLITE_NOMEM; }else{ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; int i = 1; aPhrase[0] = pLeft; do { pLeft = pLeft->pParent; aPhrase[i++] = pLeft->pRight; }while( pLeft!=pExpr ); for(i=0; i<(nPhrase-1); i++){ int nNear = aPhrase[i+1]->pParent->nNear; - fts3EvalNearMerge(aPhrase[i], aPhrase[i+1], nNear, pRc); + fts3EvalNearMerge(p->bDescIdx, aPhrase[i], aPhrase[i+1], nNear, pRc); } for(i=nPhrase-2; i>=0; i--){ int nNear = aPhrase[i+1]->pParent->nNear; - fts3EvalNearMerge(aPhrase[i+1], aPhrase[i], nNear, pRc); + fts3EvalNearMerge(p->bDescIdx, aPhrase[i+1], aPhrase[i], nNear, pRc); } sqlite3_free(aPhrase); } Index: ext/fts3/fts3Int.h ================================================================== --- ext/fts3/fts3Int.h +++ ext/fts3/fts3Int.h @@ -178,10 +178,11 @@ char *zWriteExprlist; int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ + u8 bDescIdx; /* True if doclists are in reverse order */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ /* TODO: Fix the first paragraph of this comment. @@ -234,11 +235,11 @@ Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ char *pNextId; /* Pointer into the body of aDoclist */ char *aDoclist; /* List of docids for full-text queries */ int nDoclist; /* Size of buffer at aDoclist */ - int bDesc; /* True to sort in descending order */ + u8 bDesc; /* True to sort in descending order */ int eEvalmode; /* An FTS3_EVAL_XX constant */ int nRowAvg; /* Average size of database rows, in pages */ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ u32 *aMatchinfo; /* Information about most recent match */ @@ -436,10 +437,11 @@ int sqlite3Fts3PutVarint(char *, sqlite3_int64); int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); +void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); /* fts3_tokenizer.c */ @@ -492,7 +494,8 @@ Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); + #endif /* _FTSINT_H */ Index: ext/fts3/fts3_write.c ================================================================== --- ext/fts3/fts3_write.c +++ ext/fts3/fts3_write.c @@ -60,13 +60,12 @@ typedef struct PendingList PendingList; typedef struct SegmentNode SegmentNode; typedef struct SegmentWriter SegmentWriter; /* -** Data structure used while accumulating terms in the pending-terms hash -** table. The hash table entry maps from term (a string) to a malloc'd -** instance of this structure. +** An instance of the following data structure is used to build doclists +** incrementally. See function fts3PendingListAppend() for details. */ struct PendingList { int nData; char *aData; int nSpace; @@ -128,12 +127,15 @@ char *zTerm; /* Pointer to current term */ int nTermAlloc; /* Allocated size of zTerm buffer */ char *aDoclist; /* Pointer to doclist of current entry */ int nDoclist; /* Size of doclist in current entry */ - /* The following variables are used to iterate through the current doclist */ + /* The following variables are used by fts3SegReaderNextDocid() to iterate + ** through the current doclist (aDoclist/nDoclist). + */ char *pOffsetList; + int nOffsetList; /* For descending pending seg-readers only */ sqlite3_int64 iDocid; }; #define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) #define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1]) @@ -571,15 +573,25 @@ return 1; } return 0; } +/* +** Free a PendingList object allocated by fts3PendingListAppend(). +*/ +static void fts3PendingListDelete(PendingList *pList){ + sqlite3_free(pList); +} + +/* +** Add an entry to one of the pending-terms hash tables. +*/ static int fts3PendingTermsAddOne( Fts3Table *p, int iCol, int iPos, - Fts3Hash *pHash, + Fts3Hash *pHash, /* Pending terms hash table to add entry to */ const char *zToken, int nToken ){ PendingList *pList; int rc = SQLITE_OK; @@ -711,11 +723,12 @@ int i; for(i=0; inIndex; i++){ Fts3HashElem *pElem; Fts3Hash *pHash = &p->aIndex[i].hPending; for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ - sqlite3_free(fts3HashData(pElem)); + PendingList *pList = (PendingList *)fts3HashData(pElem); + fts3PendingListDelete(pList); } fts3HashClear(pHash); } p->nPendingData = 0; } @@ -1112,17 +1125,18 @@ pReader->pBlob = p->pSegments; p->pSegments = 0; } pNext = pReader->aNode; } + + assert( !fts3SegReaderIsPending(pReader) ); rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); if( rc!=SQLITE_OK ) return rc; /* Because of the FTS3_NODE_PADDING bytes of padding, the following is - ** safe (no risk of overread) even if the node data is corrupted. - */ + ** safe (no risk of overread) even if the node data is corrupted. */ pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); if( nPrefix<0 || nSuffix<=0 || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] ){ @@ -1163,18 +1177,28 @@ /* ** Set the SegReader to point to the first docid in the doclist associated ** with the current term. */ -static int fts3SegReaderFirstDocid(Fts3SegReader *pReader){ - int rc; +static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ + int rc = SQLITE_OK; assert( pReader->aDoclist ); assert( !pReader->pOffsetList ); - rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); - if( rc==SQLITE_OK ){ - int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); - pReader->pOffsetList = &pReader->aDoclist[n]; + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + u8 bEof = 0; + pReader->iDocid = 0; + pReader->nOffsetList = 0; + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, + &pReader->iDocid, &pReader->nOffsetList, &bEof + ); + }else{ + rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; + } } return rc; } /* @@ -1186,55 +1210,87 @@ ** in the doclist entry (i.e. immediately past the docid varint). ** *pnOffsetList is set to the length of the set of column-offset ** lists, not including the nul-terminator byte. For example: */ static int fts3SegReaderNextDocid( - Fts3SegReader *pReader, - char **ppOffsetList, - int *pnOffsetList + Fts3Table *pTab, + Fts3SegReader *pReader, /* Reader to advance to next docid */ + char **ppOffsetList, /* OUT: Pointer to current position-list */ + int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ ){ int rc = SQLITE_OK; char *p = pReader->pOffsetList; char c = 0; - /* Pointer p currently points at the first byte of an offset list. The - ** following two lines advance it to point one byte past the end of - ** the same offset list. - */ - while( 1 ){ - int nRead; - int rc; - - while( *p | c ) c = *p++ & 0x80; - assert( *p==0 ); - if( pReader->pBlob==0 || (p - pReader->aNode)!=pReader->nPopulate ) break; - rc = fts3SegReaderIncrRead(pReader); - if( rc!=SQLITE_OK ) return rc; - } - p++; - - /* If required, populate the output variables with a pointer to and the - ** size of the previous offset-list. - */ - if( ppOffsetList ){ - *ppOffsetList = pReader->pOffsetList; - *pnOffsetList = (int)(p - pReader->pOffsetList - 1); - } - - /* If there are no more entries in the doclist, set pOffsetList to - ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and - ** Fts3SegReader.pOffsetList to point to the next offset list before - ** returning. - */ - if( p>=&pReader->aDoclist[pReader->nDoclist] ){ - pReader->pOffsetList = 0; - }else{ - rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); - if( rc==SQLITE_OK ){ - sqlite3_int64 iDelta; - pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); - pReader->iDocid += iDelta; + assert( p ); + + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + /* A pending-terms seg-reader for an FTS4 table that uses order=desc. + ** Pending-terms doclists are always built up in ascending order, so + ** we have to iterate through them backwards here. */ + u8 bEof = 0; + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = pReader->nOffsetList - 1; + } + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, + &pReader->nOffsetList, &bEof + ); + if( bEof ){ + pReader->pOffsetList = 0; + }else{ + pReader->pOffsetList = p; + } + }else{ + + /* Pointer p currently points at the first byte of an offset list. The + ** following block advances it to point one byte past the end of + ** the same offset list. */ + while( 1 ){ + + /* The following line of code (and the "p++" below the while() loop) is + ** normally all that is required to move pointer p to the desired + ** position. The exception is if this node is being loaded from disk + ** incrementally and pointer "p" now points to the first byte passed + ** the populated part of pReader->aNode[]. + */ + while( *p | c ) c = *p++ & 0x80; + assert( *p==0 ); + + if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; + rc = fts3SegReaderIncrRead(pReader); + if( rc!=SQLITE_OK ) return rc; + } + p++; + + /* If required, populate the output variables with a pointer to and the + ** size of the previous offset-list. + */ + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = (int)(p - pReader->pOffsetList - 1); + } + + /* If there are no more entries in the doclist, set pOffsetList to + ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and + ** Fts3SegReader.pOffsetList to point to the next offset list before + ** returning. + */ + if( p>=&pReader->aDoclist[pReader->nDoclist] ){ + pReader->pOffsetList = 0; + }else{ + rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + sqlite3_int64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); + if( pTab->bDescIdx ){ + pReader->iDocid -= iDelta; + }else{ + pReader->iDocid += iDelta; + } + } } } return SQLITE_OK; } @@ -1595,10 +1651,22 @@ if( pLhs->iDocid==pRhs->iDocid ){ rc = pRhs->iIdx - pLhs->iIdx; }else{ rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1; } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} +static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; + } } assert( pLhs->aNode && pRhs->aNode ); return rc; } @@ -2288,10 +2356,13 @@ const char *zTerm, /* Term to iterate through a doclist for */ int nTerm /* Number of bytes in zTerm */ ){ int i; int nSegment = pCsr->nSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); assert( pCsr->pFilter==0 ); assert( zTerm && nTerm>0 ); /* Advance each segment iterator until it points to the term zTerm/nTerm. */ @@ -2313,14 +2384,14 @@ } pCsr->nAdvance = i; /* Advance each of the segments to point to the first docid. */ for(i=0; inAdvance; i++){ - int rc = fts3SegReaderFirstDocid(pCsr->apSegment[i]); + int rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); if( rc!=SQLITE_OK ) return rc; } - fts3SegReaderSort(pCsr->apSegment, i, i, fts3SegReaderDoclistCmp); + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); assert( iCol<0 || iColnColumn ); pCsr->iColFilter = iCol; return SQLITE_OK; @@ -2333,10 +2404,13 @@ char **paPoslist, /* OUT: Pointer to position list */ int *pnPoslist /* OUT: Size of position list in bytes */ ){ int nMerge = pMsr->nAdvance; Fts3SegReader **apSegment = pMsr->apSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); if( nMerge==0 ){ *paPoslist = 0; return SQLITE_OK; } @@ -2354,23 +2428,22 @@ char *pList; int nList; int j; sqlite3_int64 iDocid = apSegment[0]->iDocid; - rc = fts3SegReaderNextDocid(apSegment[0], &pList, &nList); + rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); j = 1; while( rc==SQLITE_OK && jpOffsetList && apSegment[j]->iDocid==iDocid ){ - fts3SegReaderNextDocid(apSegment[j], 0, 0); + rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); j++; } if( rc!=SQLITE_OK ) return rc; - - fts3SegReaderSort(pMsr->apSegment, nMerge, j, fts3SegReaderDoclistCmp); + fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); if( pMsr->iColFilter>=0 ){ fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); } @@ -2431,10 +2504,13 @@ int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); Fts3SegReader **apSegment = pCsr->apSegment; int nSegment = pCsr->nSegment; Fts3SegFilter *pFilter = pCsr->pFilter; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); if( pCsr->nSegment==0 ) return SQLITE_OK; do { int nMerge; @@ -2481,11 +2557,14 @@ ){ nMerge++; } assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); - if( nMerge==1 && !isIgnoreEmpty ){ + if( nMerge==1 + && !isIgnoreEmpty + && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) + ){ pCsr->aDoclist = apSegment[0]->aDoclist; pCsr->nDoclist = apSegment[0]->nDoclist; rc = SQLITE_ROW; }else{ int nDoclist = 0; /* Size of doclist */ @@ -2494,56 +2573,66 @@ /* The current term of the first nMerge entries in the array ** of Fts3SegReader objects is the same. The doclists must be merged ** and a single term returned with the merged doclist. */ for(i=0; ipOffsetList ){ int j; /* Number of segments that share a docid */ char *pList; int nList; int nByte; sqlite3_int64 iDocid = apSegment[0]->iDocid; - fts3SegReaderNextDocid(apSegment[0], &pList, &nList); + fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); j = 1; while( jpOffsetList && apSegment[j]->iDocid==iDocid ){ - fts3SegReaderNextDocid(apSegment[j], 0, 0); + fts3SegReaderNextDocid(p, apSegment[j], 0, 0); j++; } if( isColFilter ){ fts3ColumnFilter(pFilter->iCol, &pList, &nList); } if( !isIgnoreEmpty || nList>0 ){ - nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0); + + /* Calculate the 'docid' delta value to write into the merged + ** doclist. */ + sqlite3_int64 iDelta; + if( p->bDescIdx && nDoclist>0 ){ + iDelta = iPrev - iDocid; + }else{ + iDelta = iDocid - iPrev; + } + assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) ); + assert( nDoclist>0 || iDelta==iDocid ); + + nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); if( nDoclist+nByte>pCsr->nBuffer ){ char *aNew; pCsr->nBuffer = (nDoclist+nByte)*2; aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); if( !aNew ){ return SQLITE_NOMEM; } pCsr->aBuffer = aNew; } - nDoclist += sqlite3Fts3PutVarint( - &pCsr->aBuffer[nDoclist], iDocid-iPrev - ); + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); iPrev = iDocid; if( isRequirePos ){ memcpy(&pCsr->aBuffer[nDoclist], pList, nList); nDoclist += nList; pCsr->aBuffer[nDoclist++] = '\0'; } } - fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp); + fts3SegReaderSort(apSegment, nMerge, j, xCmp); } if( nDoclist>0 ){ pCsr->aDoclist = pCsr->aBuffer; pCsr->nDoclist = nDoclist; rc = SQLITE_ROW; @@ -2881,36 +2970,20 @@ } *pnByte = 0; return 0; } -/* -** Helper fucntion for FreeDeferredDoclists(). This function removes all -** references to deferred doclists from within the tree of Fts3Expr -** structures headed by -*/ -static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ - if( pExpr ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3DeferredDoclistClear(pExpr->pLeft); - fts3DeferredDoclistClear(pExpr->pRight); - } -} - /* ** Delete all cached deferred doclists. Deferred doclists are cached ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. */ void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ Fts3DeferredToken *pDef; for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ - sqlite3_free(pDef->pList); + fts3PendingListDelete(pDef->pList); pDef->pList = 0; } - if( pCsr->pDeferred ){ - fts3DeferredDoclistClear(pCsr->pExpr); - } } /* ** Free all entries in the pCsr->pDeffered list. Entries are added to ** this list using sqlite3Fts3DeferToken(). @@ -2918,11 +2991,11 @@ void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ Fts3DeferredToken *pDef; Fts3DeferredToken *pNext; for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ pNext = pDef->pNext; - sqlite3_free(pDef->pList); + fts3PendingListDelete(pDef->pList); sqlite3_free(pDef); } pCsr->pDeferred = 0; } Index: test/fts3sort.test ================================================================== --- test/fts3sort.test +++ test/fts3sort.test @@ -19,21 +19,20 @@ ifcapable !fts3 { finish_test return } -set testprefix fts3sort -proc build_database {nRow} { +proc build_database {nRow param} { db close forcedelete test.db sqlite3 db test.db set vocab [list aa ab ac ba bb bc ca cb cc da] expr srand(0) - execsql { CREATE VIRTUAL TABLE t1 USING fts4 } + execsql "CREATE VIRTUAL TABLE t1 USING fts4($param)" for {set i 0} {$i < $nRow} {incr i} { set v [expr int(rand()*1000000)] set doc [list] for {set div 1} {$div < 1000000} {set div [expr $div*10]} { lappend doc [lindex $vocab [expr ($v/$div) % 10]] @@ -40,17 +39,28 @@ } execsql { INSERT INTO t1 VALUES($doc) } } } -set nRow 1000 -do_test 1.0 { - build_database $nRow - execsql { SELECT count(*) FROM t1 } -} $nRow +set testprefix fts3sort + +unset -nocomplain CONTROL +foreach {t param} { + 1 "" + 2 "order=asc" + 3 "order=desc" +} { + + set testprefix fts3sort-1.$t -foreach {tn query} { + set nRow 1000 + do_test 1.0 { + build_database $nRow $param + execsql { SELECT count(*) FROM t1 } + } $nRow + + foreach {tn query} { 1 "SELECT docid, * FROM t1" 2 "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa'" 3 "SELECT docid, * FROM t1 WHERE t1 MATCH 'a*'" 4 "SELECT docid, quote(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'a*'" 5 "SELECT docid, quote(matchinfo(t1,'pcnxals')) FROM t1 WHERE t1 MATCH 'b*'" @@ -57,52 +67,95 @@ 6 "SELECT docid, * FROM t1 WHERE t1 MATCH 'a* b* c*'" 7 "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa OR da'" 8 "SELECT docid, * FROM t1 WHERE t1 MATCH 'nosuchtoken'" 9 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR da'" 10 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR nosuchtoken'" -} { - - unset -nocomplain A B C D - set A_list [list] - set B_list [list] - set C_list [list] - set D_list [list] + 11 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa NEAR bb'" + 12 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH '\"aa bb\"'" + 13 "SELECT docid, content FROM t1 WHERE t1 MATCH 'aa NEAR/2 bb NEAR/3 cc'" + 14 "SELECT docid, content FROM t1 WHERE t1 MATCH '\"aa bb cc\"'" + } { + + unset -nocomplain A B C D + set A_list [list] + set B_list [list] + set C_list [list] + set D_list [list] + + unset -nocomplain X + db eval "$query ORDER BY rowid ASC" X { + set A($X(docid)) [array get X] + lappend A_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY rowid DESC" X { + set B($X(docid)) [array get X] + lappend B_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY docid ASC" X { + set C($X(docid)) [array get X] + lappend C_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY docid DESC" X { + set D($X(docid)) [array get X] + lappend D_list $X(docid) + } + + do_test $tn.1 { set A_list } [lsort -integer -increasing $A_list] + do_test $tn.2 { set B_list } [lsort -integer -decreasing $B_list] + do_test $tn.3 { set C_list } [lsort -integer -increasing $C_list] + do_test $tn.4 { set D_list } [lsort -integer -decreasing $D_list] + + unset -nocomplain DATA + unset -nocomplain X + db eval "$query" X { + set DATA($X(docid)) [array get X] + } + + do_test $tn.5 { lsort [array get A] } [lsort [array get DATA]] + do_test $tn.6 { lsort [array get B] } [lsort [array get DATA]] + do_test $tn.7 { lsort [array get C] } [lsort [array get DATA]] + do_test $tn.8 { lsort [array get D] } [lsort [array get DATA]] - unset -nocomplain X - db eval "$query ORDER BY rowid ASC" X { - set A($X(docid)) [array get X] - lappend A_list $X(docid) - } - unset -nocomplain X - db eval "$query ORDER BY rowid DESC" X { - set B($X(docid)) [array get X] - lappend B_list $X(docid) - } - unset -nocomplain X - db eval "$query ORDER BY docid ASC" X { - set C($X(docid)) [array get X] - lappend C_list $X(docid) + if {[info exists CONTROL($tn)]} { + do_test $tn.9 { set CONTROL($tn) } [lsort [array get DATA]] + } else { + set CONTROL($tn) [lsort [array get DATA]] + } } - unset -nocomplain X - db eval "$query ORDER BY docid DESC" X { - set D($X(docid)) [array get X] - lappend D_list $X(docid) - } +} +unset -nocomplain CONTROL + +set testprefix fts3sort - do_test 1.$tn.1 { set A_list } [lsort -integer -increasing $A_list] - do_test 1.$tn.2 { set B_list } [lsort -integer -decreasing $B_list] - do_test 1.$tn.3 { set C_list } [lsort -integer -increasing $C_list] - do_test 1.$tn.4 { set D_list } [lsort -integer -decreasing $D_list] - - unset -nocomplain DATA - unset -nocomplain X - db eval "$query" X { - set DATA($X(docid)) [array get X] - } - - do_test 1.$tn.5 { lsort [array get A] } [lsort [array get DATA]] - do_test 1.$tn.6 { lsort [array get B] } [lsort [array get DATA]] - do_test 1.$tn.7 { lsort [array get C] } [lsort [array get DATA]] - do_test 1.$tn.8 { lsort [array get D] } [lsort [array get DATA]] +#------------------------------------------------------------------------- +# Tests for parsing the "order=asc" and "order=desc" directives. +# +foreach {tn param res} { + 1 "order=asc" {0 {}} + 2 "order=desc" {0 {}} + 3 "order=dec" {1 {unrecognized order: dec}} + 4 "order=xxx, order=asc" {0 {}} +} { + execsql { DROP TABLE IF EXISTS t1 } + do_catchsql_test 2.1.$tn " + CREATE VIRTUAL TABLE t1 USING fts4(a, b, $param) + " $res } + +do_execsql_test 2.2 { + BEGIN; + CREATE VIRTUAL TABLE t2 USING fts4(order=desc); + INSERT INTO t2 VALUES('aa bb'); + INSERT INTO t2 VALUES('bb cc'); + INSERT INTO t2 VALUES('cc aa'); + SELECT docid FROM t2 WHERE t2 MATCH 'aa'; + END; +} {3 1} +do_execsql_test 2.3 { + SELECT docid FROM t2 WHERE t2 MATCH 'aa'; +} {3 1} finish_test +