Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -2345,10 +2345,71 @@ } *pnRight = p - aOut; } +/* +** When this function is called, pList points to a doclist containing position +** data, length *pnList bytes. This removes all entries from the doclist that +** do not correspond to the first token in a column and overwrites pList +** with the result. *pnList is set to the length of the new doclist before +** returning. +** +** If bDescDoclist is true, then both the input and output are in descending +** order. Otherwise, ascending. +*/ +static void fts3DoclistFirstFilter( + int bDescDoclist, /* True if pList is a descending doclist */ + char *pList, /* Buffer containing doclist */ + int *pnList /* IN/OUT: Size of doclist */ +){ + char *p = pList; + char *pOut = pList; + char *pEnd = &pList[*pnList]; + + sqlite3_int64 iDoc; + sqlite3_int64 iPrev; + int bFirstOut = 0; + + fts3GetDeltaVarint3(&p, pEnd, 0, &iDoc); + while( p ){ + int bWritten = 0; + if( *p!=0x01 ){ + if( *p==0x02 ){ + fts3PutDeltaVarint3(&pOut, bDescDoclist, &iPrev, &bFirstOut, iDoc); + *pOut++ = 0x02; + bWritten = 1; + } + fts3ColumnlistCopy(0, &p); + } + + while( *p==0x01 ){ + sqlite3_int64 iCol; + p++; + p += sqlite3Fts3GetVarint(p, &iCol); + if( *p==0x02 ){ + if( bWritten==0 ){ + fts3PutDeltaVarint3(&pOut, bDescDoclist, &iPrev, &bFirstOut, iDoc); + bWritten = 1; + } + pOut += sqlite3Fts3PutVarint(pOut, iCol); + *pOut++ = 0x02; + } + fts3ColumnlistCopy(0, &p); + } + if( bWritten ){ + *pOut++ = 0x00; + } + + assert( *p==0x00 ); + p++; + fts3GetDeltaVarint3(&p, pEnd, bDescDoclist, &iDoc); + } + + *pnList = (pOut - pList); +} + /* ** Merge all doclists in the TermSelect.aaOutput[] array into a single ** doclist stored in TermSelect.aaOutput[0]. If successful, delete all ** other doclists (except the aaOutput[0] one) and return SQLITE_OK. @@ -3515,10 +3576,14 @@ int iToken, /* Token pList/nList corresponds to */ char *pList, /* Pointer to doclist */ int nList /* Number of bytes in pList */ ){ assert( iToken!=p->iDoclistToken ); + + if( p->aToken[iToken].bFirst ){ + fts3DoclistFirstFilter(pTab->bDescIdx, pList, &nList); + } if( pList==0 ){ sqlite3_free(p->doclist.aAll); p->doclist.aAll = 0; p->doclist.nAll = 0; @@ -3719,10 +3784,11 @@ if( pCsr->bDesc==pTab->bDescIdx && bOptOk==1 && p->nToken==1 && pFirst->pSegcsr && pFirst->pSegcsr->bLookup + && pFirst->bFirst==0 ){ /* Use the incremental approach. */ int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); rc = sqlite3Fts3MsrIncrStart( pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n); Index: ext/fts3/fts3Int.h ================================================================== --- ext/fts3/fts3Int.h +++ ext/fts3/fts3Int.h @@ -308,10 +308,11 @@ */ struct Fts3PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer z */ int isPrefix; /* True if token ends with a "*" character */ + int bFirst; /* True if token must appear at position 0 */ /* Variables above this point are populated when the expression is ** parsed (by code in fts3_expr.c). Below this point the variables are ** used when evaluating the expression. */ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ Index: ext/fts3/fts3_expr.c ================================================================== --- ext/fts3/fts3_expr.c +++ ext/fts3/fts3_expr.c @@ -178,13 +178,25 @@ if( iEndpPhrase->aToken[0].isPrefix = 1; iEnd++; } - if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){ - pParse->isNot = 1; + + while( 1 ){ + if( !sqlite3_fts3_enable_parentheses + && iStart>0 && z[iStart-1]=='-' + ){ + pParse->isNot = 1; + iStart--; + }else if( iStart>0 && z[iStart-1]=='^' ){ + pRet->pPhrase->aToken[0].bFirst = 1; + iStart--; + }else{ + break; + } } + } nConsumed = iEnd; } pModule->xClose(pCursor); @@ -279,10 +291,11 @@ memcpy(&zTemp[nTemp], zByte, nByte); nTemp += nByte; pToken->n = nByte; pToken->isPrefix = (iEndbFirst = (iBegin>0 && zInput[iBegin-1]=='^'); nToken = ii+1; } } pModule->xClose(pCursor); Index: ext/fts3/fts3_write.c ================================================================== --- ext/fts3/fts3_write.c +++ ext/fts3/fts3_write.c @@ -3115,10 +3115,11 @@ pTC->pTokenizer = pT; rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ Fts3PhraseToken *pPT = pDef->pToken; if( (pDef->iCol>=p->nColumn || pDef->iCol==i) + && (pPT->bFirst==0 || iPos==0) && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) ){ fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); } Index: test/fts3defer.test ================================================================== --- test/fts3defer.test +++ test/fts3defer.test @@ -424,10 +424,17 @@ if {$fts3_simple_deferred_tokens_only==0} { do_select_test 6.2.3 { SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"' } {8 15 26 92 96} } + + do_select_test 7.1 { + SELECT rowid FROM t1 WHERE t1 MATCH '^zm mjpavjuhw' + } {56 62} + do_select_test 7.2 { + SELECT rowid FROM t1 WHERE t1 MATCH '^azavwm zm' + } {43} } set testprefix fts3defer do_execsql_test 3.1 { ADDED test/fts3first.test Index: test/fts3first.test ================================================================== --- /dev/null +++ test/fts3first.test @@ -0,0 +1,79 @@ +# 2011 October 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl + +ifcapable !fts3 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x1 USING FTS4(a, b, c); + INSERT INTO x1(docid,a,b,c) VALUES(0, 'K H D S T', 'V M N Y K', 'S Z N Q S'); + INSERT INTO x1(docid,a,b,c) VALUES(1, 'K N J L W', 'S Z W J Q', 'D U W S E'); + INSERT INTO x1(docid,a,b,c) VALUES(2, 'B P M O I', 'R P H W S', 'R J L L E'); + INSERT INTO x1(docid,a,b,c) VALUES(3, 'U R Q M L', 'M J K A V', 'Q W J T J'); + INSERT INTO x1(docid,a,b,c) VALUES(4, 'N J C Y N', 'R U D X V', 'B O U A Q'); + INSERT INTO x1(docid,a,b,c) VALUES(5, 'Q L X L U', 'I F N X S', 'U Q A N Y'); + INSERT INTO x1(docid,a,b,c) VALUES(6, 'M R G U T', 'U V I Q P', 'X Y D L S'); + INSERT INTO x1(docid,a,b,c) VALUES(7, 'D Y P O I', 'X J P K R', 'V O T H V'); + INSERT INTO x1(docid,a,b,c) VALUES(8, 'R Y D L R', 'U U E S J', 'N W L M R'); + INSERT INTO x1(docid,a,b,c) VALUES(9, 'Z P F N P', 'W A X D U', 'V A E Q A'); + INSERT INTO x1(docid,a,b,c) VALUES(10, 'Q I A Q M', 'N D K H C', 'A H T Q Z'); + INSERT INTO x1(docid,a,b,c) VALUES(11, 'T E R Q B', 'C I B C B', 'F Z U W R'); + INSERT INTO x1(docid,a,b,c) VALUES(12, 'E S V U W', 'T P F W H', 'A M D J Q'); + INSERT INTO x1(docid,a,b,c) VALUES(13, 'X S B T Y', 'U D N D P', 'X Z Y G F'); + INSERT INTO x1(docid,a,b,c) VALUES(14, 'K H A B L', 'S R C C Z', 'D W E H J'); + INSERT INTO x1(docid,a,b,c) VALUES(15, 'C E U C C', 'W F M N M', 'T Z U X T'); + INSERT INTO x1(docid,a,b,c) VALUES(16, 'Q G C G H', 'H N N B H', 'B Q I H Y'); + INSERT INTO x1(docid,a,b,c) VALUES(17, 'Q T S K B', 'W B D Y N', 'V J P E C'); + INSERT INTO x1(docid,a,b,c) VALUES(18, 'A J M O Q', 'L G Y Y A', 'G N M R N'); + INSERT INTO x1(docid,a,b,c) VALUES(19, 'T R Y P Y', 'N V Y B X', 'L Z T N T'); + + CREATE VIRTUAL TABLE x2 USING FTS4(a, b, c, order=DESC); + INSERT INTO x2(docid, a, b, c) SELECT docid, a, b, c FROM x1; +} + +foreach x {1 2} { + foreach {tn match res} { + 1 "^K" {0 1 14} + 2 "^S" {0 1 14} + 3 "^W" {9 15 17} + 4 "^J" {} + 5 "^E" {12} + 6 "V ^-E" {0 3 4 6 7 9 17 19} + 7 "V -^E" {0 3 4 6 7 9 17 19} + 8 "^-E V" {0 3 4 6 7 9 17 19} + 9 "-^E V" {0 3 4 6 7 9 17 19} + 10 "V" {0 3 4 6 7 9 12 17 19} + + 11 {"^K H"} {0 14} + 12 {"K H"} {0 10 14} + 13 {"K ^H"} {} + } { + set rev [list] + for {set ii [expr [llength $res]-1]} {$ii>=0} {incr ii -1} { + lappend rev [lindex $res $ii] + } + do_execsql_test 1.$x.$tn.1 {SELECT docid FROM x1 WHERE x1 MATCH $match} $res + do_execsql_test 1.$x.$tn.2 {SELECT docid FROM x2 WHERE x2 MATCH $match} $rev + } + + do_execsql_test 1.$x.[expr $tn+1] { + INSERT INTO x1(x1) VALUES('optimize'); + INSERT INTO x2(x2) VALUES('optimize'); + } {} +} + +finish_test Index: test/permutations.test ================================================================== --- test/permutations.test +++ test/permutations.test @@ -183,10 +183,11 @@ fts3fault.test fts3malloc.test fts3matchinfo.test fts3aux1.test fts3comp1.test fts3auto.test fts4aa.test fts4content.test fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test fts3corrupt2.test + fts3first.test } lappend ::testsuitelist xxx #-------------------------------------------------------------------------