Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -2174,11 +2174,11 @@ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ const char *zStart = ""; const char *zEnd = ""; const char *zEllipsis = "..."; int iCol = -1; - int nToken = 15; + int nToken = 15; /* Default number of tokens in snippet */ /* There must be at least one argument passed to this function (otherwise ** the non-overloaded version would have been called instead of this one). */ assert( nVal>=1 ); Index: ext/fts3/fts3_snippet.c ================================================================== --- ext/fts3/fts3_snippet.c +++ ext/fts3/fts3_snippet.c @@ -43,16 +43,14 @@ int (*x)(Fts3Expr *, void *), /* Callback function to invoke for phrases */ void *pCtx /* Second argument to pass to callback */ ){ int rc; int eType = pExpr->eType; - if( eType==FTSQUERY_NOT ){ - rc = SQLITE_OK; - }else if( eType!=FTSQUERY_PHRASE ){ + if( eType!=FTSQUERY_PHRASE ){ assert( pExpr->pLeft && pExpr->pRight ); rc = fts3ExprIterate(pExpr->pLeft, x, pCtx); - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){ rc = fts3ExprIterate(pExpr->pRight, x, pCtx); } }else{ rc = x(pExpr, pCtx); } @@ -106,11 +104,11 @@ if( pExpr->isLoaded==0 ){ rc = sqlite3Fts3ExprLoadDoclist(p->pTab, pExpr); pExpr->isLoaded = 1; if( rc==SQLITE_OK ){ - fts3ExprNearTrim(pExpr); + rc = fts3ExprNearTrim(pExpr); } } return rc; } @@ -457,14 +455,14 @@ while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){ const char *ZDUMMY; int DUMMY1, DUMMY2, DUMMY3; rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent); } pMod->xClose(pC); - if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ - return rc; - } - nShift = iCurrent-nSnippet; + if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ return rc; } + + nShift = (rc==SQLITE_DONE)+iCurrent-nSnippet; + assert( nShift<=nDesired ); if( nShift>0 ){ *piPos += nShift; *pHlmask = hlmask >> nShift; } } @@ -473,10 +471,12 @@ } static int fts3SnippetText( Fts3Cursor *pCsr, /* FTS3 Cursor */ SnippetFragment *pFragment, /* Snippet to extract */ + int iFragment, /* Fragment number */ + int isLast, /* True for final fragment in snippet */ int nSnippet, /* Number of tokens in extracted snippet */ const char *zOpen, /* String inserted before highlighted term */ const char *zClose, /* String inserted after highlighted term */ const char *zEllipsis, StrBuffer *pOut @@ -484,20 +484,19 @@ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int rc; /* Return code */ const char *zDoc; /* Document text to extract snippet from */ int nDoc; /* Size of zDoc in bytes */ int iCurrent = 0; /* Current token number of document */ - int iStart = 0; /* Byte offset of current token */ int iEnd = 0; /* Byte offset of end of current token */ int isShiftDone = 0; int iPos = pFragment->iPos; u64 hlmask = pFragment->hlmask; sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */ sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */ const char *ZDUMMY; /* Dummy arguments used with tokenizer */ - int DUMMY1, DUMMY2, DUMMY3; /* Dummy arguments used with tokenizer */ + int DUMMY1; /* Dummy arguments used with tokenizer */ zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, pFragment->iCol+1); if( zDoc==0 ){ if( sqlite3_column_type(pCsr->pStmt, pFragment->iCol+1)!=SQLITE_NULL ){ return SQLITE_NOMEM; @@ -504,66 +503,67 @@ } return SQLITE_OK; } nDoc = sqlite3_column_bytes(pCsr->pStmt, pFragment->iCol+1); - /* Open a token cursor on the document. Read all tokens up to and - ** including token iPos (the first token of the snippet). Set variable - ** iStart to the byte offset in zDoc of the start of token iPos. - */ + /* Open a token cursor on the document. */ pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC); if( rc!=SQLITE_OK ){ return rc; } pC->pTokenizer = pTab->pTokenizer; while( rc==SQLITE_OK ){ - int iBegin; - int iFin; + int iBegin; /* Offset in zDoc of start of token */ + int iFin; /* Offset in zDoc of end of token */ + int isHighlight; + rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent); - - if( rc==SQLITE_OK ){ - if( iCurrent=(iPos+nSnippet) ){ - rc = SQLITE_DONE; - }else{ - iEnd = iFin; - if( hlmask & ((u64)1 << (iCurrent-iPos)) ){ - if( fts3StringAppend(pOut, &zDoc[iStart], iBegin-iStart) - || fts3StringAppend(pOut, zOpen, -1) - || fts3StringAppend(pOut, &zDoc[iBegin], iEnd-iBegin) - || fts3StringAppend(pOut, zClose, -1) - ){ - rc = SQLITE_NOMEM; - } - iStart = iEnd; - } + if( !isShiftDone ){ + int n = nDoc - iBegin; + rc = fts3SnippetShift(pTab, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask); + isShiftDone = 1; + + /* Now that the shift has been done, check if the initial "..." are + ** required. They are required if (a) this is not the first fragment, + ** or (b) this fragment does not begin at position 0 of its column. + */ + if( rc==SQLITE_OK && (iPos>0 || iFragment>0) ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); } + if( rc!=SQLITE_OK || iCurrentxNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent); - if( rc==SQLITE_DONE ){ - rc = fts3StringAppend(pOut, &zDoc[iEnd], -1); - }else if( rc==SQLITE_OK && zEllipsis ){ + + if( iCurrent>=(iPos+nSnippet) ){ + if( isLast ){ rc = fts3StringAppend(pOut, zEllipsis, -1); } + break; } + + /* Set isHighlight to true if this term should be highlighted. */ + isHighlight = (hlmask & ((u64)1 << (iCurrent-iPos)))!=0; + + if( iCurrent>iPos ) rc = fts3StringAppend(pOut, &zDoc[iEnd], iBegin-iEnd); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zOpen, -1); + if( rc==SQLITE_OK ) rc = fts3StringAppend(pOut, &zDoc[iBegin], iFin-iBegin); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zClose, -1); + + iEnd = iFin; } pMod->xClose(pC); return rc; } @@ -801,17 +801,13 @@ }while( nSnippet0 ); for(i=0; i0 || p->iPos>0 ){ - fts3StringAppend(&res, zEllipsis, -1); - } - rc = fts3SnippetText(pCsr, p, nFToken, zStart, zEnd, zTail, &res); + rc = fts3SnippetText(pCsr, &aSnippet[i], + i, (i==nSnippet-1), nFToken, zStart, zEnd, zEllipsis, &res + ); } snippet_out: if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pCtx, rc); @@ -949,11 +945,11 @@ if( rc==SQLITE_OK ){ char aBuffer[64]; sqlite3_snprintf(sizeof(aBuffer), aBuffer, "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart ); - fts3StringAppend(&res, aBuffer, -1); + rc = fts3StringAppend(&res, aBuffer, -1); } } } if( rc==SQLITE_DONE ){ rc = SQLITE_ERROR; Index: test/fts3snippet.test ================================================================== --- test/fts3snippet.test +++ test/fts3snippet.test @@ -1,25 +1,39 @@ +# 2010 January 07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# set testdir [file dirname $argv0] source $testdir/tester.tcl -# If SQLITE_ENABLE_FTS3 is defined, omit this file. +# If SQLITE_ENABLE_FTS3 is not defined, omit this file. ifcapable !fts3 { finish_test ; return } + +# Transform the list $L to its "normal" form. So that it can be compared to +# another list with the same set of elements using [string compare]. +# +proc normalize {L} { + set ret [list] + foreach l $L {lappend ret $l} + return $ret +} do_test fts3snippet-1.1 { execsql { CREATE VIRTUAL TABLE ft USING fts3; INSERT INTO ft VALUES('xxx xxx xxx xxx'); } } {} -proc normalize {L} { - set ret [list] - foreach l $L {lappend ret $l} - return $ret -} - do_test fts3snippet-1.2 { execsql { SELECT offsets(ft) FROM ft WHERE ft MATCH 'xxx' } } {{0 0 0 3 0 0 4 3 0 0 8 3 0 0 12 3}} do_test fts3snippet-1.3 { @@ -61,8 +75,34 @@ 0 1 8 3 0 2 8 3 0 0 12 3 0 2 12 3 }]] + +do_test fts3snippet-2.1 { + execsql { + DROP TABLE IF EXISTS ft; + CREATE VIRTUAL TABLE ft USING fts3; + INSERT INTO ft VALUES('one two three four five six seven eight nine ten'); + } +} {} +foreach {tn expr res} { + 1 one "[one] two three four five..." + 2 two "one [two] three four five..." + 3 three "one two [three] four five..." + 4 four "...two three [four] five six..." + 5 five "...three four [five] six seven..." + 6 six "...four five [six] seven eight..." + 7 seven "...five six [seven] eight nine..." + 8 eight "...six seven [eight] nine ten" + 9 nine "...six seven eight [nine] ten" + 10 ten "...six seven eight nine [ten]" +} { + do_test fts3snippet-2.2.$tn { + execsql { + SELECT snippet(ft, '[', ']', '...', 0, 5) FROM ft WHERE ft MATCH $expr + } + } [list $res] +} finish_test