Index: ext/fts3/fts3.c
==================================================================
--- ext/fts3/fts3.c
+++ ext/fts3/fts3.c
@@ -2174,11 +2174,11 @@
Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */
const char *zStart = "";
const char *zEnd = "";
const char *zEllipsis = "...";
int iCol = -1;
- int nToken = 15;
+ int nToken = 15; /* Default number of tokens in snippet */
/* There must be at least one argument passed to this function (otherwise
** the non-overloaded version would have been called instead of this one).
*/
assert( nVal>=1 );
Index: ext/fts3/fts3_snippet.c
==================================================================
--- ext/fts3/fts3_snippet.c
+++ ext/fts3/fts3_snippet.c
@@ -43,16 +43,14 @@
int (*x)(Fts3Expr *, void *), /* Callback function to invoke for phrases */
void *pCtx /* Second argument to pass to callback */
){
int rc;
int eType = pExpr->eType;
- if( eType==FTSQUERY_NOT ){
- rc = SQLITE_OK;
- }else if( eType!=FTSQUERY_PHRASE ){
+ if( eType!=FTSQUERY_PHRASE ){
assert( pExpr->pLeft && pExpr->pRight );
rc = fts3ExprIterate(pExpr->pLeft, x, pCtx);
- if( rc==SQLITE_OK ){
+ if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){
rc = fts3ExprIterate(pExpr->pRight, x, pCtx);
}
}else{
rc = x(pExpr, pCtx);
}
@@ -106,11 +104,11 @@
if( pExpr->isLoaded==0 ){
rc = sqlite3Fts3ExprLoadDoclist(p->pTab, pExpr);
pExpr->isLoaded = 1;
if( rc==SQLITE_OK ){
- fts3ExprNearTrim(pExpr);
+ rc = fts3ExprNearTrim(pExpr);
}
}
return rc;
}
@@ -457,14 +455,14 @@
while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){
const char *ZDUMMY; int DUMMY1, DUMMY2, DUMMY3;
rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent);
}
pMod->xClose(pC);
- if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){
- return rc;
- }
- nShift = iCurrent-nSnippet;
+ if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ return rc; }
+
+ nShift = (rc==SQLITE_DONE)+iCurrent-nSnippet;
+ assert( nShift<=nDesired );
if( nShift>0 ){
*piPos += nShift;
*pHlmask = hlmask >> nShift;
}
}
@@ -473,10 +471,12 @@
}
static int fts3SnippetText(
Fts3Cursor *pCsr, /* FTS3 Cursor */
SnippetFragment *pFragment, /* Snippet to extract */
+ int iFragment, /* Fragment number */
+ int isLast, /* True for final fragment in snippet */
int nSnippet, /* Number of tokens in extracted snippet */
const char *zOpen, /* String inserted before highlighted term */
const char *zClose, /* String inserted after highlighted term */
const char *zEllipsis,
StrBuffer *pOut
@@ -484,20 +484,19 @@
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
int rc; /* Return code */
const char *zDoc; /* Document text to extract snippet from */
int nDoc; /* Size of zDoc in bytes */
int iCurrent = 0; /* Current token number of document */
- int iStart = 0; /* Byte offset of current token */
int iEnd = 0; /* Byte offset of end of current token */
int isShiftDone = 0;
int iPos = pFragment->iPos;
u64 hlmask = pFragment->hlmask;
sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */
sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */
const char *ZDUMMY; /* Dummy arguments used with tokenizer */
- int DUMMY1, DUMMY2, DUMMY3; /* Dummy arguments used with tokenizer */
+ int DUMMY1; /* Dummy arguments used with tokenizer */
zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, pFragment->iCol+1);
if( zDoc==0 ){
if( sqlite3_column_type(pCsr->pStmt, pFragment->iCol+1)!=SQLITE_NULL ){
return SQLITE_NOMEM;
@@ -504,66 +503,67 @@
}
return SQLITE_OK;
}
nDoc = sqlite3_column_bytes(pCsr->pStmt, pFragment->iCol+1);
- /* Open a token cursor on the document. Read all tokens up to and
- ** including token iPos (the first token of the snippet). Set variable
- ** iStart to the byte offset in zDoc of the start of token iPos.
- */
+ /* Open a token cursor on the document. */
pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule;
rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
if( rc!=SQLITE_OK ){
return rc;
}
pC->pTokenizer = pTab->pTokenizer;
while( rc==SQLITE_OK ){
- int iBegin;
- int iFin;
- rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent);
-
- if( rc==SQLITE_OK ){
- if( iCurrent=(iPos+nSnippet) ){
- rc = SQLITE_DONE;
- }else{
- iEnd = iFin;
- if( hlmask & ((u64)1 << (iCurrent-iPos)) ){
- if( fts3StringAppend(pOut, &zDoc[iStart], iBegin-iStart)
- || fts3StringAppend(pOut, zOpen, -1)
- || fts3StringAppend(pOut, &zDoc[iBegin], iEnd-iBegin)
- || fts3StringAppend(pOut, zClose, -1)
- ){
- rc = SQLITE_NOMEM;
- }
- iStart = iEnd;
- }
- }
- }
- }
- assert( rc!=SQLITE_OK );
- if( rc==SQLITE_DONE ){
- rc = fts3StringAppend(pOut, &zDoc[iStart], iEnd-iStart);
- if( rc==SQLITE_OK ){
- rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent);
- if( rc==SQLITE_DONE ){
- rc = fts3StringAppend(pOut, &zDoc[iEnd], -1);
- }else if( rc==SQLITE_OK && zEllipsis ){
+ int iBegin; /* Offset in zDoc of start of token */
+ int iFin; /* Offset in zDoc of end of token */
+ int isHighlight;
+
+ rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent);
+ if( rc!=SQLITE_OK ){
+ if( rc==SQLITE_DONE ){
+ /* Special case - the last token of the snippet is also the last token
+ ** of the column. Append any punctuation that occurred between the end
+ ** of the previous token and the end of the document to the output.
+ ** Then break out of the loop. */
+ rc = fts3StringAppend(pOut, &zDoc[iEnd], -1);
+ }
+ break;
+ }
+ if( iCurrent0 || iFragment>0) ){
+ rc = fts3StringAppend(pOut, zEllipsis, -1);
+ }
+ if( rc!=SQLITE_OK || iCurrent=(iPos+nSnippet) ){
+ if( isLast ){
rc = fts3StringAppend(pOut, zEllipsis, -1);
}
+ break;
}
+
+ /* Set isHighlight to true if this term should be highlighted. */
+ isHighlight = (hlmask & ((u64)1 << (iCurrent-iPos)))!=0;
+
+ if( iCurrent>iPos ) rc = fts3StringAppend(pOut, &zDoc[iEnd], iBegin-iEnd);
+ if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zOpen, -1);
+ if( rc==SQLITE_OK ) rc = fts3StringAppend(pOut, &zDoc[iBegin], iFin-iBegin);
+ if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zClose, -1);
+
+ iEnd = iFin;
}
pMod->xClose(pC);
return rc;
}
@@ -801,17 +801,13 @@
}while( nSnippet0 );
for(i=0; i0 || p->iPos>0 ){
- fts3StringAppend(&res, zEllipsis, -1);
- }
- rc = fts3SnippetText(pCsr, p, nFToken, zStart, zEnd, zTail, &res);
+ rc = fts3SnippetText(pCsr, &aSnippet[i],
+ i, (i==nSnippet-1), nFToken, zStart, zEnd, zEllipsis, &res
+ );
}
snippet_out:
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
@@ -949,11 +945,11 @@
if( rc==SQLITE_OK ){
char aBuffer[64];
sqlite3_snprintf(sizeof(aBuffer), aBuffer,
"%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart
);
- fts3StringAppend(&res, aBuffer, -1);
+ rc = fts3StringAppend(&res, aBuffer, -1);
}
}
}
if( rc==SQLITE_DONE ){
rc = SQLITE_ERROR;
Index: test/fts3snippet.test
==================================================================
--- test/fts3snippet.test
+++ test/fts3snippet.test
@@ -1,25 +1,39 @@
+# 2010 January 07
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
ifcapable !fts3 { finish_test ; return }
+
+# Transform the list $L to its "normal" form. So that it can be compared to
+# another list with the same set of elements using [string compare].
+#
+proc normalize {L} {
+ set ret [list]
+ foreach l $L {lappend ret $l}
+ return $ret
+}
do_test fts3snippet-1.1 {
execsql {
CREATE VIRTUAL TABLE ft USING fts3;
INSERT INTO ft VALUES('xxx xxx xxx xxx');
}
} {}
-proc normalize {L} {
- set ret [list]
- foreach l $L {lappend ret $l}
- return $ret
-}
-
do_test fts3snippet-1.2 {
execsql { SELECT offsets(ft) FROM ft WHERE ft MATCH 'xxx' }
} {{0 0 0 3 0 0 4 3 0 0 8 3 0 0 12 3}}
do_test fts3snippet-1.3 {
@@ -61,8 +75,34 @@
0 1 8 3
0 2 8 3
0 0 12 3
0 2 12 3
}]]
+
+do_test fts3snippet-2.1 {
+ execsql {
+ DROP TABLE IF EXISTS ft;
+ CREATE VIRTUAL TABLE ft USING fts3;
+ INSERT INTO ft VALUES('one two three four five six seven eight nine ten');
+ }
+} {}
+foreach {tn expr res} {
+ 1 one "[one] two three four five..."
+ 2 two "one [two] three four five..."
+ 3 three "one two [three] four five..."
+ 4 four "...two three [four] five six..."
+ 5 five "...three four [five] six seven..."
+ 6 six "...four five [six] seven eight..."
+ 7 seven "...five six [seven] eight nine..."
+ 8 eight "...six seven [eight] nine ten"
+ 9 nine "...six seven eight [nine] ten"
+ 10 ten "...six seven eight nine [ten]"
+} {
+ do_test fts3snippet-2.2.$tn {
+ execsql {
+ SELECT snippet(ft, '[', ']', '...', 0, 5) FROM ft WHERE ft MATCH $expr
+ }
+ } [list $res]
+}
finish_test