Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Handle the case where a tokenizer determines that there are zero tokens in an fts5 query term. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts5 |
| Files: | files | file ages | folders |
| SHA1: |
75f3d17f864072dfa2caee182b86cc4b |
| User & Date: | dan 2015-01-19 11:15:36 |
Context
|
2015-01-20
| ||
| 20:34 | Add extra fault injection tests to fts5. check-in: f45a0dc0 user: dan tags: fts5 | |
|
2015-01-19
| ||
| 11:15 | Handle the case where a tokenizer determines that there are zero tokens in an fts5 query term. check-in: 75f3d17f user: dan tags: fts5 | |
|
2015-01-17
| ||
| 20:01 | Ensure an up to date copy of the fts5 configuration has been loaded into memory before attempting to modify the same configuration. check-in: f30afd20 user: dan tags: fts5 | |
Changes
Changes to ext/fts5/fts5_expr.c.
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 .... 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 .... 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 .... 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 .... 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 .... 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 .... 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 |
do {
t = fts5ExprGetToken(&sParse, &z, &token);
sqlite3Fts5Parser(pEngine, t, token, &sParse);
}while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
sqlite3Fts5ParserFree(pEngine, fts5ParseFree);
assert( sParse.pExpr==0 || (sParse.rc==SQLITE_OK && sParse.zErr==0) );
if( sParse.rc==SQLITE_OK ){
*ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
if( pNew==0 ){
sParse.rc = SQLITE_NOMEM;
sqlite3Fts5ParseNodeFree(sParse.pExpr);
}else{
pNew->pRoot = sParse.pExpr;
................................................................................
** is passed a non-zero value, iteration is in ascending rowid order. Or,
** if it is zero, in descending order.
**
** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
** is not considered an error if the query does not match any documents.
*/
int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, int bAsc){
int rc;
p->pIndex = pIdx;
p->bAsc = bAsc;
rc = fts5ExprNodeFirst(p, p->pRoot);
return rc;
}
/*
** Move to the next document
**
** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
................................................................................
int sqlite3Fts5ExprNext(Fts5Expr *p){
int rc;
rc = fts5ExprNodeNext(p, p->pRoot, 0, 0);
return rc;
}
int sqlite3Fts5ExprEof(Fts5Expr *p){
return p->pRoot->bEof;
}
i64 sqlite3Fts5ExprRowid(Fts5Expr *p){
return p->pRoot->iRowid;
}
/*
................................................................................
Fts5ExprNearset *pNear, /* Existing nearset, or NULL */
Fts5ExprPhrase *pPhrase /* Recently parsed phrase */
){
const int SZALLOC = 8;
Fts5ExprNearset *pRet = 0;
if( pParse->rc==SQLITE_OK ){
if( pNear==0 ){
int nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*);
pRet = sqlite3_malloc(nByte);
if( pRet==0 ){
pParse->rc = SQLITE_NOMEM;
}else{
memset(pRet, 0, nByte);
................................................................................
/*
** This function is called by the parser to process a string token. The
** string may or may not be quoted. In any case it is tokenized and a
** phrase object consisting of all tokens returned.
*/
Fts5ExprPhrase *sqlite3Fts5ParseTerm(
Fts5Parse *pParse, /* Parse context */
Fts5ExprPhrase *pPhrase, /* Phrase to append to */
Fts5Token *pToken, /* String to tokenize */
int bPrefix /* True if there is a trailing "*" */
){
Fts5Config *pConfig = pParse->pConfig;
TokenCtx sCtx; /* Context object passed to callback */
int rc; /* Tokenize return code */
char *z = 0;
memset(&sCtx, 0, sizeof(TokenCtx));
sCtx.pPhrase = pPhrase;
if( pPhrase==0 ){
if( (pParse->nPhrase % 8)==0 ){
int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8);
Fts5ExprPhrase **apNew;
apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte);
if( apNew==0 ){
pParse->rc = SQLITE_NOMEM;
fts5ExprPhraseFree(pPhrase);
return 0;
}
pParse->apPhrase = apNew;
}
pParse->nPhrase++;
}
rc = fts5ParseStringFromToken(pToken, &z);
if( rc==SQLITE_OK ){
sqlite3Fts5Dequote(z);
rc = sqlite3Fts5Tokenize(pConfig, z, strlen(z), &sCtx, fts5ParseTokenize);
}
if( rc ){
pParse->rc = rc;
fts5ExprPhraseFree(sCtx.pPhrase);
sCtx.pPhrase = 0;
}else if( sCtx.pPhrase->nTerm>0 ){
sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
}
pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
sqlite3_free(z);
return sCtx.pPhrase;
}
/*
** Token pTok has appeared in a MATCH expression where the NEAR operator
** is expected. If token pTok does not contain "NEAR", store an error
** in the pParse object.
................................................................................
Fts5ExprNode *pLeft, /* Left hand child expression */
Fts5ExprNode *pRight, /* Right hand child expression */
Fts5ExprNearset *pNear /* For STRING expressions, the near cluster */
){
Fts5ExprNode *pRet = 0;
if( pParse->rc==SQLITE_OK ){
assert( (eType!=FTS5_STRING && pLeft && pRight && !pNear)
|| (eType==FTS5_STRING && !pLeft && !pRight && pNear)
);
pRet = (Fts5ExprNode*)sqlite3_malloc(sizeof(Fts5ExprNode));
if( pRet==0 ){
pParse->rc = SQLITE_NOMEM;
}else{
memset(pRet, 0, sizeof(*pRet));
pRet->eType = eType;
pRet->pLeft = pLeft;
................................................................................
rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr);
}
if( rc==SQLITE_OK ){
char *zText;
if( bTcl ){
zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot);
}else{
zText = fts5ExprPrint(pConfig, pExpr->pRoot);
}
if( rc==SQLITE_OK ){
sqlite3_result_text(pCtx, zText, -1, SQLITE_TRANSIENT);
sqlite3_free(zText);
|
| | > | | | > | > > > | | < < < < < < < < < < < < < < < > | > > > > > > > > > > > > > > > > > > < < < | | > > > > > | |
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 .... 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 .... 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 .... 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 .... 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 .... 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 .... 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 |
do {
t = fts5ExprGetToken(&sParse, &z, &token);
sqlite3Fts5Parser(pEngine, t, token, &sParse);
}while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
sqlite3Fts5ParserFree(pEngine, fts5ParseFree);
assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 );
if( sParse.rc==SQLITE_OK ){
*ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
if( pNew==0 ){
sParse.rc = SQLITE_NOMEM;
sqlite3Fts5ParseNodeFree(sParse.pExpr);
}else{
pNew->pRoot = sParse.pExpr;
................................................................................
** is passed a non-zero value, iteration is in ascending rowid order. Or,
** if it is zero, in descending order.
**
** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
** is not considered an error if the query does not match any documents.
*/
int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, int bAsc){
int rc = SQLITE_OK;
if( p->pRoot ){
p->pIndex = pIdx;
p->bAsc = bAsc;
rc = fts5ExprNodeFirst(p, p->pRoot);
}
return rc;
}
/*
** Move to the next document
**
** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
................................................................................
int sqlite3Fts5ExprNext(Fts5Expr *p){
int rc;
rc = fts5ExprNodeNext(p, p->pRoot, 0, 0);
return rc;
}
int sqlite3Fts5ExprEof(Fts5Expr *p){
return (p->pRoot==0 || p->pRoot->bEof);
}
i64 sqlite3Fts5ExprRowid(Fts5Expr *p){
return p->pRoot->iRowid;
}
/*
................................................................................
Fts5ExprNearset *pNear, /* Existing nearset, or NULL */
Fts5ExprPhrase *pPhrase /* Recently parsed phrase */
){
const int SZALLOC = 8;
Fts5ExprNearset *pRet = 0;
if( pParse->rc==SQLITE_OK ){
if( pPhrase==0 ){
return pNear;
}
if( pNear==0 ){
int nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*);
pRet = sqlite3_malloc(nByte);
if( pRet==0 ){
pParse->rc = SQLITE_NOMEM;
}else{
memset(pRet, 0, nByte);
................................................................................
/*
** This function is called by the parser to process a string token. The
** string may or may not be quoted. In any case it is tokenized and a
** phrase object consisting of all tokens returned.
*/
Fts5ExprPhrase *sqlite3Fts5ParseTerm(
Fts5Parse *pParse, /* Parse context */
Fts5ExprPhrase *pAppend, /* Phrase to append to */
Fts5Token *pToken, /* String to tokenize */
int bPrefix /* True if there is a trailing "*" */
){
Fts5Config *pConfig = pParse->pConfig;
TokenCtx sCtx; /* Context object passed to callback */
int rc; /* Tokenize return code */
char *z = 0;
memset(&sCtx, 0, sizeof(TokenCtx));
sCtx.pPhrase = pAppend;
rc = fts5ParseStringFromToken(pToken, &z);
if( rc==SQLITE_OK ){
sqlite3Fts5Dequote(z);
rc = sqlite3Fts5Tokenize(pConfig, z, strlen(z), &sCtx, fts5ParseTokenize);
}
sqlite3_free(z);
if( rc ){
pParse->rc = rc;
fts5ExprPhraseFree(sCtx.pPhrase);
sCtx.pPhrase = 0;
}else if( sCtx.pPhrase ){
if( pAppend==0 ){
if( (pParse->nPhrase % 8)==0 ){
int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8);
Fts5ExprPhrase **apNew;
apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte);
if( apNew==0 ){
pParse->rc = SQLITE_NOMEM;
fts5ExprPhraseFree(sCtx.pPhrase);
return 0;
}
pParse->apPhrase = apNew;
}
pParse->nPhrase++;
}
pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
assert( sCtx.pPhrase->nTerm>0 );
sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
}
return sCtx.pPhrase;
}
/*
** Token pTok has appeared in a MATCH expression where the NEAR operator
** is expected. If token pTok does not contain "NEAR", store an error
** in the pParse object.
................................................................................
Fts5ExprNode *pLeft, /* Left hand child expression */
Fts5ExprNode *pRight, /* Right hand child expression */
Fts5ExprNearset *pNear /* For STRING expressions, the near cluster */
){
Fts5ExprNode *pRet = 0;
if( pParse->rc==SQLITE_OK ){
assert( (eType!=FTS5_STRING && !pNear)
|| (eType==FTS5_STRING && !pLeft && !pRight)
);
if( eType==FTS5_STRING && pNear==0 ) return 0;
if( eType!=FTS5_STRING && pLeft==0 ) return pRight;
if( eType!=FTS5_STRING && pRight==0 ) return pLeft;
pRet = (Fts5ExprNode*)sqlite3_malloc(sizeof(Fts5ExprNode));
if( pRet==0 ){
pParse->rc = SQLITE_NOMEM;
}else{
memset(pRet, 0, sizeof(*pRet));
pRet->eType = eType;
pRet->pLeft = pLeft;
................................................................................
rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr);
}
if( rc==SQLITE_OK ){
char *zText;
if( pExpr->pRoot==0 ){
zText = sqlite3_mprintf("");
}else if( bTcl ){
zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot);
}else{
zText = fts5ExprPrint(pConfig, pExpr->pRoot);
}
if( rc==SQLITE_OK ){
sqlite3_result_text(pCtx, zText, -1, SQLITE_TRANSIENT);
sqlite3_free(zText);
|
Changes to ext/fts5/test/fts5aa.test.
16 17 18 19 20 21 22 23 24 25 26 27 28 29 ... 295 296 297 298 299 300 301 302 303 304 305 306 307 308 ... 315 316 317 318 319 320 321 322 323 324 325 |
set testprefix fts5aa
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
SELECT name, sql FROM sqlite_master;
} {
t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)}
t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)}
................................................................................
SELECT t2 FROM t2 WHERE t2 MATCH '*stuff'
} {1 {unknown special query: stuff}}
do_test 12.3 {
set res [db one { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }]
string is integer $res
} {1}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 13.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o');
................................................................................
do_execsql_test 13.4 {
DELETE FROM t1 WHERE rowid=2;
} {}
do_execsql_test 13.5 {
SELECT rowid FROM t1 WHERE t1 MATCH 'o';
} {1}
finish_test
|
> > > > > > > > |
16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 ... 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 ... 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 |
set testprefix fts5aa
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
if 0 {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
SELECT name, sql FROM sqlite_master;
} {
t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)}
t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)}
................................................................................
SELECT t2 FROM t2 WHERE t2 MATCH '*stuff'
} {1 {unknown special query: stuff}}
do_test 12.3 {
set res [db one { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }]
string is integer $res
} {1}
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 13.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o');
................................................................................
do_execsql_test 13.4 {
DELETE FROM t1 WHERE rowid=2;
} {}
do_execsql_test 13.5 {
SELECT rowid FROM t1 WHERE t1 MATCH 'o';
} {1}
do_execsql_test 13.6 {
SELECT rowid FROM t1 WHERE t1 MATCH '.';
} {}
finish_test
|
Added ext/fts5/test/fts5eb.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 |
# 2014 June 17 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5eb # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } proc do_syntax_error_test {tn expr err} { set ::se_expr $expr do_catchsql_test $tn {SELECT fts5_expr($se_expr)} [list 1 $err] } proc do_syntax_test {tn expr res} { set ::se_expr $expr do_execsql_test $tn {SELECT fts5_expr($se_expr)} [list $res] } foreach {tn expr res} { 1 {abc} {"abc"} 2 {abc .} {"abc"} 3 {.} {} 4 {abc OR .} {"abc"} 5 {abc NOT .} {"abc"} 6 {abc AND .} {"abc"} 7 {. OR abc} {"abc"} 8 {. NOT abc} {"abc"} 9 {. AND abc} {"abc"} 10 {abc + . + def} {"abc" + "def"} 11 {abc . def} {"abc" AND "def"} 12 {r+e OR w} {"r" + "e" OR "w"} } { do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res] } finish_test |