Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Fix some issues related to ORDER BY and fts tables with a non-zero languageid_bits setting. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts-languageid-bits |
| Files: | files | file ages | folders |
| SHA1: |
81527768ef9b39289cfda69d415069b7 |
| User & Date: | dan 2013-06-20 18:32:34 |
Context
|
2013-06-20
| ||
| 19:55 | Add tests for modifying the docid and languageid fields of an fts table with a non-zero languageid_bits field. Closed-Leaf check-in: 949425d4 user: dan tags: fts-languageid-bits | |
| 18:32 | Fix some issues related to ORDER BY and fts tables with a non-zero languageid_bits setting. check-in: 81527768 user: dan tags: fts-languageid-bits | |
| 16:22 | Add extra tests for fts with a non-zero languageid_bits setting. Fix querying by docid with the same. check-in: b1df00f3 user: dan tags: fts-languageid-bits | |
Changes
Changes to ext/fts3/fts3.c.
1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 |
if( iLangidCons>=0 ){
pInfo->aConstraintUsage[iLangidCons].argvIndex = 2;
}
/* Regardless of the strategy selected, FTS can deliver rows in rowid (or
** docid) order. Both ascending and descending are possible.
*/
if( pInfo->nOrderBy==1 ){
struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0];
if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){
if( pOrder->desc ){
pInfo->idxStr = "DESC";
}else{
pInfo->idxStr = "ASC";
}
pInfo->orderByConsumed = 1;
}
|
> > | > > |
1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 |
if( iLangidCons>=0 ){
pInfo->aConstraintUsage[iLangidCons].argvIndex = 2;
}
/* Regardless of the strategy selected, FTS can deliver rows in rowid (or
** docid) order. Both ascending and descending are possible.
*/
assert( pInfo->orderByConsumed==0 );
if( pInfo->nOrderBy==1 ){
struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0];
if( pOrder->iColumn<0 || (
(pOrder->iColumn==p->nColumn+1)
&& (pInfo->idxNum>=FTS3_FULLTEXT_SEARCH || p->nLanguageidBits==0)
)){
if( pOrder->desc ){
pInfo->idxStr = "DESC";
}else{
pInfo->idxStr = "ASC";
}
pInfo->orderByConsumed = 1;
}
|
Changes to ext/fts3/fts3_write.c.
5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 |
return rc;
}
/*
** Convert a docid (iDocid) and a language id (iLangid) to a rowid,
** according to the configured languageid_bits= value belonging to
** FTS table *p.
*/
i64 sqlite3Fts3DocidToRowid(Fts3Table *p, i64 iDocid, int iLangid){
i64 iRowid = iDocid;
if( p->nLanguageidBits ){
iRowid = (iRowid << p->nLanguageidBits) + iLangid;
}
return iRowid;
}
i64 sqlite3Fts3RowidToDocid(Fts3Table *p, i64 iRowid){
return (iRowid >> p->nLanguageidBits);
}
/*
** This function does the work for the xUpdate method of FTS3 virtual
** tables. The schema of the virtual table being:
**
** CREATE TABLE <table name>(
|
> > > > > > > > > > > > | > | > | > > > > > | > > > > > | > > > > > > > > > > |
5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 |
return rc; } /* ** Convert a docid (iDocid) and a language id (iLangid) to a rowid, ** according to the configured languageid_bits= value belonging to ** FTS table *p. ** ** The conversion is as follows: ** ** * The sign bit of iDocid becomes the sign bit of the rowid. ** ** * iLangid is converted to an unsigned integer and stored in ** the next most significant Fts3Table.nLanguageidBits bits ** of the returned rowid. ** ** * The least signficant (63-nLanguageidBits) of iDocid are ** copied to the (63-nLanguageidBits) least signifcant bits of ** the returned rowid. */ i64 sqlite3Fts3DocidToRowid(Fts3Table *p, i64 iDocid, int iLangid){ u64 iRet = iDocid; if( p->nLanguageidBits ){ int iShift = (63 - p->nLanguageidBits); u64 mask = ((((u64)1 << p->nLanguageidBits) - 1) << iShift); iRet &= ~mask; iRet |= (u64)iLangid << iShift; } assert( sqlite3Fts3RowidToDocid(p, (i64)iRet)==iDocid ); return (i64)iRet; } /* ** Convert a rowid (iRowid) to a docid according to the languageid_bits= ** value belonging to FTS table *p. */ i64 sqlite3Fts3RowidToDocid(Fts3Table *p, i64 iRowid){ u64 iRet = iRowid; if( p->nLanguageidBits ){ static const u64 signbit = ((u64)1 << 63); u64 mask = ((((u64)1 << p->nLanguageidBits)-1) << (63-p->nLanguageidBits)); if( iRet & signbit ){ iRet |= mask; }else{ iRet &= ~mask; } } return (i64)iRet; } /* ** This function does the work for the xUpdate method of FTS3 virtual ** tables. The schema of the virtual table being: ** ** CREATE TABLE <table name>( |
Changes to test/fts4langid2.test.
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
...
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
|
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
finish_test
return
}
#-------------------------------------------------------------------------
# Test out-of-range values for the languageid_bits= parameter.
#
do_catchsql_test 1.1 {
CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=31);
} {1 {languageid_bits parameter out of range}}
................................................................................
SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=1;
} {}
do_execsql_test 6.3.4 {
SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=2;
} {4 {1 1 1 1 1}}
do_execsql_test 6.4 {
CREATE VIRTUAL TABLE t2 USING fts4(languageid_bits=1, languageid=lid);
INSERT INTO t1(docid,lid,content) VALUES(-1, 0, 'A B C D');
INSERT INTO t1(docid,lid,content) VALUES(-2, 0, 'D C B A');
INSERT INTO t1(docid,lid,content) VALUES(-3, 0, 'C B D A');
INSERT INTO t1(docid,lid,content) VALUES(-4, 0, 'A D B C');
INSERT INTO t1(docid,lid,content) VALUES(-1, 1, 'A A A A');
INSERT INTO t1(docid,lid,content) VALUES(-2, 1, 'B B B B');
INSERT INTO t1(docid,lid,content) VALUES(-3, 1, 'C C C C');
INSERT INTO t1(docid,lid,content) VALUES(-4, 1, 'D D D D');
}
do_execsql_test 6.4.1 {
SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B';
} {
-4 {1 1 1 4 4}
-3 {1 1 1 4 4}
-2 {1 1 1 4 4}
-1 {1 1 1 4 4}
}
do_execsql_test 6.4.2 {
SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=1;
} {-2 {1 1 4 4 1}}
do_execsql_test 6.5 {
DROP TABLE t1;
DROP TABLE t2;
}
#-------------------------------------------------------------------------
# Tests for querying by docid.
#
do_execsql_test 7.1 {
CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=8, languageid=lid);
INSERT INTO t1(docid,lid,content) VALUES(10, 10, 'abc def');
}
do_execsql_test 7.2 {
SELECT docid,lid,content FROM t1 WHERE docid=10;
} {10 10 {abc def}}
finish_test
|
<
|
|
|
|
|
|
|
|
|
|
|
<
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
18
19
20
21
22
23
24
25
26
27
28
29
30
31
...
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
|
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
finish_test
return
}
#-------------------------------------------------------------------------
# Test out-of-range values for the languageid_bits= parameter.
#
do_catchsql_test 1.1 {
CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=31);
} {1 {languageid_bits parameter out of range}}
................................................................................
SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=1;
} {}
do_execsql_test 6.3.4 {
SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=2;
} {4 {1 1 1 1 1}}
do_execsql_test 6.4 {
CREATE VIRTUAL TABLE t2 USING fts4(languageid_bits=8, languageid=lid);
INSERT INTO t2(docid,lid,content) VALUES(-1, 0, 'A B C D');
INSERT INTO t2(docid,lid,content) VALUES(-2, 0, 'D C B A');
INSERT INTO t2(docid,lid,content) VALUES(-3, 0, 'C B D A');
INSERT INTO t2(docid,lid,content) VALUES(-4, 0, 'A D B C');
INSERT INTO t2(docid,lid,content) VALUES(-1, 1, 'A A A A');
INSERT INTO t2(docid,lid,content) VALUES(-2, 1, 'B B B B');
INSERT INTO t2(docid,lid,content) VALUES(-3, 1, 'C C C C');
INSERT INTO t2(docid,lid,content) VALUES(-4, 1, 'D D D D');
}
do_execsql_test 6.4.1 {
SELECT docid, mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'B';
} {
-4 {1 1 1 4 4}
-3 {1 1 1 4 4}
-2 {1 1 1 4 4}
-1 {1 1 1 4 4}
}
do_execsql_test 6.4.2 {
SELECT docid, mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'B' AND lid=1;
} {-2 {1 1 4 4 1}}
do_execsql_test 6.5 {
DROP TABLE t1;
DROP TABLE t2;
}
#-------------------------------------------------------------------------
# Tests for querying by docid.
#
do_execsql_test 7.1 {
CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=8, languageid=lid);
INSERT INTO t1(docid,lid,content) VALUES(10, 10, 'abc def');
}
do_execsql_test 7.2 {
SELECT docid,lid,content FROM t1 WHERE docid=10;
} {10 10 {abc def}}
do_execsql_test 7.3 {
SELECT docid,lid,content FROM t1 WHERE docid<11;
} {10 10 {abc def}}
do_execsql_test 7.4 {
DROP TABLE t1;
}
#-------------------------------------------------------------------------
# Tests for sorting by docid.
#
do_execsql_test 8.1 {
CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=6, languageid=lid);
INSERT INTO t1 (docid,lid,content) VALUES(1, 0, 'abc def');
INSERT INTO t1 (docid,lid,content) VALUES(3, 0, 'abc ghi');
INSERT INTO t1 (docid,lid,content) VALUES(2, 0, 'def ghi');
INSERT INTO t1 (docid,lid,content) VALUES(1, 5, 'A B');
INSERT INTO t1 (docid,lid,content) VALUES(3, 5, 'A C');
INSERT INTO t1 (docid,lid,content) VALUES(2, 5, 'B C');
}
do_execsql_test 8.2 {
SELECT docid FROM t1 ORDER BY docid;
} {1 1 2 2 3 3}
do_execsql_test 8.2 {
SELECT docid FROM t1 WHERE t1 MATCH 'ghi' ORDER BY docid;
} {2 3}
do_execsql_test 8.2 {
SELECT docid FROM t1 WHERE t1 MATCH 'ghi' ORDER BY docid DESC;
} {3 2}
finish_test
|