Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Fix a case where adjacent tokens are handled incorrectly by the fts5 snippet() function. |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA3-256: |
8f5e9c192ff2820d8cfb076ab28f3069 |
User & Date: | dan 2023-11-03 17:20:20 |
Context
2023-11-03
| ||
18:45 | Back out the ALWAYS inserted late yesterday. The fuzzer discovered a counter-example. (check-in: 57063557 user: drh tags: trunk) | |
17:20 | Fix a case where adjacent tokens are handled incorrectly by the fts5 snippet() function. (check-in: 8f5e9c19 user: dan tags: trunk) | |
13:00 | Wrap more of the stmt API behind the JNI wrapper1 API. (check-in: 8fea23dc user: stephan tags: trunk) | |
Changes
Changes to ext/fts5/fts5_aux.c.
︙ | ︙ | |||
207 208 209 210 211 212 213 214 215 216 217 218 219 220 | if( rc==SQLITE_OK ){ rc = fts5CInstIterNext(&p->iter); } } if( iPos==p->iRangeEnd ){ fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); p->iOff = iEndOff; } return rc; } | > > > > > > > > | 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 | if( rc==SQLITE_OK ){ rc = fts5CInstIterNext(&p->iter); } } if( iPos==p->iRangeEnd ){ if( p->bOpen ){ if( p->iter.iStart>=0 && iPos>=p->iter.iStart ){ fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); p->iOff = iEndOff; } fts5HighlightAppend(&rc, p, p->zClose, -1); p->bOpen = 0; } fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); p->iOff = iEndOff; } return rc; } |
︙ | ︙ |
Added ext/fts5/test/fts5tokenizer2.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | # 2023 Nov 03 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # Tests focusing on the built-in fts5 tokenizers. # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5tokenizer2 # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } sqlite3_fts5_create_tokenizer db tst get_tst_tokenizer proc get_tst_tokenizer {args} { return "tst_tokenizer" } proc tst_tokenizer {flags txt} { set token "" set lTok [list] foreach c [split $txt {}] { if {$token==""} { append token $c } else { set t1 [string is upper $token] set t2 [string is upper $c] if {$t1!=$t2} { lappend lTok $token set token "" } append token $c } } if {$token!=""} { lappend lTok $token } set iOff 0 foreach t $lTok { set n [string length $t] sqlite3_fts5_token $t $iOff [expr $iOff+$n] incr iOff $n } } do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(t, tokenize=tst); } do_execsql_test 1.1 { INSERT INTO t1 VALUES('AAdontBBmess'); } do_execsql_test 1.2 { SELECT snippet(t1, 0, '>', '<', '...', 4) FROM t1('BB'); } {AAdont>BB<mess} do_execsql_test 1.3 { SELECT highlight(t1, 0, '>', '<') FROM t1('BB'); } {AAdont>BB<mess} do_execsql_test 1.4 { SELECT highlight(t1, 0, '>', '<') FROM t1('AA'); } {>AA<dontBBmess} do_execsql_test 1.5 { SELECT highlight(t1, 0, '>', '<') FROM t1('dont'); } {AA>dont<BBmess} do_execsql_test 1.6 { SELECT highlight(t1, 0, '>', '<') FROM t1('mess'); } {AAdontBB>mess<} do_execsql_test 1.7 { SELECT highlight(t1, 0, '>', '<') FROM t1('BB mess'); } {AAdont>BBmess<} finish_test |