Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Fix a bug in merging FTS language tables for languages other than language 0. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts4-languageid |
Files: | files | file ages | folders |
SHA1: |
d281cb8984c911a4c0cce2ec299e1351 |
User & Date: | dan 2012-03-02 11:48:50.564 |
Context
2012-03-02
| ||
12:26 | Fix the FTS 'optimize' command on multi-lingual databases. (check-in: 65fa693729 user: dan tags: fts4-languageid) | |
11:48 | Fix a bug in merging FTS language tables for languages other than language 0. (check-in: d281cb8984 user: dan tags: fts4-languageid) | |
2012-03-01
| ||
19:44 | Add the "languageid=" option to fts4. This code is still largely untested and alsmost certainly buggy. (check-in: bea257f70f user: dan tags: fts4-languageid) | |
Changes
Changes to ext/fts3/fts3_write.c.
︙ | ︙ | |||
484 485 486 487 488 489 490 | getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); } }else{ /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); if( rc==SQLITE_OK ){ | | | 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 | getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); } }else{ /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); } } *ppStmt = pStmt; return rc; } |
︙ | ︙ |
Changes to test/fts4langid.test.
︙ | ︙ | |||
20 21 22 23 24 25 26 27 28 29 30 31 32 33 | ifcapable !fts3 { finish_test return } set ::testprefix fts4langid do_execsql_test 1.1 { CREATE VIRTUAL TABLE t1 USING fts4(a, b, languageid=lang_id); } do_execsql_test 1.2 { | > > > > > > > > > > > > > > > > > > > > > | 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 | ifcapable !fts3 { finish_test return } set ::testprefix fts4langid #--------------------------------------------------------------------------- # Test plan: # # 1.* - Warm-body tests created for specific purposes during development. # Passing these doesn't really prove much. # # 2.* - Test that FTS queries only ever return rows associated with # the requested language. # # 3.* - Test that the 'optimize' and 'rebuild' commands work correctly. # # 4.* - Test that if one is provided, the tokenizer xLanguage method # is called to configure the tokenizer before tokenizing query # or document text. # # 5.* - Test the fts4aux table when the associated FTS4 table contains # multiple languages. # # 6.* - Tests with content= tables. Both where there is a real # underlying content table and where there is not. # do_execsql_test 1.1 { CREATE VIRTUAL TABLE t1 USING fts4(a, b, languageid=lang_id); } do_execsql_test 1.2 { |
︙ | ︙ | |||
69 70 71 72 73 74 75 76 77 78 | do_execsql_test 1.16 { SELECT rowid FROM t1 WHERE t1 MATCH 'b' AND lang_id = 1; } {2} do_catchsql_test 1.17 { INSERT INTO t1(content, lang_id) VALUES('123', -1); } {1 {constraint failed}} finish_test | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 | do_execsql_test 1.16 { SELECT rowid FROM t1 WHERE t1 MATCH 'b' AND lang_id = 1; } {2} do_catchsql_test 1.17 { INSERT INTO t1(content, lang_id) VALUES('123', -1); } {1 {constraint failed}} do_execsql_test 1.18 { DROP TABLE t1; CREATE VIRTUAL TABLE t1 USING fts4(languageid=lang_id); INSERT INTO t1(content, lang_id) VALUES('A', 13); INSERT INTO t1(content, lang_id) VALUES('B', 13); INSERT INTO t1(content, lang_id) VALUES('C', 13); INSERT INTO t1(content, lang_id) VALUES('D', 13); INSERT INTO t1(content, lang_id) VALUES('E', 13); INSERT INTO t1(content, lang_id) VALUES('F', 13); INSERT INTO t1(content, lang_id) VALUES('G', 13); INSERT INTO t1(content, lang_id) VALUES('H', 13); INSERT INTO t1(content, lang_id) VALUES('I', 13); INSERT INTO t1(content, lang_id) VALUES('J', 13); INSERT INTO t1(content, lang_id) VALUES('K', 13); INSERT INTO t1(content, lang_id) VALUES('L', 13); INSERT INTO t1(content, lang_id) VALUES('M', 13); INSERT INTO t1(content, lang_id) VALUES('N', 13); INSERT INTO t1(content, lang_id) VALUES('O', 13); INSERT INTO t1(content, lang_id) VALUES('P', 13); INSERT INTO t1(content, lang_id) VALUES('Q', 13); INSERT INTO t1(content, lang_id) VALUES('R', 13); INSERT INTO t1(content, lang_id) VALUES('S', 13); SELECT rowid FROM t1 WHERE t1 MATCH 'A'; } {} #------------------------------------------------------------------------- # Test cases 2.* # proc build_multilingual_db_1 {db} { $db eval { CREATE VIRTUAL TABLE t2 USING fts4(x, y, languageid=l) } set xwords [list zero one two three four five six seven eight nine ten] set ywords [list alpha beta gamma delta epsilon zeta eta theta iota kappa] for {set i 0} {$i < 1000} {incr i} { set iLangid [expr $i%9] set x "" set y "" set x [list] lappend x [lindex $xwords [expr ($i / 1000) % 10]] lappend x [lindex $xwords [expr ($i / 100) % 10]] lappend x [lindex $xwords [expr ($i / 10) % 10]] lappend x [lindex $xwords [expr ($i / 1) % 10]] set y [list] lappend y [lindex $ywords [expr ($i / 1000) % 10]] lappend y [lindex $ywords [expr ($i / 100) % 10]] lappend y [lindex $ywords [expr ($i / 10) % 10]] lappend y [lindex $ywords [expr ($i / 1) % 10]] $db eval { INSERT INTO t2(docid, x, y, l) VALUES($i, $x, $y, $iLangid) } } } proc rowid_list_set_langid {langid} { set ::rowid_list_langid $langid } proc rowid_list {pattern} { set langid $::rowid_list_langid set res [list] db eval {SELECT docid, x, y FROM t2 WHERE l = $langid ORDER BY docid ASC} { if {[string match "*$pattern*" $x] || [string match "*$pattern*" $y]} { lappend res $docid } } return $res } proc or_merge_list {list1 list2} { set res [list] set i1 0 set i2 0 set n1 [llength $list1] set n2 [llength $list2] while {$i1 < $n1 && $i2 < $n2} { set e1 [lindex $list1 $i1] set e2 [lindex $list2 $i2] if {$e1==$e2} { lappend res $e1 incr i1 incr i2 } elseif {$e1 < $e2} { lappend res $e1 incr i1 } else { lappend res $e2 incr i2 } } concat $res [lrange $list1 $i1 end] [lrange $list2 $i2 end] } proc or_merge_lists {args} { set res [lindex $args 0] for {set i 1} {$i < [llength $args]} {incr i} { set res [or_merge_list $res [lindex $args $i]] } set res } proc and_merge_list {list1 list2} { foreach i $list2 { set a($i) 1 } set res [list] foreach i $list1 { if {[info exists a($i)]} {lappend res $i} } set res } proc and_merge_lists {args} { set res [lindex $args 0] for {set i 1} {$i < [llength $args]} {incr i} { set res [and_merge_list $res [lindex $args $i]] } set res } proc filter_list {list langid} { set res [list] foreach i $list { if {($i % 9) == $langid} {lappend res $i} } set res } do_test 2.0 { reset_db build_multilingual_db_1 db } {} proc do_test_2.1 {tn query res_script} { for {set langid 0} {$langid < 10} {incr langid} { rowid_list_set_langid $langid set res [eval $res_script] set actual [ execsql {SELECT docid FROM t2 WHERE t2 MATCH $query AND l = $langid} ] do_test 2.1.$tn.$langid [list set {} $actual] $res } } do_test_2.1 1 {delta} { rowid_list delta } do_test_2.1 2 {"zero one two"} { rowid_list "zero one two" } do_test_2.1 3 {zero one two} { and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] } do_test_2.1 4 {"zero one" OR "one two"} { or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] } finish_test |