# 2012 May 25 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # # The tests in this file focus on testing the "unicode" FTS tokenizer. # set testdir [file dirname $argv0] source $testdir/tester.tcl ifcapable !fts3_unicode { finish_test ; return } set ::testprefix fts4unicode proc do_unicode_token_test {tn input res} { set input [string map {' ''} $input] uplevel [list do_execsql_test $tn " SELECT fts3_tokenizer_test('unicode61', 'remove_diacritics=0', '$input'); " [list [list {*}$res]]] } proc do_unicode_token_test2 {tn input res} { set input [string map {' ''} $input] uplevel [list do_execsql_test $tn " SELECT fts3_tokenizer_test('unicode61', '$input'); " [list [list {*}$res]]] } proc do_unicode_token_test3 {tn args} { set res [lindex $args end] set sql "SELECT fts3_tokenizer_test('unicode61'" foreach a [lrange $args 0 end-1] { append sql ", '" append sql [string map {' ''} $a] append sql "'" } append sql ")" uplevel [list do_execsql_test $tn $sql [list [list {*}$res]]] } do_unicode_token_test 1.0 {a B c D} {0 a a 1 b B 2 c c 3 d D} do_unicode_token_test 1.1 "\uC4 \uD6 \uDC" \ "0 \uE4 \uC4 1 \uF6 \uD6 2 \uFC \uDC" do_unicode_token_test 1.2 "x\uC4x x\uD6x x\uDCx" \ "0 x\uE4x x\uC4x 1 x\uF6x x\uD6x 2 x\uFCx x\uDCx" # 0x00DF is a small "sharp s". 0x1E9E is a capital sharp s. do_unicode_token_test 1.3 "\uDF" "0 \uDF \uDF" do_unicode_token_test 1.4 "\u1E9E" "0 \uDF \u1E9E" do_unicode_token_test 1.5 "The quick brown fox" { 0 the The 1 quick quick 2 brown brown 3 fox fox } do_unicode_token_test 1.6 "The\u00bfquick\u224ebrown\u2263fox" { 0 the The 1 quick quick 2 brown brown 3 fox fox } do_unicode_token_test2 1.7 {a B c D} {0 a a 1 b B 2 c c 3 d D} do_unicode_token_test2 1.8 "\uC4 \uD6 \uDC" "0 a \uC4 1 o \uD6 2 u \uDC" do_unicode_token_test2 1.9 "x\uC4x x\uD6x x\uDCx" \ "0 xax x\uC4x 1 xox x\uD6x 2 xux x\uDCx" # Check that diacritics are removed if remove_diacritics=1 is specified. # And that they do not break tokens. do_unicode_token_test2 1.10 "xx\u0301xx" "0 xxxx xx\u301xx" # Title-case mappings work do_unicode_token_test 1.11 "\u01c5" "0 \u01c6 \u01c5" #------------------------------------------------------------------------- # set docs [list { Enhance the INSERT syntax to allow multiple rows to be inserted via the VALUES clause. } { Enhance the CREATE VIRTUAL TABLE command to support the IF NOT EXISTS clause. } { Added the sqlite3_stricmp() interface as a counterpart to sqlite3_strnicmp(). } { Added the sqlite3_db_readonly() interface. } { Added the SQLITE_FCNTL_PRAGMA file control, giving VFS implementations the ability to add new PRAGMA statements or to override built-in PRAGMAs. } { Queries of the form: "SELECT max(x), y FROM table" returns the value of y on the same row that contains the maximum x value. } { Added support for the FTS4 languageid option. } { Documented support for the FTS4 content option. This feature has actually been in the code since version 3.7.9 but is only now considered to be officially supported. } { Pending statements no longer block ROLLBACK. Instead, the pending statement will return SQLITE_ABORT upon next access after the ROLLBACK. } { Improvements to the handling of CSV inputs in the command-line shell } { Fix a bug introduced in version 3.7.10 that might cause a LEFT JOIN to be incorrectly converted into an INNER JOIN if the WHERE clause indexable terms connected by OR. }] set map(a) [list "\u00C4" "\u00E4"] ; # LATIN LETTER A WITH DIAERESIS set map(e) [list "\u00CB" "\u00EB"] ; # LATIN LETTER E WITH DIAERESIS set map(i) [list "\u00CF" "\u00EF"] ; # LATIN LETTER I WITH DIAERESIS set map(o) [list "\u00D6" "\u00F6"] ; # LATIN LETTER O WITH DIAERESIS set map(u) [list "\u00DC" "\u00FC"] ; # LATIN LETTER U WITH DIAERESIS set map(y) [list "\u0178" "\u00FF"] ; # LATIN LETTER Y WITH DIAERESIS set map(h) [list "\u1E26" "\u1E27"] ; # LATIN LETTER H WITH DIAERESIS set map(w) [list "\u1E84" "\u1E85"] ; # LATIN LETTER W WITH DIAERESIS set map(x) [list "\u1E8C" "\u1E8D"] ; # LATIN LETTER X WITH DIAERESIS foreach k [array names map] { lappend mappings [string toupper $k] [lindex $map($k) 0] lappend mappings $k [lindex $map($k) 1] } proc mapdoc {doc} { set doc [regsub -all {[[:space:]]+} $doc " "] string map $::mappings [string trim $doc] } do_test 2.0 { execsql { CREATE VIRTUAL TABLE t2 USING fts4(tokenize=unicode61, x); } foreach doc $docs { set d [mapdoc $doc] execsql { INSERT INTO t2 VALUES($d) } } } {} do_test 2.1 { set q [mapdoc "row"] execsql { SELECT * FROM t2 WHERE t2 MATCH $q } } [list [mapdoc { Queries of the form: "SELECT max(x), y FROM table" returns the value of y on the same row that contains the maximum x value. }]] foreach {tn query snippet} { 2 "row" { ...returns the value of y on the same [row] that contains the maximum x value. } 3 "ROW" { ...returns the value of y on the same [row] that contains the maximum x value. } 4 "rollback" { ...[ROLLBACK]. Instead, the pending statement will return SQLITE_ABORT upon next access after the [ROLLBACK]. } 5 "rOllback" { ...[ROLLBACK]. Instead, the pending statement will return SQLITE_ABORT upon next access after the [ROLLBACK]. } 6 "lang*" { Added support for the FTS4 [languageid] option. } } { do_test 2.$tn { set q [mapdoc $query] execsql { SELECT snippet(t2, '[', ']', '...') FROM t2 WHERE t2 MATCH $q } } [list [mapdoc $snippet]] } #------------------------------------------------------------------------- # Make sure the unicode61 tokenizer does not crash if it is passed a # NULL pointer. reset_db do_execsql_test 3.1 { CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61, x, y); INSERT INTO t1 VALUES(NULL, 'a b c'); } do_execsql_test 3.2 { SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH 'b' } {{a [b] c}} do_execsql_test 3.3 { BEGIN; DELETE FROM t1; INSERT INTO t1 VALUES('b b b b b b b b b b b', 'b b b b b b b b b b b b b'); INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 VALUES('a b c', NULL); INSERT INTO t1 VALUES('a x c', NULL); COMMIT; } do_execsql_test 3.4 { SELECT * FROM t1 WHERE t1 MATCH 'a b'; } {{a b c} {}} #------------------------------------------------------------------------- # reset_db do_test 4.1 { set a "abc\uFFFEdef" set b "abc\uD800def" set c "\uFFFEdef" set d "\uD800def" execsql { CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61, x); INSERT INTO t1 VALUES($a); INSERT INTO t1 VALUES($b); INSERT INTO t1 VALUES($c); INSERT INTO t1 VALUES($d); } } {} do_test 4.2 { set a [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0x62}] set b [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0x62}] set c [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}] set d [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}] execsql { INSERT INTO t1 VALUES($a); INSERT INTO t1 VALUES($b); INSERT INTO t1 VALUES($c); INSERT INTO t1 VALUES($d); } } {} do_test 4.3 { set a [binary format c* {0xF7 0xBF 0xBF 0xBF}] set b [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF}] set c [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF}] set d [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF}] execsql { INSERT INTO t1 VALUES($a); INSERT INTO t1 VALUES($b); INSERT INTO t1 VALUES($c); INSERT INTO t1 VALUES($d); } } {} #------------------------------------------------------------------------- do_unicode_token_test3 5.1 {tokenchars=} { sqlite3_reset sqlite3_column_int } { 0 sqlite3 sqlite3 1 reset reset 2 sqlite3 sqlite3 3 column column 4 int int } do_unicode_token_test3 5.2 {tokenchars=_} { sqlite3_reset sqlite3_column_int } { 0 sqlite3_reset sqlite3_reset 1 sqlite3_column_int sqlite3_column_int } do_unicode_token_test3 5.3 {separators=xyz} { Laotianxhorseyrunszfast } { 0 laotian Laotian 1 horse horse 2 runs runs 3 fast fast } do_unicode_token_test3 5.4 {tokenchars=xyz} { Laotianxhorseyrunszfast } { 0 laotianxhorseyrunszfast Laotianxhorseyrunszfast } do_unicode_token_test3 5.5 {tokenchars=_} {separators=zyx} { sqlite3_resetxsqlite3_column_intyhonda_phantom } { 0 sqlite3_reset sqlite3_reset 1 sqlite3_column_int sqlite3_column_int 2 honda_phantom honda_phantom } do_unicode_token_test3 5.6 "separators=\u05D1" "abc\u05D1def" { 0 abc abc 1 def def } do_unicode_token_test3 5.7 \ "tokenchars=\u2444\u2445" \ "separators=\u05D0\u05D1\u05D2" \ "\u2444fre\u2445sh\u05D0water\u05D2fish.\u2445timer" \ [list \ 0 \u2444fre\u2445sh \u2444fre\u2445sh \ 1 water water \ 2 fish fish \ 3 \u2445timer \u2445timer \ ] # Check that it is not possible to add a standalone diacritic codepoint # to either separators or tokenchars. do_unicode_token_test3 5.8 "separators=\u0301" \ "hello\u0301world \u0301helloworld" \ "0 helloworld hello\u0301world 1 helloworld helloworld" do_unicode_token_test3 5.9 "tokenchars=\u0301" \ "hello\u0301world \u0301helloworld" \ "0 helloworld hello\u0301world 1 helloworld helloworld" do_unicode_token_test3 5.10 "separators=\u0301" \ "remove_diacritics=0" \ "hello\u0301world \u0301helloworld" \ "0 hello\u0301world hello\u0301world 1 helloworld helloworld" do_unicode_token_test3 5.11 "tokenchars=\u0301" \ "remove_diacritics=0" \ "hello\u0301world \u0301helloworld" \ "0 hello\u0301world hello\u0301world 1 helloworld helloworld" #------------------------------------------------------------------------- proc do_tokenize {tokenizer txt} { set res [list] foreach {a b c} [db one {SELECT fts3_tokenizer_test($tokenizer, $txt)}] { lappend res $b } set res } # Argument $lCodepoint must be a list of codepoints (integers) that # correspond to whitespace characters. This command creates a string # $W from the codepoints, then tokenizes "${W}hello{$W}world${W}" # using tokenizer $tokenizer. The test passes if the tokenizer successfully # extracts the two 5 character tokens. # proc do_isspace_test {tn tokenizer lCp} { set whitespace [format [string repeat %c [llength $lCp]] {*}$lCp] set txt "${whitespace}hello${whitespace}world${whitespace}" uplevel [list do_test $tn [list do_tokenize $tokenizer $txt] {hello world}] } set tokenizers [list unicode61] ifcapable icu { lappend tokenizers icu } # Some tests to check that the tokenizers can both identify white-space # codepoints. All codepoints tested below are of type "Zs" in the # UnicodeData.txt file. # # Note that codepoint 6158 has changed from Zs to Cf in recent versions # of UnicodeData.txt. So take that into account for the "icu" tests. # foreach T $tokenizers { do_isspace_test 6.$T.1 $T 32 do_isspace_test 6.$T.2 $T 160 do_isspace_test 6.$T.3 $T 5760 if {$T!="icu"} { do_isspace_test 6.$T.4 $T 6158 } do_isspace_test 6.$T.5 $T 8192 do_isspace_test 6.$T.6 $T 8193 do_isspace_test 6.$T.7 $T 8194 do_isspace_test 6.$T.8 $T 8195 do_isspace_test 6.$T.9 $T 8196 do_isspace_test 6.$T.10 $T 8197 do_isspace_test 6.$T.11 $T 8198 do_isspace_test 6.$T.12 $T 8199 do_isspace_test 6.$T.13 $T 8200 do_isspace_test 6.$T.14 $T 8201 do_isspace_test 6.$T.15 $T 8202 do_isspace_test 6.$T.16 $T 8239 do_isspace_test 6.$T.17 $T 8287 do_isspace_test 6.$T.18 $T 12288 if {$T!="icu"} { do_isspace_test 6.$T.19 $T {32 160 5760 6158} } else { do_isspace_test 6.$T.19 $T {32 160 5760 8192} } do_isspace_test 6.$T.20 $T {8192 8193 8194 8195} do_isspace_test 6.$T.21 $T {8196 8197 8198 8199} do_isspace_test 6.$T.22 $T {8200 8201 8202 8239} do_isspace_test 6.$T.23 $T {8287 12288} } #------------------------------------------------------------------------- # Test that the private use ranges are treated as alphanumeric. # foreach {tn1 c} { 1 \ue000 2 \ue001 3 \uf000 4 \uf8fe 5 \uf8ff } { foreach {tn2 config res} { 1 "" "0 hello*world hello*world" 2 "separators=*" "0 hello hello 1 world world" } { set config [string map [list * $c] $config] set input [string map [list * $c] "hello*world"] set output [string map [list * $c] $res] do_unicode_token_test3 7.$tn1.$tn2 {*}$config $input $output } } #------------------------------------------------------------------------- # Cursory test of remove_diacritics=0. # # 00C4;LATIN CAPITAL LETTER A WITH DIAERESIS # 00D6;LATIN CAPITAL LETTER O WITH DIAERESIS # 00E4;LATIN SMALL LETTER A WITH DIAERESIS # 00F6;LATIN SMALL LETTER O WITH DIAERESIS # do_execsql_test 8.1.1 " CREATE VIRTUAL TABLE t3 USING fts4(tokenize=unicode61 'remove_diacritics=1'); INSERT INTO t3 VALUES('o'); INSERT INTO t3 VALUES('a'); INSERT INTO t3 VALUES('O'); INSERT INTO t3 VALUES('A'); INSERT INTO t3 VALUES('\xD6'); INSERT INTO t3 VALUES('\xC4'); INSERT INTO t3 VALUES('\xF6'); INSERT INTO t3 VALUES('\xE4'); " do_execsql_test 8.1.2 { SELECT rowid FROM t3 WHERE t3 MATCH 'o'; } {1 3 5 7} do_execsql_test 8.1.3 { SELECT rowid FROM t3 WHERE t3 MATCH 'a'; } {2 4 6 8} do_execsql_test 8.2.1 { CREATE VIRTUAL TABLE t4 USING fts4(tokenize=unicode61 "remove_diacritics=0"); INSERT INTO t4 SELECT * FROM t3; } do_execsql_test 8.2.2 { SELECT rowid FROM t4 WHERE t4 MATCH 'o'; } {1 3} do_execsql_test 8.2.3 { SELECT rowid FROM t4 WHERE t4 MATCH 'a'; } {2 4} #------------------------------------------------------------------------- # foreach {tn sql} { 1 { CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 [tokenchars= .]); CREATE VIRTUAL TABLE t6 USING fts4( tokenize=unicode61 [tokenchars=="] "tokenchars=[]"); CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 [separators=x\xC4]); } 2 { CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 "tokenchars= ."); CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 "tokenchars=[=""]"); CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 "separators=x\xC4"); } 3 { CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 'tokenchars= .'); CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 'tokenchars=="[]'); CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 'separators=x\xC4'); } 4 { CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 `tokenchars= .`); CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 `tokenchars=[="]`); CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 `separators=x\xC4`); } } { do_execsql_test 9.$tn.0 { DROP TABLE IF EXISTS t5; DROP TABLE IF EXISTS t5aux; DROP TABLE IF EXISTS t6; DROP TABLE IF EXISTS t6aux; DROP TABLE IF EXISTS t7; DROP TABLE IF EXISTS t7aux; } do_execsql_test 9.$tn.1 $sql do_execsql_test 9.$tn.2 { CREATE VIRTUAL TABLE t5aux USING fts4aux(t5); INSERT INTO t5 VALUES('one two three/four.five.six'); SELECT * FROM t5aux; } { four.five.six * 1 1 four.five.six 0 1 1 {one two three} * 1 1 {one two three} 0 1 1 } do_execsql_test 9.$tn.3 { CREATE VIRTUAL TABLE t6aux USING fts4aux(t6); INSERT INTO t6 VALUES('alpha=beta"gamma/delta[epsilon]zeta'); SELECT * FROM t6aux; } { {alpha=beta"gamma} * 1 1 {alpha=beta"gamma} 0 1 1 {delta[epsilon]zeta} * 1 1 {delta[epsilon]zeta} 0 1 1 } do_execsql_test 9.$tn.4 { CREATE VIRTUAL TABLE t7aux USING fts4aux(t7); INSERT INTO t7 VALUES('alephxbeth\xC4gimel'); SELECT * FROM t7aux; } { aleph * 1 1 aleph 0 1 1 beth * 1 1 beth 0 1 1 gimel * 1 1 gimel 0 1 1 } } # Check that multiple options are handled correctly. # do_execsql_test 10.1 { DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61 "tokenchars=xyz" "tokenchars=.=" "separators=.=" "separators=xy" "separators=a" "separators=a" "tokenchars=a" "tokenchars=a" ); INSERT INTO t1 VALUES('oneatwoxthreeyfour'); INSERT INTO t1 VALUES('a.single=word'); CREATE VIRTUAL TABLE t1aux USING fts4aux(t1); SELECT * FROM t1aux; } { .single=word * 1 1 .single=word 0 1 1 four * 1 1 four 0 1 1 one * 1 1 one 0 1 1 three * 1 1 three 0 1 1 two * 1 1 two 0 1 1 } # Test that case folding happens after tokenization, not before. # do_execsql_test 10.2 { DROP TABLE IF EXISTS t2; CREATE VIRTUAL TABLE t2 USING fts4(tokenize=unicode61 "separators=aB"); INSERT INTO t2 VALUES('oneatwoBthree'); INSERT INTO t2 VALUES('onebtwoAthree'); CREATE VIRTUAL TABLE t2aux USING fts4aux(t2); SELECT * FROM t2aux; } { one * 1 1 one 0 1 1 onebtwoathree * 1 1 onebtwoathree 0 1 1 three * 1 1 three 0 1 1 two * 1 1 two 0 1 1 } # Test that the tokenchars and separators options work with the # fts3tokenize table. # do_execsql_test 11.1 { CREATE VIRTUAL TABLE ft1 USING fts3tokenize( "unicode61", "tokenchars=@.", "separators=1234567890" ); SELECT token FROM ft1 WHERE input = 'berlin@street123sydney.road'; } { berlin@street sydney.road } finish_test