Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add compile time switch SQLITE_ENABLE_ICU_COLLATIONS. For enabling ICU collations without also enabling the tokenizer, the LIKE operator, the REGEXP operator, or the unicode aware upper()/lower() scalar functions. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA3-256: |
a079f914522d7bc4b3d27d70114eb09a |
User & Date: | dan 2017-12-08 16:23:38.347 |
Context
2017-12-08
| ||
19:37 | The query planner tries to avoids using indexes that use unknown collating functions. (check-in: 02013fc120 user: drh tags: trunk) | |
16:23 | Add compile time switch SQLITE_ENABLE_ICU_COLLATIONS. For enabling ICU collations without also enabling the tokenizer, the LIKE operator, the REGEXP operator, or the unicode aware upper()/lower() scalar functions. (check-in: a079f91452 user: dan tags: trunk) | |
14:07 | Make sure the bComplex variable in sqlite3DeleteFrom() is initialized when compiling with -DSQLITE_OMIT_TRIGGER. (check-in: e526d0c40b user: drh tags: trunk) | |
Changes
Changes to ext/icu/icu.c.
︙ | ︙ | |||
24 25 26 27 28 29 30 | ** ** * Integration of ICU and SQLite collation sequences. ** ** * An implementation of the LIKE operator that uses ICU to ** provide case-independent matching. */ | | > > > > > > > > > > > > > > > > > > > > > > | 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | ** ** * Integration of ICU and SQLite collation sequences. ** ** * An implementation of the LIKE operator that uses ICU to ** provide case-independent matching. */ #if !defined(SQLITE_CORE) \ || defined(SQLITE_ENABLE_ICU) \ || defined(SQLITE_ENABLE_ICU_COLLATIONS) /* Include ICU headers */ #include <unicode/utypes.h> #include <unicode/uregex.h> #include <unicode/ustring.h> #include <unicode/ucol.h> #include <assert.h> #ifndef SQLITE_CORE #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 #else #include "sqlite3.h" #endif /* ** This function is called when an ICU function called from within ** the implementation of an SQL scalar function returns an error. ** ** The scalar function context passed as the first argument is ** loaded with an error message based on the following two args. */ static void icuFunctionError( sqlite3_context *pCtx, /* SQLite scalar function context */ const char *zName, /* Name of ICU function that failed */ UErrorCode e /* Error code returned by ICU function */ ){ char zBuf[128]; sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); zBuf[127] = '\0'; sqlite3_result_error(pCtx, zBuf, -1); } #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) /* ** Maximum length (in bytes) of the pattern in a LIKE or GLOB ** operator. */ #ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH # define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 #endif |
︙ | ︙ | |||
220 221 222 223 224 225 226 | } if( zA && zB ){ sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc)); } } | < < < < < < < < < < < < < < < < < < | 242 243 244 245 246 247 248 249 250 251 252 253 254 255 | } if( zA && zB ){ sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc)); } } /* ** Function to delete compiled regexp objects. Registered as ** a destructor function with sqlite3_set_auxdata(). */ static void icuRegexpDelete(void *p){ URegularExpression *pExpr = (URegularExpression *)p; uregex_close(pExpr); |
︙ | ︙ | |||
403 404 405 406 407 408 409 410 411 412 413 414 415 416 | icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status); } return; } assert( 0 ); /* Unreachable */ } /* ** Collation sequence destructor function. The pCtx argument points to ** a UCollator structure previously allocated using ucol_open(). */ static void icuCollationDel(void *pCtx){ UCollator *p = (UCollator *)pCtx; ucol_close(p); | > > | 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 | icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status); } return; } assert( 0 ); /* Unreachable */ } #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ /* ** Collation sequence destructor function. The pCtx argument points to ** a UCollator structure previously allocated using ucol_open(). */ static void icuCollationDel(void *pCtx){ UCollator *p = (UCollator *)pCtx; ucol_close(p); |
︙ | ︙ | |||
497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 | const char *zName; /* Function name */ unsigned char nArg; /* Number of arguments */ unsigned short enc; /* Optimal text encoding */ unsigned char iContext; /* sqlite3_user_data() context */ void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } scalars[] = { {"icu_load_collation", 2, SQLITE_UTF8, 1, icuLoadCollation}, {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC, 0, icuRegexpFunc}, {"lower", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, {"lower", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, {"upper", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, {"upper", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, {"lower", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, {"lower", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, {"upper", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, {"upper", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, {"like", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, {"like", 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, }; int rc = SQLITE_OK; int i; | > > < | 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 | const char *zName; /* Function name */ unsigned char nArg; /* Number of arguments */ unsigned short enc; /* Optimal text encoding */ unsigned char iContext; /* sqlite3_user_data() context */ void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } scalars[] = { {"icu_load_collation", 2, SQLITE_UTF8, 1, icuLoadCollation}, #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC, 0, icuRegexpFunc}, {"lower", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, {"lower", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, {"upper", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, {"upper", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, {"lower", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, {"lower", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, {"upper", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, {"upper", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, {"like", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, {"like", 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ }; int rc = SQLITE_OK; int i; for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){ const struct IcuScalar *p = &scalars[i]; rc = sqlite3_create_function( db, p->zName, p->nArg, p->enc, p->iContext ? (void*)db : (void*)0, p->xFunc, 0, 0 |
︙ | ︙ |
Changes to src/main.c.
︙ | ︙ | |||
18 19 20 21 22 23 24 | #ifdef SQLITE_ENABLE_FTS3 # include "fts3.h" #endif #ifdef SQLITE_ENABLE_RTREE # include "rtree.h" #endif | | | 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | #ifdef SQLITE_ENABLE_FTS3 # include "fts3.h" #endif #ifdef SQLITE_ENABLE_RTREE # include "rtree.h" #endif #if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) # include "sqliteicu.h" #endif #ifdef SQLITE_ENABLE_JSON1 int sqlite3Json1Init(sqlite3*); #endif #ifdef SQLITE_ENABLE_STMTVTAB int sqlite3StmtVtabInit(sqlite3*); |
︙ | ︙ | |||
3046 3047 3048 3049 3050 3051 3052 | #ifdef SQLITE_ENABLE_FTS3 /* automatically defined by SQLITE_ENABLE_FTS4 */ if( !db->mallocFailed && rc==SQLITE_OK ){ rc = sqlite3Fts3Init(db); } #endif | | | 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 | #ifdef SQLITE_ENABLE_FTS3 /* automatically defined by SQLITE_ENABLE_FTS4 */ if( !db->mallocFailed && rc==SQLITE_OK ){ rc = sqlite3Fts3Init(db); } #endif #if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) if( !db->mallocFailed && rc==SQLITE_OK ){ rc = sqlite3IcuInit(db); } #endif #ifdef SQLITE_ENABLE_RTREE if( !db->mallocFailed && rc==SQLITE_OK){ |
︙ | ︙ |
Changes to src/test_config.c.
︙ | ︙ | |||
424 425 426 427 428 429 430 431 432 433 434 435 436 437 | #endif #ifdef SQLITE_ENABLE_ICU Tcl_SetVar2(interp, "sqlite_options", "icu", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "icu", "0", TCL_GLOBAL_ONLY); #endif #ifdef SQLITE_OMIT_INCRBLOB Tcl_SetVar2(interp, "sqlite_options", "incrblob", "0", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "incrblob", "1", TCL_GLOBAL_ONLY); #endif /* SQLITE_OMIT_AUTOVACUUM */ | > > > > > > | 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 | #endif #ifdef SQLITE_ENABLE_ICU Tcl_SetVar2(interp, "sqlite_options", "icu", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "icu", "0", TCL_GLOBAL_ONLY); #endif #ifdef SQLITE_ENABLE_ICU_COLLATIONS Tcl_SetVar2(interp, "sqlite_options", "icu_collations", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "icu_collations", "0", TCL_GLOBAL_ONLY); #endif #ifdef SQLITE_OMIT_INCRBLOB Tcl_SetVar2(interp, "sqlite_options", "incrblob", "0", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "incrblob", "1", TCL_GLOBAL_ONLY); #endif /* SQLITE_OMIT_AUTOVACUUM */ |
︙ | ︙ |
Changes to test/icu.test.
︙ | ︙ | |||
11 12 13 14 15 16 17 | # # $Id: icu.test,v 1.2 2008/07/12 14:52:20 drh Exp $ # set testdir [file dirname $argv0] source $testdir/tester.tcl | | > > | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | > | 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 | # # $Id: icu.test,v 1.2 2008/07/12 14:52:20 drh Exp $ # set testdir [file dirname $argv0] source $testdir/tester.tcl ifcapable !icu&&!icu_collations { finish_test return } # Create a table to work with. # execsql {CREATE TABLE test1(i1 int, i2 int, r1 real, r2 real, t1 text, t2 text)} execsql {INSERT INTO test1 VALUES(1,2,1.1,2.2,'hello','world')} proc test_expr {name settings expr result} { do_test $name [format { lindex [db eval { BEGIN; UPDATE test1 SET %s; SELECT %s FROM test1; ROLLBACK; }] 0 } $settings $expr] $result } ifcapable icu { # Tests of the REGEXP operator. # test_expr icu-1.1 {i1='hello'} {i1 REGEXP 'hello'} 1 test_expr icu-1.2 {i1='hello'} {i1 REGEXP '.ello'} 1 test_expr icu-1.3 {i1='hello'} {i1 REGEXP '.ell'} 0 test_expr icu-1.4 {i1='hello'} {i1 REGEXP '.ell.*'} 1 test_expr icu-1.5 {i1=NULL} {i1 REGEXP '.ell.*'} {} # Some non-ascii characters with defined case mappings # set ::EGRAVE "\xC8" set ::egrave "\xE8" set ::OGRAVE "\xD2" set ::ograve "\xF2" # That German letter that looks a bit like a B. The # upper-case version of which is "SS" (two characters). # set ::szlig "\xDF" # Tests of the upper()/lower() functions. # test_expr icu-2.1 {i1='HellO WorlD'} {upper(i1)} {HELLO WORLD} test_expr icu-2.2 {i1='HellO WorlD'} {lower(i1)} {hello world} test_expr icu-2.3 {i1=$::egrave} {lower(i1)} $::egrave test_expr icu-2.4 {i1=$::egrave} {upper(i1)} $::EGRAVE test_expr icu-2.5 {i1=$::ograve} {lower(i1)} $::ograve test_expr icu-2.6 {i1=$::ograve} {upper(i1)} $::OGRAVE test_expr icu-2.3 {i1=$::EGRAVE} {lower(i1)} $::egrave test_expr icu-2.4 {i1=$::EGRAVE} {upper(i1)} $::EGRAVE test_expr icu-2.5 {i1=$::OGRAVE} {lower(i1)} $::ograve test_expr icu-2.6 {i1=$::OGRAVE} {upper(i1)} $::OGRAVE test_expr icu-2.7 {i1=$::szlig} {upper(i1)} "SS" test_expr icu-2.8 {i1='SS'} {lower(i1)} "ss" do_execsql_test icu-2.9 { SELECT upper(char(0xfb04,0xfb04,0xfb04,0xfb04)); } {FFLFFLFFLFFL} # In turkish (locale="tr_TR"), the lower case version of I # is "small dotless i" (code point 0x131 (decimal 305)). # set ::small_dotless_i "\u0131" test_expr icu-3.1 {i1='I'} {lower(i1)} "i" test_expr icu-3.2 {i1='I'} {lower(i1, 'tr_tr')} $::small_dotless_i test_expr icu-3.3 {i1='I'} {lower(i1, 'en_AU')} "i" } #-------------------------------------------------------------------- # Test the collation sequence function. # do_test icu-4.1 { execsql { CREATE TABLE fruit(name); |
︙ | ︙ | |||
120 121 122 123 124 125 126 | #------------------------------------------------------------------------- # Test that it is not possible to call the ICU regex() function with # anything other than exactly two arguments. See also: # # http://src.chromium.org/viewvc/chrome/trunk/src/third_party/sqlite/icu-regexp.patch?revision=34807&view=markup # | > | | | | | | | | | | | | | < | | | | > | 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 | #------------------------------------------------------------------------- # Test that it is not possible to call the ICU regex() function with # anything other than exactly two arguments. See also: # # http://src.chromium.org/viewvc/chrome/trunk/src/third_party/sqlite/icu-regexp.patch?revision=34807&view=markup # ifcapable icu { do_catchsql_test icu-5.1 { SELECT regexp('a[abc]c.*', 'abc') } {0 1} do_catchsql_test icu-5.2 { SELECT regexp('a[abc]c.*') } {1 {wrong number of arguments to function regexp()}} do_catchsql_test icu-5.3 { SELECT regexp('a[abc]c.*', 'abc', 'c') } {1 {wrong number of arguments to function regexp()}} do_catchsql_test icu-5.4 { SELECT 'abc' REGEXP 'a[abc]c.*' } {0 1} do_catchsql_test icu-5.4 {SELECT 'abc' REGEXP } {1 {near " ": syntax error}} do_catchsql_test icu-5.5 {SELECT 'abc' REGEXP, 1} {1 {near ",": syntax error}} do_malloc_test icu-6.10 -sqlbody { SELECT upper(char(0xfb04,0xdf,0xfb04,0xe8,0xfb04)); } } finish_test |