Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Merge the latest fixes and enhancements from trunk. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | sessions |
Files: | files | file ages | folders |
SHA1: |
007e5c6df60f9743ac6914332f59925e |
User & Date: | drh 2016-01-14 14:48:17.709 |
Context
2016-01-20
| ||
11:33 | Merge recent enhancements from trunk. (check-in: 327af5f644 user: drh tags: sessions) | |
2016-01-14
| ||
14:48 | Merge the latest fixes and enhancements from trunk. (check-in: 007e5c6df6 user: drh tags: sessions) | |
14:33 | Yet another change to FTS5 trying to get it to merge successfully into sessions. (check-in: 8dedff3b9a user: drh tags: trunk) | |
2016-01-11
| ||
13:10 | Merge recent enhancements, and especially the WAL overwrite change, from trunk. (check-in: c4a858b228 user: drh tags: sessions) | |
Changes
Changes to Makefile.in.
︙ | ︙ | |||
178 179 180 181 182 183 184 | icu.lo insert.lo journal.lo json1.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo \ | | | 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 | icu.lo insert.lo journal.lo json1.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo \ sqlite3session.lo select.lo sqlite3rbu.lo status.lo \ table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \ utf.lo vtab.lo # Object files for the amalgamation. |
︙ | ︙ | |||
1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 | fts5.c: $(FTS5_SRC) $(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl cp $(TOP)/ext/fts5/fts5.h . fts5.lo: fts5.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c fts5.c # Rules to build the 'testfixture' application. # # If using the amalgamation, use sqlite3.c directly to build the test # fixture. Otherwise link against libsqlite3.la. (This distinction is # necessary because the test fixture requires non-API symbols which are # hidden when the library is built via the amalgamation). | > > > | 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 | fts5.c: $(FTS5_SRC) $(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl cp $(TOP)/ext/fts5/fts5.h . fts5.lo: fts5.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c fts5.c sqlite3rbu.lo: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c # Rules to build the 'testfixture' application. # # If using the amalgamation, use sqlite3.c directly to build the test # fixture. Otherwise link against libsqlite3.la. (This distinction is # necessary because the test fixture requires non-API symbols which are # hidden when the library is built via the amalgamation). |
︙ | ︙ | |||
1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 | $(LTLINK) -I. -o $@ $(TOP)/tool/logest.c wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.c $(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.c $(TLIBS) speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS) # This target will fail if the SQLite amalgamation contains any exported # symbols that do not begin with "sqlite3_". It is run as part of the # releasetest.tcl script. # checksymbols: sqlite3.lo nm -g --defined-only sqlite3.o | egrep -v ' sqlite3(changeset|session)?_' ; test $$? -ne 0 | > > > > > > | 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 | $(LTLINK) -I. -o $@ $(TOP)/tool/logest.c wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.c $(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.c $(TLIBS) speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS) rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo $(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS) loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la $(LTLINK) $(TOP)/tool/loadfts.c libsqlite3.la -o $@ $(TLIBS) # This target will fail if the SQLite amalgamation contains any exported # symbols that do not begin with "sqlite3_". It is run as part of the # releasetest.tcl script. # checksymbols: sqlite3.lo nm -g --defined-only sqlite3.o | egrep -v ' sqlite3(changeset|session)?_' ; test $$? -ne 0 |
︙ | ︙ |
Changes to Makefile.msc.
︙ | ︙ | |||
846 847 848 849 850 851 852 | icu.lo insert.lo journal.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo \ | | | 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 | icu.lo insert.lo journal.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo \ sqlite3session.lo select.lo sqlite3rbu.lo status.lo \ table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \ utf.lo vtab.lo # Object files for the amalgamation. |
︙ | ︙ | |||
1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 | fts5_ext.lo: fts5.c $(HDR) $(EXTHDR) $(LTCOMPILE) $(NO_WARN) -c fts5.c fts5.dll: fts5_ext.lo $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ fts5_ext.lo # Rules to build the 'testfixture' application. # # If using the amalgamation, use sqlite3.c directly to build the test # fixture. Otherwise link against libsqlite3.lib. (This distinction is # necessary because the test fixture requires non-API symbols which are # hidden when the library is built via the amalgamation). # | > > > | 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 | fts5_ext.lo: fts5.c $(HDR) $(EXTHDR) $(LTCOMPILE) $(NO_WARN) -c fts5.c fts5.dll: fts5_ext.lo $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ fts5_ext.lo sqlite3rbu.lo: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c # Rules to build the 'testfixture' application. # # If using the amalgamation, use sqlite3.c directly to build the test # fixture. Otherwise link against libsqlite3.lib. (This distinction is # necessary because the test fixture requires non-API symbols which are # hidden when the library is built via the amalgamation). # |
︙ | ︙ | |||
1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 | wordcount.exe: $(TOP)\test\wordcount.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL del /Q *.bsc *.cod *.da *.bb *.bbg gmon.out 2>NUL del /Q sqlite3.h opcodes.c opcodes.h 2>NUL del /Q lemon.* lempar.c parse.* 2>NUL del /Q mkkeywordhash.* keywordhash.h 2>NUL | > > > > | 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 | wordcount.exe: $(TOP)\test\wordcount.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -I. -DSQLITE_ENABLE_RBU -Fe$@ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) \ $(LDFLAGS) $(LTLINKOPTS) clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL del /Q *.bsc *.cod *.da *.bb *.bbg gmon.out 2>NUL del /Q sqlite3.h opcodes.c opcodes.h 2>NUL del /Q lemon.* lempar.c parse.* 2>NUL del /Q mkkeywordhash.* keywordhash.h 2>NUL |
︙ | ︙ |
Changes to autoconf/configure.ac.
︙ | ︙ | |||
69 70 71 72 73 74 75 | # AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING( [--enable-threadsafe], [build a thread-safe library [default=yes]])], [], [enable_threadsafe=yes]) THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0 if test x"$enable_threadsafe" != "xno"; then THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1" | | | 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | # AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING( [--enable-threadsafe], [build a thread-safe library [default=yes]])], [], [enable_threadsafe=yes]) THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0 if test x"$enable_threadsafe" != "xno"; then THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1" AC_SEARCH_LIBS(pthread_mutexattr_init, pthread) fi AC_SUBST(THREADSAFE_FLAGS) #----------------------------------------------------------------------- #----------------------------------------------------------------------- # --enable-dynamic-extensions # |
︙ | ︙ |
Changes to configure.
︙ | ︙ | |||
10460 10461 10462 10463 10464 10465 10466 | SQLITE_THREADSAFE=1 { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } fi if test "$SQLITE_THREADSAFE" = "1"; then | | | | | | | | | | | | | | 10460 10461 10462 10463 10464 10465 10466 10467 10468 10469 10470 10471 10472 10473 10474 10475 10476 10477 10478 10479 10480 10481 10482 10483 10484 10485 10486 10487 10488 10489 10490 10491 10492 10493 10494 10495 10496 10497 10498 10499 10500 10501 10502 10503 10504 10505 10506 10507 10508 10509 10510 10511 10512 10513 10514 10515 10516 10517 10518 10519 10520 10521 10522 10523 10524 | SQLITE_THREADSAFE=1 { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } fi if test "$SQLITE_THREADSAFE" = "1"; then { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_mutexattr_init" >&5 $as_echo_n "checking for library containing pthread_mutexattr_init... " >&6; } if ${ac_cv_search_pthread_mutexattr_init+:} false; then : $as_echo_n "(cached) " >&6 else ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ #ifdef __cplusplus extern "C" #endif char pthread_mutexattr_init (); int main () { return pthread_mutexattr_init (); ; return 0; } _ACEOF for ac_lib in '' pthread; do if test -z "$ac_lib"; then ac_res="none required" else ac_res=-l$ac_lib LIBS="-l$ac_lib $ac_func_search_save_LIBS" fi if ac_fn_c_try_link "$LINENO"; then : ac_cv_search_pthread_mutexattr_init=$ac_res fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext if ${ac_cv_search_pthread_mutexattr_init+:} false; then : break fi done if ${ac_cv_search_pthread_mutexattr_init+:} false; then : else ac_cv_search_pthread_mutexattr_init=no fi rm conftest.$ac_ext LIBS=$ac_func_search_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_mutexattr_init" >&5 $as_echo "$ac_cv_search_pthread_mutexattr_init" >&6; } ac_res=$ac_cv_search_pthread_mutexattr_init if test "$ac_res" != no; then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" fi fi |
︙ | ︙ |
Changes to configure.ac.
︙ | ︙ | |||
190 191 192 193 194 195 196 | else SQLITE_THREADSAFE=1 AC_MSG_RESULT([yes]) fi AC_SUBST(SQLITE_THREADSAFE) if test "$SQLITE_THREADSAFE" = "1"; then | | | 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 | else SQLITE_THREADSAFE=1 AC_MSG_RESULT([yes]) fi AC_SUBST(SQLITE_THREADSAFE) if test "$SQLITE_THREADSAFE" = "1"; then AC_SEARCH_LIBS(pthread_mutexattr_init, pthread) fi ########## # Do we want to support release # AC_ARG_ENABLE(releasemode, AC_HELP_STRING([--enable-releasemode],[Support libtool link to release mode]),,enable_releasemode=no) |
︙ | ︙ |
Changes to ext/fts5/fts5.h.
︙ | ︙ | |||
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 | ** *pnToken to the number of tokens in column iCol of the current row. ** ** If parameter iCol is greater than or equal to the number of columns ** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. ** an OOM condition or IO error), an appropriate SQLite error code is ** returned. ** ** xColumnText: ** This function attempts to retrieve the text of column iCol of the ** current document. If successful, (*pz) is set to point to a buffer ** containing the text in utf-8 encoding, (*pn) is set to the size in bytes ** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, ** if an error occurs, an SQLite error code is returned and the final values ** of (*pz) and (*pn) are undefined. ** ** xPhraseCount: ** Returns the number of phrases in the current query expression. ** ** xPhraseSize: ** Returns the number of tokens in phrase iPhrase of the query. Phrases ** are numbered starting from zero. ** ** xInstCount: ** Set *pnInst to the total number of occurrences of all phrases within ** the query within the current row. Return SQLITE_OK if successful, or ** an error code (i.e. SQLITE_NOMEM) if an error occurs. ** ** xInst: ** Query for the details of phrase match iIdx within the current row. ** Phrase matches are numbered starting from zero, so the iIdx argument ** should be greater than or equal to zero and smaller than the value ** output by xInstCount(). ** ** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) ** if an error occurs. ** ** xRowid: ** Returns the rowid of the current row. ** ** xTokenize: ** Tokenize text using the tokenizer belonging to the FTS5 table. ** | > > > > > > > > > > > > > > > > > | 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 | ** *pnToken to the number of tokens in column iCol of the current row. ** ** If parameter iCol is greater than or equal to the number of columns ** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. ** an OOM condition or IO error), an appropriate SQLite error code is ** returned. ** ** This function may be quite inefficient if used with an FTS5 table ** created with the "columnsize=0" option. ** ** xColumnText: ** This function attempts to retrieve the text of column iCol of the ** current document. If successful, (*pz) is set to point to a buffer ** containing the text in utf-8 encoding, (*pn) is set to the size in bytes ** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, ** if an error occurs, an SQLite error code is returned and the final values ** of (*pz) and (*pn) are undefined. ** ** xPhraseCount: ** Returns the number of phrases in the current query expression. ** ** xPhraseSize: ** Returns the number of tokens in phrase iPhrase of the query. Phrases ** are numbered starting from zero. ** ** xInstCount: ** Set *pnInst to the total number of occurrences of all phrases within ** the query within the current row. Return SQLITE_OK if successful, or ** an error code (i.e. SQLITE_NOMEM) if an error occurs. ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. If the FTS5 table is created ** with either "detail=none" or "detail=column" and "content=" option ** (i.e. if it is a contentless table), then this API always returns 0. ** ** xInst: ** Query for the details of phrase match iIdx within the current row. ** Phrase matches are numbered starting from zero, so the iIdx argument ** should be greater than or equal to zero and smaller than the value ** output by xInstCount(). ** ** Usually, output parameter *piPhrase is set to the phrase number, *piCol ** to the column in which it occurs and *piOff the token offset of the ** first token of the phrase. The exception is if the table was created ** with the offsets=0 option specified. In this case *piOff is always ** set to -1. ** ** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) ** if an error occurs. ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. ** ** xRowid: ** Returns the rowid of the current row. ** ** xTokenize: ** Tokenize text using the tokenizer belonging to the FTS5 table. ** |
︙ | ︙ | |||
192 193 194 195 196 197 198 | ** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient ** to use, this API may be faster under some circumstances. To iterate ** through instances of phrase iPhrase, use the following code: ** ** Fts5PhraseIter iter; ** int iCol, iOff; ** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 | ** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient ** to use, this API may be faster under some circumstances. To iterate ** through instances of phrase iPhrase, use the following code: ** ** Fts5PhraseIter iter; ** int iCol, iOff; ** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); ** iCol>=0; ** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) ** ){ ** // An instance of phrase iPhrase at offset iOff of column iCol ** } ** ** The Fts5PhraseIter structure is defined above. Applications should not ** modify this structure directly - it should only be used as shown above ** with the xPhraseFirst() and xPhraseNext() API methods (and by ** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. If the FTS5 table is created ** with either "detail=none" or "detail=column" and "content=" option ** (i.e. if it is a contentless table), then this API always iterates ** through an empty set (all calls to xPhraseFirst() set iCol to -1). ** ** xPhraseNext() ** See xPhraseFirst above. ** ** xPhraseFirstColumn() ** This function and xPhraseNextColumn() are similar to the xPhraseFirst() ** and xPhraseNext() APIs described above. The difference is that instead ** of iterating through all instances of a phrase in the current row, these ** APIs are used to iterate through the set of columns in the current row ** that contain one or more instances of a specified phrase. For example: ** ** Fts5PhraseIter iter; ** int iCol; ** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); ** iCol>=0; ** pApi->xPhraseNextColumn(pFts, &iter, &iCol) ** ){ ** // Column iCol contains at least one instance of phrase iPhrase ** } ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" option. If the FTS5 table is created with either ** "detail=none" "content=" option (i.e. if it is a contentless table), ** then this API always iterates through an empty set (all calls to ** xPhraseFirstColumn() set iCol to -1). ** ** The information accessed using this API and its companion ** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext ** (or xInst/xInstCount). The chief advantage of this API is that it is ** significantly more efficient than those alternatives when used with ** "detail=column" tables. ** ** xPhraseNextColumn() ** See xPhraseFirstColumn above. */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 3 */ void *(*xUserData)(Fts5Context*); int (*xColumnCount)(Fts5Context*); int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); |
︙ | ︙ | |||
236 237 238 239 240 241 242 | int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) ); int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); void *(*xGetAuxdata)(Fts5Context*, int bClear); | | > > > | 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 | int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) ); int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); void *(*xGetAuxdata)(Fts5Context*, int bClear); int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); }; /* ** CUSTOM AUXILIARY FUNCTIONS *************************************************************************/ /************************************************************************* |
︙ | ︙ |
Changes to ext/fts5/fts5Int.h.
︙ | ︙ | |||
147 148 149 150 151 152 153 154 155 156 157 158 159 160 | u8 *abUnindexed; /* True for unindexed columns */ int nPrefix; /* Number of prefix indexes */ int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ int eContent; /* An FTS5_CONTENT value */ char *zContent; /* content table */ char *zContentRowid; /* "content_rowid=" option value */ int bColumnsize; /* "columnsize=" option value (dflt==1) */ char *zContentExprlist; Fts5Tokenizer *pTok; fts5_tokenizer *pTokApi; /* Values loaded from the %_config table */ int iCookie; /* Incremented when %_config is modified */ int pgsz; /* Approximate page size used in %_data */ | > | 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 | u8 *abUnindexed; /* True for unindexed columns */ int nPrefix; /* Number of prefix indexes */ int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ int eContent; /* An FTS5_CONTENT value */ char *zContent; /* content table */ char *zContentRowid; /* "content_rowid=" option value */ int bColumnsize; /* "columnsize=" option value (dflt==1) */ int eDetail; /* FTS5_DETAIL_XXX value */ char *zContentExprlist; Fts5Tokenizer *pTok; fts5_tokenizer *pTokApi; /* Values loaded from the %_config table */ int iCookie; /* Incremented when %_config is modified */ int pgsz; /* Approximate page size used in %_data */ |
︙ | ︙ | |||
175 176 177 178 179 180 181 182 183 184 185 186 187 188 | /* Current expected value of %_config table 'version' field */ #define FTS5_CURRENT_VERSION 4 #define FTS5_CONTENT_NORMAL 0 #define FTS5_CONTENT_NONE 1 #define FTS5_CONTENT_EXTERNAL 2 int sqlite3Fts5ConfigParse( Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** ); void sqlite3Fts5ConfigFree(Fts5Config*); | > > > | 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 | /* Current expected value of %_config table 'version' field */ #define FTS5_CURRENT_VERSION 4 #define FTS5_CONTENT_NORMAL 0 #define FTS5_CONTENT_NONE 1 #define FTS5_CONTENT_EXTERNAL 2 #define FTS5_DETAIL_FULL 0 #define FTS5_DETAIL_NONE 1 #define FTS5_DETAIL_COLUMNS 2 int sqlite3Fts5ConfigParse( Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** ); void sqlite3Fts5ConfigFree(Fts5Config*); |
︙ | ︙ | |||
288 289 290 291 292 293 294 295 296 297 298 299 300 301 | /* Malloc utility */ void *sqlite3Fts5MallocZero(int *pRc, int nByte); char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn); /* Character set tests (like isspace(), isalpha() etc.) */ int sqlite3Fts5IsBareword(char t); /* ** End of interface to code in fts5_buffer.c. **************************************************************************/ /************************************************************************** ** Interface to code in fts5_index.c. fts5_index.c contains contains code ** to access the data stored in the %_data table. | > > > > > > > | 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 | /* Malloc utility */ void *sqlite3Fts5MallocZero(int *pRc, int nByte); char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn); /* Character set tests (like isspace(), isalpha() etc.) */ int sqlite3Fts5IsBareword(char t); /* Bucket of terms object used by the integrity-check in offsets=0 mode. */ typedef struct Fts5Termset Fts5Termset; int sqlite3Fts5TermsetNew(Fts5Termset**); int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent); void sqlite3Fts5TermsetFree(Fts5Termset*); /* ** End of interface to code in fts5_buffer.c. **************************************************************************/ /************************************************************************** ** Interface to code in fts5_index.c. fts5_index.c contains contains code ** to access the data stored in the %_data table. |
︙ | ︙ | |||
323 324 325 326 327 328 329 330 331 332 333 334 335 336 | ** sqlite3Fts5IndexQuery(p, "token", 5, 0, 0, &pIter); ** 0==sqlite3Fts5IterEof(pIter); ** sqlite3Fts5IterNext(pIter) ** ){ ** i64 iRowid = sqlite3Fts5IterRowid(pIter); ** } */ /* ** Open a new iterator to iterate though all rowids that match the ** specified token or token prefix. */ int sqlite3Fts5IndexQuery( Fts5Index *p, /* FTS index to query */ | > > > > > > > > > > > > > > > > > > > > > > > | 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 | ** sqlite3Fts5IndexQuery(p, "token", 5, 0, 0, &pIter); ** 0==sqlite3Fts5IterEof(pIter); ** sqlite3Fts5IterNext(pIter) ** ){ ** i64 iRowid = sqlite3Fts5IterRowid(pIter); ** } */ /* ** Return a simple checksum value based on the arguments. */ u64 sqlite3Fts5IndexEntryCksum( i64 iRowid, int iCol, int iPos, int iIdx, const char *pTerm, int nTerm ); /* ** Argument p points to a buffer containing utf-8 text that is n bytes in ** size. Return the number of bytes in the nChar character prefix of the ** buffer, or 0 if there are less than nChar characters in total. */ int sqlite3Fts5IndexCharlenToBytelen( const char *p, int nByte, int nChar ); /* ** Open a new iterator to iterate though all rowids that match the ** specified token or token prefix. */ int sqlite3Fts5IndexQuery( Fts5Index *p, /* FTS index to query */ |
︙ | ︙ | |||
409 410 411 412 413 414 415 | */ int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize); int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); /* ** Functions called by the storage module as part of integrity-check. */ | < | 443 444 445 446 447 448 449 450 451 452 453 454 455 456 | */ int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize); int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); /* ** Functions called by the storage module as part of integrity-check. */ int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum); /* ** Called during virtual module initialization to register UDF ** fts5_decode() with SQLite */ int sqlite3Fts5IndexInit(sqlite3*); |
︙ | ︙ | |||
431 432 433 434 435 436 437 438 439 440 441 442 443 444 | int sqlite3Fts5IndexReads(Fts5Index *p); int sqlite3Fts5IndexReinit(Fts5Index *p); int sqlite3Fts5IndexOptimize(Fts5Index *p); int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge); int sqlite3Fts5IndexLoadConfig(Fts5Index *p); /* ** End of interface to code in fts5_index.c. **************************************************************************/ /************************************************************************** ** Interface to code in fts5_varint.c. | > > | 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 | int sqlite3Fts5IndexReads(Fts5Index *p); int sqlite3Fts5IndexReinit(Fts5Index *p); int sqlite3Fts5IndexOptimize(Fts5Index *p); int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge); int sqlite3Fts5IndexLoadConfig(Fts5Index *p); int sqlite3Fts5IterCollist(Fts5IndexIter*, const u8 **, int*); /* ** End of interface to code in fts5_index.c. **************************************************************************/ /************************************************************************** ** Interface to code in fts5_varint.c. |
︙ | ︙ | |||
488 489 490 491 492 493 494 | ** Interface to code in fts5_hash.c. */ typedef struct Fts5Hash Fts5Hash; /* ** Create a hash table, free a hash table. */ | | | 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 | ** Interface to code in fts5_hash.c. */ typedef struct Fts5Hash Fts5Hash; /* ** Create a hash table, free a hash table. */ int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize); void sqlite3Fts5HashFree(Fts5Hash*); int sqlite3Fts5HashWrite( Fts5Hash*, i64 iRowid, /* Rowid for this entry */ int iCol, /* Column token appears in (-ve -> delete) */ int iPos, /* Position of token within column */ |
︙ | ︙ | |||
624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 | /* Called during startup to register a UDF with SQLite */ int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*); int sqlite3Fts5ExprPhraseCount(Fts5Expr*); int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**); /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written ** C code in this module. The interfaces below this point are called by ** the parser code in fts5parse.y. */ void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...); | > > > > > > > > > > | 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 | /* Called during startup to register a UDF with SQLite */ int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*); int sqlite3Fts5ExprPhraseCount(Fts5Expr*); int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); typedef struct Fts5PoslistPopulator Fts5PoslistPopulator; Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int); int sqlite3Fts5ExprPopulatePoslists( Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int ); void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64); void sqlite3Fts5ExprClearEof(Fts5Expr*); int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**); int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written ** C code in this module. The interfaces below this point are called by ** the parser code in fts5parse.y. */ void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...); |
︙ | ︙ |
Changes to ext/fts5/fts5_buffer.c.
︙ | ︙ | |||
286 287 288 289 290 291 292 293 294 | 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 0x50 .. 0x5F */ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60 .. 0x6F */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 /* 0x70 .. 0x7F */ }; return (t & 0x80) || aBareword[(int)t]; } | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 | 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 0x50 .. 0x5F */ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60 .. 0x6F */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 /* 0x70 .. 0x7F */ }; return (t & 0x80) || aBareword[(int)t]; } /************************************************************************* */ typedef struct Fts5TermsetEntry Fts5TermsetEntry; struct Fts5TermsetEntry { char *pTerm; int nTerm; int iIdx; /* Index (main or aPrefix[] entry) */ Fts5TermsetEntry *pNext; }; struct Fts5Termset { Fts5TermsetEntry *apHash[512]; }; int sqlite3Fts5TermsetNew(Fts5Termset **pp){ int rc = SQLITE_OK; *pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset)); return rc; } int sqlite3Fts5TermsetAdd( Fts5Termset *p, int iIdx, const char *pTerm, int nTerm, int *pbPresent ){ int rc = SQLITE_OK; *pbPresent = 0; if( p ){ int i; int hash; Fts5TermsetEntry *pEntry; /* Calculate a hash value for this term */ hash = 104 + iIdx; for(i=0; i<nTerm; i++){ hash += (hash << 3) + (int)pTerm[i]; } hash = hash % ArraySize(p->apHash); for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){ if( pEntry->iIdx==iIdx && pEntry->nTerm==nTerm && memcmp(pEntry->pTerm, pTerm, nTerm)==0 ){ *pbPresent = 1; break; } } if( pEntry==0 ){ pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm); if( pEntry ){ pEntry->pTerm = (char*)&pEntry[1]; pEntry->nTerm = nTerm; pEntry->iIdx = iIdx; memcpy(pEntry->pTerm, pTerm, nTerm); pEntry->pNext = p->apHash[hash]; p->apHash[hash] = pEntry; } } } return rc; } void sqlite3Fts5TermsetFree(Fts5Termset *p){ if( p ){ int i; for(i=0; i<ArraySize(p->apHash); i++){ Fts5TermsetEntry *pEntry = p->apHash[i]; while( pEntry ){ Fts5TermsetEntry *pDel = pEntry; pEntry = pEntry->pNext; sqlite3_free(pDel); } } sqlite3_free(p); } } |
Changes to ext/fts5/fts5_config.c.
︙ | ︙ | |||
10 11 12 13 14 15 16 | ** ****************************************************************************** ** ** This is an SQLite module implementing full-text search. */ | < | 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | ** ****************************************************************************** ** ** This is an SQLite module implementing full-text search. */ #include "fts5Int.h" #define FTS5_DEFAULT_PAGE_SIZE 4050 #define FTS5_DEFAULT_AUTOMERGE 4 #define FTS5_DEFAULT_CRISISMERGE 16 #define FTS5_DEFAULT_HASHSIZE (1024*1024) |
︙ | ︙ | |||
191 192 193 194 195 196 197 198 199 200 201 202 203 204 | assert( 0==fts5_iswhitespace(z[0]) ); quote = z[0]; if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ fts5Dequote(z); } } /* ** Parse a "special" CREATE VIRTUAL TABLE directive and update ** configuration object pConfig as appropriate. ** ** If successful, object pConfig is updated and SQLITE_OK returned. If ** an error occurs, an SQLite error code is returned and an error message ** may be left in *pzErr. It is the responsibility of the caller to | > > > > > > > > > > > > > > > > > > > > > > > > > > > | 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 | assert( 0==fts5_iswhitespace(z[0]) ); quote = z[0]; if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ fts5Dequote(z); } } struct Fts5Enum { const char *zName; int eVal; }; typedef struct Fts5Enum Fts5Enum; static int fts5ConfigSetEnum( const Fts5Enum *aEnum, const char *zEnum, int *peVal ){ int nEnum = strlen(zEnum); int i; int iVal = -1; for(i=0; aEnum[i].zName; i++){ if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){ if( iVal>=0 ) return SQLITE_ERROR; iVal = aEnum[i].eVal; } } *peVal = iVal; return iVal<0 ? SQLITE_ERROR : SQLITE_OK; } /* ** Parse a "special" CREATE VIRTUAL TABLE directive and update ** configuration object pConfig as appropriate. ** ** If successful, object pConfig is updated and SQLITE_OK returned. If ** an error occurs, an SQLite error code is returned and an error message ** may be left in *pzErr. It is the responsibility of the caller to |
︙ | ︙ | |||
340 341 342 343 344 345 346 347 348 349 350 351 352 353 | *pzErr = sqlite3_mprintf("malformed columnsize=... directive"); rc = SQLITE_ERROR; }else{ pConfig->bColumnsize = (zArg[0]=='1'); } return rc; } *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd); return SQLITE_ERROR; } /* ** Allocate an instance of the default tokenizer ("simple") at | > > > > > > > > > > > > > > | 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 | *pzErr = sqlite3_mprintf("malformed columnsize=... directive"); rc = SQLITE_ERROR; }else{ pConfig->bColumnsize = (zArg[0]=='1'); } return rc; } if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){ const Fts5Enum aDetail[] = { { "none", FTS5_DETAIL_NONE }, { "full", FTS5_DETAIL_FULL }, { "columns", FTS5_DETAIL_COLUMNS }, { 0, 0 } }; if( (rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail)) ){ *pzErr = sqlite3_mprintf("malformed detail=... directive"); } return rc; } *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd); return SQLITE_ERROR; } /* ** Allocate an instance of the default tokenizer ("simple") at |
︙ | ︙ | |||
496 497 498 499 500 501 502 503 504 505 506 507 508 509 | nByte = nArg * (sizeof(char*) + sizeof(u8)); pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte); pRet->abUnindexed = (u8*)&pRet->azCol[nArg]; pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); pRet->bColumnsize = 1; #ifdef SQLITE_DEBUG pRet->bPrefixIndex = 1; #endif if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){ *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName); rc = SQLITE_ERROR; } | > | 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 | nByte = nArg * (sizeof(char*) + sizeof(u8)); pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte); pRet->abUnindexed = (u8*)&pRet->azCol[nArg]; pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); pRet->bColumnsize = 1; pRet->eDetail = FTS5_DETAIL_FULL; #ifdef SQLITE_DEBUG pRet->bPrefixIndex = 1; #endif if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){ *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName); rc = SQLITE_ERROR; } |
︙ | ︙ |
Changes to ext/fts5/fts5_expr.c.
︙ | ︙ | |||
36 37 38 39 40 41 42 43 44 45 46 47 48 49 | #include <stdio.h> void sqlite3Fts5ParserTrace(FILE*, char*); #endif struct Fts5Expr { Fts5Index *pIndex; Fts5ExprNode *pRoot; int bDesc; /* Iterate in descending rowid order */ int nPhrase; /* Number of phrases in expression */ Fts5ExprPhrase **apExprPhrase; /* Pointers to phrase objects */ }; /* | > | 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | #include <stdio.h> void sqlite3Fts5ParserTrace(FILE*, char*); #endif struct Fts5Expr { Fts5Index *pIndex; Fts5Config *pConfig; Fts5ExprNode *pRoot; int bDesc; /* Iterate in descending rowid order */ int nPhrase; /* Number of phrases in expression */ Fts5ExprPhrase **apExprPhrase; /* Pointers to phrase objects */ }; /* |
︙ | ︙ | |||
231 232 233 234 235 236 237 238 239 240 241 242 243 244 | *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr)); if( pNew==0 ){ sParse.rc = SQLITE_NOMEM; sqlite3Fts5ParseNodeFree(sParse.pExpr); }else{ pNew->pRoot = sParse.pExpr; pNew->pIndex = 0; pNew->apExprPhrase = sParse.apPhrase; pNew->nPhrase = sParse.nPhrase; sParse.apPhrase = 0; } } sqlite3_free(sParse.apPhrase); | > | 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 | *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr)); if( pNew==0 ){ sParse.rc = SQLITE_NOMEM; sqlite3Fts5ParseNodeFree(sParse.pExpr); }else{ pNew->pRoot = sParse.pExpr; pNew->pIndex = 0; pNew->pConfig = pConfig; pNew->apExprPhrase = sParse.apPhrase; pNew->nPhrase = sParse.nPhrase; sParse.apPhrase = 0; } } sqlite3_free(sParse.apPhrase); |
︙ | ︙ | |||
295 296 297 298 299 300 301 | if( pbEof && bRetValid==0 ) *pbEof = 1; return iRet; } /* ** Argument pTerm must be a synonym iterator. */ | | > > > > > | | > > > | 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 | if( pbEof && bRetValid==0 ) *pbEof = 1; return iRet; } /* ** Argument pTerm must be a synonym iterator. */ static int fts5ExprSynonymList( Fts5ExprTerm *pTerm, int bCollist, Fts5Colset *pColset, i64 iRowid, int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */ u8 **pa, int *pn ){ Fts5PoslistReader aStatic[4]; Fts5PoslistReader *aIter = aStatic; int nIter = 0; int nAlloc = 4; int rc = SQLITE_OK; Fts5ExprTerm *p; assert( pTerm->pSynonym ); for(p=pTerm; p; p=p->pSynonym){ Fts5IndexIter *pIter = p->pIter; if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){ const u8 *a; int n; if( bCollist ){ rc = sqlite3Fts5IterCollist(pIter, &a, &n); }else{ i64 dummy; rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy); } if( rc!=SQLITE_OK ) goto synonym_poslist_out; if( n==0 ) continue; if( nIter==nAlloc ){ int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte); if( aNew==0 ){ rc = SQLITE_NOMEM; goto synonym_poslist_out; } |
︙ | ︙ | |||
418 419 420 421 422 423 424 | for(i=0; i<pPhrase->nTerm; i++){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; i64 dummy; int n = 0; int bFlag = 0; const u8 *a = 0; if( pTerm->pSynonym ){ | | | | 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 | for(i=0; i<pPhrase->nTerm; i++){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; i64 dummy; int n = 0; int bFlag = 0; const u8 *a = 0; if( pTerm->pSynonym ){ rc = fts5ExprSynonymList( pTerm, 0, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n ); }else{ rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy); } if( rc!=SQLITE_OK ) goto ismatch_out; sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); aIter[i].bFlag = (u8)bFlag; |
︙ | ︙ | |||
753 754 755 756 757 758 759 | static int fts5ExprNearTest( int *pRc, Fts5Expr *pExpr, /* Expression that pNear is a part of */ Fts5ExprNode *pNode /* The "NEAR" node (FTS5_STRING) */ ){ Fts5ExprNearset *pNear = pNode->pNear; int rc = *pRc; | > > > > > > > > > > > > > > > > > > > > > | | | | | | | | | | | | | | | | | | | | < | > | 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 | static int fts5ExprNearTest( int *pRc, Fts5Expr *pExpr, /* Expression that pNear is a part of */ Fts5ExprNode *pNode /* The "NEAR" node (FTS5_STRING) */ ){ Fts5ExprNearset *pNear = pNode->pNear; int rc = *pRc; if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL ){ Fts5ExprTerm *pTerm; Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; pPhrase->poslist.n = 0; for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ Fts5IndexIter *pIter = pTerm->pIter; if( sqlite3Fts5IterEof(pIter)==0 ){ int n; i64 iRowid; rc = sqlite3Fts5IterPoslist(pIter, pNear->pColset, 0, &n, &iRowid); if( rc!=SQLITE_OK ){ *pRc = rc; return 0; }else if( iRowid==pNode->iRowid && n>0 ){ pPhrase->poslist.n = 1; } } } return pPhrase->poslist.n; }else{ int i; /* Check that each phrase in the nearset matches the current row. ** Populate the pPhrase->poslist buffers at the same time. If any ** phrase is not a match, break out of the loop early. */ for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ int bMatch = 0; rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch); if( bMatch==0 ) break; }else{ rc = sqlite3Fts5IterPoslistBuffer( pPhrase->aTerm[0].pIter, &pPhrase->poslist ); } } *pRc = rc; if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){ return 1; } return 0; } } static int fts5ExprTokenTest( Fts5Expr *pExpr, /* Expression that pNear is a part of */ Fts5ExprNode *pNode /* The "NEAR" node (FTS5_TERM) */ ){ /* As this "NEAR" object is actually a single phrase that consists |
︙ | ︙ | |||
1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 | } assert( rc!=SQLITE_OK || cmp<=0 ); if( cmp || p2->bNomatch ) break; rc = fts5ExprNodeNext(pExpr, p1, 0, 0); } pNode->bEof = p1->bEof; pNode->iRowid = p1->iRowid; break; } } } return rc; } | > > > | 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 | } assert( rc!=SQLITE_OK || cmp<=0 ); if( cmp || p2->bNomatch ) break; rc = fts5ExprNodeNext(pExpr, p1, 0, 0); } pNode->bEof = p1->bEof; pNode->iRowid = p1->iRowid; if( p1->bEof ){ fts5ExprNodeZeroPoslist(p2); } break; } } } return rc; } |
︙ | ︙ | |||
1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 | sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; } } if( rc==SQLITE_OK ){ /* All the allocations succeeded. Put the expression object together. */ pNew->pIndex = pExpr->pIndex; pNew->nPhrase = 1; pNew->apExprPhrase[0] = sCtx.pPhrase; pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase; pNew->pRoot->pNear->nPhrase = 1; sCtx.pPhrase->pNode = pNew->pRoot; if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){ | > | 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 | sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; } } if( rc==SQLITE_OK ){ /* All the allocations succeeded. Put the expression object together. */ pNew->pIndex = pExpr->pIndex; pNew->pConfig = pExpr->pConfig; pNew->nPhrase = 1; pNew->apExprPhrase[0] = sCtx.pPhrase; pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase; pNew->pRoot->pNear->nPhrase = 1; sCtx.pPhrase->pNode = pNew->pRoot; if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){ |
︙ | ︙ | |||
1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 | } void sqlite3Fts5ParseSetColset( Fts5Parse *pParse, Fts5ExprNearset *pNear, Fts5Colset *pColset ){ if( pNear ){ pNear->pColset = pColset; }else{ sqlite3_free(pColset); } } | > > > > > > > > > | 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 | } void sqlite3Fts5ParseSetColset( Fts5Parse *pParse, Fts5ExprNearset *pNear, Fts5Colset *pColset ){ if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){ pParse->rc = SQLITE_ERROR; pParse->zErr = sqlite3_mprintf( "fts5: column queries are not supported (detail=none)" ); sqlite3_free(pColset); return; } if( pNear ){ pNear->pColset = pColset; }else{ sqlite3_free(pColset); } } |
︙ | ︙ | |||
1801 1802 1803 1804 1805 1806 1807 | pRet->eType = eType; pRet->pNear = pNear; if( eType==FTS5_STRING ){ int iPhrase; for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){ pNear->apPhrase[iPhrase]->pNode = pRet; } | | < | < | > > > > > > > > > > > | 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 | pRet->eType = eType; pRet->pNear = pNear; if( eType==FTS5_STRING ){ int iPhrase; for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){ pNear->apPhrase[iPhrase]->pNode = pRet; } if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){ if( pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){ pRet->eType = FTS5_TERM; } }else if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){ assert( pParse->rc==SQLITE_OK ); pParse->rc = SQLITE_ERROR; assert( pParse->zErr==0 ); pParse->zErr = sqlite3_mprintf( "fts5: %s queries are not supported (detail!=full)", pNear->nPhrase==1 ? "phrase": "NEAR" ); sqlite3_free(pRet); pRet = 0; } }else{ fts5ExprAddChildren(pRet, pLeft); fts5ExprAddChildren(pRet, pRight); } } } |
︙ | ︙ | |||
1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 | for(i=0; i<pNear->nPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; zRet = fts5PrintfAppend(zRet, " {"); for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){ char *zTerm = pPhrase->aTerm[iTerm].zTerm; zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm); } if( zRet ) zRet = fts5PrintfAppend(zRet, "}"); if( zRet==0 ) return 0; } }else{ | > > > | 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 | for(i=0; i<pNear->nPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; zRet = fts5PrintfAppend(zRet, " {"); for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){ char *zTerm = pPhrase->aTerm[iTerm].zTerm; zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm); if( pPhrase->aTerm[iTerm].bPrefix ){ zRet = fts5PrintfAppend(zRet, "*"); } } if( zRet ) zRet = fts5PrintfAppend(zRet, "}"); if( zRet==0 ) return 0; } }else{ |
︙ | ︙ | |||
2231 2232 2233 2234 2235 2236 2237 | nRet = pPhrase->poslist.n; }else{ *pa = 0; nRet = 0; } return nRet; } | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 | nRet = pPhrase->poslist.n; }else{ *pa = 0; nRet = 0; } return nRet; } struct Fts5PoslistPopulator { Fts5PoslistWriter writer; int bOk; /* True if ok to populate */ int bMiss; }; Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){ Fts5PoslistPopulator *pRet; pRet = sqlite3_malloc(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); if( pRet ){ int i; memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); for(i=0; i<pExpr->nPhrase; i++){ Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist; Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; assert( pExpr->apExprPhrase[i]->nTerm==1 ); if( bLive && (pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof) ){ pRet[i].bMiss = 1; }else{ pBuf->n = 0; } } } return pRet; } struct Fts5ExprCtx { Fts5Expr *pExpr; Fts5PoslistPopulator *aPopulator; i64 iOff; }; typedef struct Fts5ExprCtx Fts5ExprCtx; /* ** TODO: Make this more efficient! */ static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){ int i; for(i=0; i<pColset->nCol; i++){ if( pColset->aiCol[i]==iCol ) return 1; } return 0; } static int fts5ExprPopulatePoslistsCb( void *pCtx, /* Copy of 2nd argument to xTokenize() */ int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Pointer to buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Byte offset of token within input text */ int iEnd /* Byte offset of end of token within input text */ ){ Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx; Fts5Expr *pExpr = p->pExpr; int i; if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; for(i=0; i<pExpr->nPhrase; i++){ Fts5ExprTerm *pTerm; if( p->aPopulator[i].bOk==0 ) continue; for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ int nTerm = strlen(pTerm->zTerm); if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix)) && memcmp(pTerm->zTerm, pToken, nTerm)==0 ){ int rc = sqlite3Fts5PoslistWriterAppend( &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff ); if( rc ) return rc; break; } } } return SQLITE_OK; } int sqlite3Fts5ExprPopulatePoslists( Fts5Config *pConfig, Fts5Expr *pExpr, Fts5PoslistPopulator *aPopulator, int iCol, const char *z, int n ){ int i; Fts5ExprCtx sCtx; sCtx.pExpr = pExpr; sCtx.aPopulator = aPopulator; sCtx.iOff = (((i64)iCol) << 32) - 1; for(i=0; i<pExpr->nPhrase; i++){ Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; Fts5Colset *pColset = pNode->pNear->pColset; if( (pColset && 0==fts5ExprColsetTest(pColset, iCol)) || aPopulator[i].bMiss ){ aPopulator[i].bOk = 0; }else{ aPopulator[i].bOk = 1; } } return sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb ); } static void fts5ExprClearPoslists(Fts5ExprNode *pNode){ if( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING ){ pNode->pNear->apPhrase[0]->poslist.n = 0; }else{ int i; for(i=0; i<pNode->nChild; i++){ fts5ExprClearPoslists(pNode->apChild[i]); } } } static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ if( pNode ){ pNode->iRowid = iRowid; pNode->bEof = 0; switch( pNode->eType ){ case FTS5_TERM: case FTS5_STRING: return (pNode->pNear->apPhrase[0]->poslist.n>0); case FTS5_AND: { int i; for(i=0; i<pNode->nChild; i++){ if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){ fts5ExprClearPoslists(pNode); return 0; } } break; } case FTS5_OR: { int i; int bRet = 0; for(i=0; i<pNode->nChild; i++){ if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){ bRet = 1; } } if( bRet==0 ){ fts5ExprClearPoslists(pNode); } return bRet; } default: { assert( pNode->eType==FTS5_NOT ); if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid) || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid) ){ fts5ExprClearPoslists(pNode); return 0; } break; } } } return 1; } void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){ fts5ExprCheckPoslists(pExpr->pRoot, iRowid); } static void fts5ExprClearEof(Fts5ExprNode *pNode){ int i; for(i=0; i<pNode->nChild; i++){ fts5ExprClearEof(pNode->apChild[i]); } pNode->bEof = 0; } void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){ fts5ExprClearEof(pExpr->pRoot); } /* ** This function is only called for detail=columns tables. */ int sqlite3Fts5ExprPhraseCollist( Fts5Expr *pExpr, int iPhrase, const u8 **ppCollist, int *pnCollist ){ Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase]; Fts5ExprNode *pNode = pPhrase->pNode; int rc = SQLITE_OK; assert( iPhrase>=0 && iPhrase<pExpr->nPhrase ); if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid && pPhrase->poslist.n>0 ){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[0]; if( pTerm->pSynonym ){ int bDel = 0; u8 *a; rc = fts5ExprSynonymList( pTerm, 1, 0, pNode->iRowid, &bDel, &a, pnCollist ); if( bDel ){ sqlite3Fts5BufferSet(&rc, &pPhrase->poslist, *pnCollist, a); *ppCollist = pPhrase->poslist.p; sqlite3_free(a); }else{ *ppCollist = a; } }else{ sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist); } }else{ *ppCollist = 0; *pnCollist = 0; } return rc; } |
Changes to ext/fts5/fts5_hash.c.
︙ | ︙ | |||
22 23 24 25 26 27 28 29 30 31 32 33 34 35 | ** This file contains the implementation of an in-memory hash table used ** to accumuluate "term -> doclist" content before it is flused to a level-0 ** segment. */ struct Fts5Hash { int *pnByte; /* Pointer to bytes counter */ int nEntry; /* Number of entries currently in hash */ int nSlot; /* Size of aSlot[] array */ Fts5HashEntry *pScan; /* Current ordered scan item */ Fts5HashEntry **aSlot; /* Array of hash slots */ }; | > | 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | ** This file contains the implementation of an in-memory hash table used ** to accumuluate "term -> doclist" content before it is flused to a level-0 ** segment. */ struct Fts5Hash { int eDetail; /* Copy of Fts5Config.eDetail */ int *pnByte; /* Pointer to bytes counter */ int nEntry; /* Number of entries currently in hash */ int nSlot; /* Size of aSlot[] array */ Fts5HashEntry *pScan; /* Current ordered scan item */ Fts5HashEntry **aSlot; /* Array of hash slots */ }; |
︙ | ︙ | |||
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | Fts5HashEntry *pHashNext; /* Next hash entry with same hash-key */ Fts5HashEntry *pScanNext; /* Next entry in sorted order */ int nAlloc; /* Total size of allocation */ int iSzPoslist; /* Offset of space for 4-byte poslist size */ int nData; /* Total bytes of data (incl. structure) */ u8 bDel; /* Set delete-flag @ iSzPoslist */ int iCol; /* Column of last value written */ int iPos; /* Position of last value written */ i64 iRowid; /* Rowid of last value written */ char zKey[8]; /* Nul-terminated entry key */ }; /* ** Size of Fts5HashEntry without the zKey[] array. */ #define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8) /* ** Allocate a new hash table. */ | > | > | 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 | Fts5HashEntry *pHashNext; /* Next hash entry with same hash-key */ Fts5HashEntry *pScanNext; /* Next entry in sorted order */ int nAlloc; /* Total size of allocation */ int iSzPoslist; /* Offset of space for 4-byte poslist size */ int nData; /* Total bytes of data (incl. structure) */ u8 bDel; /* Set delete-flag @ iSzPoslist */ u8 bContent; /* Set content-flag (detail=none mode) */ int iCol; /* Column of last value written */ int iPos; /* Position of last value written */ i64 iRowid; /* Rowid of last value written */ char zKey[8]; /* Nul-terminated entry key */ }; /* ** Size of Fts5HashEntry without the zKey[] array. */ #define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8) /* ** Allocate a new hash table. */ int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){ int rc = SQLITE_OK; Fts5Hash *pNew; *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ int nByte; memset(pNew, 0, sizeof(Fts5Hash)); pNew->pnByte = pnByte; pNew->eDetail = pConfig->eDetail; pNew->nSlot = 1024; nByte = sizeof(Fts5HashEntry*) * pNew->nSlot; pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc(nByte); if( pNew->aSlot==0 ){ sqlite3_free(pNew); *ppNew = 0; |
︙ | ︙ | |||
178 179 180 181 182 183 184 | sqlite3_free(apOld); pHash->nSlot = nNew; pHash->aSlot = apNew; return SQLITE_OK; } | | > > > > > > > > > | | | | | | | | | | | > | > > > > > > > > > > > > > > > > < < < < > > > > > | > > > | > > > > | | | | | | | | | | | | | | | | | | | | < | > > > > | > > > | | | < | | > > > > | | | > | | | | > > > | | > | > | | > > < > | 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 | sqlite3_free(apOld); pHash->nSlot = nNew; pHash->aSlot = apNew; return SQLITE_OK; } static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){ if( p->iSzPoslist ){ u8 *pPtr = (u8*)p; if( pHash->eDetail==FTS5_DETAIL_NONE ){ assert( p->nData==p->iSzPoslist ); if( p->bDel ){ pPtr[p->nData++] = 0x00; if( p->bContent ){ pPtr[p->nData++] = 0x00; } } }else{ int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */ int nPos = nSz*2 + p->bDel; /* Value of nPos field */ assert( p->bDel==0 || p->bDel==1 ); if( nPos<=127 ){ pPtr[p->iSzPoslist] = (u8)nPos; }else{ int nByte = sqlite3Fts5GetVarintLen((u32)nPos); memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz); sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos); p->nData += (nByte-1); } } p->iSzPoslist = 0; p->bDel = 0; p->bContent = 0; } } /* ** Add an entry to the in-memory hash table. The key is the concatenation ** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos). ** ** (bByte || pToken) -> (iRowid,iCol,iPos) ** ** Or, if iCol is negative, then the value is a delete marker. */ int sqlite3Fts5HashWrite( Fts5Hash *pHash, i64 iRowid, /* Rowid for this entry */ int iCol, /* Column token appears in (-ve -> delete) */ int iPos, /* Position of token within column */ char bByte, /* First byte of token */ const char *pToken, int nToken /* Token to add or remove to or from index */ ){ unsigned int iHash; Fts5HashEntry *p; u8 *pPtr; int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ int bNew; /* If non-delete entry should be written */ bNew = (pHash->eDetail==FTS5_DETAIL_FULL); /* Attempt to locate an existing hash entry */ iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ if( p->zKey[0]==bByte && memcmp(&p->zKey[1], pToken, nToken)==0 && p->zKey[nToken+1]==0 ){ break; } } /* If an existing hash entry cannot be found, create a new one. */ if( p==0 ){ /* Figure out how much space to allocate */ int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64; if( nByte<128 ) nByte = 128; /* Grow the Fts5Hash.aSlot[] array if necessary. */ if( (pHash->nEntry*2)>=pHash->nSlot ){ int rc = fts5HashResize(pHash); if( rc!=SQLITE_OK ) return rc; iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); } /* Allocate new Fts5HashEntry and add it to the hash table. */ p = (Fts5HashEntry*)sqlite3_malloc(nByte); if( !p ) return SQLITE_NOMEM; memset(p, 0, FTS5_HASHENTRYSIZE); p->nAlloc = nByte; p->zKey[0] = bByte; memcpy(&p->zKey[1], pToken, nToken); assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) ); p->zKey[nToken+1] = '\0'; p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE; p->pHashNext = pHash->aSlot[iHash]; pHash->aSlot[iHash] = p; pHash->nEntry++; /* Add the first rowid field to the hash-entry */ p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); p->iRowid = iRowid; p->iSzPoslist = p->nData; if( pHash->eDetail!=FTS5_DETAIL_NONE ){ p->nData += 1; p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); } nIncr += p->nData; }else{ /* Appending to an existing hash-entry. Check that there is enough ** space to append the largest possible new entry. Worst case scenario ** is: ** ** + 9 bytes for a new rowid, ** + 4 byte reserved for the "poslist size" varint. ** + 1 byte for a "new column" byte, ** + 3 bytes for a new column number (16-bit max) as a varint, ** + 5 bytes for the new position offset (32-bit max). */ if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){ int nNew = p->nAlloc * 2; Fts5HashEntry *pNew; Fts5HashEntry **pp; pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew); if( pNew==0 ) return SQLITE_NOMEM; pNew->nAlloc = nNew; for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext); *pp = pNew; p = pNew; } nIncr -= p->nData; } assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) ); pPtr = (u8*)p; /* If this is a new rowid, append the 4-byte size field for the previous ** entry, and the new rowid for this entry. */ if( iRowid!=p->iRowid ){ fts5HashAddPoslistSize(pHash, p); p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid); p->iRowid = iRowid; bNew = 1; p->iSzPoslist = p->nData; if( pHash->eDetail!=FTS5_DETAIL_NONE ){ p->nData += 1; p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); p->iPos = 0; } } if( iCol>=0 ){ if( pHash->eDetail==FTS5_DETAIL_NONE ){ p->bContent = 1; }else{ /* Append a new column value, if necessary */ assert( iCol>=p->iCol ); if( iCol!=p->iCol ){ if( pHash->eDetail==FTS5_DETAIL_FULL ){ pPtr[p->nData++] = 0x01; p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); p->iCol = iCol; p->iPos = 0; }else{ bNew = 1; p->iCol = iPos = iCol; } } /* Append the new position offset, if necessary */ if( bNew ){ p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2); p->iPos = iPos; } } }else{ /* This is a delete. Set the delete flag. */ p->bDel = 1; } nIncr += p->nData; *pHash->pnByte += nIncr; return SQLITE_OK; } /* ** Arguments pLeft and pRight point to linked-lists of hash-entry objects, |
︙ | ︙ | |||
419 420 421 422 423 424 425 | Fts5HashEntry *p; for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break; } if( p ){ | | | 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 | Fts5HashEntry *p; for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break; } if( p ){ fts5HashAddPoslistSize(pHash, p); *ppDoclist = (const u8*)&p->zKey[nTerm+1]; *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); }else{ *ppDoclist = 0; *pnDoclist = 0; } |
︙ | ︙ | |||
455 456 457 458 459 460 461 | const char **pzTerm, /* OUT: term (nul-terminated) */ const u8 **ppDoclist, /* OUT: pointer to doclist */ int *pnDoclist /* OUT: size of doclist in bytes */ ){ Fts5HashEntry *p; if( (p = pHash->pScan) ){ int nTerm = (int)strlen(p->zKey); | | | 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 | const char **pzTerm, /* OUT: term (nul-terminated) */ const u8 **ppDoclist, /* OUT: pointer to doclist */ int *pnDoclist /* OUT: size of doclist in bytes */ ){ Fts5HashEntry *p; if( (p = pHash->pScan) ){ int nTerm = (int)strlen(p->zKey); fts5HashAddPoslistSize(pHash, p); *pzTerm = p->zKey; *ppDoclist = (const u8*)&p->zKey[nTerm+1]; *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); }else{ *pzTerm = 0; *ppDoclist = 0; *pnDoclist = 0; } } |
Changes to ext/fts5/fts5_index.c.
︙ | ︙ | |||
429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 | Fts5StructureSegment *pSeg; /* Segment to iterate through */ int flags; /* Mask of configuration flags */ int iLeafPgno; /* Current leaf page number */ Fts5Data *pLeaf; /* Current leaf data */ Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ int iLeafOffset; /* Byte offset within current leaf */ /* The page and offset from which the current term was read. The offset ** is the offset of the first rowid in the current doclist. */ int iTermLeafPgno; int iTermLeafOffset; int iPgidxOff; /* Next offset in pgidx */ int iEndofDoclist; /* The following are only used if the FTS5_SEGITER_REVERSE flag is set. */ int iRowidOffset; /* Current entry in aRowidOffset[] */ int nRowidOffset; /* Allocated size of aRowidOffset[] array */ int *aRowidOffset; /* Array of offset to rowid fields */ Fts5DlidxIter *pDlidx; /* If there is a doclist-index */ /* Variables populated based on current entry. */ Fts5Buffer term; /* Current term */ i64 iRowid; /* Current rowid */ int nPos; /* Number of bytes in current position list */ | > > > | < | 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 | Fts5StructureSegment *pSeg; /* Segment to iterate through */ int flags; /* Mask of configuration flags */ int iLeafPgno; /* Current leaf page number */ Fts5Data *pLeaf; /* Current leaf data */ Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ int iLeafOffset; /* Byte offset within current leaf */ /* Next method */ void (*xNext)(Fts5Index*, Fts5SegIter*, int*); /* The page and offset from which the current term was read. The offset ** is the offset of the first rowid in the current doclist. */ int iTermLeafPgno; int iTermLeafOffset; int iPgidxOff; /* Next offset in pgidx */ int iEndofDoclist; /* The following are only used if the FTS5_SEGITER_REVERSE flag is set. */ int iRowidOffset; /* Current entry in aRowidOffset[] */ int nRowidOffset; /* Allocated size of aRowidOffset[] array */ int *aRowidOffset; /* Array of offset to rowid fields */ Fts5DlidxIter *pDlidx; /* If there is a doclist-index */ /* Variables populated based on current entry. */ Fts5Buffer term; /* Current term */ i64 iRowid; /* Current rowid */ int nPos; /* Number of bytes in current position list */ u8 bDel; /* True if the delete flag is set */ }; /* ** Argument is a pointer to an Fts5Data structure that contains a ** leaf page. */ #define ASSERT_SZLEAF_OK(x) assert( \ (x)->szLeaf==(x)->nn || (x)->szLeaf==fts5GetU16(&(x)->p[2]) \ ) #define FTS5_SEGITER_ONETERM 0x01 #define FTS5_SEGITER_REVERSE 0x02 /* ** Argument is a pointer to an Fts5Data structure that contains a leaf ** page. This macro evaluates to true if the leaf contains no terms, or ** false if it contains at least one term. */ #define fts5LeafIsTermless(x) ((x)->szLeaf >= (x)->nn) |
︙ | ︙ | |||
1488 1489 1490 1491 1492 1493 1494 | ** ** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the ** position list content (if any). */ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ if( p->rc==SQLITE_OK ){ int iOff = pIter->iLeafOffset; /* Offset to read at */ | < > > > > > > > > > > > > > > > > | | | < | > > | 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 | ** ** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the ** position list content (if any). */ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ if( p->rc==SQLITE_OK ){ int iOff = pIter->iLeafOffset; /* Offset to read at */ ASSERT_SZLEAF_OK(pIter->pLeaf); if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ int iEod = MIN(pIter->iEndofDoclist, pIter->pLeaf->szLeaf); pIter->bDel = 0; pIter->nPos = 1; if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){ pIter->bDel = 1; iOff++; if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){ pIter->nPos = 1; iOff++; }else{ pIter->nPos = 0; } } }else{ int nSz; fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz); pIter->bDel = (nSz & 0x0001); pIter->nPos = nSz>>1; assert_nc( pIter->nPos>=0 ); } pIter->iLeafOffset = iOff; } } static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ int iOff = pIter->iLeafOffset; |
︙ | ︙ | |||
1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 | int nExtra; pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra); pIter->iEndofDoclist += nExtra; } fts5SegIterLoadRowid(p, pIter); } /* ** Initialize the iterator object pIter to iterate through the entries in ** segment pSeg. The iterator is left pointing to the first entry when ** this function returns. ** ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If | > > > > > > > > > > > > > > | 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 | int nExtra; pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra); pIter->iEndofDoclist += nExtra; } fts5SegIterLoadRowid(p, pIter); } static void fts5SegIterNext(Fts5Index*, Fts5SegIter*, int*); static void fts5SegIterNext_Reverse(Fts5Index*, Fts5SegIter*, int*); static void fts5SegIterNext_None(Fts5Index*, Fts5SegIter*, int*); static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ if( pIter->flags & FTS5_SEGITER_REVERSE ){ pIter->xNext = fts5SegIterNext_Reverse; }else if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ pIter->xNext = fts5SegIterNext_None; }else{ pIter->xNext = fts5SegIterNext; } } /* ** Initialize the iterator object pIter to iterate through the entries in ** segment pSeg. The iterator is left pointing to the first entry when ** this function returns. ** ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If |
︙ | ︙ | |||
1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 | ** at EOF already. */ assert( pIter->pLeaf==0 ); return; } if( p->rc==SQLITE_OK ){ memset(pIter, 0, sizeof(*pIter)); pIter->pSeg = pSeg; pIter->iLeafPgno = pSeg->pgnoFirst-1; fts5SegIterNextPage(p, pIter); } if( p->rc==SQLITE_OK ){ pIter->iLeafOffset = 4; | > | 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 | ** at EOF already. */ assert( pIter->pLeaf==0 ); return; } if( p->rc==SQLITE_OK ){ memset(pIter, 0, sizeof(*pIter)); fts5SegIterSetNext(p, pIter); pIter->pSeg = pSeg; pIter->iLeafPgno = pSeg->pgnoFirst-1; fts5SegIterNextPage(p, pIter); } if( p->rc==SQLITE_OK ){ pIter->iLeafOffset = 4; |
︙ | ︙ | |||
1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 | ** This function advances the iterator so that it points to the last ** relevant rowid on the page and, if necessary, initializes the ** aRowidOffset[] and iRowidOffset variables. At this point the iterator ** is in its regular state - Fts5SegIter.iLeafOffset points to the first ** byte of the position list content associated with said rowid. */ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ int n = pIter->pLeaf->szLeaf; int i = pIter->iLeafOffset; u8 *a = pIter->pLeaf->p; int iRowidOffset = 0; if( n>pIter->iEndofDoclist ){ n = pIter->iEndofDoclist; } ASSERT_SZLEAF_OK(pIter->pLeaf); while( 1 ){ i64 iDelta = 0; | > > > > > > > > > | | < | | > > | 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 | ** This function advances the iterator so that it points to the last ** relevant rowid on the page and, if necessary, initializes the ** aRowidOffset[] and iRowidOffset variables. At this point the iterator ** is in its regular state - Fts5SegIter.iLeafOffset points to the first ** byte of the position list content associated with said rowid. */ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ int eDetail = p->pConfig->eDetail; int n = pIter->pLeaf->szLeaf; int i = pIter->iLeafOffset; u8 *a = pIter->pLeaf->p; int iRowidOffset = 0; if( n>pIter->iEndofDoclist ){ n = pIter->iEndofDoclist; } ASSERT_SZLEAF_OK(pIter->pLeaf); while( 1 ){ i64 iDelta = 0; if( eDetail==FTS5_DETAIL_NONE ){ /* todo */ if( i<n && a[i]==0 ){ i++; if( i<n && a[i]==0 ) i++; } }else{ int nPos; int bDummy; i += fts5GetPoslistSize(&a[i], &nPos, &bDummy); i += nPos; } if( i>=n ) break; i += fts5GetVarint(&a[i], (u64*)&iDelta); pIter->iRowid += iDelta; /* If necessary, grow the pIter->aRowidOffset[] array. */ if( iRowidOffset>=pIter->nRowidOffset ){ int nNew = pIter->nRowidOffset + 8; int *aNew = (int*)sqlite3_realloc(pIter->aRowidOffset, nNew*sizeof(int)); if( aNew==0 ){ p->rc = SQLITE_NOMEM; break; } |
︙ | ︙ | |||
1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 | ** points to a delete marker. A delete marker is an entry with a 0 byte ** position-list. */ static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5IndexIter *pIter){ Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0); } /* ** Advance iterator pIter to the next entry. ** ** If an error occurs, Fts5Index.rc is set to an appropriate error code. It ** is not considered an error if the iterator reaches EOF. If an error has ** already occurred when this function is called, it is a no-op. */ static void fts5SegIterNext( Fts5Index *p, /* FTS5 backend object */ Fts5SegIter *pIter, /* Iterator to advance */ int *pbNewTerm /* OUT: Set for new term */ ){ | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > < < < < < < < < < < < < < < < < < < < < < < | | | | > > > | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | > | | | | | | | | | | | | | | | | | | | | | > > > > | | | | | | < < > > > > > | 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 | ** points to a delete marker. A delete marker is an entry with a 0 byte ** position-list. */ static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5IndexIter *pIter){ Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0); } /* ** Advance iterator pIter to the next entry. ** ** This version of fts5SegIterNext() is only used by reverse iterators. */ static void fts5SegIterNext_Reverse( Fts5Index *p, /* FTS5 backend object */ Fts5SegIter *pIter, /* Iterator to advance */ int *pbNewTerm /* OUT: Set for new term */ ){ assert( pIter->flags & FTS5_SEGITER_REVERSE ); assert( pIter->pNextLeaf==0 ); if( pIter->iRowidOffset>0 ){ u8 *a = pIter->pLeaf->p; int iOff; i64 iDelta; pIter->iRowidOffset--; pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset]; fts5SegIterLoadNPos(p, pIter); iOff = pIter->iLeafOffset; if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){ iOff += pIter->nPos; } fts5GetVarint(&a[iOff], (u64*)&iDelta); pIter->iRowid -= iDelta; }else{ fts5SegIterReverseNewPage(p, pIter); } } /* ** Advance iterator pIter to the next entry. ** ** This version of fts5SegIterNext() is only used if detail=none and the ** iterator is not a reverse direction iterator. */ static void fts5SegIterNext_None( Fts5Index *p, /* FTS5 backend object */ Fts5SegIter *pIter, /* Iterator to advance */ int *pbNewTerm /* OUT: Set for new term */ ){ int iOff; assert( p->rc==SQLITE_OK ); assert( (pIter->flags & FTS5_SEGITER_REVERSE)==0 ); assert( p->pConfig->eDetail==FTS5_DETAIL_NONE ); ASSERT_SZLEAF_OK(pIter->pLeaf); iOff = pIter->iLeafOffset; /* Next entry is on the next page */ if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( p->rc || pIter->pLeaf==0 ) return; pIter->iRowid = 0; iOff = 4; } if( iOff<pIter->iEndofDoclist ){ /* Next entry is on the current page */ i64 iDelta; iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta); pIter->iLeafOffset = iOff; pIter->iRowid += iDelta; }else if( (pIter->flags & FTS5_SEGITER_ONETERM)==0 ){ if( pIter->pSeg ){ int nKeep = 0; if( iOff!=fts5LeafFirstTermOff(pIter->pLeaf) ){ iOff += fts5GetVarint32(&pIter->pLeaf->p[iOff], nKeep); } pIter->iLeafOffset = iOff; fts5SegIterLoadTerm(p, pIter, nKeep); }else{ const u8 *pList = 0; const char *zTerm = 0; int nList; sqlite3Fts5HashScanNext(p->pHash); sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); if( pList==0 ) goto next_none_eof; pIter->pLeaf->p = (u8*)pList; pIter->pLeaf->nn = nList; pIter->pLeaf->szLeaf = nList; pIter->iEndofDoclist = nList; sqlite3Fts5BufferSet(&p->rc,&pIter->term, (int)strlen(zTerm), (u8*)zTerm); pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); } if( pbNewTerm ) *pbNewTerm = 1; }else{ goto next_none_eof; } fts5SegIterLoadNPos(p, pIter); return; next_none_eof: fts5DataRelease(pIter->pLeaf); pIter->pLeaf = 0; } /* ** Advance iterator pIter to the next entry. ** ** If an error occurs, Fts5Index.rc is set to an appropriate error code. It ** is not considered an error if the iterator reaches EOF. If an error has ** already occurred when this function is called, it is a no-op. */ static void fts5SegIterNext( Fts5Index *p, /* FTS5 backend object */ Fts5SegIter *pIter, /* Iterator to advance */ int *pbNewTerm /* OUT: Set for new term */ ){ Fts5Data *pLeaf = pIter->pLeaf; int iOff; int bNewTerm = 0; int nKeep = 0; assert( pbNewTerm==0 || *pbNewTerm==0 ); assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); /* Search for the end of the position list within the current page. */ u8 *a = pLeaf->p; int n = pLeaf->szLeaf; ASSERT_SZLEAF_OK(pLeaf); iOff = pIter->iLeafOffset + pIter->nPos; if( iOff<n ){ /* The next entry is on the current page. */ assert_nc( iOff<=pIter->iEndofDoclist ); if( iOff>=pIter->iEndofDoclist ){ bNewTerm = 1; if( iOff!=fts5LeafFirstTermOff(pLeaf) ){ iOff += fts5GetVarint32(&a[iOff], nKeep); } }else{ u64 iDelta; iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta); pIter->iRowid += iDelta; assert_nc( iDelta>0 ); } pIter->iLeafOffset = iOff; }else if( pIter->pSeg==0 ){ const u8 *pList = 0; const char *zTerm = 0; int nList = 0; assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm ); if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ sqlite3Fts5HashScanNext(p->pHash); sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); } if( pList==0 ){ fts5DataRelease(pIter->pLeaf); pIter->pLeaf = 0; }else{ pIter->pLeaf->p = (u8*)pList; pIter->pLeaf->nn = nList; pIter->pLeaf->szLeaf = nList; pIter->iEndofDoclist = nList+1; sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm), (u8*)zTerm); pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); *pbNewTerm = 1; } }else{ iOff = 0; /* Next entry is not on the current page */ while( iOff==0 ){ fts5SegIterNextPage(p, pIter); pLeaf = pIter->pLeaf; if( pLeaf==0 ) break; ASSERT_SZLEAF_OK(pLeaf); if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOff<pLeaf->szLeaf ){ iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); pIter->iLeafOffset = iOff; if( pLeaf->nn>pLeaf->szLeaf ){ pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist ); } } else if( pLeaf->nn>pLeaf->szLeaf ){ pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( &pLeaf->p[pLeaf->szLeaf], iOff ); pIter->iLeafOffset = iOff; pIter->iEndofDoclist = iOff; bNewTerm = 1; } assert_nc( iOff<pLeaf->szLeaf ); if( iOff>pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; return; } } } /* Check if the iterator is now at EOF. If so, return early. */ if( pIter->pLeaf ){ if( bNewTerm ){ if( pIter->flags & FTS5_SEGITER_ONETERM ){ fts5DataRelease(pIter->pLeaf); pIter->pLeaf = 0; }else{ fts5SegIterLoadTerm(p, pIter, nKeep); fts5SegIterLoadNPos(p, pIter); if( pbNewTerm ) *pbNewTerm = 1; } }else{ /* The following could be done by calling fts5SegIterLoadNPos(). But ** this block is particularly performance critical, so equivalent ** code is inlined. ** ** Later: Switched back to fts5SegIterLoadNPos() because it supports ** detail=none mode. Not ideal. */ int nSz; assert( p->rc==SQLITE_OK ); fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); pIter->bDel = (nSz & 0x0001); pIter->nPos = nSz>>1; assert_nc( pIter->nPos>=0 ); } } } #define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; } #define fts5IndexSkipVarint(a, iOff) { \ int iEnd = iOff+9; \ while( (a[iOff++] & 0x80) && iOff<iEnd ); \ } /* ** Iterator pIter currently points to the first rowid in a doclist. This ** function sets the iterator up so that iterates in reverse order through ** the doclist. */ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ |
︙ | ︙ | |||
1877 1878 1879 1880 1881 1882 1883 | pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); }else{ Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ /* Currently, Fts5SegIter.iLeafOffset points to the first byte of ** position-list content for the current rowid. Back it up so that it ** points to the start of the position-list size field. */ | > > > > > > > > | > > | 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 | pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); }else{ Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ /* Currently, Fts5SegIter.iLeafOffset points to the first byte of ** position-list content for the current rowid. Back it up so that it ** points to the start of the position-list size field. */ int iPoslist; if( pIter->iTermLeafPgno==pIter->iLeafPgno ){ iPoslist = pIter->iTermLeafOffset; }else{ iPoslist = 4; } fts5IndexSkipVarint(pLeaf->p, iPoslist); assert( p->pConfig->eDetail==FTS5_DETAIL_NONE || iPoslist==( pIter->iLeafOffset - sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel) )); pIter->iLeafOffset = iPoslist; /* If this condition is true then the largest rowid for the current ** term may not be stored on the current page. So search forward to ** see where said rowid really is. */ if( pIter->iEndofDoclist>=pLeaf->szLeaf ){ int pgno; Fts5StructureSegment *pSeg = pIter->pSeg; |
︙ | ︙ | |||
1961 1962 1963 1964 1965 1966 1967 | ){ return; } pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); } | < < < < < | 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 | ){ return; } pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); } /* ** The iterator object passed as the second argument currently contains ** no valid values except for the Fts5SegIter.pLeaf member variable. This ** function searches the leaf page for a term matching (pTerm/nTerm). ** ** If the specified term is found on the page, then the iterator is left ** pointing to it. If argument bGe is zero and the term is not found, |
︙ | ︙ | |||
2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 | } if( flags & FTS5INDEX_QUERY_DESC ){ fts5SegIterReverse(p, pIter); } } } /* Either: ** ** 1) an error has occurred, or ** 2) the iterator points to EOF, or ** 3) the iterator points to an entry with term (pTerm/nTerm), or ** 4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points ** to an entry with a term greater than or equal to (pTerm/nTerm). | > > | 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 | } if( flags & FTS5INDEX_QUERY_DESC ){ fts5SegIterReverse(p, pIter); } } } fts5SegIterSetNext(p, pIter); /* Either: ** ** 1) an error has occurred, or ** 2) the iterator points to EOF, or ** 3) the iterator points to an entry with term (pTerm/nTerm), or ** 4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points ** to an entry with a term greater than or equal to (pTerm/nTerm). |
︙ | ︙ | |||
2225 2226 2227 2228 2229 2230 2231 | sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z); pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); if( pLeaf==0 ) return; pLeaf->p = (u8*)pList; pLeaf->nn = pLeaf->szLeaf = nList; pIter->pLeaf = pLeaf; pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid); | | > > | 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 | sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z); pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); if( pLeaf==0 ) return; pLeaf->p = (u8*)pList; pLeaf->nn = pLeaf->szLeaf = nList; pIter->pLeaf = pLeaf; pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid); pIter->iEndofDoclist = pLeaf->nn; if( flags & FTS5INDEX_QUERY_DESC ){ pIter->flags |= FTS5_SEGITER_REVERSE; fts5SegIterReverseInitPage(p, pIter); }else{ fts5SegIterLoadNPos(p, pIter); } } fts5SegIterSetNext(p, pIter); } /* ** Zero the iterator passed as the only argument. */ static void fts5SegIterClear(Fts5SegIter *pIter){ fts5BufferFree(&pIter->term); |
︙ | ︙ | |||
2477 2478 2479 2480 2481 2482 2483 | pIter->iLeafPgno = iLeafPgno+1; fts5SegIterReverseNewPage(p, pIter); bMove = 0; } } do{ | | | 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 | pIter->iLeafPgno = iLeafPgno+1; fts5SegIterReverseNewPage(p, pIter); bMove = 0; } } do{ if( bMove && p->rc==SQLITE_OK ) pIter->xNext(p, pIter, 0); if( pIter->pLeaf==0 ) break; if( bRev==0 && pIter->iRowid>=iMatch ) break; if( bRev!=0 && pIter->iRowid<=iMatch ) break; bMove = 1; }while( p->rc==SQLITE_OK ); } |
︙ | ︙ | |||
2511 2512 2513 2514 2515 2516 2517 | int iChanged, /* Index of sub-iterator just advanced */ int iMinset /* Minimum entry in aFirst[] to set */ ){ int i; for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){ int iEq; if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){ | | > > | 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 | int iChanged, /* Index of sub-iterator just advanced */ int iMinset /* Minimum entry in aFirst[] to set */ ){ int i; for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){ int iEq; if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){ Fts5SegIter *pSeg = &pIter->aSeg[iEq]; assert( p->rc==SQLITE_OK ); pSeg->xNext(p, pSeg, 0); i = pIter->nSeg + iEq; } } } /* ** Sub-iterator iChanged of iterator pIter has just been advanced. It still |
︙ | ︙ | |||
2598 2599 2600 2601 2602 2603 2604 | int iFirst = pIter->aFirst[1].iFirst; int bNewTerm = 0; Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; assert( p->rc==SQLITE_OK ); if( bUseFrom && pSeg->pDlidx ){ fts5SegIterNextFrom(p, pSeg, iFrom); }else{ | | | 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 | int iFirst = pIter->aFirst[1].iFirst; int bNewTerm = 0; Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; assert( p->rc==SQLITE_OK ); if( bUseFrom && pSeg->pDlidx ){ fts5SegIterNextFrom(p, pSeg, iFrom); }else{ pSeg->xNext(p, pSeg, &bNewTerm); } if( pSeg->pLeaf==0 || bNewTerm || fts5MultiIterAdvanceRowid(p, pIter, iFirst) ){ fts5MultiIterAdvanced(p, pIter, iFirst, 1); fts5MultiIterSetEof(pIter); |
︙ | ︙ | |||
2626 2627 2628 2629 2630 2631 2632 | assert( pIter->bSkipEmpty ); if( p->rc==SQLITE_OK ){ do { int iFirst = pIter->aFirst[1].iFirst; Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; int bNewTerm = 0; | > | | 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 | assert( pIter->bSkipEmpty ); if( p->rc==SQLITE_OK ){ do { int iFirst = pIter->aFirst[1].iFirst; Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; int bNewTerm = 0; assert( p->rc==SQLITE_OK ); pSeg->xNext(p, pSeg, &bNewTerm); if( pSeg->pLeaf==0 || bNewTerm || fts5MultiIterAdvanceRowid(p, pIter, iFirst) ){ fts5MultiIterAdvanced(p, pIter, iFirst, 1); fts5MultiIterSetEof(pIter); *pbNewTerm = 1; }else{ |
︙ | ︙ | |||
2746 2747 2748 2749 2750 2751 2752 | ** to the first entry in its segment. In this case initialize the ** aFirst[] array. Or, if an error has occurred, free the iterator ** object and set the output variable to NULL. */ if( p->rc==SQLITE_OK ){ for(iIter=pNew->nSeg-1; iIter>0; iIter--){ int iEq; if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ | | > | 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 | ** to the first entry in its segment. In this case initialize the ** aFirst[] array. Or, if an error has occurred, free the iterator ** object and set the output variable to NULL. */ if( p->rc==SQLITE_OK ){ for(iIter=pNew->nSeg-1; iIter>0; iIter--){ int iEq; if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ Fts5SegIter *pSeg = &pNew->aSeg[iEq]; if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); fts5MultiIterAdvanced(p, pNew, iEq, iIter); } } fts5MultiIterSetEof(pNew); fts5AssertMultiIterSetup(p, pNew); if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ |
︙ | ︙ | |||
2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 | }else{ fts5SegIterLoadNPos(p, pIter); } pData = 0; }else{ pNew->bEof = 1; } *ppOut = pNew; } fts5DataRelease(pData); } | > | 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 | }else{ fts5SegIterLoadNPos(p, pIter); } pData = 0; }else{ pNew->bEof = 1; } fts5SegIterSetNext(p, pIter); *ppOut = pNew; } fts5DataRelease(pData); } |
︙ | ︙ | |||
2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 | ){ int nRem = pSeg->nPos; /* Number of bytes still to come */ Fts5Data *pData = 0; u8 *pChunk = &pSeg->pLeaf->p[pSeg->iLeafOffset]; int nChunk = MIN(nRem, pSeg->pLeaf->szLeaf - pSeg->iLeafOffset); int pgno = pSeg->iLeafPgno; int pgnoSave = 0; if( (pSeg->flags & FTS5_SEGITER_REVERSE)==0 ){ pgnoSave = pgno+1; } while( 1 ){ xChunk(p, pCtx, pChunk, nChunk); | > > > | 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 | ){ int nRem = pSeg->nPos; /* Number of bytes still to come */ Fts5Data *pData = 0; u8 *pChunk = &pSeg->pLeaf->p[pSeg->iLeafOffset]; int nChunk = MIN(nRem, pSeg->pLeaf->szLeaf - pSeg->iLeafOffset); int pgno = pSeg->iLeafPgno; int pgnoSave = 0; /* This function does notmwork with detail=none databases. */ assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); if( (pSeg->flags & FTS5_SEGITER_REVERSE)==0 ){ pgnoSave = pgno+1; } while( 1 ){ xChunk(p, pCtx, pChunk, nChunk); |
︙ | ︙ | |||
3288 3289 3290 3291 3292 3293 3294 | /* ** Append a rowid and position-list size field to the writers output. */ static void fts5WriteAppendRowid( Fts5Index *p, Fts5SegWriter *pWriter, | | < | 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 | /* ** Append a rowid and position-list size field to the writers output. */ static void fts5WriteAppendRowid( Fts5Index *p, Fts5SegWriter *pWriter, i64 iRowid ){ if( p->rc==SQLITE_OK ){ Fts5PageWriter *pPage = &pWriter->writer; if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){ fts5WriteFlushLeaf(p, pWriter); } |
︙ | ︙ | |||
3316 3317 3318 3319 3320 3321 3322 | }else{ assert( p->rc || iRowid>pWriter->iPrevRowid ); fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid); } pWriter->iPrevRowid = iRowid; pWriter->bFirstRowidInDoclist = 0; pWriter->bFirstRowidInPage = 0; | < < | 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 | }else{ assert( p->rc || iRowid>pWriter->iPrevRowid ); fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid); } pWriter->iPrevRowid = iRowid; pWriter->bFirstRowidInDoclist = 0; pWriter->bFirstRowidInPage = 0; } } static void fts5WriteAppendPoslistData( Fts5Index *p, Fts5SegWriter *pWriter, const u8 *aData, |
︙ | ︙ | |||
3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 | Fts5IndexIter *pIter = 0; /* Iterator to read input data */ int nRem = pnRem ? *pnRem : 0; /* Output leaf pages left to write */ int nInput; /* Number of input segments */ Fts5SegWriter writer; /* Writer object */ Fts5StructureSegment *pSeg; /* Output segment */ Fts5Buffer term; int bOldest; /* True if the output segment is the oldest */ assert( iLvl<pStruct->nLevel ); assert( pLvl->nMerge<=pLvl->nSeg ); memset(&writer, 0, sizeof(Fts5SegWriter)); memset(&term, 0, sizeof(Fts5Buffer)); if( pLvl->nMerge ){ | > | 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 | Fts5IndexIter *pIter = 0; /* Iterator to read input data */ int nRem = pnRem ? *pnRem : 0; /* Output leaf pages left to write */ int nInput; /* Number of input segments */ Fts5SegWriter writer; /* Writer object */ Fts5StructureSegment *pSeg; /* Output segment */ Fts5Buffer term; int bOldest; /* True if the output segment is the oldest */ int eDetail = p->pConfig->eDetail; assert( iLvl<pStruct->nLevel ); assert( pLvl->nMerge<=pLvl->nSeg ); memset(&writer, 0, sizeof(Fts5SegWriter)); memset(&term, 0, sizeof(Fts5Buffer)); if( pLvl->nMerge ){ |
︙ | ︙ | |||
3582 3583 3584 3585 3586 3587 3588 | /* This is a new term. Append a term to the output segment. */ fts5WriteAppendTerm(p, &writer, nTerm, pTerm); fts5BufferSet(&p->rc, &term, nTerm, pTerm); } /* Append the rowid to the output */ /* WRITEPOSLISTSIZE */ | < | > > > > > > > > | > > | > | 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 | /* This is a new term. Append a term to the output segment. */ fts5WriteAppendTerm(p, &writer, nTerm, pTerm); fts5BufferSet(&p->rc, &term, nTerm, pTerm); } /* Append the rowid to the output */ /* WRITEPOSLISTSIZE */ fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter)); if( eDetail==FTS5_DETAIL_NONE ){ if( pSegIter->bDel ){ fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); if( pSegIter->nPos>0 ){ fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); } } }else{ /* Append the position-list data to the output */ nPos = pSegIter->nPos*2 + pSegIter->bDel; fts5BufferAppendVarint(&p->rc, &writer.writer.buf, nPos); fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback); } } /* Flush the last leaf page to disk. Set the output segment b-tree height ** and last leaf page number at the same time. */ fts5WriteFinish(p, &writer, &pSeg->pgnoLast); if( fts5MultiIterEof(p, pIter) ){ |
︙ | ︙ | |||
3774 3775 3776 3777 3778 3779 3780 | /* Obtain a reference to the index structure and allocate a new segment-id ** for the new level-0 segment. */ pStruct = fts5StructureRead(p); iSegid = fts5AllocateSegid(p, pStruct); if( iSegid ){ const int pgsz = p->pConfig->pgsz; | | | 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 | /* Obtain a reference to the index structure and allocate a new segment-id ** for the new level-0 segment. */ pStruct = fts5StructureRead(p); iSegid = fts5AllocateSegid(p, pStruct); if( iSegid ){ const int pgsz = p->pConfig->pgsz; int eDetail = p->pConfig->eDetail; Fts5StructureSegment *pSeg; /* New segment within pStruct */ Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ Fts5SegWriter writer; fts5WriteInit(p, &writer, iSegid); |
︙ | ︙ | |||
3817 3818 3819 3820 3821 3822 3823 | i64 iDelta = 0; int iOff = 0; /* The entire doclist will not fit on this leaf. The following ** loop iterates through the poslists that make up the current ** doclist. */ while( p->rc==SQLITE_OK && iOff<nDoclist ){ | < < < < < > > > > > > > > > > > > > > > > > | | | | | | | | | | | | | | | | | | | | | | | | | | | | > | 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 | i64 iDelta = 0; int iOff = 0; /* The entire doclist will not fit on this leaf. The following ** loop iterates through the poslists that make up the current ** doclist. */ while( p->rc==SQLITE_OK && iOff<nDoclist ){ iOff += fts5GetVarint(&pDoclist[iOff], (u64*)&iDelta); iRowid += iDelta; if( writer.bFirstRowidInPage ){ fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); writer.bFirstRowidInPage = 0; fts5WriteDlidxAppend(p, &writer, iRowid); }else{ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta); } assert( pBuf->n<=pBuf->nSpace ); if( eDetail==FTS5_DETAIL_NONE ){ if( iOff<nDoclist && pDoclist[iOff]==0 ){ pBuf->p[pBuf->n++] = 0; iOff++; if( iOff<nDoclist && pDoclist[iOff]==0 ){ pBuf->p[pBuf->n++] = 0; iOff++; } } if( (pBuf->n + pPgidx->n)>=pgsz ){ fts5WriteFlushLeaf(p, &writer); } }else{ int bDummy; int nPos; int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); nCopy += nPos; if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ /* The entire poslist will fit on the current leaf. So copy ** it in one go. */ fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); }else{ /* The entire poslist will not fit on this leaf. So it needs ** to be broken into sections. The only qualification being ** that each varint must be stored contiguously. */ const u8 *pPoslist = &pDoclist[iOff]; int iPos = 0; while( p->rc==SQLITE_OK ){ int nSpace = pgsz - pBuf->n - pPgidx->n; int n = 0; if( (nCopy - iPos)<=nSpace ){ n = nCopy - iPos; }else{ n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); } assert( n>0 ); fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); iPos += n; if( (pBuf->n + pPgidx->n)>=pgsz ){ fts5WriteFlushLeaf(p, &writer); } if( iPos>=nCopy ) break; } } iOff += nCopy; } } } /* TODO2: Doclist terminator written here. */ /* pBuf->p[pBuf->n++] = '\0'; */ assert( pBuf->n<=pBuf->nSpace ); sqlite3Fts5HashScanNext(pHash); |
︙ | ︙ | |||
3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 | typedef struct PoslistCallbackCtx PoslistCallbackCtx; struct PoslistCallbackCtx { Fts5Buffer *pBuf; /* Append to this buffer */ Fts5Colset *pColset; /* Restrict matches to this column */ int eState; /* See above */ }; /* ** TODO: Make this more efficient! */ static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){ int i; for(i=0; i<pColset->nCol; i++){ if( pColset->aiCol[i]==iCol ) return 1; } return 0; } static void fts5PoslistFilterCallback( Fts5Index *p, void *pContext, const u8 *pChunk, int nChunk ){ PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext; | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 | typedef struct PoslistCallbackCtx PoslistCallbackCtx; struct PoslistCallbackCtx { Fts5Buffer *pBuf; /* Append to this buffer */ Fts5Colset *pColset; /* Restrict matches to this column */ int eState; /* See above */ }; typedef struct PoslistOffsetsCtx PoslistOffsetsCtx; struct PoslistOffsetsCtx { Fts5Buffer *pBuf; /* Append to this buffer */ Fts5Colset *pColset; /* Restrict matches to this column */ int iRead; int iWrite; }; /* ** TODO: Make this more efficient! */ static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){ int i; for(i=0; i<pColset->nCol; i++){ if( pColset->aiCol[i]==iCol ) return 1; } return 0; } static void fts5PoslistOffsetsCallback( Fts5Index *p, void *pContext, const u8 *pChunk, int nChunk ){ PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext; assert_nc( nChunk>=0 ); if( nChunk>0 ){ int i = 0; while( i<nChunk ){ int iVal; i += fts5GetVarint32(&pChunk[i], iVal); iVal += pCtx->iRead - 2; pCtx->iRead = iVal; if( fts5IndexColsetTest(pCtx->pColset, iVal) ){ fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite); pCtx->iWrite = iVal; } } } } static void fts5PoslistFilterCallback( Fts5Index *p, void *pContext, const u8 *pChunk, int nChunk ){ PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext; |
︙ | ︙ | |||
4075 4076 4077 4078 4079 4080 4081 | Fts5Colset *pColset, Fts5Buffer *pBuf ){ if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){ if( pColset==0 ){ fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); }else{ | > | | | | | | > > > > > > > | 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 | Fts5Colset *pColset, Fts5Buffer *pBuf ){ if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){ if( pColset==0 ){ fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); }else{ if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){ PoslistCallbackCtx sCtx; sCtx.pBuf = pBuf; sCtx.pColset = pColset; sCtx.eState = fts5IndexColsetTest(pColset, 0); assert( sCtx.eState==0 || sCtx.eState==1 ); fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); }else{ PoslistOffsetsCtx sCtx; memset(&sCtx, 0, sizeof(sCtx)); sCtx.pBuf = pBuf; sCtx.pColset = pColset; fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); } } } } /* ** IN/OUT parameter (*pa) points to a position list n bytes in size. If ** the position list contains entries for column iCol, then (*pa) is set |
︙ | ︙ | |||
4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 | assert( (prev & 0x80)==0 ); while( p<pEnd && ((prev & 0x80) || *p!=0x01) ){ prev = *p++; } return p - (*pa); } /* ** Iterator pMulti currently points to a valid entry (not EOF). This ** function appends the following to buffer pBuf: ** ** * The varint iDelta, and ** * the position list that currently points to, including the size field. | > > > > > > > > > > | 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 | assert( (prev & 0x80)==0 ); while( p<pEnd && ((prev & 0x80) || *p!=0x01) ){ prev = *p++; } return p - (*pa); } static int fts5AppendRowid( Fts5Index *p, i64 iDelta, Fts5IndexIter *pMulti, Fts5Colset *pColset, Fts5Buffer *pBuf ){ fts5BufferAppendVarint(&p->rc, pBuf, iDelta); return 0; } /* ** Iterator pMulti currently points to a valid entry (not EOF). This ** function appends the following to buffer pBuf: ** ** * The varint iDelta, and ** * the position list that currently points to, including the size field. |
︙ | ︙ | |||
4150 4151 4152 4153 4154 4155 4156 | Fts5Buffer *pBuf ){ if( p->rc==SQLITE_OK ){ Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ]; assert( fts5MultiIterEof(p, pMulti)==0 ); assert( pSeg->nPos>0 ); if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){ | | | | 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 | Fts5Buffer *pBuf ){ if( p->rc==SQLITE_OK ){ Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ]; assert( fts5MultiIterEof(p, pMulti)==0 ); assert( pSeg->nPos>0 ); if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){ if( p->pConfig->eDetail==FTS5_DETAIL_FULL && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf && (pColset==0 || pColset->nCol==1) ){ const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; int nPos; if( pColset ){ nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]); if( nPos==0 ) return 1; |
︙ | ︙ | |||
4196 4197 4198 4199 4200 4201 4202 | int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2)); while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; } sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2); } } } } | < > | 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 | int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2)); while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; } sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2); } } } } } } return 0; } static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist; assert( pIter->aPoslist ); if( p>=pIter->aEof ){ pIter->aPoslist = 0; |
︙ | ︙ | |||
4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 | #endif #define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \ (iLastRowid) = (iRowid); \ } /* ** Buffers p1 and p2 contain doclists. This function merges the content ** of the two doclists together and sets buffer p1 to the result before ** returning. ** ** If an error occurs, an error code is left in p->rc. If an error has | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 | #endif #define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \ (iLastRowid) = (iRowid); \ } /* ** Swap the contents of buffer *p1 with that of *p2. */ static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){ Fts5Buffer tmp = *p1; *p1 = *p2; *p2 = tmp; } static void fts5NextRowid(Fts5Buffer *pBuf, int *piOff, i64 *piRowid){ int i = *piOff; if( i>=pBuf->n ){ *piOff = -1; }else{ u64 iVal; *piOff = i + sqlite3Fts5GetVarint(&pBuf->p[i], &iVal); *piRowid += iVal; } } /* ** This is the equivalent of fts5MergePrefixLists() for detail=none mode. ** In this case the buffers consist of a delta-encoded list of rowids only. */ static void fts5MergeRowidLists( Fts5Index *p, /* FTS5 backend object */ Fts5Buffer *p1, /* First list to merge */ Fts5Buffer *p2 /* Second list to merge */ ){ int i1 = 0; int i2 = 0; i64 iRowid1 = 0; i64 iRowid2 = 0; i64 iOut = 0; Fts5Buffer out; memset(&out, 0, sizeof(out)); sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n); if( p->rc ) return; fts5NextRowid(p1, &i1, &iRowid1); fts5NextRowid(p2, &i2, &iRowid2); while( i1>=0 || i2>=0 ){ if( i1>=0 && (i2<0 || iRowid1<iRowid2) ){ assert( iOut==0 || iRowid1>iOut ); fts5BufferSafeAppendVarint(&out, iRowid1 - iOut); iOut = iRowid1; fts5NextRowid(p1, &i1, &iRowid1); }else{ assert( iOut==0 || iRowid2>iOut ); fts5BufferSafeAppendVarint(&out, iRowid2 - iOut); iOut = iRowid2; if( i1>=0 && iRowid1==iRowid2 ){ fts5NextRowid(p1, &i1, &iRowid1); } fts5NextRowid(p2, &i2, &iRowid2); } } fts5BufferSwap(&out, p1); fts5BufferFree(&out); } /* ** Buffers p1 and p2 contain doclists. This function merges the content ** of the two doclists together and sets buffer p1 to the result before ** returning. ** ** If an error occurs, an error code is left in p->rc. If an error has |
︙ | ︙ | |||
4331 4332 4333 4334 4335 4336 4337 | }else{ iNew = iPos2; sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); if( iPos1==iPos2 ){ sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1,&iPos1); } } | > | > < < < < < < > > > > > > > > > > | 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 | }else{ iNew = iPos2; sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); if( iPos1==iPos2 ){ sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1,&iPos1); } } if( iNew!=writer.iPrev || tmp.n==0 ){ p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew); } } /* WRITEPOSLISTSIZE */ fts5BufferSafeAppendVarint(&out, tmp.n * 2); fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); fts5DoclistIterNext(&i1); fts5DoclistIterNext(&i2); } } fts5BufferSet(&p->rc, p1, out.n, out.p); fts5BufferFree(&tmp); fts5BufferFree(&out); } } static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ int bDesc, /* True for "ORDER BY rowid DESC" */ const u8 *pToken, /* Buffer containing prefix to match */ int nToken, /* Size of buffer pToken in bytes */ Fts5Colset *pColset, /* Restrict matches to these columns */ Fts5IndexIter **ppIter /* OUT: New iterator */ ){ Fts5Structure *pStruct; Fts5Buffer *aBuf; const int nBuf = 32; void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*); int (*xAppend)(Fts5Index*, i64, Fts5IndexIter*, Fts5Colset*, Fts5Buffer*); if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ xMerge = fts5MergeRowidLists; xAppend = fts5AppendRowid; }else{ xMerge = fts5MergePrefixLists; xAppend = fts5AppendPoslist; } aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); pStruct = fts5StructureRead(p); if( aBuf && pStruct ){ const int flags = FTS5INDEX_QUERY_SCAN; int i; |
︙ | ︙ | |||
4398 4399 4400 4401 4402 4403 4404 | if( doclist.n>0 && iRowid<=iLastRowid ){ for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ assert( i<nBuf ); if( aBuf[i].n==0 ){ fts5BufferSwap(&doclist, &aBuf[i]); fts5BufferZero(&doclist); }else{ | | | | | 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 | if( doclist.n>0 && iRowid<=iLastRowid ){ for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ assert( i<nBuf ); if( aBuf[i].n==0 ){ fts5BufferSwap(&doclist, &aBuf[i]); fts5BufferZero(&doclist); }else{ xMerge(p, &doclist, &aBuf[i]); fts5BufferZero(&aBuf[i]); } } iLastRowid = 0; } if( !xAppend(p, iRowid-iLastRowid, p1, pColset, &doclist) ){ iLastRowid = iRowid; } } for(i=0; i<nBuf; i++){ if( p->rc==SQLITE_OK ){ xMerge(p, &doclist, &aBuf[i]); } fts5BufferFree(&aBuf[i]); } fts5MultiIterFree(p, p1); pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n); if( pData ){ |
︙ | ︙ | |||
4442 4443 4444 4445 4446 4447 4448 | ** to the document with rowid iRowid. */ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ assert( p->rc==SQLITE_OK ); /* Allocate the hash table if it has not already been allocated */ if( p->pHash==0 ){ | | | 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 | ** to the document with rowid iRowid. */ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ assert( p->rc==SQLITE_OK ); /* Allocate the hash table if it has not already been allocated */ if( p->pHash==0 ){ p->rc = sqlite3Fts5HashNew(p->pConfig, &p->pHash, &p->nPendingData); } /* Flush the hash table to disk if required */ if( iRowid<p->iWriteRowid || (iRowid==p->iWriteRowid && p->bDelete==0) || (p->nPendingData > p->pConfig->nHashSize) ){ |
︙ | ︙ | |||
4563 4564 4565 4566 4567 4568 4569 | } /* ** Argument p points to a buffer containing utf-8 text that is n bytes in ** size. Return the number of bytes in the nChar character prefix of the ** buffer, or 0 if there are less than nChar characters in total. */ | | > > > > | 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 | } /* ** Argument p points to a buffer containing utf-8 text that is n bytes in ** size. Return the number of bytes in the nChar character prefix of the ** buffer, or 0 if there are less than nChar characters in total. */ int sqlite3Fts5IndexCharlenToBytelen( const char *p, int nByte, int nChar ){ int n = 0; int i; for(i=0; i<nChar; i++){ if( n>=nByte ) return 0; /* Input contains fewer than nChar chars */ if( (unsigned char)p[n++]>=0xc0 ){ while( (p[n] & 0xc0)==0x80 ) n++; } |
︙ | ︙ | |||
4620 4621 4622 4623 4624 4625 4626 | /* Add the entry to the main terms index. */ rc = sqlite3Fts5HashWrite( p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken ); for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){ | > | | 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 | /* Add the entry to the main terms index. */ rc = sqlite3Fts5HashWrite( p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken ); for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){ const int nChar = pConfig->aPrefix[i]; int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); if( nByte ){ rc = sqlite3Fts5HashWrite(p->pHash, p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX+i+1), pToken, nByte ); } } |
︙ | ︙ | |||
4798 4799 4800 4801 4802 4803 4804 4805 4806 | Fts5IndexIter *pIter, Fts5Colset *pColset, /* Column filter (or NULL) */ const u8 **pp, /* OUT: Pointer to position-list data */ int *pn, /* OUT: Size of position-list in bytes */ i64 *piRowid /* OUT: Current rowid */ ){ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; assert( pIter->pIndex->rc==SQLITE_OK ); *piRowid = pSeg->iRowid; | > > > > > > | > > | > > > > > > > > > > > > | 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 | Fts5IndexIter *pIter, Fts5Colset *pColset, /* Column filter (or NULL) */ const u8 **pp, /* OUT: Pointer to position-list data */ int *pn, /* OUT: Size of position-list in bytes */ i64 *piRowid /* OUT: Current rowid */ ){ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; int eDetail = pIter->pIndex->pConfig->eDetail; assert( pIter->pIndex->rc==SQLITE_OK ); *piRowid = pSeg->iRowid; if( eDetail==FTS5_DETAIL_NONE ){ *pn = pSeg->nPos; }else if( eDetail==FTS5_DETAIL_FULL && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; if( pColset==0 || pIter->bFiltered ){ *pn = pSeg->nPos; *pp = pPos; }else if( pColset->nCol==1 ){ *pp = pPos; *pn = fts5IndexExtractCol(pp, pSeg->nPos, pColset->aiCol[0]); }else{ fts5BufferZero(&pIter->poslist); fts5IndexExtractColset(pColset, pPos, pSeg->nPos, &pIter->poslist); *pp = pIter->poslist.p; *pn = pIter->poslist.n; } }else{ fts5BufferZero(&pIter->poslist); fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); if( eDetail==FTS5_DETAIL_FULL ){ *pp = pIter->poslist.p; } *pn = pIter->poslist.n; } return fts5IndexReturn(pIter->pIndex); } int sqlite3Fts5IterCollist( Fts5IndexIter *pIter, const u8 **pp, /* OUT: Pointer to position-list data */ int *pn /* OUT: Size of position-list in bytes */ ){ assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); *pp = pIter->poslist.p; *pn = pIter->poslist.n; return SQLITE_OK; } /* ** This function is similar to sqlite3Fts5IterPoslist(), except that it ** copies the position list into the buffer supplied as the second ** argument. */ int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){ |
︙ | ︙ | |||
4936 4937 4938 4939 4940 4941 4942 | ** Below this point is the implementation of the integrity-check ** functionality. */ /* ** Return a simple checksum value based on the arguments. */ | | | 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 | ** Below this point is the implementation of the integrity-check ** functionality. */ /* ** Return a simple checksum value based on the arguments. */ u64 sqlite3Fts5IndexEntryCksum( i64 iRowid, int iCol, int iPos, int iIdx, const char *pTerm, int nTerm ){ |
︙ | ︙ | |||
5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 | Fts5Index *p, /* Fts5 index object */ int iIdx, const char *z, /* Index key to query for */ int n, /* Size of index key in bytes */ int flags, /* Flags for Fts5IndexQuery */ u64 *pCksum /* IN/OUT: Checksum value */ ){ u64 cksum = *pCksum; Fts5IndexIter *pIdxIter = 0; int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter); while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){ | > > < < < > > > > | | | | | | | | | | | > > > > | 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 | Fts5Index *p, /* Fts5 index object */ int iIdx, const char *z, /* Index key to query for */ int n, /* Size of index key in bytes */ int flags, /* Flags for Fts5IndexQuery */ u64 *pCksum /* IN/OUT: Checksum value */ ){ int eDetail = p->pConfig->eDetail; u64 cksum = *pCksum; Fts5IndexIter *pIdxIter = 0; Fts5Buffer buf = {0, 0, 0}; int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter); while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){ i64 rowid = sqlite3Fts5IterRowid(pIdxIter); if( eDetail==FTS5_DETAIL_NONE ){ cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n); }else{ rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf); if( rc==SQLITE_OK ){ Fts5PoslistReader sReader; for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader); sReader.bEof==0; sqlite3Fts5PoslistReaderNext(&sReader) ){ int iCol = FTS5_POS2COLUMN(sReader.iPos); int iOff = FTS5_POS2OFFSET(sReader.iPos); cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); } } } if( rc==SQLITE_OK ){ rc = sqlite3Fts5IterNext(pIdxIter); } } sqlite3Fts5IterClose(pIdxIter); fts5BufferFree(&buf); *pCksum = cksum; return rc; } /* |
︙ | ︙ | |||
5323 5324 5325 5326 5327 5328 5329 | #endif } /* ** Run internal checks to ensure that the FTS index (a) is internally ** consistent and (b) contains entries for which the XOR of the checksums | | > | 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 | #endif } /* ** Run internal checks to ensure that the FTS index (a) is internally ** consistent and (b) contains entries for which the XOR of the checksums ** as calculated by sqlite3Fts5IndexEntryCksum() is cksum. ** ** Return SQLITE_CORRUPT if any of the internal checks fail, or if the ** checksum does not match. Return SQLITE_OK if all checks pass without ** error, or some other SQLite error code if another error (e.g. OOM) ** occurs. */ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ int eDetail = p->pConfig->eDetail; u64 cksum2 = 0; /* Checksum based on contents of indexes */ Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ Fts5IndexIter *pIter; /* Used to iterate through entire index */ Fts5Structure *pStruct; /* Index structure */ #ifdef SQLITE_DEBUG /* Used by extra internal tests only run if NDEBUG is not defined */ |
︙ | ︙ | |||
5382 5383 5384 5385 5386 5387 5388 | int iOff = 0; /* Offset within poslist */ i64 iRowid = fts5MultiIterRowid(pIter); char *z = (char*)fts5MultiIterTerm(pIter, &n); /* If this is a new term, query for it. Update cksum3 with the results. */ fts5TestTerm(p, &term, z, n, cksum2, &cksum3); | > > > > > | | | | | | > < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 | int iOff = 0; /* Offset within poslist */ i64 iRowid = fts5MultiIterRowid(pIter); char *z = (char*)fts5MultiIterTerm(pIter, &n); /* If this is a new term, query for it. Update cksum3 with the results. */ fts5TestTerm(p, &term, z, n, cksum2, &cksum3); if( eDetail==FTS5_DETAIL_NONE ){ if( 0==fts5MultiIterIsEmpty(p, pIter) ){ cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n); } }else{ poslist.n = 0; fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist); while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ int iCol = FTS5_POS2COLUMN(iPos); int iTokOff = FTS5_POS2OFFSET(iPos); cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); } } } fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); fts5MultiIterFree(p, pIter); if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; fts5StructureRelease(pStruct); #ifdef SQLITE_DEBUG fts5BufferFree(&term); #endif fts5BufferFree(&poslist); return fts5IndexReturn(p); } /************************************************************************* ************************************************************************** ** Below this point is the implementation of the fts5_decode() scalar ** function only. */ /* |
︙ | ︙ | |||
5788 5789 5790 5791 5792 5793 5794 | if( rc==SQLITE_OK ){ rc = sqlite3_create_function( db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0 ); } return rc; } | < | 6088 6089 6090 6091 6092 6093 6094 | if( rc==SQLITE_OK ){ rc = sqlite3_create_function( db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0 ); } return rc; } |
Changes to ext/fts5/fts5_main.c.
︙ | ︙ | |||
222 223 224 225 226 227 228 229 230 231 232 233 234 235 | */ #define FTS5CSR_REQUIRE_CONTENT 0x01 #define FTS5CSR_REQUIRE_DOCSIZE 0x02 #define FTS5CSR_REQUIRE_INST 0x04 #define FTS5CSR_EOF 0x08 #define FTS5CSR_FREE_ZRANK 0x10 #define FTS5CSR_REQUIRE_RESEEK 0x20 #define BitFlagAllTest(x,y) (((x) & (y))==(y)) #define BitFlagTest(x,y) (((x) & (y))!=0) /* ** Macros to Set(), Clear() and Test() cursor flags. | > | 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 | */ #define FTS5CSR_REQUIRE_CONTENT 0x01 #define FTS5CSR_REQUIRE_DOCSIZE 0x02 #define FTS5CSR_REQUIRE_INST 0x04 #define FTS5CSR_EOF 0x08 #define FTS5CSR_FREE_ZRANK 0x10 #define FTS5CSR_REQUIRE_RESEEK 0x20 #define FTS5CSR_REQUIRE_POSLIST 0x40 #define BitFlagAllTest(x,y) (((x) & (y))==(y)) #define BitFlagTest(x,y) (((x) & (y))!=0) /* ** Macros to Set(), Clear() and Test() cursor flags. |
︙ | ︙ | |||
635 636 637 638 639 640 641 642 643 644 645 646 647 648 | ** specific to the previous row stored by the cursor object. */ static void fts5CsrNewrow(Fts5Cursor *pCsr){ CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE | FTS5CSR_REQUIRE_INST ); } static void fts5FreeCursorComponents(Fts5Cursor *pCsr){ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); Fts5Auxdata *pData; Fts5Auxdata *pNext; | > | 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 | ** specific to the previous row stored by the cursor object. */ static void fts5CsrNewrow(Fts5Cursor *pCsr){ CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE | FTS5CSR_REQUIRE_INST | FTS5CSR_REQUIRE_POSLIST ); } static void fts5FreeCursorComponents(Fts5Cursor *pCsr){ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); Fts5Auxdata *pData; Fts5Auxdata *pNext; |
︙ | ︙ | |||
717 718 719 720 721 722 723 | int iOff = 0; rc = SQLITE_OK; pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0); nBlob = sqlite3_column_bytes(pSorter->pStmt, 1); aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1); | > > | | | | | | | > | | | 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 | int iOff = 0; rc = SQLITE_OK; pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0); nBlob = sqlite3_column_bytes(pSorter->pStmt, 1); aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1); /* nBlob==0 in detail=none mode. */ if( nBlob>0 ){ for(i=0; i<(pSorter->nIdx-1); i++){ int iVal; a += fts5GetVarint32(a, iVal); iOff += iVal; pSorter->aIdx[i] = iOff; } pSorter->aIdx[i] = &aBlob[nBlob] - a; pSorter->aPoslist = a; } fts5CsrNewrow(pCsr); } return rc; } |
︙ | ︙ | |||
1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 | assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 ); assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 ); assert( pCsr->iLastRowid==LARGEST_INT64 ); assert( pCsr->iFirstRowid==SMALLEST_INT64 ); pCsr->ePlan = FTS5_PLAN_SOURCE; pCsr->pExpr = pTab->pSortCsr->pExpr; rc = fts5CursorFirst(pTab, pCsr, bDesc); }else if( pMatch ){ const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); if( zExpr==0 ) zExpr = ""; rc = fts5CursorParseRank(pConfig, pCsr, pRank); if( rc==SQLITE_OK ){ if( zExpr[0]=='*' ){ | > | 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 | assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 ); assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 ); assert( pCsr->iLastRowid==LARGEST_INT64 ); assert( pCsr->iFirstRowid==SMALLEST_INT64 ); pCsr->ePlan = FTS5_PLAN_SOURCE; pCsr->pExpr = pTab->pSortCsr->pExpr; rc = fts5CursorFirst(pTab, pCsr, bDesc); sqlite3Fts5ExprClearEof(pCsr->pExpr); }else if( pMatch ){ const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); if( zExpr==0 ) zExpr = ""; rc = fts5CursorParseRank(pConfig, pCsr, pRank); if( rc==SQLITE_OK ){ if( zExpr[0]=='*' ){ |
︙ | ︙ | |||
1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 | int rc; Fts5Table *pTab = (Fts5Table*)pVtab; fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0); rc = sqlite3Fts5StorageRollback(pTab->pStorage); return rc; } static void *fts5ApiUserData(Fts5Context *pCtx){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return pCsr->pAux->pUserData; } static int fts5ApiColumnCount(Fts5Context *pCtx){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; | > > | 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 | int rc; Fts5Table *pTab = (Fts5Table*)pVtab; fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0); rc = sqlite3Fts5StorageRollback(pTab->pStorage); return rc; } static int fts5CsrPoslist(Fts5Cursor*, int, const u8**, int*); static void *fts5ApiUserData(Fts5Context *pCtx){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return pCsr->pAux->pUserData; } static int fts5ApiColumnCount(Fts5Context *pCtx){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; |
︙ | ︙ | |||
1641 1642 1643 1644 1645 1646 1647 | } static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); } | > > > > > > > > > > > > > > > > > > > > > | > > > > > > > > > > > > > > > > > | > > > > > > > > > | > > > > > > > | | > | | 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 | } static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); } static int fts5ApiColumnText( Fts5Context *pCtx, int iCol, const char **pz, int *pn ){ int rc = SQLITE_OK; Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){ *pz = 0; *pn = 0; }else{ rc = fts5SeekCursor(pCsr, 0); if( rc==SQLITE_OK ){ *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); } } return rc; } static int fts5CsrPoslist( Fts5Cursor *pCsr, int iPhrase, const u8 **pa, int *pn ){ Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; int rc = SQLITE_OK; int bLive = (pCsr->pSorter==0); if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ Fts5PoslistPopulator *aPopulator; int i; aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive); if( aPopulator==0 ) rc = SQLITE_NOMEM; for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){ int n; const char *z; rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n); if( rc==SQLITE_OK ){ rc = sqlite3Fts5ExprPopulatePoslists( pConfig, pCsr->pExpr, aPopulator, i, z, n ); } } sqlite3_free(aPopulator); if( pCsr->pSorter ){ sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid); } } CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); } if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ Fts5Sorter *pSorter = pCsr->pSorter; int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); *pn = pSorter->aIdx[iPhrase] - i1; *pa = &pSorter->aPoslist[i1]; }else{ *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); } return rc; } /* ** Ensure that the Fts5Cursor.nInstCount and aInst[] variables are populated ** correctly for the current view. Return SQLITE_OK if successful, or an ** SQLite error code otherwise. */ |
︙ | ︙ | |||
1676 1677 1678 1679 1680 1681 1682 | aIter = pCsr->aInstIter; if( aIter ){ int nInst = 0; /* Number instances seen so far */ int i; /* Initialize all iterators */ | | > | > | | | | | | | | | | | | | | | | | | | | | | | | | | | | | > | 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 | aIter = pCsr->aInstIter; if( aIter ){ int nInst = 0; /* Number instances seen so far */ int i; /* Initialize all iterators */ for(i=0; i<nIter && rc==SQLITE_OK; i++){ const u8 *a; int n; rc = fts5CsrPoslist(pCsr, i, &a, &n); sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); } if( rc==SQLITE_OK ){ while( 1 ){ int *aInst; int iBest = -1; for(i=0; i<nIter; i++){ if( (aIter[i].bEof==0) && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos) ){ iBest = i; } } if( iBest<0 ) break; nInst++; if( nInst>=pCsr->nInstAlloc ){ pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32; aInst = (int*)sqlite3_realloc( pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3 ); if( aInst ){ pCsr->aInst = aInst; }else{ rc = SQLITE_NOMEM; break; } } aInst = &pCsr->aInst[3 * (nInst-1)]; aInst[0] = iBest; aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); sqlite3Fts5PoslistReaderNext(&aIter[iBest]); } } pCsr->nInstCount = nInst; CsrFlagClear(pCsr, FTS5CSR_REQUIRE_INST); } return rc; } |
︙ | ︙ | |||
1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 | Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; int rc = SQLITE_OK; if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) ){ if( iIdx<0 || iIdx>=pCsr->nInstCount ){ rc = SQLITE_RANGE; }else{ *piPhrase = pCsr->aInst[iIdx*3]; *piCol = pCsr->aInst[iIdx*3 + 1]; *piOff = pCsr->aInst[iIdx*3 + 2]; } } return rc; } static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){ return fts5CursorRowid((Fts5Cursor*)pCtx); } | > > > > > > < < < < < < < < < < < < < < < < < < < < < | 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 | Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; int rc = SQLITE_OK; if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) ){ if( iIdx<0 || iIdx>=pCsr->nInstCount ){ rc = SQLITE_RANGE; #if 0 }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){ *piPhrase = pCsr->aInst[iIdx*3]; *piCol = pCsr->aInst[iIdx*3 + 2]; *piOff = -1; #endif }else{ *piPhrase = pCsr->aInst[iIdx*3]; *piCol = pCsr->aInst[iIdx*3 + 1]; *piOff = pCsr->aInst[iIdx*3 + 2]; } } return rc; } static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){ return fts5CursorRowid((Fts5Cursor*)pCtx); } static int fts5ColumnSizeCb( void *pContext, /* Pointer to int */ int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ int iEnd /* End offset of token */ |
︙ | ︙ | |||
1923 1924 1925 1926 1927 1928 1929 | *piOff = 0; pIter->a += fts5GetVarint32(pIter->a, iVal); } *piOff += (iVal-2); } } | | > | > | | | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 | *piOff = 0; pIter->a += fts5GetVarint32(pIter->a, iVal); } *piOff += (iVal-2); } } static int fts5ApiPhraseFirst( Fts5Context *pCtx, int iPhrase, Fts5PhraseIter *pIter, int *piCol, int *piOff ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; int n; int rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); if( rc==SQLITE_OK ){ pIter->b = &pIter->a[n]; *piCol = 0; *piOff = 0; fts5ApiPhraseNext(pCtx, pIter, piCol, piOff); } return rc; } static void fts5ApiPhraseNextColumn( Fts5Context *pCtx, Fts5PhraseIter *pIter, int *piCol ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ if( pIter->a>=pIter->b ){ *piCol = -1; }else{ int iIncr; pIter->a += fts5GetVarint32(&pIter->a[0], iIncr); *piCol += (iIncr-2); } }else{ while( 1 ){ int dummy; if( pIter->a>=pIter->b ){ *piCol = -1; return; } if( pIter->a[0]==0x01 ) break; pIter->a += fts5GetVarint32(pIter->a, dummy); } pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); } } static int fts5ApiPhraseFirstColumn( Fts5Context *pCtx, int iPhrase, Fts5PhraseIter *pIter, int *piCol ){ int rc = SQLITE_OK; Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ int n; rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n); if( rc==SQLITE_OK ){ pIter->b = &pIter->a[n]; *piCol = 0; fts5ApiPhraseNextColumn(pCtx, pIter, piCol); } }else{ int n; rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); if( rc==SQLITE_OK ){ pIter->b = &pIter->a[n]; if( n<=0 ){ *piCol = -1; }else if( pIter->a[0]==0x01 ){ pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); }else{ *piCol = 0; } } } return rc; } static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); static const Fts5ExtensionApi sFts5Api = { 2, /* iVersion */ |
︙ | ︙ | |||
1960 1961 1962 1963 1964 1965 1966 1967 | fts5ApiColumnText, fts5ApiColumnSize, fts5ApiQueryPhrase, fts5ApiSetAuxdata, fts5ApiGetAuxdata, fts5ApiPhraseFirst, fts5ApiPhraseNext, }; | > > < | 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 | fts5ApiColumnText, fts5ApiColumnSize, fts5ApiQueryPhrase, fts5ApiSetAuxdata, fts5ApiGetAuxdata, fts5ApiPhraseFirst, fts5ApiPhraseNext, fts5ApiPhraseFirstColumn, fts5ApiPhraseNextColumn, }; /* ** Implementation of API function xQueryPhrase(). */ static int fts5ApiQueryPhrase( Fts5Context *pCtx, int iPhrase, |
︙ | ︙ | |||
2094 2095 2096 2097 2098 2099 2100 2101 | static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ int i; int rc = SQLITE_OK; int nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); Fts5Buffer val; memset(&val, 0, sizeof(Fts5Buffer)); | > > | | | | | | | | | | | | > > > > > > > > > > > > > > > > > > > > > > > > | 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 | static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ int i; int rc = SQLITE_OK; int nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); Fts5Buffer val; memset(&val, 0, sizeof(Fts5Buffer)); switch( ((Fts5Table*)(pCsr->base.pVtab))->pConfig->eDetail ){ case FTS5_DETAIL_FULL: /* Append the varints */ for(i=0; i<(nPhrase-1); i++){ const u8 *dummy; int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy); sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); } /* Append the position lists */ for(i=0; i<nPhrase; i++){ const u8 *pPoslist; int nPoslist; nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist); sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); } break; case FTS5_DETAIL_COLUMNS: /* Append the varints */ for(i=0; rc==SQLITE_OK && i<(nPhrase-1); i++){ const u8 *dummy; int nByte; rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &dummy, &nByte); sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); } /* Append the position lists */ for(i=0; rc==SQLITE_OK && i<nPhrase; i++){ const u8 *pPoslist; int nPoslist; rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &pPoslist, &nPoslist); sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); } break; default: break; } sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free); return rc; } /* |
︙ | ︙ |
Changes to ext/fts5/fts5_storage.c.
︙ | ︙ | |||
821 822 823 824 825 826 827 828 829 830 831 832 833 834 | */ typedef struct Fts5IntegrityCtx Fts5IntegrityCtx; struct Fts5IntegrityCtx { i64 iRowid; int iCol; int szCol; u64 cksum; Fts5Config *pConfig; }; /* ** Tokenization callback used by integrity check. */ static int fts5StorageIntegrityCallback( | > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > > | > > > > > > > > | > > > > | | 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 | */ typedef struct Fts5IntegrityCtx Fts5IntegrityCtx; struct Fts5IntegrityCtx { i64 iRowid; int iCol; int szCol; u64 cksum; Fts5Termset *pTermset; Fts5Config *pConfig; }; /* ** Tokenization callback used by integrity check. */ static int fts5StorageIntegrityCallback( void *pContext, /* Pointer to Fts5IntegrityCtx object */ int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ int iEnd /* End offset of token */ ){ Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; Fts5Termset *pTermset = pCtx->pTermset; int bPresent; int ii; int rc = SQLITE_OK; int iPos; int iCol; if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ pCtx->szCol++; } switch( pCtx->pConfig->eDetail ){ case FTS5_DETAIL_FULL: iPos = pCtx->szCol-1; iCol = pCtx->iCol; break; case FTS5_DETAIL_COLUMNS: iPos = pCtx->iCol; iCol = 0; break; default: assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE ); iPos = 0; iCol = 0; break; } rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent); if( rc==SQLITE_OK && bPresent==0 ){ pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( pCtx->iRowid, iCol, iPos, 0, pToken, nToken ); } for(ii=0; rc==SQLITE_OK && ii<pCtx->pConfig->nPrefix; ii++){ const int nChar = pCtx->pConfig->aPrefix[ii]; int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); if( nByte ){ rc = sqlite3Fts5TermsetAdd(pTermset, ii+1, pToken, nByte, &bPresent); if( bPresent==0 ){ pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( pCtx->iRowid, iCol, iPos, ii+1, pToken, nByte ); } } } return rc; } /* ** Check that the contents of the FTS index match that of the %_content ** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return ** some other SQLite error code if an error occurs while attempting to ** determine this. |
︙ | ︙ | |||
877 878 879 880 881 882 883 884 885 886 887 888 | int rc2; while( SQLITE_ROW==sqlite3_step(pScan) ){ int i; ctx.iRowid = sqlite3_column_int64(pScan, 0); ctx.szCol = 0; if( pConfig->bColumnsize ){ rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); } for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){ if( pConfig->abUnindexed[i] ) continue; ctx.iCol = i; ctx.szCol = 0; | > > > > > > > | | | | | | | > | > > > | > > > > | 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 | int rc2; while( SQLITE_ROW==sqlite3_step(pScan) ){ int i; ctx.iRowid = sqlite3_column_int64(pScan, 0); ctx.szCol = 0; if( pConfig->bColumnsize ){ rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); } if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){ rc = sqlite3Fts5TermsetNew(&ctx.pTermset); } for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){ if( pConfig->abUnindexed[i] ) continue; ctx.iCol = i; ctx.szCol = 0; if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ rc = sqlite3Fts5TermsetNew(&ctx.pTermset); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_column_text(pScan, i+1), sqlite3_column_bytes(pScan, i+1), (void*)&ctx, fts5StorageIntegrityCallback ); } if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ rc = FTS5_CORRUPT; } aTotalSize[i] += ctx.szCol; if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ sqlite3Fts5TermsetFree(ctx.pTermset); ctx.pTermset = 0; } } sqlite3Fts5TermsetFree(ctx.pTermset); ctx.pTermset = 0; if( rc!=SQLITE_OK ) break; } rc2 = sqlite3_reset(pScan); if( rc==SQLITE_OK ) rc = rc2; } /* Test that the "totals" (sometimes called "averages") record looks Ok */ |
︙ | ︙ |
Changes to ext/fts5/fts5_tcl.c.
︙ | ︙ | |||
231 232 233 234 235 236 237 238 239 240 241 242 243 244 | { "xColumnText", 1, "COL" }, /* 9 */ { "xColumnSize", 1, "COL" }, /* 10 */ { "xQueryPhrase", 2, "PHRASE SCRIPT" }, /* 11 */ { "xSetAuxdata", 1, "VALUE" }, /* 12 */ { "xGetAuxdata", 1, "CLEAR" }, /* 13 */ { "xSetAuxdataInt", 1, "INTEGER" }, /* 14 */ { "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */ { 0, 0, 0} }; int rc; int iSub = 0; F5tApi *p = (F5tApi*)clientData; | > > | 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 | { "xColumnText", 1, "COL" }, /* 9 */ { "xColumnSize", 1, "COL" }, /* 10 */ { "xQueryPhrase", 2, "PHRASE SCRIPT" }, /* 11 */ { "xSetAuxdata", 1, "VALUE" }, /* 12 */ { "xGetAuxdata", 1, "CLEAR" }, /* 13 */ { "xSetAuxdataInt", 1, "INTEGER" }, /* 14 */ { "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */ { "xPhraseForeach", 4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */ { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */ { 0, 0, 0} }; int rc; int iSub = 0; F5tApi *p = (F5tApi*)clientData; |
︙ | ︙ | |||
424 425 426 427 428 429 430 431 432 433 434 435 436 437 | } CASE(15, "xGetAuxdataInt") { int iVal; int bClear; if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR; iVal = ((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0); Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal)); break; } default: assert( 0 ); break; } | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 | } CASE(15, "xGetAuxdataInt") { int iVal; int bClear; if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR; iVal = ((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0); Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal)); break; } CASE(16, "xPhraseForeach") { int iPhrase; int iCol; int iOff; const char *zColvar; const char *zOffvar; Tcl_Obj *pScript = objv[5]; Fts5PhraseIter iter; if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; zColvar = Tcl_GetString(objv[3]); zOffvar = Tcl_GetString(objv[4]); for(p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff); iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){ Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0); Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0); rc = Tcl_EvalObjEx(interp, pScript, 0); if( rc==TCL_CONTINUE ) rc = TCL_OK; if( rc!=TCL_OK ){ if( rc==TCL_BREAK ) rc = TCL_OK; break; } } break; } CASE(17, "xPhraseColumnForeach") { int iPhrase; int iCol; const char *zColvar; Tcl_Obj *pScript = objv[4]; Fts5PhraseIter iter; if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; zColvar = Tcl_GetString(objv[3]); for(p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol); iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol) ){ Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0); rc = Tcl_EvalObjEx(interp, pScript, 0); if( rc==TCL_CONTINUE ) rc = TCL_OK; if( rc!=TCL_OK ){ if( rc==TCL_BREAK ) rc = TCL_OK; break; } } break; } default: assert( 0 ); break; } |
︙ | ︙ |
Changes to ext/fts5/fts5_test_mi.c.
︙ | ︙ | |||
130 131 132 133 134 135 136 | ){ Fts5PhraseIter iter; int iCol, iOff; u32 *aOut = (u32*)pUserData; int iPrev = -1; for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff); | | | 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 | ){ Fts5PhraseIter iter; int iCol, iOff; u32 *aOut = (u32*)pUserData; int iPrev = -1; for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff); iCol>=0; pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) ){ aOut[iCol*3+1]++; if( iCol!=iPrev ) aOut[iCol*3 + 2]++; iPrev = iCol; } |
︙ | ︙ | |||
207 208 209 210 211 212 213 | char f, u32 *aOut ){ int i; int rc = SQLITE_OK; switch( f ){ | | > > > > > > > > > > > > > > > > > > < < < < | < | 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 | char f, u32 *aOut ){ int i; int rc = SQLITE_OK; switch( f ){ case 'b': { int iPhrase; int nInt = ((p->nCol + 31) / 32) * p->nPhrase; for(i=0; i<nInt; i++) aOut[i] = 0; for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){ Fts5PhraseIter iter; int iCol; for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); iCol>=0; pApi->xPhraseNextColumn(pFts, &iter, &iCol) ){ aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32); } } break; } case 'x': case 'y': { int nMul = (f=='x' ? 3 : 1); int iPhrase; for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0; for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){ Fts5PhraseIter iter; int iOff, iCol; for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); iOff>=0; pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) |
︙ | ︙ |
Changes to ext/fts5/fts5_vocab.c.
︙ | ︙ | |||
375 376 377 378 379 380 381 | int rc = SQLITE_OK; int nCol = pCsr->pConfig->nCol; pCsr->rowid++; if( pTab->eType==FTS5_VOCAB_COL ){ for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){ | | | 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 | int rc = SQLITE_OK; int nCol = pCsr->pConfig->nCol; pCsr->rowid++; if( pTab->eType==FTS5_VOCAB_COL ){ for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){ if( pCsr->aDoc[pCsr->iCol] ) break; } } if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=nCol ){ if( sqlite3Fts5IterEof(pCsr->pIter) ){ pCsr->bEof = 1; }else{ |
︙ | ︙ | |||
408 409 410 411 412 413 414 | assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW ); while( rc==SQLITE_OK ){ i64 dummy; const u8 *pPos; int nPos; /* Position list */ i64 iPos = 0; /* 64-bit position read from poslist */ int iOff = 0; /* Current offset within position list */ | > > | | | | | | | | | | | | | | | | | | > > > > > > > > > > > > > > > > > > > > > > > > > > | | 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 | assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW ); while( rc==SQLITE_OK ){ i64 dummy; const u8 *pPos; int nPos; /* Position list */ i64 iPos = 0; /* 64-bit position read from poslist */ int iOff = 0; /* Current offset within position list */ switch( pCsr->pConfig->eDetail ){ case FTS5_DETAIL_FULL: rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy); if( rc==SQLITE_OK ){ if( pTab->eType==FTS5_VOCAB_ROW ){ while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ pCsr->aCnt[0]++; } pCsr->aDoc[0]++; }else{ int iCol = -1; while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ int ii = FTS5_POS2COLUMN(iPos); pCsr->aCnt[ii]++; if( iCol!=ii ){ pCsr->aDoc[ii]++; iCol = ii; } } } } break; case FTS5_DETAIL_COLUMNS: if( pTab->eType==FTS5_VOCAB_ROW ){ pCsr->aDoc[0]++; }else{ Fts5Buffer buf = {0, 0, 0}; rc = sqlite3Fts5IterPoslistBuffer(pCsr->pIter, &buf); if( rc==SQLITE_OK ){ while( 0==sqlite3Fts5PoslistNext64(buf.p, buf.n, &iOff,&iPos) ){ assert_nc( iPos>=0 && iPos<nCol ); if( iPos<nCol ) pCsr->aDoc[iPos]++; } } sqlite3Fts5BufferFree(&buf); } break; default: assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE ); pCsr->aDoc[0]++; break; } if( rc==SQLITE_OK ){ rc = sqlite3Fts5IterNextScan(pCsr->pIter); } if( rc==SQLITE_OK ){ zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ){ break; } if( sqlite3Fts5IterEof(pCsr->pIter) ) break; } } } } if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){ while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++; assert( pCsr->iCol<pCsr->pConfig->nCol ); } return rc; } /* ** This is the xFilter implementation for the virtual table. |
︙ | ︙ | |||
521 522 523 524 525 526 527 528 529 530 531 532 | static int fts5VocabColumnMethod( sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; if( iCol==0 ){ sqlite3_result_text( pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT ); | > > > < | > | | > | | | | > > | 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 | static int fts5VocabColumnMethod( sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; int eDetail = pCsr->pConfig->eDetail; int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType; i64 iVal = 0; if( iCol==0 ){ sqlite3_result_text( pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT ); }else if( eType==FTS5_VOCAB_COL ){ assert( iCol==1 || iCol==2 || iCol==3 ); if( iCol==1 ){ if( eDetail!=FTS5_DETAIL_NONE ){ const char *z = pCsr->pConfig->azCol[pCsr->iCol]; sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); } }else if( iCol==2 ){ iVal = pCsr->aDoc[pCsr->iCol]; }else{ iVal = pCsr->aCnt[pCsr->iCol]; } }else{ assert( iCol==1 || iCol==2 ); if( iCol==1 ){ iVal = pCsr->aDoc[0]; }else{ iVal = pCsr->aCnt[0]; } } if( iVal>0 ) sqlite3_result_int64(pCtx, iVal); return SQLITE_OK; } /* ** This is the xRowid method. The SQLite core calls this routine to ** retrieve the rowid for the current row of the result set. The ** rowid should be written to *pRowid. |
︙ | ︙ |
Changes to ext/fts5/test/fts5_common.tcl.
︙ | ︙ | |||
23 24 25 26 27 28 29 30 31 32 33 34 35 36 | proc fts5_test_poslist {cmd} { set res [list] for {set i 0} {$i < [$cmd xInstCount]} {incr i} { lappend res [string map {{ } .} [$cmd xInst $i]] } set res } proc fts5_test_columnsize {cmd} { set res [list] for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { lappend res [$cmd xColumnSize $i] } set res | > > > > > > > > > > > > > > > > > > > > > > | 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | proc fts5_test_poslist {cmd} { set res [list] for {set i 0} {$i < [$cmd xInstCount]} {incr i} { lappend res [string map {{ } .} [$cmd xInst $i]] } set res } proc fts5_test_poslist2 {cmd} { set res [list] for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} { $cmd xPhraseForeach $i c o { lappend res $i.$c.$o } } set res } proc fts5_test_collist {cmd} { set res [list] for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} { $cmd xPhraseColumnForeach $i c { lappend res $i.$c } } set res } proc fts5_test_columnsize {cmd} { set res [list] for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { lappend res [$cmd xColumnSize $i] } set res |
︙ | ︙ | |||
109 110 111 112 113 114 115 116 117 118 119 120 121 122 | proc fts5_aux_test_functions {db} { foreach f { fts5_test_columnsize fts5_test_columntext fts5_test_columntotalsize fts5_test_poslist fts5_test_tokenize fts5_test_rowcount fts5_test_all fts5_test_queryphrase fts5_test_phrasecount } { | > > | 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 | proc fts5_aux_test_functions {db} { foreach f { fts5_test_columnsize fts5_test_columntext fts5_test_columntotalsize fts5_test_poslist fts5_test_poslist2 fts5_test_collist fts5_test_tokenize fts5_test_rowcount fts5_test_all fts5_test_queryphrase fts5_test_phrasecount } { |
︙ | ︙ | |||
174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 | # <phrase number> . <column number> . <token offset> # # Options: # # -near N (NEAR distance. Default 10) # -col C (List of column indexes to match against) # -pc VARNAME (variable in caller frame to use for phrase numbering) # proc nearset {aCol args} { set O(-near) 10 set O(-col) {} set O(-pc) "" set nOpt [lsearch -exact $args --] if {$nOpt<0} { error "no -- option" } foreach {k v} [lrange $args 0 [expr $nOpt-1]] { if {[info exists O($k)]==0} { error "unrecognized option $k" } set O($k) $v } if {$O(-pc) == ""} { set counter 0 } else { upvar $O(-pc) counter } | > > > > > > > > > | < < > | | < > > > > | > > > > > > > > > > > > > > > > > > | | < < | > | | | > > > > > | | > | 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 | # <phrase number> . <column number> . <token offset> # # Options: # # -near N (NEAR distance. Default 10) # -col C (List of column indexes to match against) # -pc VARNAME (variable in caller frame to use for phrase numbering) # -dict VARNAME (array in caller frame to use for synonyms) # proc nearset {aCol args} { # Process the command line options. # set O(-near) 10 set O(-col) {} set O(-pc) "" set O(-dict) "" set nOpt [lsearch -exact $args --] if {$nOpt<0} { error "no -- option" } # Set $lPhrase to be a list of phrases. $nPhrase its length. set lPhrase [lrange $args [expr $nOpt+1] end] set nPhrase [llength $lPhrase] foreach {k v} [lrange $args 0 [expr $nOpt-1]] { if {[info exists O($k)]==0} { error "unrecognized option $k" } set O($k) $v } if {$O(-pc) == ""} { set counter 0 } else { upvar $O(-pc) counter } if {$O(-dict)!=""} { upvar $O(-dict) aDict } for {set j 0} {$j < [llength $aCol]} {incr j} { for {set i 0} {$i < $nPhrase} {incr i} { set A($j,$i) [list] } } # Loop through each column of the current row. for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} { # If there is a column filter, test whether this column is excluded. If # so, skip to the next iteration of this loop. Otherwise, set zCol to the # column value and nToken to the number of tokens that comprise it. if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue set zCol [lindex $aCol $iCol] set nToken [llength $zCol] # Each iteration of the following loop searches a substring of the # column value for phrase matches. The last token of the substring # is token $iLast of the column value. The first token is: # # iFirst = ($iLast - $O(-near) - 1) # # where $sz is the length of the phrase being searched for. A phrase # counts as matching the substring if its first token lies on or before # $iLast and its last token on or after $iFirst. # # For example, if the query is "NEAR(a+b c, 2)" and the column value: # # "x x x x A B x x C x" # 0 1 2 3 4 5 6 7 8 9" # # when (iLast==8 && iFirst=5) the range will contain both phrases and # so both instances can be added to the output poslists. # set iLast [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)] for { } {$iLast < $nToken} {incr iLast} { catch { array unset B } for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { set p [lindex $lPhrase $iPhrase] set nPm1 [expr {[llength $p] - 1}] set iFirst [expr $iLast - $O(-near) - [llength $p]] for {set i $iFirst} {$i <= $iLast} {incr i} { set lCand [lrange $zCol $i [expr $i+$nPm1]] set bMatch 1 foreach tok $p term $lCand { if {[nearset_match aDict $tok $term]==0} { set bMatch 0 ; break } } if {$bMatch} { lappend B($iPhrase) $i } } if {![info exists B($iPhrase)]} break } if {$iPhrase==$nPhrase} { for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { set A($iCol,$iPhrase) [concat $A($iCol,$iPhrase) $B($iPhrase)] set A($iCol,$iPhrase) [lsort -integer -uniq $A($iCol,$iPhrase)] } |
︙ | ︙ | |||
248 249 250 251 252 253 254 | foreach a $A($iCol,$iPhrase) { lappend res "$counter.$iCol.$a" } } incr counter } | | > > > > > > > > > > > > | 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 | foreach a $A($iCol,$iPhrase) { lappend res "$counter.$iCol.$a" } } incr counter } #puts "$aCol -> $res" sort_poslist $res } proc nearset_match {aDictVar tok term} { if {[string match $tok $term]} { return 1 } upvar $aDictVar aDict if {[info exists aDict($tok)]} { foreach s $aDict($tok) { if {[string match $s $term]} { return 1 } } } return 0; } #------------------------------------------------------------------------- # Usage: # # sort_poslist LIST # # Sort a position list of the type returned by command [nearset] |
︙ | ︙ | |||
322 323 324 325 326 327 328 329 | lappend ret $word $iOff [expr $iOff+$nToken] incr iOff $nToken incr iOff [gobble_whitespace text] } set ret } | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 | lappend ret $word $iOff [expr $iOff+$nToken] incr iOff $nToken incr iOff [gobble_whitespace text] } set ret } #------------------------------------------------------------------------- # proc foreach_detail_mode {prefix script} { set saved $::testprefix foreach d [list full col none] { set s [string map [list %DETAIL% $d] $script] set ::detail $d set ::testprefix "$prefix-$d" reset_db uplevel $s unset ::detail } set ::testprefix $saved } proc detail_check {} { if {$::detail != "none" && $::detail!="full" && $::detail!="col"} { error "not in foreach_detail_mode {...} block" } } proc detail_is_none {} { detail_check ; expr {$::detail == "none"} } proc detail_is_col {} { detail_check ; expr {$::detail == "col" } } proc detail_is_full {} { detail_check ; expr {$::detail == "full"} } #------------------------------------------------------------------------- # Convert a poslist of the type returned by fts5_test_poslist() to a # collist as returned by fts5_test_collist(). # proc fts5_poslist2collist {poslist} { set res [list] foreach h $poslist { regexp {(.*)\.[1234567890]+} $h -> cand lappend res $cand } set res [lsort -command fts5_collist_elem_compare -unique $res] return $res } # Comparison function used by fts5_poslist2collist to sort collist entries. proc fts5_collist_elem_compare {a b} { foreach {a1 a2} [split $a .] {} foreach {b1 b2} [split $b .] {} if {$a1==$b1} { return [expr $a2 - $b2] } return [expr $a1 - $b1] } #-------------------------------------------------------------------------- # Construct and return a tcl list equivalent to that returned by the SQL # query executed against database handle [db]: # # SELECT # rowid, # fts5_test_poslist($tbl), # fts5_test_collist($tbl) # FROM $tbl('$expr') # ORDER BY rowid $order; # proc fts5_query_data {expr tbl {order ASC} {aDictVar ""}} { # Figure out the set of columns in the FTS5 table. This routine does # not handle tables with UNINDEXED columns, but if it did, it would # have to be here. db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) } set d "" if {$aDictVar != ""} { upvar $aDictVar aDict set d aDict } set cols "" foreach e $lCols { append cols ", '$e'" } set tclexpr [db one [subst -novar { SELECT fts5_expr_tcl( $expr, 'nearset $cols -dict $d -pc ::pc' [set cols] ) }]] set res [list] db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x { set cols [list] foreach col $lCols { lappend cols $x($col) } set ::pc 0 set rowdata [eval $tclexpr] if {$rowdata != ""} { lappend res $x(rowid) $rowdata [fts5_poslist2collist $rowdata] } } set res } #------------------------------------------------------------------------- # Similar to [fts5_query_data], but omit the collist field. # proc fts5_poslist_data {expr tbl {order ASC} {aDictVar ""}} { set res [list] if {$aDictVar!=""} { upvar $aDictVar aDict set dict aDict } else { set dict "" } foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] { lappend res $rowid $poslist } set res } #------------------------------------------------------------------------- # # This command will only work inside a [foreach_detail_mode] block. It tests # whether or not expression $expr run on FTS5 table $tbl is supported by # the current mode. If so, 1 is returned. If not, 0. # # detail=full (all queries supported) # detail=col (all but phrase queries and NEAR queries) # detail=none (all but phrase queries, NEAR queries, and column filters) # proc fts5_expr_ok {expr tbl} { if {![detail_is_full]} { set nearset "nearset_rc" if {[detail_is_col]} { set nearset "nearset_rf" } set ::expr_not_ok 0 db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) } set cols "" foreach e $lCols { append cols ", '$e'" } set ::pc 0 set tclexpr [db one [subst -novar { SELECT fts5_expr_tcl( $expr, '[set nearset] $cols -pc ::pc' [set cols] ) }]] eval $tclexpr if {$::expr_not_ok} { return 0 } } return 1 } # Helper for [fts5_expr_ok] proc nearset_rf {aCol args} { set idx [lsearch -exact $args --] if {$idx != [llength $args]-2 || [llength [lindex $args end]]!=1} { set ::expr_not_ok 1 } list } # Helper for [fts5_expr_ok] proc nearset_rc {aCol args} { nearset_rf $aCol {*}$args if {[lsearch $args -col]>=0} { set ::expr_not_ok 1 } list } |
Changes to ext/fts5/test/fts5aa.test.
︙ | ︙ | |||
17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 | # If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return } do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); SELECT name, sql FROM sqlite_master; } { t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)} t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)} t1_idx {CREATE TABLE 't1_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID} t1_content {CREATE TABLE 't1_content'(id INTEGER PRIMARY KEY, c0, c1, c2)} t1_docsize {CREATE TABLE 't1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)} t1_config {CREATE TABLE 't1_config'(k PRIMARY KEY, v) WITHOUT ROWID} } do_execsql_test 1.1 { DROP TABLE t1; SELECT name, sql FROM sqlite_master; } { } #------------------------------------------------------------------------- # | > > | | | 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | # If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return } foreach_detail_mode $::testprefix { do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); SELECT name, sql FROM sqlite_master; } { t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)} t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)} t1_idx {CREATE TABLE 't1_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID} t1_content {CREATE TABLE 't1_content'(id INTEGER PRIMARY KEY, c0, c1, c2)} t1_docsize {CREATE TABLE 't1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)} t1_config {CREATE TABLE 't1_config'(k PRIMARY KEY, v) WITHOUT ROWID} } do_execsql_test 1.1 { DROP TABLE t1; SELECT name, sql FROM sqlite_master; } { } #------------------------------------------------------------------------- # do_execsql_test 2.0 { CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); } do_execsql_test 2.1 { INSERT INTO t1 VALUES('a b c', 'd e f'); } do_test 2.2 { execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 } |
︙ | ︙ | |||
61 62 63 64 65 66 67 68 69 70 71 72 | SELECT rowid FROM t1 WHERE t1 MATCH $w ORDER BY rowid DESC; } {1} } do_execsql_test 2.4 { INSERT INTO t1(t1) VALUES('integrity-check'); } #------------------------------------------------------------------------- # reset_db do_execsql_test 3.0 { | > | | 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | SELECT rowid FROM t1 WHERE t1 MATCH $w ORDER BY rowid DESC; } {1} } do_execsql_test 2.4 { INSERT INTO t1(t1) VALUES('integrity-check'); } #------------------------------------------------------------------------- # reset_db do_execsql_test 3.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); } foreach {i x y} { 1 {g f d b f} {h h e i a} 2 {f i g j e} {i j c f f} 3 {e e i f a} {e h f d f} 4 {h j f j i} {h a c f j} 5 {d b j c g} {f e i b e} |
︙ | ︙ | |||
89 90 91 92 93 94 95 | if {[set_test_counter errors]} break } #------------------------------------------------------------------------- # reset_db do_execsql_test 4.0 { | | | 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | if {[set_test_counter errors]} break } #------------------------------------------------------------------------- # reset_db do_execsql_test 4.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } foreach {i x y} { 1 {g f d b f} {h h e i a} 2 {f i g j e} {i j c f f} 3 {e e i f a} {e h f d f} 4 {h j f j i} {h a c f j} |
︙ | ︙ | |||
113 114 115 116 117 118 119 | if {[set_test_counter errors]} break } #------------------------------------------------------------------------- # reset_db do_execsql_test 5.0 { | | | 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | if {[set_test_counter errors]} break } #------------------------------------------------------------------------- # reset_db do_execsql_test 5.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } foreach {i x y} { 1 {dd abc abc abc abcde} {aaa dd ddd ddd aab} 2 {dd aab d aaa b} {abcde c aaa aaa aaa} 3 {abcde dd b b dd} {abc abc d abc ddddd} 4 {aaa abcde dddd dddd abcde} {abc b b abcde abc} |
︙ | ︙ | |||
137 138 139 140 141 142 143 | if {[set_test_counter errors]} break } #------------------------------------------------------------------------- # reset_db do_execsql_test 6.0 { | | | 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 | if {[set_test_counter errors]} break } #------------------------------------------------------------------------- # reset_db do_execsql_test 6.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } do_execsql_test 6.1 { INSERT INTO t1(rowid, x, y) VALUES(22, 'a b c', 'c b a'); REPLACE INTO t1(rowid, x, y) VALUES(22, 'd e f', 'f e d'); } |
︙ | ︙ | |||
272 273 274 275 276 277 278 | } #------------------------------------------------------------------------- # reset_db do_execsql_test 10.0 { | | | 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 | } #------------------------------------------------------------------------- # reset_db do_execsql_test 10.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); } set d10 { 1 {g f d b f} {h h e i a} 2 {f i g j e} {i j c f f} 3 {e e i f a} {e h f d f} 4 {h j f j i} {h a c f j} 5 {d b j c g} {f e i b e} |
︙ | ︙ | |||
305 306 307 308 309 310 311 | do_execsql_test 10.4.1 { DELETE FROM t1 } do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') } #------------------------------------------------------------------------- # do_catchsql_test 11.1 { | | | | | | | 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 | do_execsql_test 10.4.1 { DELETE FROM t1 } do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') } #------------------------------------------------------------------------- # do_catchsql_test 11.1 { CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%); } {1 {reserved fts5 column name: rank}} do_catchsql_test 11.2 { CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%); } {1 {reserved fts5 table name: rank}} do_catchsql_test 11.3 { CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%); } {1 {reserved fts5 column name: rowid}} #------------------------------------------------------------------------- # do_execsql_test 12.1 { CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%); } {} do_catchsql_test 12.2 { SELECT t2 FROM t2 WHERE t2 MATCH '*stuff' } {1 {unknown special query: stuff}} do_test 12.3 { set res [db eval { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }] string is integer $res } {1} #------------------------------------------------------------------------- # reset_db do_execsql_test 13.1 { CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o'); } {} do_execsql_test 13.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'o'; } {1 2} |
︙ | ︙ | |||
357 358 359 360 361 362 363 | SELECT rowid FROM t1 WHERE t1 MATCH '""'; } {} #------------------------------------------------------------------------- # reset_db do_execsql_test 14.1 { | | > > > > | 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 | SELECT rowid FROM t1 WHERE t1 MATCH '""'; } {} #------------------------------------------------------------------------- # reset_db do_execsql_test 14.1 { CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); WITH d(x,y) AS ( SELECT NULL, 'xyz xyz xyz xyz xyz xyz' UNION ALL SELECT NULL, 'xyz xyz xyz xyz xyz xyz' FROM d ) INSERT INTO t1 SELECT * FROM d LIMIT 200; } do_execsql_test 15.x { INSERT INTO t1(t1) VALUES('integrity-check'); } do_test 14.2 { set nRow 0 db eval { SELECT * FROM t1 WHERE t1 MATCH 'xyz' } { db eval { BEGIN; CREATE TABLE t2(a, b); |
︙ | ︙ | |||
413 414 415 416 417 418 419 420 421 422 423 | # do_execsql_test 16.1 { CREATE VIRTUAL TABLE n1 USING fts5(a); INSERT INTO n1 VALUES('a b c d'); } proc funk {} { set fd [db incrblob main n1_data block 10] fconfigure $fd -encoding binary -translation binary puts -nonewline $fd "\x44\x45" close $fd | > < | > | | | | | | > | | 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 | # do_execsql_test 16.1 { CREATE VIRTUAL TABLE n1 USING fts5(a); INSERT INTO n1 VALUES('a b c d'); } proc funk {} { db eval { UPDATE n1_config SET v=50 WHERE k='version' } set fd [db incrblob main n1_data block 10] fconfigure $fd -encoding binary -translation binary puts -nonewline $fd "\x44\x45" close $fd } db func funk funk do_catchsql_test 16.2 { SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d' } {1 {SQL logic error or missing database}} #------------------------------------------------------------------------- # reset_db do_execsql_test 17.1 { CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%); INSERT INTO b2 VALUES('a'); INSERT INTO b2 VALUES('b'); INSERT INTO b2 VALUES('c'); } do_test 17.2 { set res [list] db eval { SELECT * FROM b2 ORDER BY rowid ASC } { lappend res [execsql { SELECT * FROM b2 ORDER BY rowid ASC }] } set res } {{a b c} {a b c} {a b c}} if {[string match n* %DETAIL%]==0} { reset_db do_execsql_test 17.3 { CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%); INSERT INTO c2 VALUES('x x x', 'x x x'); SELECT rowid FROM c2 WHERE c2 MATCH 'y:x'; } {1} } #------------------------------------------------------------------------- # reset_db do_execsql_test 17.1 { CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%); INSERT INTO uio VALUES(NULL); INSERT INTO uio SELECT NULL FROM uio; INSERT INTO uio SELECT NULL FROM uio; INSERT INTO uio SELECT NULL FROM uio; INSERT INTO uio SELECT NULL FROM uio; INSERT INTO uio SELECT NULL FROM uio; INSERT INTO uio SELECT NULL FROM uio; |
︙ | ︙ | |||
501 502 503 504 505 506 507 | do_execsql_test 17.9 { SELECT min(rowid), max(rowid), count(*) FROM uio WHERE rowid < 10; } {-9223372036854775808 9 10} #-------------------------------------------------------------------- # do_execsql_test 18.1 { | | | | | > | 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 | do_execsql_test 17.9 { SELECT min(rowid), max(rowid), count(*) FROM uio WHERE rowid < 10; } {-9223372036854775808 9 10} #-------------------------------------------------------------------- # do_execsql_test 18.1 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%); INSERT INTO t1 VALUES('abc*', NULL); INSERT INTO t2 VALUES(1, 'abcdefg'); } do_execsql_test 18.2 { SELECT t1.rowid, t2.rowid FROM t1, t2 WHERE t2 MATCH t1.a AND t1.rowid = t2.c } {1 1} do_execsql_test 18.3 { SELECT t1.rowid, t2.rowid FROM t2, t1 WHERE t2 MATCH t1.a AND t1.rowid = t2.c } {1 1} #-------------------------------------------------------------------- # fts5 table in the temp schema. # reset_db do_execsql_test 19.0 { CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1 VALUES('x y z'); INSERT INTO t1 VALUES('w x 1'); SELECT rowid FROM t1 WHERE t1 MATCH 'x'; } {1 2} #-------------------------------------------------------------------- # Test that 6 and 7 byte varints can be read. # reset_db do_execsql_test 20.0 { CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%); } set ::ids [list \ 0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43] ] do_test 20.1 { foreach id $::ids { execsql { INSERT INTO tmp(rowid, x) VALUES($id, 'x y z') } } execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' } } $::ids } finish_test |
Changes to ext/fts5/test/fts5ab.test.
︙ | ︙ | |||
18 19 20 21 22 23 24 25 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } do_execsql_test 1.0 { | > > | | 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } foreach_detail_mode $testprefix { do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1 VALUES('hello', 'world'); INSERT INTO t1 VALUES('one two', 'three four'); INSERT INTO t1(rowid, a, b) VALUES(45, 'forty', 'five'); } do_execsql_test 1.1 { SELECT * FROM t1 ORDER BY rowid DESC; |
︙ | ︙ | |||
53 54 55 56 57 58 59 | SELECT * FROM t1 WHERE rowid=1.99; } {} #------------------------------------------------------------------------- reset_db do_execsql_test 2.1 { | | | 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | SELECT * FROM t1 WHERE rowid=1.99; } {} #------------------------------------------------------------------------- reset_db do_execsql_test 2.1 { CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); INSERT INTO t1 VALUES('one'); INSERT INTO t1 VALUES('two'); INSERT INTO t1 VALUES('three'); } do_catchsql_test 2.2 { |
︙ | ︙ | |||
155 156 157 158 159 160 161 | } #------------------------------------------------------------------------- # Documents with more than 2M tokens. # do_execsql_test 4.0 { | | > | | > > > > > | | 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 | } #------------------------------------------------------------------------- # Documents with more than 2M tokens. # do_execsql_test 4.0 { CREATE VIRTUAL TABLE s1 USING fts5(x, detail=%DETAIL%); } foreach {tn doc} [list \ 1 [string repeat {a x } 1500000] \ 2 "[string repeat {a a } 1500000] x" \ ] { do_execsql_test 4.$tn { INSERT INTO s1 VALUES($doc) } } do_execsql_test 4.3 { SELECT rowid FROM s1 WHERE s1 MATCH 'x' } {1 2} if {[detail_is_full]} { do_execsql_test 4.4 { SELECT rowid FROM s1 WHERE s1 MATCH '"a x"' } {1 2} } do_execsql_test 4.5 { SELECT rowid FROM s1 WHERE s1 MATCH 'a x' } {1 2} #------------------------------------------------------------------------- # Check that a special case of segment promotion works. The case is where # a new segment is written to level L, but the oldest segment within level # (L-2) is larger than it. # do_execsql_test 5.0 { CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%); INSERT INTO s2(s2, rank) VALUES('pgsz', 32); INSERT INTO s2(s2, rank) VALUES('automerge', 0); } proc rnddoc {n} { set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j] set doc [list] |
︙ | ︙ | |||
218 219 220 221 222 223 224 | } {8 0 0} # Test also the other type of segment promotion - when a new segment is written # that is larger than segments immediately following it. do_test 5.3 { execsql { DROP TABLE s2; | | | | 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 | } {8 0 0} # Test also the other type of segment promotion - when a new segment is written # that is larger than segments immediately following it. do_test 5.3 { execsql { DROP TABLE s2; CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%); INSERT INTO s2(s2, rank) VALUES('pgsz', 32); INSERT INTO s2(s2, rank) VALUES('automerge', 0); } for {set i 1} {$i <= 16} {incr i} { execsql { INSERT INTO s2 VALUES(rnddoc(5)) } } fts5_level_segs s2 } {0 1} do_test 5.4 { execsql { INSERT INTO s2 VALUES(rnddoc(160)) } fts5_level_segs s2 } {2 0} #------------------------------------------------------------------------- # do_execsql_test 6.0 { CREATE VIRTUAL TABLE s3 USING fts5(x, detail=%DETAIL%); BEGIN; INSERT INTO s3 VALUES('a b c'); INSERT INTO s3 VALUES('A B C'); } do_execsql_test 6.1.1 { SELECT rowid FROM s3 WHERE s3 MATCH 'a' |
︙ | ︙ | |||
272 273 274 275 276 277 278 | ROLLBACK; } } {} #------------------------------------------------------------------------- # set doc [string repeat "a b c " 500] | < | > | 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 | ROLLBACK; } } {} #------------------------------------------------------------------------- # set doc [string repeat "a b c " 500] do_execsql_test 7.0 { CREATE VIRTUAL TABLE x1 USING fts5(x, detail=%DETAIL%); INSERT INTO x1(x1, rank) VALUES('pgsz', 32); INSERT INTO x1 VALUES($doc); } } ;# foreach_detail_mode... finish_test |
Changes to ext/fts5/test/fts5ac.test.
︙ | ︙ | |||
18 19 20 21 22 23 24 25 26 27 28 29 30 31 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } set data { 0 {p o q e z k z p n f y u z y n y} {l o o l v v k} 1 {p k h h p y l l h i p v n} {p p l u r i f a j g e r r x w} 2 {l s z j k i m p s} {l w e j t j e e i t w r o p o} 3 {x g y m y m h p} {k j j b r e y y a k y} 4 {q m a i y i z} {o w a g k x g j m w e u k} 5 {k o a w y b s z} {s g l m m l m g p} | > > | 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } foreach_detail_mode $testprefix { set data { 0 {p o q e z k z p n f y u z y n y} {l o o l v v k} 1 {p k h h p y l l h i p v n} {p p l u r i f a j g e r r x w} 2 {l s z j k i m p s} {l w e j t j e e i t w r o p o} 3 {x g y m y m h p} {k j j b r e y y a k y} 4 {q m a i y i z} {o w a g k x g j m w e u k} 5 {k o a w y b s z} {s g l m m l m g p} |
︙ | ︙ | |||
120 121 122 123 124 125 126 127 | 94 {s f e a e t i h h d q p z t q} {b k m k w h c} 95 {h b n j t k i h o q u} {w n g i t o k c a m y p f l x c p} 96 {f c x p y r b m o l m o a} {p c a q s u n n x d c f a o} 97 {u h h k m n k} {u b v n u a o c} 98 {s p e t c z d f n w f} {l s f j b l c e s h} 99 {r c v w i v h a t a c v c r e} {h h u m g o f b a e o} } | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | | | | < < < < < < < < < < < < < < < < | < < > < > < | | | > > | | | | | | | | > | < | > | > > > > < < > > > > | < | | | > | | | | | > | | | | > > | > | | | | < < < < < | > > > < < | < < | | | | | | | | | > > > > | | | | < < | < < | < | < < < | | < < | < > | | | > | | | | | < | | | < < < < < < < < < < | < < < < < | < < | | | | > | > > > > > > > > > > > | 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 | 94 {s f e a e t i h h d q p z t q} {b k m k w h c} 95 {h b n j t k i h o q u} {w n g i t o k c a m y p f l x c p} 96 {f c x p y r b m o l m o a} {p c a q s u n n x d c f a o} 97 {u h h k m n k} {u b v n u a o c} 98 {s p e t c z d f n w f} {l s f j b l c e s h} 99 {r c v w i v h a t a c v c r e} {h h u m g o f b a e o} } foreach {tn2 sql} { 1 {} 2 {BEGIN} } { reset_db fts5_aux_test_functions db do_execsql_test 1.$tn2.0 { CREATE VIRTUAL TABLE xx USING fts5(x,y, detail=%DETAIL%); INSERT INTO xx(xx, rank) VALUES('pgsz', 32); } execsql $sql do_test 1.$tn2.1.1 { foreach {id x y} $data { execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) } } execsql { INSERT INTO xx(xx) VALUES('integrity-check') } } {} #------------------------------------------------------------------------- # do_execsql_test 1.$tn2.integrity { INSERT INTO xx(xx) VALUES('integrity-check'); } #------------------------------------------------------------------------- # foreach {tn expr} { 1.2 "a OR b" 1.1 "a AND b" 1.3 "o" 1.4 "b q" 1.5 "e a e" 1.6 "m d g q q b k b w f q q p p" 1.7 "l o o l v v k" 1.8 "a" 1.9 "b" 1.10 "c" 1.11 "no" 1.12 "L O O L V V K" 1.13 "a AND b AND c" 1.14 "x:a" 2.1 "x:a" 2.2 "y:a" 2.3 "x:b" 2.4 "y:b" 3.1 "{x}:a" 3.2 "{y}:a" 3.3 "{x}:b" 3.4 "{y}:b" 4.1 "{x y}:a" 4.2 "{y x}:a" 4.3 "{x x}:b" 4.4 "{y y}:b" 5.1 {{"x" "y"}:a} 5.2 {{"y" x}:a} 5.3 {{x "x"}:b} 5.4 {{"y" y}:b} 6.1 "b + q" 6.2 "e + a + e" 6.3 "m + d + g + q + q + b + k + b + w + f + q + q + p + p" 6.4 "l + o + o + l + v + v + k" 6.5 "L + O + O + L + V + V + K" 7.1 "a+b AND c" 7.2 "d+c AND u" 7.3 "d+c AND u+d" 7.4 "a+b OR c" 7.5 "d+c OR u" 7.6 "d+c OR u+d" 8.1 "NEAR(a b)" 8.2 "NEAR(r c)" 8.2 { NEAR(r c, 5) } 8.3 { NEAR(r c, 3) } 8.4 { NEAR(r c, 2) } 8.5 { NEAR(r c, 0) } 8.6 { NEAR(a b c) } 8.7 { NEAR(a b c, 8) } 8.8 { x : NEAR(r c) } 8.9 { y : NEAR(r c) } 9.1 { NEAR(r c) } 9.2 { NEAR(r c, 5) } 9.3 { NEAR(r c, 3) } 9.4 { NEAR(r c, 2) } 9.5 { NEAR(r c, 0) } 9.6 { NEAR(a b c) } 9.7 { NEAR(a b c, 8) } 9.8 { x : NEAR(r c) } 9.9 { y : NEAR(r c) } 9.10 { x : "r c" } 9.11 { y : "r c" } 9.12 { a AND b } 9.13 { a AND b AND c } 9.14a { a } 9.14b { a OR b } 9.15 { a OR b AND c } 9.16 { c AND b OR a } 9.17 { c AND (b OR a) } 9.18 { c NOT (b OR a) } 9.19 { (c NOT b) OR (a AND d) } } { if {[fts5_expr_ok $expr xx]==0} { do_test 1.$tn2.$tn.OMITTED { list } [list] continue } set res [fts5_query_data $expr xx] do_execsql_test 1.$tn2.$tn.[llength $res].asc { SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) FROM xx WHERE xx match $expr } $res set res [fts5_query_data $expr xx DESC] do_execsql_test 1.$tn2.$tn.[llength $res].desc { SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) FROM xx WHERE xx match $expr ORDER BY 1 DESC } $res } } } do_execsql_test 2.1 { SELECT fts5_expr_tcl('a AND b'); } {{AND [nearset -- {a}] [nearset -- {b}]}} do_test 2.2.1 { nearset {{a b c}} -- a } {0.0.0} do_test 2.2.2 { nearset {{a b c}} -- c } {0.0.2} foreach {tn expr tclexpr} { 1 {a b} {AND [N $x -- {a}] [N $x -- {b}]} } { do_execsql_test 2.3.$tn { SELECT fts5_expr_tcl($expr, 'N $x') } [list $tclexpr] } finish_test |
Changes to ext/fts5/test/fts5ad.test.
︙ | ︙ | |||
18 19 20 21 22 23 24 25 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } do_execsql_test 1.0 { | > > | | 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } foreach_detail_mode $testprefix { do_execsql_test 1.0 { CREATE VIRTUAL TABLE yy USING fts5(x, y, detail=%DETAIL%); INSERT INTO yy VALUES('Changes the result to be', 'the list of all matching'); INSERT INTO yy VALUES('indices (or all matching', 'values if -inline is'); INSERT INTO yy VALUES('specified as well.) If', 'indices are returned, the'); } {} foreach {tn match res} { 1 {c*} {1} |
︙ | ︙ | |||
49 50 51 52 53 54 55 | do_execsql_test 1.$tn { SELECT rowid FROM yy WHERE yy MATCH $match } $res } foreach {T create} { 2 { | | | | | | 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | do_execsql_test 1.$tn { SELECT rowid FROM yy WHERE yy MATCH $match } $res } foreach {T create} { 2 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } 3 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } 4 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); BEGIN; } 5 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); BEGIN; } } { do_test $T.1 { |
︙ | ︙ | |||
229 230 231 232 233 234 235 236 237 238 239 | set n [llength $res] if {$T==5} breakpoint do_execsql_test $T.$bAsc.$tn.$n $sql $res } } catchsql COMMIT } finish_test | > > | 231 232 233 234 235 236 237 238 239 240 241 242 243 | set n [llength $res] if {$T==5} breakpoint do_execsql_test $T.$bAsc.$tn.$n $sql $res } } catchsql COMMIT } } finish_test |
Changes to ext/fts5/test/fts5ae.test.
︙ | ︙ | |||
18 19 20 21 22 23 24 25 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } do_execsql_test 1.0 { | > > | | 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } foreach_detail_mode $testprefix { do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } do_execsql_test 1.1 { INSERT INTO t1 VALUES('hello', 'world'); SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC; } {1} |
︙ | ︙ | |||
51 52 53 54 55 56 57 | } {1 2 4} fts5_aux_test_functions db #------------------------------------------------------------------------- # do_execsql_test 2.0 { | | > | | | | | > | > | | | | | | | | | > | | | 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | } {1 2 4} fts5_aux_test_functions db #------------------------------------------------------------------------- # do_execsql_test 2.0 { CREATE VIRTUAL TABLE t2 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t2 VALUES('u t l w w m s', 'm f m o l t k o p e'); INSERT INTO t2 VALUES('f g q e l n d m z x q', 'z s i i i m f w w f n g p'); } do_execsql_test 2.1 { SELECT rowid, fts5_test_poslist(t2) FROM t2 WHERE t2 MATCH 'm' ORDER BY rowid; } { 1 {0.0.5 0.1.0 0.1.2} 2 {0.0.7 0.1.5} } do_execsql_test 2.2 { SELECT rowid, fts5_test_poslist(t2) FROM t2 WHERE t2 MATCH 'u OR q' ORDER BY rowid; } { 1 {0.0.0} 2 {1.0.2 1.0.10} } if {[detail_is_full]} { do_execsql_test 2.3 { SELECT rowid, fts5_test_poslist(t2) FROM t2 WHERE t2 MATCH 'y:o' ORDER BY rowid; } { 1 {0.1.3 0.1.7} } } #------------------------------------------------------------------------- # do_execsql_test 3.0 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t3 VALUES( 'j f h o x x a z g b a f a m i b', 'j z c z y x w t'); INSERT INTO t3 VALUES( 'r c', ''); } if {[detail_is_full]} { do_execsql_test 3.1 { SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)'; } { 1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15} } do_execsql_test 3.2 { SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)'; } { 2 {0.0.0 1.0.1} } } do_execsql_test 3.3 { INSERT INTO t3 VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o'); SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'a OR b AND c'; } { 1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15 2.1.2} 3 0.0.5 } #------------------------------------------------------------------------- # do_execsql_test 4.0 { CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t4 VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o'); } do_execsql_test 4.1 { SELECT rowid, fts5_test_poslist(t4) FROM t4 WHERE t4 MATCH 'a OR b AND c'; } { 1 0.0.5 } #------------------------------------------------------------------------- # Test that the xColumnSize() and xColumnAvgsize() APIs work. # reset_db fts5_aux_test_functions db do_execsql_test 5.1 { CREATE VIRTUAL TABLE t5 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t5 VALUES('a b c d', 'e f g h i j'); INSERT INTO t5 VALUES('', 'a'); INSERT INTO t5 VALUES('a', ''); } do_execsql_test 5.2 { SELECT rowid, fts5_test_columnsize(t5) FROM t5 WHERE t5 MATCH 'a' ORDER BY rowid DESC; |
︙ | ︙ | |||
178 179 180 181 182 183 184 | #------------------------------------------------------------------------- # Test the xTokenize() API # reset_db fts5_aux_test_functions db do_execsql_test 6.1 { | | | | 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 | #------------------------------------------------------------------------- # Test the xTokenize() API # reset_db fts5_aux_test_functions db do_execsql_test 6.1 { CREATE VIRTUAL TABLE t6 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t6 VALUES('There are more', 'things in heaven and earth'); INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.'); } do_execsql_test 6.2 { SELECT rowid, fts5_test_tokenize(t6) FROM t6 WHERE t6 MATCH 't*' } { 1 {{there are more} {things in heaven and earth}} 2 {{horatio than are} {dreamt of in your philosophy}} } #------------------------------------------------------------------------- # Test the xQueryPhrase() API # reset_db fts5_aux_test_functions db do_execsql_test 7.1 { CREATE VIRTUAL TABLE t7 USING fts5(x, y, detail=%DETAIL%); } do_test 7.2 { foreach {x y} { {q i b w s a a e l o} {i b z a l f p t e u} {b a z t a l o x d i} {b p a d b f h d w y} {z m h n p p u i e g} {v h d v b x j j c z} {a g i m v a u c b i} {p k s o t l r t b m} |
︙ | ︙ | |||
236 237 238 239 240 241 242 | # SELECT rowid, bm25debug(t7) FROM t7 WHERE t7 MATCH 'a'; #} {5 5 5 5} # #------------------------------------------------------------------------- # do_test 8.1 { | | | 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 | # SELECT rowid, bm25debug(t7) FROM t7 WHERE t7 MATCH 'a'; #} {5 5 5 5} # #------------------------------------------------------------------------- # do_test 8.1 { execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y, detail=%DETAIL%) } foreach {rowid x y} { 0 {A o} {o o o C o o o o o o o o} 1 {o o B} {o o o C C o o o o o o o} 2 {A o o} {o o o o D D o o o o o o} 3 {o B} {o o o o o D o o o o o o} 4 {E o G} {H o o o o o o o o o o o} 5 {F o G} {I o J o o o o o o o o o} |
︙ | ︙ | |||
294 295 296 297 298 299 300 301 302 303 304 | 2 {a OR b} 2 3 {a OR b OR c} 3 4 {NEAR(a b)} 2 } { do_execsql_test 9.2.$tn { SELECT fts5_test_phrasecount(t9) FROM t9 WHERE t9 MATCH $q LIMIT 1 } $cnt } finish_test | > > | 300 301 302 303 304 305 306 307 308 309 310 311 312 | 2 {a OR b} 2 3 {a OR b OR c} 3 4 {NEAR(a b)} 2 } { do_execsql_test 9.2.$tn { SELECT fts5_test_phrasecount(t9) FROM t9 WHERE t9 MATCH $q LIMIT 1 } $cnt } } finish_test |
Changes to ext/fts5/test/fts5af.test.
︙ | ︙ | |||
20 21 22 23 24 25 26 27 28 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } do_execsql_test 1.0 { | > | | 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } foreach_detail_mode $testprefix { do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); } proc do_snippet_test {tn doc match res} { uplevel #0 [list set v1 $doc] uplevel #0 [list set v2 $match] |
︙ | ︙ | |||
107 108 109 110 111 112 113 | 7.4 {o o o X o o X o o} {...o [X] o o [X] o o} 7.5 {o o o o X o o X o} {...o o [X] o o [X] o} 7.6 {o o o o o X o o X} {...o o o [X] o o [X]} } { do_snippet_test 1.$tn $doc X $res } | > | | | | | | | | | | | | | | | | | | | | | | < | | | | > > > | 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 | 7.4 {o o o X o o X o o} {...o [X] o o [X] o o} 7.5 {o o o o X o o X o} {...o o [X] o o [X] o} 7.6 {o o o o o X o o X} {...o o o [X] o o [X]} } { do_snippet_test 1.$tn $doc X $res } if {[detail_is_full]} { foreach {tn doc res} { 1.1 {X Y o o o o o} {[X Y] o o o o o} 1.2 {o X Y o o o o} {o [X Y] o o o o} 1.3 {o o X Y o o o} {o o [X Y] o o o} 1.4 {o o o X Y o o} {o o o [X Y] o o} 1.5 {o o o o X Y o} {o o o o [X Y] o} 1.6 {o o o o o X Y} {o o o o o [X Y]} 2.1 {X Y o o o o o o} {[X Y] o o o o o...} 2.2 {o X Y o o o o o} {o [X Y] o o o o...} 2.3 {o o X Y o o o o} {o o [X Y] o o o...} 2.4 {o o o X Y o o o} {...o o [X Y] o o o} 2.5 {o o o o X Y o o} {...o o o [X Y] o o} 2.6 {o o o o o X Y o} {...o o o o [X Y] o} 2.7 {o o o o o o X Y} {...o o o o o [X Y]} 3.1 {X Y o o o o o o o} {[X Y] o o o o o...} 3.2 {o X Y o o o o o o} {o [X Y] o o o o...} 3.3 {o o X Y o o o o o} {o o [X Y] o o o...} 3.4 {o o o X Y o o o o} {...o o [X Y] o o o...} 3.5 {o o o o X Y o o o} {...o o [X Y] o o o} 3.6 {o o o o o X Y o o} {...o o o [X Y] o o} 3.7 {o o o o o o X Y o} {...o o o o [X Y] o} 3.8 {o o o o o o o X Y} {...o o o o o [X Y]} } { do_snippet_test 2.$tn $doc "X + Y" $res } } } ;# foreach_detail_mode finish_test |
Changes to ext/fts5/test/fts5ag.test.
︙ | ︙ | |||
29 30 31 32 33 34 35 36 | # ... WHERE fts MATCH ? ORDER BY bm25(fts) [ASC|DESC] # # and # # ... WHERE fts MATCH ? ORDER BY rank [ASC|DESC] # do_execsql_test 1.0 { | > > | | 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | # ... WHERE fts MATCH ? ORDER BY bm25(fts) [ASC|DESC] # # and # # ... WHERE fts MATCH ? ORDER BY rank [ASC|DESC] # foreach_detail_mode $testprefix { do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x, y, z, detail=%DETAIL%); } do_test 1.1 { foreach {x y z} { {j s m y m r n l u k} {z k f u z g h s w g} {r n o s s b v n w w} {m v g n d x q r r s} {q t d a q a v l h j} {s k l f s i n v q v} {m f f d h h s o h a} {y e v r q i u m h d} {b c k q m z l z h n} |
︙ | ︙ | |||
115 116 117 118 119 120 121 | foreach {tn expr} { 2.1 a 2.2 b 2.3 c 2.4 d | < < < > > > > > > > > | | | | < > | 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 | foreach {tn expr} { 2.1 a 2.2 b 2.3 c 2.4 d 3.0 {a AND b} 3.1 {a OR b} 3.2 {b OR c AND d} } { do_fts5ag_test $tn $expr } if {[detail_is_full]} { foreach {tn expr} { 4.1 {"m m"} 4.2 {e + s} 4.3 {NEAR(c d)} } { do_fts5ag_test $tn $expr } } } ;# foreach_detail_mode finish_test |
Changes to ext/fts5/test/fts5ah.test.
︙ | ︙ | |||
16 17 18 19 20 21 22 23 24 25 26 27 28 | set testprefix fts5ah # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } #------------------------------------------------------------------------- # This file contains tests for very large doclists. # do_test 1.0 { | > > > > | | 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 | set testprefix fts5ah # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } foreach_detail_mode $testprefix { #------------------------------------------------------------------------- # This file contains tests for very large doclists. # set Y [list] set W [list] do_test 1.0 { execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%) } execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 128) } set v {w w w w w w w w w w w w w w w w w w w w} execsql { INSERT INTO t1(rowid, a) VALUES(0, $v) } for {set i 1} {$i <= 10000} {incr i} { set v {x x x x x x x x x x x x x x x x x x x x} if {($i % 2139)==0} {lset v 3 Y ; lappend Y $i} if {($i % 1577)==0} {lset v 5 W ; lappend W $i} |
︙ | ︙ | |||
66 67 68 69 70 71 72 | expr [reads] - $nRead } do_test 1.4 { set nRead [reads] execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'x' } set nReadX [expr [reads] - $nRead] | > > > > > | > > > > > | | | | | | | | | | | > > > > > > | 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 | expr [reads] - $nRead } do_test 1.4 { set nRead [reads] execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'x' } set nReadX [expr [reads] - $nRead] #puts -nonewline "(nReadX=$nReadX)" if {[detail_is_full]} { set expect 1000 } if {[detail_is_col]} { set expect 250 } if {[detail_is_none]} { set expect 80 } expr $nReadX>$expect } {1} do_test 1.5 { set fwd [execsql_reads {SELECT rowid FROM t1 WHERE t1 MATCH 'x' }] set bwd [execsql_reads { SELECT rowid FROM t1 WHERE t1 MATCH 'x' ORDER BY 1 ASC }] expr {$bwd < $fwd + 12} } {1} foreach {tn q res} " 1 { SELECT rowid FROM t1 WHERE t1 MATCH 'w + x' } [list $W] 2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x + w' } [list $W] 3 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w' } [list $W] 4 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x' } [list $Y] " { if {[detail_is_full]==0 && ($tn==1 || $tn==2)} continue if {[detail_is_full]} { set ratio 8 } if {[detail_is_col]} { set ratio 4 } if {[detail_is_none]} { set ratio 2 } do_test 1.6.$tn.1 { set n [execsql_reads $q] #puts -nonewline "(n=$n nReadX=$nReadX)" expr {$n < ($nReadX / $ratio)} } {1} do_test 1.6.$tn.2 { set n [execsql_reads "$q ORDER BY rowid DESC"] #puts -nonewline "(n=$n nReadX=$nReadX)" expr {$n < ($nReadX / $ratio)} } {1} do_execsql_test 1.6.$tn.3 $q [lsort -int -incr $res] do_execsql_test 1.6.$tn.4 "$q ORDER BY rowid DESC" [lsort -int -decr $res] } #------------------------------------------------------------------------- # Now test that adding range constraints on the rowid field reduces the # number of pages loaded from disk. # foreach {tn fraction tail cnt} { 1 0.6 {rowid > 5000} 5000 2 0.2 {rowid > 9000} 1000 3 0.2 {rowid < 1000} 999 4 0.2 {rowid BETWEEN 4000 AND 5000} 1001 5 0.6 {rowid >= 5000} 5001 6 0.2 {rowid >= 9000} 1001 7 0.2 {rowid <= 1000} 1000 8 0.6 {rowid > '5000'} 5000 9 0.2 {rowid > '9000'} 1000 10 0.1 {rowid = 444} 1 } { set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail" set n [execsql_reads $q] set ret [llength [execsql $q]] # Because the position lists for 'x' are quite long in this db, the # advantage is a bit smaller in detail=none mode. Update $fraction to # reflect this. if {[detail_is_none] && $fraction<0.5} { set fraction [expr $fraction*2] } do_test "1.7.$tn.asc.(n=$n ret=$ret)" { expr {$n < ($fraction*$nReadX) && $ret==$cnt} } {1} set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail ORDER BY rowid DESC" set n [execsql_reads $q] set ret [llength [execsql $q]] do_test "1.7.$tn.desc.(n=$n ret=$ret)" { expr {$n < 2*$fraction*$nReadX && $ret==$cnt} } {1} } do_execsql_test 1.8.1 { SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND +rowid < 'text'; } {10000} do_execsql_test 1.8.2 { SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text'; } {10000} } ;# foreach_detail_mode #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} finish_test |
Changes to ext/fts5/test/fts5ai.test.
︙ | ︙ | |||
19 20 21 22 23 24 25 26 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } do_execsql_test 1.0 { | > > | | 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } foreach_detail_mode $testprefix { do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%); } {} do_execsql_test 1.1 { BEGIN; INSERT INTO t1 VALUES('a b c'); INSERT INTO t1 VALUES('d e f'); SAVEPOINT one; |
︙ | ︙ | |||
44 45 46 47 48 49 50 51 52 53 54 55 | INSERT INTO t1 VALUES('s t u'); ROLLBACK TO one; COMMIT; } do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check'); } finish_test | > | 46 47 48 49 50 51 52 53 54 55 56 57 58 | INSERT INTO t1 VALUES('s t u'); ROLLBACK TO one; COMMIT; } do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check'); } } finish_test |
Changes to ext/fts5/test/fts5ak.test.
︙ | ︙ | |||
19 20 21 22 23 24 25 26 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } do_execsql_test 1.1 { | > > | > > > < < < < < < < < < < < < < | > > > > > > > > > > > > > > > > > > | > > > > > > > | < < > > > | | | | < < < < < < < < < < < < | > | 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } foreach_detail_mode $testprefix { do_execsql_test 1.1 { CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%); INSERT INTO ft1 VALUES('i d d a g i b g d d'); INSERT INTO ft1 VALUES('h d b j c c g a c a'); INSERT INTO ft1 VALUES('e j a e f h b f h h'); INSERT INTO ft1 VALUES('j f h d g h i b d f'); INSERT INTO ft1 VALUES('d c j d c j b c g e'); INSERT INTO ft1 VALUES('i a d e g j g d a a'); INSERT INTO ft1 VALUES('j f c e d a h j d b'); INSERT INTO ft1 VALUES('i c c f a d g h j e'); INSERT INTO ft1 VALUES('i d i g c d c h b f'); INSERT INTO ft1 VALUES('g d a e h a b c f j'); CREATE VIRTUAL TABLE ft2 USING fts5(x, detail=%DETAIL%); INSERT INTO ft2 VALUES('a b c d e f g h i j'); } do_execsql_test 1.2 { SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'e'; } { {[e] j a [e] f h b f h h} {d c j d c j b c g [e]} {i a d [e] g j g d a a} {j f c [e] d a h j d b} {i c c f a d g h j [e]} {g d a [e] h a b c f j} } do_execsql_test 1.3 { SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'e e e' } { {[e] j a [e] f h b f h h} {d c j d c j b c g [e]} {i a d [e] g j g d a a} {j f c [e] d a h j d b} {i c c f a d g h j [e]} {g d a [e] h a b c f j} } do_execsql_test 1.4 { SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'f d' } { {a b c [d] e [f] g h i j} } do_execsql_test 1.5 { SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'd f' } { {a b c [d] e [f] g h i j} } #------------------------------------------------------------------------- # Tests below this point require detail=full. #------------------------------------------------------------------------- if {[detail_is_full]==0} continue do_execsql_test 2.1 { SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'h + d'; } { {[h d] b j c c g a c a} {j f [h d] g h i b d f} } do_execsql_test 2.2 { SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d'; } { {i [d d] a g i b g [d d]} } do_execsql_test 2.3 { SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d d + d'; } { {i [d d] a g i b g [d d]} } do_execsql_test 2.4 { SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c+d+e' } {{a [b c d e] f g h i j}} do_execsql_test 2.5 { SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d e+f+g' } { {a [b c d] [e f g] h i j} } do_execsql_test 2.6 { SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c' } { {a [b c d] e f g h i j} } do_execsql_test 2.7 { SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c c+d+e' } { {a [b c d e] f g h i j} } #------------------------------------------------------------------------- # The example from the docs. # do_execsql_test 3.1 { -- Assuming this: CREATE VIRTUAL TABLE ft USING fts5(a, detail=%DETAIL%); INSERT INTO ft VALUES('a b c x c d e'); INSERT INTO ft VALUES('a b c c d e'); INSERT INTO ft VALUES('a b c d e'); -- The following SELECT statement returns these three rows: -- '[a b c] x [c d e]' -- '[a b c] [c d e]' -- '[a b c d e]' SELECT highlight(ft, 0, '[', ']') FROM ft WHERE ft MATCH 'a+b+c AND c+d+e'; } { {[a b c] x [c d e]} {[a b c] [c d e]} {[a b c d e]} } } finish_test |
Changes to ext/fts5/test/fts5al.test.
︙ | ︙ | |||
19 20 21 22 23 24 25 26 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } do_execsql_test 1.1 { | > > | | 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } foreach_detail_mode $testprefix { do_execsql_test 1.1 { CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%); SELECT * FROM ft1_config; } {version 4} do_execsql_test 1.2 { INSERT INTO ft1(ft1, rank) VALUES('pgsz', 32); SELECT * FROM ft1_config; } {pgsz 32 version 4} |
︙ | ︙ | |||
79 80 81 82 83 84 85 | } #------------------------------------------------------------------------- # Assorted tests of the tcl interface for creating extension functions. # do_execsql_test 3.1 { | | | 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 | } #------------------------------------------------------------------------- # Assorted tests of the tcl interface for creating extension functions. # do_execsql_test 3.1 { CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1 VALUES('q w e r t y'); INSERT INTO t1 VALUES('y t r e w q'); } proc argtest {cmd args} { return $args } sqlite3_fts5_create_function db argtest argtest |
︙ | ︙ | |||
118 119 120 121 122 123 124 | do_execsql_test 3.4.1 { SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'q' } { {{0 0 0}} {{0 0 5}} } | > | | | | | > | 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 | do_execsql_test 3.4.1 { SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'q' } { {{0 0 0}} {{0 0 5}} } if {[detail_is_full]} { do_execsql_test 3.4.2 { SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'r+e OR w' } { {{1 0 1}} {{0 0 2} {1 0 4}} } } proc coltest {cmd} { list [$cmd xColumnSize 0] [$cmd xColumnText 0] } sqlite3_fts5_create_function db coltest coltest |
︙ | ︙ | |||
145 146 147 148 149 150 151 | # Tests for remapping the "rank" column. # # 4.1.*: Mapped to a function with no arguments. # 4.2.*: Mapped to a function with one or more arguments. # do_execsql_test 4.0 { | | | 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 | # Tests for remapping the "rank" column. # # 4.1.*: Mapped to a function with no arguments. # 4.2.*: Mapped to a function with one or more arguments. # do_execsql_test 4.0 { CREATE VIRTUAL TABLE t2 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t2 VALUES('a s h g s b j m r h', 's b p a d b b a o e'); INSERT INTO t2 VALUES('r h n t a g r d d i', 'l d n j r c f t o q'); INSERT INTO t2 VALUES('q k n i k c a a e m', 'c h n j p g s c i t'); INSERT INTO t2 VALUES('h j g t r e l s g s', 'k q k c i i c k n s'); INSERT INTO t2 VALUES('b l k h d n n n m i', 'p t i a r b t q o l'); INSERT INTO t2 VALUES('k r i l j b g i p a', 't q c h a i m g n l'); INSERT INTO t2 VALUES('a e c q n m o m d g', 'l c t g i s q g q e'); |
︙ | ︙ | |||
214 215 216 217 218 219 220 | } proc rowidplus {cmd ival} { expr [$cmd xRowid] + $ival } sqlite3_fts5_create_function db rowidplus rowidplus | > | | | | | | | | | | | | | | | | | > | | 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 | } proc rowidplus {cmd ival} { expr [$cmd xRowid] + $ival } sqlite3_fts5_create_function db rowidplus rowidplus if {[detail_is_full]} { do_execsql_test 4.2.1 { INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(100) '); SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g' } { 10 110 } do_execsql_test 4.2.2 { INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(111) '); SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g' } { 10 121 } do_execsql_test 4.2.3 { SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g' AND rank MATCH 'rowidplus(112)' } { 10 122 } } proc rowidmod {cmd imod} { expr [$cmd xRowid] % $imod } sqlite3_fts5_create_function db rowidmod rowidmod do_execsql_test 4.3.1 { CREATE VIRTUAL TABLE t3 USING fts5(x, detail=%DETAIL%); INSERT INTO t3 VALUES('a one'); INSERT INTO t3 VALUES('a two'); INSERT INTO t3 VALUES('a three'); INSERT INTO t3 VALUES('a four'); INSERT INTO t3 VALUES('a five'); INSERT INTO t3(t3, rank) VALUES('rank', 'bm25()'); } |
︙ | ︙ | |||
283 284 285 286 287 288 289 290 291 292 293 | do_catchsql_test 4.4.3 { SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'xyz(3)' } {1 {no such function: xyz}} do_catchsql_test 4.4.4 { SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL } {1 {parse error in rank function: }} finish_test | > | 289 290 291 292 293 294 295 296 297 298 299 300 | do_catchsql_test 4.4.3 { SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'xyz(3)' } {1 {no such function: xyz}} do_catchsql_test 4.4.4 { SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL } {1 {parse error in rank function: }} } ;# foreach_detail_mode finish_test |
Changes to ext/fts5/test/fts5auto.test.
︙ | ︙ | |||
18 19 20 21 22 23 24 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } | < | 18 19 20 21 22 23 24 25 26 27 28 29 30 31 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } set data { -4026076 {n x w k b p x b n t t d s} {f j j s p j o} {w v i y r} {i p y s} {a o q v e n q r} {q v g u c y a z y} 3995120 {c} {e e w d t} |
︙ | ︙ | |||
228 229 230 231 232 233 234 | do_execsql_test 1.0 { CREATE VIRTUAL TABLE tt USING fts5(a, b, c, d, e, f); } {} fts5_aux_test_functions db | < < < < < < < < < < < < < < < < < < < < < < < < < < < < | | < < | 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 | do_execsql_test 1.0 { CREATE VIRTUAL TABLE tt USING fts5(a, b, c, d, e, f); } {} fts5_aux_test_functions db proc do_auto_test {tn tbl expr} { foreach order {asc desc} { set res [fts5_poslist_data $expr $tbl $order] set testname "$tn.[string range $order 0 0].rows=[expr [llength $res]/2]" set ::autotest_expr $expr do_execsql_test $testname [subst -novar { SELECT rowid, fts5_test_poslist([set tbl]) FROM [set tbl] WHERE [set tbl] MATCH $::autotest_expr ORDER BY rowid [set order] }] $res } } #------------------------------------------------------------------------- # for {set fold 0} {$fold < 3} {incr fold} { switch $fold { |
︙ | ︙ | |||
328 329 330 331 332 333 334 | B.4 { a OR (b AND {a b c}:c) } B.5 { a OR "b c" } B.6 { a OR b OR c } C.1 { a OR (b AND "b c") } C.2 { a OR (b AND "z c") } } { | | | 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 | B.4 { a OR (b AND {a b c}:c) } B.5 { a OR "b c" } B.6 { a OR b OR c } C.1 { a OR (b AND "b c") } C.2 { a OR (b AND "z c") } } { do_auto_test 3.$fold.$tn tt $expr } } proc replace_elems {list args} { set ret $list foreach {idx elem} $args { set ret [lreplace $ret $idx $idx $elem] |
︙ | ︙ | |||
362 363 364 365 366 367 368 | 1 x 2 y 3 z 4 {c1 : x} 5 {c2 : x} 6 {c3 : x} 7 {c1 : y} 8 {c2 : y} 9 {c3 : y} 10 {c1 : z} 11 {c2 : z} 12 {c3 : z} | < < < | | 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 | 1 x 2 y 3 z 4 {c1 : x} 5 {c2 : x} 6 {c3 : x} 7 {c1 : y} 8 {c2 : y} 9 {c3 : y} 10 {c1 : z} 11 {c2 : z} 12 {c3 : z} } { do_auto_test 4.$tn yy $expr } finish_test |
Added ext/fts5/test/fts5detail.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 | # 2015 December 18 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # This file implements regression tests for SQLite library. The # focus of this script is testing the FTS5 module. # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5detail # If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return } fts5_aux_test_functions db #-------------------------------------------------------------------------- # Simple tests. # do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col); INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1 INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2 INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3 INSERT INTO t1 VALUES('e g g', 'g e d h i', 'e d b e g d c'); -- 4 INSERT INTO t1 VALUES('b c c', 'd i h a f', 'd i j f a b c'); -- 5 INSERT INTO t1 VALUES('e d e', 'b c j g d', 'a i f d h b d'); -- 6 INSERT INTO t1 VALUES('g h e', 'b c d i d', 'e f c i f i c'); -- 7 INSERT INTO t1 VALUES('c f j', 'j j i e a', 'h a c f d h e'); -- 8 INSERT INTO t1 VALUES('a h i', 'c i a f a', 'c f d h g d g'); -- 9 INSERT INTO t1 VALUES('j g g', 'e f e f f', 'h j b i c g e'); -- 10 } do_execsql_test 1.1 { INSERT INTO t1(t1) VALUES('integrity-check'); } foreach {tn match res} { 1 "a:a" {9} 2 "b:g" {1 4 6} 3 "c:h" {1 3 6 8 9 10} } { do_execsql_test 1.2.$tn.1 { SELECT rowid FROM t1($match); } $res do_execsql_test 1.2.$tn.2 { SELECT rowid FROM t1($match || '*'); } $res } do_catchsql_test 1.3.1 { SELECT rowid FROM t1('h + d'); } {1 {fts5: phrase queries are not supported (detail!=full)}} do_catchsql_test 1.3.2 { SELECT rowid FROM t1('NEAR(h d)'); } {1 {fts5: NEAR queries are not supported (detail!=full)}} #------------------------------------------------------------------------- # integrity-check with both detail= and prefix= options. # do_execsql_test 2.0 { CREATE VIRTUAL TABLE t2 USING fts5(a, detail=col, prefix="1"); INSERT INTO t2(a) VALUES('aa ab'); } #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t2_data} {puts $r} do_execsql_test 2.1 { INSERT INTO t2(t2) VALUES('integrity-check'); } do_execsql_test 2.2 { SELECT fts5_test_poslist(t2) FROM t2('aa'); } {0.0.0} set ::pc 0 #puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*] #exit #------------------------------------------------------------------------- # Check that the xInstCount, xInst, xPhraseFirst and xPhraseNext APIs # work with detail=col tables. # set data { 1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca} 2 {bca aca acb} {ccb bcc bca aab bcc} {bab aaa aac cbb bba aca abc} 3 {cca abc cab} {aab aba bcc cac baa} {bab cbb acb aba aab ccc cca} 4 {ccb bcb aba} {aba bbb bcc cac bbb} {cbb aaa bca bcc aab cac aca} 5 {bca bbc cac} {aba cbb cac cca aca} {cab acb cbc ccb cac bbb bcb} 6 {acc bba cba} {bab bbc bbb bcb aca} {bca ccc cbb aca bac ccc ccb} 7 {aba bab aaa} {abb bca aac bcb bcc} {bcb bbc aba aaa cba abc acc} 8 {cab aba aaa} {ccb aca caa bbc bcc} {aaa abc ccb bbb cac cca abb} 9 {bcb bab bac} {bcb cba cac bbb abc} {aba aca cbb acb abb ccc ccb} 10 {aba aab ccc} {abc ccc bcc cab bbb} {aab bcc cbb ccc aaa bac baa} 11 {bab acb cba} {aac cab cab bca cbc} {aab cbc aac baa ccb acc cac} 12 {ccc cbb cbc} {aaa aab bcc aac bbc} {cbc cbc bac bac ccc bbc acc} 13 {cab bbc abc} {bbb bab bba aca bab} {baa bbb aab bbb ccb bbb ccc} 14 {bbc cab caa} {acb aac abb cba acc} {cba bba bba acb abc abb baa} 15 {aba cca bcc} {aaa acb abc aab ccb} {cca bcb acc aaa caa cca cbc} 16 {bcb bba aba} {cbc acb cab caa ccb} {aac aaa bbc cab cca cba abc} 17 {caa cbb acc} {ccb bcb bca aaa bcc} {bbb aca bcb bca cbc cbc cca} 18 {cbb bbc aac} {ccc bbc aaa aab baa} {cab cab cac cca bbc abc bbc} 19 {ccc acc aaa} {aab cbb bca cca caa} {bcb aca aca cab acc bac bcc} 20 {aab ccc bcb} {bbc cbb bbc aaa bcc} {cbc aab ccc aaa bcb bac cbc} 21 {aba cab ccc} {bbc cbc cba acc bbb} {acc aab aac acb aca bca acb} 22 {bcb bca baa} {cca bbc aca ccb cbb} {aab abc bbc aaa cab bcc bcc} 23 {cac cbb caa} {bbc aba bbb bcc ccb} {bbc bbb cab bbc cac abb acc} 24 {ccb acb caa} {cab bba cac bbc aac} {aac bca abc cab bca cab bcb} 25 {bbb aca bca} {bcb acc ccc cac aca} {ccc acb acc cac cac bba bbc} 26 {bab acc caa} {caa cab cac bac aca} {aba cac caa acc bac ccc aaa} 27 {bca bca aaa} {ccb aca bca aaa baa} {bab acc aaa cca cba cca bac} 28 {ccb cac cac} {bca abb bba bbc baa} {aca ccb aac cab ccc cab caa} 29 {abc bca cab} {cac cbc cbb ccc bcc} {bcc aaa aaa acc aac cac aac} 30 {aca acc acb} {aab aac cbb caa acb} {acb bbc bbc acc cbb bbc aac} 31 {aba aca baa} {aca bcc cab bab acb} {bcc acb baa bcb bbc acc aba} 32 {abb cbc caa} {cba abb bbb cbb aca} {bac aca caa cac caa ccb bbc} 33 {bcc bcb bcb} {cca cab cbc abb bab} {caa bbc aac bbb cab cba aaa} 34 {caa cab acc} {ccc ccc bcc acb bcc} {bac bba aca bcb bba bcb cac} 35 {bac bcb cba} {bcc acb bbc cba bab} {abb cbb abc abc bac acc cbb} 36 {cab bab ccb} {bca bba bab cca acc} {acc aab bcc bac acb cbb caa} 37 {aca cbc cab} {bba aac aca aac aaa} {baa cbb cba aba cab bca bcb} 38 {acb aab baa} {baa bab bca bbc bbb} {abc baa acc aba cab baa cac} 39 {bcb aac cba} {bcb baa caa cac bbc} {cbc ccc bab ccb bbb caa aba} 40 {cba ccb abc} {cbb caa cba aac bab} {cbb bbb bca bbb bac cac bca} } set data { 1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca} } proc matchdata {expr {bAsc 1}} { set tclexpr [db one { SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y', 'z') }] set res [list] #puts "$expr -> $tclexpr" foreach {id x y z} $::data { set cols [list $x $y $z] set ::pc 0 #set hits [lsort -command instcompare [eval $tclexpr]] set hits [eval $tclexpr] if {[llength $hits]>0} { lappend res [list $id $hits] } } if {$bAsc} { set res [lsort -integer -increasing -index 0 $res] } else { set res [lsort -integer -decreasing -index 0 $res] } return [concat {*}$res] } foreach {tn tbl} { 1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=col) } 2 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=none) } } { reset_db fts5_aux_test_functions db execsql $tbl foreach {id x y z} $data { execsql { INSERT INTO t3(rowid, x, y, z) VALUES($id, $x, $y, $z) } } foreach {tn2 expr} { 1 aaa 2 ccc 3 bab 4 aac 5 aa* 6 cc* 7 ba* 8 aa* 9 a* 10 b* 11 c* } { set res [matchdata $expr] do_execsql_test 3.$tn.$tn2.1 { SELECT rowid, fts5_test_poslist(t3) FROM t3($expr) } $res do_execsql_test 3.$tn.$tn2.2 { SELECT rowid, fts5_test_poslist2(t3) FROM t3($expr) } $res } } #------------------------------------------------------------------------- # Simple tests for detail=none tables. # do_execsql_test 4.0 { CREATE VIRTUAL TABLE t4 USING fts5(a, b, c, detail=none); INSERT INTO t4 VALUES('a b c', 'b c d', 'e f g'); INSERT INTO t4 VALUES('1 2 3', '4 5 6', '7 8 9'); } do_catchsql_test 4.1 { SELECT * FROM t4('a:a') } {1 {fts5: column queries are not supported (detail=none)}} #------------------------------------------------------------------------- # Test that for the same content detail=none uses less space than # detail=col, and that detail=col uses less space than detail=full # reset_db do_test 5.1 { foreach {tbl detail} {t1 none t2 col t3 full} { execsql "CREATE VIRTUAL TABLE $tbl USING fts5(x, y, z, detail=$detail)" foreach {rowid x y z} $::data { execsql "INSERT INTO $tbl (rowid, x, y, z) VALUES(\$rowid, \$x, \$y, \$z)" } } } {} do_execsql_test 5.2 { SELECT (SELECT sum(length(block)) from t1_data) < (SELECT sum(length(block)) from t2_data) } {1} do_execsql_test 5.3 { SELECT (SELECT sum(length(block)) from t2_data) < (SELECT sum(length(block)) from t3_data) } {1} finish_test |
Changes to ext/fts5/test/fts5dlidx.test.
︙ | ︙ | |||
22 23 24 25 26 27 28 | } if { $tcl_platform(wordSize)<8 } { finish_test return } | | | | 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | } if { $tcl_platform(wordSize)<8 } { finish_test return } foreach_detail_mode $testprefix { proc do_fb_test {tn sql res} { set res2 [lsort -integer -decr $res] uplevel [list do_execsql_test $tn.1 $sql $res] uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2] } # This test populates the FTS5 table with $nEntry entries. Rows are # numbered from 0 to ($nEntry-1). The rowid for row $i is: # # ($iFirst + $i*$nStep) # # Each document is of the form "a b c a b c a b c...". If the row number ($i) # is an integer multiple of $spc1, then an "x" token is appended to the # document. If it is *also* a multiple of $spc2, a "y" token is also appended. |
︙ | ︙ | |||
73 74 75 76 77 78 79 | do_fb_test $tn.3.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND x' } $xdoc do_fb_test $tn.3.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND a' } $xdoc do_fb_test $tn.4.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND y' } $ydoc do_fb_test $tn.4.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND a' } $ydoc | > | | | | > | | | 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 | do_fb_test $tn.3.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND x' } $xdoc do_fb_test $tn.3.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND a' } $xdoc do_fb_test $tn.4.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND y' } $ydoc do_fb_test $tn.4.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND a' } $ydoc if {[detail_is_full]} { do_fb_test $tn.5.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc do_fb_test $tn.5.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc } } foreach {tn pgsz} { 1 32 2 200 } { do_execsql_test $tn.0 { DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz); } do_dlidx_test1 1.$tn.1 10 100 10000 0 1000 do_dlidx_test1 1.$tn.2 10 10 10000 0 128 do_dlidx_test1 1.$tn.3 10 10 66 0 36028797018963970 do_dlidx_test1 1.$tn.4 10 10 50 0 150000000000000000 do_dlidx_test1 1.$tn.5 10 10 200 0 [expr 1<<55] do_dlidx_test1 1.$tn.6 10 10 30 0 [expr 1<<58] } proc do_dlidx_test2 {tn nEntry iFirst nStep} { set str [string repeat "a " 500] execsql { BEGIN; DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 64); INSERT INTO t1 VALUES('b a'); WITH iii(ii, i) AS ( SELECT 1, $iFirst UNION ALL SELECT ii+1, i+$nStep FROM iii WHERE ii<$nEntry ) |
︙ | ︙ | |||
126 127 128 129 130 131 132 | do_execsql_test $tn.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC } {1} } do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128] | < < | 128 129 130 131 132 133 134 135 136 137 138 139 140 141 | do_execsql_test $tn.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC } {1} } do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128] #-------------------------------------------------------------------- # reset_db set ::vocab [list \ IteratorpItercurrentlypointstothefirstrowidofadoclist \ Thereisadoclistindexassociatedwiththefinaltermonthecurrent \ |
︙ | ︙ | |||
154 155 156 157 158 159 160 | lappend ret [lindex $vocab [expr $i % $nVocab]] } set ret } db func rnddoc rnddoc do_execsql_test 3.1 { | | | 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 | lappend ret [lindex $vocab [expr $i % $nVocab]] } set ret } db func rnddoc rnddoc do_execsql_test 3.1 { CREATE VIRTUAL TABLE abc USING fts5(a, detail=%DETAIL%); INSERT INTO abc(abc, rank) VALUES('pgsz', 32); INSERT INTO abc VALUES ( rnddoc() ); INSERT INTO abc VALUES ( rnddoc() ); INSERT INTO abc VALUES ( rnddoc() ); INSERT INTO abc VALUES ( rnddoc() ); |
︙ | ︙ | |||
187 188 189 190 191 192 193 194 195 196 197 | set v [lindex $vocab 0] set i 0 foreach v $vocab { do_execsql_test 3.3.[incr i] { SELECT rowid FROM abc WHERE abc MATCH $v } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16} } finish_test | > > > | 187 188 189 190 191 192 193 194 195 196 197 198 199 200 | set v [lindex $vocab 0] set i 0 foreach v $vocab { do_execsql_test 3.3.[incr i] { SELECT rowid FROM abc WHERE abc MATCH $v } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16} } } ;# foreach_detail_mode finish_test |
Changes to ext/fts5/test/fts5eb.test.
︙ | ︙ | |||
38 39 40 41 42 43 44 45 46 47 48 49 50 51 | 6 {abc AND ""} {"abc"} 7 {"" OR abc} {"abc"} 8 {"" NOT abc} {"abc"} 9 {"" AND abc} {"abc"} 10 {abc + "" + def} {"abc" + "def"} 11 {abc "" def} {"abc" AND "def"} 12 {r+e OR w} {"r" + "e" OR "w"} } { do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res] } do_catchsql_test 2.1 { SELECT fts5_expr() } {1 {wrong number of arguments to function fts5_expr}} | > > > > > > > > > | 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 | 6 {abc AND ""} {"abc"} 7 {"" OR abc} {"abc"} 8 {"" NOT abc} {"abc"} 9 {"" AND abc} {"abc"} 10 {abc + "" + def} {"abc" + "def"} 11 {abc "" def} {"abc" AND "def"} 12 {r+e OR w} {"r" + "e" OR "w"} 13 {a AND b NOT c} {"a" AND ("b" NOT "c")} 14 {a OR b NOT c} {"a" OR ("b" NOT "c")} 15 {a NOT b AND c} {("a" NOT "b") AND "c"} 16 {a NOT b OR c} {("a" NOT "b") OR "c"} 17 {a AND b OR c} {("a" AND "b") OR "c"} 18 {a OR b AND c} {"a" OR ("b" AND "c")} } { do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res] } do_catchsql_test 2.1 { SELECT fts5_expr() } {1 {wrong number of arguments to function fts5_expr}} |
︙ | ︙ |
Added ext/fts5/test/fts5fault8.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | # 2015 September 3 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # # This file is focused on OOM errors. # source [file join [file dirname [info script]] fts5_common.tcl] source $testdir/malloc_common.tcl set testprefix fts5fault8 # If SQLITE_ENABLE_FTS3 is defined, omit this file. ifcapable !fts5 { finish_test return } foreach_detail_mode $testprefix { if {[detail_is_none]==0} continue fts5_aux_test_functions db do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1 VALUES('a b c d', '1 2 3 4'); INSERT INTO t1 VALUES('a b a b', NULL); INSERT INTO t1 VALUES(NULL, '1 2 1 2'); } do_faultsim_test 1 -faults oom-t* -body { execsql { SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2' } } -test { faultsim_test_result {0 {1 {0.0.1 1.1.1} 2 {0.0.1 0.0.3} 3 {1.1.1 1.1.3}}} \ {1 SQLITE_NOMEM} } } finish_test |
Changes to ext/fts5/test/fts5matchinfo.test.
︙ | ︙ | |||
12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5matchinfo # If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test ; return } proc mit {blob} { set scan(littleEndian) i* set scan(bigEndian) I* binary scan $blob $scan($::tcl_platform(byteOrder)) r return $r } db func mit mit sqlite3_fts5_register_matchinfo db do_execsql_test 1.0 { | > > | | | 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5matchinfo # If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test ; return } foreach_detail_mode $testprefix { proc mit {blob} { set scan(littleEndian) i* set scan(bigEndian) I* binary scan $blob $scan($::tcl_platform(byteOrder)) r return $r } db func mit mit sqlite3_fts5_register_matchinfo db do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(content, detail=%DETAIL%); } do_execsql_test 1.1 { INSERT INTO t1(content) VALUES('I wandered lonely as a cloud'); INSERT INTO t1(content) VALUES('That floats on high o''er vales and hills,'); INSERT INTO t1(content) VALUES('When all at once I saw a crowd,'); INSERT INTO t1(content) VALUES('A host, of golden daffodils,'); SELECT mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'I'; } {{1 1 1 2 2} {1 1 1 2 2}} # Now create an FTS4 table that does not specify matchinfo=fts3. # do_execsql_test 1.2 { CREATE VIRTUAL TABLE t2 USING fts5(content, detail=%DETAIL%); INSERT INTO t2 SELECT * FROM t1; SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'I'; } {{1 1 1 2 2} {1 1 1 2 2}} #-------------------------------------------------------------------------- # Proc [do_matchinfo_test] is used to test the FTSX matchinfo() function. |
︙ | ︙ | |||
145 146 147 148 149 150 151 | set res [list] foreach elem $list_of_lists { lappend res [list {*}$elem] } return $res } | > > | > > > > > > | | 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 | set res [list] foreach elem $list_of_lists { lappend res [list {*}$elem] } return $res } # Similar to [do_matchinfo_test], except that this is a no-op if the FTS5 # mode is not detail=full. # proc do_matchinfo_p_test {tn tbl expr results} { if {[detail_is_full]} { uplevel [list do_matchinfo_test $tn $tbl $expr $results] } } do_execsql_test 4.1.0 { CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t4 VALUES('a b c d e', 'f g h i j'); INSERT INTO t4 VALUES('f g h i j', 'a b c d e'); } do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} { s {{3 0} {0 3}} } |
︙ | ︙ | |||
181 182 183 184 185 186 187 | s {{3 0} {0 3}} xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc - xpxsscplax - } | | | | | | | | | | | 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 | s {{3 0} {0 3}} xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc - xpxsscplax - } do_matchinfo_p_test 4.1.2 t4 {t4 MATCH '"g h i"'} { p {1 1} c {2 2} x { {0 1 1 1 1 1} {1 1 1 0 1 1} } n {2 2} l {{5 5} {5 5}} a {{5 5} {5 5}} s {{0 1} {1 0}} xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc - sxsxs - } do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'} { s {{2 0} {0 2}} } do_matchinfo_p_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} } do_matchinfo_p_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} } do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'} { s {{1 0} {0 1}} } do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} { x { {0 1 1 1 1 1 0 0 0 0 0 0} {1 1 1 0 1 1 0 0 0 0 0 0} } } do_matchinfo_test 4.1.8 t4 {t4 MATCH 'f NOT abcd'} { x { {0 1 1 1 1 1 0 0 0 0 0 0} {1 1 1 0 1 1 0 0 0 0 0 0} } } do_execsql_test 4.2.0 { CREATE VIRTUAL TABLE t5 USING fts5(content, detail=%DETAIL%); INSERT INTO t5 VALUES('a a a a a'); INSERT INTO t5 VALUES('a b a b a'); INSERT INTO t5 VALUES('c b c b c'); INSERT INTO t5 VALUES('x x x x x'); } do_matchinfo_test 4.2.1 t5 {t5 MATCH 'a a'} { x {{5 8 2 5 8 2} {3 8 2 3 8 2}} s {2 1} } do_matchinfo_test 4.2.2 t5 {t5 MATCH 'a b'} { s {2} } do_matchinfo_test 4.2.3 t5 {t5 MATCH 'a b a'} { s {3} } do_matchinfo_test 4.2.4 t5 {t5 MATCH 'a a a'} { s {3 1} } do_matchinfo_p_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1} } do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')"; # It used to be that the second 'a' token would be deferred. That doesn't # work any longer. if 0 { do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { x {{5 8 2 5 5 5} {3 8 2 3 5 5}} s {2 1} } } do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'} { s {2} } do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'} { s {3} } do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'} { s {3 1} } do_matchinfo_p_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1 1} } do_execsql_test 4.4.0.1 { INSERT INTO t5(t5) VALUES('optimize') } do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'} { s {2} } do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'} { s {2 1} } do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'} { s {2} } do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'} { s {3} } do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'} { s {3 1} } do_matchinfo_p_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } do_execsql_test 4.5.0 { CREATE VIRTUAL TABLE t6 USING fts5(a, b, c, detail=%DETAIL%); INSERT INTO t6 VALUES('a', 'b', 'c'); } do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'} { s {{1 1 1}} } #------------------------------------------------------------------------- # Test the outcome of matchinfo() when used within a query that does not # use the full-text index (i.e. lookup by rowid or full-table scan). # do_execsql_test 7.1 { CREATE VIRTUAL TABLE t10 USING fts5(content, detail=%DETAIL%); INSERT INTO t10 VALUES('first record'); INSERT INTO t10 VALUES('second record'); } do_execsql_test 7.2 { SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) FROM t10; } {blob 8 blob 8} do_execsql_test 7.3 { |
︙ | ︙ | |||
295 296 297 298 299 300 301 | # This was causing a problem at one point in the obscure case where the # total number of bytes of data stored in an fts3 table was greater than # the number of rows. i.e. when the following query returns true: # # SELECT sum(length(content)) < count(*) FROM fts4table; # do_execsql_test 8.1 { | | > | | | | | | | | | | | | | | | | > | | | 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 | # This was causing a problem at one point in the obscure case where the # total number of bytes of data stored in an fts3 table was greater than # the number of rows. i.e. when the following query returns true: # # SELECT sum(length(content)) < count(*) FROM fts4table; # do_execsql_test 8.1 { CREATE VIRTUAL TABLE t11 USING fts5(content, detail=%DETAIL%); INSERT INTO t11(t11, rank) VALUES('pgsz', 32); INSERT INTO t11 VALUES('quitealongstringoftext'); INSERT INTO t11 VALUES('anotherquitealongstringoftext'); INSERT INTO t11 VALUES('athirdlongstringoftext'); INSERT INTO t11 VALUES('andonemoreforgoodluck'); } do_test 8.2 { for {set i 0} {$i < 200} {incr i} { execsql { INSERT INTO t11 VALUES('') } } execsql { INSERT INTO t11(t11) VALUES('optimize') } } {} do_execsql_test 8.3 { SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*' } {{204 1 3 3 0} {204 1 3 3 0} {204 1 3 3 0}} #------------------------------------------------------------------------- if {[detail_is_full]} { do_execsql_test 9.1 { CREATE VIRTUAL TABLE t12 USING fts5(content, detail=%DETAIL%); INSERT INTO t12 VALUES('a b c d'); SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; } {{0 1 1 0 1 1 1 1 1}} do_execsql_test 9.2 { INSERT INTO t12 VALUES('a d c d'); SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; } { {0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2} } do_execsql_test 9.3 { INSERT INTO t12 VALUES('a d d a'); SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; } { {0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3} } } #--------------------------------------------------------------------------- # Test for a memory leak # do_execsql_test 10.1 { DROP TABLE t10; CREATE VIRTUAL TABLE t10 USING fts5(idx, value, detail=%DETAIL%); INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three'); SELECT t10.rowid, t10.* FROM t10 JOIN (SELECT 1 AS idx UNION SELECT 2 UNION SELECT 3) AS x WHERE t10 MATCH x.idx AND matchinfo(t10) not null GROUP BY t10.rowid ORDER BY 1; } {1 1 one 2 2 two 3 3 three} #--------------------------------------------------------------------------- # Test the 'y' matchinfo flag # reset_db sqlite3_fts5_register_matchinfo db do_execsql_test 11.0 { CREATE VIRTUAL TABLE tt USING fts5(x, y, detail=%DETAIL%); INSERT INTO tt VALUES('c d a c d d', 'e a g b d a'); -- 1 INSERT INTO tt VALUES('c c g a e b', 'c g d g e c'); -- 2 INSERT INTO tt VALUES('b e f d e g', 'b a c b c g'); -- 3 INSERT INTO tt VALUES('a c f f g d', 'd b f d e g'); -- 4 INSERT INTO tt VALUES('g a c f c f', 'd g g b c c'); -- 5 INSERT INTO tt VALUES('g a c e b b', 'd b f b g g'); -- 6 INSERT INTO tt VALUES('f d a a f c', 'e e a d c f'); -- 7 |
︙ | ︙ | |||
406 407 408 409 410 411 412 413 414 415 416 417 418 419 | 7 "a OR (a AND b)" { 1 {1 2 1 2 0 1} 2 {1 0 1 0 1 0} 3 {0 1 0 1 1 2} 4 {1 0 1 0 0 1} 5 {1 0 1 0 0 1} 6 {1 0 1 0 2 2} 7 {2 1 0 0 0 0} 8 {1 2 1 2 2 1} 9 {1 1 1 1 1 3} 10 {1 3 0 0 0 0} } } { do_execsql_test 11.1.$tn.1 { SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr } $res set r2 [list] foreach {rowid L} $res { lappend r2 $rowid | > > | 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 | 7 "a OR (a AND b)" { 1 {1 2 1 2 0 1} 2 {1 0 1 0 1 0} 3 {0 1 0 1 1 2} 4 {1 0 1 0 0 1} 5 {1 0 1 0 0 1} 6 {1 0 1 0 2 2} 7 {2 1 0 0 0 0} 8 {1 2 1 2 2 1} 9 {1 1 1 1 1 3} 10 {1 3 0 0 0 0} } } { if {[string match *:* $expr] && [detail_is_none]} continue do_execsql_test 11.1.$tn.1 { SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr } $res set r2 [list] foreach {rowid L} $res { lappend r2 $rowid |
︙ | ︙ | |||
439 440 441 442 443 444 445 | reset_db sqlite3_fts5_register_matchinfo db db func mit mit do_test 12.0 { set cols [list] for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" } | | > > | 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 | reset_db sqlite3_fts5_register_matchinfo db db func mit mit do_test 12.0 { set cols [list] for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" } execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,], detail=%DETAIL%)" } {} do_execsql_test 12.1 { INSERT INTO tt (rowid, c4, c45) VALUES(1, 'abc', 'abc'); SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc'; } [list [list [expr 1<<4] [expr 1<<(45-32)]]] } ;# foreach_detail_mode finish_test |
Changes to ext/fts5/test/fts5simple.test.
︙ | ︙ | |||
14 15 16 17 18 19 20 | set testprefix fts5simple # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } | | > > | 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 | set testprefix fts5simple # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } if 1 { #------------------------------------------------------------------------- # set doc "x x [string repeat {y } 50]z z" do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); BEGIN; |
︙ | ︙ | |||
346 347 348 349 350 351 352 | INSERT INTO x1 SELECT rnddoc(5) FROM ii; } do_execsql_test 4.1 { SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads' } {0 {} 4} | > > > > > | > > > > > > > > > > > > > > > > > > > > > > > > | 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 | INSERT INTO x1 SELECT rnddoc(5) FROM ii; } do_execsql_test 4.1 { SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads' } {0 {} 4} #------------------------------------------------------------------------- reset_db do_execsql_test 15.0 { CREATE VIRTUAL TABLE x2 USING fts5(x, prefix=1); INSERT INTO x2 VALUES('ab'); } do_execsql_test 15.1 { INSERT INTO x2(x2) VALUES('integrity-check'); } } #------------------------------------------------------------------------- foreach_detail_mode $testprefix { reset_db fts5_aux_test_functions db do_execsql_test 16.0 { CREATE VIRTUAL TABLE x3 USING fts5(x, detail=%DETAIL%); INSERT INTO x3 VALUES('a b c d e f'); } do_execsql_test 16.1 { SELECT fts5_test_poslist(x3) FROM x3('(a NOT b) OR c'); } {2.0.2} do_execsql_test 16.1 { SELECT fts5_test_poslist(x3) FROM x3('a OR c'); } {{0.0.0 1.0.2}} } finish_test |
Added ext/fts5/test/fts5simple2.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 | # 2015 September 05 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5simple2 # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } if 1 { do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); INSERT INTO t1 VALUES('a b c'); } do_execsql_test 1.1 { SELECT rowid FROM t1('c a b') } {1} #------------------------------------------------------------------------- # reset_db do_execsql_test 2.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); BEGIN; INSERT INTO t1 VALUES('b c d'); INSERT INTO t1 VALUES('b c d'); COMMIT; } do_execsql_test 2.1 { SELECT rowid FROM t1('b c d') } {1 2} #------------------------------------------------------------------------- # reset_db do_execsql_test 3.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); BEGIN; INSERT INTO t1 VALUES('b c d'); INSERT INTO t1 VALUES('b c d'); } do_execsql_test 3.1 { SELECT rowid FROM t1('b c d'); COMMIT; } {1 2} #------------------------------------------------------------------------- # reset_db do_execsql_test 4.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); BEGIN; INSERT INTO t1 VALUES('a1 b1 c1'); INSERT INTO t1 VALUES('a2 b2 c2'); INSERT INTO t1 VALUES('a3 b3 c3'); COMMIT; } do_execsql_test 4.1 { SELECT rowid FROM t1('b*'); } {1 2 3} #------------------------------------------------------------------------- # reset_db do_execsql_test 5.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); BEGIN; INSERT INTO t1 VALUES('a1 b1 c1'); INSERT INTO t1 VALUES('a2 b2 c2'); INSERT INTO t1 VALUES('a1 b1 c1'); COMMIT; } do_execsql_test 5.1 { SELECT rowid FROM t1('b*') } {1 2 3} #------------------------------------------------------------------------- # reset_db do_execsql_test 6.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=full); BEGIN; INSERT INTO t1 VALUES('a1 b1 c1'); INSERT INTO t1 VALUES('a1 b1 c1'); INSERT INTO t1 VALUES('a1 b1 c1'); COMMIT; } do_execsql_test 6.1 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 2 1} do_execsql_test 6.2 { SELECT rowid FROM t1('b1') ORDER BY rowid DESC } {3 2 1} do_execsql_test 6.3 { SELECT rowid FROM t1('c1') ORDER BY rowid DESC } {3 2 1} #------------------------------------------------------------------------- # reset_db do_execsql_test 7.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); BEGIN; INSERT INTO t1 VALUES('a1 b1'); INSERT INTO t1 VALUES('a1 b2'); COMMIT; } do_execsql_test 7.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {2 1} do_execsql_test 7.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {2 1} #------------------------------------------------------------------------- # reset_db do_execsql_test 8.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); INSERT INTO t1 VALUES('a1 b1 c1'); INSERT INTO t1 VALUES('a2 b2 c2'); INSERT INTO t1 VALUES('a1 b1 c1'); } do_execsql_test 8.0.1 { SELECT rowid FROM t1('b*') } {1 2 3} do_execsql_test 8.0.2 { SELECT rowid FROM t1('a1') } {1 3} do_execsql_test 8.0.3 { SELECT rowid FROM t1('c2') } {2} do_execsql_test 8.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {3 2 1} do_execsql_test 8.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 1} do_execsql_test 8.0.8 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC } {2} do_execsql_test 8.1.0 { INSERT INTO t1(t1) VALUES('optimize') } do_execsql_test 8.1.1 { SELECT rowid FROM t1('b*') } {1 2 3} do_execsql_test 8.1.2 { SELECT rowid FROM t1('a1') } {1 3} do_execsql_test 8.1.3 { SELECT rowid FROM t1('c2') } {2} do_execsql_test 8.2.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC} {3 2 1} do_execsql_test 8.2.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC} {3 1} do_execsql_test 8.2.3 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC} {2} #-------------------------------------------------------------------------- # reset_db do_execsql_test 9.0.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); INSERT INTO t1 VALUES('a1 b1 c1'); INSERT INTO t1 VALUES('a2 b2 c2'); INSERT INTO t1 VALUES('a1 b1 c1'); } do_execsql_test 9.0.1 { INSERT INTO t1(t1) VALUES('integrity-check'); } {} reset_db do_execsql_test 9.1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=none); INSERT INTO t1 VALUES('a1 b1 c1', 'x y z'); INSERT INTO t1 VALUES('a2 b2 c2', '1 2 3'); INSERT INTO t1 VALUES('a1 b1 c1', 'x 2 z'); } do_execsql_test 9.2.1 { INSERT INTO t1(t1) VALUES('integrity-check'); } {} #-------------------------------------------------------------------------- # reset_db do_execsql_test 10.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); INSERT INTO t1 VALUES('b1'); INSERT INTO t1 VALUES('b1'); DELETE FROM t1 WHERE rowid=1; } do_execsql_test 10.1 { SELECT rowid FROM t1('b1'); } {2} do_execsql_test 10.2 { SELECT rowid FROM t1('b1') ORDER BY rowid DESC; } {2} do_execsql_test 10.3 { INSERT INTO t1(t1) VALUES('integrity-check'); } {} #-------------------------------------------------------------------------- # reset_db do_execsql_test 11.1 { CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=none); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); WITH d(x,y) AS ( SELECT NULL, 'xyz' UNION ALL SELECT NULL, 'xyz' FROM d ) INSERT INTO t1 SELECT * FROM d LIMIT 23; } #db eval { SELECT rowid AS r, quote(block) AS b FROM t1_data } { puts "$r: $b" } do_execsql_test 11.2 { SELECT rowid FROM t1; } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23} do_execsql_test 11.3 { SELECT rowid FROM t1('xyz'); } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23} do_execsql_test 11.4 { INSERT INTO t1(t1) VALUES('integrity-check'); } #------------------------------------------------------------------------- # reset_db do_execsql_test 12.0 { CREATE VIRTUAL TABLE yy USING fts5(x, detail=none); INSERT INTO yy VALUES('in if'); INSERT INTO yy VALUES('if'); } {} do_execsql_test 12.1 { SELECT rowid FROM yy('i*'); } {1 2} #------------------------------------------------------------------------- # reset_db do_execsql_test 13.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, prefix=1, detail=none); } {} foreach {rowid a} { 0 {f} 1 {u} 2 {k} 3 {a} 4 {a} 5 {u} 6 {u} 7 {u} 8 {f} 9 {f} 10 {a} 11 {p} 12 {f} 13 {u} 14 {a} 15 {a} } { do_execsql_test 13.1.$rowid { INSERT INTO t1(rowid, a) VALUES($rowid, $a); } } #------------------------------------------------------------------------- # reset_db fts5_aux_test_functions db do_execsql_test 14.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); INSERT INTO t1 VALUES('a b c d'); } {} do_execsql_test 14.1 { SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank; } {0.0.1} } #------------------------------------------------------------------------- # reset_db do_execsql_test 15.1 { CREATE VIRTUAL TABLE t1 USING fts5(x, detail=none); BEGIN; INSERT INTO t1(rowid, x) VALUES(1, 'sqlite'); INSERT INTO t1(rowid, x) VALUES(2, 'sqlite'); COMMIT; } {} do_test 15.1 { execsql { INSERT INTO t1(t1) VALUES('integrity-check') } } {} do_test 15.2 { execsql { DELETE FROM t1 } } {} do_execsql_test 15.3.1 { SELECT rowid FROM t1('sqlite'); } {} do_execsql_test 15.3.2 { SELECT rowid FROM t1('sqlite') ORDER BY rowid DESC; } {} do_test 15.4 { execsql { INSERT INTO t1(t1) VALUES('integrity-check') } } {} finish_test |
Added ext/fts5/test/fts5synonym2.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 | # 2014 Dec 20 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # Tests focusing on custom tokenizers that support synonyms. # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5synonym2 # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } #------------------------------------------------------------------------- # Code for a simple Tcl tokenizer that supports synonyms at query time. # foreach SYNDICT { {zero 0} {one 1 i} {two 2 ii} {three 3 iii} {four 4 iv} {five 5 v} {six 6 vi} {seven 7 vii} {eight 8 viii} {nine 9 ix} } { foreach s $SYNDICT { set o [list] foreach x $SYNDICT {if {$x!=$s} {lappend o $x}} set ::syn($s) $o } } proc tcl_tokenize {tflags text} { foreach {w iStart iEnd} [fts5_tokenize_split $text] { sqlite3_fts5_token $w $iStart $iEnd if {$tflags == "query"} { foreach s $::syn($w) { sqlite3_fts5_token -colo $s $iStart $iEnd } } } } proc tcl_create {args} { return "tcl_tokenize" } # # End of tokenizer code. #------------------------------------------------------------------------- foreach_detail_mode $testprefix { sqlite3_fts5_create_tokenizer db tcl tcl_create fts5_aux_test_functions db do_execsql_test 1.0 { CREATE VIRTUAL TABLE ss USING fts5(a, b, tokenize=tcl, detail=%DETAIL%); INSERT INTO ss VALUES('5 5 five seven 3 seven i', '2 1 5 0 two 1 i'); INSERT INTO ss VALUES('six ix iii 7 i vii iii', 'one seven nine 4 9 1 vi'); INSERT INTO ss VALUES('6 viii i five six zero seven', '5 v iii iv iv 3'); INSERT INTO ss VALUES('9 ii six 8 1 6', 'six 4 iv iv 7'); INSERT INTO ss VALUES('1 5 4 eight ii iv iii', 'nine 2 eight ix v vii'); INSERT INTO ss VALUES('one 7 seven six 2 two', '1 2 four 7 4 3 4'); INSERT INTO ss VALUES('eight iv 4 nine vii six 1', '5 6 v one zero 4'); INSERT INTO ss VALUES('v 9 8 iii 4', '9 4 seven two vi vii'); INSERT INTO ss VALUES('3 ix two 9 0 nine i', 'five ii nine two viii i five'); INSERT INTO ss VALUES('six iii 9 two eight 2', 'nine i nine vii nine'); INSERT INTO ss VALUES('6 three zero seven vii five', '8 vii ix 0 7 seven'); INSERT INTO ss VALUES('8 vii 8 7 3 4', 'eight iii four viii nine iv three'); INSERT INTO ss VALUES('4 v 7 two 0 one 8', 'vii 1 two five i zero 9'); INSERT INTO ss VALUES('3 ii vii vi eight', '8 4 ix one three eight'); INSERT INTO ss VALUES('iv eight seven 6 9 seven', 'one vi two five seven'); INSERT INTO ss VALUES('i i 5 i v vii eight', '2 seven i 2 2 four'); INSERT INTO ss VALUES('0 i iii nine 3 ix five', '0 eight iv 0 six 2'); INSERT INTO ss VALUES('iv vii three 3 9 one 8', '2 ii 6 eight ii six six'); INSERT INTO ss VALUES('eight one two nine six', '8 9 3 viii vi'); INSERT INTO ss VALUES('one 0 four ii eight one 3', 'iii eight vi vi vi'); INSERT INTO ss VALUES('4 0 eight 0 0', '1 four one vii seven ii'); INSERT INTO ss VALUES('1 zero nine 2 2', 'viii iv two vi nine v iii'); INSERT INTO ss VALUES('5 five viii four four vi', '8 five 7 vii 6 4'); INSERT INTO ss VALUES('7 ix four 8 vii', 'nine three nine ii ix vii'); INSERT INTO ss VALUES('nine iv v i 0 v', 'two iv vii six i ix 4'); INSERT INTO ss VALUES('one v v one viii 3 8', '2 1 3 five iii'); INSERT INTO ss VALUES('six ii 5 nine 4 viii seven', 'eight i ix ix 7 four'); INSERT INTO ss VALUES('9 ii two seven three 7 0', 'six viii seven 7 five'); INSERT INTO ss VALUES('five two 4 viii nine', '9 7 nine zero 1 two one'); INSERT INTO ss VALUES('viii 8 iii i ii 8 3', '4 2 7 v 8 8'); INSERT INTO ss VALUES('four vii 4 iii zero 0 vii', '3 viii iii zero 9 i'); INSERT INTO ss VALUES('0 seven v five i five v', 'one 4 2 ix 9'); INSERT INTO ss VALUES('two 5 two two ix 4 1', '3 nine ii v nine 3 five'); INSERT INTO ss VALUES('five 5 7 4 6 vii', 'three 2 ix 2 8 6'); INSERT INTO ss VALUES('six iii vi iv seven eight', '8 six 7 0 4'); INSERT INTO ss VALUES('vi vi iv 3 0 one one', '9 6 eight ix iv'); INSERT INTO ss VALUES('7 2 2 iii 0', '0 0 seven 1 nine'); INSERT INTO ss VALUES('8 6 iv six ii', 'iv 6 3 4 ii five'); INSERT INTO ss VALUES('0 two two seven ii', 'vii ix four 4 zero vi vi'); INSERT INTO ss VALUES('2 one eight 8 9 7', 'vi 3 0 3 vii'); INSERT INTO ss VALUES('iii ii ix iv three', 'vi i 6 1 two'); INSERT INTO ss VALUES('eight four nine 8 seven', 'one three i nine iii one'); INSERT INTO ss VALUES('iii seven five ix 8', 'ii 7 seven 0 four ii'); INSERT INTO ss VALUES('four 0 1 5 two', 'iii 9 5 ii ii 2 4'); INSERT INTO ss VALUES('iii nine four vi 8 five six', 'i i ii seven vi vii'); INSERT INTO ss VALUES('eight vii eight six 3', 'i vii 1 six 9 vii'); INSERT INTO ss VALUES('9 0 viii viii five', 'i 1 viii ix 3 4'); INSERT INTO ss VALUES('three nine 5 nine viii four zero', 'ii i 1 5 2 viii'); INSERT INTO ss VALUES('5 vii three 9 four', 'three five one 7 2 eight one'); } foreach {tn expr} { 1.1 "one" 1.2 "two" 1.3 "three" 1.4 "four" 1.5 "v" 1.6 "vi" 1.7 "vii" 1.8 "viii" 1.9 "9" 1.10 "0" 1.11 "1" 1.12 "2" 2.1 "one OR two OR three OR four" 2.2 "(one AND two) OR (three AND four)" 2.3 "(one AND two) OR (three AND four) NOT five" 2.4 "(one AND two) NOT 6" 3.1 "b:one AND a:two" 3.2 "b:one OR a:two" 3.3 "a:one OR b:1 OR {a b} : i" 4.1 "NEAR(one two, 2)" 4.2 "NEAR(one two three, 2)" 4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)" } { if {[fts5_expr_ok $expr ss]==0} { do_test 1.$tn.OMITTED { list } [list] continue } set res [fts5_query_data $expr ss ASC ::syn] breakpoint do_execsql_test 1.$tn.[llength $res].asc { SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr) } $res } } finish_test |
Changes to ext/fts5/test/fts5vocab.test.
︙ | ︙ | |||
17 18 19 20 21 22 23 24 25 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } do_execsql_test 1.1.1 { | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 | # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } foreach_detail_mode $testprefix { proc null_list_entries {iFirst nInterval L} { for {set i $iFirst} {$i < [llength $L]} {incr i $nInterval} { lset L $i {} } return $L } proc star_from_row {L} { if {[detail_is_full]==0} { set L [null_list_entries 2 3 $L] } return $L } proc star_from_col {L} { if {[detail_is_col]} { set L [null_list_entries 3 4 $L] } if {[detail_is_none]} { set L [null_list_entries 1 4 $L] set L [null_list_entries 3 4 $L] } return $L } proc row_to_col {L} { if {[detail_is_none]==0} { error "this is for detail=none mode" } set ret [list] foreach {a b c} $L { lappend ret $a {} $b {} } set ret } do_execsql_test 1.1.1 { CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1, detail=%DETAIL%); CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row'); PRAGMA table_info = v1; } { 0 term {} 0 {} 0 1 doc {} 0 {} 0 2 cnt {} 0 {} 0 } |
︙ | ︙ | |||
48 49 50 51 52 53 54 | do_execsql_test 1.3 { INSERT INTO t1 VALUES('x y z'); INSERT INTO t1 VALUES('x x x'); } do_execsql_test 1.4.1 { SELECT * FROM v1; | | | | | | | > > > > > | | > | < < < | 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | do_execsql_test 1.3 { INSERT INTO t1 VALUES('x y z'); INSERT INTO t1 VALUES('x x x'); } do_execsql_test 1.4.1 { SELECT * FROM v1; } [star_from_row {x 2 4 y 1 1 z 1 1}] do_execsql_test 1.4.2 { SELECT * FROM v2; } [star_from_col {x one 2 4 y one 1 1 z one 1 1}] do_execsql_test 1.5.1 { BEGIN; INSERT INTO t1 VALUES('a b c'); SELECT * FROM v1 WHERE term<'d'; } [star_from_row {a 1 1 b 1 1 c 1 1}] do_execsql_test 1.5.2 { SELECT * FROM v2 WHERE term<'d'; COMMIT; } [star_from_col {a one 1 1 b one 1 1 c one 1 1}] do_execsql_test 1.6 { DELETE FROM t1 WHERE one = 'a b c'; SELECT * FROM v1; } [star_from_row {x 2 4 y 1 1 z 1 1}] #------------------------------------------------------------------------- # do_execsql_test 2.0 { CREATE VIRTUAL TABLE tt USING fts5(a, b, detail=%DETAIL%); INSERT INTO tt VALUES('d g b f d f', 'f c e c d a'); INSERT INTO tt VALUES('f a e a a b', 'e d c f d d'); INSERT INTO tt VALUES('b c a a a b', 'f f c c b c'); INSERT INTO tt VALUES('f d c a c e', 'd g d e g d'); INSERT INTO tt VALUES('g d e f a g x', 'f f d a a b'); INSERT INTO tt VALUES('g c f b c g', 'a g f d c b'); INSERT INTO tt VALUES('c e c f g b', 'f e d b g a'); INSERT INTO tt VALUES('g d e f d e', 'a c d b a g'); INSERT INTO tt VALUES('e f a c c b', 'b f e a f d y'); INSERT INTO tt VALUES('c c a a c f', 'd g a e b g'); } set res_row [star_from_row { a 10 20 b 9 14 c 9 20 d 9 19 e 8 13 f 10 20 g 7 14 x 1 1 y 1 1 }] set res_col [star_from_col { a a 6 11 a b 7 9 b a 6 7 b b 7 7 c a 6 12 c b 5 8 d a 4 6 d b 9 13 e a 6 7 e b 6 6 f a 9 10 f b 7 10 g a 5 7 g b 5 7 x a 1 1 y b 1 1 }] if {[detail_is_none]} { set res_col [row_to_col $res_row] } foreach {tn tbl resname} { 1 "fts5vocab(tt, 'col')" res_col 2 "fts5vocab(tt, 'row')" res_row 3 "fts5vocab(tt, \"row\")" res_row 4 "fts5vocab(tt, [row])" res_row |
︙ | ︙ | |||
149 150 151 152 153 154 155 | #------------------------------------------------------------------------- # Test fts5vocab tables created in the temp schema. # reset_db forcedelete test.db2 do_execsql_test 5.0 { ATTACH 'test.db2' AS aux; | | | | | 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 | #------------------------------------------------------------------------- # Test fts5vocab tables created in the temp schema. # reset_db forcedelete test.db2 do_execsql_test 5.0 { ATTACH 'test.db2' AS aux; CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%); CREATE VIRTUAL TABLE aux.t1 USING fts5(x, detail=%DETAIL%); INSERT INTO main.t1 VALUES('a b c'); INSERT INTO main.t1 VALUES('d e f'); INSERT INTO main.t1 VALUES('a e c'); INSERT INTO temp.t1 VALUES('1 2 3'); INSERT INTO temp.t1 VALUES('4 5 6'); |
︙ | ︙ | |||
174 175 176 177 178 179 180 | do_execsql_test 5.1 { CREATE VIRTUAL TABLE temp.vm USING fts5vocab(main, t1, row); CREATE VIRTUAL TABLE temp.vt1 USING fts5vocab(t1, row); CREATE VIRTUAL TABLE temp.vt2 USING fts5vocab(temp, t1, row); CREATE VIRTUAL TABLE temp.va USING fts5vocab(aux, t1, row); } | | | | | | | | | | 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 | do_execsql_test 5.1 { CREATE VIRTUAL TABLE temp.vm USING fts5vocab(main, t1, row); CREATE VIRTUAL TABLE temp.vt1 USING fts5vocab(t1, row); CREATE VIRTUAL TABLE temp.vt2 USING fts5vocab(temp, t1, row); CREATE VIRTUAL TABLE temp.va USING fts5vocab(aux, t1, row); } do_execsql_test 5.2 { SELECT * FROM vm } [star_from_row { a 2 2 b 1 1 c 2 2 d 1 1 e 2 2 f 1 1 }] do_execsql_test 5.3 { SELECT * FROM vt1 } [star_from_row { 1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1 }] do_execsql_test 5.4 { SELECT * FROM vt2 } [star_from_row { 1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1 }] do_execsql_test 5.5 { SELECT * FROM va } [star_from_row { m 1 1 n 2 2 o 1 1 x 2 2 y 1 1 z 2 2 }] #------------------------------------------------------------------------- # do_execsql_test 6.0 { CREATE TABLE iii(iii); CREATE TABLE jjj(x); } |
︙ | ︙ | |||
214 215 216 217 218 219 220 | } {1 {no such fts5 table: main.lll}} #------------------------------------------------------------------------- # Test single term queries on fts5vocab tables (i.e. those with term=? # constraints in the WHERE clause). # do_execsql_test 7.0 { | | | 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 | } {1 {no such fts5 table: main.lll}} #------------------------------------------------------------------------- # Test single term queries on fts5vocab tables (i.e. those with term=? # constraints in the WHERE clause). # do_execsql_test 7.0 { CREATE VIRTUAL TABLE tx USING fts5(one, two, detail=%DETAIL%); INSERT INTO tx VALUES('g a ggg g a b eee', 'cc d aa ff g ee'); INSERT INTO tx VALUES('dd fff i a i jjj', 'f fff hh jj e f'); INSERT INTO tx VALUES('ggg a f f fff dd aa', 'd ggg f f j gg ddd'); INSERT INTO tx VALUES('e bb h jjj ii gg', 'e aa e f c fff'); INSERT INTO tx VALUES('j ff aa a h', 'h a j bbb bb'); INSERT INTO tx VALUES('cc i ff c d f', 'dd ii fff f c cc d'); INSERT INTO tx VALUES('jjj g i bb cc eee', 'hhh iii aaa b bbb aaa'); |
︙ | ︙ | |||
272 273 274 275 276 277 278 279 280 281 282 283 284 285 | set r2 [db eval { SELECT $term, 'two', sum(cont(two, $term)>0), sum(cont(two, $term)) FROM tx }] if {[lindex $r2 2]==0} {set r2 [list]} set resc [concat $r1 $r2] do_execsql_test 7.$term.1 {SELECT * FROM txc WHERE term=$term} $resc do_execsql_test 7.$term.2 {SELECT * FROM txr WHERE term=$term} $resr } do_execsql_test 7.1 { CREATE TABLE txr_c AS SELECT * FROM txr; CREATE TABLE txc_c AS SELECT * FROM txc; | > > > > | 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 | set r2 [db eval { SELECT $term, 'two', sum(cont(two, $term)>0), sum(cont(two, $term)) FROM tx }] if {[lindex $r2 2]==0} {set r2 [list]} set resc [concat $r1 $r2] set resc [star_from_col $resc] set resr [star_from_row $resr] if {[detail_is_none]} { set resc [row_to_col $resr] } do_execsql_test 7.$term.1 {SELECT * FROM txc WHERE term=$term} $resc do_execsql_test 7.$term.2 {SELECT * FROM txr WHERE term=$term} $resr } do_execsql_test 7.1 { CREATE TABLE txr_c AS SELECT * FROM txr; CREATE TABLE txc_c AS SELECT * FROM txc; |
︙ | ︙ | |||
336 337 338 339 340 341 342 | } [db eval {SELECT * FROM txc_c WHERE term>$a AND term <$b}] } do_execsql_test 7.3.1 { SELECT count(*) FROM txr, txr_c WHERE txr.term = txr_c.term; } {30} | > | | | | > > > | 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 | } [db eval {SELECT * FROM txc_c WHERE term>$a AND term <$b}] } do_execsql_test 7.3.1 { SELECT count(*) FROM txr, txr_c WHERE txr.term = txr_c.term; } {30} if {![detail_is_none]} { do_execsql_test 7.3.2 { SELECT count(*) FROM txc, txc_c WHERE txc.term = txc_c.term AND txc.col=txc_c.col; } {57} } } finish_test |
Added ext/fts5/tool/fts5speed.tcl.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | set Q { {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'enron'"} {25 "SELECT count(*) FROM t1 WHERE t1 MATCH 'hours'"} {300 "SELECT count(*) FROM t1 WHERE t1 MATCH 'acid'"} {100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'loaned OR mobility OR popcore OR sunk'"} {100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'enron AND myapps'"} {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'en* AND my*'"} {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'c:t*'"} {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t* OR b:t* OR c:t* OR d:t* OR e:t* OR f:t* OR g:t*'"} {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t*'"} } proc usage {} { global Q puts stderr "Usage: $::argv0 DATABASE QUERY" puts stderr "" for {set i 1} {$i <= [llength $Q]} {incr i} { puts stderr " $i. [lindex $Q [expr $i-1]]" } puts stderr "" exit -1 } set nArg [llength $argv] if {$nArg!=2 && $nArg!=3} usage set database [lindex $argv 0] set iquery [lindex $argv 1] if {$iquery<1 || $iquery>[llength $Q]} usage set nRepeat 0 if {$nArg==3} { set nRepeat [lindex $argv 2] } sqlite3 db $database catch { load_static_extension db fts5 } incr iquery -1 set sql [lindex $Q $iquery 1] if {$nRepeat==0} { set nRepeat [lindex $Q $iquery 0] } puts "sql: $sql" puts "nRepeat: $nRepeat" if {[regexp matchinfo $sql]} { sqlite3_fts5_register_matchinfo db db eval $sql } else { puts "result: [db eval $sql]" } for {set i 1} {$i < $nRepeat} {incr i} { db eval $sql } |
Changes to ext/rbu/rbu.c.
︙ | ︙ | |||
71 72 73 74 75 76 77 | int nStep = 0; /* Maximum number of step() calls */ int rc; sqlite3_int64 nProgress = 0; /* Process command line arguments. Following this block local variables ** zTarget, zRbu and nStep are all set. */ if( argc==5 ){ | | | 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 | int nStep = 0; /* Maximum number of step() calls */ int rc; sqlite3_int64 nProgress = 0; /* Process command line arguments. Following this block local variables ** zTarget, zRbu and nStep are all set. */ if( argc==5 ){ size_t nArg1 = strlen(argv[1]); if( nArg1>5 || nArg1<2 || memcmp("-step", argv[1], nArg1) ) usage(argv[0]); nStep = atoi(argv[2]); }else if( argc!=3 ){ usage(argv[0]); } zTarget = argv[argc-2]; zRbu = argv[argc-1]; |
︙ | ︙ | |||
99 100 101 102 103 104 105 | /* Let the user know what happened. */ switch( rc ){ case SQLITE_OK: sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_OK: rbu update incomplete (%lld operations so far)\n", nProgress ); | | | < | 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | /* Let the user know what happened. */ switch( rc ){ case SQLITE_OK: sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_OK: rbu update incomplete (%lld operations so far)\n", nProgress ); fprintf(stdout, "%s", zBuf); break; case SQLITE_DONE: sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_DONE: rbu update completed (%lld operations)\n", nProgress ); fprintf(stdout, "%s", zBuf); break; default: fprintf(stderr, "error=%d: %s\n", rc, zErrmsg); break; } sqlite3_free(zErrmsg); return (rc==SQLITE_OK || rc==SQLITE_DONE) ? 0 : 1; } |
Changes to ext/rbu/sqlite3rbu.c.
︙ | ︙ | |||
931 932 933 934 935 936 937 | ** immediately without attempting the allocation or modifying the stored ** error code. */ static void *rbuMalloc(sqlite3rbu *p, int nByte){ void *pRet = 0; if( p->rc==SQLITE_OK ){ assert( nByte>0 ); | | | 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 | ** immediately without attempting the allocation or modifying the stored ** error code. */ static void *rbuMalloc(sqlite3rbu *p, int nByte){ void *pRet = 0; if( p->rc==SQLITE_OK ){ assert( nByte>0 ); pRet = sqlite3_malloc64(nByte); if( pRet==0 ){ p->rc = SQLITE_NOMEM; }else{ memset(pRet, 0, nByte); } } return pRet; |
︙ | ︙ | |||
977 978 979 980 981 982 983 | ** if the allocation succeeds, (*pRc) is left unchanged. */ static char *rbuStrndup(const char *zStr, int *pRc){ char *zRet = 0; assert( *pRc==SQLITE_OK ); if( zStr ){ | | | | 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 | ** if the allocation succeeds, (*pRc) is left unchanged. */ static char *rbuStrndup(const char *zStr, int *pRc){ char *zRet = 0; assert( *pRc==SQLITE_OK ); if( zStr ){ size_t nCopy = strlen(zStr) + 1; zRet = (char*)sqlite3_malloc64(nCopy); if( zRet ){ memcpy(zRet, zStr, nCopy); }else{ *pRc = SQLITE_NOMEM; } } |
︙ | ︙ | |||
2326 2327 2328 2329 2330 2331 2332 | return SQLITE_INTERNAL; } pRbu->pgsz = iAmt; if( pRbu->nFrame==pRbu->nFrameAlloc ){ int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2; RbuFrame *aNew; | | | 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 | return SQLITE_INTERNAL; } pRbu->pgsz = iAmt; if( pRbu->nFrame==pRbu->nFrameAlloc ){ int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2; RbuFrame *aNew; aNew = (RbuFrame*)sqlite3_realloc64(pRbu->aFrame, nNew * sizeof(RbuFrame)); if( aNew==0 ) return SQLITE_NOMEM; pRbu->aFrame = aNew; pRbu->nFrameAlloc = nNew; } iFrame = (u32)((iOff-32) / (i64)(iAmt+24)) + 1; if( pRbu->iMaxFrame<iFrame ) pRbu->iMaxFrame = iFrame; |
︙ | ︙ | |||
2391 2392 2393 2394 2395 2396 2397 | int nChar; LPWSTR zWideFilename; nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); if( nChar==0 ){ return 0; } | | | 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 | int nChar; LPWSTR zWideFilename; nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); if( nChar==0 ){ return 0; } zWideFilename = sqlite3_malloc64( nChar*sizeof(zWideFilename[0]) ); if( zWideFilename==0 ){ return 0; } memset(zWideFilename, 0, nChar*sizeof(zWideFilename[0])); nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename, nChar); if( nChar==0 ){ |
︙ | ︙ | |||
3025 3026 3027 3028 3029 3030 3031 | */ sqlite3rbu *sqlite3rbu_open( const char *zTarget, const char *zRbu, const char *zState ){ sqlite3rbu *p; | | | | > | | 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 | */ sqlite3rbu *sqlite3rbu_open( const char *zTarget, const char *zRbu, const char *zState ){ sqlite3rbu *p; size_t nTarget = strlen(zTarget); size_t nRbu = strlen(zRbu); size_t nState = zState ? strlen(zState) : 0; size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1+ nState+1; p = (sqlite3rbu*)sqlite3_malloc64(nByte); if( p ){ RbuState *pState = 0; /* Create the custom VFS. */ memset(p, 0, sizeof(sqlite3rbu)); rbuCreateVfs(p); |
︙ | ︙ | |||
3166 3167 3168 3169 3170 3171 3172 | ** If the error code currently stored in the RBU handle is SQLITE_CONSTRAINT, ** then edit any error message string so as to remove all occurrences of ** the pattern "rbu_imp_[0-9]*". */ static void rbuEditErrmsg(sqlite3rbu *p){ if( p->rc==SQLITE_CONSTRAINT && p->zErrmsg ){ int i; | | | 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 | ** If the error code currently stored in the RBU handle is SQLITE_CONSTRAINT, ** then edit any error message string so as to remove all occurrences of ** the pattern "rbu_imp_[0-9]*". */ static void rbuEditErrmsg(sqlite3rbu *p){ if( p->rc==SQLITE_CONSTRAINT && p->zErrmsg ){ int i; size_t nErrmsg = strlen(p->zErrmsg); for(i=0; i<(nErrmsg-8); i++){ if( memcmp(&p->zErrmsg[i], "rbu_imp_", 8)==0 ){ int nDel = 8; while( p->zErrmsg[i+nDel]>='0' && p->zErrmsg[i+nDel]<='9' ) nDel++; memmove(&p->zErrmsg[i], &p->zErrmsg[i+nDel], nErrmsg + 1 - i - nDel); nErrmsg -= nDel; } |
︙ | ︙ | |||
3630 3631 3632 3633 3634 3635 3636 | /* If not in RBU_STAGE_OAL, allow this call to pass through. Or, if this ** rbu is in the RBU_STAGE_OAL state, use heap memory for *-shm space ** instead of a file on disk. */ assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) ); if( eStage==RBU_STAGE_OAL || eStage==RBU_STAGE_MOVE ){ if( iRegion<=p->nShm ){ int nByte = (iRegion+1) * sizeof(char*); | | | | 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 | /* If not in RBU_STAGE_OAL, allow this call to pass through. Or, if this ** rbu is in the RBU_STAGE_OAL state, use heap memory for *-shm space ** instead of a file on disk. */ assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) ); if( eStage==RBU_STAGE_OAL || eStage==RBU_STAGE_MOVE ){ if( iRegion<=p->nShm ){ int nByte = (iRegion+1) * sizeof(char*); char **apNew = (char**)sqlite3_realloc64(p->apShm, nByte); if( apNew==0 ){ rc = SQLITE_NOMEM; }else{ memset(&apNew[p->nShm], 0, sizeof(char*) * (1 + iRegion - p->nShm)); p->apShm = apNew; p->nShm = iRegion+1; } } if( rc==SQLITE_OK && p->apShm[iRegion]==0 ){ char *pNew = (char*)sqlite3_malloc64(szRegion); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ memset(pNew, 0, szRegion); p->apShm[iRegion] = pNew; } } |
︙ | ︙ | |||
3751 3752 3753 3754 3755 3756 3757 | if( zName ){ if( flags & SQLITE_OPEN_MAIN_DB ){ /* A main database has just been opened. The following block sets ** (pFd->zWal) to point to a buffer owned by SQLite that contains ** the name of the *-wal file this db connection will use. SQLite ** happens to pass a pointer to this buffer when using xAccess() ** or xOpen() to operate on the *-wal file. */ | | | 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 | if( zName ){ if( flags & SQLITE_OPEN_MAIN_DB ){ /* A main database has just been opened. The following block sets ** (pFd->zWal) to point to a buffer owned by SQLite that contains ** the name of the *-wal file this db connection will use. SQLite ** happens to pass a pointer to this buffer when using xAccess() ** or xOpen() to operate on the *-wal file. */ int n = (int)strlen(zName); const char *z = &zName[n]; if( flags & SQLITE_OPEN_URI ){ int odd = 0; while( 1 ){ if( z[0]==0 ){ odd = 1 - odd; if( odd && z[1]==0 ) break; |
︙ | ︙ | |||
3777 3778 3779 3780 3781 3782 3783 | rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName); if( pDb ){ if( pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){ /* This call is to open a *-wal file. Intead, open the *-oal. This ** code ensures that the string passed to xOpen() is terminated by a ** pair of '\0' bytes in case the VFS attempts to extract a URI ** parameter from it. */ | | | | 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 | rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName); if( pDb ){ if( pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){ /* This call is to open a *-wal file. Intead, open the *-oal. This ** code ensures that the string passed to xOpen() is terminated by a ** pair of '\0' bytes in case the VFS attempts to extract a URI ** parameter from it. */ size_t nCopy = strlen(zName); char *zCopy = sqlite3_malloc64(nCopy+2); if( zCopy ){ memcpy(zCopy, zName, nCopy); zCopy[nCopy-3] = 'o'; zCopy[nCopy] = '\0'; zCopy[nCopy+1] = '\0'; zOpen = (const char*)(pFd->zDel = zCopy); }else{ |
︙ | ︙ | |||
4007 4008 4009 4010 4011 4012 4013 | rbuVfsCurrentTime, /* xCurrentTime */ rbuVfsGetLastError, /* xGetLastError */ 0, /* xCurrentTimeInt64 (version 2) */ 0, 0, 0 /* Unimplemented version 3 methods */ }; rbu_vfs *pNew = 0; /* Newly allocated VFS */ | < | | > | | 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 | rbuVfsCurrentTime, /* xCurrentTime */ rbuVfsGetLastError, /* xGetLastError */ 0, /* xCurrentTimeInt64 (version 2) */ 0, 0, 0 /* Unimplemented version 3 methods */ }; rbu_vfs *pNew = 0; /* Newly allocated VFS */ int rc = SQLITE_OK; size_t nName; size_t nByte; nName = strlen(zName); nByte = sizeof(rbu_vfs) + nName + 1; pNew = (rbu_vfs*)sqlite3_malloc64(nByte); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ sqlite3_vfs *pParent; /* Parent VFS */ memset(pNew, 0, nByte); pParent = sqlite3_vfs_find(zParent); if( pParent==0 ){ |
︙ | ︙ |
Changes to src/btree.c.
︙ | ︙ | |||
1075 1076 1077 1078 1079 1080 1081 | static void btreeParseCellPtrNoPayload( MemPage *pPage, /* Page containing the cell */ u8 *pCell, /* Pointer to the cell text. */ CellInfo *pInfo /* Fill in this structure */ ){ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 ); | < | 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 | static void btreeParseCellPtrNoPayload( MemPage *pPage, /* Page containing the cell */ u8 *pCell, /* Pointer to the cell text. */ CellInfo *pInfo /* Fill in this structure */ ){ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 ); assert( pPage->childPtrSize==4 ); #ifndef SQLITE_DEBUG UNUSED_PARAMETER(pPage); #endif pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey); pInfo->nPayload = 0; pInfo->nLocal = 0; |
︙ | ︙ | |||
1097 1098 1099 1100 1101 1102 1103 | ){ u8 *pIter; /* For scanning through pCell */ u32 nPayload; /* Number of bytes of cell payload */ u64 iKey; /* Extracted Key value */ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 || pPage->leaf==1 ); | < < | 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 | ){ u8 *pIter; /* For scanning through pCell */ u32 nPayload; /* Number of bytes of cell payload */ u64 iKey; /* Extracted Key value */ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 || pPage->leaf==1 ); assert( pPage->intKeyLeaf ); assert( pPage->childPtrSize==0 ); pIter = pCell; /* The next block of code is equivalent to: ** ** pIter += getVarint32(pIter, nPayload); |
︙ | ︙ | |||
1167 1168 1169 1170 1171 1172 1173 | ){ u8 *pIter; /* For scanning through pCell */ u32 nPayload; /* Number of bytes of cell payload */ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 || pPage->leaf==1 ); assert( pPage->intKeyLeaf==0 ); | < | 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 | ){ u8 *pIter; /* For scanning through pCell */ u32 nPayload; /* Number of bytes of cell payload */ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 || pPage->leaf==1 ); assert( pPage->intKeyLeaf==0 ); pIter = pCell + pPage->childPtrSize; nPayload = *pIter; if( nPayload>=0x80 ){ u8 *pEnd = &pIter[8]; nPayload &= 0x7f; do{ nPayload = (nPayload<<7) | (*++pIter & 0x7f); |
︙ | ︙ | |||
1228 1229 1230 1231 1232 1233 1234 | ** the (CellInfo.nSize) value found by doing a full parse of the ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of ** this function verifies that this invariant is not violated. */ CellInfo debuginfo; pPage->xParseCell(pPage, pCell, &debuginfo); #endif | < | 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 | ** the (CellInfo.nSize) value found by doing a full parse of the ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of ** this function verifies that this invariant is not violated. */ CellInfo debuginfo; pPage->xParseCell(pPage, pCell, &debuginfo); #endif nSize = *pIter; if( nSize>=0x80 ){ pEnd = &pIter[8]; nSize &= 0x7f; do{ nSize = (nSize<<7) | (*++pIter & 0x7f); }while( *(pIter)>=0x80 && pIter<pEnd ); |
︙ | ︙ | |||
1686 1687 1688 1689 1690 1691 1692 | assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); /* EVIDENCE-OF: R-20501-61796 A value of 13 means the page is a leaf ** table b-tree page. */ assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); pPage->intKey = 1; if( pPage->leaf ){ pPage->intKeyLeaf = 1; | < < < | 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 | assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); /* EVIDENCE-OF: R-20501-61796 A value of 13 means the page is a leaf ** table b-tree page. */ assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); pPage->intKey = 1; if( pPage->leaf ){ pPage->intKeyLeaf = 1; pPage->xParseCell = btreeParseCellPtr; }else{ pPage->intKeyLeaf = 0; pPage->xCellSize = cellSizePtrNoPayload; pPage->xParseCell = btreeParseCellPtrNoPayload; } pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; }else if( flagByte==PTF_ZERODATA ){ /* EVIDENCE-OF: R-27225-53936 A value of 2 means the page is an interior ** index b-tree page. */ assert( (PTF_ZERODATA)==2 ); /* EVIDENCE-OF: R-16571-11615 A value of 10 means the page is a leaf ** index b-tree page. */ assert( (PTF_ZERODATA|PTF_LEAF)==10 ); pPage->intKey = 0; pPage->intKeyLeaf = 0; pPage->xParseCell = btreeParseCellPtrIndex; pPage->maxLocal = pBt->maxLocal; pPage->minLocal = pBt->minLocal; }else{ /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is ** an error. */ return SQLITE_CORRUPT_BKPT; |
︙ | ︙ |
Changes to src/btreeInt.h.
︙ | ︙ | |||
272 273 274 275 276 277 278 | ** stored in MemPage.pBt->mutex. */ struct MemPage { u8 isInit; /* True if previously initialized. MUST BE FIRST! */ u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ u8 intKey; /* True if table b-trees. False for index b-trees */ u8 intKeyLeaf; /* True if the leaf of an intKey table */ | < | 272 273 274 275 276 277 278 279 280 281 282 283 284 285 | ** stored in MemPage.pBt->mutex. */ struct MemPage { u8 isInit; /* True if previously initialized. MUST BE FIRST! */ u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ u8 intKey; /* True if table b-trees. False for index b-trees */ u8 intKeyLeaf; /* True if the leaf of an intKey table */ u8 leaf; /* True if a leaf page */ u8 hdrOffset; /* 100 for page 1. 0 otherwise */ u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ u8 max1bytePayload; /* min(maxLocal,127) */ u8 bBusy; /* Prevent endless loops on corrupt database files */ u16 maxLocal; /* Copy of BtShared.maxLocal or BtShared.maxLeaf */ u16 minLocal; /* Copy of BtShared.minLocal or BtShared.minLeaf */ |
︙ | ︙ |
Changes to src/main.c.
︙ | ︙ | |||
3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 | assert( fd!=0 ); if( op==SQLITE_FCNTL_FILE_POINTER ){ *(sqlite3_file**)pArg = fd; rc = SQLITE_OK; }else if( op==SQLITE_FCNTL_VFS_POINTER ){ *(sqlite3_vfs**)pArg = sqlite3PagerVfs(pPager); rc = SQLITE_OK; }else if( fd->pMethods ){ rc = sqlite3OsFileControl(fd, op, pArg); }else{ rc = SQLITE_NOTFOUND; } sqlite3BtreeLeave(pBtree); } | > > > | 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 | assert( fd!=0 ); if( op==SQLITE_FCNTL_FILE_POINTER ){ *(sqlite3_file**)pArg = fd; rc = SQLITE_OK; }else if( op==SQLITE_FCNTL_VFS_POINTER ){ *(sqlite3_vfs**)pArg = sqlite3PagerVfs(pPager); rc = SQLITE_OK; }else if( op==SQLITE_FCNTL_JOURNAL_POINTER ){ *(sqlite3_file**)pArg = sqlite3PagerJrnlFile(pPager); rc = SQLITE_OK; }else if( fd->pMethods ){ rc = sqlite3OsFileControl(fd, op, pArg); }else{ rc = SQLITE_NOTFOUND; } sqlite3BtreeLeave(pBtree); } |
︙ | ︙ |
Changes to src/os_unix.c.
︙ | ︙ | |||
426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 | { "mkdir", (sqlite3_syscall_ptr)mkdir, 0 }, #define osMkdir ((int(*)(const char*,mode_t))aSyscall[18].pCurrent) { "rmdir", (sqlite3_syscall_ptr)rmdir, 0 }, #define osRmdir ((int(*)(const char*))aSyscall[19].pCurrent) { "fchown", (sqlite3_syscall_ptr)fchown, 0 }, #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) { "geteuid", (sqlite3_syscall_ptr)geteuid, 0 }, #define osGeteuid ((uid_t(*)(void))aSyscall[21].pCurrent) #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, | > > > > | 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 | { "mkdir", (sqlite3_syscall_ptr)mkdir, 0 }, #define osMkdir ((int(*)(const char*,mode_t))aSyscall[18].pCurrent) { "rmdir", (sqlite3_syscall_ptr)rmdir, 0 }, #define osRmdir ((int(*)(const char*))aSyscall[19].pCurrent) #if defined(HAVE_FCHOWN) { "fchown", (sqlite3_syscall_ptr)fchown, 0 }, #else { "fchown", (sqlite3_syscall_ptr)0, 0 }, #endif #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) { "geteuid", (sqlite3_syscall_ptr)geteuid, 0 }, #define osGeteuid ((uid_t(*)(void))aSyscall[21].pCurrent) #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, |
︙ | ︙ | |||
460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 | #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "getpagesize", (sqlite3_syscall_ptr)unixGetpagesize, 0 }, #else { "getpagesize", (sqlite3_syscall_ptr)0, 0 }, #endif #define osGetpagesize ((int(*)(void))aSyscall[25].pCurrent) { "readlink", (sqlite3_syscall_ptr)readlink, 0 }, #define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[26].pCurrent) }; /* End of the overrideable system calls */ /* ** On some systems, calls to fchown() will trigger a message in a security ** log if they come from non-root processes. So avoid calling fchown() if ** we are not running as root. */ static int robustFchown(int fd, uid_t uid, gid_t gid){ | > > > > | | | | 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 | #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "getpagesize", (sqlite3_syscall_ptr)unixGetpagesize, 0 }, #else { "getpagesize", (sqlite3_syscall_ptr)0, 0 }, #endif #define osGetpagesize ((int(*)(void))aSyscall[25].pCurrent) #if defined(HAVE_READLINK) { "readlink", (sqlite3_syscall_ptr)readlink, 0 }, #else { "readlink", (sqlite3_syscall_ptr)0, 0 }, #endif #define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[26].pCurrent) }; /* End of the overrideable system calls */ /* ** On some systems, calls to fchown() will trigger a message in a security ** log if they come from non-root processes. So avoid calling fchown() if ** we are not running as root. */ static int robustFchown(int fd, uid_t uid, gid_t gid){ #if defined(HAVE_FCHOWN) return osGeteuid() ? 0 : osFchown(fd,uid,gid); #else return 0; #endif } /* ** This is the xSetSystemCall() method of sqlite3_vfs for all of the ** "unix" VFSes. Return SQLITE_OK opon successfully updating the ** system call pointer, or SQLITE_NOTFOUND if there is no configurable |
︙ | ︙ | |||
5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 | ** current working directory has been unlinked. */ SimulateIOError( return SQLITE_ERROR ); assert( pVfs->mxPathname==MAX_PATHNAME ); UNUSED_PARAMETER(pVfs); /* Attempt to resolve the path as if it were a symbolic link. If it is ** a symbolic link, the resolved path is stored in buffer zOut[]. Or, if ** the identified file is not a symbolic link or does not exist, then ** zPath is copied directly into zOut. Either way, nByte is left set to ** the size of the string copied into zOut[] in bytes. */ nByte = osReadlink(zPath, zOut, nOut-1); if( nByte<0 ){ if( errno!=EINVAL && errno!=ENOENT ){ return unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zPath); } sqlite3_snprintf(nOut, zOut, "%s", zPath); nByte = sqlite3Strlen30(zOut); }else{ zOut[nByte] = '\0'; } /* If buffer zOut[] now contains an absolute path there is nothing more ** to do. If it contains a relative path, do the following: ** ** * move the relative path string so that it is at the end of th ** zOut[] buffer. ** * Call getcwd() to read the path of the current working directory | > > | 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 | ** current working directory has been unlinked. */ SimulateIOError( return SQLITE_ERROR ); assert( pVfs->mxPathname==MAX_PATHNAME ); UNUSED_PARAMETER(pVfs); #if defined(HAVE_READLINK) /* Attempt to resolve the path as if it were a symbolic link. If it is ** a symbolic link, the resolved path is stored in buffer zOut[]. Or, if ** the identified file is not a symbolic link or does not exist, then ** zPath is copied directly into zOut. Either way, nByte is left set to ** the size of the string copied into zOut[] in bytes. */ nByte = osReadlink(zPath, zOut, nOut-1); if( nByte<0 ){ if( errno!=EINVAL && errno!=ENOENT ){ return unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zPath); } sqlite3_snprintf(nOut, zOut, "%s", zPath); nByte = sqlite3Strlen30(zOut); }else{ zOut[nByte] = '\0'; } #endif /* If buffer zOut[] now contains an absolute path there is nothing more ** to do. If it contains a relative path, do the following: ** ** * move the relative path string so that it is at the end of th ** zOut[] buffer. ** * Call getcwd() to read the path of the current working directory |
︙ | ︙ |
Changes to src/pager.c.
︙ | ︙ | |||
5623 5624 5625 5626 5627 5628 5629 | ** exclusive lock on the database is not already held, obtain it now. */ if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){ rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ return rc; } | | | 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 | ** exclusive lock on the database is not already held, obtain it now. */ if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){ rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ return rc; } (void)sqlite3WalExclusiveMode(pPager->pWal, 1); } /* Grab the write lock on the log file. If successful, upgrade to ** PAGER_RESERVED state. Otherwise, return an error code to the caller. ** The busy-handler is not invoked if another connection already ** holds the write-lock. If possible, the upper layer will call it. */ |
︙ | ︙ | |||
6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 | ** Return the file handle for the database file associated ** with the pager. This might return NULL if the file has ** not yet been opened. */ sqlite3_file *sqlite3PagerFile(Pager *pPager){ return pPager->fd; } /* ** Return the full pathname of the journal file. */ const char *sqlite3PagerJournalname(Pager *pPager){ return pPager->zJournal; } | > > > > > > > > > > > > | 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 | ** Return the file handle for the database file associated ** with the pager. This might return NULL if the file has ** not yet been opened. */ sqlite3_file *sqlite3PagerFile(Pager *pPager){ return pPager->fd; } /* ** Return the file handle for the journal file (if it exists). ** This will be either the rollback journal or the WAL file. */ sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ #if SQLITE_OMIT_WAL return pPager->jfd; #else return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; #endif } /* ** Return the full pathname of the journal file. */ const char *sqlite3PagerJournalname(Pager *pPager){ return pPager->zJournal; } |
︙ | ︙ |
Changes to src/pager.h.
︙ | ︙ | |||
184 185 186 187 188 189 190 191 192 193 194 195 196 197 | #ifdef SQLITE_DEBUG int sqlite3PagerRefcount(Pager*); #endif int sqlite3PagerMemUsed(Pager*); const char *sqlite3PagerFilename(Pager*, int); sqlite3_vfs *sqlite3PagerVfs(Pager*); sqlite3_file *sqlite3PagerFile(Pager*); const char *sqlite3PagerJournalname(Pager*); int sqlite3PagerNosync(Pager*); void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerIsMemdb(Pager*); void sqlite3PagerCacheStat(Pager *, int, int, int *); void sqlite3PagerClearCache(Pager *); int sqlite3SectorSize(sqlite3_file *); | > | 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 | #ifdef SQLITE_DEBUG int sqlite3PagerRefcount(Pager*); #endif int sqlite3PagerMemUsed(Pager*); const char *sqlite3PagerFilename(Pager*, int); sqlite3_vfs *sqlite3PagerVfs(Pager*); sqlite3_file *sqlite3PagerFile(Pager*); sqlite3_file *sqlite3PagerJrnlFile(Pager*); const char *sqlite3PagerJournalname(Pager*); int sqlite3PagerNosync(Pager*); void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerIsMemdb(Pager*); void sqlite3PagerCacheStat(Pager *, int, int, int *); void sqlite3PagerClearCache(Pager *); int sqlite3SectorSize(sqlite3_file *); |
︙ | ︙ |
Changes to src/select.c.
︙ | ︙ | |||
50 51 52 53 54 55 56 57 58 59 60 61 62 63 | struct SortCtx { ExprList *pOrderBy; /* The ORDER BY (or GROUP BY clause) */ int nOBSat; /* Number of ORDER BY terms satisfied by indices */ int iECursor; /* Cursor number for the sorter */ int regReturn; /* Register holding block-output return address */ int labelBkOut; /* Start label for the block-output subroutine */ int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ u8 sortFlags; /* Zero or more SORTFLAG_* bits */ }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ /* ** Delete all the content of a Select structure. Deallocate the structure ** itself only if bFree is true. | > | 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | struct SortCtx { ExprList *pOrderBy; /* The ORDER BY (or GROUP BY clause) */ int nOBSat; /* Number of ORDER BY terms satisfied by indices */ int iECursor; /* Cursor number for the sorter */ int regReturn; /* Register holding block-output return address */ int labelBkOut; /* Start label for the block-output subroutine */ int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ int labelDone; /* Jump here when done, ex: LIMIT reached */ u8 sortFlags; /* Zero or more SORTFLAG_* bits */ }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ /* ** Delete all the content of a Select structure. Deallocate the structure ** itself only if bFree is true. |
︙ | ︙ | |||
120 121 122 123 124 125 126 127 128 129 130 131 132 133 | pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ASTERISK,0)); } pNew->pEList = pEList; pNew->op = TK_SELECT; pNew->selFlags = selFlags; pNew->iLimit = 0; pNew->iOffset = 0; pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = 0; if( pSrc==0 ) pSrc = sqlite3DbMallocZero(db, sizeof(*pSrc)); pNew->pSrc = pSrc; pNew->pWhere = pWhere; pNew->pGroupBy = pGroupBy; | > > > | 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 | pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ASTERISK,0)); } pNew->pEList = pEList; pNew->op = TK_SELECT; pNew->selFlags = selFlags; pNew->iLimit = 0; pNew->iOffset = 0; #if SELECTTRACE_ENABLED pNew->zSelName[0] = 0; #endif pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = 0; if( pSrc==0 ) pSrc = sqlite3DbMallocZero(db, sizeof(*pSrc)); pNew->pSrc = pSrc; pNew->pWhere = pWhere; pNew->pGroupBy = pGroupBy; |
︙ | ︙ | |||
509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 | int bSeq = ((pSort->sortFlags & SORTFLAG_UseSorter)==0); int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ int nBase = nExpr + bSeq + nData; /* Fields in sorter record */ int regBase; /* Regs for sorter record */ int regRecord = ++pParse->nMem; /* Assembled sorter record */ int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */ int op; /* Opcode to add sorter record to sorter */ assert( bSeq==0 || bSeq==1 ); assert( nData==1 || regData==regOrigData ); if( nPrefixReg ){ assert( nPrefixReg==nExpr+bSeq ); regBase = regData - nExpr - bSeq; }else{ regBase = pParse->nMem + 1; pParse->nMem += nBase; } sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, SQLITE_ECEL_DUP|SQLITE_ECEL_REF); if( bSeq ){ sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); } if( nPrefixReg==0 ){ sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData); } | > > > > < | 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 | int bSeq = ((pSort->sortFlags & SORTFLAG_UseSorter)==0); int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ int nBase = nExpr + bSeq + nData; /* Fields in sorter record */ int regBase; /* Regs for sorter record */ int regRecord = ++pParse->nMem; /* Assembled sorter record */ int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */ int op; /* Opcode to add sorter record to sorter */ int iLimit; /* LIMIT counter */ assert( bSeq==0 || bSeq==1 ); assert( nData==1 || regData==regOrigData ); if( nPrefixReg ){ assert( nPrefixReg==nExpr+bSeq ); regBase = regData - nExpr - bSeq; }else{ regBase = pParse->nMem + 1; pParse->nMem += nBase; } assert( pSelect->iOffset==0 || pSelect->iLimit!=0 ); iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit; pSort->labelDone = sqlite3VdbeMakeLabel(v); sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, SQLITE_ECEL_DUP|SQLITE_ECEL_REF); if( bSeq ){ sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); } if( nPrefixReg==0 ){ sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ int addrFirst; /* Address of the OP_IfNot opcode */ int addrJmp; /* Address of the OP_Jump opcode */ VdbeOp *pOp; /* Opcode that opens the sorter */ int nKey; /* Number of sorting key columns, including OP_Sequence */ |
︙ | ︙ | |||
562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 | pKI->nXField-1); addrJmp = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); pSort->labelBkOut = sqlite3VdbeMakeLabel(v); pSort->regReturn = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); sqlite3VdbeJumpHere(v, addrFirst); sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat); sqlite3VdbeJumpHere(v, addrJmp); } if( pSort->sortFlags & SORTFLAG_UseSorter ){ op = OP_SorterInsert; }else{ op = OP_IdxInsert; } sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); | > > > > | < < < < < < | 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 | pKI->nXField-1); addrJmp = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); pSort->labelBkOut = sqlite3VdbeMakeLabel(v); pSort->regReturn = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); if( iLimit ){ sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, pSort->labelDone); VdbeCoverage(v); } sqlite3VdbeJumpHere(v, addrFirst); sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat); sqlite3VdbeJumpHere(v, addrJmp); } if( pSort->sortFlags & SORTFLAG_UseSorter ){ op = OP_SorterInsert; }else{ op = OP_IdxInsert; } sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); if( iLimit ){ int addr; addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v); sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); sqlite3VdbeJumpHere(v, addr); } } |
︙ | ︙ | |||
1183 1184 1185 1186 1187 1188 1189 | Parse *pParse, /* Parsing context */ Select *p, /* The SELECT statement */ SortCtx *pSort, /* Information on the ORDER BY clause */ int nColumn, /* Number of columns of data */ SelectDest *pDest /* Write the sorted results here */ ){ Vdbe *v = pParse->pVdbe; /* The prepared statement */ | | > | 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 | Parse *pParse, /* Parsing context */ Select *p, /* The SELECT statement */ SortCtx *pSort, /* Information on the ORDER BY clause */ int nColumn, /* Number of columns of data */ SelectDest *pDest /* Write the sorted results here */ ){ Vdbe *v = pParse->pVdbe; /* The prepared statement */ int addrBreak = pSort->labelDone; /* Jump here to exit loop */ int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ int addr; int addrOnce = 0; int iTab; ExprList *pOrderBy = pSort->pOrderBy; int eDest = pDest->eDest; int iParm = pDest->iSDParm; int regRow; int regRowid; int nKey; int iSortTab; /* Sorter cursor to read from */ int nSortData; /* Trailing values to read from sorter */ int i; int bSeq; /* True if sorter record includes seq. no. */ #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS struct ExprList_item *aOutEx = p->pEList->a; #endif assert( addrBreak<0 ); if( pSort->labelBkOut ){ sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeGoto(v, addrBreak); sqlite3VdbeResolveLabel(v, pSort->labelBkOut); } iTab = pSort->iECursor; if( eDest==SRT_Output || eDest==SRT_Coroutine ){ |
︙ | ︙ |
Changes to src/shell.c.
︙ | ︙ | |||
1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 | ".tables ?TABLE? List names of tables\n" " If TABLE specified, only list tables matching\n" " LIKE pattern TABLE.\n" ".timeout MS Try opening locked tables for MS milliseconds\n" ".timer on|off Turn SQL timer on or off\n" ".trace FILE|off Output each SQL statement as it is run\n" ".vfsinfo ?AUX? Information about the top-level VFS\n" ".vfsname ?AUX? Print the name of the VFS stack\n" ".width NUM1 NUM2 ... Set column widths for \"column\" mode\n" " Negative values right-justify\n" ; #if defined(SQLITE_ENABLE_SESSION) /* | > | 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 | ".tables ?TABLE? List names of tables\n" " If TABLE specified, only list tables matching\n" " LIKE pattern TABLE.\n" ".timeout MS Try opening locked tables for MS milliseconds\n" ".timer on|off Turn SQL timer on or off\n" ".trace FILE|off Output each SQL statement as it is run\n" ".vfsinfo ?AUX? Information about the top-level VFS\n" ".vfslist List all available VFSes\n" ".vfsname ?AUX? Print the name of the VFS stack\n" ".width NUM1 NUM2 ... Set column widths for \"column\" mode\n" " Negative values right-justify\n" ; #if defined(SQLITE_ENABLE_SESSION) /* |
︙ | ︙ | |||
4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 | sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs); if( pVfs ){ utf8_printf(p->out, "vfs.zName = \"%s\"\n", pVfs->zName); raw_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); raw_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); } } }else if( c=='v' && strncmp(azArg[0], "vfsname", n)==0 ){ const char *zDbName = nArg==2 ? azArg[1] : "main"; char *zVfsName = 0; if( p->db ){ | > > > > > > > > > > > > > > > > > > | 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 | sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs); if( pVfs ){ utf8_printf(p->out, "vfs.zName = \"%s\"\n", pVfs->zName); raw_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); raw_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); } } }else if( c=='v' && strncmp(azArg[0], "vfslist", n)==0 ){ sqlite3_vfs *pVfs; sqlite3_vfs *pCurrent = 0; if( p->db ){ sqlite3_file_control(p->db, "main", SQLITE_FCNTL_VFS_POINTER, &pCurrent); } for(pVfs=sqlite3_vfs_find(0); pVfs; pVfs=pVfs->pNext){ utf8_printf(p->out, "vfs.zName = \"%s\"%s\n", pVfs->zName, pVfs==pCurrent ? " <--- CURRENT" : ""); raw_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); raw_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); if( pVfs->pNext ){ raw_printf(p->out, "-----------------------------------\n"); } } }else if( c=='v' && strncmp(azArg[0], "vfsname", n)==0 ){ const char *zDbName = nArg==2 ? azArg[1] : "main"; char *zVfsName = 0; if( p->db ){ |
︙ | ︙ |
Changes to src/sqlite.h.in.
︙ | ︙ | |||
790 791 792 793 794 795 796 | ** for the nominated database. Allocating database file space in large ** chunks (say 1MB at a time), may reduce file-system fragmentation and ** improve performance on some systems. ** ** <li>[[SQLITE_FCNTL_FILE_POINTER]] ** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer ** to the [sqlite3_file] object associated with a particular database | | | > > > > > | 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 | ** for the nominated database. Allocating database file space in large ** chunks (say 1MB at a time), may reduce file-system fragmentation and ** improve performance on some systems. ** ** <li>[[SQLITE_FCNTL_FILE_POINTER]] ** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer ** to the [sqlite3_file] object associated with a particular database ** connection. See also [SQLITE_FCNTL_JOURNAL_POINTER]. ** ** <li>[[SQLITE_FCNTL_JOURNAL_POINTER]] ** The [SQLITE_FCNTL_JOURNAL_POINTER] opcode is used to obtain a pointer ** to the [sqlite3_file] object associated with the journal file (either ** the [rollback journal] or the [write-ahead log]) for a particular database ** connection. See also [SQLITE_FCNTL_FILE_POINTER]. ** ** <li>[[SQLITE_FCNTL_SYNC_OMITTED]] ** No longer in use. ** ** <li>[[SQLITE_FCNTL_SYNC]] ** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and ** sent to the VFS immediately before the xSync method is invoked on a |
︙ | ︙ | |||
1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 | #define SQLITE_FCNTL_SYNC 21 #define SQLITE_FCNTL_COMMIT_PHASETWO 22 #define SQLITE_FCNTL_WIN32_SET_HANDLE 23 #define SQLITE_FCNTL_WAL_BLOCK 24 #define SQLITE_FCNTL_ZIPVFS 25 #define SQLITE_FCNTL_RBU 26 #define SQLITE_FCNTL_VFS_POINTER 27 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE #define SQLITE_SET_LOCKPROXYFILE SQLITE_FCNTL_SET_LOCKPROXYFILE #define SQLITE_LAST_ERRNO SQLITE_FCNTL_LAST_ERRNO | > | 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 | #define SQLITE_FCNTL_SYNC 21 #define SQLITE_FCNTL_COMMIT_PHASETWO 22 #define SQLITE_FCNTL_WIN32_SET_HANDLE 23 #define SQLITE_FCNTL_WAL_BLOCK 24 #define SQLITE_FCNTL_ZIPVFS 25 #define SQLITE_FCNTL_RBU 26 #define SQLITE_FCNTL_VFS_POINTER 27 #define SQLITE_FCNTL_JOURNAL_POINTER 28 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE #define SQLITE_SET_LOCKPROXYFILE SQLITE_FCNTL_SET_LOCKPROXYFILE #define SQLITE_LAST_ERRNO SQLITE_FCNTL_LAST_ERRNO |
︙ | ︙ |
Changes to src/vdbe.c.
︙ | ︙ | |||
2380 2381 2382 2383 2384 2385 2386 | const u8 *zData; /* Part of the record being decoded */ const u8 *zHdr; /* Next unparsed byte of the header */ const u8 *zEndHdr; /* Pointer to first byte after the header */ u32 offset; /* Offset into the data */ u64 offset64; /* 64-bit offset */ u32 avail; /* Number of bytes of available data */ u32 t; /* A type code from the record header */ | < | 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 | const u8 *zData; /* Part of the record being decoded */ const u8 *zHdr; /* Next unparsed byte of the header */ const u8 *zEndHdr; /* Pointer to first byte after the header */ u32 offset; /* Offset into the data */ u64 offset64; /* 64-bit offset */ u32 avail; /* Number of bytes of available data */ u32 t; /* A type code from the record header */ Mem *pReg; /* PseudoTable input register */ p2 = pOp->p2; assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); pDest = &aMem[pOp->p3]; memAboutToChange(p, pDest); assert( pOp->p1>=0 && pOp->p1<p->nCursor ); |
︙ | ︙ | |||
2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 | ** all valid. */ assert( p2<pC->nHdrParsed ); assert( rc==SQLITE_OK ); assert( sqlite3VdbeCheckMemInvariants(pDest) ); if( VdbeMemDynamic(pDest) ) sqlite3VdbeMemSetNull(pDest); assert( t==pC->aType[p2] ); if( pC->szRow>=aOffset[p2+1] ){ /* This is the common case where the desired content fits on the original ** page - where the content is not on an overflow page */ | > > > | > > > > > > > > > > > > > > > > > > | | < | | < < | | | | < | < < < < < < < < < < < < < < < | > | 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 | ** all valid. */ assert( p2<pC->nHdrParsed ); assert( rc==SQLITE_OK ); assert( sqlite3VdbeCheckMemInvariants(pDest) ); if( VdbeMemDynamic(pDest) ) sqlite3VdbeMemSetNull(pDest); assert( t==pC->aType[p2] ); pDest->enc = encoding; if( pC->szRow>=aOffset[p2+1] ){ /* This is the common case where the desired content fits on the original ** page - where the content is not on an overflow page */ zData = pC->aRow + aOffset[p2]; if( t<12 ){ sqlite3VdbeSerialGet(zData, t, pDest); }else{ /* If the column value is a string, we need a persistent value, not ** a MEM_Ephem value. This branch is a fast short-cut that is equivalent ** to calling sqlite3VdbeSerialGet() and sqlite3VdbeDeephemeralize(). */ static const u16 aFlag[] = { MEM_Blob, MEM_Str|MEM_Term }; pDest->n = len = (t-12)/2; if( pDest->szMalloc < len+2 ){ pDest->flags = MEM_Null; if( sqlite3VdbeMemGrow(pDest, len+2, 0) ) goto no_mem; }else{ pDest->z = pDest->zMalloc; } memcpy(pDest->z, zData, len); pDest->z[len] = 0; pDest->z[len+1] = 0; pDest->flags = aFlag[t&1]; } }else{ /* This branch happens only when content is on overflow pages */ if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0)) || (len = sqlite3VdbeSerialTypeLen(t))==0 ){ /* Content is irrelevant for ** 1. the typeof() function, ** 2. the length(X) function if X is a blob, and ** 3. if the content length is zero. ** So we might as well use bogus content rather than reading ** content from disk. */ static u8 aZero[8]; /* This is the bogus content */ sqlite3VdbeSerialGet(aZero, t, pDest); }else{ rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, !pC->isTable, pDest); if( rc==SQLITE_OK ){ sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); pDest->flags &= ~MEM_Ephem; } } } op_column_out: op_column_error: UPDATE_MAX_BLOBSIZE(pDest); REGISTER_TRACE(pOp->p3, pDest); break; } /* Opcode: Affinity P1 P2 * P4 * |
︙ | ︙ |
Changes to src/vdbeaux.c.
︙ | ︙ | |||
302 303 304 305 306 307 308 | ** as having been used. ** ** The zWhere string must have been obtained from sqlite3_malloc(). ** This routine will take ownership of the allocated memory. */ void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){ int j; | | < | 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 | ** as having been used. ** ** The zWhere string must have been obtained from sqlite3_malloc(). ** This routine will take ownership of the allocated memory. */ void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){ int j; sqlite3VdbeAddOp4(p, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); for(j=0; j<p->db->nDb; j++) sqlite3VdbeUsesBtree(p, j); } /* ** Add an opcode that includes the p4 value as an integer. */ int sqlite3VdbeAddOp4Int( |
︙ | ︙ | |||
802 803 804 805 806 807 808 | ** opcodes contained within. If aOp is not NULL it is assumed to contain ** nOp entries. */ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ if( aOp ){ Op *pOp; for(pOp=aOp; pOp<&aOp[nOp]; pOp++){ | | | 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 | ** opcodes contained within. If aOp is not NULL it is assumed to contain ** nOp entries. */ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ if( aOp ){ Op *pOp; for(pOp=aOp; pOp<&aOp[nOp]; pOp++){ if( pOp->p4type ) freeP4(db, pOp->p4type, pOp->p4.p); #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS sqlite3DbFree(db, pOp->zComment); #endif } } sqlite3DbFree(db, aOp); } |
︙ | ︙ | |||
864 865 866 867 868 869 870 871 872 873 874 875 876 | ** ** Other values of n (P4_STATIC, P4_COLLSEQ etc.) indicate that zP4 points ** to a string or structure that is guaranteed to exist for the lifetime of ** the Vdbe. In these cases we can just copy the pointer. ** ** If addr<0 then change P4 on the most recently inserted instruction. */ void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ Op *pOp; sqlite3 *db; assert( p!=0 ); db = p->db; assert( p->magic==VDBE_MAGIC_INIT ); | > > > > > > > > > > > > > > > > > > > | | | < < | | | < > | < < < < < < < < < < < < < < < < < < | < < | < < | 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 | ** ** Other values of n (P4_STATIC, P4_COLLSEQ etc.) indicate that zP4 points ** to a string or structure that is guaranteed to exist for the lifetime of ** the Vdbe. In these cases we can just copy the pointer. ** ** If addr<0 then change P4 on the most recently inserted instruction. */ static void SQLITE_NOINLINE vdbeChangeP4Full( Vdbe *p, Op *pOp, const char *zP4, int n ){ if( pOp->p4type ){ freeP4(p->db, pOp->p4type, pOp->p4.p); pOp->p4type = 0; pOp->p4.p = 0; } if( n<0 ){ sqlite3VdbeChangeP4(p, (int)(pOp - p->aOp), zP4, n); }else{ if( n==0 ) n = sqlite3Strlen30(zP4); pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n); pOp->p4type = P4_DYNAMIC; } } void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ Op *pOp; sqlite3 *db; assert( p!=0 ); db = p->db; assert( p->magic==VDBE_MAGIC_INIT ); assert( p->aOp!=0 || db->mallocFailed ); if( db->mallocFailed ){ if( n!=P4_VTAB ) freeP4(db, n, (void*)*(char**)&zP4); return; } assert( p->nOp>0 ); assert( addr<p->nOp ); if( addr<0 ){ addr = p->nOp - 1; } pOp = &p->aOp[addr]; if( n>=0 || pOp->p4type ){ vdbeChangeP4Full(p, pOp, zP4, n); return; } if( n==P4_INT32 ){ /* Note: this cast is safe, because the origin data point was an int ** that was cast to a (const char *). */ pOp->p4.i = SQLITE_PTR_TO_INT(zP4); pOp->p4type = P4_INT32; }else if( zP4!=0 ){ assert( n<0 ); pOp->p4.p = (void*)zP4; pOp->p4type = (signed char)n; if( n==P4_VTAB ) sqlite3VtabLock((VTable*)zP4); } } /* ** Set the P4 on the most recently added opcode to the KeyInfo for the ** index given. */ |
︙ | ︙ |
Changes to src/vdbesort.c.
︙ | ︙ | |||
733 734 735 736 737 738 739 | assert( pFile->iEof>iStart ); assert( pReadr->aAlloc==0 && pReadr->nAlloc==0 ); assert( pReadr->aBuffer==0 ); assert( pReadr->aMap==0 ); rc = vdbePmaReaderSeek(pTask, pReadr, pFile, iStart); if( rc==SQLITE_OK ){ | | | 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 | assert( pFile->iEof>iStart ); assert( pReadr->aAlloc==0 && pReadr->nAlloc==0 ); assert( pReadr->aBuffer==0 ); assert( pReadr->aMap==0 ); rc = vdbePmaReaderSeek(pTask, pReadr, pFile, iStart); if( rc==SQLITE_OK ){ u64 nByte = 0; /* Size of PMA in bytes */ rc = vdbePmaReadVarint(pReadr, &nByte); pReadr->iEof = pReadr->iReadOff + nByte; *pnByte += nByte; } if( rc==SQLITE_OK ){ rc = vdbePmaReaderNext(pReadr); |
︙ | ︙ |
Changes to src/vxworks.h.
︙ | ︙ | |||
22 23 24 25 26 27 28 29 | #define SQLITE_HOMEGROWN_RECURSIVE_MUTEX 1 #define SQLITE_OMIT_LOAD_EXTENSION 1 #define SQLITE_ENABLE_LOCKING_STYLE 0 #define HAVE_UTIME 1 #else /* This is not VxWorks. */ #define OS_VXWORKS 0 #endif /* defined(_WRS_KERNEL) */ | > > | 22 23 24 25 26 27 28 29 30 31 | #define SQLITE_HOMEGROWN_RECURSIVE_MUTEX 1 #define SQLITE_OMIT_LOAD_EXTENSION 1 #define SQLITE_ENABLE_LOCKING_STYLE 0 #define HAVE_UTIME 1 #else /* This is not VxWorks. */ #define OS_VXWORKS 0 #define HAVE_FCHOWN 1 #define HAVE_READLINK 1 #endif /* defined(_WRS_KERNEL) */ |
Changes to src/wal.c.
︙ | ︙ | |||
704 705 706 707 708 709 710 711 712 713 714 715 716 717 | nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); sqlite3Put4byte(&aFrame[16], aCksum[0]); sqlite3Put4byte(&aFrame[20], aCksum[1]); } } /* ** Check to see if the frame with header in aFrame[] and content ** in aData[] is valid. If it is a valid frame, fill *piPage and ** *pnTruncate and return true. Return if the frame is not valid. | > > | 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 | nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); sqlite3Put4byte(&aFrame[16], aCksum[0]); sqlite3Put4byte(&aFrame[20], aCksum[1]); }else{ memset(&aFrame[8], 0, 16); } } /* ** Check to see if the frame with header in aFrame[] and content ** in aData[] is valid. If it is a valid frame, fill *piPage and ** *pnTruncate and return true. Return if the frame is not valid. |
︙ | ︙ | |||
3400 3401 3402 3403 3404 3405 3406 3407 3408 | ** or zero if it is not (or if pWal is NULL). */ int sqlite3WalFramesize(Wal *pWal){ assert( pWal==0 || pWal->readLock>=0 ); return (pWal ? pWal->szPage : 0); } #endif #endif /* #ifndef SQLITE_OMIT_WAL */ | > > > > > > | 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 | ** or zero if it is not (or if pWal is NULL). */ int sqlite3WalFramesize(Wal *pWal){ assert( pWal==0 || pWal->readLock>=0 ); return (pWal ? pWal->szPage : 0); } #endif /* Return the sqlite3_file object for the WAL file */ sqlite3_file *sqlite3WalFile(Wal *pWal){ return pWal->pWalFd; } #endif /* #ifndef SQLITE_OMIT_WAL */ |
Changes to src/wal.h.
︙ | ︙ | |||
40 41 42 43 44 45 46 47 48 49 50 51 52 53 | # define sqlite3WalFrames(u,v,w,x,y,z) 0 # define sqlite3WalCheckpoint(r,s,t,u,v,w,x,y,z) 0 # define sqlite3WalCallback(z) 0 # define sqlite3WalExclusiveMode(y,z) 0 # define sqlite3WalHeapMemory(z) 0 # define sqlite3WalFramesize(z) 0 # define sqlite3WalFindFrame(x,y,z) 0 #else #define WAL_SAVEPOINT_NDATA 4 /* Connection to a write-ahead log (WAL) file. ** There is one object of this type for each pager. */ | > | 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 | # define sqlite3WalFrames(u,v,w,x,y,z) 0 # define sqlite3WalCheckpoint(r,s,t,u,v,w,x,y,z) 0 # define sqlite3WalCallback(z) 0 # define sqlite3WalExclusiveMode(y,z) 0 # define sqlite3WalHeapMemory(z) 0 # define sqlite3WalFramesize(z) 0 # define sqlite3WalFindFrame(x,y,z) 0 # define sqlite3WalFile(x) 0 #else #define WAL_SAVEPOINT_NDATA 4 /* Connection to a write-ahead log (WAL) file. ** There is one object of this type for each pager. */ |
︙ | ︙ | |||
133 134 135 136 137 138 139 140 141 142 | #ifdef SQLITE_ENABLE_ZIPVFS /* If the WAL file is not empty, return the number of bytes of content ** stored in each frame (i.e. the db page-size when the WAL was created). */ int sqlite3WalFramesize(Wal *pWal); #endif #endif /* ifndef SQLITE_OMIT_WAL */ #endif /* _WAL_H_ */ | > > > | 134 135 136 137 138 139 140 141 142 143 144 145 146 | #ifdef SQLITE_ENABLE_ZIPVFS /* If the WAL file is not empty, return the number of bytes of content ** stored in each frame (i.e. the db page-size when the WAL was created). */ int sqlite3WalFramesize(Wal *pWal); #endif /* Return the sqlite3_file object for the WAL file */ sqlite3_file *sqlite3WalFile(Wal *pWal); #endif /* ifndef SQLITE_OMIT_WAL */ #endif /* _WAL_H_ */ |
Changes to src/walker.c.
︙ | ︙ | |||
32 33 34 35 36 37 38 | ** ** WRC_Abort Do no more callbacks. Unwind the stack and ** return the top-level walk call. ** ** The return value from this routine is WRC_Abort to abandon the tree walk ** and WRC_Continue to continue. */ | | < > > > | 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | ** ** WRC_Abort Do no more callbacks. Unwind the stack and ** return the top-level walk call. ** ** The return value from this routine is WRC_Abort to abandon the tree walk ** and WRC_Continue to continue. */ static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ int rc; testcase( ExprHasProperty(pExpr, EP_TokenOnly) ); testcase( ExprHasProperty(pExpr, EP_Reduced) ); rc = pWalker->xExprCallback(pWalker, pExpr); if( rc==WRC_Continue && !ExprHasProperty(pExpr,EP_TokenOnly) ){ if( sqlite3WalkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort; if( sqlite3WalkExpr(pWalker, pExpr->pRight) ) return WRC_Abort; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort; }else{ if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; } } return rc & WRC_Abort; } int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ return pExpr ? walkExpr(pWalker,pExpr) : WRC_Continue; } /* ** Call sqlite3WalkExpr() for every expression in list p or until ** an abort request is seen. */ int sqlite3WalkExprList(Walker *pWalker, ExprList *p){ int i; |
︙ | ︙ |
Changes to src/where.c.
︙ | ︙ | |||
4314 4315 4316 4317 4318 4319 4320 | assert( pTabItem->iCursor==pLevel->iTabCur ); testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 ); testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS ); if( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol<BMS && HasRowid(pTab) ){ Bitmask b = pTabItem->colUsed; int n = 0; for(; b; b=b>>1, n++){} | < | | 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 | assert( pTabItem->iCursor==pLevel->iTabCur ); testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 ); testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS ); if( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol<BMS && HasRowid(pTab) ){ Bitmask b = pTabItem->colUsed; int n = 0; for(; b; b=b>>1, n++){} sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32); assert( n<=pTab->nCol ); } #ifdef SQLITE_ENABLE_CURSOR_HINTS if( pLoop->u.btree.pIndex!=0 ){ sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete); }else #endif |
︙ | ︙ |
Changes to test/orderby1.test.
︙ | ︙ | |||
523 524 525 526 527 528 529 530 531 | INSERT INTO t1 SELECT i%2, randomblob(500) FROM cnt; } do_test 8.3 { db eval { SELECT * FROM t1 ORDER BY a, b } { incr res $a } set res } 5000 finish_test | > > > > > > > > > > > > > > > > > | 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 | INSERT INTO t1 SELECT i%2, randomblob(500) FROM cnt; } do_test 8.3 { db eval { SELECT * FROM t1 ORDER BY a, b } { incr res $a } set res } 5000 #--------------------------------------------------------------------------- # https://www.sqlite.org/src/tktview/cb3aa0641d9a413841c004293a4fc06cdc122029 # # Adverse interaction between scalar subqueries and the partial-sorting # logic. # do_execsql_test 9.0 { DROP TABLE IF EXISTS t1; CREATE TABLE t1(x INTEGER PRIMARY KEY); INSERT INTO t1 VALUES(1),(2); DROP TABLE IF EXISTS t2; CREATE TABLE t2(y); INSERT INTO t2 VALUES(9),(8),(3),(4); SELECT (SELECT x||y FROM t2, t1 ORDER BY x, y); } {13} finish_test |
Changes to test/waloverwrite.test.
︙ | ︙ | |||
150 151 152 153 154 155 156 | do_test 1.$tn.9 { db2 close forcecopy test.db-wal test.db2-wal sqlite3 db2 test.db2 execsql { SELECT sum(length(y)) FROM t1 } db2 } [expr 20*798] | | | 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 | do_test 1.$tn.9 { db2 close forcecopy test.db-wal test.db2-wal sqlite3 db2 test.db2 execsql { SELECT sum(length(y)) FROM t1 } db2 } [expr 20*798] do_test 1.$tn.10 { execsql { PRAGMA integrity_check } db2 } ok db2 close } finish_test |
Changes to tool/sqldiff.c.
︙ | ︙ | |||
990 991 992 993 994 995 996 | if( lenSrc<=NHASH ){ putInt(lenOut, &zDelta); *(zDelta++) = ':'; memcpy(zDelta, zOut, lenOut); zDelta += lenOut; putInt(checksum(zOut, lenOut), &zDelta); *(zDelta++) = ';'; | | | 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 | if( lenSrc<=NHASH ){ putInt(lenOut, &zDelta); *(zDelta++) = ':'; memcpy(zDelta, zOut, lenOut); zDelta += lenOut; putInt(checksum(zOut, lenOut), &zDelta); *(zDelta++) = ';'; return (int)(zDelta - zOrigDelta); } /* Compute the hash table used to locate matching sections in the ** source file. */ nHash = lenSrc/NHASH; collide = sqlite3_malloc( nHash*2*sizeof(int) ); |
︙ | ︙ | |||
1137 1138 1139 1140 1141 1142 1143 | memcpy(zDelta, &zOut[base], lenOut-base); zDelta += lenOut-base; } /* Output the final checksum record. */ putInt(checksum(zOut, lenOut), &zDelta); *(zDelta++) = ';'; sqlite3_free(collide); | | | 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 | memcpy(zDelta, &zOut[base], lenOut-base); zDelta += lenOut-base; } /* Output the final checksum record. */ putInt(checksum(zOut, lenOut), &zDelta); *(zDelta++) = ';'; sqlite3_free(collide); return (int)(zDelta - zOrigDelta); } /* ** End of code copied from fossil. **************************************************************************/ static void strPrintfArray( |
︙ | ︙ |