Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Merge latest trunk into this branch. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | schemalint |
| Files: | files | file ages | folders |
| SHA3-256: |
cb721d0b36268a7b0ef493fa4d7f6bcb |
| User & Date: | dan 2017-03-31 11:20:20 |
Context
|
2017-04-04
| ||
| 04:23 | Add the sqlite3_whereinfo_hook() API - an experimental API replacing the DBCONFIG_WHEREINFO hack on this branch. (check-in: a54aef35 user: dan tags: schemalint) | |
|
2017-03-31
| ||
| 11:20 | Merge latest trunk into this branch. (check-in: cb721d0b user: dan tags: schemalint) | |
| 08:00 | Update shell6.test to account for the fact that tests are now run in a separate directory. (check-in: 1e3622de user: dan tags: schemalint) | |
|
2017-03-30
| ||
| 17:13 | Declare the Lemon-generated parser object as itself. (Duh) (check-in: c8000e94 user: drh tags: trunk) | |
Changes
Changes to Makefile.in.
26 26 # will run on the target platform. (BCC and TCC are usually the 27 27 # same unless your are cross-compiling.) Separate CC and CFLAGS macros 28 28 # are provide so that these aspects of the build process can be changed 29 29 # on the "make" command-line. Ex: "make CC=clang CFLAGS=-fsanitize=undefined" 30 30 # 31 31 CC = @CC@ 32 32 CFLAGS = @CPPFLAGS@ @CFLAGS@ 33 -TCC = $(CC) $(CFLAGS) -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/fts3 33 +TCC = ${CC} ${CFLAGS} -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/icu 34 +TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session 34 35 35 36 # Define this for the autoconf-based build, so that the code knows it can 36 37 # include the generated config.h 37 38 # 38 39 TCC += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite 39 40 40 41 # Define -DNDEBUG to compile without debugging (i.e., for production usage) ................................................................................ 175 176 func.lo global.lo hash.lo \ 176 177 icu.lo insert.lo json1.lo legacy.lo loadext.lo \ 177 178 main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ 178 179 memjournal.lo \ 179 180 mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ 180 181 notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ 181 182 pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ 182 - random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \ 183 + random.lo resolve.lo rowset.lo rtree.lo \ 184 + sqlite3session.lo select.lo sqlite3rbu.lo status.lo \ 183 185 table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ 184 186 update.lo util.lo vacuum.lo \ 185 187 vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ 186 188 vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \ 187 189 utf.lo vtab.lo 188 190 189 191 # Object files for the amalgamation. ................................................................................ 338 340 SRC += \ 339 341 $(TOP)/ext/icu/sqliteicu.h \ 340 342 $(TOP)/ext/icu/icu.c 341 343 SRC += \ 342 344 $(TOP)/ext/rtree/rtree.h \ 343 345 $(TOP)/ext/rtree/rtree.c 344 346 SRC += \ 347 + $(TOP)/ext/session/sqlite3session.c \ 348 + $(TOP)/ext/session/sqlite3session.h 349 +SRC += \ 345 350 $(TOP)/ext/rbu/sqlite3rbu.h \ 346 351 $(TOP)/ext/rbu/sqlite3rbu.c 347 352 SRC += \ 348 353 $(TOP)/ext/misc/json1.c 349 - 350 - 351 354 352 355 # Generated source code files 353 356 # 354 357 SRC += \ 355 358 keywordhash.h \ 356 359 opcodes.c \ 357 360 opcodes.h \ ................................................................................ 375 378 $(TOP)/src/test_autoext.c \ 376 379 $(TOP)/src/test_async.c \ 377 380 $(TOP)/src/test_backup.c \ 378 381 $(TOP)/src/test_bestindex.c \ 379 382 $(TOP)/src/test_blob.c \ 380 383 $(TOP)/src/test_btree.c \ 381 384 $(TOP)/src/test_config.c \ 385 + $(TOP)/src/test_delete.c \ 382 386 $(TOP)/src/test_demovfs.c \ 383 387 $(TOP)/src/test_devsym.c \ 384 388 $(TOP)/src/test_fs.c \ 385 389 $(TOP)/src/test_func.c \ 386 390 $(TOP)/src/test_hexio.c \ 387 391 $(TOP)/src/test_init.c \ 388 392 $(TOP)/src/test_intarray.c \ ................................................................................ 401 405 $(TOP)/src/test_syscall.c \ 402 406 $(TOP)/src/test_tclvar.c \ 403 407 $(TOP)/src/test_thread.c \ 404 408 $(TOP)/src/test_vfs.c \ 405 409 $(TOP)/src/test_windirent.c \ 406 410 $(TOP)/src/test_wsd.c \ 407 411 $(TOP)/ext/fts3/fts3_term.c \ 408 - $(TOP)/ext/fts3/fts3_test.c \ 412 + $(TOP)/ext/fts3/fts3_test.c \ 413 + $(TOP)/ext/session/test_session.c \ 409 414 $(TOP)/ext/rbu/test_rbu.c 410 415 411 416 # Statically linked extensions 412 417 # 413 418 TESTSRC += \ 414 419 $(TOP)/ext/misc/amatch.c \ 420 + $(TOP)/ext/misc/carray.c \ 415 421 $(TOP)/ext/misc/closure.c \ 422 + $(TOP)/ext/misc/csv.c \ 416 423 $(TOP)/ext/misc/eval.c \ 417 424 $(TOP)/ext/misc/fileio.c \ 418 425 $(TOP)/ext/misc/fuzzer.c \ 419 426 $(TOP)/ext/fts5/fts5_tcl.c \ 420 427 $(TOP)/ext/fts5/fts5_test_mi.c \ 421 428 $(TOP)/ext/fts5/fts5_test_tok.c \ 422 429 $(TOP)/ext/misc/ieee754.c \ 423 430 $(TOP)/ext/misc/nextchar.c \ 424 431 $(TOP)/ext/misc/percentile.c \ 425 432 $(TOP)/ext/misc/regexp.c \ 433 + $(TOP)/ext/misc/remember.c \ 426 434 $(TOP)/ext/misc/series.c \ 427 435 $(TOP)/ext/misc/spellfix.c \ 428 436 $(TOP)/ext/misc/totype.c \ 429 437 $(TOP)/ext/misc/wholenumber.c 430 438 431 439 # Source code to the library files needed by the test fixture 432 440 # ................................................................................ 470 478 parse.c \ 471 479 $(TOP)/ext/fts3/fts3.c \ 472 480 $(TOP)/ext/fts3/fts3_aux.c \ 473 481 $(TOP)/ext/fts3/fts3_expr.c \ 474 482 $(TOP)/ext/fts3/fts3_term.c \ 475 483 $(TOP)/ext/fts3/fts3_tokenizer.c \ 476 484 $(TOP)/ext/fts3/fts3_write.c \ 477 - $(TOP)/ext/async/sqlite3async.c 485 + $(TOP)/ext/async/sqlite3async.c \ 486 + $(TOP)/ext/session/sqlite3session.c 478 487 479 488 # Header files used by all library source files. 480 489 # 481 490 HDR = \ 482 491 $(TOP)/src/btree.h \ 483 492 $(TOP)/src/btreeInt.h \ 484 493 $(TOP)/src/hash.h \ ................................................................................ 529 538 530 539 # executables needed for testing 531 540 # 532 541 TESTPROGS = \ 533 542 testfixture$(TEXE) \ 534 543 sqlite3$(TEXE) \ 535 544 sqlite3_analyzer$(TEXE) \ 536 - sqldiff$(TEXE) 545 + sqldiff$(TEXE) \ 546 + dbhash$(TEXE) 537 547 538 548 # Databases containing fuzzer test cases 539 549 # 540 550 FUZZDATA = \ 541 551 $(TOP)/test/fuzzdata1.db \ 542 552 $(TOP)/test/fuzzdata2.db \ 543 553 $(TOP)/test/fuzzdata3.db \ 544 - $(TOP)/test/fuzzdata4.db 554 + $(TOP)/test/fuzzdata4.db \ 555 + $(TOP)/test/fuzzdata5.db 545 556 546 557 # Standard options to testfixture 547 558 # 548 559 TESTOPTS = --verbose=file --output=test-out.txt 549 560 550 561 # Extra compiler options for various shell tools 551 562 # 552 563 SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 564 +# SHELL_OPT += -DSQLITE_ENABLE_FTS5 553 565 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS 566 +SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION 554 567 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 555 -FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 568 +FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ 569 +FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 570 +FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c 571 +DBFUZZ_OPT = 556 572 557 573 # This is the default Makefile target. The objects listed here 558 574 # are what get build when you type just "make" with no arguments. 559 575 # 560 576 all: sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la) 561 577 562 578 Makefile: $(TOP)/Makefile.in ................................................................................ 577 593 -avoid-version 578 594 579 595 sqlite3$(TEXE): $(TOP)/src/shell.c sqlite3.c 580 596 $(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \ 581 597 $(TOP)/src/shell.c sqlite3.c \ 582 598 $(LIBREADLINE) $(TLIBS) -rpath "$(libdir)" 583 599 584 -sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h 585 - $(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) 600 +sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h 601 + $(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS) 602 + 603 +dbhash$(TEXE): $(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h 604 + $(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS) 605 + 606 +scrub$(TEXE): $(TOP)/ext/misc/scrub.c sqlite3.lo 607 + $(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \ 608 + $(TOP)/ext/misc/scrub.c sqlite3.lo $(TLIBS) 586 609 587 610 srcck1$(BEXE): $(TOP)/tool/srcck1.c 588 611 $(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c 589 612 590 613 sourcetest: srcck1$(BEXE) sqlite3.c 591 614 ./srcck1 sqlite3.c 592 615 593 616 fuzzershell$(TEXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h 594 617 $(LTLINK) -o $@ $(FUZZERSHELL_OPT) \ 595 618 $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS) 596 619 597 -fuzzcheck$(TEXE): $(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h 598 - $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS) 620 +fuzzcheck$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h 621 + $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS) 599 622 600 -mptester$(TEXE): sqlite3.c $(TOP)/mptest/mptest.c 601 - $(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \ 623 +ossshell$(TEXE): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h 624 + $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \ 625 + $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS) 626 + 627 +dbfuzz$(TEXE): $(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h 628 + $(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS) 629 + 630 +mptester$(TEXE): sqlite3.lo $(TOP)/mptest/mptest.c 631 + $(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \ 602 632 $(TLIBS) -rpath "$(libdir)" 603 633 604 634 MPTEST1=./mptester$(TEXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20 605 635 MPTEST2=./mptester$(TEXE) mptest.db $(TOP)/mptest/multiwrite01.test --repeat 20 606 636 mptest: mptester$(TEXE) 607 637 rm -f mptest.db 608 638 $(MPTEST1) --journalmode DELETE ................................................................................ 630 660 mv vdbe.new tsrc/vdbe.c 631 661 cp fts5.c fts5.h tsrc 632 662 touch .target_source 633 663 634 664 sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl 635 665 $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl 636 666 cp tsrc/shell.c tsrc/sqlite3ext.h . 667 + cp $(TOP)/ext/session/sqlite3session.h . 637 668 638 669 sqlite3ext.h: .target_source 639 670 cp tsrc/sqlite3ext.h . 640 671 641 672 tclsqlite3.c: sqlite3.c 642 673 echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c 643 674 cat sqlite3.c >>tclsqlite3.c ................................................................................ 993 1024 994 1025 fts3_write.lo: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR) 995 1026 $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c 996 1027 997 1028 rtree.lo: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) 998 1029 $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c 999 1030 1031 +sqlite3session.lo: $(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR) 1032 + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c 1033 + 1000 1034 json1.lo: $(TOP)/ext/misc/json1.c 1001 1035 $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c 1002 1036 1003 1037 # FTS5 things 1004 1038 # 1005 1039 FTS5_SRC = \ 1006 1040 $(TOP)/ext/fts5/fts5.h \ ................................................................................ 1018 1052 $(TOP)/ext/fts5/fts5_unicode2.c \ 1019 1053 $(TOP)/ext/fts5/fts5_varint.c \ 1020 1054 $(TOP)/ext/fts5/fts5_vocab.c \ 1021 1055 1022 1056 fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon 1023 1057 cp $(TOP)/ext/fts5/fts5parse.y . 1024 1058 rm -f fts5parse.h 1025 - ./lemon $(OPTS) fts5parse.y 1059 + ./lemon$(BEXE) $(OPTS) fts5parse.y 1026 1060 1027 1061 fts5parse.h: fts5parse.c 1028 1062 1029 1063 fts5.c: $(FTS5_SRC) 1030 1064 $(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl 1031 1065 cp $(TOP)/ext/fts5/fts5.h . 1032 1066 ................................................................................ 1077 1111 1078 1112 fastfuzztest: fuzzcheck$(TEXE) $(FUZZDATA) 1079 1113 ./fuzzcheck$(TEXE) --limit-mem 100M $(FUZZDATA) 1080 1114 1081 1115 valgrindfuzz: fuzzcheck$(TEXT) $(FUZZDATA) 1082 1116 valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M --timeout 600 $(FUZZDATA) 1083 1117 1118 +# The veryquick.test TCL tests. 1119 +# 1120 +tcltest: ./testfixture$(TEXE) 1121 + ./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS) 1122 + 1084 1123 # Minimal testing that runs in less than 3 minutes 1085 1124 # 1086 1125 quicktest: ./testfixture$(TEXE) 1087 1126 ./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS) 1088 1127 1089 1128 # This is the common case. Run many tests that do not take too long, 1090 1129 # including fuzzcheck, sqlite3_analyzer, and sqldiff tests. 1091 1130 # 1092 -test: $(TESTPROGS) sourcetest fastfuzztest 1093 - ./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS) 1131 +test: fastfuzztest sourcetest $(TESTPROGS) tcltest 1094 1132 1095 1133 # Run a test using valgrind. This can take a really long time 1096 1134 # because valgrind is so much slower than a native machine. 1097 1135 # 1098 1136 valgrindtest: $(TESTPROGS) valgrindfuzz 1099 1137 OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind $(TESTOPTS) 1100 1138 ................................................................................ 1113 1151 echo "static const char *zMainloop = " >> $@ 1114 1152 $(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@ 1115 1153 echo "; return zMainloop; }" >> $@ 1116 1154 1117 1155 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c 1118 1156 $(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS) 1119 1157 1158 +dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo 1159 + $(LTLINK) -DDBDUMP_STANDALONE -o $@ \ 1160 + $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS) 1161 + 1120 1162 showdb$(TEXE): $(TOP)/tool/showdb.c sqlite3.lo 1121 1163 $(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS) 1122 1164 1123 1165 showstat4$(TEXE): $(TOP)/tool/showstat4.c sqlite3.lo 1124 1166 $(LTLINK) -o $@ $(TOP)/tool/showstat4.c sqlite3.lo $(TLIBS) 1125 1167 1126 1168 showjournal$(TEXE): $(TOP)/tool/showjournal.c sqlite3.lo 1127 1169 $(LTLINK) -o $@ $(TOP)/tool/showjournal.c sqlite3.lo $(TLIBS) 1128 1170 1129 1171 showwal$(TEXE): $(TOP)/tool/showwal.c sqlite3.lo 1130 1172 $(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS) 1131 1173 1174 +changeset$(TEXE): $(TOP)/ext/session/changeset.c sqlite3.lo 1175 + $(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS) 1176 + 1132 1177 rollback-test$(TEXE): $(TOP)/tool/rollback-test.c sqlite3.lo 1133 1178 $(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS) 1134 1179 1135 1180 LogEst$(TEXE): $(TOP)/tool/logest.c sqlite3.h 1136 1181 $(LTLINK) -I. -o $@ $(TOP)/tool/logest.c 1137 1182 1138 -wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.c 1139 - $(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.c $(TLIBS) 1183 +wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.lo 1184 + $(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS) 1140 1185 1141 -speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.lo 1142 - $(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS) 1186 +speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.c 1187 + $(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS) 1188 + 1189 +KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ 1190 + 1191 +kvtest$(TEXE): $(TOP)/test/kvtest.c sqlite3.c 1192 + $(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS) 1143 1193 1144 1194 rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo 1145 1195 $(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS) 1146 1196 1147 1197 loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la 1148 1198 $(LTLINK) $(TOP)/tool/loadfts.c libsqlite3.la -o $@ $(TLIBS) 1149 1199 1150 1200 # This target will fail if the SQLite amalgamation contains any exported 1151 1201 # symbols that do not begin with "sqlite3_". It is run as part of the 1152 1202 # releasetest.tcl script. 1153 1203 # 1204 +VALIDIDS=' sqlite3(changeset|changegroup|session)?_' 1154 1205 checksymbols: sqlite3.lo 1155 - nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0 1206 + nm -g --defined-only sqlite3.lo | egrep -v $(VALIDIDS); test $$? -ne 0 1156 1207 echo '0 errors out of 1 tests' 1157 1208 1158 1209 # Build the amalgamation-autoconf package. The amalamgation-tarball target builds 1159 1210 # a tarball named for the version number. Ex: sqlite-autoconf-3110000.tar.gz. 1160 1211 # The snapshot-tarball target builds a tarball named by the SHA1 hash 1161 1212 # 1162 1213 amalgamation-tarball: sqlite3.c ................................................................................ 1211 1262 clean: 1212 1263 rm -f *.lo *.la *.o sqlite3$(TEXE) libsqlite3.la 1213 1264 rm -f sqlite3.h opcodes.* 1214 1265 rm -rf .libs .deps 1215 1266 rm -f lemon$(BEXE) lempar.c parse.* sqlite*.tar.gz 1216 1267 rm -f mkkeywordhash$(BEXE) keywordhash.h 1217 1268 rm -f *.da *.bb *.bbg gmon.out 1218 - rm -rf quota2a quota2b quota2c 1219 1269 rm -rf tsrc .target_source 1220 1270 rm -f tclsqlite3$(TEXE) 1221 1271 rm -f testfixture$(TEXE) test.db 1222 1272 rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE) 1223 1273 rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE) 1224 - rm -f wordcount$(TEXE) 1274 + rm -f wordcount$(TEXE) changeset$(TEXE) 1225 1275 rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def 1226 1276 rm -f sqlite3.c 1227 1277 rm -f sqlite3rc.h 1228 1278 rm -f shell.c sqlite3ext.h 1229 1279 rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c 1230 1280 rm -f sqlite-*-output.vsix 1231 1281 rm -f mptester mptester.exe 1232 1282 rm -f rbu rbu.exe 1233 1283 rm -f srcck1 srcck1.exe 1234 1284 rm -f fuzzershell fuzzershell.exe 1235 1285 rm -f fuzzcheck fuzzcheck.exe 1236 1286 rm -f sqldiff sqldiff.exe 1287 + rm -f dbhash dbhash.exe 1237 1288 rm -f fts5.* fts5parse.* 1238 1289 1239 1290 distclean: clean 1240 1291 rm -f config.h config.log config.status libtool Makefile sqlite3.pc 1241 1292 1242 1293 # 1243 1294 # Windows section
Changes to Makefile.msc.
19 19 # <</mark>> 20 20 21 21 # Set this non-0 to enable full warnings (-W4, etc) when compiling. 22 22 # 23 23 !IFNDEF USE_FULLWARN 24 24 USE_FULLWARN = 0 25 25 !ENDIF 26 + 27 +# Set this non-0 to enable treating warnings as errors (-WX, etc) when 28 +# compiling. 29 +# 30 +!IFNDEF USE_FATAL_WARN 31 +USE_FATAL_WARN = 0 32 +!ENDIF 33 + 34 +# Set this non-0 to enable full runtime error checks (-RTC1, etc). This 35 +# has no effect if (any) optimizations are enabled. 36 +# 37 +!IFNDEF USE_RUNTIME_CHECKS 38 +USE_RUNTIME_CHECKS = 0 39 +!ENDIF 40 + 41 +# Set this non-0 to create a SQLite amalgamation file that excludes the 42 +# various built-in extensions. 43 +# 44 +!IFNDEF MINIMAL_AMALGAMATION 45 +MINIMAL_AMALGAMATION = 0 46 +!ENDIF 26 47 27 48 # Set this non-0 to use "stdcall" calling convention for the core library 28 49 # and shell executable. 29 50 # 30 51 !IFNDEF USE_STDCALL 31 52 USE_STDCALL = 0 32 53 !ENDIF ................................................................................ 193 214 # Enable use of available compiler optimizations? Normally, this should be 194 215 # non-zero. Setting this to zero, thus disabling all compiler optimizations, 195 216 # can be useful for testing. 196 217 # 197 218 !IFNDEF OPTIMIZATIONS 198 219 OPTIMIZATIONS = 2 199 220 !ENDIF 221 + 222 +# Set this to non-0 to enable support for the session extension. 223 +# 224 +!IFNDEF SESSION 225 +SESSION = 0 226 +!ENDIF 200 227 201 228 # Set the source code file to be used by executables and libraries when 202 229 # they need the amalgamation. 203 230 # 204 231 !IFNDEF SQLITE3C 205 232 !IF $(SPLIT_AMALGAMATION)!=0 206 233 SQLITE3C = sqlite3-all.c ................................................................................ 253 280 !IF $(FOR_WIN10)!=0 254 281 SQLITE3EXEPDB = 255 282 !ELSE 256 283 SQLITE3EXEPDB = /pdb:sqlite3sh.pdb 257 284 !ENDIF 258 285 !ENDIF 259 286 287 +# <<mark>> 288 +# These are the names of the customized Tcl header files used by various parts 289 +# of this makefile when the stdcall calling convention is in use. It is not 290 +# used for any other purpose. 291 +# 292 +!IFNDEF SQLITETCLH 293 +SQLITETCLH = sqlite_tcl.h 294 +!ENDIF 295 + 296 +!IFNDEF SQLITETCLDECLSH 297 +SQLITETCLDECLSH = sqlite_tclDecls.h 298 +!ENDIF 299 + 300 +# These are the additional targets that the targets that integrate with the 301 +# Tcl library should depend on when compiling, etc. 302 +# 303 +!IFNDEF SQLITE_TCL_DEP 304 +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 305 +SQLITE_TCL_DEP = $(SQLITETCLDECLSH) $(SQLITETCLH) 306 +!ELSE 307 +SQLITE_TCL_DEP = 308 +!ENDIF 309 +!ENDIF 310 +# <</mark>> 311 + 260 312 # These are the "standard" SQLite compilation options used when compiling for 261 313 # the Windows platform. 262 314 # 263 315 !IFNDEF OPT_FEATURE_FLAGS 316 +!IF $(MINIMAL_AMALGAMATION)==0 264 317 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 265 318 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 319 +!ENDIF 266 320 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 267 321 !ENDIF 322 + 323 +# Should the session extension be enabled? If so, add compilation options 324 +# to enable it. 325 +# 326 +!IF $(SESSION)!=0 327 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1 328 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1 329 +!ENDIF 268 330 269 331 # These are the "extended" SQLite compilation options used when compiling for 270 332 # the Windows 10 platform. 271 333 # 272 334 !IFNDEF EXT_FEATURE_FLAGS 273 335 !IF $(FOR_WIN10)!=0 274 336 EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS4=1 ................................................................................ 433 495 # same unless your are cross-compiling.) 434 496 # 435 497 !IF $(USE_FULLWARN)!=0 436 498 TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS) 437 499 !ELSE 438 500 TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS) 439 501 !ENDIF 502 + 503 +# Check if warnings should be treated as errors when compiling. 504 +# 505 +!IF $(USE_FATAL_WARN)!=0 506 +TCC = $(TCC) -WX 507 +!ENDIF 440 508 441 509 TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src -fp:precise 442 510 RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS) 443 511 444 512 # Check if we want to use the "stdcall" calling convention when compiling. 445 513 # This is not supported by the compilers for non-x86 platforms. It should 446 514 # also be noted here that building any target with these "stdcall" options 447 515 # will most likely fail if the Tcl library is also required. This is due 448 516 # to how the Tcl library functions are declared and exported (i.e. without 449 517 # an explicit calling convention, which results in "cdecl"). 450 518 # 451 519 !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 452 520 !IF "$(PLATFORM)"=="x86" 453 -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall 454 -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall 521 +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall 522 +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall 523 +# <<mark>> 524 +TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl 525 +# <</mark>> 455 526 !ELSE 456 527 !IFNDEF PLATFORM 457 -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall 458 -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall 528 +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall 529 +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall 530 +# <<mark>> 531 +TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl 532 +# <</mark>> 459 533 !ELSE 460 534 CORE_CCONV_OPTS = 461 535 SHELL_CCONV_OPTS = 536 +# <<mark>> 537 +TEST_CCONV_OPTS = 538 +# <</mark>> 462 539 !ENDIF 463 540 !ENDIF 464 541 !ELSE 465 542 CORE_CCONV_OPTS = 466 543 SHELL_CCONV_OPTS = 544 +# <<mark>> 545 +TEST_CCONV_OPTS = 546 +# <</mark>> 467 547 !ENDIF 468 548 469 549 # These are additional compiler options used for the core library. 470 550 # 471 551 !IFNDEF CORE_COMPILE_OPTS 472 552 !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 473 553 CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) -DSQLITE_API=__declspec(dllexport) ................................................................................ 603 683 # builds, we need to make sure the compiler can find these. 604 684 # 605 685 !IF $(USE_AMALGAMATION)==0 606 686 TCC = $(TCC) -I$(TOP)\ext\fts3 607 687 RCC = $(RCC) -I$(TOP)\ext\fts3 608 688 TCC = $(TCC) -I$(TOP)\ext\rtree 609 689 RCC = $(RCC) -I$(TOP)\ext\rtree 690 +TCC = $(TCC) -I$(TOP)\ext\session 691 +RCC = $(RCC) -I$(TOP)\ext\session 610 692 !ENDIF 611 693 612 694 # The mksqlite3c.tcl script accepts some options on the command 613 695 # line. When compiling with debugging enabled, some of these 614 696 # options are necessary in order to allow debugging symbols to 615 697 # work correctly with Visual Studio when using the amalgamation. 616 698 # 699 +!IFNDEF MKSQLITE3C_TOOL 700 +!IF $(MINIMAL_AMALGAMATION)!=0 701 +MKSQLITE3C_TOOL = $(TOP)\tool\mksqlite3c-noext.tcl 702 +!ELSE 703 +MKSQLITE3C_TOOL = $(TOP)\tool\mksqlite3c.tcl 704 +!ENDIF 705 +!ENDIF 706 + 617 707 !IFNDEF MKSQLITE3C_ARGS 618 708 !IF $(DEBUG)>1 619 709 MKSQLITE3C_ARGS = --linemacros 620 710 !ELSE 621 711 MKSQLITE3C_ARGS = 712 +!ENDIF 713 +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 714 +MKSQLITE3C_ARGS = $(MKSQLITE3C_ARGS) --useapicall 715 +!ENDIF 716 +!ENDIF 717 + 718 +# The mksqlite3h.tcl script accepts some options on the command line. 719 +# When compiling with stdcall support, some of these options are 720 +# necessary. 721 +# 722 +!IFNDEF MKSQLITE3H_ARGS 723 +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 724 +MKSQLITE3H_ARGS = --useapicall 725 +!ELSE 726 +MKSQLITE3H_ARGS = 622 727 !ENDIF 623 728 !ENDIF 624 729 # <</mark>> 625 730 626 731 # Define -DNDEBUG to compile without debugging (i.e., for production usage) 627 732 # Omitting the define will cause extra debugging code to be inserted and 628 733 # includes extra comments when "EXPLAIN stmt" is used. ................................................................................ 637 742 TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1 638 743 RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 639 744 !ENDIF 640 745 641 746 !IF $(DEBUG)>2 642 747 TCC = $(TCC) -DSQLITE_DEBUG=1 643 748 RCC = $(RCC) -DSQLITE_DEBUG=1 749 +!IF $(DYNAMIC_SHELL)==0 750 +TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE 751 +RCC = $(RCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE 752 +!ENDIF 644 753 !ENDIF 645 754 646 755 !IF $(DEBUG)>4 || $(OSTRACE)!=0 647 756 TCC = $(TCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1 648 757 RCC = $(RCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1 649 758 !ENDIF 650 759 ................................................................................ 698 807 !ENDIF 699 808 700 809 !IFNDEF TCLLIBDIR 701 810 TCLLIBDIR = c:\tcl\lib 702 811 !ENDIF 703 812 704 813 !IFNDEF LIBTCL 705 -LIBTCL = tcl85.lib 814 +LIBTCL = tcl86.lib 706 815 !ENDIF 707 816 708 817 !IFNDEF LIBTCLSTUB 709 -LIBTCLSTUB = tclstub85.lib 818 +LIBTCLSTUB = tclstub86.lib 710 819 !ENDIF 711 820 712 821 !IFNDEF LIBTCLPATH 713 822 LIBTCLPATH = c:\tcl\bin 714 823 !ENDIF 715 824 716 825 # The locations of the ICU header and library files. These variables ................................................................................ 732 841 733 842 # This is the command to use for tclsh - normally just "tclsh", but we may 734 843 # know the specific version we want to use. This variable (TCLSH_CMD) may be 735 844 # overridden via the environment prior to running nmake in order to select a 736 845 # specific Tcl shell to use. 737 846 # 738 847 !IFNDEF TCLSH_CMD 739 -TCLSH_CMD = tclsh85 848 +TCLSH_CMD = tclsh 740 849 !ENDIF 741 850 # <</mark>> 742 851 743 852 # Compiler options needed for programs that use the readline() library. 744 853 # 745 854 !IFNDEF READLINE_FLAGS 746 855 READLINE_FLAGS = -DHAVE_READLINE=0 ................................................................................ 815 924 816 925 # If optimizations are enabled or disabled (either implicitly or 817 926 # explicitly), add the necessary flags. 818 927 # 819 928 !IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0 820 929 TCC = $(TCC) -Od 821 930 BCC = $(BCC) -Od 931 +!IF $(USE_RUNTIME_CHECKS)!=0 932 +TCC = $(TCC) -RTC1 933 +BCC = $(BCC) -RTC1 934 +!ENDIF 822 935 !ELSEIF $(OPTIMIZATIONS)>=3 823 936 TCC = $(TCC) -Ox 824 937 BCC = $(BCC) -Ox 825 938 !ELSEIF $(OPTIMIZATIONS)==2 826 939 TCC = $(TCC) -O2 827 940 BCC = $(BCC) -O2 828 941 !ELSEIF $(OPTIMIZATIONS)==1 ................................................................................ 987 1100 func.lo global.lo hash.lo \ 988 1101 icu.lo insert.lo legacy.lo loadext.lo \ 989 1102 main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ 990 1103 memjournal.lo \ 991 1104 mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ 992 1105 notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ 993 1106 pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ 994 - random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \ 1107 + random.lo resolve.lo rowset.lo rtree.lo \ 1108 + sqlite3session.lo select.lo sqlite3rbu.lo status.lo \ 995 1109 table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ 996 1110 update.lo util.lo vacuum.lo \ 997 1111 vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ 998 1112 vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \ 999 1113 utf.lo vtab.lo 1000 1114 # <</mark>> 1001 1115 ................................................................................ 1174 1288 $(TOP)\ext\fts3\fts3_tokenizer1.c \ 1175 1289 $(TOP)\ext\fts3\fts3_tokenize_vtab.c \ 1176 1290 $(TOP)\ext\fts3\fts3_unicode.c \ 1177 1291 $(TOP)\ext\fts3\fts3_unicode2.c \ 1178 1292 $(TOP)\ext\fts3\fts3_write.c \ 1179 1293 $(TOP)\ext\icu\icu.c \ 1180 1294 $(TOP)\ext\rtree\rtree.c \ 1295 + $(TOP)\ext\session\sqlite3session.c \ 1181 1296 $(TOP)\ext\rbu\sqlite3rbu.c \ 1182 1297 $(TOP)\ext\misc\json1.c 1183 1298 1184 1299 # Extension header files, part 1. 1185 1300 # 1186 1301 SRC08 = \ 1187 1302 $(TOP)\ext\fts1\fts1.h \ ................................................................................ 1196 1311 SRC09 = \ 1197 1312 $(TOP)\ext\fts3\fts3.h \ 1198 1313 $(TOP)\ext\fts3\fts3Int.h \ 1199 1314 $(TOP)\ext\fts3\fts3_hash.h \ 1200 1315 $(TOP)\ext\fts3\fts3_tokenizer.h \ 1201 1316 $(TOP)\ext\icu\sqliteicu.h \ 1202 1317 $(TOP)\ext\rtree\rtree.h \ 1203 - $(TOP)\ext\rbu\sqlite3rbu.h 1318 + $(TOP)\ext\rbu\sqlite3rbu.h \ 1319 + $(TOP)\ext\session\sqlite3session.h 1204 1320 1205 1321 # Generated source code files 1206 1322 # 1207 1323 SRC10 = \ 1208 1324 opcodes.c \ 1209 1325 parse.c 1210 1326 ................................................................................ 1211 1327 # Generated header files 1212 1328 # 1213 1329 SRC11 = \ 1214 1330 keywordhash.h \ 1215 1331 opcodes.h \ 1216 1332 parse.h \ 1217 1333 $(SQLITE3H) 1334 + 1335 +# Generated Tcl header files 1336 +# 1337 +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 1338 +SRC12 = \ 1339 + $(SQLITETCLH) \ 1340 + $(SQLITETCLDECLSH) 1341 +!ELSE 1342 +SRC12 = 1343 +!ENDIF 1218 1344 1219 1345 # All source code files. 1220 1346 # 1221 1347 SRC = $(SRC00) $(SRC01) $(SRC02) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11) 1222 1348 1223 1349 # Source code to the test files. 1224 1350 # ................................................................................ 1235 1361 $(TOP)\src\test_autoext.c \ 1236 1362 $(TOP)\src\test_async.c \ 1237 1363 $(TOP)\src\test_backup.c \ 1238 1364 $(TOP)\src\test_bestindex.c \ 1239 1365 $(TOP)\src\test_blob.c \ 1240 1366 $(TOP)\src\test_btree.c \ 1241 1367 $(TOP)\src\test_config.c \ 1368 + $(TOP)\src\test_delete.c \ 1242 1369 $(TOP)\src\test_demovfs.c \ 1243 1370 $(TOP)\src\test_devsym.c \ 1244 1371 $(TOP)\src\test_fs.c \ 1245 1372 $(TOP)\src\test_func.c \ 1246 1373 $(TOP)\src\test_hexio.c \ 1247 1374 $(TOP)\src\test_init.c \ 1248 1375 $(TOP)\src\test_intarray.c \ ................................................................................ 1262 1389 $(TOP)\src\test_tclvar.c \ 1263 1390 $(TOP)\src\test_thread.c \ 1264 1391 $(TOP)\src\test_vfs.c \ 1265 1392 $(TOP)\src\test_windirent.c \ 1266 1393 $(TOP)\src\test_wsd.c \ 1267 1394 $(TOP)\ext\fts3\fts3_term.c \ 1268 1395 $(TOP)\ext\fts3\fts3_test.c \ 1269 - $(TOP)\ext\rbu\test_rbu.c 1396 + $(TOP)\ext\rbu\test_rbu.c \ 1397 + $(TOP)\ext\session\test_session.c 1270 1398 1271 1399 # Statically linked extensions. 1272 1400 # 1273 1401 TESTEXT = \ 1274 1402 $(TOP)\ext\misc\amatch.c \ 1403 + $(TOP)\ext\misc\carray.c \ 1275 1404 $(TOP)\ext\misc\closure.c \ 1405 + $(TOP)\ext\misc\csv.c \ 1276 1406 $(TOP)\ext\misc\eval.c \ 1277 1407 $(TOP)\ext\misc\fileio.c \ 1278 1408 $(TOP)\ext\misc\fuzzer.c \ 1279 1409 $(TOP)\ext\fts5\fts5_tcl.c \ 1280 1410 $(TOP)\ext\fts5\fts5_test_mi.c \ 1281 1411 $(TOP)\ext\fts5\fts5_test_tok.c \ 1282 1412 $(TOP)\ext\misc\ieee754.c \ 1283 1413 $(TOP)\ext\misc\nextchar.c \ 1284 1414 $(TOP)\ext\misc\percentile.c \ 1285 1415 $(TOP)\ext\misc\regexp.c \ 1416 + $(TOP)\ext\misc\remember.c \ 1286 1417 $(TOP)\ext\misc\series.c \ 1287 1418 $(TOP)\ext\misc\spellfix.c \ 1288 1419 $(TOP)\ext\misc\totype.c \ 1289 1420 $(TOP)\ext\misc\wholenumber.c 1290 1421 1291 1422 # Source code to the library files needed by the test fixture 1423 +# (non-amalgamation) 1292 1424 # 1293 1425 TESTSRC2 = \ 1294 1426 $(SRC00) \ 1295 1427 $(SRC01) \ 1296 1428 $(SRC06) \ 1297 1429 $(SRC07) \ 1298 1430 $(SRC10) \ ................................................................................ 1314 1446 $(TOP)\src\os_setup.h \ 1315 1447 $(TOP)\src\os_win.h \ 1316 1448 $(TOP)\src\pager.h \ 1317 1449 $(TOP)\src\pcache.h \ 1318 1450 parse.h \ 1319 1451 $(TOP)\src\pragma.h \ 1320 1452 $(SQLITE3H) \ 1321 - $(TOP)\src\sqlite3ext.h \ 1453 + sqlite3ext.h \ 1322 1454 $(TOP)\src\sqliteInt.h \ 1323 1455 $(TOP)\src\sqliteLimit.h \ 1324 1456 $(TOP)\src\vdbe.h \ 1325 1457 $(TOP)\src\vdbeInt.h \ 1326 1458 $(TOP)\src\vxworks.h \ 1327 1459 $(TOP)\src\whereInt.h 1328 1460 ................................................................................ 1343 1475 $(TOP)\ext\fts3\fts3_tokenizer.h 1344 1476 EXTHDR = $(EXTHDR) \ 1345 1477 $(TOP)\ext\rtree\rtree.h 1346 1478 EXTHDR = $(EXTHDR) \ 1347 1479 $(TOP)\ext\icu\sqliteicu.h 1348 1480 EXTHDR = $(EXTHDR) \ 1349 1481 $(TOP)\ext\rtree\sqlite3rtree.h 1482 +EXTHDR = $(EXTHDR) \ 1483 + $(TOP)\ext\session\sqlite3session.h 1350 1484 1351 1485 # executables needed for testing 1352 1486 # 1353 1487 TESTPROGS = \ 1354 1488 testfixture.exe \ 1355 1489 $(SQLITE3EXE) \ 1356 1490 sqlite3_analyzer.exe \ 1357 - sqldiff.exe 1491 + sqldiff.exe \ 1492 + dbhash.exe 1358 1493 1359 1494 # Databases containing fuzzer test cases 1360 1495 # 1361 1496 FUZZDATA = \ 1362 1497 $(TOP)\test\fuzzdata1.db \ 1363 1498 $(TOP)\test\fuzzdata2.db \ 1364 1499 $(TOP)\test\fuzzdata3.db \ 1365 - $(TOP)\test\fuzzdata4.db 1500 + $(TOP)\test\fuzzdata4.db \ 1501 + $(TOP)\test\fuzzdata5.db 1366 1502 # <</mark>> 1367 1503 1368 1504 # Additional compiler options for the shell. These are only effective 1369 1505 # when the shell is not being dynamically linked. 1370 1506 # 1371 1507 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0 1372 1508 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS ................................................................................ 1373 1509 !ENDIF 1374 1510 1375 1511 # <<mark>> 1376 1512 # Extra compiler options for various test tools. 1377 1513 # 1378 1514 MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5 1379 1515 FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 1380 -FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 1516 +FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 1517 +FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c 1518 +OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c 1519 +DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION 1520 +KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ 1521 +DBSELFTEST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 1522 +ST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 1381 1523 1382 1524 # Standard options to testfixture. 1383 1525 # 1384 1526 TESTOPTS = --verbose=file --output=test-out.txt 1385 1527 1386 1528 # Extra targets for the "all" target that require Tcl. 1387 1529 # ................................................................................ 1416 1558 $(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) 1417 1559 $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) 1418 1560 1419 1561 # <<block2>> 1420 1562 sqlite3.def: libsqlite3.lib 1421 1563 echo EXPORTS > sqlite3.def 1422 1564 dumpbin /all libsqlite3.lib \ 1423 - | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_[^@]*)(?:@\d+)?$$" \1 \ 1565 + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset)?_[^@]*)(?:@\d+)?$$" \1 \ 1424 1566 | sort >> sqlite3.def 1425 1567 # <</block2>> 1426 1568 1427 1569 $(SQLITE3EXE): $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) 1428 1570 $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \ 1429 1571 /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) 1430 1572 1431 1573 # <<mark>> 1432 1574 sqldiff.exe: $(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H) 1433 1575 $(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 1434 1576 1577 +dbhash.exe: $(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H) 1578 + $(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 1579 + 1580 +scrub.exe: $(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H) 1581 + $(LTLINK) $(NO_WARN) $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 1582 + 1435 1583 srcck1.exe: $(TOP)\tool\srcck1.c 1436 1584 $(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c 1437 1585 1438 1586 sourcetest: srcck1.exe sqlite3.c 1439 1587 srcck1.exe sqlite3.c 1440 1588 1441 1589 fuzzershell.exe: $(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H) 1442 1590 $(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 1443 1591 1444 -fuzzcheck.exe: $(TOP)\test\fuzzcheck.c $(SQLITE3C) $(SQLITE3H) 1445 - $(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(TOP)\test\fuzzcheck.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 1592 +dbfuzz.exe: $(TOP)\test\dbfuzz.c $(SQLITE3C) $(SQLITE3H) 1593 + $(LTLINK) $(NO_WARN) $(DBFUZZ_COMPILE_OPTS) $(TOP)\test\dbfuzz.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 1594 + 1595 +fuzzcheck.exe: $(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H) 1596 + $(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 1597 + 1598 +ossshell.exe: $(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H) 1599 + $(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 1446 1600 1447 1601 mptester.exe: $(TOP)\mptest\mptest.c $(SQLITE3C) $(SQLITE3H) 1448 1602 $(LTLINK) $(NO_WARN) $(MPTESTER_COMPILE_OPTS) $(TOP)\mptest\mptest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 1449 1603 1450 1604 MPTEST1 = mptester mptest.db $(TOP)\mptest\crash01.test --repeat 20 1451 1605 MPTEST2 = mptester mptest.db $(TOP)\mptest\multiwrite01.test --repeat 20 1452 1606 ................................................................................ 1463 1617 1464 1618 # This target creates a directory named "tsrc" and fills it with 1465 1619 # copies of all of the C source code and header files needed to 1466 1620 # build on the target system. Some of the C source code and header 1467 1621 # files are automatically generated. This target takes care of 1468 1622 # all that automatic generation. 1469 1623 # 1470 -.target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c 1624 +.target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c $(SQLITE_TCL_DEP) 1471 1625 -rmdir /Q/S tsrc 2>NUL 1472 1626 -mkdir tsrc 1473 1627 for %i in ($(SRC00)) do copy /Y %i tsrc 1474 1628 for %i in ($(SRC01)) do copy /Y %i tsrc 1475 1629 for %i in ($(SRC02)) do copy /Y %i tsrc 1476 1630 for %i in ($(SRC03)) do copy /Y %i tsrc 1477 1631 for %i in ($(SRC04)) do copy /Y %i tsrc ................................................................................ 1478 1632 for %i in ($(SRC05)) do copy /Y %i tsrc 1479 1633 for %i in ($(SRC06)) do copy /Y %i tsrc 1480 1634 for %i in ($(SRC07)) do copy /Y %i tsrc 1481 1635 for %i in ($(SRC08)) do copy /Y %i tsrc 1482 1636 for %i in ($(SRC09)) do copy /Y %i tsrc 1483 1637 for %i in ($(SRC10)) do copy /Y %i tsrc 1484 1638 for %i in ($(SRC11)) do copy /Y %i tsrc 1639 + for %i in ($(SRC12)) do copy /Y %i tsrc 1485 1640 copy /Y fts5.c tsrc 1486 1641 copy /Y fts5.h tsrc 1487 1642 del /Q tsrc\sqlite.h.in tsrc\parse.y 2>NUL 1488 1643 $(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl $(OPTS) < tsrc\vdbe.c > vdbe.new 1489 1644 move vdbe.new tsrc\vdbe.c 1490 1645 echo > .target_source 1491 1646 1492 -sqlite3.c: .target_source sqlite3ext.h $(TOP)\tool\mksqlite3c.tcl 1493 - $(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl $(MKSQLITE3C_ARGS) 1647 +sqlite3.c: .target_source sqlite3ext.h $(MKSQLITE3C_TOOL) 1648 + $(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS) 1494 1649 copy tsrc\shell.c . 1650 + copy $(TOP)\ext\session\sqlite3session.h . 1495 1651 1496 1652 sqlite3-all.c: sqlite3.c $(TOP)\tool\split-sqlite3c.tcl 1497 1653 $(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl 1498 1654 # <</mark>> 1499 1655 1500 1656 # Rule to build the amalgamation 1501 1657 # ................................................................................ 1758 1914 1759 1915 wherecode.lo: $(TOP)\src\wherecode.c $(HDR) 1760 1916 $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\wherecode.c 1761 1917 1762 1918 whereexpr.lo: $(TOP)\src\whereexpr.c $(HDR) 1763 1919 $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\whereexpr.c 1764 1920 1765 -tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) 1921 +tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP) 1766 1922 $(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c 1767 1923 1768 -tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) 1924 +tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP) 1769 1925 $(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c 1770 1926 1771 1927 tclsqlite3.exe: tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) 1772 1928 $(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS) 1773 1929 1774 1930 # Rules to build opcodes.c and opcodes.h 1775 1931 # ................................................................................ 1787 1943 del /Q parse.y parse.h parse.h.temp 2>NUL 1788 1944 copy $(TOP)\src\parse.y . 1789 1945 .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) parse.y 1790 1946 move parse.h parse.h.temp 1791 1947 $(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h 1792 1948 1793 1949 $(SQLITE3H): $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION 1794 - $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H) 1950 + $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H) $(MKSQLITE3H_ARGS) 1795 1951 1796 1952 sqlite3ext.h: .target_source 1797 - copy tsrc\sqlite3ext.h . 1953 +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 1954 + type tsrc\sqlite3ext.h | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*\)" "(SQLITE_CALLBACK *)" \ 1955 + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*" "(SQLITE_APICALL *" > sqlite3ext.h 1956 + copy /Y sqlite3ext.h tsrc\sqlite3ext.h 1957 +!ELSE 1958 + copy /Y tsrc\sqlite3ext.h sqlite3ext.h 1959 +!ENDIF 1798 1960 1799 1961 mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c 1800 1962 $(BCC) $(NO_WARN) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) \ 1801 1963 $(TOP)\tool\mkkeywordhash.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS) 1802 1964 1803 1965 keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe 1804 1966 .\mkkeywordhash.exe > keywordhash.h ................................................................................ 1866 2028 1867 2029 fts3_write.lo: $(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR) 1868 2030 $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_write.c 1869 2031 1870 2032 rtree.lo: $(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR) 1871 2033 $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\rtree\rtree.c 1872 2034 2035 +sqlite3session.lo: $(TOP)\ext\session\sqlite3session.c $(HDR) $(EXTHDR) 2036 + $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\session\sqlite3session.c 2037 + 1873 2038 # FTS5 things 1874 2039 # 1875 2040 FTS5_SRC = \ 1876 2041 $(TOP)\ext\fts5\fts5.h \ 1877 2042 $(TOP)\ext\fts5\fts5Int.h \ 1878 2043 $(TOP)\ext\fts5\fts5_aux.c \ 1879 2044 $(TOP)\ext\fts5\fts5_buffer.c \ ................................................................................ 1920 2085 # hidden when the library is built via the amalgamation). 1921 2086 # 1922 2087 TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 1923 2088 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" 1924 2089 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN) 1925 2090 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 1926 2091 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024 2092 +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS) 1927 2093 1928 2094 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) 1929 2095 TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C) 1930 2096 !IF $(USE_AMALGAMATION)==0 1931 2097 TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0) 1932 2098 !ELSE 1933 2099 TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1) 1934 2100 !ENDIF 1935 2101 1936 -testfixture.exe: $(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) 2102 +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 2103 +sqlite_tclDecls.h: 2104 + echo #ifndef SQLITE_TCLAPI > $(SQLITETCLDECLSH) 2105 + echo # define SQLITE_TCLAPI >> $(SQLITETCLDECLSH) 2106 + echo #endif >> $(SQLITETCLDECLSH) 2107 + type "$(TCLINCDIR)\tclDecls.h" \ 2108 + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "^(EXTERN(?: CONST\d+?)?\s+?[^\(]*?\s+?)Tcl_" "\1 SQLITE_TCLAPI Tcl_" \ 2109 + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "^(EXTERN\s+?(?:void|VOID)\s+?)TclFreeObj" "\1 SQLITE_TCLAPI TclFreeObj" \ 2110 + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*tcl_" "(SQLITE_TCLAPI *tcl_" \ 2111 + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*tclFreeObj" "(SQLITE_TCLAPI *tclFreeObj" \ 2112 + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*" "(SQLITE_TCLAPI *" >> $(SQLITETCLDECLSH) 2113 + 2114 +sqlite_tcl.h: 2115 + type "$(TCLINCDIR)\tcl.h" | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact tclDecls.h sqlite_tclDecls.h \ 2116 + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "typedef (.*?)\(Tcl_" "typedef \1 (SQLITE_TCLAPI Tcl_" \ 2117 + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "void (*freeProc)" "void (SQLITE_TCLAPI *freeProc)" \ 2118 + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*findProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *findProc)" \ 2119 + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*createProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *createProc)" >> $(SQLITETCLH) 2120 +!ENDIF 2121 + 2122 +testfixture.exe: $(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP) 1937 2123 $(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ 1938 2124 -DBUILD_sqlite -I$(TCLINCDIR) \ 1939 2125 $(TESTFIXTURE_SRC) \ 1940 2126 /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) 1941 2127 1942 2128 extensiontest: testfixture.exe testloadext.dll 1943 2129 @set PATH=$(LIBTCLPATH);$(PATH) ................................................................................ 1978 2164 @set PATH=$(LIBTCLPATH);$(PATH) 1979 2165 .\testfixture.exe $(TOP)\test\veryquick.test $(TESTOPTS) 1980 2166 1981 2167 smoketest: $(TESTPROGS) 1982 2168 @set PATH=$(LIBTCLPATH);$(PATH) 1983 2169 .\testfixture.exe $(TOP)\test\main.test $(TESTOPTS) 1984 2170 1985 -sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl 2171 +sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(SQLITE_TCL_DEP) 1986 2172 echo #define TCLSH 2 > $@ 1987 2173 echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@ 1988 2174 copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@ 1989 2175 echo static const char *tclsh_main_loop(void){ >> $@ 1990 2176 echo static const char *zMainloop = >> $@ 1991 2177 $(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\spaceanal.tcl >> $@ 1992 2178 echo ; return zMainloop; } >> $@ 1993 2179 1994 2180 sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) 1995 2181 $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \ 1996 2182 /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) 1997 2183 2184 +dbdump.exe: $(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H) 2185 + $(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \ 2186 + /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) 2187 + 1998 2188 testloadext.lo: $(TOP)\src\test_loadext.c 1999 2189 $(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c 2000 2190 2001 2191 testloadext.dll: testloadext.lo 2002 2192 $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo 2003 2193 2004 2194 showdb.exe: $(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H) 2005 - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ 2195 + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ 2006 2196 $(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 2007 2197 2008 2198 showstat4.exe: $(TOP)\tool\showstat4.c $(SQLITE3C) $(SQLITE3H) 2009 - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ 2199 + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ 2010 2200 $(TOP)\tool\showstat4.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 2011 2201 2012 2202 showjournal.exe: $(TOP)\tool\showjournal.c $(SQLITE3C) $(SQLITE3H) 2013 - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ 2203 + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ 2014 2204 $(TOP)\tool\showjournal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 2015 2205 2016 2206 showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H) 2017 - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ 2207 + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ 2018 2208 $(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 2019 2209 2210 +changeset.exe: $(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H) 2211 + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ 2212 + -DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \ 2213 + $(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 2214 + 2020 2215 fts3view.exe: $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H) 2021 - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ 2216 + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ 2022 2217 $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 2023 2218 2024 2219 rollback-test.exe: $(TOP)\tool\rollback-test.c $(SQLITE3C) $(SQLITE3H) 2025 - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ 2220 + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ 2026 2221 $(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 2027 2222 2028 2223 LogEst.exe: $(TOP)\tool\logest.c $(SQLITE3H) 2029 - $(LTLINK) $(NO_WARN) -Fe$@ $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS) 2224 + $(LTLINK) $(NO_WARN) $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS) 2030 2225 2031 2226 wordcount.exe: $(TOP)\test\wordcount.c $(SQLITE3C) $(SQLITE3H) 2032 - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ 2227 + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ 2033 2228 $(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 2034 2229 2035 2230 speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) $(SQLITE3H) 2036 - $(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ 2231 + $(LTLINK) $(NO_WARN) $(ST_COMPILE_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION \ 2037 2232 $(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 2038 2233 2234 +kvtest.exe: $(TOP)\test\kvtest.c $(SQLITE3C) $(SQLITE3H) 2235 + $(LTLINK) $(NO_WARN) $(KV_COMPILE_OPTS) \ 2236 + $(TOP)\test\kvtest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 2237 + 2238 +dbselftest.exe: $(TOP)\test\dbselftest.c $(SQLITE3C) $(SQLITE3H) 2239 + $(LTLINK) $(NO_WARN) $(DBSELFTEST_COMPILE_OPTS) $(TOP)\test\dbselftest.c $(SQLITE3C) 2240 + 2039 2241 rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H) 2040 - $(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ \ 2242 + $(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU \ 2041 2243 $(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 2042 2244 2043 2245 moreclean: clean 2044 2246 del /Q $(SQLITE3C) $(SQLITE3H) 2>NUL 2045 2247 # <</mark>> 2046 2248 2047 2249 clean: ................................................................................ 2052 2254 del /Q sqlite3.c sqlite3.h 2>NUL 2053 2255 del /Q opcodes.c opcodes.h 2>NUL 2054 2256 del /Q lemon.* lempar.c parse.* 2>NUL 2055 2257 del /Q mkkeywordhash.* keywordhash.h 2>NUL 2056 2258 del /Q notasharedlib.* 2>NUL 2057 2259 -rmdir /Q/S .deps 2>NUL 2058 2260 -rmdir /Q/S .libs 2>NUL 2059 - -rmdir /Q/S quota2a 2>NUL 2060 - -rmdir /Q/S quota2b 2>NUL 2061 - -rmdir /Q/S quota2c 2>NUL 2062 2261 -rmdir /Q/S tsrc 2>NUL 2063 2262 del /Q .target_source 2>NUL 2064 - del /Q tclsqlite3.exe 2>NUL 2263 + del /Q tclsqlite3.exe $(SQLITETCLH) $(SQLITETCLDECLSH) 2>NUL 2065 2264 del /Q testloadext.dll 2>NUL 2066 2265 del /Q testfixture.exe test.db 2>NUL 2067 - del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe 2>NUL 2266 + del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe dbdump.exe 2>NUL 2267 + del /Q changeset.exe 2>NUL 2068 2268 del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL 2069 2269 del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL 2070 2270 del /Q sqlite3.c sqlite3-*.c 2>NUL 2071 2271 del /Q sqlite3rc.h 2>NUL 2072 - del /Q shell.c sqlite3ext.h 2>NUL 2272 + del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL 2073 2273 del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL 2074 2274 del /Q sqlite-*-output.vsix 2>NUL 2075 - del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL 2275 + del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL 2076 2276 del /Q fts5.* fts5parse.* 2>NUL 2077 2277 # <</mark>>
Changes to README.md.
4 4 engine. Some test scripts are also include. However, many other test scripts 5 5 and most of the documentation are managed separately. 6 6 7 7 If you are reading this on a Git mirror someplace, you are doing it wrong. 8 8 The [official repository](https://www.sqlite.org/src/) is better. Go there 9 9 now. 10 10 11 +## Obtaining The Code 12 + 13 +SQLite sources are managed using the 14 +[Fossil](https://www.fossil-scm.org/), a distributed version control system 15 +that was specifically designed to support SQLite development. 16 +If you do not want to use Fossil, you can download tarballs or ZIP 17 +archives as follows: 18 + 19 + * Lastest trunk check-in: 20 + <https://www.sqlite.org/src/tarball/sqlite.tar.gz> or 21 + <https://www.sqlite.org/src/zip/sqlite.zip>. 22 + 23 + * Latest release: 24 + <https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release> or 25 + <https://www.sqlite.org/src/zip/sqlite.zip?r=release>. 26 + 27 + * For other check-ins, substitute an appropriate branch name or 28 + tag or hash prefix for "release" in the URLs of the previous 29 + bullet. Or browse the [timeline](https://www.sqlite.org/src/timeline) 30 + to locate the check-in desired, click on its information page link, 31 + then click on the "Tarball" or "ZIP Archive" links on the information 32 + page. 33 + 34 +If you do want to use Fossil to check out the source tree, 35 +first install Fossil version 2.0 or later. 36 +(Source tarballs and precompiled binaries available 37 +[here](https://www.fossil-scm.org/fossil/uv/download.html).) 38 +Then run commands like this: 39 + 40 + mkdir ~/sqlite 41 + cd ~/sqlite 42 + fossil clone https://www.sqlite.org/src sqlite.fossil 43 + fossil open sqlite.fossil 44 + 45 +After setting up a repository using the steps above, you can always 46 +update to the lastest version using: 47 + 48 + fossil update trunk ;# latest trunk check-in 49 + fossil update release ;# latest official release 50 + 51 +Or type "fossil ui" to get a web-based user interface. 52 + 11 53 ## Compiling 12 54 13 55 First create a directory in which to place 14 56 the build products. It is recommended, but not required, that the 15 57 build directory be separate from the source directory. Cd into the 16 58 build directory and then from the build directory run the configure 17 59 script found at the root of the source tree. Then run "make". 18 60 19 61 For example: 20 62 21 - tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" 22 - mkdir bld ;# Build will occur in a sibling directory 23 - cd bld ;# Change to the build directory 24 - ../sqlite/configure ;# Run the configure script 25 - make ;# Run the makefile. 26 - make sqlite3.c ;# Build the "amalgamation" source file 27 - make test ;# Run some tests (requires Tcl) 63 + tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" 64 + mkdir bld ;# Build will occur in a sibling directory 65 + cd bld ;# Change to the build directory 66 + ../sqlite/configure ;# Run the configure script 67 + make ;# Run the makefile. 68 + make sqlite3.c ;# Build the "amalgamation" source file 69 + make test ;# Run some tests (requires Tcl) 28 70 29 71 See the makefile for additional targets. 30 72 31 73 The configure script uses autoconf 2.61 and libtool. If the configure 32 74 script does not work out for you, there is a generic makefile named 33 75 "Makefile.linux-gcc" in the top directory of the source tree that you 34 76 can copy and edit to suit your needs. Comments on the generic makefile ................................................................................ 39 81 On Windows, all applicable build products can be compiled with MSVC. 40 82 First open the command prompt window associated with the desired compiler 41 83 version (e.g. "Developer Command Prompt for VS2013"). Next, use NMAKE 42 84 with the provided "Makefile.msc" to build one of the supported targets. 43 85 44 86 For example: 45 87 46 - mkdir bld 47 - cd bld 48 - nmake /f Makefile.msc TOP=..\sqlite 49 - nmake /f Makefile.msc sqlite3.c TOP=..\sqlite 50 - nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite 51 - nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite 52 - nmake /f Makefile.msc test TOP=..\sqlite 88 + mkdir bld 89 + cd bld 90 + nmake /f Makefile.msc TOP=..\sqlite 91 + nmake /f Makefile.msc sqlite3.c TOP=..\sqlite 92 + nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite 93 + nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite 94 + nmake /f Makefile.msc test TOP=..\sqlite 53 95 54 96 There are several build options that can be set via the NMAKE command 55 97 line. For example, to build for WinRT, simply add "FOR_WINRT=1" argument 56 98 to the "sqlite3.dll" command line above. When debugging into the SQLite 57 99 code, adding the "DEBUG=1" argument to one of the above command lines is 58 100 recommended. 59 101
Changes to VERSION.
1 -3.12.0 1 +3.19.0
Changes to autoconf/Makefile.am.
1 1 2 -AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE 2 +AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE 3 3 4 4 lib_LTLIBRARIES = libsqlite3.la 5 5 libsqlite3_la_SOURCES = sqlite3.c 6 6 libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8 7 7 8 8 bin_PROGRAMS = sqlite3 9 9 sqlite3_SOURCES = shell.c sqlite3.h
Changes to autoconf/Makefile.msc.
19 19 20 20 21 21 # Set this non-0 to enable full warnings (-W4, etc) when compiling. 22 22 # 23 23 !IFNDEF USE_FULLWARN 24 24 USE_FULLWARN = 0 25 25 !ENDIF 26 + 27 +# Set this non-0 to enable treating warnings as errors (-WX, etc) when 28 +# compiling. 29 +# 30 +!IFNDEF USE_FATAL_WARN 31 +USE_FATAL_WARN = 0 32 +!ENDIF 33 + 34 +# Set this non-0 to enable full runtime error checks (-RTC1, etc). This 35 +# has no effect if (any) optimizations are enabled. 36 +# 37 +!IFNDEF USE_RUNTIME_CHECKS 38 +USE_RUNTIME_CHECKS = 0 39 +!ENDIF 40 + 41 +# Set this non-0 to create a SQLite amalgamation file that excludes the 42 +# various built-in extensions. 43 +# 44 +!IFNDEF MINIMAL_AMALGAMATION 45 +MINIMAL_AMALGAMATION = 0 46 +!ENDIF 26 47 27 48 # Set this non-0 to use "stdcall" calling convention for the core library 28 49 # and shell executable. 29 50 # 30 51 !IFNDEF USE_STDCALL 31 52 USE_STDCALL = 0 32 53 !ENDIF ................................................................................ 178 199 # Enable use of available compiler optimizations? Normally, this should be 179 200 # non-zero. Setting this to zero, thus disabling all compiler optimizations, 180 201 # can be useful for testing. 181 202 # 182 203 !IFNDEF OPTIMIZATIONS 183 204 OPTIMIZATIONS = 2 184 205 !ENDIF 206 + 207 +# Set this to non-0 to enable support for the session extension. 208 +# 209 +!IFNDEF SESSION 210 +SESSION = 0 211 +!ENDIF 185 212 186 213 # Set the source code file to be used by executables and libraries when 187 214 # they need the amalgamation. 188 215 # 189 216 !IFNDEF SQLITE3C 190 217 !IF $(SPLIT_AMALGAMATION)!=0 191 218 SQLITE3C = sqlite3-all.c ................................................................................ 237 264 !IFNDEF SQLITE3EXEPDB 238 265 !IF $(FOR_WIN10)!=0 239 266 SQLITE3EXEPDB = 240 267 !ELSE 241 268 SQLITE3EXEPDB = /pdb:sqlite3sh.pdb 242 269 !ENDIF 243 270 !ENDIF 271 + 244 272 245 273 # These are the "standard" SQLite compilation options used when compiling for 246 274 # the Windows platform. 247 275 # 248 276 !IFNDEF OPT_FEATURE_FLAGS 277 +!IF $(MINIMAL_AMALGAMATION)==0 249 278 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 250 279 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 280 +!ENDIF 251 281 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 252 282 !ENDIF 283 + 284 +# Should the session extension be enabled? If so, add compilation options 285 +# to enable it. 286 +# 287 +!IF $(SESSION)!=0 288 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1 289 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1 290 +!ENDIF 253 291 254 292 # These are the "extended" SQLite compilation options used when compiling for 255 293 # the Windows 10 platform. 256 294 # 257 295 !IFNDEF EXT_FEATURE_FLAGS 258 296 !IF $(FOR_WIN10)!=0 259 297 EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS4=1 ................................................................................ 418 456 # same unless your are cross-compiling.) 419 457 # 420 458 !IF $(USE_FULLWARN)!=0 421 459 TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS) 422 460 !ELSE 423 461 TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS) 424 462 !ENDIF 463 + 464 +# Check if warnings should be treated as errors when compiling. 465 +# 466 +!IF $(USE_FATAL_WARN)!=0 467 +TCC = $(TCC) -WX 468 +!ENDIF 425 469 426 470 TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -fp:precise 427 471 RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS) 428 472 429 473 # Check if we want to use the "stdcall" calling convention when compiling. 430 474 # This is not supported by the compilers for non-x86 platforms. It should 431 475 # also be noted here that building any target with these "stdcall" options 432 476 # will most likely fail if the Tcl library is also required. This is due 433 477 # to how the Tcl library functions are declared and exported (i.e. without 434 478 # an explicit calling convention, which results in "cdecl"). 435 479 # 436 480 !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 437 481 !IF "$(PLATFORM)"=="x86" 438 -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall 439 -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall 482 +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall 483 +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall 440 484 !ELSE 441 485 !IFNDEF PLATFORM 442 -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall 443 -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall 486 +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall 487 +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall 444 488 !ELSE 445 489 CORE_CCONV_OPTS = 446 490 SHELL_CCONV_OPTS = 447 491 !ENDIF 448 492 !ENDIF 449 493 !ELSE 450 494 CORE_CCONV_OPTS = ................................................................................ 597 641 TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1 598 642 RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 599 643 !ENDIF 600 644 601 645 !IF $(DEBUG)>2 602 646 TCC = $(TCC) -DSQLITE_DEBUG=1 603 647 RCC = $(RCC) -DSQLITE_DEBUG=1 648 +!IF $(DYNAMIC_SHELL)==0 649 +TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE 650 +RCC = $(RCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE 651 +!ENDIF 604 652 !ENDIF 605 653 606 654 !IF $(DEBUG)>4 || $(OSTRACE)!=0 607 655 TCC = $(TCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1 608 656 RCC = $(RCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1 609 657 !ENDIF 610 658 ................................................................................ 722 770 723 771 # If optimizations are enabled or disabled (either implicitly or 724 772 # explicitly), add the necessary flags. 725 773 # 726 774 !IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0 727 775 TCC = $(TCC) -Od 728 776 BCC = $(BCC) -Od 777 +!IF $(USE_RUNTIME_CHECKS)!=0 778 +TCC = $(TCC) -RTC1 779 +BCC = $(BCC) -RTC1 780 +!ENDIF 729 781 !ELSEIF $(OPTIMIZATIONS)>=3 730 782 TCC = $(TCC) -Ox 731 783 BCC = $(BCC) -Ox 732 784 !ELSEIF $(OPTIMIZATIONS)==2 733 785 TCC = $(TCC) -O2 734 786 BCC = $(BCC) -O2 735 787 !ELSEIF $(OPTIMIZATIONS)==1 ................................................................................ 898 950 899 951 Replace.exe: 900 952 $(CSC) /target:exe $(TOP)\Replace.cs 901 953 902 954 sqlite3.def: Replace.exe $(LIBOBJ) 903 955 echo EXPORTS > sqlite3.def 904 956 dumpbin /all $(LIBOBJ) \ 905 - | .\Replace.exe "^\s+/EXPORT:_?(sqlite3_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \ 957 + | .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \ 906 958 | sort >> sqlite3.def 907 959 908 960 $(SQLITE3EXE): $(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) 909 961 $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \ 910 962 /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) 911 963 912 964
Changes to autoconf/configure.ac.
26 26 # Check for library functions that SQLite can optionally use. 27 27 AC_CHECK_FUNCS([fdatasync usleep fullfsync localtime_r gmtime_r]) 28 28 AC_FUNC_STRERROR_R 29 29 30 30 AC_CONFIG_FILES([Makefile sqlite3.pc]) 31 31 AC_SUBST(BUILD_CFLAGS) 32 32 33 -#----------------------------------------------------------------------- 33 +#------------------------------------------------------------------------- 34 +# Two options to enable readline compatible libraries: 35 +# 34 36 # --enable-editline 35 37 # --enable-readline 36 38 # 37 -AC_ARG_ENABLE(editline, [AS_HELP_STRING( 38 - [--enable-editline], 39 - [use BSD libedit])], 40 - [], [enable_editline=yes]) 41 -AC_ARG_ENABLE(readline, [AS_HELP_STRING( 42 - [--enable-readline], 43 - [use readline])], 44 - [], [enable_readline=no]) 45 -if test x"$enable_editline" != xno ; then 46 - sLIBS=$LIBS 47 - LIBS="" 48 - AC_SEARCH_LIBS([readline],[edit],[enable_readline=no],[enable_editline=no]) 49 - READLINE_LIBS=$LIBS 50 - if test x"$LIBS" != "x"; then 51 - AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline) 52 - else 53 - unset ac_cv_search_readline 54 - fi 55 - LIBS=$sLIBS 56 -fi 57 -if test x"$enable_readline" != xno ; then 58 - sLIBS=$LIBS 59 - LIBS="" 60 - AC_SEARCH_LIBS(tgetent, curses ncurses ncursesw, [], []) 61 - AC_SEARCH_LIBS(readline, readline, [], [enable_readline=no]) 62 - AC_CHECK_FUNCS(readline, [], []) 63 - READLINE_LIBS=$LIBS 64 - LIBS=$sLIBS 65 -fi 39 +# Both are enabled by default. If, after command line processing both are 40 +# still enabled, the script searches for editline first and automatically 41 +# disables readline if it is found. So, to use readline explicitly, the 42 +# user must pass "--disable-editline". To disable command line editing 43 +# support altogether, "--disable-editline --disable-readline". 44 +# 45 +# When searching for either library, check for headers before libraries 46 +# as some distros supply packages that contain libraries but not header 47 +# files, which come as a separate development package. 48 +# 49 +AC_ARG_ENABLE(editline, [AS_HELP_STRING([--enable-editline],[use BSD libedit])]) 50 +AC_ARG_ENABLE(readline, [AS_HELP_STRING([--enable-readline],[use readline])]) 51 + 52 +AS_IF([ test x"$enable_editline" != xno ],[ 53 + AC_CHECK_HEADERS([editline/readline.h],[ 54 + sLIBS=$LIBS 55 + LIBS="" 56 + AC_SEARCH_LIBS([readline],[edit],[ 57 + AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline) 58 + READLINE_LIBS="$LIBS -ltinfo" 59 + enable_readline=no 60 + ],[],[-ltinfo]) 61 + AS_UNSET(ac_cv_search_readline) 62 + LIBS=$sLIBS 63 + ]) 64 +]) 65 + 66 +AS_IF([ test x"$enable_readline" != xno ],[ 67 + AC_CHECK_HEADERS([readline/readline.h],[ 68 + sLIBS=$LIBS 69 + LIBS="" 70 + AC_SEARCH_LIBS(tgetent, termcap curses ncurses ncursesw, [], []) 71 + AC_SEARCH_LIBS(readline,[readline edit], [ 72 + AC_DEFINE([HAVE_READLINE],1,Define to use readline or wrapper) 73 + READLINE_LIBS=$LIBS 74 + ]) 75 + LIBS=$sLIBS 76 + ]) 77 +]) 78 + 66 79 AC_SUBST(READLINE_LIBS) 67 80 #----------------------------------------------------------------------- 68 81 69 82 #----------------------------------------------------------------------- 70 83 # --enable-threadsafe 71 84 # 72 85 AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING( ................................................................................ 99 112 100 113 #----------------------------------------------------------------------- 101 114 # --enable-fts5 102 115 # 103 116 AC_ARG_ENABLE(fts5, [AS_HELP_STRING( 104 117 [--enable-fts5], [include fts5 support [default=no]])], 105 118 [], [enable_fts5=no]) 106 -if test x"$enable_fts5" == "xyes"; then 119 +if test x"$enable_fts5" = "xyes"; then 107 120 AC_SEARCH_LIBS(log, m) 108 121 FTS5_FLAGS=-DSQLITE_ENABLE_FTS5 109 122 fi 110 123 AC_SUBST(FTS5_FLAGS) 111 124 #----------------------------------------------------------------------- 112 125 113 126 #----------------------------------------------------------------------- 114 127 # --enable-json1 115 128 # 116 129 AC_ARG_ENABLE(json1, [AS_HELP_STRING( 117 130 [--enable-json1], [include json1 support [default=no]])], 118 131 [], [enable_json1=no]) 119 -if test x"$enable_json1" == "xyes"; then 132 +if test x"$enable_json1" = "xyes"; then 120 133 JSON1_FLAGS=-DSQLITE_ENABLE_JSON1 121 134 fi 122 135 AC_SUBST(JSON1_FLAGS) 123 136 #----------------------------------------------------------------------- 137 + 138 +#----------------------------------------------------------------------- 139 +# --enable-session 140 +# 141 +AC_ARG_ENABLE(session, [AS_HELP_STRING( 142 + [--enable-session], [enable the session extension [default=no]])], 143 + [], [enable_session=no]) 144 +if test x"$enable_session" = "xyes"; then 145 + SESSION_FLAGS="-DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK" 146 +fi 147 +AC_SUBST(SESSION_FLAGS) 148 +#----------------------------------------------------------------------- 124 149 125 150 #----------------------------------------------------------------------- 126 151 # --enable-static-shell 127 152 # 128 153 AC_ARG_ENABLE(static-shell, [AS_HELP_STRING( 129 154 [--enable-static-shell], 130 155 [statically link libsqlite3 into shell tool [default=yes]])], 131 156 [], [enable_static_shell=yes]) 132 -if test x"$enable_static_shell" == "xyes"; then 157 +if test x"$enable_static_shell" = "xyes"; then 133 158 EXTRA_SHELL_OBJ=sqlite3-sqlite3.$OBJEXT 134 159 else 135 160 EXTRA_SHELL_OBJ=libsqlite3.la 136 161 fi 137 162 AC_SUBST(EXTRA_SHELL_OBJ) 138 163 #----------------------------------------------------------------------- 139 164
Changes to autoconf/tea/configure.ac.
74 74 TEA_ADD_SOURCES([tclsqlite3.c]) 75 75 TEA_ADD_HEADERS([]) 76 76 TEA_ADD_INCLUDES([-I\"`\${CYGPATH} \${srcdir}/generic`\"]) 77 77 TEA_ADD_LIBS([]) 78 78 TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS3=1]) 79 79 TEA_ADD_CFLAGS([-DSQLITE_3_SUFFIX_ONLY=1]) 80 80 TEA_ADD_CFLAGS([-DSQLITE_ENABLE_RTREE=1]) 81 -TEA_ADD_CFLAGS([-DSQLITE_OMIT_DEPRECATED=1]) 82 81 TEA_ADD_STUB_SOURCES([]) 83 82 TEA_ADD_TCL_SOURCES([]) 84 83 85 84 #-------------------------------------------------------------------- 86 85 # The --with-system-sqlite causes the TCL bindings to SQLite to use 87 86 # the system shared library for SQLite rather than statically linking 88 87 # against its own private copy. This is dangerous and leads to
Changes to configure.
1 1 #! /bin/sh 2 2 # Guess values for system-dependent variables and create Makefiles. 3 -# Generated by GNU Autoconf 2.69 for sqlite 3.12.0. 3 +# Generated by GNU Autoconf 2.69 for sqlite 3.19.0. 4 4 # 5 5 # 6 6 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. 7 7 # 8 8 # 9 9 # This configure script is free software; the Free Software Foundation 10 10 # gives unlimited permission to copy, distribute and modify it. ................................................................................ 722 722 subdirs= 723 723 MFLAGS= 724 724 MAKEFLAGS= 725 725 726 726 # Identity of this package. 727 727 PACKAGE_NAME='sqlite' 728 728 PACKAGE_TARNAME='sqlite' 729 -PACKAGE_VERSION='3.12.0' 730 -PACKAGE_STRING='sqlite 3.12.0' 729 +PACKAGE_VERSION='3.19.0' 730 +PACKAGE_STRING='sqlite 3.19.0' 731 731 PACKAGE_BUGREPORT='' 732 732 PACKAGE_URL='' 733 733 734 734 # Factoring default headers for most tests. 735 735 ac_includes_default="\ 736 736 #include <stdio.h> 737 737 #ifdef HAVE_SYS_TYPES_H ................................................................................ 899 899 enable_editline 900 900 enable_readline 901 901 with_readline_lib 902 902 with_readline_inc 903 903 enable_debug 904 904 enable_amalgamation 905 905 enable_load_extension 906 +enable_memsys5 907 +enable_memsys3 906 908 enable_fts3 907 909 enable_fts4 908 910 enable_fts5 909 911 enable_json1 910 912 enable_rtree 913 +enable_session 911 914 enable_gcov 912 915 ' 913 916 ac_precious_vars='build_alias 914 917 host_alias 915 918 target_alias 916 919 CC 917 920 CFLAGS ................................................................................ 1456 1459 # 1457 1460 # Report the --help message. 1458 1461 # 1459 1462 if test "$ac_init_help" = "long"; then 1460 1463 # Omit some internal or obsolete options to make the list less imposing. 1461 1464 # This message is too long to be a string in the A/UX 3.1 sh. 1462 1465 cat <<_ACEOF 1463 -\`configure' configures sqlite 3.12.0 to adapt to many kinds of systems. 1466 +\`configure' configures sqlite 3.19.0 to adapt to many kinds of systems. 1464 1467 1465 1468 Usage: $0 [OPTION]... [VAR=VALUE]... 1466 1469 1467 1470 To assign environment variables (e.g., CC, CFLAGS...), specify them as 1468 1471 VAR=VALUE. See below for descriptions of some of the useful variables. 1469 1472 1470 1473 Defaults for the options are specified in brackets. ................................................................................ 1521 1524 --build=BUILD configure for building on BUILD [guessed] 1522 1525 --host=HOST cross-compile to build programs to run on HOST [BUILD] 1523 1526 _ACEOF 1524 1527 fi 1525 1528 1526 1529 if test -n "$ac_init_help"; then 1527 1530 case $ac_init_help in 1528 - short | recursive ) echo "Configuration of sqlite 3.12.0:";; 1531 + short | recursive ) echo "Configuration of sqlite 3.19.0:";; 1529 1532 esac 1530 1533 cat <<\_ACEOF 1531 1534 1532 1535 Optional Features: 1533 1536 --disable-option-checking ignore unrecognized --enable/--with options 1534 1537 --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) 1535 1538 --enable-FEATURE[=ARG] include FEATURE [ARG=yes] ................................................................................ 1547 1550 --enable-editline enable BSD editline support 1548 1551 --disable-readline disable readline support 1549 1552 --enable-debug enable debugging & verbose explain 1550 1553 --disable-amalgamation Disable the amalgamation and instead build all files 1551 1554 separately 1552 1555 --disable-load-extension 1553 1556 Disable loading of external extensions 1557 + --enable-memsys5 Enable MEMSYS5 1558 + --enable-memsys3 Enable MEMSYS3 1554 1559 --enable-fts3 Enable the FTS3 extension 1555 1560 --enable-fts4 Enable the FTS4 extension 1556 1561 --enable-fts5 Enable the FTS5 extension 1557 1562 --enable-json1 Enable the JSON1 extension 1558 1563 --enable-rtree Enable the RTREE extension 1564 + --enable-session Enable the SESSION extension 1559 1565 --enable-gcov Enable coverage testing using gcov 1560 1566 1561 1567 Optional Packages: 1562 1568 --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] 1563 1569 --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) 1564 1570 --with-pic try to use only PIC/non-PIC objects [default=use 1565 1571 both] ................................................................................ 1642 1648 cd "$ac_pwd" || { ac_status=$?; break; } 1643 1649 done 1644 1650 fi 1645 1651 1646 1652 test -n "$ac_init_help" && exit $ac_status 1647 1653 if $ac_init_version; then 1648 1654 cat <<\_ACEOF 1649 -sqlite configure 3.12.0 1655 +sqlite configure 3.19.0 1650 1656 generated by GNU Autoconf 2.69 1651 1657 1652 1658 Copyright (C) 2012 Free Software Foundation, Inc. 1653 1659 This configure script is free software; the Free Software Foundation 1654 1660 gives unlimited permission to copy, distribute and modify it. 1655 1661 _ACEOF 1656 1662 exit ................................................................................ 2061 2067 eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno 2062 2068 2063 2069 } # ac_fn_c_check_header_mongrel 2064 2070 cat >config.log <<_ACEOF 2065 2071 This file contains any messages produced by compilers while 2066 2072 running configure, to aid debugging if configure makes a mistake. 2067 2073 2068 -It was created by sqlite $as_me 3.12.0, which was 2074 +It was created by sqlite $as_me 3.19.0, which was 2069 2075 generated by GNU Autoconf 2.69. Invocation command line was 2070 2076 2071 2077 $ $0 $@ 2072 2078 2073 2079 _ACEOF 2074 2080 exec 5>>config.log 2075 2081 { ................................................................................ 3919 3925 { $as_echo "$as_me:${as_lineno-$LINENO}: checking the name lister ($NM) interface" >&5 3920 3926 $as_echo_n "checking the name lister ($NM) interface... " >&6; } 3921 3927 if ${lt_cv_nm_interface+:} false; then : 3922 3928 $as_echo_n "(cached) " >&6 3923 3929 else 3924 3930 lt_cv_nm_interface="BSD nm" 3925 3931 echo "int some_variable = 0;" > conftest.$ac_ext 3926 - (eval echo "\"\$as_me:3926: $ac_compile\"" >&5) 3932 + (eval echo "\"\$as_me:3932: $ac_compile\"" >&5) 3927 3933 (eval "$ac_compile" 2>conftest.err) 3928 3934 cat conftest.err >&5 3929 - (eval echo "\"\$as_me:3929: $NM \\\"conftest.$ac_objext\\\"\"" >&5) 3935 + (eval echo "\"\$as_me:3935: $NM \\\"conftest.$ac_objext\\\"\"" >&5) 3930 3936 (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) 3931 3937 cat conftest.err >&5 3932 - (eval echo "\"\$as_me:3932: output\"" >&5) 3938 + (eval echo "\"\$as_me:3938: output\"" >&5) 3933 3939 cat conftest.out >&5 3934 3940 if $GREP 'External.*some_variable' conftest.out > /dev/null; then 3935 3941 lt_cv_nm_interface="MS dumpbin" 3936 3942 fi 3937 3943 rm -f conftest* 3938 3944 fi 3939 3945 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_nm_interface" >&5 ................................................................................ 5131 5137 ;; 5132 5138 esac 5133 5139 fi 5134 5140 rm -rf conftest* 5135 5141 ;; 5136 5142 *-*-irix6*) 5137 5143 # Find out which ABI we are using. 5138 - echo '#line 5138 "configure"' > conftest.$ac_ext 5144 + echo '#line 5144 "configure"' > conftest.$ac_ext 5139 5145 if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 5140 5146 (eval $ac_compile) 2>&5 5141 5147 ac_status=$? 5142 5148 $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 5143 5149 test $ac_status = 0; }; then 5144 5150 if test "$lt_cv_prog_gnu_ld" = yes; then 5145 5151 case `/usr/bin/file conftest.$ac_objext` in ................................................................................ 6656 6662 # Note that $ac_compile itself does not contain backslashes and begins 6657 6663 # with a dollar sign (not a hyphen), so the echo should work correctly. 6658 6664 # The option is referenced via a variable to avoid confusing sed. 6659 6665 lt_compile=`echo "$ac_compile" | $SED \ 6660 6666 -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ 6661 6667 -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ 6662 6668 -e 's:$: $lt_compiler_flag:'` 6663 - (eval echo "\"\$as_me:6663: $lt_compile\"" >&5) 6669 + (eval echo "\"\$as_me:6669: $lt_compile\"" >&5) 6664 6670 (eval "$lt_compile" 2>conftest.err) 6665 6671 ac_status=$? 6666 6672 cat conftest.err >&5 6667 - echo "$as_me:6667: \$? = $ac_status" >&5 6673 + echo "$as_me:6673: \$? = $ac_status" >&5 6668 6674 if (exit $ac_status) && test -s "$ac_outfile"; then 6669 6675 # The compiler can only warn and ignore the option if not recognized 6670 6676 # So say no if there are warnings other than the usual output. 6671 6677 $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' >conftest.exp 6672 6678 $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 6673 6679 if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then 6674 6680 lt_cv_prog_compiler_rtti_exceptions=yes ................................................................................ 6995 7001 # Note that $ac_compile itself does not contain backslashes and begins 6996 7002 # with a dollar sign (not a hyphen), so the echo should work correctly. 6997 7003 # The option is referenced via a variable to avoid confusing sed. 6998 7004 lt_compile=`echo "$ac_compile" | $SED \ 6999 7005 -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ 7000 7006 -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ 7001 7007 -e 's:$: $lt_compiler_flag:'` 7002 - (eval echo "\"\$as_me:7002: $lt_compile\"" >&5) 7008 + (eval echo "\"\$as_me:7008: $lt_compile\"" >&5) 7003 7009 (eval "$lt_compile" 2>conftest.err) 7004 7010 ac_status=$? 7005 7011 cat conftest.err >&5 7006 - echo "$as_me:7006: \$? = $ac_status" >&5 7012 + echo "$as_me:7012: \$? = $ac_status" >&5 7007 7013 if (exit $ac_status) && test -s "$ac_outfile"; then 7008 7014 # The compiler can only warn and ignore the option if not recognized 7009 7015 # So say no if there are warnings other than the usual output. 7010 7016 $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' >conftest.exp 7011 7017 $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 7012 7018 if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then 7013 7019 lt_cv_prog_compiler_pic_works=yes ................................................................................ 7100 7106 # (2) before a word containing "conftest.", or (3) at the end. 7101 7107 # Note that $ac_compile itself does not contain backslashes and begins 7102 7108 # with a dollar sign (not a hyphen), so the echo should work correctly. 7103 7109 lt_compile=`echo "$ac_compile" | $SED \ 7104 7110 -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ 7105 7111 -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ 7106 7112 -e 's:$: $lt_compiler_flag:'` 7107 - (eval echo "\"\$as_me:7107: $lt_compile\"" >&5) 7113 + (eval echo "\"\$as_me:7113: $lt_compile\"" >&5) 7108 7114 (eval "$lt_compile" 2>out/conftest.err) 7109 7115 ac_status=$? 7110 7116 cat out/conftest.err >&5 7111 - echo "$as_me:7111: \$? = $ac_status" >&5 7117 + echo "$as_me:7117: \$? = $ac_status" >&5 7112 7118 if (exit $ac_status) && test -s out/conftest2.$ac_objext 7113 7119 then 7114 7120 # The compiler can only warn and ignore the option if not recognized 7115 7121 # So say no if there are warnings 7116 7122 $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' > out/conftest.exp 7117 7123 $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2 7118 7124 if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then ................................................................................ 7155 7161 # (2) before a word containing "conftest.", or (3) at the end. 7156 7162 # Note that $ac_compile itself does not contain backslashes and begins 7157 7163 # with a dollar sign (not a hyphen), so the echo should work correctly. 7158 7164 lt_compile=`echo "$ac_compile" | $SED \ 7159 7165 -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ 7160 7166 -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ 7161 7167 -e 's:$: $lt_compiler_flag:'` 7162 - (eval echo "\"\$as_me:7162: $lt_compile\"" >&5) 7168 + (eval echo "\"\$as_me:7168: $lt_compile\"" >&5) 7163 7169 (eval "$lt_compile" 2>out/conftest.err) 7164 7170 ac_status=$? 7165 7171 cat out/conftest.err >&5 7166 - echo "$as_me:7166: \$? = $ac_status" >&5 7172 + echo "$as_me:7172: \$? = $ac_status" >&5 7167 7173 if (exit $ac_status) && test -s out/conftest2.$ac_objext 7168 7174 then 7169 7175 # The compiler can only warn and ignore the option if not recognized 7170 7176 # So say no if there are warnings 7171 7177 $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' > out/conftest.exp 7172 7178 $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2 7173 7179 if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then ................................................................................ 9535 9541 else 9536 9542 if test "$cross_compiling" = yes; then : 9537 9543 lt_cv_dlopen_self=cross 9538 9544 else 9539 9545 lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 9540 9546 lt_status=$lt_dlunknown 9541 9547 cat > conftest.$ac_ext <<_LT_EOF 9542 -#line 9542 "configure" 9548 +#line 9548 "configure" 9543 9549 #include "confdefs.h" 9544 9550 9545 9551 #if HAVE_DLFCN_H 9546 9552 #include <dlfcn.h> 9547 9553 #endif 9548 9554 9549 9555 #include <stdio.h> ................................................................................ 9631 9637 else 9632 9638 if test "$cross_compiling" = yes; then : 9633 9639 lt_cv_dlopen_self_static=cross 9634 9640 else 9635 9641 lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 9636 9642 lt_status=$lt_dlunknown 9637 9643 cat > conftest.$ac_ext <<_LT_EOF 9638 -#line 9638 "configure" 9644 +#line 9644 "configure" 9639 9645 #include "confdefs.h" 9640 9646 9641 9647 #if HAVE_DLFCN_H 9642 9648 #include <dlfcn.h> 9643 9649 #endif 9644 9650 9645 9651 #include <stdio.h> ................................................................................ 10750 10756 if test -f "$i/tclConfig.sh" ; then 10751 10757 ac_cv_c_tclconfig="$i" 10752 10758 break 10753 10759 fi 10754 10760 done 10755 10761 fi 10756 10762 fi 10763 + 10764 + # Recent versions of Xcode on Macs hid the tclConfig.sh file 10765 + # in a strange place. 10766 + if test x"${ac_cv_c_tclconfig}" = x ; then 10767 + if test x"$cross_compiling" = xno; then 10768 + for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib 10769 + do 10770 + if test -f "$i/tclConfig.sh" ; then 10771 + ac_cv_c_tclconfig="$i" 10772 + break 10773 + fi 10774 + done 10775 + fi 10776 + fi 10757 10777 10758 10778 # then check for a private Tcl installation 10759 10779 if test x"${ac_cv_c_tclconfig}" = x ; then 10760 10780 for i in \ 10761 10781 ../tcl \ 10762 10782 `ls -dr ../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \ 10763 10783 `ls -dr ../tcl[8-9].[0-9] 2>/dev/null` \ ................................................................................ 11228 11248 if test "${enable_debug+set}" = set; then : 11229 11249 enableval=$enable_debug; use_debug=$enableval 11230 11250 else 11231 11251 use_debug=no 11232 11252 fi 11233 11253 11234 11254 if test "${use_debug}" = "yes" ; then 11235 - TARGET_DEBUG="-DSQLITE_DEBUG=1" 11255 + TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE" 11236 11256 else 11237 11257 TARGET_DEBUG="-DNDEBUG" 11238 11258 fi 11239 11259 11240 11260 11241 11261 ######### 11242 11262 # See whether we should use the amalgamation to build ................................................................................ 11318 11338 test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" 11319 11339 11320 11340 fi 11321 11341 11322 11342 else 11323 11343 OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1" 11324 11344 fi 11345 + 11346 +########## 11347 +# Do we want to support memsys3 and/or memsys5 11348 +# 11349 +# Check whether --enable-memsys5 was given. 11350 +if test "${enable_memsys5+set}" = set; then : 11351 + enableval=$enable_memsys5; enable_memsys5=yes 11352 +else 11353 + enable_memsys5=no 11354 +fi 11355 + 11356 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS5" >&5 11357 +$as_echo_n "checking whether to support MEMSYS5... " >&6; } 11358 +if test "${enable_memsys5}" = "yes"; then 11359 + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS5" 11360 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 11361 +$as_echo "yes" >&6; } 11362 +else 11363 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 11364 +$as_echo "no" >&6; } 11365 +fi 11366 +# Check whether --enable-memsys3 was given. 11367 +if test "${enable_memsys3+set}" = set; then : 11368 + enableval=$enable_memsys3; enable_memsys3=yes 11369 +else 11370 + enable_memsys3=no 11371 +fi 11372 + 11373 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS3" >&5 11374 +$as_echo_n "checking whether to support MEMSYS3... " >&6; } 11375 +if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then 11376 + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS3" 11377 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 11378 +$as_echo "yes" >&6; } 11379 +else 11380 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 11381 +$as_echo "no" >&6; } 11382 +fi 11325 11383 11326 11384 ######### 11327 11385 # See whether we should enable Full Text Search extensions 11328 11386 # Check whether --enable-fts3 was given. 11329 11387 if test "${enable_fts3+set}" = set; then : 11330 11388 enableval=$enable_fts3; enable_fts3=yes 11331 11389 else ................................................................................ 11489 11547 else 11490 11548 enable_rtree=no 11491 11549 fi 11492 11550 11493 11551 if test "${enable_rtree}" = "yes" ; then 11494 11552 OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE" 11495 11553 fi 11554 + 11555 +######### 11556 +# See whether we should enable the SESSION extension 11557 +# Check whether --enable-session was given. 11558 +if test "${enable_session+set}" = set; then : 11559 + enableval=$enable_session; enable_session=yes 11560 +else 11561 + enable_session=no 11562 +fi 11563 + 11564 +if test "${enable_session}" = "yes" ; then 11565 + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_SESSION" 11566 + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_PREUPDATE_HOOK" 11567 +fi 11496 11568 11497 11569 ######### 11498 11570 # attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter 11499 11571 for option in $CFLAGS $CPPFLAGS 11500 11572 do 11501 11573 case $option in 11502 11574 -DSQLITE_OMIT*) OPT_FEATURE_FLAGS="$OPT_FEATURE_FLAGS $option";; ................................................................................ 12075 12147 test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1 12076 12148 12077 12149 cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 12078 12150 # Save the log message, to keep $0 and so on meaningful, and to 12079 12151 # report actual input values of CONFIG_FILES etc. instead of their 12080 12152 # values after options handling. 12081 12153 ac_log=" 12082 -This file was extended by sqlite $as_me 3.12.0, which was 12154 +This file was extended by sqlite $as_me 3.19.0, which was 12083 12155 generated by GNU Autoconf 2.69. Invocation command line was 12084 12156 12085 12157 CONFIG_FILES = $CONFIG_FILES 12086 12158 CONFIG_HEADERS = $CONFIG_HEADERS 12087 12159 CONFIG_LINKS = $CONFIG_LINKS 12088 12160 CONFIG_COMMANDS = $CONFIG_COMMANDS 12089 12161 $ $0 $@ ................................................................................ 12141 12213 12142 12214 Report bugs to the package provider." 12143 12215 12144 12216 _ACEOF 12145 12217 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 12146 12218 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" 12147 12219 ac_cs_version="\\ 12148 -sqlite config.status 3.12.0 12220 +sqlite config.status 3.19.0 12149 12221 configured by $0, generated by GNU Autoconf 2.69, 12150 12222 with options \\"\$ac_cs_config\\" 12151 12223 12152 12224 Copyright (C) 2012 Free Software Foundation, Inc. 12153 12225 This config.status script is free software; the Free Software Foundation 12154 12226 gives unlimited permission to copy, distribute and modify it." 12155 12227
Changes to configure.ac.
329 329 if test -f "$i/tclConfig.sh" ; then 330 330 ac_cv_c_tclconfig="$i" 331 331 break 332 332 fi 333 333 done 334 334 fi 335 335 fi 336 + 337 + # Recent versions of Xcode on Macs hid the tclConfig.sh file 338 + # in a strange place. 339 + if test x"${ac_cv_c_tclconfig}" = x ; then 340 + if test x"$cross_compiling" = xno; then 341 + for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib 342 + do 343 + if test -f "$i/tclConfig.sh" ; then 344 + ac_cv_c_tclconfig="$i" 345 + break 346 + fi 347 + done 348 + fi 349 + fi 336 350 337 351 # then check for a private Tcl installation 338 352 if test x"${ac_cv_c_tclconfig}" = x ; then 339 353 for i in \ 340 354 ../tcl \ 341 355 `ls -dr ../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ 342 356 `ls -dr ../tcl[[8-9]].[[0-9]] 2>/dev/null` \ ................................................................................ 542 556 AC_SEARCH_LIBS(fdatasync, [rt]) 543 557 544 558 ######### 545 559 # check for debug enabled 546 560 AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]), 547 561 [use_debug=$enableval],[use_debug=no]) 548 562 if test "${use_debug}" = "yes" ; then 549 - TARGET_DEBUG="-DSQLITE_DEBUG=1" 563 + TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE" 550 564 else 551 565 TARGET_DEBUG="-DNDEBUG" 552 566 fi 553 567 AC_SUBST(TARGET_DEBUG) 554 568 555 569 ######### 556 570 # See whether we should use the amalgamation to build ................................................................................ 569 583 [use_loadextension=$enableval],[use_loadextension=yes]) 570 584 if test "${use_loadextension}" = "yes" ; then 571 585 OPT_FEATURE_FLAGS="" 572 586 AC_SEARCH_LIBS(dlopen, dl) 573 587 else 574 588 OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1" 575 589 fi 590 + 591 +########## 592 +# Do we want to support memsys3 and/or memsys5 593 +# 594 +AC_ARG_ENABLE(memsys5, 595 + AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]), 596 + [enable_memsys5=yes],[enable_memsys5=no]) 597 +AC_MSG_CHECKING([whether to support MEMSYS5]) 598 +if test "${enable_memsys5}" = "yes"; then 599 + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS5" 600 + AC_MSG_RESULT([yes]) 601 +else 602 + AC_MSG_RESULT([no]) 603 +fi 604 +AC_ARG_ENABLE(memsys3, 605 + AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]), 606 + [enable_memsys3=yes],[enable_memsys3=no]) 607 +AC_MSG_CHECKING([whether to support MEMSYS3]) 608 +if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then 609 + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS3" 610 + AC_MSG_RESULT([yes]) 611 +else 612 + AC_MSG_RESULT([no]) 613 +fi 576 614 577 615 ######### 578 616 # See whether we should enable Full Text Search extensions 579 617 AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3], 580 618 [Enable the FTS3 extension]), 581 619 [enable_fts3=yes],[enable_fts3=no]) 582 620 if test "${enable_fts3}" = "yes" ; then ................................................................................ 610 648 # See whether we should enable RTREE 611 649 AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree], 612 650 [Enable the RTREE extension]), 613 651 [enable_rtree=yes],[enable_rtree=no]) 614 652 if test "${enable_rtree}" = "yes" ; then 615 653 OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE" 616 654 fi 655 + 656 +######### 657 +# See whether we should enable the SESSION extension 658 +AC_ARG_ENABLE(session, AC_HELP_STRING([--enable-session], 659 + [Enable the SESSION extension]), 660 + [enable_session=yes],[enable_session=no]) 661 +if test "${enable_session}" = "yes" ; then 662 + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_SESSION" 663 + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_PREUPDATE_HOOK" 664 +fi 617 665 618 666 ######### 619 667 # attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter 620 668 for option in $CFLAGS $CPPFLAGS 621 669 do 622 670 case $option in 623 671 -DSQLITE_OMIT*) OPT_FEATURE_FLAGS="$OPT_FEATURE_FLAGS $option";;
Changes to doc/lemon.html.
1 1 <html> 2 2 <head> 3 3 <title>The Lemon Parser Generator</title> 4 4 </head> 5 5 <body bgcolor=white> 6 6 <h1 align=center>The Lemon Parser Generator</h1> 7 7 8 -<p>Lemon is an LALR(1) parser generator for C or C++. 9 -It does the same job as ``bison'' and ``yacc''. 10 -But lemon is not another bison or yacc clone. It 8 +<p>Lemon is an LALR(1) parser generator for C. 9 +It does the same job as "bison" and "yacc". 10 +But lemon is not a bison or yacc clone. Lemon 11 11 uses a different grammar syntax which is designed to 12 -reduce the number of coding errors. Lemon also uses a more 13 -sophisticated parsing engine that is faster than yacc and 14 -bison and which is both reentrant and thread-safe. 15 -Furthermore, Lemon implements features that can be used 12 +reduce the number of coding errors. Lemon also uses a 13 +parsing engine that is faster than yacc and 14 +bison and which is both reentrant and threadsafe. 15 +(Update: Since the previous sentence was written, bison 16 +has also been updated so that it too can generate a 17 +reentrant and threadsafe parser.) 18 +Lemon also implements features that can be used 16 19 to eliminate resource leaks, making is suitable for use 17 20 in long-running programs such as graphical user interfaces 18 21 or embedded controllers.</p> 19 22 20 23 <p>This document is an introduction to the Lemon 21 24 parser generator.</p> 22 25 ................................................................................ 40 43 <ul> 41 44 <li>C code to implement the parser. 42 45 <li>A header file defining an integer ID for each terminal symbol. 43 46 <li>An information file that describes the states of the generated parser 44 47 automaton. 45 48 </ul> 46 49 By default, all three of these output files are generated. 47 -The header file is suppressed if the ``-m'' command-line option is 48 -used and the report file is omitted when ``-q'' is selected.</p> 50 +The header file is suppressed if the "-m" command-line option is 51 +used and the report file is omitted when "-q" is selected.</p> 49 52 50 -<p>The grammar specification file uses a ``.y'' suffix, by convention. 53 +<p>The grammar specification file uses a ".y" suffix, by convention. 51 54 In the examples used in this document, we'll assume the name of the 52 -grammar file is ``gram.y''. A typical use of Lemon would be the 55 +grammar file is "gram.y". A typical use of Lemon would be the 53 56 following command: 54 57 <pre> 55 58 lemon gram.y 56 59 </pre> 57 -This command will generate three output files named ``gram.c'', 58 -``gram.h'' and ``gram.out''. 60 +This command will generate three output files named "gram.c", 61 +"gram.h" and "gram.out". 59 62 The first is C code to implement the parser. The second 60 63 is the header file that defines numerical values for all 61 64 terminal symbols, and the last is the report that explains 62 65 the states used by the parser automaton.</p> 63 66 64 67 <h3>Command Line Options</h3> 65 68 ................................................................................ 67 70 You can obtain a list of the available command-line options together 68 71 with a brief explanation of what each does by typing 69 72 <pre> 70 73 lemon -? 71 74 </pre> 72 75 As of this writing, the following command-line options are supported: 73 76 <ul> 74 -<li><tt>-b</tt> 75 -<li><tt>-c</tt> 76 -<li><tt>-g</tt> 77 -<li><tt>-m</tt> 78 -<li><tt>-q</tt> 79 -<li><tt>-s</tt> 80 -<li><tt>-x</tt> 81 -</ul> 82 -The ``-b'' option reduces the amount of text in the report file by 83 -printing only the basis of each parser state, rather than the full 84 -configuration. 85 -The ``-c'' option suppresses action table compression. Using -c 86 -will make the parser a little larger and slower but it will detect 87 -syntax errors sooner. 88 -The ``-g'' option causes no output files to be generated at all. 89 -Instead, the input grammar file is printed on standard output but 90 -with all comments, actions and other extraneous text deleted. This 91 -is a useful way to get a quick summary of a grammar. 92 -The ``-m'' option causes the output C source file to be compatible 93 -with the ``makeheaders'' program. 94 -Makeheaders is a program that automatically generates header files 95 -from C source code. When the ``-m'' option is used, the header 96 -file is not output since the makeheaders program will take care 97 -of generated all header files automatically. 98 -The ``-q'' option suppresses the report file. 99 -Using ``-s'' causes a brief summary of parser statistics to be 100 -printed. Like this: 101 -<pre> 102 - Parser statistics: 74 terminals, 70 nonterminals, 179 rules 103 - 340 states, 2026 parser table entries, 0 conflicts 104 -</pre> 105 -Finally, the ``-x'' option causes Lemon to print its version number 106 -and then stops without attempting to read the grammar or generate a parser.</p> 77 +<li><b>-b</b> 78 +Show only the basis for each parser state in the report file. 79 +<li><b>-c</b> 80 +Do not compress the generated action tables. 81 +<li><b>-D<i>name</i></b> 82 +Define C preprocessor macro <i>name</i>. This macro is useable by 83 +"%ifdef" lines in the grammar file. 84 +<li><b>-g</b> 85 +Do not generate a parser. Instead write the input grammar to standard 86 +output with all comments, actions, and other extraneous text removed. 87 +<li><b>-l</b> 88 +Omit "#line" directives in the generated parser C code. 89 +<li><b>-m</b> 90 +Cause the output C source code to be compatible with the "makeheaders" 91 +program. 92 +<li><b>-p</b> 93 +Display all conflicts that are resolved by 94 +<a href='#precrules'>precedence rules</a>. 95 +<li><b>-q</b> 96 +Suppress generation of the report file. 97 +<li><b>-r</b> 98 +Do not sort or renumber the parser states as part of optimization. 99 +<li><b>-s</b> 100 +Show parser statistics before existing. 101 +<li><b>-T<i>file</i></b> 102 +Use <i>file</i> as the template for the generated C-code parser implementation. 103 +<li><b>-x</b> 104 +Print the Lemon version number. 105 +</ul> 107 106 108 107 <h3>The Parser Interface</h3> 109 108 110 109 <p>Lemon doesn't generate a complete, working program. It only generates 111 110 a few subroutines that implement a parser. This section describes 112 111 the interface to those subroutines. It is up to the programmer to 113 112 call these subroutines in an appropriate way in order to produce a ................................................................................ 117 116 must first create the parser. 118 117 A new parser is created as follows: 119 118 <pre> 120 119 void *pParser = ParseAlloc( malloc ); 121 120 </pre> 122 121 The ParseAlloc() routine allocates and initializes a new parser and 123 122 returns a pointer to it. 124 -The actual data structure used to represent a parser is opaque -- 123 +The actual data structure used to represent a parser is opaque — 125 124 its internal structure is not visible or usable by the calling routine. 126 125 For this reason, the ParseAlloc() routine returns a pointer to void 127 126 rather than a pointer to some particular structure. 128 127 The sole argument to the ParseAlloc() routine is a pointer to the 129 -subroutine used to allocate memory. Typically this means ``malloc()''.</p> 128 +subroutine used to allocate memory. Typically this means malloc().</p> 130 129 131 130 <p>After a program is finished using a parser, it can reclaim all 132 131 memory allocated by that parser by calling 133 132 <pre> 134 133 ParseFree(pParser, free); 135 134 </pre> 136 135 The first argument is the same pointer returned by ParseAlloc(). The ................................................................................ 147 146 The first argument to the Parse() routine is the pointer returned by 148 147 ParseAlloc(). 149 148 The second argument is a small positive integer that tells the parse the 150 149 type of the next token in the data stream. 151 150 There is one token type for each terminal symbol in the grammar. 152 151 The gram.h file generated by Lemon contains #define statements that 153 152 map symbolic terminal symbol names into appropriate integer values. 154 -(A value of 0 for the second argument is a special flag to the 155 -parser to indicate that the end of input has been reached.) 153 +A value of 0 for the second argument is a special flag to the 154 +parser to indicate that the end of input has been reached. 156 155 The third argument is the value of the given token. By default, 157 156 the type of the third argument is integer, but the grammar will 158 157 usually redefine this type to be some kind of structure. 159 158 Typically the second argument will be a broad category of tokens 160 -such as ``identifier'' or ``number'' and the third argument will 159 +such as "identifier" or "number" and the third argument will 161 160 be the name of the identifier or the value of the number.</p> 162 161 163 162 <p>The Parse() function may have either three or four arguments, 164 163 depending on the grammar. If the grammar specification file requests 165 164 it (via the <a href='#extraarg'><tt>extra_argument</tt> directive</a>), 166 165 the Parse() function will have a fourth parameter that can be 167 166 of any type chosen by the programmer. The parser doesn't do anything ................................................................................ 189 188 15 ParseFree(pParser, free ); 190 189 16 TokenizerFree(pTokenizer); 191 190 17 return sState.treeRoot; 192 191 18 } 193 192 </pre> 194 193 This example shows a user-written routine that parses a file of 195 194 text and returns a pointer to the parse tree. 196 -(We've omitted all error-handling from this example to keep it 195 +(All error-handling code is omitted from this example to keep it 197 196 simple.) 198 197 We assume the existence of some kind of tokenizer which is created 199 198 using TokenizerCreate() on line 8 and deleted by TokenizerFree() 200 199 on line 16. The GetNextToken() function on line 11 retrieves the 201 200 next token from the input file and puts its type in the 202 201 integer variable hTokenId. The sToken variable is assumed to be 203 202 some kind of structure that contains details about each token, ................................................................................ 283 282 declaration can occur at any point in the file. 284 283 Lemon ignores whitespace (except where it is needed to separate 285 284 tokens) and it honors the same commenting conventions as C and C++.</p> 286 285 287 286 <h3>Terminals and Nonterminals</h3> 288 287 289 288 <p>A terminal symbol (token) is any string of alphanumeric 290 -and underscore characters 289 +and/or underscore characters 291 290 that begins with an upper case letter. 292 291 A terminal can contain lowercase letters after the first character, 293 292 but the usual convention is to make terminals all upper case. 294 293 A nonterminal, on the other hand, is any string of alphanumeric 295 294 and underscore characters than begins with a lower case letter. 296 295 Again, the usual convention is to make nonterminals use all lower 297 296 case letters.</p> ................................................................................ 310 309 must have alphanumeric names.</p> 311 310 312 311 <h3>Grammar Rules</h3> 313 312 314 313 <p>The main component of a Lemon grammar file is a sequence of grammar 315 314 rules. 316 315 Each grammar rule consists of a nonterminal symbol followed by 317 -the special symbol ``::='' and then a list of terminals and/or nonterminals. 316 +the special symbol "::=" and then a list of terminals and/or nonterminals. 318 317 The rule is terminated by a period. 319 318 The list of terminals and nonterminals on the right-hand side of the 320 319 rule can be empty. 321 320 Rules can occur in any order, except that the left-hand side of the 322 321 first rule is assumed to be the start symbol for the grammar (unless 323 322 specified otherwise using the <tt>%start</tt> directive described below.) 324 323 A typical sequence of grammar rules might look something like this: ................................................................................ 326 325 expr ::= expr PLUS expr. 327 326 expr ::= expr TIMES expr. 328 327 expr ::= LPAREN expr RPAREN. 329 328 expr ::= VALUE. 330 329 </pre> 331 330 </p> 332 331 333 -<p>There is one non-terminal in this example, ``expr'', and five 334 -terminal symbols or tokens: ``PLUS'', ``TIMES'', ``LPAREN'', 335 -``RPAREN'' and ``VALUE''.</p> 332 +<p>There is one non-terminal in this example, "expr", and five 333 +terminal symbols or tokens: "PLUS", "TIMES", "LPAREN", 334 +"RPAREN" and "VALUE".</p> 336 335 337 336 <p>Like yacc and bison, Lemon allows the grammar to specify a block 338 337 of C code that will be executed whenever a grammar rule is reduced 339 338 by the parser. 340 339 In Lemon, this action is specified by putting the C code (contained 341 340 within curly braces <tt>{...}</tt>) immediately after the 342 341 period that closes the rule. ................................................................................ 344 343 <pre> 345 344 expr ::= expr PLUS expr. { printf("Doing an addition...\n"); } 346 345 </pre> 347 346 </p> 348 347 349 348 <p>In order to be useful, grammar actions must normally be linked to 350 349 their associated grammar rules. 351 -In yacc and bison, this is accomplished by embedding a ``$$'' in the 350 +In yacc and bison, this is accomplished by embedding a "$$" in the 352 351 action to stand for the value of the left-hand side of the rule and 353 -symbols ``$1'', ``$2'', and so forth to stand for the value of 352 +symbols "$1", "$2", and so forth to stand for the value of 354 353 the terminal or nonterminal at position 1, 2 and so forth on the 355 354 right-hand side of the rule. 356 355 This idea is very powerful, but it is also very error-prone. The 357 356 single most common source of errors in a yacc or bison grammar is 358 357 to miscount the number of symbols on the right-hand side of a grammar 359 -rule and say ``$7'' when you really mean ``$8''.</p> 358 +rule and say "$7" when you really mean "$8".</p> 360 359 361 360 <p>Lemon avoids the need to count grammar symbols by assigning symbolic 362 361 names to each symbol in a grammar rule and then using those symbolic 363 362 names in the action. 364 363 In yacc or bison, one would write this: 365 364 <pre> 366 365 expr -> expr PLUS expr { $$ = $1 + $3; }; ................................................................................ 382 381 includes a linking symbol in parentheses but that linking symbol 383 382 is not actually used in the reduce action, then an error message 384 383 is generated. 385 384 For example, the rule 386 385 <pre> 387 386 expr(A) ::= expr(B) PLUS expr(C). { A = B; } 388 387 </pre> 389 -will generate an error because the linking symbol ``C'' is used 388 +will generate an error because the linking symbol "C" is used 390 389 in the grammar rule but not in the reduce action.</p> 391 390 392 391 <p>The Lemon notation for linking grammar rules to reduce actions 393 392 also facilitates the use of destructors for reclaiming memory 394 393 allocated by the values of terminals and nonterminals on the 395 394 right-hand side of a rule.</p> 396 395 396 +<a name='precrules'></a> 397 397 <h3>Precedence Rules</h3> 398 398 399 399 <p>Lemon resolves parsing ambiguities in exactly the same way as 400 400 yacc and bison. A shift-reduce conflict is resolved in favor 401 401 of the shift, and a reduce-reduce conflict is resolved by reducing 402 402 whichever rule comes first in the grammar file.</p> 403 403 404 404 <p>Just like in 405 405 yacc and bison, Lemon allows a measure of control 406 406 over the resolution of paring conflicts using precedence rules. 407 407 A precedence value can be assigned to any terminal symbol 408 -using the %left, %right or %nonassoc directives. Terminal symbols 408 +using the 409 +<a href='#pleft'>%left</a>, 410 +<a href='#pright'>%right</a> or 411 +<a href='#pnonassoc'>%nonassoc</a> directives. Terminal symbols 409 412 mentioned in earlier directives have a lower precedence that 410 413 terminal symbols mentioned in later directives. For example:</p> 411 414 412 415 <p><pre> 413 416 %left AND. 414 417 %left OR. 415 418 %nonassoc EQ NE GT GE LT LE. ................................................................................ 521 524 522 525 <p>Lemon supports the following special directives: 523 526 <ul> 524 527 <li><tt>%code</tt> 525 528 <li><tt>%default_destructor</tt> 526 529 <li><tt>%default_type</tt> 527 530 <li><tt>%destructor</tt> 531 +<li><tt>%endif</tt> 528 532 <li><tt>%extra_argument</tt> 533 +<li><tt>%fallback</tt> 534 +<li><tt>%ifdef</tt> 535 +<li><tt>%ifndef</tt> 529 536 <li><tt>%include</tt> 530 537 <li><tt>%left</tt> 531 538 <li><tt>%name</tt> 532 539 <li><tt>%nonassoc</tt> 533 540 <li><tt>%parse_accept</tt> 534 541 <li><tt>%parse_failure </tt> 535 542 <li><tt>%right</tt> 536 543 <li><tt>%stack_overflow</tt> 537 544 <li><tt>%stack_size</tt> 538 545 <li><tt>%start_symbol</tt> 539 546 <li><tt>%syntax_error</tt> 547 +<li><tt>%token_class</tt> 540 548 <li><tt>%token_destructor</tt> 541 549 <li><tt>%token_prefix</tt> 542 550 <li><tt>%token_type</tt> 543 551 <li><tt>%type</tt> 552 +<li><tt>%wildcard</tt> 544 553 </ul> 545 554 Each of these directives will be described separately in the 546 555 following sections:</p> 547 556 557 +<a name='pcode'></a> 548 558 <h4>The <tt>%code</tt> directive</h4> 549 559 550 -<p>The %code directive is used to specify addition C/C++ code that 560 +<p>The %code directive is used to specify addition C code that 551 561 is added to the end of the main output file. This is similar to 552 -the %include directive except that %include is inserted at the 553 -beginning of the main output file.</p> 562 +the <a href='#pinclude'>%include</a> directive except that %include 563 +is inserted at the beginning of the main output file.</p> 554 564 555 565 <p>%code is typically used to include some action routines or perhaps 556 -a tokenizer as part of the output file.</p> 566 +a tokenizer or even the "main()" function 567 +as part of the output file.</p> 557 568 569 +<a name='default_destructor'></a> 558 570 <h4>The <tt>%default_destructor</tt> directive</h4> 559 571 560 572 <p>The %default_destructor directive specifies a destructor to 561 573 use for non-terminals that do not have their own destructor 562 574 specified by a separate %destructor directive. See the documentation 563 -on the %destructor directive below for additional information.</p> 575 +on the <a name='#destructor'>%destructor</a> directive below for 576 +additional information.</p> 564 577 565 578 <p>In some grammers, many different non-terminal symbols have the 566 579 same datatype and hence the same destructor. This directive is 567 580 a convenience way to specify the same destructor for all those 568 581 non-terminals using a single statement.</p> 569 582 583 +<a name='default_type'></a> 570 584 <h4>The <tt>%default_type</tt> directive</h4> 571 585 572 586 <p>The %default_type directive specifies the datatype of non-terminal 573 587 symbols that do no have their own datatype defined using a separate 574 -%type directive. See the documentation on %type below for addition 575 -information.</p> 588 +<a href='#ptype'>%type</a> directive. 589 +</p> 576 590 591 +<a name='destructor'></a> 577 592 <h4>The <tt>%destructor</tt> directive</h4> 578 593 579 594 <p>The %destructor directive is used to specify a destructor for 580 595 a non-terminal symbol. 581 -(See also the %token_destructor directive which is used to 582 -specify a destructor for terminal symbols.)</p> 596 +(See also the <a href='#token_destructor'>%token_destructor</a> 597 +directive which is used to specify a destructor for terminal symbols.)</p> 583 598 584 599 <p>A non-terminal's destructor is called to dispose of the 585 600 non-terminal's value whenever the non-terminal is popped from 586 601 the stack. This includes all of the following circumstances: 587 602 <ul> 588 603 <li> When a rule reduces and the value of a non-terminal on 589 604 the right-hand side is not linked to C code. ................................................................................ 598 613 <pre> 599 614 %type nt {void*} 600 615 %destructor nt { free($$); } 601 616 nt(A) ::= ID NUM. { A = malloc( 100 ); } 602 617 </pre> 603 618 This example is a bit contrived but it serves to illustrate how 604 619 destructors work. The example shows a non-terminal named 605 -``nt'' that holds values of type ``void*''. When the rule for 606 -an ``nt'' reduces, it sets the value of the non-terminal to 620 +"nt" that holds values of type "void*". When the rule for 621 +an "nt" reduces, it sets the value of the non-terminal to 607 622 space obtained from malloc(). Later, when the nt non-terminal 608 623 is popped from the stack, the destructor will fire and call 609 624 free() on this malloced space, thus avoiding a memory leak. 610 -(Note that the symbol ``$$'' in the destructor code is replaced 625 +(Note that the symbol "$$" in the destructor code is replaced 611 626 by the value of the non-terminal.)</p> 612 627 613 628 <p>It is important to note that the value of a non-terminal is passed 614 629 to the destructor whenever the non-terminal is removed from the 615 630 stack, unless the non-terminal is used in a C-code action. If 616 631 the non-terminal is used by C-code, then it is assumed that the 617 -C-code will take care of destroying it if it should really 618 -be destroyed. More commonly, the value is used to build some 632 +C-code will take care of destroying it. 633 +More commonly, the value is used to build some 619 634 larger structure and we don't want to destroy it, which is why 620 635 the destructor is not called in this circumstance.</p> 621 636 622 -<p>By appropriate use of destructors, it is possible to 623 -build a parser using Lemon that can be used within a long-running 624 -program, such as a GUI, that will not leak memory or other resources. 637 +<p>Destructors help avoid memory leaks by automatically freeing 638 +allocated objects when they go out of scope. 625 639 To do the same using yacc or bison is much more difficult.</p> 626 640 627 641 <a name="extraarg"></a> 628 642 <h4>The <tt>%extra_argument</tt> directive</h4> 629 643 630 644 The %extra_argument directive instructs Lemon to add a 4th parameter 631 645 to the parameter list of the Parse() function it generates. Lemon ................................................................................ 634 648 and so forth. For example, if the grammar file contains:</p> 635 649 636 650 <p><pre> 637 651 %extra_argument { MyStruct *pAbc } 638 652 </pre></p> 639 653 640 654 <p>Then the Parse() function generated will have an 4th parameter 641 -of type ``MyStruct*'' and all action routines will have access to 642 -a variable named ``pAbc'' that is the value of the 4th parameter 655 +of type "MyStruct*" and all action routines will have access to 656 +a variable named "pAbc" that is the value of the 4th parameter 643 657 in the most recent call to Parse().</p> 644 658 659 +<a name='pfallback'></a> 660 +<h4>The <tt>%fallback</tt> directive</h4> 661 + 662 +<p>The %fallback directive specifies an alternative meaning for one 663 +or more tokens. The alternative meaning is tried if the original token 664 +would have generated a syntax error. 665 + 666 +<p>The %fallback directive was added to support robust parsing of SQL 667 +syntax in <a href="https://www.sqlite.org/">SQLite</a>. 668 +The SQL language contains a large assortment of keywords, each of which 669 +appears as a different token to the language parser. SQL contains so 670 +many keywords, that it can be difficult for programmers to keep up with 671 +them all. Programmers will, therefore, sometimes mistakenly use an 672 +obscure language keyword for an identifier. The %fallback directive 673 +provides a mechanism to tell the parser: "If you are unable to parse 674 +this keyword, try treating it as an identifier instead." 675 + 676 +<p>The syntax of %fallback is as follows: 677 + 678 +<blockquote> 679 +<tt>%fallback</tt> <i>ID</i> <i>TOKEN...</i> <b>.</b> 680 +</blockquote> 681 + 682 +<p>In words, the %fallback directive is followed by a list of token names 683 +terminated by a period. The first token name is the fallback token - the 684 +token to which all the other tokens fall back to. The second and subsequent 685 +arguments are tokens which fall back to the token identified by the first 686 +argument. 687 + 688 +<a name='pifdef'></a> 689 +<h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives.</h4> 690 + 691 +<p>The %ifdef, %ifndef, and %endif directives are similar to 692 +#ifdef, #ifndef, and #endif in the C-preprocessor, just not as general. 693 +Each of these directives must begin at the left margin. No whitespace 694 +is allowed between the "%" and the directive name. 695 + 696 +<p>Grammar text in between "%ifdef MACRO" and the next nested "%endif" is 697 +ignored unless the "-DMACRO" command-line option is used. Grammar text 698 +betwen "%ifndef MACRO" and the next nested "%endif" is included except when 699 +the "-DMACRO" command-line option is used. 700 + 701 +<p>Note that the argument to %ifdef and %ifndef must be a single 702 +preprocessor symbol name, not a general expression. There is no "%else" 703 +directive. 704 + 705 + 706 +<a name='pinclude'></a> 645 707 <h4>The <tt>%include</tt> directive</h4> 646 708 647 709 <p>The %include directive specifies C code that is included at the 648 710 top of the generated parser. You can include any text you want -- 649 711 the Lemon parser generator copies it blindly. If you have multiple 650 -%include directives in your grammar file the value of the last 651 -%include directive overwrites all the others.</p. 712 +%include directives in your grammar file, their values are concatenated 713 +so that all %include code ultimately appears near the top of the 714 +generated parser, in the same order as it appeared in the grammer.</p> 652 715 653 716 <p>The %include directive is very handy for getting some extra #include 654 717 preprocessor statements at the beginning of the generated parser. 655 718 For example:</p> 656 719 657 720 <p><pre> 658 721 %include {#include <unistd.h>} 659 722 </pre></p> 660 723 661 724 <p>This might be needed, for example, if some of the C actions in the 662 725 grammar call functions that are prototyed in unistd.h.</p> 663 726 727 +<a name='pleft'></a> 664 728 <h4>The <tt>%left</tt> directive</h4> 665 729 666 -The %left directive is used (along with the %right and 667 -%nonassoc directives) to declare precedences of terminal 668 -symbols. Every terminal symbol whose name appears after 669 -a %left directive but before the next period (``.'') is 730 +The %left directive is used (along with the <a href='#pright'>%right</a> and 731 +<a href='#pnonassoc'>%nonassoc</a> directives) to declare precedences of 732 +terminal symbols. Every terminal symbol whose name appears after 733 +a %left directive but before the next period (".") is 670 734 given the same left-associative precedence value. Subsequent 671 735 %left directives have higher precedence. For example:</p> 672 736 673 737 <p><pre> 674 738 %left AND. 675 739 %left OR. 676 740 %nonassoc EQ NE GT GE LT LE. ................................................................................ 683 747 directive.</p> 684 748 685 749 <p>LALR(1) grammars can get into a situation where they require 686 750 a large amount of stack space if you make heavy use or right-associative 687 751 operators. For this reason, it is recommended that you use %left 688 752 rather than %right whenever possible.</p> 689 753 754 +<a name='pname'></a> 690 755 <h4>The <tt>%name</tt> directive</h4> 691 756 692 757 <p>By default, the functions generated by Lemon all begin with the 693 -five-character string ``Parse''. You can change this string to something 758 +five-character string "Parse". You can change this string to something 694 759 different using the %name directive. For instance:</p> 695 760 696 761 <p><pre> 697 762 %name Abcde 698 763 </pre></p> 699 764 700 765 <p>Putting this directive in the grammar file will cause Lemon to generate ................................................................................ 705 770 <li> AbcdeTrace(), and 706 771 <li> Abcde(). 707 772 </ul> 708 773 The %name directive allows you to generator two or more different 709 774 parsers and link them all into the same executable. 710 775 </p> 711 776 777 +<a name='pnonassoc'></a> 712 778 <h4>The <tt>%nonassoc</tt> directive</h4> 713 779 714 780 <p>This directive is used to assign non-associative precedence to 715 -one or more terminal symbols. See the section on precedence rules 716 -or on the %left directive for additional information.</p> 781 +one or more terminal symbols. See the section on 782 +<a href='#precrules'>precedence rules</a> 783 +or on the <a href='#pleft'>%left</a> directive for additional information.</p> 717 784 785 +<a name='parse_accept'></a> 718 786 <h4>The <tt>%parse_accept</tt> directive</h4> 719 787 720 788 <p>The %parse_accept directive specifies a block of C code that is 721 -executed whenever the parser accepts its input string. To ``accept'' 789 +executed whenever the parser accepts its input string. To "accept" 722 790 an input string means that the parser was able to process all tokens 723 791 without error.</p> 724 792 725 793 <p>For example:</p> 726 794 727 795 <p><pre> 728 796 %parse_accept { 729 797 printf("parsing complete!\n"); 730 798 } 731 799 </pre></p> 732 800 733 - 801 +<a name='parse_failure'></a> 734 802 <h4>The <tt>%parse_failure</tt> directive</h4> 735 803 736 804 <p>The %parse_failure directive specifies a block of C code that 737 805 is executed whenever the parser fails complete. This code is not 738 806 executed until the parser has tried and failed to resolve an input 739 807 error using is usual error recovery strategy. The routine is 740 808 only invoked when parsing is unable to continue.</p> ................................................................................ 741 809 742 810 <p><pre> 743 811 %parse_failure { 744 812 fprintf(stderr,"Giving up. Parser is hopelessly lost...\n"); 745 813 } 746 814 </pre></p> 747 815 816 +<a name='pright'></a> 748 817 <h4>The <tt>%right</tt> directive</h4> 749 818 750 819 <p>This directive is used to assign right-associative precedence to 751 -one or more terminal symbols. See the section on precedence rules 752 -or on the %left directive for additional information.</p> 820 +one or more terminal symbols. See the section on 821 +<a href='#precrules'>precedence rules</a> 822 +or on the <a href='#pleft'>%left</a> directive for additional information.</p> 753 823 824 +<a name='stack_overflow'></a> 754 825 <h4>The <tt>%stack_overflow</tt> directive</h4> 755 826 756 827 <p>The %stack_overflow directive specifies a block of C code that 757 828 is executed if the parser's internal stack ever overflows. Typically 758 829 this just prints an error message. After a stack overflow, the parser 759 830 will be unable to continue and must be reset.</p> 760 831 ................................................................................ 775 846 </pre> 776 847 Not like this: 777 848 <pre> 778 849 list ::= element list. // right-recursion. Bad! 779 850 list ::= . 780 851 </pre> 781 852 853 +<a name='stack_size'></a> 782 854 <h4>The <tt>%stack_size</tt> directive</h4> 783 855 784 856 <p>If stack overflow is a problem and you can't resolve the trouble 785 857 by using left-recursion, then you might want to increase the size 786 858 of the parser's stack using this directive. Put an positive integer 787 859 after the %stack_size directive and Lemon will generate a parse 788 860 with a stack of the requested size. The default value is 100.</p> 789 861 790 862 <p><pre> 791 863 %stack_size 2000 792 864 </pre></p> 793 865 866 +<a name='start_symbol'></a> 794 867 <h4>The <tt>%start_symbol</tt> directive</h4> 795 868 796 869 <p>By default, the start-symbol for the grammar that Lemon generates 797 870 is the first non-terminal that appears in the grammar file. But you 798 871 can choose a different start-symbol using the %start_symbol directive.</p> 799 872 800 873 <p><pre> 801 874 %start_symbol prog 802 875 </pre></p> 803 876 877 +<a name='token_destructor'></a> 804 878 <h4>The <tt>%token_destructor</tt> directive</h4> 805 879 806 880 <p>The %destructor directive assigns a destructor to a non-terminal 807 881 symbol. (See the description of the %destructor directive above.) 808 882 This directive does the same thing for all terminal symbols.</p> 809 883 810 884 <p>Unlike non-terminal symbols which may each have a different data type 811 885 for their values, terminals all use the same data type (defined by 812 886 the %token_type directive) and so they use a common destructor. Other 813 887 than that, the token destructor works just like the non-terminal 814 888 destructors.</p> 815 889 890 +<a name='token_prefix'></a> 816 891 <h4>The <tt>%token_prefix</tt> directive</h4> 817 892 818 893 <p>Lemon generates #defines that assign small integer constants 819 894 to each terminal symbol in the grammar. If desired, Lemon will 820 895 add a prefix specified by this directive 821 896 to each of the #defines it generates. 822 897 So if the default output of Lemon looked like this: ................................................................................ 834 909 <pre> 835 910 #define TOKEN_AND 1 836 911 #define TOKEN_MINUS 2 837 912 #define TOKEN_OR 3 838 913 #define TOKEN_PLUS 4 839 914 </pre> 840 915 916 +<a name='token_type'></a><a name='ptype'></a> 841 917 <h4>The <tt>%token_type</tt> and <tt>%type</tt> directives</h4> 842 918 843 919 <p>These directives are used to specify the data types for values 844 920 on the parser's stack associated with terminal and non-terminal 845 921 symbols. The values of all terminal symbols must be of the same 846 922 type. This turns out to be the same data type as the 3rd parameter 847 923 to the Parse() function generated by Lemon. Typically, you will ................................................................................ 849 925 token structure. Like this:</p> 850 926 851 927 <p><pre> 852 928 %token_type {Token*} 853 929 </pre></p> 854 930 855 931 <p>If the data type of terminals is not specified, the default value 856 -is ``int''.</p> 932 +is "void*".</p> 857 933 858 934 <p>Non-terminal symbols can each have their own data types. Typically 859 935 the data type of a non-terminal is a pointer to the root of a parse-tree 860 936 structure that contains all information about that non-terminal. 861 937 For example:</p> 862 938 863 939 <p><pre> ................................................................................ 870 946 on what the corresponding non-terminal or terminal symbol is. But 871 947 the grammar designer should keep in mind that the size of the union 872 948 will be the size of its largest element. So if you have a single 873 949 non-terminal whose data type requires 1K of storage, then your 100 874 950 entry parser stack will require 100K of heap space. If you are willing 875 951 and able to pay that price, fine. You just need to know.</p> 876 952 953 +<a name='pwildcard'></a> 954 +<h4>The <tt>%wildcard</tt> directive</h4> 955 + 956 +<p>The %wildcard directive is followed by a single token name and a 957 +period. This directive specifies that the identified token should 958 +match any input token. 959 + 960 +<p>When the generated parser has the choice of matching an input against 961 +the wildcard token and some other token, the other token is always used. 962 +The wildcard token is only matched if there are no other alternatives. 963 + 877 964 <h3>Error Processing</h3> 878 965 879 966 <p>After extensive experimentation over several years, it has been 880 967 discovered that the error recovery strategy used by yacc is about 881 968 as good as it gets. And so that is what Lemon uses.</p> 882 969 883 970 <p>When a Lemon-generated parser encounters a syntax error, it 884 971 first invokes the code specified by the %syntax_error directive, if 885 972 any. It then enters its error recovery strategy. The error recovery 886 973 strategy is to begin popping the parsers stack until it enters a 887 974 state where it is permitted to shift a special non-terminal symbol 888 -named ``error''. It then shifts this non-terminal and continues 975 +named "error". It then shifts this non-terminal and continues 889 976 parsing. But the %syntax_error routine will not be called again 890 977 until at least three new tokens have been successfully shifted.</p> 891 978 892 979 <p>If the parser pops its stack until the stack is empty, and it still 893 980 is unable to shift the error symbol, then the %parse_failed routine 894 981 is invoked and the parser resets itself to its start state, ready 895 982 to begin parsing a new file. This is what will happen at the very 896 983 first syntax error, of course, if there are no instances of the 897 -``error'' non-terminal in your grammar.</p> 984 +"error" non-terminal in your grammar.</p> 898 985 899 986 </body> 900 987 </html>
Name change from ext/README.txt to ext/README.md.
1 -Version loadable extensions to SQLite are found in subfolders 2 -of this folder. 1 +## Loadable Extensions 2 + 3 +Various [loadable extensions](https://www.sqlite.org/loadext.html) for 4 +SQLite are found in subfolders. 5 + 6 +Most subfolders are dedicated to a single loadable extension (for 7 +example FTS5, or RTREE). But the misc/ subfolder contains a collection 8 +of smaller single-file extensions.
Changes to ext/fts2/fts2_tokenizer.c.
95 95 } 96 96 97 97 sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); 98 98 } 99 99 100 100 #ifdef SQLITE_TEST 101 101 102 -#include <tcl.h> 102 +#if defined(INCLUDE_SQLITE_TCL_H) 103 +# include "sqlite_tcl.h" 104 +#else 105 +# include "tcl.h" 106 +#endif 103 107 #include <string.h> 104 108 105 109 /* 106 110 ** Implementation of a special SQL scalar function for testing tokenizers 107 111 ** designed to be used in concert with the Tcl testing framework. This 108 112 ** function must be called with two arguments: 109 113 **
Changes to ext/fts3/fts3.c.
345 345 if( (v & mask2)==0 ){ var = v; return ret; } 346 346 347 347 /* 348 348 ** Read a 64-bit variable-length integer from memory starting at p[0]. 349 349 ** Return the number of bytes read, or 0 on error. 350 350 ** The value is stored in *v. 351 351 */ 352 -int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ 353 - const char *pStart = p; 352 +int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){ 353 + const unsigned char *p = (const unsigned char*)pBuf; 354 + const unsigned char *pStart = p; 354 355 u32 a; 355 356 u64 b; 356 357 int shift; 357 358 358 359 GETVARINT_INIT(a, p, 0, 0x00, 0x80, *v, 1); 359 360 GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *v, 2); 360 361 GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *v, 3); ................................................................................ 488 489 Fts3Table *p = (Fts3Table *)pVtab; 489 490 int i; 490 491 491 492 assert( p->nPendingData==0 ); 492 493 assert( p->pSegments==0 ); 493 494 494 495 /* Free any prepared statements held */ 496 + sqlite3_finalize(p->pSeekStmt); 495 497 for(i=0; i<SizeofArray(p->aStmt); i++){ 496 498 sqlite3_finalize(p->aStmt[i]); 497 499 } 498 500 sqlite3_free(p->zSegmentsTbl); 499 501 sqlite3_free(p->zReadExprlist); 500 502 sqlite3_free(p->zWriteExprlist); 501 503 sqlite3_free(p->zContentTbl); ................................................................................ 1359 1361 p->db = db; 1360 1362 p->nColumn = nCol; 1361 1363 p->nPendingData = 0; 1362 1364 p->azColumn = (char **)&p[1]; 1363 1365 p->pTokenizer = pTokenizer; 1364 1366 p->nMaxPendingData = FTS3_MAX_PENDING_DATA; 1365 1367 p->bHasDocsize = (isFts4 && bNoDocsize==0); 1366 - p->bHasStat = isFts4; 1367 - p->bFts4 = isFts4; 1368 - p->bDescIdx = bDescIdx; 1368 + p->bHasStat = (u8)isFts4; 1369 + p->bFts4 = (u8)isFts4; 1370 + p->bDescIdx = (u8)bDescIdx; 1369 1371 p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ 1370 1372 p->zContentTbl = zContent; 1371 1373 p->zLanguageid = zLanguageid; 1372 1374 zContent = 0; 1373 1375 zLanguageid = 0; 1374 1376 TESTONLY( p->inTransaction = -1 ); 1375 1377 TESTONLY( p->mxSavepoint = -1 ); ................................................................................ 1392 1394 zCsr += nDb; 1393 1395 1394 1396 /* Fill in the azColumn array */ 1395 1397 for(iCol=0; iCol<nCol; iCol++){ 1396 1398 char *z; 1397 1399 int n = 0; 1398 1400 z = (char *)sqlite3Fts3NextToken(aCol[iCol], &n); 1399 - memcpy(zCsr, z, n); 1401 + if( n>0 ){ 1402 + memcpy(zCsr, z, n); 1403 + } 1400 1404 zCsr[n] = '\0'; 1401 1405 sqlite3Fts3Dequote(zCsr); 1402 1406 p->azColumn[iCol] = zCsr; 1403 1407 zCsr += n+1; 1404 1408 assert( zCsr <= &((char *)p)[nByte] ); 1405 1409 } 1406 1410 ................................................................................ 1675 1679 *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); 1676 1680 if( !pCsr ){ 1677 1681 return SQLITE_NOMEM; 1678 1682 } 1679 1683 memset(pCsr, 0, sizeof(Fts3Cursor)); 1680 1684 return SQLITE_OK; 1681 1685 } 1686 + 1687 +/* 1688 +** Finalize the statement handle at pCsr->pStmt. 1689 +** 1690 +** Or, if that statement handle is one created by fts3CursorSeekStmt(), 1691 +** and the Fts3Table.pSeekStmt slot is currently NULL, save the statement 1692 +** pointer there instead of finalizing it. 1693 +*/ 1694 +static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){ 1695 + if( pCsr->bSeekStmt ){ 1696 + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; 1697 + if( p->pSeekStmt==0 ){ 1698 + p->pSeekStmt = pCsr->pStmt; 1699 + sqlite3_reset(pCsr->pStmt); 1700 + pCsr->pStmt = 0; 1701 + } 1702 + pCsr->bSeekStmt = 0; 1703 + } 1704 + sqlite3_finalize(pCsr->pStmt); 1705 +} 1682 1706 1683 1707 /* 1684 1708 ** Close the cursor. For additional information see the documentation 1685 1709 ** on the xClose method of the virtual table interface. 1686 1710 */ 1687 1711 static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ 1688 1712 Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; 1689 1713 assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); 1690 - sqlite3_finalize(pCsr->pStmt); 1714 + fts3CursorFinalizeStmt(pCsr); 1691 1715 sqlite3Fts3ExprFree(pCsr->pExpr); 1692 1716 sqlite3Fts3FreeDeferredTokens(pCsr); 1693 1717 sqlite3_free(pCsr->aDoclist); 1694 1718 sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); 1695 1719 assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); 1696 1720 sqlite3_free(pCsr); 1697 1721 return SQLITE_OK; ................................................................................ 1701 1725 ** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then 1702 1726 ** compose and prepare an SQL statement of the form: 1703 1727 ** 1704 1728 ** "SELECT <columns> FROM %_content WHERE rowid = ?" 1705 1729 ** 1706 1730 ** (or the equivalent for a content=xxx table) and set pCsr->pStmt to 1707 1731 ** it. If an error occurs, return an SQLite error code. 1708 -** 1709 -** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK. 1710 1732 */ 1711 -static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ 1733 +static int fts3CursorSeekStmt(Fts3Cursor *pCsr){ 1712 1734 int rc = SQLITE_OK; 1713 1735 if( pCsr->pStmt==0 ){ 1714 1736 Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; 1715 1737 char *zSql; 1716 - zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); 1717 - if( !zSql ) return SQLITE_NOMEM; 1718 - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); 1719 - sqlite3_free(zSql); 1738 + if( p->pSeekStmt ){ 1739 + pCsr->pStmt = p->pSeekStmt; 1740 + p->pSeekStmt = 0; 1741 + }else{ 1742 + zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); 1743 + if( !zSql ) return SQLITE_NOMEM; 1744 + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); 1745 + sqlite3_free(zSql); 1746 + } 1747 + if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1; 1720 1748 } 1721 - *ppStmt = pCsr->pStmt; 1722 1749 return rc; 1723 1750 } 1724 1751 1725 1752 /* 1726 1753 ** Position the pCsr->pStmt statement so that it is on the row 1727 1754 ** of the %_content table that contains the last match. Return 1728 1755 ** SQLITE_OK on success. 1729 1756 */ 1730 1757 static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ 1731 1758 int rc = SQLITE_OK; 1732 1759 if( pCsr->isRequireSeek ){ 1733 - sqlite3_stmt *pStmt = 0; 1734 - 1735 - rc = fts3CursorSeekStmt(pCsr, &pStmt); 1760 + rc = fts3CursorSeekStmt(pCsr); 1736 1761 if( rc==SQLITE_OK ){ 1737 1762 sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); 1738 1763 pCsr->isRequireSeek = 0; 1739 1764 if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ 1740 1765 return SQLITE_OK; 1741 1766 }else{ 1742 1767 rc = sqlite3_reset(pCsr->pStmt); ................................................................................ 3186 3211 if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++]; 3187 3212 if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++]; 3188 3213 if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++]; 3189 3214 if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++]; 3190 3215 assert( iIdx==nVal ); 3191 3216 3192 3217 /* In case the cursor has been used before, clear it now. */ 3193 - sqlite3_finalize(pCsr->pStmt); 3218 + fts3CursorFinalizeStmt(pCsr); 3194 3219 sqlite3_free(pCsr->aDoclist); 3195 3220 sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); 3196 3221 sqlite3Fts3ExprFree(pCsr->pExpr); 3197 3222 memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); 3198 3223 3199 3224 /* Set the lower and upper bounds on docids to return */ 3200 3225 pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); ................................................................................ 3254 3279 if( zSql ){ 3255 3280 rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); 3256 3281 sqlite3_free(zSql); 3257 3282 }else{ 3258 3283 rc = SQLITE_NOMEM; 3259 3284 } 3260 3285 }else if( eSearch==FTS3_DOCID_SEARCH ){ 3261 - rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt); 3286 + rc = fts3CursorSeekStmt(pCsr); 3262 3287 if( rc==SQLITE_OK ){ 3263 3288 rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); 3264 3289 } 3265 3290 } 3266 3291 if( rc!=SQLITE_OK ) return rc; 3267 3292 3268 3293 return fts3NextMethod(pCursor); ................................................................................ 3382 3407 ** of blocks from the segments table. But this is not considered overhead 3383 3408 ** as it would also be required by a crisis-merge that used the same input 3384 3409 ** segments. 3385 3410 */ 3386 3411 const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ 3387 3412 3388 3413 Fts3Table *p = (Fts3Table*)pVtab; 3389 - int rc = sqlite3Fts3PendingTermsFlush(p); 3414 + int rc; 3415 + i64 iLastRowid = sqlite3_last_insert_rowid(p->db); 3390 3416 3417 + rc = sqlite3Fts3PendingTermsFlush(p); 3391 3418 if( rc==SQLITE_OK 3392 3419 && p->nLeafAdd>(nMinMerge/16) 3393 3420 && p->nAutoincrmerge && p->nAutoincrmerge!=0xff 3394 3421 ){ 3395 3422 int mxLevel = 0; /* Maximum relative level value in db */ 3396 3423 int A; /* Incr-merge parameter A */ 3397 3424 ................................................................................ 3398 3425 rc = sqlite3Fts3MaxLevel(p, &mxLevel); 3399 3426 assert( rc==SQLITE_OK || mxLevel==0 ); 3400 3427 A = p->nLeafAdd * mxLevel; 3401 3428 A += (A/2); 3402 3429 if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); 3403 3430 } 3404 3431 sqlite3Fts3SegmentsClose(p); 3432 + sqlite3_set_last_insert_rowid(p->db, iLastRowid); 3405 3433 return rc; 3406 3434 } 3407 3435 3408 3436 /* 3409 3437 ** If it is currently unknown whether or not the FTS table has an %_stat 3410 3438 ** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat 3411 3439 ** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code ................................................................................ 3418 3446 char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName); 3419 3447 if( zSql ){ 3420 3448 sqlite3_stmt *pStmt = 0; 3421 3449 rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); 3422 3450 if( rc==SQLITE_OK ){ 3423 3451 int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); 3424 3452 rc = sqlite3_finalize(pStmt); 3425 - if( rc==SQLITE_OK ) p->bHasStat = bHasStat; 3453 + if( rc==SQLITE_OK ) p->bHasStat = (u8)bHasStat; 3426 3454 } 3427 3455 sqlite3_free(zSql); 3428 3456 }else{ 3429 3457 rc = SQLITE_NOMEM; 3430 3458 } 3431 3459 } 3432 3460 return rc;
Changes to ext/fts3/fts3Int.h.
226 226 int nAutoincrmerge; /* Value configured by 'automerge' */ 227 227 u32 nLeafAdd; /* Number of leaf blocks added this trans */ 228 228 229 229 /* Precompiled statements used by the implementation. Each of these 230 230 ** statements is run and reset within a single virtual table API call. 231 231 */ 232 232 sqlite3_stmt *aStmt[40]; 233 + sqlite3_stmt *pSeekStmt; /* Cache for fts3CursorSeekStmt() */ 233 234 234 235 char *zReadExprlist; 235 236 char *zWriteExprlist; 236 237 237 238 int nNodeSize; /* Soft limit for node size */ 238 239 u8 bFts4; /* True for FTS4, false for FTS3 */ 239 240 u8 bHasStat; /* True if %_stat table exists (2==unknown) */ ................................................................................ 295 296 ** the xOpen method. Cursors are destroyed using the xClose method. 296 297 */ 297 298 struct Fts3Cursor { 298 299 sqlite3_vtab_cursor base; /* Base class used by SQLite core */ 299 300 i16 eSearch; /* Search strategy (see below) */ 300 301 u8 isEof; /* True if at End Of Results */ 301 302 u8 isRequireSeek; /* True if must seek pStmt to %_content row */ 303 + u8 bSeekStmt; /* True if pStmt is a seek */ 302 304 sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ 303 305 Fts3Expr *pExpr; /* Parsed MATCH query string */ 304 306 int iLangid; /* Language being queried for */ 305 307 int nPhrase; /* Number of matchable phrases in query */ 306 308 Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ 307 309 sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ 308 310 char *pNextId; /* Pointer into the body of aDoclist */
Changes to ext/fts3/fts3_test.c.
14 14 ** testing. It contains a Tcl command that can be used to test if a document 15 15 ** matches an FTS NEAR expression. 16 16 ** 17 17 ** As of March 2012, it also contains a version 1 tokenizer used for testing 18 18 ** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly. 19 19 */ 20 20 21 -#include <tcl.h> 21 +#if defined(INCLUDE_SQLITE_TCL_H) 22 +# include "sqlite_tcl.h" 23 +#else 24 +# include "tcl.h" 25 +# ifndef SQLITE_TCLAPI 26 +# define SQLITE_TCLAPI 27 +# endif 28 +#endif 22 29 #include <string.h> 23 30 #include <assert.h> 24 31 25 32 #if defined(SQLITE_TEST) 26 33 #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) 27 34 28 35 /* Required so that the "ifdef SQLITE_ENABLE_FTS3" below works */ ................................................................................ 139 146 140 147 return nOcc; 141 148 } 142 149 143 150 /* 144 151 ** Tclcmd: fts3_near_match DOCUMENT EXPR ?OPTIONS? 145 152 */ 146 -static int fts3_near_match_cmd( 153 +static int SQLITE_TCLAPI fts3_near_match_cmd( 147 154 ClientData clientData, 148 155 Tcl_Interp *interp, 149 156 int objc, 150 157 Tcl_Obj *CONST objv[] 151 158 ){ 152 159 int nTotal = 0; 153 160 int rc; ................................................................................ 274 281 ** set cfg [fts3_configure_incr_load $new_chunksize $new_threshold] 275 282 ** 276 283 ** .... run tests .... 277 284 ** 278 285 ** # Restore initial incr-load settings: 279 286 ** eval fts3_configure_incr_load $cfg 280 287 */ 281 -static int fts3_configure_incr_load_cmd( 288 +static int SQLITE_TCLAPI fts3_configure_incr_load_cmd( 282 289 ClientData clientData, 283 290 Tcl_Interp *interp, 284 291 int objc, 285 292 Tcl_Obj *CONST objv[] 286 293 ){ 287 294 #ifdef SQLITE_ENABLE_FTS3 288 295 extern int test_fts3_node_chunksize; ................................................................................ 454 461 rc = SQLITE_NOMEM; 455 462 }else{ 456 463 int i; 457 464 458 465 if( pCsr->iLangid & 0x00000001 ){ 459 466 for(i=0; i<nToken; i++) pCsr->aBuffer[i] = pToken[i]; 460 467 }else{ 461 - for(i=0; i<nToken; i++) pCsr->aBuffer[i] = testTolower(pToken[i]); 468 + for(i=0; i<nToken; i++) pCsr->aBuffer[i] = (char)testTolower(pToken[i]); 462 469 } 463 470 pCsr->iToken++; 464 471 pCsr->iInput = (int)(p - pCsr->aInput); 465 472 466 473 *ppToken = pCsr->aBuffer; 467 474 *pnBytes = nToken; 468 475 *piStartOffset = (int)(pToken - pCsr->aInput); ................................................................................ 484 491 if( pCsr->iLangid>=100 ){ 485 492 rc = SQLITE_ERROR; 486 493 } 487 494 return rc; 488 495 } 489 496 #endif 490 497 491 -static int fts3_test_tokenizer_cmd( 498 +static int SQLITE_TCLAPI fts3_test_tokenizer_cmd( 492 499 ClientData clientData, 493 500 Tcl_Interp *interp, 494 501 int objc, 495 502 Tcl_Obj *CONST objv[] 496 503 ){ 497 504 #ifdef SQLITE_ENABLE_FTS3 498 505 static const sqlite3_tokenizer_module testTokenizerModule = { ................................................................................ 513 520 (const unsigned char *)&pPtr, sizeof(sqlite3_tokenizer_module *) 514 521 )); 515 522 #endif 516 523 UNUSED_PARAMETER(clientData); 517 524 return TCL_OK; 518 525 } 519 526 520 -static int fts3_test_varint_cmd( 527 +static int SQLITE_TCLAPI fts3_test_varint_cmd( 521 528 ClientData clientData, 522 529 Tcl_Interp *interp, 523 530 int objc, 524 531 Tcl_Obj *CONST objv[] 525 532 ){ 526 533 #ifdef SQLITE_ENABLE_FTS3 527 534 char aBuf[24];
Changes to ext/fts3/fts3_tokenizer.c.
220 220 sqlite3_free(zCopy); 221 221 return rc; 222 222 } 223 223 224 224 225 225 #ifdef SQLITE_TEST 226 226 227 -#include <tcl.h> 227 +#if defined(INCLUDE_SQLITE_TCL_H) 228 +# include "sqlite_tcl.h" 229 +#else 230 +# include "tcl.h" 231 +#endif 228 232 #include <string.h> 229 233 230 234 /* 231 235 ** Implementation of a special SQL scalar function for testing tokenizers 232 236 ** designed to be used in concert with the Tcl testing framework. This 233 237 ** function must be called with two or more arguments: 234 238 **
Changes to ext/fts3/fts3_unicode.c.
132 132 unicode_tokenizer *p, /* Tokenizer to add exceptions to */ 133 133 int bAlnum, /* Replace Isalnum() return value with this */ 134 134 const char *zIn, /* Array of characters to make exceptions */ 135 135 int nIn /* Length of z in bytes */ 136 136 ){ 137 137 const unsigned char *z = (const unsigned char *)zIn; 138 138 const unsigned char *zTerm = &z[nIn]; 139 - int iCode; 139 + unsigned int iCode; 140 140 int nEntry = 0; 141 141 142 142 assert( bAlnum==0 || bAlnum==1 ); 143 143 144 144 while( z<zTerm ){ 145 145 READ_UTF8(z, zTerm, iCode); 146 - assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 ); 147 - if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum 148 - && sqlite3FtsUnicodeIsdiacritic(iCode)==0 146 + assert( (sqlite3FtsUnicodeIsalnum((int)iCode) & 0xFFFFFFFE)==0 ); 147 + if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum 148 + && sqlite3FtsUnicodeIsdiacritic((int)iCode)==0 149 149 ){ 150 150 nEntry++; 151 151 } 152 152 } 153 153 154 154 if( nEntry ){ 155 155 int *aNew; /* New aiException[] array */ ................................................................................ 158 158 aNew = sqlite3_realloc(p->aiException, (p->nException+nEntry)*sizeof(int)); 159 159 if( aNew==0 ) return SQLITE_NOMEM; 160 160 nNew = p->nException; 161 161 162 162 z = (const unsigned char *)zIn; 163 163 while( z<zTerm ){ 164 164 READ_UTF8(z, zTerm, iCode); 165 - if( sqlite3FtsUnicodeIsalnum(iCode)!=bAlnum 166 - && sqlite3FtsUnicodeIsdiacritic(iCode)==0 165 + if( sqlite3FtsUnicodeIsalnum((int)iCode)!=bAlnum 166 + && sqlite3FtsUnicodeIsdiacritic((int)iCode)==0 167 167 ){ 168 168 int i, j; 169 - for(i=0; i<nNew && aNew[i]<iCode; i++); 169 + for(i=0; i<nNew && aNew[i]<(int)iCode; i++); 170 170 for(j=nNew; j>i; j--) aNew[j] = aNew[j-1]; 171 - aNew[i] = iCode; 171 + aNew[i] = (int)iCode; 172 172 nNew++; 173 173 } 174 174 } 175 175 p->aiException = aNew; 176 176 p->nException = nNew; 177 177 } 178 178 ................................................................................ 314 314 int *pnToken, /* OUT: Number of bytes at *paToken */ 315 315 int *piStart, /* OUT: Starting offset of token */ 316 316 int *piEnd, /* OUT: Ending offset of token */ 317 317 int *piPos /* OUT: Position integer of token */ 318 318 ){ 319 319 unicode_cursor *pCsr = (unicode_cursor *)pC; 320 320 unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); 321 - int iCode = 0; 321 + unsigned int iCode = 0; 322 322 char *zOut; 323 323 const unsigned char *z = &pCsr->aInput[pCsr->iOff]; 324 324 const unsigned char *zStart = z; 325 325 const unsigned char *zEnd; 326 326 const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput]; 327 327 328 328 /* Scan past any delimiter characters before the start of the next token. 329 329 ** Return SQLITE_DONE early if this takes us all the way to the end of 330 330 ** the input. */ 331 331 while( z<zTerm ){ 332 332 READ_UTF8(z, zTerm, iCode); 333 - if( unicodeIsAlnum(p, iCode) ) break; 333 + if( unicodeIsAlnum(p, (int)iCode) ) break; 334 334 zStart = z; 335 335 } 336 336 if( zStart>=zTerm ) return SQLITE_DONE; 337 337 338 338 zOut = pCsr->zToken; 339 339 do { 340 340 int iOut; ................................................................................ 346 346 zOut = &zNew[zOut - pCsr->zToken]; 347 347 pCsr->zToken = zNew; 348 348 pCsr->nAlloc += 64; 349 349 } 350 350 351 351 /* Write the folded case of the last character read to the output */ 352 352 zEnd = z; 353 - iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic); 353 + iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic); 354 354 if( iOut ){ 355 355 WRITE_UTF8(zOut, iOut); 356 356 } 357 357 358 358 /* If the cursor is not at EOF, read the next character */ 359 359 if( z>=zTerm ) break; 360 360 READ_UTF8(z, zTerm, iCode); 361 - }while( unicodeIsAlnum(p, iCode) 362 - || sqlite3FtsUnicodeIsdiacritic(iCode) 361 + }while( unicodeIsAlnum(p, (int)iCode) 362 + || sqlite3FtsUnicodeIsdiacritic((int)iCode) 363 363 ); 364 364 365 365 /* Set the output variables and return. */ 366 366 pCsr->iOff = (int)(z - pCsr->aInput); 367 367 *paToken = pCsr->zToken; 368 368 *pnToken = (int)(zOut - pCsr->zToken); 369 369 *piStart = (int)(zStart - pCsr->aInput);
Changes to ext/fts3/fts3_unicode2.c.
123 123 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060, 124 124 0x380400F0, 125 125 }; 126 126 static const unsigned int aAscii[4] = { 127 127 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, 128 128 }; 129 129 130 - if( c<128 ){ 131 - return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); 132 - }else if( c<(1<<22) ){ 130 + if( (unsigned int)c<128 ){ 131 + return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 ); 132 + }else if( (unsigned int)c<(1<<22) ){ 133 133 unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; 134 134 int iRes = 0; 135 135 int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; 136 136 int iLo = 0; 137 137 while( iHi>=iLo ){ 138 138 int iTest = (iHi + iLo) / 2; 139 139 if( key >= aEntry[iTest] ){ ................................................................................ 318 318 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, 319 319 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, 320 320 65514, 65521, 65527, 65528, 65529, 321 321 }; 322 322 323 323 int ret = c; 324 324 325 - assert( c>=0 ); 326 325 assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); 327 326 328 327 if( c<128 ){ 329 328 if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); 330 329 }else if( c<65536 ){ 330 + const struct TableEntry *p; 331 331 int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; 332 332 int iLo = 0; 333 333 int iRes = -1; 334 334 335 + assert( c>aEntry[0].iCode ); 335 336 while( iHi>=iLo ){ 336 337 int iTest = (iHi + iLo) / 2; 337 338 int cmp = (c - aEntry[iTest].iCode); 338 339 if( cmp>=0 ){ 339 340 iRes = iTest; 340 341 iLo = iTest+1; 341 342 }else{ 342 343 iHi = iTest-1; 343 344 } 344 345 } 345 - assert( iRes<0 || c>=aEntry[iRes].iCode ); 346 346 347 - if( iRes>=0 ){ 348 - const struct TableEntry *p = &aEntry[iRes]; 349 - if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ 350 - ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; 351 - assert( ret>0 ); 352 - } 347 + assert( iRes>=0 && c>=aEntry[iRes].iCode ); 348 + p = &aEntry[iRes]; 349 + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ 350 + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; 351 + assert( ret>0 ); 353 352 } 354 353 355 354 if( bRemoveDiacritic ) ret = remove_diacritic(ret); 356 355 } 357 356 358 357 else if( c>=66560 && c<66600 ){ 359 358 ret = c + 40; 360 359 } 361 360 362 361 return ret; 363 362 } 364 363 #endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */ 365 364 #endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */
Changes to ext/fts3/fts3_write.c.
4952 4952 return rc; 4953 4953 } 4954 4954 4955 4955 /* 4956 4956 ** Convert the text beginning at *pz into an integer and return 4957 4957 ** its value. Advance *pz to point to the first character past 4958 4958 ** the integer. 4959 +** 4960 +** This function used for parameters to merge= and incrmerge= 4961 +** commands. 4959 4962 */ 4960 4963 static int fts3Getint(const char **pz){ 4961 4964 const char *z = *pz; 4962 4965 int i = 0; 4963 - while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0'; 4966 + while( (*z)>='0' && (*z)<='9' && i<214748363 ) i = 10*i + *(z++) - '0'; 4964 4967 *pz = z; 4965 4968 return i; 4966 4969 } 4967 4970 4968 4971 /* 4969 4972 ** Process statements of the form: 4970 4973 **
Added ext/fts3/tool/fts3cov.sh.
1 +#!/bin/sh 2 + 3 +set -e 4 + 5 +srcdir=`dirname $(dirname $(dirname $(dirname $0)))` 6 +./testfixture $srcdir/test/fts3.test --output=fts3cov-out.txt 7 + 8 +echo "" 9 + 10 +for f in `ls $srcdir/ext/fts3/*.c` 11 +do 12 + f=`basename $f` 13 + echo -ne "$f: " 14 + gcov -b $f | grep Taken | sed 's/Taken at least once://' 15 +done 16 +
Changes to ext/fts3/tool/fts3view.c.
394 394 " WHERE (a.blockid BETWEEN b.start_block" 395 395 " AND b.leaves_end_block)" 396 396 " AND (b.level%%1024)==%d)", 397 397 pgsz-45, zTab, zTab, i); 398 398 if( sqlite3_step(pStmt)==SQLITE_ROW 399 399 && (nLeaf = sqlite3_column_int(pStmt, 0))>0 400 400 ){ 401 - nIdx = sqlite3_column_int(pStmt, 5); 402 401 sqlite3_int64 sz; 402 + nIdx = sqlite3_column_int(pStmt, 5); 403 403 printf("For level %d:\n", i); 404 404 printf(" Number of indexes...................... %9d\n", nIdx); 405 405 printf(" Number of leaf segments................ %9d\n", nLeaf); 406 406 if( nIdx>1 ){ 407 407 printf(" Average leaf segments per index........ %11.1f\n", 408 408 (double)nLeaf/(double)nIdx); 409 409 }
Changes to ext/fts3/unicode/mkunicode.tcl.
223 223 puts "** is less than zero." 224 224 puts "*/" 225 225 puts "int ${zFunc}\(int c)\{" 226 226 an_print_range_array $lRange 227 227 an_print_ascii_bitmap $lRange 228 228 puts { 229 229 if( (unsigned int)c<128 ){ 230 - return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); 230 + return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 ); 231 231 }else if( (unsigned int)c<(1<<22) ){ 232 232 unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; 233 233 int iRes = 0; 234 234 int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; 235 235 int iLo = 0; 236 236 while( iHi>=iLo ){ 237 237 int iTest = (iHi + iLo) / 2;
Changes to ext/fts5/fts5.h.
139 139 ** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): 140 140 ** This API function is used to query the FTS table for phrase iPhrase 141 141 ** of the current query. Specifically, a query equivalent to: 142 142 ** 143 143 ** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid 144 144 ** 145 145 ** with $p set to a phrase equivalent to the phrase iPhrase of the 146 -** current query is executed. For each row visited, the callback function 147 -** passed as the fourth argument is invoked. The context and API objects 148 -** passed to the callback function may be used to access the properties of 149 -** each matched row. Invoking Api.xUserData() returns a copy of the pointer 150 -** passed as the third argument to pUserData. 146 +** current query is executed. Any column filter that applies to 147 +** phrase iPhrase of the current query is included in $p. For each 148 +** row visited, the callback function passed as the fourth argument 149 +** is invoked. The context and API objects passed to the callback 150 +** function may be used to access the properties of each matched row. 151 +** Invoking Api.xUserData() returns a copy of the pointer passed as 152 +** the third argument to pUserData. 151 153 ** 152 154 ** If the callback function returns any value other than SQLITE_OK, the 153 155 ** query is abandoned and the xQueryPhrase function returns immediately. 154 156 ** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. 155 157 ** Otherwise, the error code is propagated upwards. 156 158 ** 157 159 ** If the query runs to completion without incident, SQLITE_OK is returned. ................................................................................ 312 314 ** Applications may also register custom tokenizer types. A tokenizer 313 315 ** is registered by providing fts5 with a populated instance of the 314 316 ** following structure. All structure methods must be defined, setting 315 317 ** any member of the fts5_tokenizer struct to NULL leads to undefined 316 318 ** behaviour. The structure methods are expected to function as follows: 317 319 ** 318 320 ** xCreate: 319 -** This function is used to allocate and inititalize a tokenizer instance. 321 +** This function is used to allocate and initialize a tokenizer instance. 320 322 ** A tokenizer instance is required to actually tokenize text. 321 323 ** 322 324 ** The first argument passed to this function is a copy of the (void*) 323 325 ** pointer provided by the application when the fts5_tokenizer object 324 326 ** was registered with FTS5 (the third argument to xCreateTokenizer()). 325 327 ** The second and third arguments are an array of nul-terminated strings 326 328 ** containing the tokenizer arguments, if any, specified following the ................................................................................ 571 573 *************************************************************************/ 572 574 573 575 #ifdef __cplusplus 574 576 } /* end of the 'extern "C"' block */ 575 577 #endif 576 578 577 579 #endif /* _FTS5_H */ 578 -
Changes to ext/fts5/fts5Int.h.
26 26 typedef unsigned char u8; 27 27 typedef unsigned int u32; 28 28 typedef unsigned short u16; 29 29 typedef short i16; 30 30 typedef sqlite3_int64 i64; 31 31 typedef sqlite3_uint64 u64; 32 32 33 -#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0]))) 33 +#ifndef ArraySize 34 +# define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0]))) 35 +#endif 34 36 35 37 #define testcase(x) 36 38 #define ALWAYS(x) 1 37 39 #define NEVER(x) 0 38 40 39 41 #define MIN(x,y) (((x) < (y)) ? (x) : (y)) 40 42 #define MAX(x,y) (((x) > (y)) ? (x) : (y)) ................................................................................ 43 45 ** Constants for the largest and smallest possible 64-bit signed integers. 44 46 */ 45 47 # define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) 46 48 # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) 47 49 48 50 #endif 49 51 52 +/* Truncate very long tokens to this many bytes. Hard limit is 53 +** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset 54 +** field that occurs at the start of each leaf page (see fts5_index.c). */ 55 +#define FTS5_MAX_TOKEN_SIZE 32768 50 56 51 57 /* 52 58 ** Maximum number of prefix indexes on single FTS5 table. This must be 53 59 ** less than 32. If it is set to anything large than that, an #error 54 60 ** directive in fts5_index.c will cause the build to fail. 55 61 */ 56 62 #define FTS5_MAX_PREFIX_INDEXES 31 ................................................................................ 476 482 ** this connection since it was created. 477 483 */ 478 484 int sqlite3Fts5IndexReads(Fts5Index *p); 479 485 480 486 int sqlite3Fts5IndexReinit(Fts5Index *p); 481 487 int sqlite3Fts5IndexOptimize(Fts5Index *p); 482 488 int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge); 489 +int sqlite3Fts5IndexReset(Fts5Index *p); 483 490 484 491 int sqlite3Fts5IndexLoadConfig(Fts5Index *p); 485 492 486 493 /* 487 494 ** End of interface to code in fts5_index.c. 488 495 **************************************************************************/ 489 496 ................................................................................ 618 625 Fts5Storage *p, const char*, sqlite3_value*, int 619 626 ); 620 627 621 628 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p); 622 629 int sqlite3Fts5StorageRebuild(Fts5Storage *p); 623 630 int sqlite3Fts5StorageOptimize(Fts5Storage *p); 624 631 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge); 632 +int sqlite3Fts5StorageReset(Fts5Storage *p); 625 633 626 634 /* 627 635 ** End of interface to code in fts5_storage.c. 628 636 **************************************************************************/ 629 637 630 638 631 639 /************************************************************************** ................................................................................ 676 684 677 685 typedef struct Fts5PoslistPopulator Fts5PoslistPopulator; 678 686 Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int); 679 687 int sqlite3Fts5ExprPopulatePoslists( 680 688 Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int 681 689 ); 682 690 void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64); 683 -void sqlite3Fts5ExprClearEof(Fts5Expr*); 684 691 685 692 int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**); 686 693 687 694 int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); 688 695 689 696 /******************************************* 690 697 ** The fts5_expr.c API above this point is used by the other hand-written ................................................................................ 728 735 729 736 void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*); 730 737 void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*); 731 738 void sqlite3Fts5ParseNodeFree(Fts5ExprNode*); 732 739 733 740 void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*); 734 741 void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*); 742 +Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*); 735 743 void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p); 736 744 void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*); 737 745 738 746 /* 739 747 ** End of interface to code in fts5_expr.c. 740 748 **************************************************************************/ 741 749
Changes to ext/fts5/fts5_aux.c.
185 185 rc = fts5CInstIterNext(&p->iter); 186 186 } 187 187 } 188 188 189 189 if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){ 190 190 fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); 191 191 p->iOff = iEndOff; 192 - if( iPos<p->iter.iEnd ){ 192 + if( iPos>=p->iter.iStart && iPos<p->iter.iEnd ){ 193 193 fts5HighlightAppend(&rc, p, p->zClose, -1); 194 194 } 195 195 } 196 196 197 197 return rc; 198 198 } 199 199 ................................................................................ 241 241 if( rc!=SQLITE_OK ){ 242 242 sqlite3_result_error_code(pCtx, rc); 243 243 } 244 244 } 245 245 /* 246 246 ** End of highlight() implementation. 247 247 **************************************************************************/ 248 + 249 +/* 250 +** Context object passed to the fts5SentenceFinderCb() function. 251 +*/ 252 +typedef struct Fts5SFinder Fts5SFinder; 253 +struct Fts5SFinder { 254 + int iPos; /* Current token position */ 255 + int nFirstAlloc; /* Allocated size of aFirst[] */ 256 + int nFirst; /* Number of entries in aFirst[] */ 257 + int *aFirst; /* Array of first token in each sentence */ 258 + const char *zDoc; /* Document being tokenized */ 259 +}; 260 + 261 +/* 262 +** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if 263 +** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an 264 +** error occurs. 265 +*/ 266 +static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){ 267 + if( p->nFirstAlloc==p->nFirst ){ 268 + int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64; 269 + int *aNew; 270 + 271 + aNew = (int*)sqlite3_realloc(p->aFirst, nNew*sizeof(int)); 272 + if( aNew==0 ) return SQLITE_NOMEM; 273 + p->aFirst = aNew; 274 + p->nFirstAlloc = nNew; 275 + } 276 + p->aFirst[p->nFirst++] = iAdd; 277 + return SQLITE_OK; 278 +} 279 + 280 +/* 281 +** This function is an xTokenize() callback used by the auxiliary snippet() 282 +** function. Its job is to identify tokens that are the first in a sentence. 283 +** For each such token, an entry is added to the SFinder.aFirst[] array. 284 +*/ 285 +static int fts5SentenceFinderCb( 286 + void *pContext, /* Pointer to HighlightContext object */ 287 + int tflags, /* Mask of FTS5_TOKEN_* flags */ 288 + const char *pToken, /* Buffer containing token */ 289 + int nToken, /* Size of token in bytes */ 290 + int iStartOff, /* Start offset of token */ 291 + int iEndOff /* End offset of token */ 292 +){ 293 + int rc = SQLITE_OK; 294 + 295 + UNUSED_PARAM2(pToken, nToken); 296 + UNUSED_PARAM(iEndOff); 297 + 298 + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ 299 + Fts5SFinder *p = (Fts5SFinder*)pContext; 300 + if( p->iPos>0 ){ 301 + int i; 302 + char c = 0; 303 + for(i=iStartOff-1; i>=0; i--){ 304 + c = p->zDoc[i]; 305 + if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break; 306 + } 307 + if( i!=iStartOff-1 && (c=='.' || c==':') ){ 308 + rc = fts5SentenceFinderAdd(p, p->iPos); 309 + } 310 + }else{ 311 + rc = fts5SentenceFinderAdd(p, 0); 312 + } 313 + p->iPos++; 314 + } 315 + return rc; 316 +} 317 + 318 +static int fts5SnippetScore( 319 + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ 320 + Fts5Context *pFts, /* First arg to pass to pApi functions */ 321 + int nDocsize, /* Size of column in tokens */ 322 + unsigned char *aSeen, /* Array with one element per query phrase */ 323 + int iCol, /* Column to score */ 324 + int iPos, /* Starting offset to score */ 325 + int nToken, /* Max tokens per snippet */ 326 + int *pnScore, /* OUT: Score */ 327 + int *piPos /* OUT: Adjusted offset */ 328 +){ 329 + int rc; 330 + int i; 331 + int ip = 0; 332 + int ic = 0; 333 + int iOff = 0; 334 + int iFirst = -1; 335 + int nInst; 336 + int nScore = 0; 337 + int iLast = 0; 338 + 339 + rc = pApi->xInstCount(pFts, &nInst); 340 + for(i=0; i<nInst && rc==SQLITE_OK; i++){ 341 + rc = pApi->xInst(pFts, i, &ip, &ic, &iOff); 342 + if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOff<(iPos+nToken) ){ 343 + nScore += (aSeen[ip] ? 1 : 1000); 344 + aSeen[ip] = 1; 345 + if( iFirst<0 ) iFirst = iOff; 346 + iLast = iOff + pApi->xPhraseSize(pFts, ip); 347 + } 348 + } 349 + 350 + *pnScore = nScore; 351 + if( piPos ){ 352 + int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2; 353 + if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken; 354 + if( iAdj<0 ) iAdj = 0; 355 + *piPos = iAdj; 356 + } 357 + 358 + return rc; 359 +} 248 360 249 361 /* 250 362 ** Implementation of snippet() function. 251 363 */ 252 364 static void fts5SnippetFunction( 253 365 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ 254 366 Fts5Context *pFts, /* First arg to pass to pApi functions */ ................................................................................ 263 375 int nToken; /* 5th argument to snippet() */ 264 376 int nInst = 0; /* Number of instance matches this row */ 265 377 int i; /* Used to iterate through instances */ 266 378 int nPhrase; /* Number of phrases in query */ 267 379 unsigned char *aSeen; /* Array of "seen instance" flags */ 268 380 int iBestCol; /* Column containing best snippet */ 269 381 int iBestStart = 0; /* First token of best snippet */ 270 - int iBestLast; /* Last token of best snippet */ 271 382 int nBestScore = 0; /* Score of best snippet */ 272 383 int nColSize = 0; /* Total size of iBestCol in tokens */ 384 + Fts5SFinder sFinder; /* Used to find the beginnings of sentences */ 385 + int nCol; 273 386 274 387 if( nVal!=5 ){ 275 388 const char *zErr = "wrong number of arguments to function snippet()"; 276 389 sqlite3_result_error(pCtx, zErr, -1); 277 390 return; 278 391 } 279 392 393 + nCol = pApi->xColumnCount(pFts); 280 394 memset(&ctx, 0, sizeof(HighlightContext)); 281 395 iCol = sqlite3_value_int(apVal[0]); 282 396 ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); 283 397 ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); 284 398 zEllips = (const char*)sqlite3_value_text(apVal[3]); 285 399 nToken = sqlite3_value_int(apVal[4]); 286 - iBestLast = nToken-1; 287 400 288 401 iBestCol = (iCol>=0 ? iCol : 0); 289 402 nPhrase = pApi->xPhraseCount(pFts); 290 403 aSeen = sqlite3_malloc(nPhrase); 291 404 if( aSeen==0 ){ 292 405 rc = SQLITE_NOMEM; 293 406 } 294 - 295 407 if( rc==SQLITE_OK ){ 296 408 rc = pApi->xInstCount(pFts, &nInst); 297 409 } 298 - for(i=0; rc==SQLITE_OK && i<nInst; i++){ 299 - int ip, iSnippetCol, iStart; 300 - memset(aSeen, 0, nPhrase); 301 - rc = pApi->xInst(pFts, i, &ip, &iSnippetCol, &iStart); 302 - if( rc==SQLITE_OK && (iCol<0 || iSnippetCol==iCol) ){ 303 - int nScore = 1000; 304 - int iLast = iStart - 1 + pApi->xPhraseSize(pFts, ip); 305 - int j; 306 - aSeen[ip] = 1; 307 - 308 - for(j=i+1; rc==SQLITE_OK && j<nInst; j++){ 309 - int ic; int io; int iFinal; 310 - rc = pApi->xInst(pFts, j, &ip, &ic, &io); 311 - iFinal = io + pApi->xPhraseSize(pFts, ip) - 1; 312 - if( rc==SQLITE_OK && ic==iSnippetCol && iLast<iStart+nToken ){ 313 - nScore += aSeen[ip] ? 1000 : 1; 314 - aSeen[ip] = 1; 315 - if( iFinal>iLast ) iLast = iFinal; 410 + 411 + memset(&sFinder, 0, sizeof(Fts5SFinder)); 412 + for(i=0; i<nCol; i++){ 413 + if( iCol<0 || iCol==i ){ 414 + int nDoc; 415 + int nDocsize; 416 + int ii; 417 + sFinder.iPos = 0; 418 + sFinder.nFirst = 0; 419 + rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc); 420 + if( rc!=SQLITE_OK ) break; 421 + rc = pApi->xTokenize(pFts, 422 + sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb 423 + ); 424 + if( rc!=SQLITE_OK ) break; 425 + rc = pApi->xColumnSize(pFts, i, &nDocsize); 426 + if( rc!=SQLITE_OK ) break; 427 + 428 + for(ii=0; rc==SQLITE_OK && ii<nInst; ii++){ 429 + int ip, ic, io; 430 + int iAdj; 431 + int nScore; 432 + int jj; 433 + 434 + rc = pApi->xInst(pFts, ii, &ip, &ic, &io); 435 + if( ic!=i || rc!=SQLITE_OK ) continue; 436 + memset(aSeen, 0, nPhrase); 437 + rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, 438 + io, nToken, &nScore, &iAdj 439 + ); 440 + if( rc==SQLITE_OK && nScore>nBestScore ){ 441 + nBestScore = nScore; 442 + iBestCol = i; 443 + iBestStart = iAdj; 444 + nColSize = nDocsize; 316 445 } 317 - } 318 446 319 - if( rc==SQLITE_OK && nScore>nBestScore ){ 320 - iBestCol = iSnippetCol; 321 - iBestStart = iStart; 322 - iBestLast = iLast; 323 - nBestScore = nScore; 447 + if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){ 448 + for(jj=0; jj<(sFinder.nFirst-1); jj++){ 449 + if( sFinder.aFirst[jj+1]>io ) break; 450 + } 451 + 452 + if( sFinder.aFirst[jj]<io ){ 453 + memset(aSeen, 0, nPhrase); 454 + rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, 455 + sFinder.aFirst[jj], nToken, &nScore, 0 456 + ); 457 + 458 + nScore += (sFinder.aFirst[jj]==0 ? 120 : 100); 459 + if( rc==SQLITE_OK && nScore>nBestScore ){ 460 + nBestScore = nScore; 461 + iBestCol = i; 462 + iBestStart = sFinder.aFirst[jj]; 463 + nColSize = nDocsize; 464 + } 465 + } 466 + } 324 467 } 325 468 } 326 469 } 327 470 328 471 if( rc==SQLITE_OK ){ 329 - rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); 472 + rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn); 330 473 } 331 - if( rc==SQLITE_OK ){ 332 - rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn); 474 + if( rc==SQLITE_OK && nColSize==0 ){ 475 + rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); 333 476 } 334 477 if( ctx.zIn ){ 335 478 if( rc==SQLITE_OK ){ 336 479 rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter); 337 480 } 338 481 339 - if( (iBestStart+nToken-1)>iBestLast ){ 340 - iBestStart -= (iBestStart+nToken-1-iBestLast) / 2; 341 - } 342 - if( iBestStart+nToken>nColSize ){ 343 - iBestStart = nColSize - nToken; 344 - } 345 - if( iBestStart<0 ) iBestStart = 0; 346 - 347 482 ctx.iRangeStart = iBestStart; 348 483 ctx.iRangeEnd = iBestStart + nToken - 1; 349 484 350 485 if( iBestStart>0 ){ 351 486 fts5HighlightAppend(&rc, &ctx, zEllips, -1); 352 487 } 488 + 489 + /* Advance iterator ctx.iter so that it points to the first coalesced 490 + ** phrase instance at or following position iBestStart. */ 491 + while( ctx.iter.iStart>=0 && ctx.iter.iStart<iBestStart && rc==SQLITE_OK ){ 492 + rc = fts5CInstIterNext(&ctx.iter); 493 + } 494 + 353 495 if( rc==SQLITE_OK ){ 354 496 rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); 355 497 } 356 498 if( ctx.iRangeEnd>=(nColSize-1) ){ 357 499 fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); 358 500 }else{ 359 501 fts5HighlightAppend(&rc, &ctx, zEllips, -1); 360 502 } 361 - 362 - if( rc==SQLITE_OK ){ 363 - sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); 364 - }else{ 365 - sqlite3_result_error_code(pCtx, rc); 366 - } 367 - sqlite3_free(ctx.zOut); 503 + } 504 + if( rc==SQLITE_OK ){ 505 + sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); 506 + }else{ 507 + sqlite3_result_error_code(pCtx, rc); 368 508 } 509 + sqlite3_free(ctx.zOut); 369 510 sqlite3_free(aSeen); 511 + sqlite3_free(sFinder.aFirst); 370 512 } 371 513 372 514 /************************************************************************/ 373 515 374 516 /* 375 517 ** The first time the bm25() function is called for a query, an instance 376 518 ** of the following structure is allocated and populated.
Changes to ext/fts5/fts5_expr.c.
163 163 case ')': tok = FTS5_RP; break; 164 164 case '{': tok = FTS5_LCP; break; 165 165 case '}': tok = FTS5_RCP; break; 166 166 case ':': tok = FTS5_COLON; break; 167 167 case ',': tok = FTS5_COMMA; break; 168 168 case '+': tok = FTS5_PLUS; break; 169 169 case '*': tok = FTS5_STAR; break; 170 + case '-': tok = FTS5_MINUS; break; 170 171 case '\0': tok = FTS5_EOF; break; 171 172 172 173 case '"': { 173 174 const char *z2; 174 175 tok = FTS5_STRING; 175 176 176 177 for(z2=&z[1]; 1; z2++){ ................................................................................ 741 742 } 742 743 743 744 744 745 /* 745 746 ** Initialize all term iterators in the pNear object. If any term is found 746 747 ** to match no documents at all, return immediately without initializing any 747 748 ** further iterators. 749 +** 750 +** If an error occurs, return an SQLite error code. Otherwise, return 751 +** SQLITE_OK. It is not considered an error if some term matches zero 752 +** documents. 748 753 */ 749 754 static int fts5ExprNearInitAll( 750 755 Fts5Expr *pExpr, 751 756 Fts5ExprNode *pNode 752 757 ){ 753 758 Fts5ExprNearset *pNear = pNode->pNear; 754 - int i, j; 755 - int rc = SQLITE_OK; 759 + int i; 756 760 757 761 assert( pNode->bNomatch==0 ); 758 - for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){ 762 + for(i=0; i<pNear->nPhrase; i++){ 759 763 Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; 760 - for(j=0; j<pPhrase->nTerm; j++){ 761 - Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; 762 - Fts5ExprTerm *p; 763 - int bEof = 1; 764 - 765 - for(p=pTerm; p && rc==SQLITE_OK; p=p->pSynonym){ 766 - if( p->pIter ){ 767 - sqlite3Fts5IterClose(p->pIter); 768 - p->pIter = 0; 769 - } 770 - rc = sqlite3Fts5IndexQuery( 771 - pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm), 772 - (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | 773 - (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), 774 - pNear->pColset, 775 - &p->pIter 776 - ); 777 - assert( rc==SQLITE_OK || p->pIter==0 ); 778 - if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){ 779 - bEof = 0; 780 - } 781 - } 782 - 783 - if( bEof ){ 784 - pNode->bEof = 1; 785 - return rc; 786 - } 787 - } 788 - } 789 - 790 - return rc; 764 + if( pPhrase->nTerm==0 ){ 765 + pNode->bEof = 1; 766 + return SQLITE_OK; 767 + }else{ 768 + int j; 769 + for(j=0; j<pPhrase->nTerm; j++){ 770 + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; 771 + Fts5ExprTerm *p; 772 + int bHit = 0; 773 + 774 + for(p=pTerm; p; p=p->pSynonym){ 775 + int rc; 776 + if( p->pIter ){ 777 + sqlite3Fts5IterClose(p->pIter); 778 + p->pIter = 0; 779 + } 780 + rc = sqlite3Fts5IndexQuery( 781 + pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm), 782 + (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | 783 + (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), 784 + pNear->pColset, 785 + &p->pIter 786 + ); 787 + assert( (rc==SQLITE_OK)==(p->pIter!=0) ); 788 + if( rc!=SQLITE_OK ) return rc; 789 + if( 0==sqlite3Fts5IterEof(p->pIter) ){ 790 + bHit = 1; 791 + } 792 + } 793 + 794 + if( bHit==0 ){ 795 + pNode->bEof = 1; 796 + return SQLITE_OK; 797 + } 798 + } 799 + } 800 + } 801 + 802 + pNode->bEof = 0; 803 + return SQLITE_OK; 791 804 } 792 805 793 806 /* 794 807 ** If pExpr is an ASC iterator, this function returns a value with the 795 808 ** same sign as: 796 809 ** 797 810 ** (iLhs - iRhs) ................................................................................ 916 929 if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){ 917 930 pNode->bNomatch = 0; 918 931 pNode->bEof = 1; 919 932 return rc; 920 933 } 921 934 }else{ 922 935 Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; 923 - if( pIter->iRowid==iLast ) continue; 936 + if( pIter->iRowid==iLast || pIter->bEof ) continue; 924 937 bMatch = 0; 925 938 if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ 926 939 return rc; 927 940 } 928 941 } 929 942 } 930 943 } ................................................................................ 1093 1106 Fts5ExprNode *p1 = pNode->apChild[i]; 1094 1107 assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 ); 1095 1108 if( p1->bEof==0 ){ 1096 1109 if( (p1->iRowid==iLast) 1097 1110 || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0) 1098 1111 ){ 1099 1112 int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom); 1100 - if( rc!=SQLITE_OK ) return rc; 1113 + if( rc!=SQLITE_OK ){ 1114 + pNode->bNomatch = 0; 1115 + return rc; 1116 + } 1101 1117 } 1102 1118 } 1103 1119 } 1104 1120 1105 1121 fts5ExprNodeTest_OR(pExpr, pNode); 1106 1122 return SQLITE_OK; 1107 1123 } ................................................................................ 1124 1140 bMatch = 1; 1125 1141 for(iChild=0; iChild<pAnd->nChild; iChild++){ 1126 1142 Fts5ExprNode *pChild = pAnd->apChild[iChild]; 1127 1143 int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid); 1128 1144 if( cmp>0 ){ 1129 1145 /* Advance pChild until it points to iLast or laster */ 1130 1146 rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast); 1131 - if( rc!=SQLITE_OK ) return rc; 1147 + if( rc!=SQLITE_OK ){ 1148 + pAnd->bNomatch = 0; 1149 + return rc; 1150 + } 1132 1151 } 1133 1152 1134 1153 /* If the child node is now at EOF, so is the parent AND node. Otherwise, 1135 1154 ** the child node is guaranteed to have advanced at least as far as 1136 1155 ** rowid iLast. So if it is not at exactly iLast, pChild->iRowid is the 1137 1156 ** new lastest rowid seen so far. */ 1138 1157 assert( pChild->bEof || fts5RowidCmp(pExpr, iLast, pChild->iRowid)<=0 ); ................................................................................ 1163 1182 Fts5ExprNode *pNode, 1164 1183 int bFromValid, 1165 1184 i64 iFrom 1166 1185 ){ 1167 1186 int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); 1168 1187 if( rc==SQLITE_OK ){ 1169 1188 rc = fts5ExprNodeTest_AND(pExpr, pNode); 1189 + }else{ 1190 + pNode->bNomatch = 0; 1170 1191 } 1171 1192 return rc; 1172 1193 } 1173 1194 1174 1195 static int fts5ExprNodeTest_NOT( 1175 1196 Fts5Expr *pExpr, /* Expression pPhrase belongs to */ 1176 1197 Fts5ExprNode *pNode /* FTS5_NOT node to advance */ ................................................................................ 1205 1226 int bFromValid, 1206 1227 i64 iFrom 1207 1228 ){ 1208 1229 int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); 1209 1230 if( rc==SQLITE_OK ){ 1210 1231 rc = fts5ExprNodeTest_NOT(pExpr, pNode); 1211 1232 } 1233 + if( rc!=SQLITE_OK ){ 1234 + pNode->bNomatch = 0; 1235 + } 1212 1236 return rc; 1213 1237 } 1214 1238 1215 1239 /* 1216 1240 ** If pNode currently points to a match, this function returns SQLITE_OK 1217 1241 ** without modifying it. Otherwise, pNode is advanced until it does point 1218 1242 ** to a match or EOF is reached. ................................................................................ 1327 1351 1328 1352 p->pIndex = pIdx; 1329 1353 p->bDesc = bDesc; 1330 1354 rc = fts5ExprNodeFirst(p, pRoot); 1331 1355 1332 1356 /* If not at EOF but the current rowid occurs earlier than iFirst in 1333 1357 ** the iteration order, move to document iFirst or later. */ 1334 - if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){ 1358 + if( rc==SQLITE_OK 1359 + && 0==pRoot->bEof 1360 + && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 1361 + ){ 1335 1362 rc = fts5ExprNodeNext(p, pRoot, 1, iFirst); 1336 1363 } 1337 1364 1338 1365 /* If the iterator is not at a real match, skip forward until it is. */ 1339 1366 while( pRoot->bNomatch ){ 1340 1367 assert( pRoot->bEof==0 && rc==SQLITE_OK ); 1341 1368 rc = fts5ExprNodeNext(p, pRoot, 0, 0); ................................................................................ 1489 1516 TokenCtx *pCtx = (TokenCtx*)pContext; 1490 1517 Fts5ExprPhrase *pPhrase = pCtx->pPhrase; 1491 1518 1492 1519 UNUSED_PARAM2(iUnused1, iUnused2); 1493 1520 1494 1521 /* If an error has already occurred, this is a no-op */ 1495 1522 if( pCtx->rc!=SQLITE_OK ) return pCtx->rc; 1523 + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; 1496 1524 1497 1525 if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED) ){ 1498 1526 Fts5ExprTerm *pSyn; 1499 1527 int nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1; 1500 1528 pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte); 1501 1529 if( pSyn==0 ){ 1502 1530 rc = SQLITE_NOMEM; ................................................................................ 1580 1608 char *z = 0; 1581 1609 1582 1610 memset(&sCtx, 0, sizeof(TokenCtx)); 1583 1611 sCtx.pPhrase = pAppend; 1584 1612 1585 1613 rc = fts5ParseStringFromToken(pToken, &z); 1586 1614 if( rc==SQLITE_OK ){ 1587 - int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0); 1615 + int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0); 1588 1616 int n; 1589 1617 sqlite3Fts5Dequote(z); 1590 1618 n = (int)strlen(z); 1591 1619 rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize); 1592 1620 } 1593 1621 sqlite3_free(z); 1594 1622 if( rc || (rc = sCtx.rc) ){ ................................................................................ 1632 1660 int sqlite3Fts5ExprClonePhrase( 1633 1661 Fts5Expr *pExpr, 1634 1662 int iPhrase, 1635 1663 Fts5Expr **ppNew 1636 1664 ){ 1637 1665 int rc = SQLITE_OK; /* Return code */ 1638 1666 Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ 1639 - int i; /* Used to iterate through phrase terms */ 1640 1667 Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ 1641 1668 TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */ 1642 1669 1643 1670 pOrig = pExpr->apExprPhrase[iPhrase]; 1644 1671 pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); 1645 1672 if( rc==SQLITE_OK ){ 1646 1673 pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, ................................................................................ 1650 1677 pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, 1651 1678 sizeof(Fts5ExprNode)); 1652 1679 } 1653 1680 if( rc==SQLITE_OK ){ 1654 1681 pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, 1655 1682 sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); 1656 1683 } 1657 - 1658 - for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){ 1659 - int tflags = 0; 1660 - Fts5ExprTerm *p; 1661 - for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ 1662 - const char *zTerm = p->zTerm; 1663 - rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm), 1664 - 0, 0); 1665 - tflags = FTS5_TOKEN_COLOCATED; 1666 - } 1667 - if( rc==SQLITE_OK ){ 1668 - sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; 1669 - } 1684 + if( rc==SQLITE_OK ){ 1685 + Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset; 1686 + if( pColsetOrig ){ 1687 + int nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int); 1688 + Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte); 1689 + if( pColset ){ 1690 + memcpy(pColset, pColsetOrig, nByte); 1691 + } 1692 + pNew->pRoot->pNear->pColset = pColset; 1693 + } 1694 + } 1695 + 1696 + if( pOrig->nTerm ){ 1697 + int i; /* Used to iterate through phrase terms */ 1698 + for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){ 1699 + int tflags = 0; 1700 + Fts5ExprTerm *p; 1701 + for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ 1702 + const char *zTerm = p->zTerm; 1703 + rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm), 1704 + 0, 0); 1705 + tflags = FTS5_TOKEN_COLOCATED; 1706 + } 1707 + if( rc==SQLITE_OK ){ 1708 + sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; 1709 + } 1710 + } 1711 + }else{ 1712 + /* This happens when parsing a token or quoted phrase that contains 1713 + ** no token characters at all. (e.g ... MATCH '""'). */ 1714 + sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase)); 1670 1715 } 1671 1716 1672 1717 if( rc==SQLITE_OK ){ 1673 1718 /* All the allocations succeeded. Put the expression object together. */ 1674 1719 pNew->pIndex = pExpr->pIndex; 1675 1720 pNew->pConfig = pExpr->pConfig; 1676 1721 pNew->nPhrase = 1; ................................................................................ 1776 1821 /* Check that the array is in order and contains no duplicate entries. */ 1777 1822 for(i=1; i<pNew->nCol; i++) assert( pNew->aiCol[i]>pNew->aiCol[i-1] ); 1778 1823 #endif 1779 1824 } 1780 1825 1781 1826 return pNew; 1782 1827 } 1828 + 1829 +/* 1830 +** Allocate and return an Fts5Colset object specifying the inverse of 1831 +** the colset passed as the second argument. Free the colset passed 1832 +** as the second argument before returning. 1833 +*/ 1834 +Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p){ 1835 + Fts5Colset *pRet; 1836 + int nCol = pParse->pConfig->nCol; 1837 + 1838 + pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc, 1839 + sizeof(Fts5Colset) + sizeof(int)*nCol 1840 + ); 1841 + if( pRet ){ 1842 + int i; 1843 + int iOld = 0; 1844 + for(i=0; i<nCol; i++){ 1845 + if( iOld>=p->nCol || p->aiCol[iOld]!=i ){ 1846 + pRet->aiCol[pRet->nCol++] = i; 1847 + }else{ 1848 + iOld++; 1849 + } 1850 + } 1851 + } 1852 + 1853 + sqlite3_free(p); 1854 + return pRet; 1855 +} 1783 1856 1784 1857 Fts5Colset *sqlite3Fts5ParseColset( 1785 1858 Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */ 1786 1859 Fts5Colset *pColset, /* Existing colset object */ 1787 1860 Fts5Token *p 1788 1861 ){ 1789 1862 Fts5Colset *pRet = 0; ................................................................................ 2491 2564 ){ 2492 2565 Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx; 2493 2566 Fts5Expr *pExpr = p->pExpr; 2494 2567 int i; 2495 2568 2496 2569 UNUSED_PARAM2(iUnused1, iUnused2); 2497 2570 2571 + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; 2498 2572 if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; 2499 2573 for(i=0; i<pExpr->nPhrase; i++){ 2500 2574 Fts5ExprTerm *pTerm; 2501 2575 if( p->aPopulator[i].bOk==0 ) continue; 2502 2576 for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ 2503 - int nTerm = strlen(pTerm->zTerm); 2577 + int nTerm = (int)strlen(pTerm->zTerm); 2504 2578 if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix)) 2505 2579 && memcmp(pTerm->zTerm, pToken, nTerm)==0 2506 2580 ){ 2507 2581 int rc = sqlite3Fts5PoslistWriterAppend( 2508 2582 &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff 2509 2583 ); 2510 2584 if( rc ) return rc; ................................................................................ 2600 2674 return 1; 2601 2675 } 2602 2676 2603 2677 void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){ 2604 2678 fts5ExprCheckPoslists(pExpr->pRoot, iRowid); 2605 2679 } 2606 2680 2607 -static void fts5ExprClearEof(Fts5ExprNode *pNode){ 2608 - int i; 2609 - for(i=0; i<pNode->nChild; i++){ 2610 - fts5ExprClearEof(pNode->apChild[i]); 2611 - } 2612 - pNode->bEof = 0; 2613 -} 2614 -void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){ 2615 - fts5ExprClearEof(pExpr->pRoot); 2616 -} 2617 - 2618 2681 /* 2619 2682 ** This function is only called for detail=columns tables. 2620 2683 */ 2621 2684 int sqlite3Fts5ExprPhraseCollist( 2622 2685 Fts5Expr *pExpr, 2623 2686 int iPhrase, 2624 2687 const u8 **ppCollist,
Changes to ext/fts5/fts5_hash.c.
341 341 }else{ 342 342 /* Append a new column value, if necessary */ 343 343 assert( iCol>=p->iCol ); 344 344 if( iCol!=p->iCol ){ 345 345 if( pHash->eDetail==FTS5_DETAIL_FULL ){ 346 346 pPtr[p->nData++] = 0x01; 347 347 p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); 348 - p->iCol = iCol; 348 + p->iCol = (i16)iCol; 349 349 p->iPos = 0; 350 350 }else{ 351 351 bNew = 1; 352 - p->iCol = iPos = iCol; 352 + p->iCol = (i16)(iPos = iCol); 353 353 } 354 354 } 355 355 356 356 /* Append the new position offset, if necessary */ 357 357 if( bNew ){ 358 358 p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2); 359 359 p->iPos = iPos;
Changes to ext/fts5/fts5_index.c.
300 300 sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ 301 301 sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */ 302 302 sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */ 303 303 sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */ 304 304 sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */ 305 305 sqlite3_stmt *pIdxSelect; 306 306 int nRead; /* Total number of blocks read */ 307 + 308 + sqlite3_stmt *pDataVersion; 309 + i64 iStructVersion; /* data_version when pStruct read */ 310 + Fts5Structure *pStruct; /* Current db structure (or NULL) */ 307 311 }; 308 312 309 313 struct Fts5DoclistIter { 310 314 u8 *aEof; /* Pointer to 1 byte past end of doclist */ 311 315 312 316 /* Output variables. aPoslist==0 at EOF */ 313 317 i64 iRowid; ................................................................................ 694 698 p->nRead++; 695 699 } 696 700 697 701 assert( (pRet==0)==(p->rc!=SQLITE_OK) ); 698 702 return pRet; 699 703 } 700 704 701 - 702 705 /* 703 706 ** Release a reference to data record returned by an earlier call to 704 707 ** fts5DataRead(). 705 708 */ 706 709 static void fts5DataRelease(Fts5Data *pData){ 707 710 sqlite3_free(pData); 708 711 } 712 + 713 +static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){ 714 + Fts5Data *pRet = fts5DataRead(p, iRowid); 715 + if( pRet ){ 716 + if( pRet->szLeaf>pRet->nn ){ 717 + p->rc = FTS5_CORRUPT; 718 + fts5DataRelease(pRet); 719 + pRet = 0; 720 + } 721 + } 722 + return pRet; 723 +} 709 724 710 725 static int fts5IndexPrepareStmt( 711 726 Fts5Index *p, 712 727 sqlite3_stmt **ppStmt, 713 728 char *zSql 714 729 ){ 715 730 if( p->rc==SQLITE_OK ){ ................................................................................ 954 969 } 955 970 pLvl->aSeg = aNew; 956 971 }else{ 957 972 *pRc = SQLITE_NOMEM; 958 973 } 959 974 } 960 975 } 976 + 977 +static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){ 978 + Fts5Structure *pRet = 0; 979 + Fts5Config *pConfig = p->pConfig; 980 + int iCookie; /* Configuration cookie */ 981 + Fts5Data *pData; 982 + 983 + pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); 984 + if( p->rc==SQLITE_OK ){ 985 + /* TODO: Do we need this if the leaf-index is appended? Probably... */ 986 + memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); 987 + p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); 988 + if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ 989 + p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); 990 + } 991 + fts5DataRelease(pData); 992 + if( p->rc!=SQLITE_OK ){ 993 + fts5StructureRelease(pRet); 994 + pRet = 0; 995 + } 996 + } 997 + 998 + return pRet; 999 +} 1000 + 1001 +static i64 fts5IndexDataVersion(Fts5Index *p){ 1002 + i64 iVersion = 0; 1003 + 1004 + if( p->rc==SQLITE_OK ){ 1005 + if( p->pDataVersion==0 ){ 1006 + p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion, 1007 + sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb) 1008 + ); 1009 + if( p->rc ) return 0; 1010 + } 1011 + 1012 + if( SQLITE_ROW==sqlite3_step(p->pDataVersion) ){ 1013 + iVersion = sqlite3_column_int64(p->pDataVersion, 0); 1014 + } 1015 + p->rc = sqlite3_reset(p->pDataVersion); 1016 + } 1017 + 1018 + return iVersion; 1019 +} 961 1020 962 1021 /* 963 1022 ** Read, deserialize and return the structure record. 964 1023 ** 965 1024 ** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array 966 1025 ** are over-allocated as described for function fts5StructureDecode() 967 1026 ** above. 968 1027 ** 969 1028 ** If an error occurs, NULL is returned and an error code left in the 970 1029 ** Fts5Index handle. If an error has already occurred when this function 971 1030 ** is called, it is a no-op. 972 1031 */ 973 1032 static Fts5Structure *fts5StructureRead(Fts5Index *p){ 974 - Fts5Config *pConfig = p->pConfig; 975 - Fts5Structure *pRet = 0; /* Object to return */ 976 - int iCookie; /* Configuration cookie */ 977 - Fts5Data *pData; 978 - 979 - pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); 980 - if( p->rc ) return 0; 981 - /* TODO: Do we need this if the leaf-index is appended? Probably... */ 982 - memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); 983 - p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); 984 - if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ 985 - p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); 986 - } 987 - 988 - fts5DataRelease(pData); 989 - if( p->rc!=SQLITE_OK ){ 990 - fts5StructureRelease(pRet); 991 - pRet = 0; 992 - } 993 - return pRet; 1033 + 1034 + if( p->pStruct==0 ){ 1035 + p->iStructVersion = fts5IndexDataVersion(p); 1036 + if( p->rc==SQLITE_OK ){ 1037 + p->pStruct = fts5StructureReadUncached(p); 1038 + } 1039 + } 1040 + 1041 +#if 0 1042 + else{ 1043 + Fts5Structure *pTest = fts5StructureReadUncached(p); 1044 + if( pTest ){ 1045 + int i, j; 1046 + assert_nc( p->pStruct->nSegment==pTest->nSegment ); 1047 + assert_nc( p->pStruct->nLevel==pTest->nLevel ); 1048 + for(i=0; i<pTest->nLevel; i++){ 1049 + assert_nc( p->pStruct->aLevel[i].nMerge==pTest->aLevel[i].nMerge ); 1050 + assert_nc( p->pStruct->aLevel[i].nSeg==pTest->aLevel[i].nSeg ); 1051 + for(j=0; j<pTest->aLevel[i].nSeg; j++){ 1052 + Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j]; 1053 + Fts5StructureSegment *p2 = &p->pStruct->aLevel[i].aSeg[j]; 1054 + assert_nc( p1->iSegid==p2->iSegid ); 1055 + assert_nc( p1->pgnoFirst==p2->pgnoFirst ); 1056 + assert_nc( p1->pgnoLast==p2->pgnoLast ); 1057 + } 1058 + } 1059 + fts5StructureRelease(pTest); 1060 + } 1061 + } 1062 +#endif 1063 + 1064 + if( p->rc!=SQLITE_OK ) return 0; 1065 + assert( p->iStructVersion!=0 ); 1066 + assert( p->pStruct!=0 ); 1067 + fts5StructureRef(p->pStruct); 1068 + return p->pStruct; 1069 +} 1070 + 1071 +static void fts5StructureInvalidate(Fts5Index *p){ 1072 + if( p->pStruct ){ 1073 + fts5StructureRelease(p->pStruct); 1074 + p->pStruct = 0; 1075 + } 994 1076 } 995 1077 996 1078 /* 997 1079 ** Return the total number of segments in index structure pStruct. This 998 1080 ** function is only ever used as part of assert() conditions. 999 1081 */ 1000 1082 #ifdef SQLITE_DEBUG ................................................................................ 1444 1526 Fts5StructureSegment *pSeg = pIter->pSeg; 1445 1527 fts5DataRelease(pIter->pLeaf); 1446 1528 pIter->iLeafPgno++; 1447 1529 if( pIter->pNextLeaf ){ 1448 1530 pIter->pLeaf = pIter->pNextLeaf; 1449 1531 pIter->pNextLeaf = 0; 1450 1532 }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){ 1451 - pIter->pLeaf = fts5DataRead(p, 1533 + pIter->pLeaf = fts5LeafRead(p, 1452 1534 FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno) 1453 1535 ); 1454 1536 }else{ 1455 1537 pIter->pLeaf = 0; 1456 1538 } 1457 1539 pLeaf = pIter->pLeaf; 1458 1540 ................................................................................ 1947 2029 if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOff<pLeaf->szLeaf ){ 1948 2030 iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); 1949 2031 pIter->iLeafOffset = iOff; 1950 2032 1951 2033 if( pLeaf->nn>pLeaf->szLeaf ){ 1952 2034 pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( 1953 2035 &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist 1954 - ); 2036 + ); 1955 2037 } 1956 - 1957 2038 } 1958 2039 else if( pLeaf->nn>pLeaf->szLeaf ){ 1959 2040 pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( 1960 2041 &pLeaf->p[pLeaf->szLeaf], iOff 1961 - ); 2042 + ); 1962 2043 pIter->iLeafOffset = iOff; 1963 2044 pIter->iEndofDoclist = iOff; 1964 2045 bNewTerm = 1; 1965 2046 } 1966 2047 assert_nc( iOff<pLeaf->szLeaf ); 1967 2048 if( iOff>pLeaf->szLeaf ){ 1968 2049 p->rc = FTS5_CORRUPT; ................................................................................ 1988 2069 ** code is inlined. 1989 2070 ** 1990 2071 ** Later: Switched back to fts5SegIterLoadNPos() because it supports 1991 2072 ** detail=none mode. Not ideal. 1992 2073 */ 1993 2074 int nSz; 1994 2075 assert( p->rc==SQLITE_OK ); 2076 + assert( pIter->iLeafOffset<=pIter->pLeaf->nn ); 1995 2077 fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); 1996 2078 pIter->bDel = (nSz & 0x0001); 1997 2079 pIter->nPos = nSz>>1; 1998 2080 assert_nc( pIter->nPos>=0 ); 1999 2081 } 2000 2082 } 2001 2083 } ................................................................................ 2193 2275 bEndOfPage = 1; 2194 2276 break; 2195 2277 } 2196 2278 2197 2279 iPgidx += fts5GetVarint32(&a[iPgidx], nKeep); 2198 2280 iTermOff += nKeep; 2199 2281 iOff = iTermOff; 2282 + 2283 + if( iOff>=n ){ 2284 + p->rc = FTS5_CORRUPT; 2285 + return; 2286 + } 2200 2287 2201 2288 /* Read the nKeep field of the next term. */ 2202 2289 fts5FastGetVarint32(a, iOff, nKeep); 2203 2290 } 2204 2291 2205 2292 search_failed: 2206 2293 if( bGe==0 ){ ................................................................................ 2245 2332 pIter->iEndofDoclist = iTermOff + nExtra; 2246 2333 } 2247 2334 pIter->iPgidxOff = iPgidx; 2248 2335 2249 2336 fts5SegIterLoadRowid(p, pIter); 2250 2337 fts5SegIterLoadNPos(p, pIter); 2251 2338 } 2339 + 2340 +static sqlite3_stmt *fts5IdxSelectStmt(Fts5Index *p){ 2341 + if( p->pIdxSelect==0 ){ 2342 + Fts5Config *pConfig = p->pConfig; 2343 + fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf( 2344 + "SELECT pgno FROM '%q'.'%q_idx' WHERE " 2345 + "segid=? AND term<=? ORDER BY term DESC LIMIT 1", 2346 + pConfig->zDb, pConfig->zName 2347 + )); 2348 + } 2349 + return p->pIdxSelect; 2350 +} 2252 2351 2253 2352 /* 2254 2353 ** Initialize the object pIter to point to term pTerm/nTerm within segment 2255 2354 ** pSeg. If there is no such term in the index, the iterator is set to EOF. 2256 2355 ** 2257 2356 ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If 2258 2357 ** an error has already occurred when this function is called, it is a no-op. ................................................................................ 2263 2362 int flags, /* Mask of FTS5INDEX_XXX flags */ 2264 2363 Fts5StructureSegment *pSeg, /* Description of segment */ 2265 2364 Fts5SegIter *pIter /* Object to populate */ 2266 2365 ){ 2267 2366 int iPg = 1; 2268 2367 int bGe = (flags & FTS5INDEX_QUERY_SCAN); 2269 2368 int bDlidx = 0; /* True if there is a doclist-index */ 2369 + sqlite3_stmt *pIdxSelect = 0; 2270 2370 2271 2371 assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); 2272 2372 assert( pTerm && nTerm ); 2273 2373 memset(pIter, 0, sizeof(*pIter)); 2274 2374 pIter->pSeg = pSeg; 2275 2375 2276 2376 /* This block sets stack variable iPg to the leaf page number that may 2277 2377 ** contain term (pTerm/nTerm), if it is present in the segment. */ 2278 - if( p->pIdxSelect==0 ){ 2279 - Fts5Config *pConfig = p->pConfig; 2280 - fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf( 2281 - "SELECT pgno FROM '%q'.'%q_idx' WHERE " 2282 - "segid=? AND term<=? ORDER BY term DESC LIMIT 1", 2283 - pConfig->zDb, pConfig->zName 2284 - )); 2285 - } 2378 + pIdxSelect = fts5IdxSelectStmt(p); 2286 2379 if( p->rc ) return; 2287 - sqlite3_bind_int(p->pIdxSelect, 1, pSeg->iSegid); 2288 - sqlite3_bind_blob(p->pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC); 2289 - if( SQLITE_ROW==sqlite3_step(p->pIdxSelect) ){ 2290 - i64 val = sqlite3_column_int(p->pIdxSelect, 0); 2380 + sqlite3_bind_int(pIdxSelect, 1, pSeg->iSegid); 2381 + sqlite3_bind_blob(pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC); 2382 + if( SQLITE_ROW==sqlite3_step(pIdxSelect) ){ 2383 + i64 val = sqlite3_column_int(pIdxSelect, 0); 2291 2384 iPg = (int)(val>>1); 2292 2385 bDlidx = (val & 0x0001); 2293 2386 } 2294 - p->rc = sqlite3_reset(p->pIdxSelect); 2387 + p->rc = sqlite3_reset(pIdxSelect); 2295 2388 2296 2389 if( iPg<pSeg->pgnoFirst ){ 2297 2390 iPg = pSeg->pgnoFirst; 2298 2391 bDlidx = 0; 2299 2392 } 2300 2393 2301 2394 pIter->iLeafPgno = iPg - 1; ................................................................................ 2744 2837 static void fts5MultiIterNext( 2745 2838 Fts5Index *p, 2746 2839 Fts5Iter *pIter, 2747 2840 int bFrom, /* True if argument iFrom is valid */ 2748 2841 i64 iFrom /* Advance at least as far as this */ 2749 2842 ){ 2750 2843 int bUseFrom = bFrom; 2844 + assert( pIter->base.bEof==0 ); 2751 2845 while( p->rc==SQLITE_OK ){ 2752 2846 int iFirst = pIter->aFirst[1].iFirst; 2753 2847 int bNewTerm = 0; 2754 2848 Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; 2755 2849 assert( p->rc==SQLITE_OK ); 2756 2850 if( bUseFrom && pSeg->pDlidx ){ 2757 2851 fts5SegIterNextFrom(p, pSeg, iFrom); ................................................................................ 2970 3064 xChunk(p, pCtx, pChunk, nChunk); 2971 3065 nRem -= nChunk; 2972 3066 fts5DataRelease(pData); 2973 3067 if( nRem<=0 ){ 2974 3068 break; 2975 3069 }else{ 2976 3070 pgno++; 2977 - pData = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno)); 3071 + pData = fts5LeafRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno)); 2978 3072 if( pData==0 ) break; 2979 3073 pChunk = &pData->p[4]; 2980 3074 nChunk = MIN(nRem, pData->szLeaf - 4); 2981 3075 if( pgno==pgnoSave ){ 2982 3076 assert( pSeg->pNextLeaf==0 ); 2983 3077 pSeg->pNextLeaf = pData; 2984 3078 pData = 0; ................................................................................ 3113 3207 ** Fts5Iter.poslist buffer and then set the output pointer to point 3114 3208 ** to this buffer. */ 3115 3209 fts5BufferZero(&pIter->poslist); 3116 3210 fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist); 3117 3211 pIter->base.pData = pIter->poslist.p; 3118 3212 } 3119 3213 } 3214 + 3215 +/* 3216 +** xSetOutputs callback used when the Fts5Colset object has nCol==0 (match 3217 +** against no columns at all). 3218 +*/ 3219 +static void fts5IterSetOutputs_ZeroColset(Fts5Iter *pIter, Fts5SegIter *pSeg){ 3220 + UNUSED_PARAM(pSeg); 3221 + pIter->base.nData = 0; 3222 +} 3120 3223 3121 3224 /* 3122 3225 ** xSetOutputs callback used by detail=col when there is a column filter 3123 3226 ** and there are 100 or more columns. Also called as a fallback from 3124 3227 ** fts5IterSetOutputs_Col100 if the column-list spans more than one page. 3125 3228 */ 3126 3229 static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){ ................................................................................ 3163 3266 while( a<pEnd ){ 3164 3267 iPrev += (int)a++[0] - 2; 3165 3268 while( *aiCol<iPrev ){ 3166 3269 aiCol++; 3167 3270 if( aiCol==aiColEnd ) goto setoutputs_col_out; 3168 3271 } 3169 3272 if( *aiCol==iPrev ){ 3170 - *aOut++ = (iPrev - iPrevOut) + 2; 3273 + *aOut++ = (u8)((iPrev - iPrevOut) + 2); 3171 3274 iPrevOut = iPrev; 3172 3275 } 3173 3276 } 3174 3277 3175 3278 setoutputs_col_out: 3176 3279 pIter->base.pData = pIter->poslist.p; 3177 3280 pIter->base.nData = aOut - pIter->poslist.p; ................................................................................ 3218 3321 if( pConfig->eDetail==FTS5_DETAIL_NONE ){ 3219 3322 pIter->xSetOutputs = fts5IterSetOutputs_None; 3220 3323 } 3221 3324 3222 3325 else if( pIter->pColset==0 ){ 3223 3326 pIter->xSetOutputs = fts5IterSetOutputs_Nocolset; 3224 3327 } 3328 + 3329 + else if( pIter->pColset->nCol==0 ){ 3330 + pIter->xSetOutputs = fts5IterSetOutputs_ZeroColset; 3331 + } 3225 3332 3226 3333 else if( pConfig->eDetail==FTS5_DETAIL_FULL ){ 3227 3334 pIter->xSetOutputs = fts5IterSetOutputs_Full; 3228 3335 } 3229 3336 3230 3337 else{ 3231 3338 assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS ); ................................................................................ 3449 3556 static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ 3450 3557 int iSegid = 0; 3451 3558 3452 3559 if( p->rc==SQLITE_OK ){ 3453 3560 if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){ 3454 3561 p->rc = SQLITE_FULL; 3455 3562 }else{ 3456 - while( iSegid==0 ){ 3457 - int iLvl, iSeg; 3458 - sqlite3_randomness(sizeof(u32), (void*)&iSegid); 3459 - iSegid = iSegid & ((1 << FTS5_DATA_ID_B)-1); 3460 - for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ 3461 - for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ 3462 - if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){ 3463 - iSegid = 0; 3464 - } 3563 + /* FTS5_MAX_SEGMENT is currently defined as 2000. So the following 3564 + ** array is 63 elements, or 252 bytes, in size. */ 3565 + u32 aUsed[(FTS5_MAX_SEGMENT+31) / 32]; 3566 + int iLvl, iSeg; 3567 + int i; 3568 + u32 mask; 3569 + memset(aUsed, 0, sizeof(aUsed)); 3570 + for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ 3571 + for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ 3572 + int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid; 3573 + if( iId<=FTS5_MAX_SEGMENT ){ 3574 + aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32); 3465 3575 } 3466 3576 } 3467 3577 } 3578 + 3579 + for(i=0; aUsed[i]==0xFFFFFFFF; i++); 3580 + mask = aUsed[i]; 3581 + for(iSegid=0; mask & (1 << iSegid); iSegid++); 3582 + iSegid += 1 + i*32; 3583 + 3584 +#ifdef SQLITE_DEBUG 3585 + for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ 3586 + for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ 3587 + assert( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ); 3588 + } 3589 + } 3590 + assert( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT ); 3591 + 3592 + { 3593 + sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p); 3594 + if( p->rc==SQLITE_OK ){ 3595 + u8 aBlob[2] = {0xff, 0xff}; 3596 + sqlite3_bind_int(pIdxSelect, 1, iSegid); 3597 + sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC); 3598 + assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW ); 3599 + p->rc = sqlite3_reset(pIdxSelect); 3600 + } 3601 + } 3602 +#endif 3468 3603 } 3469 3604 } 3470 3605 3471 3606 return iSegid; 3472 3607 } 3473 3608 3474 3609 /* ................................................................................ 3705 3840 } 3706 3841 } 3707 3842 3708 3843 static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ 3709 3844 static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; 3710 3845 Fts5PageWriter *pPage = &pWriter->writer; 3711 3846 i64 iRowid; 3847 + 3848 +static int nCall = 0; 3849 +nCall++; 3712 3850 3713 3851 assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) ); 3714 3852 3715 3853 /* Set the szLeaf header field. */ 3716 3854 assert( 0==fts5GetU16(&pPage->buf.p[2]) ); 3717 3855 fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n); 3718 3856 ................................................................................ 3905 4043 Fts5PageWriter *pLeaf = &pWriter->writer; 3906 4044 if( p->rc==SQLITE_OK ){ 3907 4045 assert( pLeaf->pgno>=1 ); 3908 4046 if( pLeaf->buf.n>4 ){ 3909 4047 fts5WriteFlushLeaf(p, pWriter); 3910 4048 } 3911 4049 *pnLeaf = pLeaf->pgno-1; 3912 - fts5WriteFlushBtree(p, pWriter); 4050 + if( pLeaf->pgno>1 ){ 4051 + fts5WriteFlushBtree(p, pWriter); 4052 + } 3913 4053 } 3914 4054 fts5BufferFree(&pLeaf->term); 3915 4055 fts5BufferFree(&pLeaf->buf); 3916 4056 fts5BufferFree(&pLeaf->pgidx); 3917 4057 fts5BufferFree(&pWriter->btterm); 3918 4058 3919 4059 for(i=0; i<pWriter->nDlidx; i++){ ................................................................................ 4324 4464 int iSegid; 4325 4465 int pgnoLast = 0; /* Last leaf page number in segment */ 4326 4466 4327 4467 /* Obtain a reference to the index structure and allocate a new segment-id 4328 4468 ** for the new level-0 segment. */ 4329 4469 pStruct = fts5StructureRead(p); 4330 4470 iSegid = fts5AllocateSegid(p, pStruct); 4471 + fts5StructureInvalidate(p); 4331 4472 4332 4473 if( iSegid ){ 4333 4474 const int pgsz = p->pConfig->pgsz; 4334 4475 int eDetail = p->pConfig->eDetail; 4335 4476 Fts5StructureSegment *pSeg; /* New segment within pStruct */ 4336 4477 Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ 4337 4478 Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ ................................................................................ 4508 4649 } 4509 4650 4510 4651 nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); 4511 4652 pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); 4512 4653 4513 4654 if( pNew ){ 4514 4655 Fts5StructureLevel *pLvl; 4515 - int nByte = nSeg * sizeof(Fts5StructureSegment); 4656 + nByte = nSeg * sizeof(Fts5StructureSegment); 4516 4657 pNew->nLevel = pStruct->nLevel+1; 4517 4658 pNew->nRef = 1; 4518 4659 pNew->nWriteCounter = pStruct->nWriteCounter; 4519 4660 pLvl = &pNew->aLevel[pStruct->nLevel]; 4520 4661 pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); 4521 4662 if( pLvl->aSeg ){ 4522 4663 int iLvl, iSeg; ................................................................................ 4543 4684 int sqlite3Fts5IndexOptimize(Fts5Index *p){ 4544 4685 Fts5Structure *pStruct; 4545 4686 Fts5Structure *pNew = 0; 4546 4687 4547 4688 assert( p->rc==SQLITE_OK ); 4548 4689 fts5IndexFlush(p); 4549 4690 pStruct = fts5StructureRead(p); 4691 + fts5StructureInvalidate(p); 4550 4692 4551 4693 if( pStruct ){ 4552 4694 pNew = fts5IndexOptimizeStruct(p, pStruct); 4553 4695 } 4554 4696 fts5StructureRelease(pStruct); 4555 4697 4556 4698 assert( pNew==0 || pNew->nSegment>0 ); ................................................................................ 4573 4715 ** This is called to implement the special "VALUES('merge', $nMerge)" 4574 4716 ** INSERT command. 4575 4717 */ 4576 4718 int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ 4577 4719 Fts5Structure *pStruct = fts5StructureRead(p); 4578 4720 if( pStruct ){ 4579 4721 int nMin = p->pConfig->nUsermerge; 4722 + fts5StructureInvalidate(p); 4580 4723 if( nMerge<0 ){ 4581 4724 Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct); 4582 4725 fts5StructureRelease(pStruct); 4583 4726 pStruct = pNew; 4584 4727 nMin = 2; 4585 4728 nMerge = nMerge*-1; 4586 4729 } ................................................................................ 5000 5143 ** to the database. Additionally, assume that the contents of the %_data 5001 5144 ** table may have changed on disk. So any in-memory caches of %_data 5002 5145 ** records must be invalidated. 5003 5146 */ 5004 5147 int sqlite3Fts5IndexRollback(Fts5Index *p){ 5005 5148 fts5CloseReader(p); 5006 5149 fts5IndexDiscardData(p); 5150 + fts5StructureInvalidate(p); 5007 5151 /* assert( p->rc==SQLITE_OK ); */ 5008 5152 return SQLITE_OK; 5009 5153 } 5010 5154 5011 5155 /* 5012 5156 ** The %_data table is completely empty when this function is called. This 5013 5157 ** function populates it with the initial structure objects for each index, 5014 5158 ** and the initial version of the "averages" record (a zero-byte blob). 5015 5159 */ 5016 5160 int sqlite3Fts5IndexReinit(Fts5Index *p){ 5017 5161 Fts5Structure s; 5162 + fts5StructureInvalidate(p); 5018 5163 memset(&s, 0, sizeof(Fts5Structure)); 5019 5164 fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); 5020 5165 fts5StructureWrite(p, &s); 5021 5166 return fts5IndexReturn(p); 5022 5167 } 5023 5168 5024 5169 /* ................................................................................ 5069 5214 /* 5070 5215 ** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen(). 5071 5216 */ 5072 5217 int sqlite3Fts5IndexClose(Fts5Index *p){ 5073 5218 int rc = SQLITE_OK; 5074 5219 if( p ){ 5075 5220 assert( p->pReader==0 ); 5221 + fts5StructureInvalidate(p); 5076 5222 sqlite3_finalize(p->pWriter); 5077 5223 sqlite3_finalize(p->pDeleter); 5078 5224 sqlite3_finalize(p->pIdxWriter); 5079 5225 sqlite3_finalize(p->pIdxDeleter); 5080 5226 sqlite3_finalize(p->pIdxSelect); 5227 + sqlite3_finalize(p->pDataVersion); 5081 5228 sqlite3Fts5HashFree(p->pHash); 5082 5229 sqlite3_free(p->zDataTbl); 5083 5230 sqlite3_free(p); 5084 5231 } 5085 5232 return rc; 5086 5233 } 5087 5234 ................................................................................ 5679 5826 int iIdxLeaf = sqlite3_column_int(pStmt, 2); 5680 5827 int bIdxDlidx = sqlite3_column_int(pStmt, 3); 5681 5828 5682 5829 /* If the leaf in question has already been trimmed from the segment, 5683 5830 ** ignore this b-tree entry. Otherwise, load it into memory. */ 5684 5831 if( iIdxLeaf<pSeg->pgnoFirst ) continue; 5685 5832 iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, iIdxLeaf); 5686 - pLeaf = fts5DataRead(p, iRow); 5833 + pLeaf = fts5LeafRead(p, iRow); 5687 5834 if( pLeaf==0 ) break; 5688 5835 5689 5836 /* Check that the leaf contains at least one term, and that it is equal 5690 5837 ** to or larger than the split-key in zIdxTerm. Also check that if there 5691 5838 ** is also a rowid pointer within the leaf page header, it points to a 5692 5839 ** location before the term. */ 5693 5840 if( pLeaf->nn<=pLeaf->szLeaf ){ ................................................................................ 6329 6476 if( rc==SQLITE_OK ){ 6330 6477 rc = sqlite3_create_function( 6331 6478 db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0 6332 6479 ); 6333 6480 } 6334 6481 return rc; 6335 6482 } 6483 + 6484 + 6485 +int sqlite3Fts5IndexReset(Fts5Index *p){ 6486 + assert( p->pStruct==0 || p->iStructVersion!=0 ); 6487 + if( fts5IndexDataVersion(p)!=p->iStructVersion ){ 6488 + fts5StructureInvalidate(p); 6489 + } 6490 + return fts5IndexReturn(p); 6491 +}
Changes to ext/fts5/fts5_main.c.
592 592 pInfo->aConstraintUsage[pC->iConsIndex].omit = (unsigned char)pC->omit; 593 593 } 594 594 } 595 595 596 596 pInfo->idxNum = idxFlags; 597 597 return SQLITE_OK; 598 598 } 599 + 600 +static int fts5NewTransaction(Fts5Table *pTab){ 601 + Fts5Cursor *pCsr; 602 + for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ 603 + if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK; 604 + } 605 + return sqlite3Fts5StorageReset(pTab->pStorage); 606 +} 599 607 600 608 /* 601 609 ** Implementation of xOpen method. 602 610 */ 603 611 static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ 604 612 Fts5Table *pTab = (Fts5Table*)pVTab; 605 613 Fts5Config *pConfig = pTab->pConfig; 606 - Fts5Cursor *pCsr; /* New cursor object */ 614 + Fts5Cursor *pCsr = 0; /* New cursor object */ 607 615 int nByte; /* Bytes of space to allocate */ 608 - int rc = SQLITE_OK; /* Return code */ 616 + int rc; /* Return code */ 609 617 610 - nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int); 611 - pCsr = (Fts5Cursor*)sqlite3_malloc(nByte); 612 - if( pCsr ){ 613 - Fts5Global *pGlobal = pTab->pGlobal; 614 - memset(pCsr, 0, nByte); 615 - pCsr->aColumnSize = (int*)&pCsr[1]; 616 - pCsr->pNext = pGlobal->pCsr; 617 - pGlobal->pCsr = pCsr; 618 - pCsr->iCsrId = ++pGlobal->iNextId; 619 - }else{ 620 - rc = SQLITE_NOMEM; 618 + rc = fts5NewTransaction(pTab); 619 + if( rc==SQLITE_OK ){ 620 + nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int); 621 + pCsr = (Fts5Cursor*)sqlite3_malloc(nByte); 622 + if( pCsr ){ 623 + Fts5Global *pGlobal = pTab->pGlobal; 624 + memset(pCsr, 0, nByte); 625 + pCsr->aColumnSize = (int*)&pCsr[1]; 626 + pCsr->pNext = pGlobal->pCsr; 627 + pGlobal->pCsr = pCsr; 628 + pCsr->iCsrId = ++pGlobal->iNextId; 629 + }else{ 630 + rc = SQLITE_NOMEM; 631 + } 621 632 } 622 633 *ppCsr = (sqlite3_vtab_cursor*)pCsr; 623 634 return rc; 624 635 } 625 636 626 637 static int fts5StmtType(Fts5Cursor *pCsr){ 627 638 if( pCsr->ePlan==FTS5_PLAN_SCAN ){ ................................................................................ 1171 1182 assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 ); 1172 1183 assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 ); 1173 1184 assert( pCsr->iLastRowid==LARGEST_INT64 ); 1174 1185 assert( pCsr->iFirstRowid==SMALLEST_INT64 ); 1175 1186 pCsr->ePlan = FTS5_PLAN_SOURCE; 1176 1187 pCsr->pExpr = pTab->pSortCsr->pExpr; 1177 1188 rc = fts5CursorFirst(pTab, pCsr, bDesc); 1178 - sqlite3Fts5ExprClearEof(pCsr->pExpr); 1179 1189 }else if( pMatch ){ 1180 1190 const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); 1181 1191 if( zExpr==0 ) zExpr = ""; 1182 1192 1183 1193 rc = fts5CursorParseRank(pConfig, pCsr, pRank); 1184 1194 if( rc==SQLITE_OK ){ 1185 1195 if( zExpr[0]=='*' ){ ................................................................................ 1574 1584 return rc; 1575 1585 } 1576 1586 1577 1587 /* 1578 1588 ** Implementation of xBegin() method. 1579 1589 */ 1580 1590 static int fts5BeginMethod(sqlite3_vtab *pVtab){ 1581 - UNUSED_PARAM(pVtab); /* Call below is a no-op for NDEBUG builds */ 1582 1591 fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0); 1592 + fts5NewTransaction((Fts5Table*)pVtab); 1583 1593 return SQLITE_OK; 1584 1594 } 1585 1595 1586 1596 /* 1587 1597 ** Implementation of xCommit() method. This is a no-op. The contents of 1588 1598 ** the pending-terms hash-table have already been flushed into the database 1589 1599 ** by fts5SyncMethod().
Changes to ext/fts5/fts5_storage.c.
141 141 if( rc!=SQLITE_OK && pzErrMsg ){ 142 142 *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db)); 143 143 } 144 144 } 145 145 } 146 146 147 147 *ppStmt = p->aStmt[eStmt]; 148 + sqlite3_reset(*ppStmt); 148 149 return rc; 149 150 } 150 151 151 152 152 153 static int fts5ExecPrintf( 153 154 sqlite3 *db, 154 155 char **pzErr, ................................................................................ 242 243 int bWithout, /* True for without rowid */ 243 244 char **pzErr /* OUT: Error message */ 244 245 ){ 245 246 int rc; 246 247 char *zErr = 0; 247 248 248 249 rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s", 249 - pConfig->zDb, pConfig->zName, zPost, zDefn, bWithout?" WITHOUT ROWID":"" 250 + pConfig->zDb, pConfig->zName, zPost, zDefn, 251 +#ifndef SQLITE_FTS5_NO_WITHOUT_ROWID 252 + bWithout?" WITHOUT ROWID": 253 +#endif 254 + "" 250 255 ); 251 256 if( zErr ){ 252 257 *pzErr = sqlite3_mprintf( 253 258 "fts5: error creating shadow table %q_%s: %s", 254 259 pConfig->zName, zPost, zErr 255 260 ); 256 261 sqlite3_free(zErr); ................................................................................ 364 369 int nToken, /* Size of token in bytes */ 365 370 int iUnused1, /* Start offset of token */ 366 371 int iUnused2 /* End offset of token */ 367 372 ){ 368 373 Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext; 369 374 Fts5Index *pIdx = pCtx->pStorage->pIndex; 370 375 UNUSED_PARAM2(iUnused1, iUnused2); 376 + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; 371 377 if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ 372 378 pCtx->szCol++; 373 379 } 374 380 return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); 375 381 } 376 382 377 383 /* ................................................................................ 535 541 if( rc==SQLITE_OK ){ 536 542 sqlite3_bind_int64(pDel, 1, iDel); 537 543 sqlite3_step(pDel); 538 544 rc = sqlite3_reset(pDel); 539 545 } 540 546 } 541 547 542 - /* Write the averages record */ 543 - if( rc==SQLITE_OK ){ 544 - rc = fts5StorageSaveTotals(p); 545 - } 546 - 547 548 return rc; 548 549 } 549 550 550 551 /* 551 552 ** Delete all entries in the FTS5 index. 552 553 */ 553 554 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){ ................................................................................ 634 635 int sqlite3Fts5StorageOptimize(Fts5Storage *p){ 635 636 return sqlite3Fts5IndexOptimize(p->pIndex); 636 637 } 637 638 638 639 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){ 639 640 return sqlite3Fts5IndexMerge(p->pIndex, nMerge); 640 641 } 642 + 643 +int sqlite3Fts5StorageReset(Fts5Storage *p){ 644 + return sqlite3Fts5IndexReset(p->pIndex); 645 +} 641 646 642 647 /* 643 648 ** Allocate a new rowid. This is used for "external content" tables when 644 649 ** a NULL value is inserted into the rowid column. The new rowid is allocated 645 650 ** by inserting a dummy row into the %_docsize table. The dummy will be 646 651 ** overwritten later. 647 652 ** ................................................................................ 739 744 740 745 /* Write the %_docsize record */ 741 746 if( rc==SQLITE_OK ){ 742 747 rc = fts5StorageInsertDocsize(p, iRowid, &buf); 743 748 } 744 749 sqlite3_free(buf.p); 745 750 746 - /* Write the averages record */ 747 - if( rc==SQLITE_OK ){ 748 - rc = fts5StorageSaveTotals(p); 749 - } 750 - 751 751 return rc; 752 752 } 753 753 754 754 static int fts5StorageCount(Fts5Storage *p, const char *zSuffix, i64 *pnRow){ 755 755 Fts5Config *pConfig = p->pConfig; 756 756 char *zSql; 757 757 int rc; ................................................................................ 806 806 int bPresent; 807 807 int ii; 808 808 int rc = SQLITE_OK; 809 809 int iPos; 810 810 int iCol; 811 811 812 812 UNUSED_PARAM2(iUnused1, iUnused2); 813 + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; 813 814 814 815 if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ 815 816 pCtx->szCol++; 816 817 } 817 818 818 819 switch( pCtx->pConfig->eDetail ){ 819 820 case FTS5_DETAIL_FULL: ................................................................................ 1077 1078 return rc; 1078 1079 } 1079 1080 1080 1081 /* 1081 1082 ** Flush any data currently held in-memory to disk. 1082 1083 */ 1083 1084 int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit){ 1084 - if( bCommit && p->bTotalsValid ){ 1085 - int rc = fts5StorageSaveTotals(p); 1086 - p->bTotalsValid = 0; 1087 - if( rc!=SQLITE_OK ) return rc; 1085 + int rc = SQLITE_OK; 1086 + i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db); 1087 + if( p->bTotalsValid ){ 1088 + rc = fts5StorageSaveTotals(p); 1089 + if( bCommit ) p->bTotalsValid = 0; 1090 + } 1091 + if( rc==SQLITE_OK ){ 1092 + rc = sqlite3Fts5IndexSync(p->pIndex, bCommit); 1088 1093 } 1089 - return sqlite3Fts5IndexSync(p->pIndex, bCommit); 1094 + sqlite3_set_last_insert_rowid(p->pConfig->db, iLastRowid); 1095 + return rc; 1090 1096 } 1091 1097 1092 1098 int sqlite3Fts5StorageRollback(Fts5Storage *p){ 1093 1099 p->bTotalsValid = 0; 1094 1100 return sqlite3Fts5IndexRollback(p->pIndex); 1095 1101 } 1096 1102 ................................................................................ 1117 1123 rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew); 1118 1124 if( rc==SQLITE_OK ){ 1119 1125 p->pConfig->iCookie = iNew; 1120 1126 } 1121 1127 } 1122 1128 return rc; 1123 1129 } 1124 - 1125 -
Changes to ext/fts5/fts5_tcl.c.
10 10 ** 11 11 ****************************************************************************** 12 12 ** 13 13 */ 14 14 15 15 16 16 #ifdef SQLITE_TEST 17 -#include <tcl.h> 17 +#if defined(INCLUDE_SQLITE_TCL_H) 18 +# include "sqlite_tcl.h" 19 +#else 20 +# include "tcl.h" 21 +# ifndef SQLITE_TCLAPI 22 +# define SQLITE_TCLAPI 23 +# endif 24 +#endif 18 25 19 26 #ifdef SQLITE_ENABLE_FTS5 20 27 21 28 #include "fts5.h" 22 29 #include <string.h> 23 30 #include <assert.h> 24 31 ................................................................................ 74 81 return aErr[i].rc; 75 82 } 76 83 } 77 84 78 85 return SQLITE_ERROR; 79 86 } 80 87 81 -static int f5tDbAndApi( 88 +static int SQLITE_TCLAPI f5tDbAndApi( 82 89 Tcl_Interp *interp, 83 90 Tcl_Obj *pObj, 84 91 sqlite3 **ppDb, 85 92 fts5_api **ppApi 86 93 ){ 87 94 sqlite3 *db = 0; 88 95 int rc = f5tDbPointer(interp, pObj, &db); ................................................................................ 160 167 if( rc==TCL_OK ){ 161 168 rc = f5tResultToErrorCode(Tcl_GetStringResult(p->interp)); 162 169 } 163 170 164 171 return rc; 165 172 } 166 173 167 -static int xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []); 174 +static int SQLITE_TCLAPI xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []); 168 175 169 176 static int xQueryPhraseCb( 170 177 const Fts5ExtensionApi *pApi, 171 178 Fts5Context *pFts, 172 179 void *pCtx 173 180 ){ 174 181 F5tFunction *p = (F5tFunction*)pCtx; ................................................................................ 205 212 } 206 213 207 214 /* 208 215 ** api sub-command... 209 216 ** 210 217 ** Description... 211 218 */ 212 -static int xF5tApi( 219 +static int SQLITE_TCLAPI xF5tApi( 213 220 void * clientData, 214 221 Tcl_Interp *interp, 215 222 int objc, 216 223 Tcl_Obj *CONST objv[] 217 224 ){ 218 225 struct Sub { 219 226 const char *zName; ................................................................................ 598 605 } 599 606 600 607 /* 601 608 ** sqlite3_fts5_create_function DB NAME SCRIPT 602 609 ** 603 610 ** Description... 604 611 */ 605 -static int f5tCreateFunction( 612 +static int SQLITE_TCLAPI f5tCreateFunction( 606 613 void * clientData, 607 614 Tcl_Interp *interp, 608 615 int objc, 609 616 Tcl_Obj *CONST objv[] 610 617 ){ 611 618 char *zName; 612 619 Tcl_Obj *pScript; ................................................................................ 668 675 669 676 670 677 /* 671 678 ** sqlite3_fts5_tokenize DB TOKENIZER TEXT 672 679 ** 673 680 ** Description... 674 681 */ 675 -static int f5tTokenize( 682 +static int SQLITE_TCLAPI f5tTokenize( 676 683 void * clientData, 677 684 Tcl_Interp *interp, 678 685 int objc, 679 686 Tcl_Obj *CONST objv[] 680 687 ){ 681 688 char *zText; 682 689 int nText; ................................................................................ 874 881 pInst->pContext->xToken = xOldToken; 875 882 return rc; 876 883 } 877 884 878 885 /* 879 886 ** sqlite3_fts5_token ?-colocated? TEXT START END 880 887 */ 881 -static int f5tTokenizerReturn( 888 +static int SQLITE_TCLAPI f5tTokenizerReturn( 882 889 void * clientData, 883 890 Tcl_Interp *interp, 884 891 int objc, 885 892 Tcl_Obj *CONST objv[] 886 893 ){ 887 894 F5tTokenizerContext *p = (F5tTokenizerContext*)clientData; 888 895 int iStart; ................................................................................ 945 952 ** tokenizer instance "returned" by SCRIPT. Specifically, to tokenize 946 953 ** text SCRIPT2 is invoked with a single argument appended to it - the 947 954 ** text to tokenize. 948 955 ** 949 956 ** SCRIPT2 should invoke the [sqlite3_fts5_token] command once for each 950 957 ** token within the tokenized text. 951 958 */ 952 -static int f5tCreateTokenizer( 959 +static int SQLITE_TCLAPI f5tCreateTokenizer( 953 960 ClientData clientData, 954 961 Tcl_Interp *interp, 955 962 int objc, 956 963 Tcl_Obj *CONST objv[] 957 964 ){ 958 965 F5tTokenizerContext *pContext = (F5tTokenizerContext*)clientData; 959 966 sqlite3 *db; ................................................................................ 988 995 Tcl_AppendResult(interp, "error in fts5_api.xCreateTokenizer()", 0); 989 996 return TCL_ERROR; 990 997 } 991 998 992 999 return TCL_OK; 993 1000 } 994 1001 995 -static void xF5tFree(ClientData clientData){ 1002 +static void SQLITE_TCLAPI xF5tFree(ClientData clientData){ 996 1003 ckfree(clientData); 997 1004 } 998 1005 999 1006 /* 1000 1007 ** sqlite3_fts5_may_be_corrupt BOOLEAN 1001 1008 ** 1002 1009 ** Set or clear the global "may-be-corrupt" flag. Return the old value. 1003 1010 */ 1004 -static int f5tMayBeCorrupt( 1011 +static int SQLITE_TCLAPI f5tMayBeCorrupt( 1005 1012 void * clientData, 1006 1013 Tcl_Interp *interp, 1007 1014 int objc, 1008 1015 Tcl_Obj *CONST objv[] 1009 1016 ){ 1010 1017 int bOld = sqlite3_fts5_may_be_corrupt; 1011 1018 ................................................................................ 1029 1036 unsigned int h = 13; 1030 1037 for(i=n-1; i>=0; i--){ 1031 1038 h = (h << 3) ^ h ^ p[i]; 1032 1039 } 1033 1040 return (h % nSlot); 1034 1041 } 1035 1042 1036 -static int f5tTokenHash( 1043 +static int SQLITE_TCLAPI f5tTokenHash( 1037 1044 void * clientData, 1038 1045 Tcl_Interp *interp, 1039 1046 int objc, 1040 1047 Tcl_Obj *CONST objv[] 1041 1048 ){ 1042 1049 char *z; 1043 1050 int n; ................................................................................ 1054 1061 z = Tcl_GetStringFromObj(objv[2], &n); 1055 1062 1056 1063 iVal = f5t_fts5HashKey(nSlot, z, n); 1057 1064 Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal)); 1058 1065 return TCL_OK; 1059 1066 } 1060 1067 1061 -static int f5tRegisterMatchinfo( 1068 +static int SQLITE_TCLAPI f5tRegisterMatchinfo( 1062 1069 void * clientData, 1063 1070 Tcl_Interp *interp, 1064 1071 int objc, 1065 1072 Tcl_Obj *CONST objv[] 1066 1073 ){ 1067 1074 int rc; 1068 1075 sqlite3 *db = 0; ................................................................................ 1079 1086 if( rc!=SQLITE_OK ){ 1080 1087 Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); 1081 1088 return TCL_ERROR; 1082 1089 } 1083 1090 return TCL_OK; 1084 1091 } 1085 1092 1086 -static int f5tRegisterTok( 1093 +static int SQLITE_TCLAPI f5tRegisterTok( 1087 1094 void * clientData, 1088 1095 Tcl_Interp *interp, 1089 1096 int objc, 1090 1097 Tcl_Obj *CONST objv[] 1091 1098 ){ 1092 1099 int rc; 1093 1100 sqlite3 *db = 0;
Changes to ext/fts5/fts5_vocab.c.
274 274 } 275 275 if( iTermLe>=0 ){ 276 276 idxNum |= FTS5_VOCAB_TERM_LE; 277 277 pInfo->aConstraintUsage[iTermLe].argvIndex = ++nArg; 278 278 pInfo->estimatedCost = pInfo->estimatedCost / 2; 279 279 } 280 280 } 281 + 282 + /* This virtual table always delivers results in ascending order of 283 + ** the "term" column (column 0). So if the user has requested this 284 + ** specifically - "ORDER BY term" or "ORDER BY term ASC" - set the 285 + ** sqlite3_index_info.orderByConsumed flag to tell the core the results 286 + ** are already in sorted order. */ 287 + if( pInfo->nOrderBy==1 288 + && pInfo->aOrderBy[0].iColumn==0 289 + && pInfo->aOrderBy[0].desc==0 290 + ){ 291 + pInfo->orderByConsumed = 1; 292 + } 281 293 282 294 pInfo->idxNum = idxNum; 283 - 284 295 return SQLITE_OK; 285 296 } 286 297 287 298 /* 288 299 ** Implementation of xOpen method. 289 300 */ 290 301 static int fts5VocabOpenMethod(
Changes to ext/fts5/fts5parse.y.
116 116 } 117 117 118 118 %type colset {Fts5Colset*} 119 119 %destructor colset { sqlite3_free($$); } 120 120 %type colsetlist {Fts5Colset*} 121 121 %destructor colsetlist { sqlite3_free($$); } 122 122 123 +colset(A) ::= MINUS LCP colsetlist(X) RCP. { 124 + A = sqlite3Fts5ParseColsetInvert(pParse, X); 125 +} 123 126 colset(A) ::= LCP colsetlist(X) RCP. { A = X; } 124 127 colset(A) ::= STRING(X). { 125 128 A = sqlite3Fts5ParseColset(pParse, 0, &X); 126 129 } 130 +colset(A) ::= MINUS STRING(X). { 131 + A = sqlite3Fts5ParseColset(pParse, 0, &X); 132 + A = sqlite3Fts5ParseColsetInvert(pParse, A); 133 +} 127 134 128 135 colsetlist(A) ::= colsetlist(Y) STRING(X). { 129 136 A = sqlite3Fts5ParseColset(pParse, Y, &X); } 130 137 colsetlist(A) ::= STRING(X). { 131 138 A = sqlite3Fts5ParseColset(pParse, 0, &X); 132 139 } 133 - 134 140 135 141 %type nearset {Fts5ExprNearset*} 136 142 %type nearphrases {Fts5ExprNearset*} 137 143 %destructor nearset { sqlite3Fts5ParseNearsetFree($$); } 138 144 %destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); } 139 145 140 146 nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }
Changes to ext/fts5/test/fts5aa.test.
428 428 set fd [db incrblob main n1_data block 10] 429 429 fconfigure $fd -encoding binary -translation binary 430 430 puts -nonewline $fd "\x44\x45" 431 431 close $fd 432 432 } 433 433 db func funk funk 434 434 435 +# This test case corrupts the structure record within the first invocation 436 +# of function funk(). Which used to cause the bm25() function to throw an 437 +# exception. But since bm25() can now used the cached structure record, 438 +# it never sees the corruption introduced by funk() and so the following 439 +# statement no longer fails. 440 +# 435 441 do_catchsql_test 16.2 { 436 442 SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d' 437 -} {1 {SQL logic error or missing database}} 443 +} {0 {{} -1e-06 {}}} 444 +# {1 {SQL logic error or missing database}} 438 445 439 446 #------------------------------------------------------------------------- 440 447 # 441 448 reset_db 442 449 do_execsql_test 17.1 { 443 450 CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%); 444 451 INSERT INTO b2 VALUES('a');
Changes to ext/fts5/test/fts5af.test.
68 68 1.6 {o o o o o X o} {o o o o o [X] o} 69 69 1.7 {o o o o o o X} {o o o o o o [X]} 70 70 71 71 2.1 {X o o o o o o o} {[X] o o o o o o...} 72 72 2.2 {o X o o o o o o} {o [X] o o o o o...} 73 73 2.3 {o o X o o o o o} {o o [X] o o o o...} 74 74 2.4 {o o o X o o o o} {o o o [X] o o o...} 75 - 2.5 {o o o o X o o o} {...o o o [X] o o o} 76 - 2.6 {o o o o o X o o} {...o o o o [X] o o} 77 - 2.7 {o o o o o o X o} {...o o o o o [X] o} 75 + 2.5 {o o o o X o o o} {o o o o [X] o o...} 76 + 2.6 {o o o o o X o o} {o o o o o [X] o...} 77 + 2.7 {o o o o o o X o} {o o o o o o [X]...} 78 78 2.8 {o o o o o o o X} {...o o o o o o [X]} 79 + 80 + 2.9 {o o o o o o o X o} {...o o o o o [X] o} 81 + 2.10 {o o o o o o o X o o} {...o o o o [X] o o} 82 + 2.11 {o o o o o o o X o o o} {...o o o [X] o o o} 83 + 2.12 {o o o o o o o X o o o o} {...o o o [X] o o o...} 84 + 79 85 80 86 3.1 {X o o o o o o o o} {[X] o o o o o o...} 81 87 3.2 {o X o o o o o o o} {o [X] o o o o o...} 82 88 3.3 {o o X o o o o o o} {o o [X] o o o o...} 83 89 3.4 {o o o X o o o o o} {o o o [X] o o o...} 84 - 3.5 {o o o o X o o o o} {...o o o [X] o o o...} 85 - 3.6 {o o o o o X o o o} {...o o o [X] o o o} 86 - 3.7 {o o o o o o X o o} {...o o o o [X] o o} 87 - 3.8 {o o o o o o o X o} {...o o o o o [X] o} 88 - 3.9 {o o o o o o o o X} {...o o o o o o [X]} 90 + 91 + 3.5 {o o o o o o o X o o o o} {...o o o [X] o o o...} 92 + 3.6 {o o o o o o o o X o o o} {...o o o [X] o o o} 93 + 3.7 {o o o o o o o o o X o o} {...o o o o [X] o o} 94 + 3.8 {o o o o o o o o o o X o} {...o o o o o [X] o} 95 + 3.9 {o o o o o o o o o o o X} {...o o o o o o [X]} 89 96 90 97 4.1 {X o o o o o X o o} {[X] o o o o o [X]...} 91 - 4.2 {o X o o o o o X o} {...[X] o o o o o [X]...} 92 - 4.3 {o o X o o o o o X} {...[X] o o o o o [X]} 98 + 4.2 {o o o o o o o X o o o o o X o} {...[X] o o o o o [X]...} 99 + 4.3 {o o o o o o o o X o o o o o X} {...[X] o o o o o [X]} 93 100 94 101 5.1 {X o o o o X o o o} {[X] o o o o [X] o...} 95 - 5.2 {o X o o o o X o o} {...[X] o o o o [X] o...} 96 - 5.3 {o o X o o o o X o} {...[X] o o o o [X] o} 97 - 5.4 {o o o X o o o o X} {...o [X] o o o o [X]} 102 + 5.2 {o o o o o o o X o o o o X o o} {...[X] o o o o [X] o...} 103 + 5.3 {o o o o o o o o X o o o o X o} {...[X] o o o o [X] o} 104 + 5.4 {o o o o o o o o o X o o o o X} {...o [X] o o o o [X]} 98 105 99 106 6.1 {X o o o X o o o} {[X] o o o [X] o o...} 100 107 6.2 {o X o o o X o o o} {o [X] o o o [X] o...} 101 - 6.3 {o o X o o o X o o} {...o [X] o o o [X] o...} 102 - 6.4 {o o o X o o o X o} {...o [X] o o o [X] o} 103 - 6.5 {o o o o X o o o X} {...o o [X] o o o [X]} 108 + 6.3 {o o o o o o o X o o o X o o} {...o [X] o o o [X] o...} 109 + 6.4 {o o o o o o o o X o o o X o} {...o [X] o o o [X] o} 110 + 6.5 {o o o o o o o o o X o o o X} {...o o [X] o o o [X]} 104 111 105 112 7.1 {X o o X o o o o o} {[X] o o [X] o o o...} 106 113 7.2 {o X o o X o o o o} {o [X] o o [X] o o...} 107 - 7.3 {o o X o o X o o o} {...o [X] o o [X] o o...} 108 - 7.4 {o o o X o o X o o} {...o [X] o o [X] o o} 109 - 7.5 {o o o o X o o X o} {...o o [X] o o [X] o} 110 - 7.6 {o o o o o X o o X} {...o o o [X] o o [X]} 114 + 7.3 {o o o o o o o X o o X o o o} {...o [X] o o [X] o o...} 115 + 7.4 {o o o o o o o o X o o X o o} {...o [X] o o [X] o o} 116 + 7.5 {o o o o o o o o o X o o X o} {...o o [X] o o [X] o} 117 + 7.6 {o o o o o o o o o o X o o X} {...o o o [X] o o [X]} 118 + 119 + 8.1 {o o o o o o o o o X o o o o o o o o o o o o o o o o X X X o o o} 120 + {...o o [X] [X] [X] o o...} 121 + 8.2 {o o o o o o o. o o X o o o o o o o o o o o o o o o o X X X o o o} 122 + {...o o [X] o o o o...} 123 + 8.3 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o} 124 + {o o o o [X] o o...} 111 125 } { 112 126 do_snippet_test 1.$tn $doc X $res 113 127 } 114 128 115 129 if {[detail_is_full]} { 116 130 foreach {tn doc res} { 117 131 1.1 {X Y o o o o o} {[X Y] o o o o o} ................................................................................ 120 134 1.4 {o o o X Y o o} {o o o [X Y] o o} 121 135 1.5 {o o o o X Y o} {o o o o [X Y] o} 122 136 1.6 {o o o o o X Y} {o o o o o [X Y]} 123 137 124 138 2.1 {X Y o o o o o o} {[X Y] o o o o o...} 125 139 2.2 {o X Y o o o o o} {o [X Y] o o o o...} 126 140 2.3 {o o X Y o o o o} {o o [X Y] o o o...} 127 - 2.4 {o o o X Y o o o} {...o o [X Y] o o o} 128 - 2.5 {o o o o X Y o o} {...o o o [X Y] o o} 129 - 2.6 {o o o o o X Y o} {...o o o o [X Y] o} 130 - 2.7 {o o o o o o X Y} {...o o o o o [X Y]} 141 + 2.4 {o o o o o o o X Y o o o} {...o o [X Y] o o o} 142 + 2.5 {o o o o o o o o X Y o o} {...o o o [X Y] o o} 143 + 2.6 {o o o o o o o o o X Y o} {...o o o o [X Y] o} 144 + 2.7 {o o o o o o o o o o X Y} {...o o o o o [X Y]} 131 145 132 146 3.1 {X Y o o o o o o o} {[X Y] o o o o o...} 133 147 3.2 {o X Y o o o o o o} {o [X Y] o o o o...} 134 148 3.3 {o o X Y o o o o o} {o o [X Y] o o o...} 135 - 3.4 {o o o X Y o o o o} {...o o [X Y] o o o...} 136 - 3.5 {o o o o X Y o o o} {...o o [X Y] o o o} 137 - 3.6 {o o o o o X Y o o} {...o o o [X Y] o o} 138 - 3.7 {o o o o o o X Y o} {...o o o o [X Y] o} 139 - 3.8 {o o o o o o o X Y} {...o o o o o [X Y]} 149 + 3.4 {o o o o o o o X Y o o o o} {...o o [X Y] o o o...} 150 + 3.5 {o o o o o o o o X Y o o o} {...o o [X Y] o o o} 151 + 3.6 {o o o o o o o o o X Y o o} {...o o o [X Y] o o} 152 + 3.7 {o o o o o o o o o o X Y o} {...o o o o [X Y] o} 153 + 3.8 {o o o o o o o o o o o X Y} {...o o o o o [X Y]} 140 154 } { 141 155 do_snippet_test 2.$tn $doc "X + Y" $res 142 156 } 143 157 } 144 158 159 +do_execsql_test 4.0 { 160 + CREATE VIRTUAL TABLE x1 USING fts5(a, b); 161 + INSERT INTO x1 VALUES('xyz', '1 2 3 4 5 6 7 8 9 10 11 12 13'); 162 + SELECT snippet(x1, 1, '[', ']', '...', 5) FROM x1('xyz'); 163 +} { 164 + {1 2 3 4 5...} 165 +} 166 + 167 +do_execsql_test 5.0 { 168 + CREATE VIRTUAL TABLE p1 USING fts5(a, b); 169 + INSERT INTO p1 VALUES( 170 + 'x a a a a a a a a a a', 171 + 'a a a a a a a a a a a a a a a a a a a x' 172 + ); 173 +} 174 +do_execsql_test 5.1 { 175 + SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x'); 176 +} {{[x] a a a a a...}} 177 + 145 178 } ;# foreach_detail_mode 146 179 147 180 finish_test 148 181
Changes to ext/fts5/test/fts5aux.test.
241 241 execsql { DELETE FROM x1 } 242 242 foreach row $lRow { execsql { INSERT INTO x1 VALUES($row) } } 243 243 breakpoint 244 244 do_execsql_test 8.$tn { 245 245 SELECT highlight(x1, 0, '[', ']') FROM x1 WHERE x1 MATCH 'a OR (b AND d)'; 246 246 } $res 247 247 } 248 + 249 +#------------------------------------------------------------------------- 250 +# Test the built-in bm25() demo. 251 +# 252 +reset_db 253 +do_execsql_test 9.1 { 254 + CREATE VIRTUAL TABLE t1 USING fts5(a, b); 255 + INSERT INTO t1 VALUES('a', NULL); -- 1 256 + INSERT INTO t1 VALUES('a', NULL); -- 2 257 + INSERT INTO t1 VALUES('a', NULL); -- 3 258 + INSERT INTO t1 VALUES('a', NULL); -- 4 259 + INSERT INTO t1 VALUES('a', NULL); -- 5 260 + INSERT INTO t1 VALUES('a', NULL); -- 6 261 + INSERT INTO t1 VALUES('a', NULL); -- 7 262 + INSERT INTO t1 VALUES('a', NULL); -- 8 263 + INSERT INTO t1 VALUES(NULL, 'a a b'); -- 9 264 + INSERT INTO t1 VALUES(NULL, 'b b a'); -- 10 265 +} 266 + 267 +do_execsql_test 9.2 { 268 + SELECT rowid FROM t1('a AND b') ORDER BY rank; 269 +} { 270 + 10 9 271 +} 272 + 273 +do_execsql_test 9.3 { 274 + SELECT rowid FROM t1('b:a AND b:b') ORDER BY rank; 275 +} { 276 + 9 10 277 +} 278 + 279 + 248 280 249 281 finish_test 250 282
Added ext/fts5/test/fts5colset.test.
1 +# 2016 August 10 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#************************************************************************* 11 +# This file implements regression tests for SQLite library. The 12 +# focus of this script is testing the FTS5 module. 13 +# 14 + 15 +source [file join [file dirname [info script]] fts5_common.tcl] 16 +set testprefix fts5colset 17 + 18 +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. 19 +ifcapable !fts5 { 20 + finish_test 21 + return 22 +} 23 + 24 +foreach_detail_mode $::testprefix { 25 + if {[detail_is_none]} continue 26 + 27 + do_execsql_test 1.0 { 28 + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d, detail=%DETAIL%); 29 + INSERT INTO t1 VALUES('a', 'b', 'c', 'd'); -- 1 30 + INSERT INTO t1 VALUES('d', 'a', 'b', 'c'); -- 2 31 + INSERT INTO t1 VALUES('c', 'd', 'a', 'b'); -- 3 32 + INSERT INTO t1 VALUES('b', 'c', 'd', 'a'); -- 4 33 + } 34 + 35 + foreach {tn q res} { 36 + 1 "a" {1 2 3 4} 37 + 2 "{a} : a" {1} 38 + 3 "-{a} : a" {2 3 4} 39 + 4 "- {a c} : a" {2 4} 40 + 5 " - {d d c} : a" {1 2} 41 + 6 "- {d c b a} : a" {} 42 + 7 "-{\"a\"} : b" {1 2 3} 43 + 8 "- c : a" {1 2 4} 44 + 9 "-c : a" {1 2 4} 45 + 10 "-\"c\" : a" {1 2 4} 46 + } { 47 + breakpoint 48 + do_execsql_test 1.$tn { 49 + SELECT rowid FROM t1($q) 50 + } $res 51 + } 52 + 53 + 54 +} 55 + 56 + 57 +finish_test 58 + 59 +
Changes to ext/fts5/test/fts5corrupt2.test.
33 33 CREATE VIRTUAL TABLE t1 USING fts5(x); 34 34 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); 35 35 WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100) 36 36 INSERT INTO t1 SELECT rnddoc(10) FROM ii; 37 37 } 38 38 set mask [expr 31 << 31] 39 39 40 -if 1 { 40 +if 0 { 41 41 42 42 # Test 1: 43 43 # 44 44 # For each page in the t1_data table, open a transaction and DELETE 45 45 # the t1_data entry. Then run: 46 46 # 47 47 # * an integrity-check, and ................................................................................ 78 78 } 79 79 80 80 do_execsql_test 1.$tno.$tn.3.$rowid { 81 81 ROLLBACK; 82 82 INSERT INTO t1(t1) VALUES('integrity-check'); 83 83 } {} 84 84 } 85 +} 86 + 85 87 } 86 88 87 89 # Using the same database as the 1.* tests. 88 90 # 89 91 # Run N-1 tests, where N is the number of bytes in the rightmost leaf page 90 92 # of the fts index. For test $i, truncate the rightmost leafpage to $i 91 93 # bytes. Then test both the integrity-check detects the corruption. ................................................................................ 206 208 set {} 1 207 209 } {1} 208 210 209 211 execsql ROLLBACK 210 212 } 211 213 212 214 # do_test 4.$tn.x { expr $nCorrupt>0 } 1 213 -} 214 - 215 215 } 216 216 217 217 set doc [string repeat "A B C " 1000] 218 218 do_execsql_test 5.0 { 219 219 CREATE VIRTUAL TABLE x5 USING fts5(tt); 220 220 INSERT INTO x5(x5, rank) VALUES('pgsz', 32); 221 221 WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10)
Changes to ext/fts5/test/fts5corrupt3.test.
175 175 for {set i 1} {1} {incr i} { 176 176 set struct [db one {SELECT block FROM t1_data WHERE id=10}] 177 177 binary scan $struct c* var 178 178 set end [lindex $var end] 179 179 if {$end<=$i} break 180 180 lset var end [expr $end - $i] 181 181 set struct [binary format c* $var] 182 + 183 + db close 184 + sqlite3 db test.db 185 + 182 186 db eval { 183 187 BEGIN; 184 188 UPDATE t1_data SET block = $struct WHERE id=10; 185 189 } 186 190 do_test 4.1.$i { 187 191 incr nErr [catch { db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x*' } }] 188 192 set {} {}
Added ext/fts5/test/fts5determin.test.
1 +# 2016 March 21 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#************************************************************************* 11 +# This file implements regression tests for SQLite library. The 12 +# focus of this script is testing the FTS5 module. 13 +# 14 +# Specifically, that the fts5 module is deterministic. At one point, when 15 +# segment ids were allocated using sqlite3_randomness(), this was not the 16 +# case. 17 +# 18 + 19 +source [file join [file dirname [info script]] fts5_common.tcl] 20 +set testprefix fts5aa 21 +return_if_no_fts5 22 + 23 +proc do_determin_test {tn} { 24 + uplevel [list 25 + do_execsql_test $tn { 26 + SELECT (SELECT md5sum(id, block) FROM t1_data)== 27 + (SELECT md5sum(id, block) FROM t2_data), 28 + (SELECT md5sum(id, block) FROM t1_data)== 29 + (SELECT md5sum(id, block) FROM t3_data) 30 + } {1 1} 31 + ] 32 +} 33 + 34 +foreach_detail_mode $::testprefix { 35 + do_execsql_test 1.0 { 36 + CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1 2", detail=%DETAIL%); 37 + CREATE VIRTUAL TABLE t2 USING fts5(a, b, prefix="1 2", detail=%DETAIL%); 38 + CREATE VIRTUAL TABLE t3 USING fts5(a, b, prefix="1 2", detail=%DETAIL%); 39 + } 40 + 41 + do_test 1.1 { 42 + foreach t {t1 t2 t3} { 43 + execsql [string map [list TBL $t] { 44 + INSERT INTO TBL VALUES('a b c', 'd e f'); 45 + INSERT INTO TBL VALUES('c1 c2 c3', 'c1 c2 c3'); 46 + INSERT INTO TBL VALUES('xyzxyzxyz', 'xyzxyzxyz'); 47 + }] 48 + } 49 + } {} 50 + 51 + do_determin_test 1.2 52 + 53 + do_test 1.3 { 54 + foreach t {t1 t2 t3} { 55 + execsql [string map [list TBL $t] { 56 + INSERT INTO TBL(TBL) VALUES('optimize'); 57 + }] 58 + } 59 + } {} 60 + 61 + do_determin_test 1.4 62 +} 63 + 64 + 65 +finish_test 66 + 67 +
Changes to ext/fts5/test/fts5dlidx.test.
174 174 175 175 do_execsql_test 3.2 { 176 176 SELECT rowid FROM abc WHERE abc 177 177 MATCH 'IteratorpItercurrentlypointstothefirstrowidofadoclist' 178 178 ORDER BY rowid DESC; 179 179 } {16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1} 180 180 181 -do_execsql_test 3.2 { 181 +do_execsql_test 3.3 { 182 182 INSERT INTO abc(abc) VALUES('integrity-check'); 183 183 INSERT INTO abc(abc) VALUES('optimize'); 184 184 INSERT INTO abc(abc) VALUES('integrity-check'); 185 185 } 186 186 187 187 set v [lindex $vocab 0] 188 188 set i 0 189 189 foreach v $vocab { 190 - do_execsql_test 3.3.[incr i] { 190 + do_execsql_test 3.4.[incr i] { 191 191 SELECT rowid FROM abc WHERE abc MATCH $v 192 192 } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16} 193 193 } 194 194 195 195 } ;# foreach_detail_mode 196 196 197 197 198 198 199 199 finish_test 200 200
Changes to ext/fts5/test/fts5eb.test.
58 58 do_catchsql_test 2.1 { 59 59 SELECT fts5_expr() 60 60 } {1 {wrong number of arguments to function fts5_expr}} 61 61 62 62 do_catchsql_test 2.1 { 63 63 SELECT fts5_expr_tcl() 64 64 } {1 {wrong number of arguments to function fts5_expr_tcl}} 65 + 66 + 67 +do_execsql_test 3.0 { 68 + CREATE VIRTUAL TABLE e1 USING fts5(text, tokenize = 'porter unicode61'); 69 + INSERT INTO e1 VALUES ("just a few words with a / inside"); 70 +} 71 +do_execsql_test 3.1 { 72 + SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank; 73 +} {1 -1e-06} 74 +do_execsql_test 3.2 { 75 + SELECT rowid FROM e1 WHERE e1 MATCH '"/" OR "just"' 76 +} 1 77 +do_execsql_test 3.3 { 78 + SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank; 79 +} {1 -1e-06} 80 + 81 + 65 82 66 83 finish_test 67 84 68 85 69 86
Changes to ext/fts5/test/fts5fault4.test.
82 82 } 83 83 84 84 set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}] 85 85 86 86 do_faultsim_test 4 -faults oom-* -body { 87 87 db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'} 88 88 } -test { 89 - faultsim_test_result {0 {0 {} 4}} 89 + faultsim_test_result {0 {0 {} 3}} 90 90 } 91 91 92 92 #------------------------------------------------------------------------- 93 93 # An OOM within a query that uses a custom rank function. 94 94 # 95 95 reset_db 96 96 do_execsql_test 5.0 {
Changes to ext/fts5/test/fts5faultB.test.
74 74 75 75 do_faultsim_test 2.4 -faults oom* -body { 76 76 execsql { SELECT mit(matchinfo(t1, 's')) FROM t1('a b c') } 77 77 } -test { 78 78 faultsim_test_result {0 {{3 2} {2 3}}} 79 79 } 80 80 81 +#------------------------------------------------------------------------- 82 +# 83 +reset_db 84 +do_execsql_test 3.0 { 85 + CREATE VIRTUAL TABLE x1 USING fts5(z); 86 +} 87 + 88 +do_faultsim_test 3.1 -faults oom* -body { 89 + execsql { 90 + SELECT rowid FROM x1('c') WHERE rowid>1; 91 + } 92 +} -test { 93 + faultsim_test_result {0 {}} 94 +} 95 + 96 +do_execsql_test 3.2 { 97 + INSERT INTO x1 VALUES('a b c'); 98 + INSERT INTO x1 VALUES('b c d'); 99 + INSERT INTO x1 VALUES('c d e'); 100 + INSERT INTO x1 VALUES('d e f'); 101 +} 102 +do_faultsim_test 3.3 -faults oom* -body { 103 + execsql { 104 + SELECT rowid FROM x1('c') WHERE rowid>1; 105 + } 106 +} -test { 107 + faultsim_test_result {0 {2 3}} 108 +} 81 109 82 110 finish_test 83 111
Added ext/fts5/test/fts5faultD.test.
1 +# 2016 February 2 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#************************************************************************* 11 +# 12 +# This file is focused on OOM errors. 13 +# 14 + 15 +source [file join [file dirname [info script]] fts5_common.tcl] 16 +source $testdir/malloc_common.tcl 17 +set testprefix fts5faultA 18 + 19 +# If SQLITE_ENABLE_FTS3 is defined, omit this file. 20 +ifcapable !fts5 { 21 + finish_test 22 + return 23 +} 24 + 25 +foreach_detail_mode $testprefix { 26 + if {"%DETAIL%"=="none"} continue 27 + 28 + do_execsql_test 1.0 { 29 + CREATE VIRTUAL TABLE o1 USING fts5(a, b, c, detail=%DETAIL%); 30 + INSERT INTO o1(o1, rank) VALUES('pgsz', 32); 31 + 32 + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 ) 33 + INSERT INTO o1 SELECT 'A', 'B', 'C' FROM s; 34 + 35 + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 ) 36 + INSERT INTO o1 SELECT 'C', 'A', 'B' FROM s; 37 + 38 + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 ) 39 + INSERT INTO o1 SELECT 'B', 'C', 'A' FROM s; 40 + } 41 + 42 + do_faultsim_test 1 -faults int* -prep { 43 + sqlite3 db test.db 44 + } -body { 45 + execsql { SELECT count(*) FROM o1('a') } 46 + } -test { 47 + faultsim_test_result {0 180} {1 {vtable constructor failed: o1}} 48 + } 49 + 50 + do_faultsim_test 2 -faults int* -prep { 51 + sqlite3 db test.db 52 + } -body { 53 + execsql { SELECT * FROM o1('a:a AND {b c}:b') ORDER BY rank } 54 + expr 1 55 + } -test { 56 + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} 57 + } 58 + 59 + do_faultsim_test 3 -faults int* -prep { 60 + sqlite3 db test.db 61 + } -body { 62 + execsql { SELECT * FROM o1('{b c}:b NOT a:a') ORDER BY rank } 63 + expr 1 64 + } -test { 65 + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} 66 + } 67 + 68 + do_faultsim_test 4 -faults int* -prep { 69 + sqlite3 db test.db 70 + } -body { 71 + execsql { SELECT * FROM o1('b:b OR a:a') } 72 + expr 1 73 + } -test { 74 + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} 75 + } 76 + 77 + do_faultsim_test 5 -faults int* -prep { 78 + sqlite3 db test.db 79 + } -body { 80 + execsql { SELECT count(*) FROM o1('c:b') } 81 + expr 1 82 + } -test { 83 + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} 84 + } 85 +} 86 + 87 +finish_test
Added ext/fts5/test/fts5lastrowid.test.
1 +# 2017 Feb 27 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#*********************************************************************** 11 +# 12 +# Tests of the last_insert_rowid functionality with fts5. 13 +# 14 + 15 +source [file join [file dirname [info script]] fts5_common.tcl] 16 +set testprefix fts5lastrowid 17 + 18 +# If SQLITE_ENABLE_FTS5 is defined, omit this file. 19 +ifcapable !fts5 { 20 + finish_test 21 + return 22 +} 23 + 24 +do_execsql_test 1.0 { 25 + CREATE VIRTUAL TABLE t1 USING fts5(str); 26 +} 27 + 28 +do_execsql_test 1.1 { 29 + INSERT INTO t1 VALUES('one string'); 30 + INSERT INTO t1 VALUES('two string'); 31 + INSERT INTO t1 VALUES('three string'); 32 + SELECT last_insert_rowid(); 33 +} {3} 34 + 35 +do_execsql_test 1.2 { 36 + BEGIN; 37 + INSERT INTO t1 VALUES('one string'); 38 + INSERT INTO t1 VALUES('two string'); 39 + INSERT INTO t1 VALUES('three string'); 40 + COMMIT; 41 + SELECT last_insert_rowid(); 42 +} {6} 43 + 44 +do_execsql_test 1.3 { 45 + INSERT INTO t1(rowid, str) VALUES(-22, 'some more text'); 46 + SELECT last_insert_rowid(); 47 +} {-22} 48 + 49 +do_execsql_test 1.4 { 50 + BEGIN; 51 + INSERT INTO t1(rowid, str) VALUES(45, 'some more text'); 52 + INSERT INTO t1(rowid, str) VALUES(46, 'some more text'); 53 + INSERT INTO t1(rowid, str) VALUES(222, 'some more text'); 54 + SELECT last_insert_rowid(); 55 + COMMIT; 56 + SELECT last_insert_rowid(); 57 +} {222 222} 58 + 59 +do_execsql_test 1.5 { 60 + CREATE TABLE x1(x); 61 + INSERT INTO x1 VALUES('john'), ('paul'), ('george'), ('ringo'); 62 + INSERT INTO t1 SELECT x FROM x1; 63 + SELECT last_insert_rowid(); 64 +} {226} 65 + 66 +do_execsql_test 1.6 { 67 + INSERT INTO t1(rowid, str) SELECT rowid+10, x FROM x1; 68 + SELECT last_insert_rowid(); 69 +} {14} 70 + 71 + 72 +finish_test 73 +
Added ext/fts5/test/fts5multiclient.test.
1 +# 2016 March 17 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#************************************************************************* 11 +# 12 + 13 +source [file join [file dirname [info script]] fts5_common.tcl] 14 +source $testdir/lock_common.tcl 15 + 16 +set testprefix fts5multiclient 17 +return_if_no_fts5 18 + 19 +foreach_detail_mode $testprefix { 20 + 21 +do_multiclient_test tn { 22 + 23 + do_test 1.$tn.1 { 24 + sql1 { CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%) } 25 + sql1 { INSERT INTO t1 VALUES('a b c') } 26 + sql2 { SELECT rowid FROM t1('b') } 27 + } {1} 28 + 29 + do_test 1.$tn.2 { 30 + sql2 { INSERT INTO t1 VALUES('a b c') } 31 + sql1 { SELECT rowid FROM t1('b') } 32 + } {1 2} 33 + 34 + do_test 1.$tn.3 { 35 + sql2 { INSERT INTO t1 VALUES('a b c') } 36 + sql1 { SELECT rowid FROM t1('b') } 37 + } {1 2 3} 38 + 39 + do_test 1.$tn.4 { 40 + sql2 { INSERT INTO t1 VALUES('a b c') } 41 + sql1 { INSERT INTO t1 VALUES('a b c') } 42 + sql3 { INSERT INTO t1(t1) VALUES('integrity-check') } 43 + } {} 44 + 45 +};# do_multiclient_test 46 +};# foreach_detail_mode 47 +finish_test 48 +
Changes to ext/fts5/test/fts5prefix.test.
5 5 # 6 6 # May you do good and not evil. 7 7 # May you find forgiveness for yourself and forgive others. 8 8 # May you share freely, never taking more than you give. 9 9 # 10 10 #*********************************************************************** 11 11 # 12 -# This file containst tests focused on prefix indexes. 12 +# This file contains tests focused on prefix indexes. 13 13 # 14 14 15 15 source [file join [file dirname [info script]] fts5_common.tcl] 16 16 set testprefix fts5prefix 17 17 18 18 # If SQLITE_ENABLE_FTS5 is defined, omit this file. 19 19 ifcapable !fts5 {
Changes to ext/fts5/test/fts5rank.test.
87 87 } {1 3 2} 88 88 89 89 do_test 2.7 { 90 90 execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db 91 91 } {1 3 2} 92 92 93 93 94 +#-------------------------------------------------------------------------- 95 +# At one point there was a problem with queries such as: 96 +# 97 +# ... MATCH 'x OR y' ORDER BY rank; 98 +# 99 +# if there were zero occurrences of token 'y' in the dataset. The 100 +# following tests verify that that problem has been addressed. 101 +# 102 +foreach_detail_mode $::testprefix { 103 + do_execsql_test 3.1.0 { 104 + CREATE VIRTUAL TABLE y1 USING fts5(z, detail=%DETAIL%); 105 + INSERT INTO y1 VALUES('test xyz'); 106 + INSERT INTO y1 VALUES('test test xyz test'); 107 + INSERT INTO y1 VALUES('test test xyz'); 108 + } 109 + 110 + do_execsql_test 3.1.1 { 111 + SELECT rowid FROM y1('test OR tset'); 112 + } {1 2 3} 113 + 114 + do_execsql_test 3.1.2 { 115 + SELECT rowid FROM y1('test OR tset') ORDER BY bm25(y1) 116 + } {2 3 1} 117 + 118 + do_execsql_test 3.1.3 { 119 + SELECT rowid FROM y1('test OR tset') ORDER BY +rank 120 + } {2 3 1} 121 + 122 + do_execsql_test 3.1.4 { 123 + SELECT rowid FROM y1('test OR tset') ORDER BY rank 124 + } {2 3 1} 125 + 126 + do_execsql_test 3.1.5 { 127 + SELECT rowid FROM y1('test OR xyz') ORDER BY rank 128 + } {3 2 1} 129 + 130 + 131 + do_execsql_test 3.2.1 { 132 + CREATE VIRTUAL TABLE z1 USING fts5(a, detail=%DETAIL%); 133 + INSERT INTO z1 VALUES('wrinkle in time'); 134 + SELECT * FROM z1 WHERE z1 MATCH 'wrinkle in time OR a wrinkle in time'; 135 + } {{wrinkle in time}} 136 +} 137 + 138 +do_execsql_test 4.1 { 139 + DROP TABLE IF EXISTS VTest; 140 + CREATE virtual TABLE VTest USING FTS5( 141 + Title, AUthor, tokenize ='porter unicode61 remove_diacritics 1', 142 + columnsize='1', detail=full 143 + ); 144 + INSERT INTO VTest (Title, Author) VALUES ('wrinkle in time', 'Bill Smith'); 94 145 146 + SELECT * FROM VTest WHERE 147 + VTest MATCH 'wrinkle in time OR a wrinkle in time' ORDER BY rank; 148 +} {{wrinkle in time} {Bill Smith}} 95 149 96 150 97 151 98 152 99 153 finish_test 100 154
Changes to ext/fts5/test/fts5simple.test.
261 261 # Test that character 0x1A is allowed in fts5 barewords. 262 262 # 263 263 do_test 11.0 { 264 264 execsql "CREATE VIRTUAL TABLE t4 USING fts5(x, tokenize=\"ascii tokenchars '\x1A'\")" 265 265 execsql " 266 266 INSERT INTO t4 VALUES('a b c \x1A'); 267 267 INSERT INTO t4 VALUES('a b c d\x1A'); 268 - INSERT INTO t4 VALUES('a b c \x1Ad'); 268 + INSERT INTO t4 VALUES('a b c \x1Ag'); 269 269 INSERT INTO t4 VALUES('a b c d'); 270 270 " 271 271 } {} 272 272 273 273 do_test 11.1 { 274 274 execsql "SELECT rowid FROM t4('\x1A')" 275 275 } {1} ................................................................................ 336 336 } 337 337 do_test 14.2 { 338 338 fts5_level_segs ttt 339 339 } {1} 340 340 341 341 #------------------------------------------------------------------------- 342 342 db func rnddoc fts5_rnddoc 343 -do_execsql_test 4.0 { 343 +do_execsql_test 14.3 { 344 344 CREATE VIRTUAL TABLE x1 USING fts5(x); 345 345 INSERT INTO x1(x1, rank) VALUES('pgsz', 32); 346 346 347 347 WITH ii(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10 ) 348 348 INSERT INTO x1 SELECT rnddoc(5) FROM ii; 349 349 } 350 350 351 -do_execsql_test 4.1 { 351 +do_execsql_test 14.4 { 352 352 SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads' 353 -} {0 {} 4} 353 +} {0 {} 3} 354 354 355 355 #------------------------------------------------------------------------- 356 356 reset_db 357 357 do_execsql_test 15.0 { 358 358 CREATE VIRTUAL TABLE x2 USING fts5(x, prefix=1); 359 359 INSERT INTO x2 VALUES('ab'); 360 360 } ................................................................................ 443 443 execsql { INSERT INTO x1(x1) VALUES('optimize'); } 444 444 execsql { DELETE FROM x1 WHERE rowid = 4; } 445 445 } {} 446 446 do_execsql_test 20.2 { 447 447 INSERT INTO x1(x1) VALUES('optimize'); 448 448 INSERT INTO x1(x1) VALUES('integrity-check'); 449 449 } {} 450 + 451 +#------------------------------------------------------------------------- 452 +reset_db 453 +set doc "a b [string repeat x 100000]" 454 +do_execsql_test 21.0 { 455 + CREATE VIRTUAL TABLE x1 USING fts5(x); 456 + INSERT INTO x1(rowid, x) VALUES(11111, $doc); 457 + INSERT INTO x1(rowid, x) VALUES(11112, $doc); 458 +} 459 +do_execsql_test 21.1 { 460 + INSERT INTO x1(x1) VALUES('integrity-check'); 461 +} 462 +do_execsql_test 21.2 { 463 + SELECT rowid FROM x1($doc); 464 +} {11111 11112} 465 +do_execsql_test 21.3 { 466 + DELETE FROM x1 WHERE rowid=11111; 467 + INSERT INTO x1(x1) VALUES('integrity-check'); 468 + SELECT rowid FROM x1($doc); 469 +} {11112} 450 470 451 471 finish_test
Changes to ext/fts5/test/fts5simple2.test.
327 327 INSERT INTO t2(rowid, x) VALUES(1, 'a b c'); 328 328 INSERT INTO t2(rowid, x) VALUES(456, 'a b c'); 329 329 INSERT INTO t2(rowid, x) VALUES(1000, 'a b c'); 330 330 COMMIT; 331 331 UPDATE t2 SET x=x; 332 332 DELETE FROM t2; 333 333 } 334 + 335 +#------------------------------------------------------------------------- 336 +# 337 +reset_db 338 +do_execsql_test 17.0 { 339 + CREATE VIRTUAL TABLE t2 USING fts5(x, y); 340 + BEGIN; 341 + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); 342 + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); 343 + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); 344 + COMMIT; 345 +} 346 +do_execsql_test 17.1 { SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 } 347 +do_execsql_test 17.2 { 348 + BEGIN; 349 + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); 350 + SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 ; 351 +} 352 +do_execsql_test 17.3 { 353 + COMMIT 354 +} 355 + 356 +reset_db 357 +do_execsql_test 17.4 { 358 + CREATE VIRTUAL TABLE t2 USING fts5(x, y); 359 + BEGIN; 360 + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); 361 + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); 362 + SELECT * FROM t2('y:a*') WHERE rowid>66; 363 +} 364 +do_execsql_test 17.5 { SELECT * FROM t2('x:b* OR y:a*') } 365 +do_execsql_test 17.5 { COMMIT ; SELECT * FROM t2('x:b* OR y:a*') } 366 +do_execsql_test 17.6 { 367 + SELECT * FROM t2('x:b* OR y:a*') WHERE rowid>55 368 +} 334 369 335 370 #db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM t2_data} {puts $r} 336 371 337 372 finish_test 338 373
Changes to ext/fts5/test/fts5simple3.test.
75 75 do_execsql_test 3.0 { 76 76 CREATE VIRTUAL TABLE x3 USING fts5(one); 77 77 INSERT INTO x3 VALUES('a b c'); 78 78 INSERT INTO x3 VALUES('c b a'); 79 79 INSERT INTO x3 VALUES('o t t'); 80 80 SELECT * FROM x3('x OR y OR z'); 81 81 } 82 + 83 +#------------------------------------------------------------------------- 84 +# Test that a crash occuring when the second or subsequent tokens in a 85 +# phrase matched zero rows has been fixed. 86 +# 87 +do_execsql_test 4.0 { 88 + CREATE VIRTUAL TABLE t1 USING fts5(x); 89 + INSERT INTO t1 VALUES('ab'); 90 + INSERT INTO t1 VALUES('cd'); 91 + INSERT INTO t1 VALUES('ab cd'); 92 + INSERT INTO t1 VALUES('ab cdXXX'); 93 + INSERT INTO t1 VALUES('abXXX cd'); 94 +} 95 +do_execsql_test 4.1 { 96 + SELECT * FROM t1('"ab cd" OR "ab cd" *'); 97 +} {{ab cd} {ab cdXXX}} 98 +do_execsql_test 4.2 { 99 + SELECT * FROM t1('"xy zz" OR "ab cd" *'); 100 +} {{ab cd} {ab cdXXX}} 101 +do_execsql_test 4.3 { 102 + SELECT * FROM t1('"xy zz" OR "xy zz" *'); 103 +} 104 +do_execsql_test 4.4 { 105 + SELECT * FROM t1('"ab cd" OR "xy zz" *'); 106 +} {{ab cd}} 107 +do_execsql_test 4.5 { 108 + CREATE VIRTUAL TABLE t2 USING fts5(x); 109 + INSERT INTO t2 VALUES('ab'); 110 + INSERT INTO t2 VALUES('cd'); 111 + INSERT INTO t2 VALUES('ef'); 112 +} 113 +do_execsql_test 4.6 { 114 + SELECT * FROM t2('ab + xyz'); 115 +} 82 116 83 117 84 118 finish_test 85 119
Changes to ext/fts5/test/fts5synonym.test.
148 148 reset_db 149 149 fts5_tclnum_register db 150 150 151 151 foreach {tn expr res} { 152 152 1 {abc} {"abc"} 153 153 2 {one} {"one"|"i"|"1"} 154 154 3 {3} {"3"|"iii"|"three"} 155 - 4 {3*} {"3"|"iii"|"three" *} 155 + 4 {3*} {"3" *} 156 156 } { 157 157 do_execsql_test 4.1.$tn { 158 158 SELECT fts5_expr($expr, 'tokenize=tclnum') 159 159 } [list $res] 160 160 } 161 161 162 162 do_execsql_test 4.2.1 {
Changes to ext/fts5/test/fts5tokenizer.test.
257 257 INSERT INTO e6 VALUES('theAquickBbrownCfoxDjumpedWoverXtheYlazyZdog'); 258 258 CREATE VIRTUAL TABLE e7 USING fts5vocab(e6, 'row'); 259 259 SELECT term FROM e7; 260 260 ROLLBACK; 261 261 } { 262 262 brown dog fox jump lazi over quick the 263 263 } 264 + 265 +#------------------------------------------------------------------------- 266 +# Check that the FTS5_TOKENIZE_PREFIX flag is passed to the tokenizer 267 +# implementation. 268 +# 269 +reset_db 270 +proc tcl_create {args} { return "tcl_tokenize" } 271 +sqlite3_fts5_create_tokenizer db tcl tcl_create 272 +set ::flags [list] 273 +proc tcl_tokenize {tflags text} { 274 + lappend ::flags $tflags 275 + foreach {w iStart iEnd} [fts5_tokenize_split $text] { 276 + sqlite3_fts5_token $w $iStart $iEnd 277 + } 278 +} 279 + 280 +do_execsql_test 9.1.1 { 281 + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl); 282 + INSERT INTO t1 VALUES('abc'); 283 + INSERT INTO t1 VALUES('xyz'); 284 +} {} 285 +do_test 9.1.2 { set ::flags } {document document} 286 + 287 +set ::flags [list] 288 +do_execsql_test 9.2.1 { SELECT * FROM t1('abc'); } {abc} 289 +do_test 9.2.2 { set ::flags } {query} 290 + 291 +set ::flags [list] 292 +do_execsql_test 9.3.1 { SELECT * FROM t1('ab*'); } {abc} 293 +do_test 9.3.2 { set ::flags } {prefixquery} 294 + 295 +set ::flags [list] 296 +do_execsql_test 9.4.1 { SELECT * FROM t1('"abc xyz" *'); } {} 297 +do_test 9.4.2 { set ::flags } {prefixquery} 298 + 299 +set ::flags [list] 300 +do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {} 301 +do_test 9.5.2 { set ::flags } {query} 302 + 264 303 265 304 finish_test 266 305
Changes to ext/fts5/test/fts5unicode2.test.
156 156 the maximum x value. 157 157 } 158 158 3 "ROW" { 159 159 ...returns the value of y on the same [row] that contains 160 160 the maximum x value. 161 161 } 162 162 4 "rollback" { 163 - ...[ROLLBACK]. Instead, the pending statement 164 - will return SQLITE_ABORT upon next access after the [ROLLBACK]. 163 + Pending statements no longer block [ROLLBACK]. Instead, the pending 164 + statement will return SQLITE_ABORT upon... 165 165 } 166 166 5 "rOllback" { 167 - ...[ROLLBACK]. Instead, the pending statement 168 - will return SQLITE_ABORT upon next access after the [ROLLBACK]. 167 + Pending statements no longer block [ROLLBACK]. Instead, the pending 168 + statement will return SQLITE_ABORT upon... 169 169 } 170 170 6 "lang*" { 171 171 Added support for the FTS4 [languageid] option. 172 172 } 173 173 } { 174 174 do_test 2.$tn { 175 175 set q [mapdoc $query]
Changes to ext/fts5/test/fts5vocab.test.
438 438 } else { 439 439 do_catchsql_test 8.2.2 { 440 440 SELECT * FROM x1_c 441 441 } {1 {database disk image is malformed}} 442 442 } 443 443 444 444 sqlite3_fts5_may_be_corrupt 0 445 +} 446 + 447 +#------------------------------------------------------------------------- 448 +# Test that both "ORDER BY term" and "ORDER BY term DESC" work. 449 +# 450 +reset_db 451 +do_execsql_test 9.1 { 452 + CREATE VIRTUAL TABLE x1 USING fts5(x); 453 + INSERT INTO x1 VALUES('def ABC ghi'); 454 + INSERT INTO x1 VALUES('DEF abc GHI'); 455 +} 445 456 457 +do_execsql_test 9.2 { 458 + CREATE VIRTUAL TABLE rrr USING fts5vocab(x1, row); 459 + SELECT * FROM rrr 460 +} { 461 + abc 2 2 def 2 2 ghi 2 2 462 +} 463 +do_execsql_test 9.3 { 464 + SELECT * FROM rrr ORDER BY term ASC 465 +} { 466 + abc 2 2 def 2 2 ghi 2 2 467 +} 468 +do_execsql_test 9.4 { 469 + SELECT * FROM rrr ORDER BY term DESC 470 +} { 471 + ghi 2 2 def 2 2 abc 2 2 446 472 } 473 +do_test 9.5 { 474 + set e2 [db eval { EXPLAIN SELECT * FROM rrr ORDER BY term ASC }] 475 + expr [lsearch $e2 SorterSort]<0 476 +} 1 477 +do_test 9.6 { 478 + set e2 [db eval { EXPLAIN SELECT * FROM rrr ORDER BY term DESC }] 479 + expr [lsearch $e2 SorterSort]<0 480 +} 0 481 + 482 + 447 483 448 484 finish_test 449 485
Changes to ext/fts5/tool/fts5txt2db.tcl.
13 13 {fts5 "use fts5 (this is the default)"} 14 14 {fts4 "use fts4"} 15 15 {colsize "10 10 10" "list of column sizes"} 16 16 {tblname "t1" "table name to create"} 17 17 {detail "full" "Fts5 detail mode to use"} 18 18 {repeat 1 "Load each file this many times"} 19 19 {prefix "" "Fts prefix= option"} 20 + {trans 1 "True to use a transaction"} 20 21 database 21 22 file... 22 23 } { 23 24 This script is designed to create fts4/5 tables with more than one column. 24 25 The -colsize option should be set to a Tcl list of integer values, one for 25 26 each column in the table. Each value is the number of tokens that will be 26 27 inserted into the column value for each row. For example, setting the -colsize ................................................................................ 210 211 set cols [create_table] 211 212 set sql "INSERT INTO $A(tblname) VALUES(\$R([lindex $cols 0])" 212 213 foreach c [lrange $cols 1 end] { 213 214 append sql ", \$R($c)" 214 215 } 215 216 append sql ")" 216 217 217 -db eval BEGIN 218 +if {$A(trans)} { db eval BEGIN } 218 219 while {$i < $N} { 219 220 foreach c $cols s $A(colsize) { 220 221 set R($c) [lrange $tokens $i [expr $i+$s-1]] 221 222 incr i $s 222 223 } 223 224 db eval $sql 224 225 } 225 -db eval COMMIT 226 +if {$A(trans)} { db eval COMMIT } 226 227 227 228 228 229
Changes to ext/icu/icu.c.
345 345 ** To access ICU "language specific" case mapping, upper() or lower() 346 346 ** should be invoked with two arguments. The second argument is the name 347 347 ** of the locale to use. Passing an empty string ("") or SQL NULL value 348 348 ** as the second argument is the same as invoking the 1 argument version 349 349 ** of upper() or lower(). 350 350 ** 351 351 ** lower('I', 'en_us') -> 'i' 352 -** lower('I', 'tr_tr') -> 'ı' (small dotless i) 352 +** lower('I', 'tr_tr') -> '\u131' (small dotless i) 353 353 ** 354 354 ** http://www.icu-project.org/userguide/posix.html#case_mappings 355 355 */ 356 356 static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ 357 - const UChar *zInput; 358 - UChar *zOutput = 0; 359 - int nInput; 360 - int nOut; 357 + const UChar *zInput; /* Pointer to input string */ 358 + UChar *zOutput = 0; /* Pointer to output buffer */ 359 + int nInput; /* Size of utf-16 input string in bytes */ 360 + int nOut; /* Size of output buffer in bytes */ 361 361 int cnt; 362 + int bToUpper; /* True for toupper(), false for tolower() */ 362 363 UErrorCode status; 363 364 const char *zLocale = 0; 364 365 365 366 assert(nArg==1 || nArg==2); 367 + bToUpper = (sqlite3_user_data(p)!=0); 366 368 if( nArg==2 ){ 367 369 zLocale = (const char *)sqlite3_value_text(apArg[1]); 368 370 } 369 371 370 372 zInput = sqlite3_value_text16(apArg[0]); 371 373 if( !zInput ){ 372 374 return; ................................................................................ 382 384 if( zNew==0 ){ 383 385 sqlite3_free(zOutput); 384 386 sqlite3_result_error_nomem(p); 385 387 return; 386 388 } 387 389 zOutput = zNew; 388 390 status = U_ZERO_ERROR; 389 - if( sqlite3_user_data(p) ){ 391 + if( bToUpper ){ 390 392 nOut = 2*u_strToUpper(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); 391 393 }else{ 392 394 nOut = 2*u_strToLower(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); 393 395 } 394 - if( !U_SUCCESS(status) ){ 395 - if( status==U_BUFFER_OVERFLOW_ERROR ) continue; 396 - icuFunctionError(p, 397 - sqlite3_user_data(p) ? "u_strToUpper" : "u_strToLower", status); 398 - return; 396 + 397 + if( U_SUCCESS(status) ){ 398 + sqlite3_result_text16(p, zOutput, nOut, xFree); 399 + }else if( status==U_BUFFER_OVERFLOW_ERROR ){ 400 + assert( cnt==0 ); 401 + continue; 402 + }else{ 403 + icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status); 399 404 } 405 + return; 400 406 } 401 - sqlite3_result_text16(p, zOutput, nOut, xFree); 407 + assert( 0 ); /* Unreachable */ 402 408 } 403 409 404 410 /* 405 411 ** Collation sequence destructor function. The pCtx argument points to 406 412 ** a UCollator structure previously allocated using ucol_open(). 407 413 */ 408 414 static void icuCollationDel(void *pCtx){ ................................................................................ 483 489 } 484 490 } 485 491 486 492 /* 487 493 ** Register the ICU extension functions with database db. 488 494 */ 489 495 int sqlite3IcuInit(sqlite3 *db){ 490 - struct IcuScalar { 496 + static const struct IcuScalar { 491 497 const char *zName; /* Function name */ 492 - int nArg; /* Number of arguments */ 493 - int enc; /* Optimal text encoding */ 494 - void *pContext; /* sqlite3_user_data() context */ 498 + unsigned char nArg; /* Number of arguments */ 499 + unsigned short enc; /* Optimal text encoding */ 500 + unsigned char iContext; /* sqlite3_user_data() context */ 495 501 void (*xFunc)(sqlite3_context*,int,sqlite3_value**); 496 502 } scalars[] = { 497 - {"regexp", 2, SQLITE_ANY, 0, icuRegexpFunc}, 498 - 499 - {"lower", 1, SQLITE_UTF16, 0, icuCaseFunc16}, 500 - {"lower", 2, SQLITE_UTF16, 0, icuCaseFunc16}, 501 - {"upper", 1, SQLITE_UTF16, (void*)1, icuCaseFunc16}, 502 - {"upper", 2, SQLITE_UTF16, (void*)1, icuCaseFunc16}, 503 - 504 - {"lower", 1, SQLITE_UTF8, 0, icuCaseFunc16}, 505 - {"lower", 2, SQLITE_UTF8, 0, icuCaseFunc16}, 506 - {"upper", 1, SQLITE_UTF8, (void*)1, icuCaseFunc16}, 507 - {"upper", 2, SQLITE_UTF8, (void*)1, icuCaseFunc16}, 508 - 509 - {"like", 2, SQLITE_UTF8, 0, icuLikeFunc}, 510 - {"like", 3, SQLITE_UTF8, 0, icuLikeFunc}, 511 - 512 - {"icu_load_collation", 2, SQLITE_UTF8, (void*)db, icuLoadCollation}, 503 + {"icu_load_collation", 2, SQLITE_UTF8, 1, icuLoadCollation}, 504 + {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC, 0, icuRegexpFunc}, 505 + {"lower", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, 506 + {"lower", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, 507 + {"upper", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, 508 + {"upper", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, 509 + {"lower", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, 510 + {"lower", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, 511 + {"upper", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, 512 + {"upper", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, 513 + {"like", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, 514 + {"like", 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, 513 515 }; 514 - 515 516 int rc = SQLITE_OK; 516 517 int i; 517 518 519 + 518 520 for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){ 519 - struct IcuScalar *p = &scalars[i]; 521 + const struct IcuScalar *p = &scalars[i]; 520 522 rc = sqlite3_create_function( 521 - db, p->zName, p->nArg, p->enc, p->pContext, p->xFunc, 0, 0 523 + db, p->zName, p->nArg, p->enc, 524 + p->iContext ? (void*)db : (void*)0, 525 + p->xFunc, 0, 0 522 526 ); 523 527 } 524 528 525 529 return rc; 526 530 } 527 531 528 532 #if !SQLITE_CORE
Added ext/misc/README.md.
1 +## Miscellaneous Extensions 2 + 3 +This folder contains a collection of smaller loadable extensions. 4 +See <https://www.sqlite.org/loadext.html> for instructions on how 5 +to compile and use loadable extensions. 6 +Each extension in this folder is implemented in a single file of C code. 7 + 8 +Each source file contains a description in its header comment. See the 9 +header comments for details about each extension. Additional notes are 10 +as follows: 11 + 12 + * **carray.c** — This module implements the 13 + [carray](https://www.sqlite.org/carray.html) table-valued function. 14 + It is a good example of how to go about implementing a custom 15 + [table-valued function](https://www.sqlite.org/vtab.html#tabfunc2). 16 + 17 + * **dbdump.c** — This is not actually a loadable extension, but 18 + rather a library that implements an approximate equivalent to the 19 + ".dump" command of the 20 + [command-line shell](https://www.sqlite.org/cli.html). 21 + 22 + * **memvfs.c** — This file implements a custom 23 + [VFS](https://www.sqlite.org/vfs.html) that stores an entire database 24 + file in a single block of RAM. It serves as a good example of how 25 + to implement a simple custom VFS. 26 + 27 + * **rot13.c** — This file implements the very simple rot13() 28 + substitution function. This file makes a good template for implementing 29 + new custom SQL functions for SQLite. 30 + 31 + * **series.c** — This is an implementation of the 32 + "generate_series" [virtual table](https://www.sqlite.org/vtab.html). 33 + It can make a good template for new custom virtual table implementations. 34 + 35 + * **shathree.c** — An implementation of the sha3() and 36 + sha3_query() SQL functions. The file is named "shathree.c" instead 37 + of "sha3.c" because the default entry point names in SQLite are based 38 + on the source filename with digits removed, so if we used the name 39 + "sha3.c" then the entry point would conflict with the prior "sha1.c" 40 + extension.
Changes to ext/misc/amatch.c.
621 621 { 622 622 pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo ); 623 623 if( pRule==0 ){ 624 624 rc = SQLITE_NOMEM; 625 625 }else{ 626 626 memset(pRule, 0, sizeof(*pRule)); 627 627 pRule->zFrom = &pRule->zTo[nTo+1]; 628 - pRule->nFrom = nFrom; 628 + pRule->nFrom = (amatch_len)nFrom; 629 629 memcpy(pRule->zFrom, zFrom, nFrom+1); 630 630 memcpy(pRule->zTo, zTo, nTo+1); 631 - pRule->nTo = nTo; 631 + pRule->nTo = (amatch_len)nTo; 632 632 pRule->rCost = rCost; 633 633 pRule->iLang = (int)iLang; 634 634 } 635 635 } 636 636 637 637 *ppRule = pRule; 638 638 return rc; ................................................................................ 1077 1077 } 1078 1078 pWord = sqlite3_malloc( sizeof(*pWord) + nBase + nTail - 1 ); 1079 1079 if( pWord==0 ) return; 1080 1080 memset(pWord, 0, sizeof(*pWord)); 1081 1081 pWord->rCost = rCost; 1082 1082 pWord->iSeq = pCur->nWord++; 1083 1083 amatchWriteCost(pWord); 1084 - pWord->nMatch = nMatch; 1084 + pWord->nMatch = (short)nMatch; 1085 1085 pWord->pNext = pCur->pAllWords; 1086 1086 pCur->pAllWords = pWord; 1087 1087 pWord->sCost.zKey = pWord->zCost; 1088 1088 pWord->sCost.pWord = pWord; 1089 1089 pOther = amatchAvlInsert(&pCur->pCost, &pWord->sCost); 1090 1090 assert( pOther==0 ); (void)pOther; 1091 1091 pWord->sWord.zKey = pWord->zWord; ................................................................................ 1158 1158 #ifdef AMATCH_TRACE_1 1159 1159 printf("PROCESS [%s][%.*s^%s] %d (\"%s\" \"%s\")\n", 1160 1160 pWord->zWord+2, pWord->nMatch, pCur->zInput, pCur->zInput+pWord->nMatch, 1161 1161 pWord->rCost, pWord->zWord, pWord->zCost); 1162 1162 #endif 1163 1163 nWord = (int)strlen(pWord->zWord+2); 1164 1164 if( nWord+20>nBuf ){ 1165 - nBuf = nWord+100; 1165 + nBuf = (char)(nWord+100); 1166 1166 zBuf = sqlite3_realloc(zBuf, nBuf); 1167 1167 if( zBuf==0 ) return SQLITE_NOMEM; 1168 1168 } 1169 1169 amatchStrcpy(zBuf, pWord->zWord+2); 1170 1170 zNext[0] = 0; 1171 1171 zNextIn[0] = pCur->zInput[pWord->nMatch]; 1172 1172 if( zNextIn[0] ){
Added ext/misc/carray.c.
1 +/* 2 +** 2016-06-29 3 +** 4 +** The author disclaims copyright to this source code. In place of 5 +** a legal notice, here is a blessing: 6 +** 7 +** May you do good and not evil. 8 +** May you find forgiveness for yourself and forgive others. 9 +** May you share freely, never taking more than you give. 10 +** 11 +************************************************************************* 12 +** 13 +** This file demonstrates how to create a table-valued-function that 14 +** returns the values in a C-language array. 15 +** Examples: 16 +** 17 +** SELECT * FROM carray($ptr,5) 18 +** 19 +** The query above returns 5 integers contained in a C-language array 20 +** at the address $ptr. $ptr is a pointer to the array of integers that 21 +** has been cast to an integer. 22 +** 23 +** There is an optional third parameter to determine the datatype of 24 +** the C-language array. Allowed values of the third parameter are 25 +** 'int32', 'int64', 'double', 'char*'. Example: 26 +** 27 +** SELECT * FROM carray($ptr,10,'char*'); 28 +** 29 +** HOW IT WORKS 30 +** 31 +** The carray "function" is really a virtual table with the 32 +** following schema: 33 +** 34 +** CREATE TABLE carray( 35 +** value, 36 +** pointer HIDDEN, 37 +** count HIDDEN, 38 +** ctype TEXT HIDDEN 39 +** ); 40 +** 41 +** If the hidden columns "pointer" and "count" are unconstrained, then 42 +** the virtual table has no rows. Otherwise, the virtual table interprets 43 +** the integer value of "pointer" as a pointer to the array and "count" 44 +** as the number of elements in the array. The virtual table steps through 45 +** the array, element by element. 46 +*/ 47 +#include "sqlite3ext.h" 48 +SQLITE_EXTENSION_INIT1 49 +#include <assert.h> 50 +#include <string.h> 51 + 52 +#ifndef SQLITE_OMIT_VIRTUALTABLE 53 + 54 +/* 55 +** Allowed datatypes 56 +*/ 57 +#define CARRAY_INT32 0 58 +#define CARRAY_INT64 1 59 +#define CARRAY_DOUBLE 2 60 +#define CARRAY_TEXT 3 61 + 62 +/* 63 +** Names of types 64 +*/ 65 +static const char *azType[] = { "int32", "int64", "double", "char*" }; 66 + 67 + 68 +/* carray_cursor is a subclass of sqlite3_vtab_cursor which will 69 +** serve as the underlying representation of a cursor that scans 70 +** over rows of the result 71 +*/ 72 +typedef struct carray_cursor carray_cursor; 73 +struct carray_cursor { 74 + sqlite3_vtab_cursor base; /* Base class - must be first */ 75 + sqlite3_int64 iRowid; /* The rowid */ 76 + sqlite3_int64 iPtr; /* Pointer to array of values */ 77 + sqlite3_int64 iCnt; /* Number of integers in the array */ 78 + unsigned char eType; /* One of the CARRAY_type values */ 79 +}; 80 + 81 +/* 82 +** The carrayConnect() method is invoked to create a new 83 +** carray_vtab that describes the carray virtual table. 84 +** 85 +** Think of this routine as the constructor for carray_vtab objects. 86 +** 87 +** All this routine needs to do is: 88 +** 89 +** (1) Allocate the carray_vtab object and initialize all fields. 90 +** 91 +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the 92 +** result set of queries against carray will look like. 93 +*/ 94 +static int carrayConnect( 95 + sqlite3 *db, 96 + void *pAux, 97 + int argc, const char *const*argv, 98 + sqlite3_vtab **ppVtab, 99 + char **pzErr 100 +){ 101 + sqlite3_vtab *pNew; 102 + int rc; 103 + 104 +/* Column numbers */ 105 +#define CARRAY_COLUMN_VALUE 0 106 +#define CARRAY_COLUMN_POINTER 1 107 +#define CARRAY_COLUMN_COUNT 2 108 +#define CARRAY_COLUMN_CTYPE 3 109 + 110 + rc = sqlite3_declare_vtab(db, 111 + "CREATE TABLE x(value,pointer hidden,count hidden,ctype hidden)"); 112 + if( rc==SQLITE_OK ){ 113 + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); 114 + if( pNew==0 ) return SQLITE_NOMEM; 115 + memset(pNew, 0, sizeof(*pNew)); 116 + } 117 + return rc; 118 +} 119 + 120 +/* 121 +** This method is the destructor for carray_cursor objects. 122 +*/ 123 +static int carrayDisconnect(sqlite3_vtab *pVtab){ 124 + sqlite3_free(pVtab); 125 + return SQLITE_OK; 126 +} 127 + 128 +/* 129 +** Constructor for a new carray_cursor object. 130 +*/ 131 +static int carrayOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ 132 + carray_cursor *pCur; 133 + pCur = sqlite3_malloc( sizeof(*pCur) ); 134 + if( pCur==0 ) return SQLITE_NOMEM; 135 + memset(pCur, 0, sizeof(*pCur)); 136 + *ppCursor = &pCur->base; 137 + return SQLITE_OK; 138 +} 139 + 140 +/* 141 +** Destructor for a carray_cursor. 142 +*/ 143 +static int carrayClose(sqlite3_vtab_cursor *cur){ 144 + sqlite3_free(cur); 145 + return SQLITE_OK; 146 +} 147 + 148 + 149 +/* 150 +** Advance a carray_cursor to its next row of output. 151 +*/ 152 +static int carrayNext(sqlite3_vtab_cursor *cur){ 153 + carray_cursor *pCur = (carray_cursor*)cur; 154 + pCur->iRowid++; 155 + return SQLITE_OK; 156 +} 157 + 158 +/* 159 +** Return values of columns for the row at which the carray_cursor 160 +** is currently pointing. 161 +*/ 162 +static int carrayColumn( 163 + sqlite3_vtab_cursor *cur, /* The cursor */ 164 + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ 165 + int i /* Which column to return */ 166 +){ 167 + carray_cursor *pCur = (carray_cursor*)cur; 168 + sqlite3_int64 x = 0; 169 + switch( i ){ 170 + case CARRAY_COLUMN_POINTER: x = pCur->iPtr; break; 171 + case CARRAY_COLUMN_COUNT: x = pCur->iCnt; break; 172 + case CARRAY_COLUMN_CTYPE: { 173 + sqlite3_result_text(ctx, azType[pCur->eType], -1, SQLITE_STATIC); 174 + return SQLITE_OK; 175 + } 176 + default: { 177 + switch( pCur->eType ){ 178 + case CARRAY_INT32: { 179 + int *p = (int*)pCur->iPtr; 180 + sqlite3_result_int(ctx, p[pCur->iRowid-1]); 181 + return SQLITE_OK; 182 + } 183 + case CARRAY_INT64: { 184 + sqlite3_int64 *p = (sqlite3_int64*)pCur->iPtr; 185 + sqlite3_result_int64(ctx, p[pCur->iRowid-1]); 186 + return SQLITE_OK; 187 + } 188 + case CARRAY_DOUBLE: { 189 + double *p = (double*)pCur->iPtr; 190 + sqlite3_result_double(ctx, p[pCur->iRowid-1]); 191 + return SQLITE_OK; 192 + } 193 + case CARRAY_TEXT: { 194 + const char **p = (const char**)pCur->iPtr; 195 + sqlite3_result_text(ctx, p[pCur->iRowid-1], -1, SQLITE_TRANSIENT); 196 + return SQLITE_OK; 197 + } 198 + } 199 + } 200 + } 201 + sqlite3_result_int64(ctx, x); 202 + return SQLITE_OK; 203 +} 204 + 205 +/* 206 +** Return the rowid for the current row. In this implementation, the 207 +** rowid is the same as the output value. 208 +*/ 209 +static int carrayRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ 210 + carray_cursor *pCur = (carray_cursor*)cur; 211 + *pRowid = pCur->iRowid; 212 + return SQLITE_OK; 213 +} 214 + 215 +/* 216 +** Return TRUE if the cursor has been moved off of the last 217 +** row of output. 218 +*/ 219 +static int carrayEof(sqlite3_vtab_cursor *cur){ 220 + carray_cursor *pCur = (carray_cursor*)cur; 221 + return pCur->iRowid>pCur->iCnt; 222 +} 223 + 224 +/* 225 +** This method is called to "rewind" the carray_cursor object back 226 +** to the first row of output. 227 +*/ 228 +static int carrayFilter( 229 + sqlite3_vtab_cursor *pVtabCursor, 230 + int idxNum, const char *idxStr, 231 + int argc, sqlite3_value **argv 232 +){ 233 + carray_cursor *pCur = (carray_cursor *)pVtabCursor; 234 + if( idxNum ){ 235 + pCur->iPtr = sqlite3_value_int64(argv[0]); 236 + pCur->iCnt = sqlite3_value_int64(argv[1]); 237 + if( idxNum<3 ){ 238 + pCur->eType = CARRAY_INT32; 239 + }else{ 240 + unsigned char i; 241 + const char *zType = (const char*)sqlite3_value_text(argv[2]); 242 + for(i=0; i<sizeof(azType)/sizeof(azType[0]); i++){ 243 + if( sqlite3_stricmp(zType, azType[i])==0 ) break; 244 + } 245 + if( i>=sizeof(azType)/sizeof(azType[0]) ){ 246 + pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf( 247 + "unknown datatype: %Q", zType); 248 + return SQLITE_ERROR; 249 + }else{ 250 + pCur->eType = i; 251 + } 252 + } 253 + }else{ 254 + pCur->iPtr = 0; 255 + pCur->iCnt = 0; 256 + } 257 + pCur->iRowid = 1; 258 + return SQLITE_OK; 259 +} 260 + 261 +/* 262 +** SQLite will invoke this method one or more times while planning a query 263 +** that uses the carray virtual table. This routine needs to create 264 +** a query plan for each invocation and compute an estimated cost for that 265 +** plan. 266 +** 267 +** In this implementation idxNum is used to represent the 268 +** query plan. idxStr is unused. 269 +** 270 +** idxNum is 2 if the pointer= and count= constraints exist, 271 +** 3 if the ctype= constraint also exists, and is 0 otherwise. 272 +** If idxNum is 0, then carray becomes an empty table. 273 +*/ 274 +static int carrayBestIndex( 275 + sqlite3_vtab *tab, 276 + sqlite3_index_info *pIdxInfo 277 +){ 278 + int i; /* Loop over constraints */ 279 + int ptrIdx = -1; /* Index of the pointer= constraint, or -1 if none */ 280 + int cntIdx = -1; /* Index of the count= constraint, or -1 if none */ 281 + int ctypeIdx = -1; /* Index of the ctype= constraint, or -1 if none */ 282 + 283 + const struct sqlite3_index_constraint *pConstraint; 284 + pConstraint = pIdxInfo->aConstraint; 285 + for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){ 286 + if( pConstraint->usable==0 ) continue; 287 + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; 288 + switch( pConstraint->iColumn ){ 289 + case CARRAY_COLUMN_POINTER: 290 + ptrIdx = i; 291 + break; 292 + case CARRAY_COLUMN_COUNT: 293 + cntIdx = i; 294 + break; 295 + case CARRAY_COLUMN_CTYPE: 296 + ctypeIdx = i; 297 + break; 298 + } 299 + } 300 + if( ptrIdx>=0 && cntIdx>=0 ){ 301 + pIdxInfo->aConstraintUsage[ptrIdx].argvIndex = 1; 302 + pIdxInfo->aConstraintUsage[ptrIdx].omit = 1; 303 + pIdxInfo->aConstraintUsage[cntIdx].argvIndex = 2; 304 + pIdxInfo->aConstraintUsage[cntIdx].omit = 1; 305 + pIdxInfo->estimatedCost = (double)1; 306 + pIdxInfo->estimatedRows = 100; 307 + pIdxInfo->idxNum = 2; 308 + if( ctypeIdx>=0 ){ 309 + pIdxInfo->aConstraintUsage[ctypeIdx].argvIndex = 3; 310 + pIdxInfo->aConstraintUsage[ctypeIdx].omit = 1; 311 + pIdxInfo->idxNum = 3; 312 + } 313 + }else{ 314 + pIdxInfo->estimatedCost = (double)2147483647; 315 + pIdxInfo->estimatedRows = 2147483647; 316 + pIdxInfo->idxNum = 0; 317 + } 318 + return SQLITE_OK; 319 +} 320 + 321 +/* 322 +** This following structure defines all the methods for the 323 +** carray virtual table. 324 +*/ 325 +static sqlite3_module carrayModule = { 326 + 0, /* iVersion */ 327 + 0, /* xCreate */ 328 + carrayConnect, /* xConnect */ 329 + carrayBestIndex, /* xBestIndex */ 330 + carrayDisconnect, /* xDisconnect */ 331 + 0, /* xDestroy */ 332 + carrayOpen, /* xOpen - open a cursor */ 333 + carrayClose, /* xClose - close a cursor */ 334 + carrayFilter, /* xFilter - configure scan constraints */ 335 + carrayNext, /* xNext - advance a cursor */ 336 + carrayEof, /* xEof - check for end of scan */ 337 + carrayColumn, /* xColumn - read data */ 338 + carrayRowid, /* xRowid - read data */ 339 + 0, /* xUpdate */ 340 + 0, /* xBegin */ 341 + 0, /* xSync */ 342 + 0, /* xCommit */ 343 + 0, /* xRollback */ 344 + 0, /* xFindMethod */ 345 + 0, /* xRename */ 346 +}; 347 + 348 +#endif /* SQLITE_OMIT_VIRTUALTABLE */ 349 + 350 +#ifdef _WIN32 351 +__declspec(dllexport) 352 +#endif 353 +int sqlite3_carray_init( 354 + sqlite3 *db, 355 + char **pzErrMsg, 356 + const sqlite3_api_routines *pApi 357 +){ 358 + int rc = SQLITE_OK; 359 + SQLITE_EXTENSION_INIT2(pApi); 360 +#ifndef SQLITE_OMIT_VIRTUALTABLE 361 + rc = sqlite3_create_module(db, "carray", &carrayModule, 0); 362 +#endif 363 + return rc; 364 +}
Added ext/misc/csv.c.
1 +/* 2 +** 2016-05-28 3 +** 4 +** The author disclaims copyright to this source code. In place of 5 +** a legal notice, here is a blessing: 6 +** 7 +** May you do good and not evil. 8 +** May you find forgiveness for yourself and forgive others. 9 +** May you share freely, never taking more than you give. 10 +** 11 +****************************************************************************** 12 +** 13 +** This file contains the implementation of an SQLite virtual table for 14 +** reading CSV files. 15 +** 16 +** Usage: 17 +** 18 +** .load ./csv 19 +** CREATE VIRTUAL TABLE temp.csv USING csv(filename=FILENAME); 20 +** SELECT * FROM csv; 21 +** 22 +** The columns are named "c1", "c2", "c3", ... by default. But the 23 +** application can define its own CREATE TABLE statement as an additional 24 +** parameter. For example: 25 +** 26 +** CREATE VIRTUAL TABLE temp.csv2 USING csv( 27 +** filename = "../http.log", 28 +** schema = "CREATE TABLE x(date,ipaddr,url,referrer,userAgent)" 29 +** ); 30 +** 31 +** Instead of specifying a file, the text of the CSV can be loaded using 32 +** the data= parameter. 33 +** 34 +** If the columns=N parameter is supplied, then the CSV file is assumed to have 35 +** N columns. If the columns parameter is omitted, the CSV file is opened 36 +** as soon as the virtual table is constructed and the first row of the CSV 37 +** is read in order to count the tables. 38 +** 39 +** Some extra debugging features (used for testing virtual tables) are available 40 +** if this module is compiled with -DSQLITE_TEST. 41 +*/ 42 +#include <sqlite3ext.h> 43 +SQLITE_EXTENSION_INIT1 44 +#include <string.h> 45 +#include <stdlib.h> 46 +#include <assert.h> 47 +#include <stdarg.h> 48 +#include <ctype.h> 49 +#include <stdio.h> 50 + 51 +#ifndef SQLITE_OMIT_VIRTUALTABLE 52 + 53 +/* 54 +** A macro to hint to the compiler that a function should not be 55 +** inlined. 56 +*/ 57 +#if defined(__GNUC__) 58 +# define CSV_NOINLINE __attribute__((noinline)) 59 +#elif defined(_MSC_VER) && _MSC_VER>=1310 60 +# define CSV_NOINLINE __declspec(noinline) 61 +#else 62 +# define CSV_NOINLINE 63 +#endif 64 + 65 + 66 +/* Max size of the error message in a CsvReader */ 67 +#define CSV_MXERR 200 68 + 69 +/* Size of the CsvReader input buffer */ 70 +#define CSV_INBUFSZ 1024 71 + 72 +/* A context object used when read a CSV file. */ 73 +typedef struct CsvReader CsvReader; 74 +struct CsvReader { 75 + FILE *in; /* Read the CSV text from this input stream */ 76 + char *z; /* Accumulated text for a field */ 77 + int n; /* Number of bytes in z */ 78 + int nAlloc; /* Space allocated for z[] */ 79 + int nLine; /* Current line number */ 80 + char cTerm; /* Character that terminated the most recent field */ 81 + size_t iIn; /* Next unread character in the input buffer */ 82 + size_t nIn; /* Number of characters in the input buffer */ 83 + char *zIn; /* The input buffer */ 84 + char zErr[CSV_MXERR]; /* Error message */ 85 +}; 86 + 87 +/* Initialize a CsvReader object */ 88 +static void csv_reader_init(CsvReader *p){ 89 + p->in = 0; 90 + p->z = 0; 91 + p->n = 0; 92 + p->nAlloc = 0; 93 + p->nLine = 0; 94 + p->nIn = 0; 95 + p->zIn = 0; 96 + p->zErr[0] = 0; 97 +} 98 + 99 +/* Close and reset a CsvReader object */ 100 +static void csv_reader_reset(CsvReader *p){ 101 + if( p->in ){ 102 + fclose(p->in); 103 + sqlite3_free(p->zIn); 104 + } 105 + sqlite3_free(p->z); 106 + csv_reader_init(p); 107 +} 108 + 109 +/* Report an error on a CsvReader */ 110 +static void csv_errmsg(CsvReader *p, const char *zFormat, ...){ 111 + va_list ap; 112 + va_start(ap, zFormat); 113 + sqlite3_vsnprintf(CSV_MXERR, p->zErr, zFormat, ap); 114 + va_end(ap); 115 +} 116 + 117 +/* Open the file associated with a CsvReader 118 +** Return the number of errors. 119 +*/ 120 +static int csv_reader_open( 121 + CsvReader *p, /* The reader to open */ 122 + const char *zFilename, /* Read from this filename */ 123 + const char *zData /* ... or use this data */ 124 +){ 125 + if( zFilename ){ 126 + p->zIn = sqlite3_malloc( CSV_INBUFSZ ); 127 + if( p->zIn==0 ){ 128 + csv_errmsg(p, "out of memory"); 129 + return 1; 130 + } 131 + p->in = fopen(zFilename, "rb"); 132 + if( p->in==0 ){ 133 + csv_reader_reset(p); 134 + csv_errmsg(p, "cannot open '%s' for reading", zFilename); 135 + return 1; 136 + } 137 + }else{ 138 + assert( p->in==0 ); 139 + p->zIn = (char*)zData; 140 + p->nIn = strlen(zData); 141 + } 142 + return 0; 143 +} 144 + 145 +/* The input buffer has overflowed. Refill the input buffer, then 146 +** return the next character 147 +*/ 148 +static CSV_NOINLINE int csv_getc_refill(CsvReader *p){ 149 + size_t got; 150 + 151 + assert( p->iIn>=p->nIn ); /* Only called on an empty input buffer */ 152 + assert( p->in!=0 ); /* Only called if reading froma file */ 153 + 154 + got = fread(p->zIn, 1, CSV_INBUFSZ, p->in); 155 + if( got==0 ) return EOF; 156 + p->nIn = got; 157 + p->iIn = 1; 158 + return p->zIn[0]; 159 +} 160 + 161 +/* Return the next character of input. Return EOF at end of input. */ 162 +static int csv_getc(CsvReader *p){ 163 + if( p->iIn >= p->nIn ){ 164 + if( p->in!=0 ) return csv_getc_refill(p); 165 + return EOF; 166 + } 167 + return p->zIn[p->iIn++]; 168 +} 169 + 170 +/* Increase the size of p->z and append character c to the end. 171 +** Return 0 on success and non-zero if there is an OOM error */ 172 +static CSV_NOINLINE int csv_resize_and_append(CsvReader *p, char c){ 173 + char *zNew; 174 + int nNew = p->nAlloc*2 + 100; 175 + zNew = sqlite3_realloc64(p->z, nNew); 176 + if( zNew ){ 177 + p->z = zNew; 178 + p->nAlloc = nNew; 179 + p->z[p->n++] = c; 180 + return 0; 181 + }else{ 182 + csv_errmsg(p, "out of memory"); 183 + return 1; 184 + } 185 +} 186 + 187 +/* Append a single character to the CsvReader.z[] array. 188 +** Return 0 on success and non-zero if there is an OOM error */ 189 +static int csv_append(CsvReader *p, char c){ 190 + if( p->n>=p->nAlloc-1 ) return csv_resize_and_append(p, c); 191 + p->z[p->n++] = c; 192 + return 0; 193 +} 194 + 195 +/* Read a single field of CSV text. Compatible with rfc4180 and extended 196 +** with the option of having a separator other than ",". 197 +** 198 +** + Input comes from p->in. 199 +** + Store results in p->z of length p->n. Space to hold p->z comes 200 +** from sqlite3_malloc64(). 201 +** + Keep track of the line number in p->nLine. 202 +** + Store the character that terminates the field in p->cTerm. Store 203 +** EOF on end-of-file. 204 +** 205 +** Return "" at EOF. Return 0 on an OOM error. 206 +*/ 207 +static char *csv_read_one_field(CsvReader *p){ 208 + int c; 209 + p->n = 0; 210 + c = csv_getc(p); 211 + if( c==EOF ){ 212 + p->cTerm = EOF; 213 + return ""; 214 + } 215 + if( c=='"' ){ 216 + int pc, ppc; 217 + int startLine = p->nLine; 218 + pc = ppc = 0; 219 + while( 1 ){ 220 + c = csv_getc(p); 221 + if( c<='"' || pc=='"' ){ 222 + if( c=='\n' ) p->nLine++; 223 + if( c=='"' ){ 224 + if( pc=='"' ){ 225 + pc = 0; 226 + continue; 227 + } 228 + } 229 + if( (c==',' && pc=='"') 230 + || (c=='\n' && pc=='"') 231 + || (c=='\n' && pc=='\r' && ppc=='"') 232 + || (c==EOF && pc=='"') 233 + ){ 234 + do{ p->n--; }while( p->z[p->n]!='"' ); 235 + p->cTerm = (char)c; 236 + break; 237 + } 238 + if( pc=='"' && c!='\r' ){ 239 + csv_errmsg(p, "line %d: unescaped %c character", p->nLine, '"'); 240 + break; 241 + } 242 + if( c==EOF ){ 243 + csv_errmsg(p, "line %d: unterminated %c-quoted field\n", 244 + startLine, '"'); 245 + p->cTerm = (char)c; 246 + break; 247 + } 248 + } 249 + if( csv_append(p, (char)c) ) return 0; 250 + ppc = pc; 251 + pc = c; 252 + } 253 + }else{ 254 + while( c>',' || (c!=EOF && c!=',' && c!='\n') ){ 255 + if( csv_append(p, (char)c) ) return 0; 256 + c = csv_getc(p); 257 + } 258 + if( c=='\n' ){ 259 + p->nLine++; 260 + if( p->n>0 && p->z[p->n-1]=='\r' ) p->n--; 261 + } 262 + p->cTerm = (char)c; 263 + } 264 + if( p->z ) p->z[p->n] = 0; 265 + return p->z; 266 +} 267 + 268 + 269 +/* Forward references to the various virtual table methods implemented 270 +** in this file. */ 271 +static int csvtabCreate(sqlite3*, void*, int, const char*const*, 272 + sqlite3_vtab**,char**); 273 +static int csvtabConnect(sqlite3*, void*, int, const char*const*, 274 + sqlite3_vtab**,char**); 275 +static int csvtabBestIndex(sqlite3_vtab*,sqlite3_index_info*); 276 +static int csvtabDisconnect(sqlite3_vtab*); 277 +static int csvtabOpen(sqlite3_vtab*, sqlite3_vtab_cursor**); 278 +static int csvtabClose(sqlite3_vtab_cursor*); 279 +static int csvtabFilter(sqlite3_vtab_cursor*, int idxNum, const char *idxStr, 280 + int argc, sqlite3_value **argv); 281 +static int csvtabNext(sqlite3_vtab_cursor*); 282 +static int csvtabEof(sqlite3_vtab_cursor*); 283 +static int csvtabColumn(sqlite3_vtab_cursor*,sqlite3_context*,int); 284 +static int csvtabRowid(sqlite3_vtab_cursor*,sqlite3_int64*); 285 + 286 +/* An instance of the CSV virtual table */ 287 +typedef struct CsvTable { 288 + sqlite3_vtab base; /* Base class. Must be first */ 289 + char *zFilename; /* Name of the CSV file */ 290 + char *zData; /* Raw CSV data in lieu of zFilename */ 291 + long iStart; /* Offset to start of data in zFilename */ 292 + int nCol; /* Number of columns in the CSV file */ 293 + unsigned int tstFlags; /* Bit values used for testing */ 294 +} CsvTable; 295 + 296 +/* Allowed values for tstFlags */ 297 +#define CSVTEST_FIDX 0x0001 /* Pretend that constrained searchs cost less*/ 298 + 299 +/* A cursor for the CSV virtual table */ 300 +typedef struct CsvCursor { 301 + sqlite3_vtab_cursor base; /* Base class. Must be first */ 302 + CsvReader rdr; /* The CsvReader object */ 303 + char **azVal; /* Value of the current row */ 304 + int *aLen; /* Length of each entry */ 305 + sqlite3_int64 iRowid; /* The current rowid. Negative for EOF */ 306 +} CsvCursor; 307 + 308 +/* Transfer error message text from a reader into a CsvTable */ 309 +static void csv_xfer_error(CsvTable *pTab, CsvReader *pRdr){ 310 + sqlite3_free(pTab->base.zErrMsg); 311 + pTab->base.zErrMsg = sqlite3_mprintf("%s", pRdr->zErr); 312 +} 313 + 314 +/* 315 +** This method is the destructor fo a CsvTable object. 316 +*/ 317 +static int csvtabDisconnect(sqlite3_vtab *pVtab){ 318 + CsvTable *p = (CsvTable*)pVtab; 319 + sqlite3_free(p->zFilename); 320 + sqlite3_free(p->zData); 321 + sqlite3_free(p); 322 + return SQLITE_OK; 323 +} 324 + 325 +/* Skip leading whitespace. Return a pointer to the first non-whitespace 326 +** character, or to the zero terminator if the string has only whitespace */ 327 +static const char *csv_skip_whitespace(const char *z){ 328 + while( isspace((unsigned char)z[0]) ) z++; 329 + return z; 330 +} 331 + 332 +/* Remove trailing whitespace from the end of string z[] */ 333 +static void csv_trim_whitespace(char *z){ 334 + size_t n = strlen(z); 335 + while( n>0 && isspace((unsigned char)z[n]) ) n--; 336 + z[n] = 0; 337 +} 338 + 339 +/* Dequote the string */ 340 +static void csv_dequote(char *z){ 341 + int j; 342 + char cQuote = z[0]; 343 + size_t i, n; 344 + 345 + if( cQuote!='\'' && cQuote!='"' ) return; 346 + n = strlen(z); 347 + if( n<2 || z[n-1]!=z[0] ) return; 348 + for(i=1, j=0; i<n-1; i++){ 349 + if( z[i]==cQuote && z[i+1]==cQuote ) i++; 350 + z[j++] = z[i]; 351 + } 352 + z[j] = 0; 353 +} 354 + 355 +/* Check to see if the string is of the form: "TAG = VALUE" with optional 356 +** whitespace before and around tokens. If it is, return a pointer to the 357 +** first character of VALUE. If it is not, return NULL. 358 +*/ 359 +static const char *csv_parameter(const char *zTag, int nTag, const char *z){ 360 + z = csv_skip_whitespace(z); 361 + if( strncmp(zTag, z, nTag)!=0 ) return 0; 362 + z = csv_skip_whitespace(z+nTag); 363 + if( z[0]!='=' ) return 0; 364 + return csv_skip_whitespace(z+1); 365 +} 366 + 367 +/* Decode a parameter that requires a dequoted string. 368 +** 369 +** Return 1 if the parameter is seen, or 0 if not. 1 is returned 370 +** even if there is an error. If an error occurs, then an error message 371 +** is left in p->zErr. If there are no errors, p->zErr[0]==0. 372 +*/ 373 +static int csv_string_parameter( 374 + CsvReader *p, /* Leave the error message here, if there is one */ 375 + const char *zParam, /* Parameter we are checking for */ 376 + const char *zArg, /* Raw text of the virtual table argment */ 377 + char **pzVal /* Write the dequoted string value here */ 378 +){ 379 + const char *zValue; 380 + zValue = csv_parameter(zParam,(int)strlen(zParam),zArg); 381 + if( zValue==0 ) return 0; 382 + p->zErr[0] = 0; 383 + if( *pzVal ){ 384 + csv_errmsg(p, "more than one '%s' parameter", zParam); 385 + return 1; 386 + } 387 + *pzVal = sqlite3_mprintf("%s", zValue); 388 + if( *pzVal==0 ){ 389 + csv_errmsg(p, "out of memory"); 390 + return 1; 391 + } 392 + csv_trim_whitespace(*pzVal); 393 + csv_dequote(*pzVal); 394 + return 1; 395 +} 396 + 397 + 398 +/* Return 0 if the argument is false and 1 if it is true. Return -1 if 399 +** we cannot really tell. 400 +*/ 401 +static int csv_boolean(const char *z){ 402 + if( sqlite3_stricmp("yes",z)==0 403 + || sqlite3_stricmp("on",z)==0 404 + || sqlite3_stricmp("true",z)==0 405 + || (z[0]=='1' && z[1]==0) 406 + ){ 407 + return 1; 408 + } 409 + if( sqlite3_stricmp("no",z)==0 410 + || sqlite3_stricmp("off",z)==0 411 + || sqlite3_stricmp("false",z)==0 412 + || (z[0]=='0' && z[1]==0) 413 + ){ 414 + return 0; 415 + } 416 + return -1; 417 +} 418 + 419 + 420 +/* 421 +** Parameters: 422 +** filename=FILENAME Name of file containing CSV content 423 +** data=TEXT Direct CSV content. 424 +** schema=SCHEMA Alternative CSV schema. 425 +** header=YES|NO First row of CSV defines the names of 426 +** columns if "yes". Default "no". 427 +** columns=N Assume the CSV file contains N columns. 428 +** 429 +** Only available if compiled with SQLITE_TEST: 430 +** 431 +** testflags=N Bitmask of test flags. Optional 432 +** 433 +** If schema= is omitted, then the columns are named "c0", "c1", "c2", 434 +** and so forth. If columns=N is omitted, then the file is opened and 435 +** the number of columns in the first row is counted to determine the 436 +** column count. If header=YES, then the first row is skipped. 437 +*/ 438 +static int csvtabConnect( 439 + sqlite3 *db, 440 + void *pAux, 441 + int argc, const char *const*argv, 442 + sqlite3_vtab **ppVtab, 443 + char **pzErr 444 +){ 445 + CsvTable *pNew = 0; /* The CsvTable object to construct */ 446 + int bHeader = -1; /* header= flags. -1 means not seen yet */ 447 + int rc = SQLITE_OK; /* Result code from this routine */ 448 + int i, j; /* Loop counters */ 449 +#ifdef SQLITE_TEST 450 + int tstFlags = 0; /* Value for testflags=N parameter */ 451 +#endif 452 + int nCol = -99; /* Value of the columns= parameter */ 453 + CsvReader sRdr; /* A CSV file reader used to store an error 454 + ** message and/or to count the number of columns */ 455 + static const char *azParam[] = { 456 + "filename", "data", "schema", 457 + }; 458 + char *azPValue[3]; /* Parameter values */ 459 +# define CSV_FILENAME (azPValue[0]) 460 +# define CSV_DATA (azPValue[1]) 461 +# define CSV_SCHEMA (azPValue[2]) 462 + 463 + 464 + assert( sizeof(azPValue)==sizeof(azParam) ); 465 + memset(&sRdr, 0, sizeof(sRdr)); 466 + memset(azPValue, 0, sizeof(azPValue)); 467 + for(i=3; i<argc; i++){ 468 + const char *z = argv[i]; 469 + const char *zValue; 470 + for(j=0; j<sizeof(azParam)/sizeof(azParam[0]); j++){ 471 + if( csv_string_parameter(&sRdr, azParam[j], z, &azPValue[j]) ) break; 472 + } 473 + if( j<sizeof(azParam)/sizeof(azParam[0]) ){ 474 + if( sRdr.zErr[0] ) goto csvtab_connect_error; 475 + }else 476 + if( (zValue = csv_parameter("header",6,z))!=0 ){ 477 + int x; 478 + if( bHeader>=0 ){ 479 + csv_errmsg(&sRdr, "more than one 'header' parameter"); 480 + goto csvtab_connect_error; 481 + } 482 + x = csv_boolean(zValue); 483 + if( x==1 ){ 484 + bHeader = 1; 485 + }else if( x==0 ){ 486 + bHeader = 0; 487 + }else{ 488 + csv_errmsg(&sRdr, "unrecognized argument to 'header': %s", zValue); 489 + goto csvtab_connect_error; 490 + } 491 + }else 492 +#ifdef SQLITE_TEST 493 + if( (zValue = csv_parameter("testflags",9,z))!=0 ){ 494 + tstFlags = (unsigned int)atoi(zValue); 495 + }else 496 +#endif 497 + if( (zValue = csv_parameter("columns",7,z))!=0 ){ 498 + if( nCol>0 ){ 499 + csv_errmsg(&sRdr, "more than one 'columns' parameter"); 500 + goto csvtab_connect_error; 501 + } 502 + nCol = atoi(zValue); 503 + if( nCol<=0 ){ 504 + csv_errmsg(&sRdr, "must have at least one column"); 505 + goto csvtab_connect_error; 506 + } 507 + }else 508 + { 509 + csv_errmsg(&sRdr, "unrecognized parameter '%s'", z); 510 + goto csvtab_connect_error; 511 + } 512 + } 513 + if( (CSV_FILENAME==0)==(CSV_DATA==0) ){ 514 + csv_errmsg(&sRdr, "must either filename= or data= but not both"); 515 + goto csvtab_connect_error; 516 + } 517 + if( nCol<=0 && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA) ){ 518 + goto csvtab_connect_error; 519 + } 520 + pNew = sqlite3_malloc( sizeof(*pNew) ); 521 + *ppVtab = (sqlite3_vtab*)pNew; 522 + if( pNew==0 ) goto csvtab_connect_oom; 523 + memset(pNew, 0, sizeof(*pNew)); 524 + if( nCol>0 ){ 525 + pNew->nCol = nCol; 526 + }else{ 527 + do{ 528 + const char *z = csv_read_one_field(&sRdr); 529 + if( z==0 ) goto csvtab_connect_oom; 530 + pNew->nCol++; 531 + }while( sRdr.cTerm==',' ); 532 + } 533 + pNew->zFilename = CSV_FILENAME; CSV_FILENAME = 0; 534 + pNew->zData = CSV_DATA; CSV_DATA = 0; 535 +#ifdef SQLITE_TEST 536 + pNew->tstFlags = tstFlags; 537 +#endif 538 + pNew->iStart = bHeader==1 ? ftell(sRdr.in) : 0; 539 + csv_reader_reset(&sRdr); 540 + if( CSV_SCHEMA==0 ){ 541 + char *zSep = ""; 542 + CSV_SCHEMA = sqlite3_mprintf("CREATE TABLE x("); 543 + if( CSV_SCHEMA==0 ) goto csvtab_connect_oom; 544 + for(i=0; i<pNew->nCol; i++){ 545 + CSV_SCHEMA = sqlite3_mprintf("%z%sc%d TEXT",CSV_SCHEMA, zSep, i); 546 + zSep = ","; 547 + } 548 + CSV_SCHEMA = sqlite3_mprintf("%z);", CSV_SCHEMA); 549 + } 550 + rc = sqlite3_declare_vtab(db, CSV_SCHEMA); 551 + if( rc ) goto csvtab_connect_error; 552 + for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){ 553 + sqlite3_free(azPValue[i]); 554 + } 555 + return SQLITE_OK; 556 + 557 +csvtab_connect_oom: 558 + rc = SQLITE_NOMEM; 559 + csv_errmsg(&sRdr, "out of memory"); 560 + 561 +csvtab_connect_error: 562 + if( pNew ) csvtabDisconnect(&pNew->base); 563 + for(i=0; i<sizeof(azPValue)/sizeof(azPValue[0]); i++){ 564 + sqlite3_free(azPValue[i]); 565 + } 566 + if( sRdr.zErr[0] ){ 567 + sqlite3_free(*pzErr); 568 + *pzErr = sqlite3_mprintf("%s", sRdr.zErr); 569 + } 570 + csv_reader_reset(&sRdr); 571 + if( rc==SQLITE_OK ) rc = SQLITE_ERROR; 572 + return rc; 573 +} 574 + 575 +/* 576 +** Reset the current row content held by a CsvCursor. 577 +*/ 578 +static void csvtabCursorRowReset(CsvCursor *pCur){ 579 + CsvTable *pTab = (CsvTable*)pCur->base.pVtab; 580 + int i; 581 + for(i=0; i<pTab->nCol; i++){ 582 + sqlite3_free(pCur->azVal[i]); 583 + pCur->azVal[i] = 0; 584 + pCur->aLen[i] = 0; 585 + } 586 +} 587 + 588 +/* 589 +** The xConnect and xCreate methods do the same thing, but they must be 590 +** different so that the virtual table is not an eponymous virtual table. 591 +*/ 592 +static int csvtabCreate( 593 + sqlite3 *db, 594 + void *pAux, 595 + int argc, const char *const*argv, 596 + sqlite3_vtab **ppVtab, 597 + char **pzErr 598 +){ 599 + return csvtabConnect(db, pAux, argc, argv, ppVtab, pzErr); 600 +} 601 + 602 +/* 603 +** Destructor for a CsvCursor. 604 +*/ 605 +static int csvtabClose(sqlite3_vtab_cursor *cur){ 606 + CsvCursor *pCur = (CsvCursor*)cur; 607 + csvtabCursorRowReset(pCur); 608 + csv_reader_reset(&pCur->rdr); 609 + sqlite3_free(cur); 610 + return SQLITE_OK; 611 +} 612 + 613 +/* 614 +** Constructor for a new CsvTable cursor object. 615 +*/ 616 +static int csvtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ 617 + CsvTable *pTab = (CsvTable*)p; 618 + CsvCursor *pCur; 619 + size_t nByte; 620 + nByte = sizeof(*pCur) + (sizeof(char*)+sizeof(int))*pTab->nCol; 621 + pCur = sqlite3_malloc64( nByte ); 622 + if( pCur==0 ) return SQLITE_NOMEM; 623 + memset(pCur, 0, nByte); 624 + pCur->azVal = (char**)&pCur[1]; 625 + pCur->aLen = (int*)&pCur->azVal[pTab->nCol]; 626 + *ppCursor = &pCur->base; 627 + if( csv_reader_open(&pCur->rdr, pTab->zFilename, pTab->zData) ){ 628 + csv_xfer_error(pTab, &pCur->rdr); 629 + return SQLITE_ERROR; 630 + } 631 + return SQLITE_OK; 632 +} 633 + 634 + 635 +/* 636 +** Advance a CsvCursor to its next row of input. 637 +** Set the EOF marker if we reach the end of input. 638 +*/ 639 +static int csvtabNext(sqlite3_vtab_cursor *cur){ 640 + CsvCursor *pCur = (CsvCursor*)cur; 641 + CsvTable *pTab = (CsvTable*)cur->pVtab; 642 + int i = 0; 643 + char *z; 644 + do{ 645 + z = csv_read_one_field(&pCur->rdr); 646 + if( z==0 ){ 647 + csv_xfer_error(pTab, &pCur->rdr); 648 + break; 649 + } 650 + if( i<pTab->nCol ){ 651 + if( pCur->aLen[i] < pCur->rdr.n+1 ){ 652 + char *zNew = sqlite3_realloc64(pCur->azVal[i], pCur->rdr.n+1); 653 + if( zNew==0 ){ 654 + csv_errmsg(&pCur->rdr, "out of memory"); 655 + csv_xfer_error(pTab, &pCur->rdr); 656 + break; 657 + } 658 + pCur->azVal[i] = zNew; 659 + pCur->aLen[i] = pCur->rdr.n+1; 660 + } 661 + memcpy(pCur->azVal[i], z, pCur->rdr.n+1); 662 + i++; 663 + } 664 + }while( pCur->rdr.cTerm==',' ); 665 + while( i<pTab->nCol ){ 666 + sqlite3_free(pCur->azVal[i]); 667 + pCur->azVal[i] = 0; 668 + pCur->aLen[i] = 0; 669 + i++; 670 + } 671 + if( z==0 || pCur->rdr.cTerm==EOF ){ 672 + pCur->iRowid = -1; 673 + }else{ 674 + pCur->iRowid++; 675 + } 676 + return SQLITE_OK; 677 +} 678 + 679 +/* 680 +** Return values of columns for the row at which the CsvCursor 681 +** is currently pointing. 682 +*/ 683 +static int csvtabColumn( 684 + sqlite3_vtab_cursor *cur, /* The cursor */ 685 + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ 686 + int i /* Which column to return */ 687 +){ 688 + CsvCursor *pCur = (CsvCursor*)cur; 689 + CsvTable *pTab = (CsvTable*)cur->pVtab; 690 + if( i>=0 && i<pTab->nCol && pCur->azVal[i]!=0 ){ 691 + sqlite3_result_text(ctx, pCur->azVal[i], -1, SQLITE_STATIC); 692 + } 693 + return SQLITE_OK; 694 +} 695 + 696 +/* 697 +** Return the rowid for the current row. 698 +*/ 699 +static int csvtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ 700 + CsvCursor *pCur = (CsvCursor*)cur; 701 + *pRowid = pCur->iRowid; 702 + return SQLITE_OK; 703 +} 704 + 705 +/* 706 +** Return TRUE if the cursor has been moved off of the last 707 +** row of output. 708 +*/ 709 +static int csvtabEof(sqlite3_vtab_cursor *cur){ 710 + CsvCursor *pCur = (CsvCursor*)cur; 711 + return pCur->iRowid<0; 712 +} 713 + 714 +/* 715 +** Only a full table scan is supported. So xFilter simply rewinds to 716 +** the beginning. 717 +*/ 718 +static int csvtabFilter( 719 + sqlite3_vtab_cursor *pVtabCursor, 720 + int idxNum, const char *idxStr, 721 + int argc, sqlite3_value **argv 722 +){ 723 + CsvCursor *pCur = (CsvCursor*)pVtabCursor; 724 + CsvTable *pTab = (CsvTable*)pVtabCursor->pVtab; 725 + pCur->iRowid = 0; 726 + if( pCur->rdr.in==0 ){ 727 + assert( pCur->rdr.zIn==pTab->zData ); 728 + assert( pTab->iStart>=0 ); 729 + assert( (size_t)pTab->iStart<=pCur->rdr.nIn ); 730 + pCur->rdr.iIn = pTab->iStart; 731 + }else{ 732 + fseek(pCur->rdr.in, pTab->iStart, SEEK_SET); 733 + pCur->rdr.iIn = 0; 734 + pCur->rdr.nIn = 0; 735 + } 736 + return csvtabNext(pVtabCursor); 737 +} 738 + 739 +/* 740 +** Only a forward full table scan is supported. xBestIndex is mostly 741 +** a no-op. If CSVTEST_FIDX is set, then the presence of equality 742 +** constraints lowers the estimated cost, which is fiction, but is useful 743 +** for testing certain kinds of virtual table behavior. 744 +*/ 745 +static int csvtabBestIndex( 746 + sqlite3_vtab *tab, 747 + sqlite3_index_info *pIdxInfo 748 +){ 749 + pIdxInfo->estimatedCost = 1000000; 750 +#ifdef SQLITE_TEST 751 + if( (((CsvTable*)tab)->tstFlags & CSVTEST_FIDX)!=0 ){ 752 + /* The usual (and sensible) case is to always do a full table scan. 753 + ** The code in this branch only runs when testflags=1. This code 754 + ** generates an artifical and unrealistic plan which is useful 755 + ** for testing virtual table logic but is not helpful to real applications. 756 + ** 757 + ** Any ==, LIKE, or GLOB constraint is marked as usable by the virtual 758 + ** table (even though it is not) and the cost of running the virtual table 759 + ** is reduced from 1 million to just 10. The constraints are *not* marked 760 + ** as omittable, however, so the query planner should still generate a 761 + ** plan that gives a correct answer, even if they plan is not optimal. 762 + */ 763 + int i; 764 + int nConst = 0; 765 + for(i=0; i<pIdxInfo->nConstraint; i++){ 766 + unsigned char op; 767 + if( pIdxInfo->aConstraint[i].usable==0 ) continue; 768 + op = pIdxInfo->aConstraint[i].op; 769 + if( op==SQLITE_INDEX_CONSTRAINT_EQ 770 + || op==SQLITE_INDEX_CONSTRAINT_LIKE 771 + || op==SQLITE_INDEX_CONSTRAINT_GLOB 772 + ){ 773 + pIdxInfo->estimatedCost = 10; 774 + pIdxInfo->aConstraintUsage[nConst].argvIndex = nConst+1; 775 + nConst++; 776 + } 777 + } 778 + } 779 +#endif 780 + return SQLITE_OK; 781 +} 782 + 783 + 784 +static sqlite3_module CsvModule = { 785 + 0, /* iVersion */ 786 + csvtabCreate, /* xCreate */ 787 + csvtabConnect, /* xConnect */ 788 + csvtabBestIndex, /* xBestIndex */ 789 + csvtabDisconnect, /* xDisconnect */ 790 + csvtabDisconnect, /* xDestroy */ 791 + csvtabOpen, /* xOpen - open a cursor */ 792 + csvtabClose, /* xClose - close a cursor */ 793 + csvtabFilter, /* xFilter - configure scan constraints */ 794 + csvtabNext, /* xNext - advance a cursor */ 795 + csvtabEof, /* xEof - check for end of scan */ 796 + csvtabColumn, /* xColumn - read data */ 797 + csvtabRowid, /* xRowid - read data */ 798 + 0, /* xUpdate */ 799 + 0, /* xBegin */ 800 + 0, /* xSync */ 801 + 0, /* xCommit */ 802 + 0, /* xRollback */ 803 + 0, /* xFindMethod */ 804 + 0, /* xRename */ 805 +}; 806 + 807 +#ifdef SQLITE_TEST 808 +/* 809 +** For virtual table testing, make a version of the CSV virtual table 810 +** available that has an xUpdate function. But the xUpdate always returns 811 +** SQLITE_READONLY since the CSV file is not really writable. 812 +*/ 813 +static int csvtabUpdate(sqlite3_vtab *p,int n,sqlite3_value**v,sqlite3_int64*x){ 814 + return SQLITE_READONLY; 815 +} 816 +static sqlite3_module CsvModuleFauxWrite = { 817 + 0, /* iVersion */ 818 + csvtabCreate, /* xCreate */ 819 + csvtabConnect, /* xConnect */ 820 + csvtabBestIndex, /* xBestIndex */ 821 + csvtabDisconnect, /* xDisconnect */ 822 + csvtabDisconnect, /* xDestroy */ 823 + csvtabOpen, /* xOpen - open a cursor */ 824 + csvtabClose, /* xClose - close a cursor */ 825 + csvtabFilter, /* xFilter - configure scan constraints */ 826 + csvtabNext, /* xNext - advance a cursor */ 827 + csvtabEof, /* xEof - check for end of scan */ 828 + csvtabColumn, /* xColumn - read data */ 829 + csvtabRowid, /* xRowid - read data */ 830 + csvtabUpdate, /* xUpdate */ 831 + 0, /* xBegin */ 832 + 0, /* xSync */ 833 + 0, /* xCommit */ 834 + 0, /* xRollback */ 835 + 0, /* xFindMethod */ 836 + 0, /* xRename */ 837 +}; 838 +#endif /* SQLITE_TEST */ 839 + 840 +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ 841 + 842 + 843 +#ifdef _WIN32 844 +__declspec(dllexport) 845 +#endif 846 +/* 847 +** This routine is called when the extension is loaded. The new 848 +** CSV virtual table module is registered with the calling database 849 +** connection. 850 +*/ 851 +int sqlite3_csv_init( 852 + sqlite3 *db, 853 + char **pzErrMsg, 854 + const sqlite3_api_routines *pApi 855 +){ 856 +#ifndef SQLITE_OMIT_VIRTUALTABLE 857 + int rc; 858 + SQLITE_EXTENSION_INIT2(pApi); 859 + rc = sqlite3_create_module(db, "csv", &CsvModule, 0); 860 +#ifdef SQLITE_TEST 861 + if( rc==SQLITE_OK ){ 862 + rc = sqlite3_create_module(db, "csv_wr", &CsvModuleFauxWrite, 0); 863 + } 864 +#endif 865 + return rc; 866 +#else 867 + return SQLITE_OK; 868 +#endif 869 +}
Added ext/misc/dbdump.c.
1 +/* 2 +** 2016-03-13 3 +** 4 +** The author disclaims copyright to this source code. In place of 5 +** a legal notice, here is a blessing: 6 +** 7 +** May you do good and not evil. 8 +** May you find forgiveness for yourself and forgive others. 9 +** May you share freely, never taking more than you give. 10 +** 11 +****************************************************************************** 12 +** 13 +** This file implements a C-language subroutine that converts the content 14 +** of an SQLite database into UTF-8 text SQL statements that can be used 15 +** to exactly recreate the original database. ROWID values are preserved. 16 +** 17 +** A prototype of the implemented subroutine is this: 18 +** 19 +** int sqlite3_db_dump( 20 +** sqlite3 *db, 21 +** const char *zSchema, 22 +** const char *zTable, 23 +** void (*xCallback)(void*, const char*), 24 +** void *pArg 25 +** ); 26 +** 27 +** The db parameter is the database connection. zSchema is the schema within 28 +** that database which is to be dumped. Usually the zSchema is "main" but 29 +** can also be "temp" or any ATTACH-ed database. If zTable is not NULL, then 30 +** only the content of that one table is dumped. If zTable is NULL, then all 31 +** tables are dumped. 32 +** 33 +** The generate text is passed to xCallback() in multiple calls. The second 34 +** argument to xCallback() is a copy of the pArg parameter. The first 35 +** argument is some of the output text that this routine generates. The 36 +** signature to xCallback() is designed to make it compatible with fputs(). 37 +** 38 +** The sqlite3_db_dump() subroutine returns SQLITE_OK on success or some error 39 +** code if it encounters a problem. 40 +** 41 +** If this file is compiled with -DDBDUMP_STANDALONE then a "main()" routine 42 +** is included so that this routine becomes a command-line utility. The 43 +** command-line utility takes two or three arguments which are the name 44 +** of the database file, the schema, and optionally the table, forming the 45 +** first three arguments of a single call to the library routine. 46 +*/ 47 +#include "sqlite3.h" 48 +#include <stdarg.h> 49 +#include <string.h> 50 +#include <ctype.h> 51 + 52 +/* 53 +** The state of the dump process. 54 +*/ 55 +typedef struct DState DState; 56 +struct DState { 57 + sqlite3 *db; /* The database connection */ 58 + int nErr; /* Number of errors seen so far */ 59 + int rc; /* Error code */ 60 + int writableSchema; /* True if in writable_schema mode */ 61 + int (*xCallback)(const char*,void*); /* Send output here */ 62 + void *pArg; /* Argument to xCallback() */ 63 +}; 64 + 65 +/* 66 +** A variable length string to which one can append text. 67 +*/ 68 +typedef struct DText DText; 69 +struct DText { 70 + char *z; /* The text */ 71 + int n; /* Number of bytes of content in z[] */ 72 + int nAlloc; /* Number of bytes allocated to z[] */ 73 +}; 74 + 75 +/* 76 +** Initialize and destroy a DText object 77 +*/ 78 +static void initText(DText *p){ 79 + memset(p, 0, sizeof(*p)); 80 +} 81 +static void freeText(DText *p){ 82 + sqlite3_free(p->z); 83 + initText(p); 84 +} 85 + 86 +/* zIn is either a pointer to a NULL-terminated string in memory obtained 87 +** from malloc(), or a NULL pointer. The string pointed to by zAppend is 88 +** added to zIn, and the result returned in memory obtained from malloc(). 89 +** zIn, if it was not NULL, is freed. 90 +** 91 +** If the third argument, quote, is not '\0', then it is used as a 92 +** quote character for zAppend. 93 +*/ 94 +static void appendText(DText *p, char const *zAppend, char quote){ 95 + int len; 96 + int i; 97 + int nAppend = (int)(strlen(zAppend) & 0x3fffffff); 98 + 99 + len = nAppend+p->n+1; 100 + if( quote ){ 101 + len += 2; 102 + for(i=0; i<nAppend; i++){ 103 + if( zAppend[i]==quote ) len++; 104 + } 105 + } 106 + 107 + if( p->n+len>=p->nAlloc ){ 108 + char *zNew; 109 + p->nAlloc = p->nAlloc*2 + len + 20; 110 + zNew = sqlite3_realloc(p->z, p->nAlloc); 111 + if( zNew==0 ){ 112 + freeText(p); 113 + return; 114 + } 115 + p->z = zNew; 116 + } 117 + 118 + if( quote ){ 119 + char *zCsr = p->z+p->n; 120 + *zCsr++ = quote; 121 + for(i=0; i<nAppend; i++){ 122 + *zCsr++ = zAppend[i]; 123 + if( zAppend[i]==quote ) *zCsr++ = quote; 124 + } 125 + *zCsr++ = quote; 126 + p->n = (int)(zCsr - p->z); 127 + *zCsr = '\0'; 128 + }else{ 129 + memcpy(p->z+p->n, zAppend, nAppend); 130 + p->n += nAppend; 131 + p->z[p->n] = '\0'; 132 + } 133 +} 134 + 135 +/* 136 +** Attempt to determine if identifier zName needs to be quoted, either 137 +** because it contains non-alphanumeric characters, or because it is an 138 +** SQLite keyword. Be conservative in this estimate: When in doubt assume 139 +** that quoting is required. 140 +** 141 +** Return '"' if quoting is required. Return 0 if no quoting is required. 142 +*/ 143 +static char quoteChar(const char *zName){ 144 + /* All SQLite keywords, in alphabetical order */ 145 + static const char *azKeywords[] = { 146 + "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS", 147 + "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY", 148 + "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT", 149 + "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE", 150 + "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE", 151 + "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH", 152 + "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN", 153 + "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF", 154 + "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER", 155 + "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY", 156 + "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL", 157 + "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA", 158 + "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP", 159 + "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT", 160 + "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP", 161 + "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE", 162 + "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE", 163 + "WITH", "WITHOUT", 164 + }; 165 + int i, lwr, upr, mid, c; 166 + if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; 167 + for(i=0; zName[i]; i++){ 168 + if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; 169 + } 170 + lwr = 0; 171 + upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1; 172 + while( lwr<=upr ){ 173 + mid = (lwr+upr)/2; 174 + c = sqlite3_stricmp(azKeywords[mid], zName); 175 + if( c==0 ) return '"'; 176 + if( c<0 ){ 177 + lwr = mid+1; 178 + }else{ 179 + upr = mid-1; 180 + } 181 + } 182 + return 0; 183 +} 184 + 185 + 186 +/* 187 +** Release memory previously allocated by tableColumnList(). 188 +*/ 189 +static void freeColumnList(char **azCol){ 190 + int i; 191 + for(i=1; azCol[i]; i++){ 192 + sqlite3_free(azCol[i]); 193 + } 194 + /* azCol[0] is a static string */ 195 + sqlite3_free(azCol); 196 +} 197 + 198 +/* 199 +** Return a list of pointers to strings which are the names of all 200 +** columns in table zTab. The memory to hold the names is dynamically 201 +** allocated and must be released by the caller using a subsequent call 202 +** to freeColumnList(). 203 +** 204 +** The azCol[0] entry is usually NULL. However, if zTab contains a rowid 205 +** value that needs to be preserved, then azCol[0] is filled in with the 206 +** name of the rowid column. 207 +** 208 +** The first regular column in the table is azCol[1]. The list is terminated 209 +** by an entry with azCol[i]==0. 210 +*/ 211 +static char **tableColumnList(DState *p, const char *zTab){ 212 + char **azCol = 0; 213 + sqlite3_stmt *pStmt = 0; 214 + char *zSql; 215 + int nCol = 0; 216 + int nAlloc = 0; 217 + int nPK = 0; /* Number of PRIMARY KEY columns seen */ 218 + int isIPK = 0; /* True if one PRIMARY KEY column of type INTEGER */ 219 + int preserveRowid = 1; 220 + int rc; 221 + 222 + zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab); 223 + if( zSql==0 ) return 0; 224 + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); 225 + sqlite3_free(zSql); 226 + if( rc ) return 0; 227 + while( sqlite3_step(pStmt)==SQLITE_ROW ){ 228 + if( nCol>=nAlloc-2 ){ 229 + char **azNew; 230 + nAlloc = nAlloc*2 + nCol + 10; 231 + azNew = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0])); 232 + if( azNew==0 ) goto col_oom; 233 + azCol = azNew; 234 + azCol[0] = 0; 235 + } 236 + azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); 237 + if( azCol[nCol]==0 ) goto col_oom; 238 + if( sqlite3_column_int(pStmt, 5) ){ 239 + nPK++; 240 + if( nPK==1 241 + && sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2), 242 + "INTEGER")==0 243 + ){ 244 + isIPK = 1; 245 + }else{ 246 + isIPK = 0; 247 + } 248 + } 249 + } 250 + sqlite3_finalize(pStmt); 251 + pStmt = 0; 252 + azCol[nCol+1] = 0; 253 + 254 + /* The decision of whether or not a rowid really needs to be preserved 255 + ** is tricky. We never need to preserve a rowid for a WITHOUT ROWID table 256 + ** or a table with an INTEGER PRIMARY KEY. We are unable to preserve 257 + ** rowids on tables where the rowid is inaccessible because there are other 258 + ** columns in the table named "rowid", "_rowid_", and "oid". 259 + */ 260 + if( isIPK ){ 261 + /* If a single PRIMARY KEY column with type INTEGER was seen, then it 262 + ** might be an alise for the ROWID. But it might also be a WITHOUT ROWID 263 + ** table or a INTEGER PRIMARY KEY DESC column, neither of which are 264 + ** ROWID aliases. To distinguish these cases, check to see if 265 + ** there is a "pk" entry in "PRAGMA index_list". There will be 266 + ** no "pk" index if the PRIMARY KEY really is an alias for the ROWID. 267 + */ 268 + zSql = sqlite3_mprintf("SELECT 1 FROM pragma_index_list(%Q)" 269 + " WHERE origin='pk'", zTab); 270 + if( zSql==0 ) goto col_oom; 271 + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); 272 + sqlite3_free(zSql); 273 + if( rc ){ 274 + freeColumnList(azCol); 275 + return 0; 276 + } 277 + rc = sqlite3_step(pStmt); 278 + sqlite3_finalize(pStmt); 279 + pStmt = 0; 280 + preserveRowid = rc==SQLITE_ROW; 281 + } 282 + if( preserveRowid ){ 283 + /* Only preserve the rowid if we can find a name to use for the 284 + ** rowid */ 285 + static char *azRowid[] = { "rowid", "_rowid_", "oid" }; 286 + int i, j; 287 + for(j=0; j<3; j++){ 288 + for(i=1; i<=nCol; i++){ 289 + if( sqlite3_stricmp(azRowid[j],azCol[i])==0 ) break; 290 + } 291 + if( i>nCol ){ 292 + /* At this point, we know that azRowid[j] is not the name of any 293 + ** ordinary column in the table. Verify that azRowid[j] is a valid 294 + ** name for the rowid before adding it to azCol[0]. WITHOUT ROWID 295 + ** tables will fail this last check */ 296 + int rc; 297 + rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0); 298 + if( rc==SQLITE_OK ) azCol[0] = azRowid[j]; 299 + break; 300 + } 301 + } 302 + } 303 + return azCol; 304 + 305 +col_oom: 306 + sqlite3_finalize(pStmt); 307 + freeColumnList(azCol); 308 + p->nErr++; 309 + p->rc = SQLITE_NOMEM; 310 + return 0; 311 +} 312 + 313 +/* 314 +** Send mprintf-formatted content to the output callback. 315 +*/ 316 +static void output_formatted(DState *p, const char *zFormat, ...){ 317 + va_list ap; 318 + char *z; 319 + va_start(ap, zFormat); 320 + z = sqlite3_vmprintf(zFormat, ap); 321 + va_end(ap); 322 + p->xCallback(z, p->pArg); 323 + sqlite3_free(z); 324 +} 325 + 326 +/* 327 +** Output the given string as a quoted string using SQL quoting conventions. 328 +** 329 +** The "\n" and "\r" characters are converted to char(10) and char(13) 330 +** to prevent them from being transformed by end-of-line translators. 331 +*/ 332 +static void output_quoted_string(DState *p, const unsigned char *z){ 333 + int i; 334 + char c; 335 + int inQuote = 0; 336 + int bStarted = 0; 337 + 338 + for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){} 339 + if( c==0 ){ 340 + output_formatted(p, "'%s'", z); 341 + return; 342 + } 343 + while( *z ){ 344 + for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){} 345 + if( c=='\'' ) i++; 346 + if( i ){ 347 + if( !inQuote ){ 348 + if( bStarted ) p->xCallback("||", p->pArg); 349 + p->xCallback("'", p->pArg); 350 + inQuote = 1; 351 + } 352 + output_formatted(p, "%.*s", i, z); 353 + z += i; 354 + bStarted = 1; 355 + } 356 + if( c=='\'' ){ 357 + p->xCallback("'", p->pArg); 358 + continue; 359 + } 360 + if( inQuote ){ 361 + p->xCallback("'", p->pArg); 362 + inQuote = 0; 363 + } 364 + if( c==0 ){ 365 + break; 366 + } 367 + for(i=0; (c = z[i])=='\r' || c=='\n'; i++){ 368 + if( bStarted ) p->xCallback("||", p->pArg); 369 + output_formatted(p, "char(%d)", c); 370 + bStarted = 1; 371 + } 372 + z += i; 373 + } 374 + if( inQuote ) p->xCallback("'", p->pArg); 375 +} 376 + 377 +/* 378 +** This is an sqlite3_exec callback routine used for dumping the database. 379 +** Each row received by this callback consists of a table name, 380 +** the table type ("index" or "table") and SQL to create the table. 381 +** This routine should print text sufficient to recreate the table. 382 +*/ 383 +static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ 384 + int rc; 385 + const char *zTable; 386 + const char *zType; 387 + const char *zSql; 388 + DState *p = (DState*)pArg; 389 + sqlite3_stmt *pStmt; 390 + 391 + (void)azCol; 392 + if( nArg!=3 ) return 1; 393 + zTable = azArg[0]; 394 + zType = azArg[1]; 395 + zSql = azArg[2]; 396 + 397 + if( strcmp(zTable, "sqlite_sequence")==0 ){ 398 + p->xCallback("DELETE FROM sqlite_sequence;\n", p->pArg); 399 + }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 ){ 400 + p->xCallback("ANALYZE sqlite_master;\n", p->pArg); 401 + }else if( strncmp(zTable, "sqlite_", 7)==0 ){ 402 + return 0; 403 + }else if( strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ 404 + if( !p->writableSchema ){ 405 + p->xCallback("PRAGMA writable_schema=ON;\n", p->pArg); 406 + p->writableSchema = 1; 407 + } 408 + output_formatted(p, 409 + "INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql)" 410 + "VALUES('table','%q','%q',0,'%q');", 411 + zTable, zTable, zSql); 412 + return 0; 413 + }else{ 414 + if( sqlite3_strglob("CREATE TABLE ['\"]*", zSql)==0 ){ 415 + p->xCallback("CREATE TABLE IF NOT EXISTS ", p->pArg); 416 + p->xCallback(zSql+13, p->pArg); 417 + }else{ 418 + p->xCallback(zSql, p->pArg); 419 + } 420 + p->xCallback(";\n", p->pArg); 421 + } 422 + 423 + if( strcmp(zType, "table")==0 ){ 424 + DText sSelect; 425 + DText sTable; 426 + char **azCol; 427 + int i; 428 + int nCol; 429 + 430 + azCol = tableColumnList(p, zTable); 431 + if( azCol==0 ) return 0; 432 + 433 + initText(&sTable); 434 + appendText(&sTable, "INSERT INTO ", 0); 435 + 436 + /* Always quote the table name, even if it appears to be pure ascii, 437 + ** in case it is a keyword. Ex: INSERT INTO "table" ... */ 438 + appendText(&sTable, zTable, quoteChar(zTable)); 439 + 440 + /* If preserving the rowid, add a column list after the table name. 441 + ** In other words: "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)" 442 + ** instead of the usual "INSERT INTO tab VALUES(...)". 443 + */ 444 + if( azCol[0] ){ 445 + appendText(&sTable, "(", 0); 446 + appendText(&sTable, azCol[0], 0); 447 + for(i=1; azCol[i]; i++){ 448 + appendText(&sTable, ",", 0); 449 + appendText(&sTable, azCol[i], quoteChar(azCol[i])); 450 + } 451 + appendText(&sTable, ")", 0); 452 + } 453 + appendText(&sTable, " VALUES(", 0); 454 + 455 + /* Build an appropriate SELECT statement */ 456 + initText(&sSelect); 457 + appendText(&sSelect, "SELECT ", 0); 458 + if( azCol[0] ){ 459 + appendText(&sSelect, azCol[0], 0); 460 + appendText(&sSelect, ",", 0); 461 + } 462 + for(i=1; azCol[i]; i++){ 463 + appendText(&sSelect, azCol[i], quoteChar(azCol[i])); 464 + if( azCol[i+1] ){ 465 + appendText(&sSelect, ",", 0); 466 + } 467 + } 468 + nCol = i; 469 + if( azCol[0]==0 ) nCol--; 470 + freeColumnList(azCol); 471 + appendText(&sSelect, " FROM ", 0); 472 + appendText(&sSelect, zTable, quoteChar(zTable)); 473 + 474 + rc = sqlite3_prepare_v2(p->db, sSelect.z, -1, &pStmt, 0); 475 + if( rc!=SQLITE_OK ){ 476 + p->nErr++; 477 + if( p->rc==SQLITE_OK ) p->rc = rc; 478 + }else{ 479 + while( SQLITE_ROW==sqlite3_step(pStmt) ){ 480 + p->xCallback(sTable.z, p->pArg); 481 + for(i=0; i<nCol; i++){ 482 + if( i ) p->xCallback(",", p->pArg); 483 + switch( sqlite3_column_type(pStmt,i) ){ 484 + case SQLITE_INTEGER: { 485 + output_formatted(p, "%lld", sqlite3_column_int64(pStmt,i)); 486 + break; 487 + } 488 + case SQLITE_FLOAT: { 489 + double r = sqlite3_column_double(pStmt,i); 490 + output_formatted(p, "%!.20g", r); 491 + break; 492 + } 493 + case SQLITE_NULL: { 494 + p->xCallback("NULL", p->pArg); 495 + break; 496 + } 497 + case SQLITE_TEXT: { 498 + output_quoted_string(p, sqlite3_column_text(pStmt,i)); 499 + break; 500 + } 501 + case SQLITE_BLOB: { 502 + int nByte = sqlite3_column_bytes(pStmt,i); 503 + unsigned char *a = (unsigned char*)sqlite3_column_blob(pStmt,i); 504 + int j; 505 + p->xCallback("x'", p->pArg); 506 + for(j=0; j<nByte; j++){ 507 + char zWord[3]; 508 + zWord[0] = "0123456789abcdef"[(a[j]>>4)&15]; 509 + zWord[1] = "0123456789abcdef"[a[j]&15]; 510 + zWord[2] = 0; 511 + p->xCallback(zWord, p->pArg); 512 + } 513 + p->xCallback("'", p->pArg); 514 + break; 515 + } 516 + } 517 + } 518 + p->xCallback(");\n", p->pArg); 519 + } 520 + } 521 + sqlite3_finalize(pStmt); 522 + freeText(&sTable); 523 + freeText(&sSelect); 524 + } 525 + return 0; 526 +} 527 + 528 + 529 +/* 530 +** Execute a query statement that will generate SQL output. Print 531 +** the result columns, comma-separated, on a line and then add a 532 +** semicolon terminator to the end of that line. 533 +** 534 +** If the number of columns is 1 and that column contains text "--" 535 +** then write the semicolon on a separate line. That way, if a 536 +** "--" comment occurs at the end of the statement, the comment 537 +** won't consume the semicolon terminator. 538 +*/ 539 +static void output_sql_from_query( 540 + DState *p, /* Query context */ 541 + const char *zSelect, /* SELECT statement to extract content */ 542 + ... 543 +){ 544 + sqlite3_stmt *pSelect; 545 + int rc; 546 + int nResult; 547 + int i; 548 + const char *z; 549 + char *zSql; 550 + va_list ap; 551 + va_start(ap, zSelect); 552 + zSql = sqlite3_vmprintf(zSelect, ap); 553 + va_end(ap); 554 + if( zSql==0 ){ 555 + p->rc = SQLITE_NOMEM; 556 + p->nErr++; 557 + return; 558 + } 559 + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pSelect, 0); 560 + sqlite3_free(zSql); 561 + if( rc!=SQLITE_OK || !pSelect ){ 562 + output_formatted(p, "/**** ERROR: (%d) %s *****/\n", rc, 563 + sqlite3_errmsg(p->db)); 564 + p->nErr++; 565 + return; 566 + } 567 + rc = sqlite3_step(pSelect); 568 + nResult = sqlite3_column_count(pSelect); 569 + while( rc==SQLITE_ROW ){ 570 + z = (const char*)sqlite3_column_text(pSelect, 0); 571 + p->xCallback(z, p->pArg); 572 + for(i=1; i<nResult; i++){ 573 + p->xCallback(",", p->pArg); 574 + p->xCallback((const char*)sqlite3_column_text(pSelect,i), p->pArg); 575 + } 576 + if( z==0 ) z = ""; 577 + while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; 578 + if( z[0] ){ 579 + p->xCallback("\n;\n", p->pArg); 580 + }else{ 581 + p->xCallback(";\n", p->pArg); 582 + } 583 + rc = sqlite3_step(pSelect); 584 + } 585 + rc = sqlite3_finalize(pSelect); 586 + if( rc!=SQLITE_OK ){ 587 + output_formatted(p, "/**** ERROR: (%d) %s *****/\n", rc, 588 + sqlite3_errmsg(p->db)); 589 + if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; 590 + } 591 +} 592 + 593 +/* 594 +** Run zQuery. Use dump_callback() as the callback routine so that 595 +** the contents of the query are output as SQL statements. 596 +** 597 +** If we get a SQLITE_CORRUPT error, rerun the query after appending 598 +** "ORDER BY rowid DESC" to the end. 599 +*/ 600 +static void run_schema_dump_query( 601 + DState *p, 602 + const char *zQuery, 603 + ... 604 +){ 605 + char *zErr = 0; 606 + char *z; 607 + va_list ap; 608 + va_start(ap, zQuery); 609 + z = sqlite3_vmprintf(zQuery, ap); 610 + va_end(ap); 611 + sqlite3_exec(p->db, z, dump_callback, p, &zErr); 612 + sqlite3_free(z); 613 + if( zErr ){ 614 + output_formatted(p, "/****** %s ******/\n", zErr); 615 + sqlite3_free(zErr); 616 + p->nErr++; 617 + zErr = 0; 618 + } 619 +} 620 + 621 +/* 622 +** Convert an SQLite database into SQL statements that will recreate that 623 +** database. 624 +*/ 625 +int sqlite3_db_dump( 626 + sqlite3 *db, /* The database connection */ 627 + const char *zSchema, /* Which schema to dump. Usually "main". */ 628 + const char *zTable, /* Which table to dump. NULL means everything. */ 629 + int (*xCallback)(const char*,void*), /* Output sent to this callback */ 630 + void *pArg /* Second argument of the callback */ 631 +){ 632 + DState x; 633 + memset(&x, 0, sizeof(x)); 634 + x.rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); 635 + if( x.rc ) return x.rc; 636 + x.db = db; 637 + x.xCallback = xCallback; 638 + x.pArg = pArg; 639 + xCallback("PRAGMA foreign_keys=OFF;\nBEGIN TRANSACTION;\n", pArg); 640 + if( zTable==0 ){ 641 + run_schema_dump_query(&x, 642 + "SELECT name, type, sql FROM \"%w\".sqlite_master " 643 + "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'", 644 + zSchema 645 + ); 646 + run_schema_dump_query(&x, 647 + "SELECT name, type, sql FROM \"%w\".sqlite_master " 648 + "WHERE name=='sqlite_sequence'", zSchema 649 + ); 650 + output_sql_from_query(&x, 651 + "SELECT sql FROM sqlite_master " 652 + "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 653 + ); 654 + }else{ 655 + run_schema_dump_query(&x, 656 + "SELECT name, type, sql FROM \"%w\".sqlite_master " 657 + "WHERE tbl_name=%Q COLLATE nocase AND type=='table'" 658 + " AND sql NOT NULL", 659 + zSchema, zTable 660 + ); 661 + output_sql_from_query(&x, 662 + "SELECT sql FROM \"%w\".sqlite_master " 663 + "WHERE sql NOT NULL" 664 + " AND type IN ('index','trigger','view')" 665 + " AND tbl_name=%Q COLLATE nocase", 666 + zSchema, zTable 667 + ); 668 + } 669 + if( x.writableSchema ){ 670 + xCallback("PRAGMA writable_schema=OFF;\n", pArg); 671 + } 672 + xCallback(x.nErr ? "ROLLBACK; -- due to errors\n" : "COMMIT;\n", pArg); 673 + sqlite3_exec(db, "COMMIT", 0, 0, 0); 674 + return x.rc; 675 +} 676 + 677 + 678 + 679 +/* The generic subroutine is above. The code the follows implements 680 +** the command-line interface. 681 +*/ 682 +#ifdef DBDUMP_STANDALONE 683 +#include <stdio.h> 684 + 685 +/* 686 +** Command-line interface 687 +*/ 688 +int main(int argc, char **argv){ 689 + sqlite3 *db; 690 + const char *zDb; 691 + const char *zSchema; 692 + const char *zTable = 0; 693 + int rc; 694 + 695 + if( argc<2 || argc>4 ){ 696 + fprintf(stderr, "Usage: %s DATABASE ?SCHEMA? ?TABLE?\n", argv[0]); 697 + return 1; 698 + } 699 + zDb = argv[1]; 700 + zSchema = argc>=3 ? argv[2] : "main"; 701 + zTable = argc==4 ? argv[3] : 0; 702 + 703 + rc = sqlite3_open(zDb, &db); 704 + if( rc ){ 705 + fprintf(stderr, "Cannot open \"%s\": %s\n", zDb, sqlite3_errmsg(db)); 706 + sqlite3_close(db); 707 + return 1; 708 + } 709 + rc = sqlite3_db_dump(db, zSchema, zTable, 710 + (int(*)(const char*,void*))fputs, (void*)stdout); 711 + if( rc ){ 712 + fprintf(stderr, "Error: sqlite3_db_dump() returns %d\n", rc); 713 + } 714 + sqlite3_close(db); 715 + return rc!=SQLITE_OK; 716 +} 717 +#endif /* DBDUMP_STANDALONE */
Changes to ext/misc/fuzzer.c.
340 340 pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo ); 341 341 if( pRule==0 ){ 342 342 rc = SQLITE_NOMEM; 343 343 }else{ 344 344 memset(pRule, 0, sizeof(*pRule)); 345 345 pRule->zFrom = pRule->zTo; 346 346 pRule->zFrom += nTo + 1; 347 - pRule->nFrom = nFrom; 347 + pRule->nFrom = (fuzzer_len)nFrom; 348 348 memcpy(pRule->zFrom, zFrom, nFrom+1); 349 349 memcpy(pRule->zTo, zTo, nTo+1); 350 - pRule->nTo = nTo; 350 + pRule->nTo = (fuzzer_len)nTo; 351 351 pRule->rCost = nCost; 352 352 pRule->iRuleset = (int)iRuleset; 353 353 } 354 354 } 355 355 356 356 *ppRule = pRule; 357 357 return rc;
Changes to ext/misc/json1.c.
18 18 ** For the time being, all JSON is stored as pure text. (We might add 19 19 ** a JSONB type in the future which stores a binary encoding of JSON in 20 20 ** a BLOB, but there is no support for JSONB in the current implementation. 21 21 ** This implementation parses JSON text at 250 MB/s, so it is hard to see 22 22 ** how JSONB might improve on that.) 23 23 */ 24 24 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1) 25 -#if !defined(_SQLITEINT_H_) 25 +#if !defined(SQLITEINT_H) 26 26 #include "sqlite3ext.h" 27 27 #endif 28 28 SQLITE_EXTENSION_INIT1 29 29 #include <assert.h> 30 30 #include <string.h> 31 31 #include <stdlib.h> 32 32 #include <stdarg.h> ................................................................................ 45 45 /* 46 46 ** Versions of isspace(), isalnum() and isdigit() to which it is safe 47 47 ** to pass signed char values. 48 48 */ 49 49 #ifdef sqlite3Isdigit 50 50 /* Use the SQLite core versions if this routine is part of the 51 51 ** SQLite amalgamation */ 52 -# define safe_isdigit(x) sqlite3Isdigit(x) 53 -# define safe_isalnum(x) sqlite3Isalnum(x) 52 +# define safe_isdigit(x) sqlite3Isdigit(x) 53 +# define safe_isalnum(x) sqlite3Isalnum(x) 54 +# define safe_isxdigit(x) sqlite3Isxdigit(x) 54 55 #else 55 56 /* Use the standard library for separate compilation */ 56 57 #include <ctype.h> /* amalgamator: keep */ 57 -# define safe_isdigit(x) isdigit((unsigned char)(x)) 58 -# define safe_isalnum(x) isalnum((unsigned char)(x)) 58 +# define safe_isdigit(x) isdigit((unsigned char)(x)) 59 +# define safe_isalnum(x) isalnum((unsigned char)(x)) 60 +# define safe_isxdigit(x) isxdigit((unsigned char)(x)) 59 61 #endif 60 62 61 63 /* 62 64 ** Growing our own isspace() routine this way is twice as fast as 63 65 ** the library isspace() function, resulting in a 7% overall performance 64 66 ** increase for the parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). 65 67 */ ................................................................................ 132 134 }; 133 135 134 136 /* Bit values for the JsonNode.jnFlag field 135 137 */ 136 138 #define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ 137 139 #define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ 138 140 #define JNODE_REMOVE 0x04 /* Do not output */ 139 -#define JNODE_REPLACE 0x08 /* Replace with JsonNode.iVal */ 140 -#define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ 141 -#define JNODE_LABEL 0x20 /* Is a label of an object */ 141 +#define JNODE_REPLACE 0x08 /* Replace with JsonNode.u.iReplace */ 142 +#define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ 143 +#define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ 144 +#define JNODE_LABEL 0x40 /* Is a label of an object */ 142 145 143 146 144 147 /* A single node of parsed JSON 145 148 */ 146 149 struct JsonNode { 147 150 u8 eType; /* One of the JSON_ type values */ 148 151 u8 jnFlags; /* JNODE flags */ 149 - u8 iVal; /* Replacement value when JNODE_REPLACE */ 150 152 u32 n; /* Bytes of content, or number of sub-nodes */ 151 153 union { 152 154 const char *zJContent; /* Content for INT, REAL, and STRING */ 153 155 u32 iAppend; /* More terms for ARRAY and OBJECT */ 154 156 u32 iKey; /* Key for ARRAY objects in json_tree() */ 157 + u32 iReplace; /* Replacement content for JNODE_REPLACE */ 158 + JsonNode *pPatch; /* Node chain of patch for JNODE_PATCH */ 155 159 } u; 156 160 }; 157 161 158 162 /* A completely parsed JSON string 159 163 */ 160 164 struct JsonParse { 161 165 u32 nNode; /* Number of slots of aNode[] used */ ................................................................................ 404 408 ** the number of JsonNode objects that are encoded. 405 409 */ 406 410 static void jsonRenderNode( 407 411 JsonNode *pNode, /* The node to render */ 408 412 JsonString *pOut, /* Write JSON here */ 409 413 sqlite3_value **aReplace /* Replacement values */ 410 414 ){ 415 + if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){ 416 + if( pNode->jnFlags & JNODE_REPLACE ){ 417 + jsonAppendValue(pOut, aReplace[pNode->u.iReplace]); 418 + return; 419 + } 420 + pNode = pNode->u.pPatch; 421 + } 411 422 switch( pNode->eType ){ 412 423 default: { 413 424 assert( pNode->eType==JSON_NULL ); 414 425 jsonAppendRaw(pOut, "null", 4); 415 426 break; 416 427 } 417 428 case JSON_TRUE: { ................................................................................ 435 446 break; 436 447 } 437 448 case JSON_ARRAY: { 438 449 u32 j = 1; 439 450 jsonAppendChar(pOut, '['); 440 451 for(;;){ 441 452 while( j<=pNode->n ){ 442 - if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){ 443 - if( pNode[j].jnFlags & JNODE_REPLACE ){ 444 - jsonAppendSeparator(pOut); 445 - jsonAppendValue(pOut, aReplace[pNode[j].iVal]); 446 - } 447 - }else{ 453 + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){ 448 454 jsonAppendSeparator(pOut); 449 455 jsonRenderNode(&pNode[j], pOut, aReplace); 450 456 } 451 457 j += jsonNodeSize(&pNode[j]); 452 458 } 453 459 if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; 454 460 pNode = &pNode[pNode->u.iAppend]; ................................................................................ 462 468 jsonAppendChar(pOut, '{'); 463 469 for(;;){ 464 470 while( j<=pNode->n ){ 465 471 if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ 466 472 jsonAppendSeparator(pOut); 467 473 jsonRenderNode(&pNode[j], pOut, aReplace); 468 474 jsonAppendChar(pOut, ':'); 469 - if( pNode[j+1].jnFlags & JNODE_REPLACE ){ 470 - jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]); 471 - }else{ 472 - jsonRenderNode(&pNode[j+1], pOut, aReplace); 473 - } 475 + jsonRenderNode(&pNode[j+1], pOut, aReplace); 474 476 } 475 477 j += 1 + jsonNodeSize(&pNode[j+1]); 476 478 } 477 479 if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; 478 480 pNode = &pNode[pNode->u.iAppend]; 479 481 j = 1; 480 482 } ................................................................................ 589 591 char c = z[i]; 590 592 if( c!='\\' ){ 591 593 zOut[j++] = c; 592 594 }else{ 593 595 c = z[++i]; 594 596 if( c=='u' ){ 595 597 u32 v = 0, k; 596 - for(k=0; k<4 && i<n-2; i++, k++){ 598 + for(k=0; k<4; i++, k++){ 599 + assert( i<n-2 ); 597 600 c = z[i+1]; 598 - if( c>='0' && c<='9' ) v = v*16 + c - '0'; 599 - else if( c>='A' && c<='F' ) v = v*16 + c - 'A' + 10; 600 - else if( c>='a' && c<='f' ) v = v*16 + c - 'a' + 10; 601 - else break; 601 + assert( safe_isxdigit(c) ); 602 + if( c<='9' ) v = v*16 + c - '0'; 603 + else if( c<='F' ) v = v*16 + c - 'A' + 10; 604 + else v = v*16 + c - 'a' + 10; 602 605 } 603 606 if( v==0 ) break; 604 607 if( v<=0x7f ){ 605 608 zOut[j++] = (char)v; 606 609 }else if( v<=0x7ff ){ 607 610 zOut[j++] = (char)(0xc0 | (v>>6)); 608 611 zOut[j++] = 0x80 | (v&0x3f); ................................................................................ 692 695 JsonNode *p; 693 696 if( pParse->nNode>=pParse->nAlloc ){ 694 697 return jsonParseAddNodeExpand(pParse, eType, n, zContent); 695 698 } 696 699 p = &pParse->aNode[pParse->nNode]; 697 700 p->eType = (u8)eType; 698 701 p->jnFlags = 0; 699 - p->iVal = 0; 700 702 p->n = n; 701 703 p->u.zJContent = zContent; 702 704 return pParse->nNode++; 703 705 } 706 + 707 +/* 708 +** Return true if z[] begins with 4 (or more) hexadecimal digits 709 +*/ 710 +static int jsonIs4Hex(const char *z){ 711 + int i; 712 + for(i=0; i<4; i++) if( !safe_isxdigit(z[i]) ) return 0; 713 + return 1; 714 +} 704 715 705 716 /* 706 717 ** Parse a single JSON value which begins at pParse->zJson[i]. Return the 707 718 ** index of the first character past the end of the value parsed. 708 719 ** 709 720 ** Return negative for a syntax error. Special cases: return -2 if the 710 721 ** first non-whitespace character is '}' and return -3 if the first ................................................................................ 772 783 u8 jnFlags = 0; 773 784 j = i+1; 774 785 for(;;){ 775 786 c = pParse->zJson[j]; 776 787 if( c==0 ) return -1; 777 788 if( c=='\\' ){ 778 789 c = pParse->zJson[++j]; 779 - if( c==0 ) return -1; 780 - jnFlags = JNODE_ESCAPE; 790 + if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' 791 + || c=='n' || c=='r' || c=='t' 792 + || (c=='u' && jsonIs4Hex(pParse->zJson+j+1)) ){ 793 + jnFlags = JNODE_ESCAPE; 794 + }else{ 795 + return -1; 796 + } 781 797 }else if( c=='"' ){ 782 798 break; 783 799 } 784 800 j++; 785 801 } 786 802 jsonParseAddNode(pParse, JSON_STRING, j+1-i, &pParse->zJson[i]); 787 803 if( !pParse->oom ) pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; ................................................................................ 1144 1160 ){ 1145 1161 char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", 1146 1162 zFuncName); 1147 1163 sqlite3_result_error(pCtx, zMsg, -1); 1148 1164 sqlite3_free(zMsg); 1149 1165 } 1150 1166 1167 +/* 1168 +** Mark all NULL entries in the Object passed in as JNODE_REMOVE. 1169 +*/ 1170 +static void jsonRemoveAllNulls(JsonNode *pNode){ 1171 + int i, n; 1172 + assert( pNode->eType==JSON_OBJECT ); 1173 + n = pNode->n; 1174 + for(i=2; i<=n; i += jsonNodeSize(&pNode[i])+1){ 1175 + switch( pNode[i].eType ){ 1176 + case JSON_NULL: 1177 + pNode[i].jnFlags |= JNODE_REMOVE; 1178 + break; 1179 + case JSON_OBJECT: 1180 + jsonRemoveAllNulls(&pNode[i]); 1181 + break; 1182 + } 1183 + } 1184 +} 1185 + 1151 1186 1152 1187 /**************************************************************************** 1153 1188 ** SQL functions used for testing and debugging 1154 1189 ****************************************************************************/ 1155 1190 1156 1191 #ifdef SQLITE_DEBUG 1157 1192 /* ................................................................................ 1206 1241 sqlite3_result_int(ctx, sqlite3_value_subtype(argv[0])==JSON_SUBTYPE); 1207 1242 } 1208 1243 #endif /* SQLITE_DEBUG */ 1209 1244 1210 1245 /**************************************************************************** 1211 1246 ** Scalar SQL function implementations 1212 1247 ****************************************************************************/ 1248 + 1249 +/* 1250 +** Implementation of the json_QUOTE(VALUE) function. Return a JSON value 1251 +** corresponding to the SQL value input. Mostly this means putting 1252 +** double-quotes around strings and returning the unquoted string "null" 1253 +** when given a NULL input. 1254 +*/ 1255 +static void jsonQuoteFunc( 1256 + sqlite3_context *ctx, 1257 + int argc, 1258 + sqlite3_value **argv 1259 +){ 1260 + JsonString jx; 1261 + UNUSED_PARAM(argc); 1262 + 1263 + jsonInit(&jx, ctx); 1264 + jsonAppendValue(&jx, argv[0]); 1265 + jsonResult(&jx); 1266 + sqlite3_result_subtype(ctx, JSON_SUBTYPE); 1267 +} 1213 1268 1214 1269 /* 1215 1270 ** Implementation of the json_array(VALUE,...) function. Return a JSON 1216 1271 ** array that contains all values given in arguments. Or if any argument 1217 1272 ** is a BLOB, throw an error. 1218 1273 */ 1219 1274 static void jsonArrayFunc( ................................................................................ 1315 1370 jsonAppendChar(&jx, ']'); 1316 1371 jsonResult(&jx); 1317 1372 sqlite3_result_subtype(ctx, JSON_SUBTYPE); 1318 1373 } 1319 1374 jsonReset(&jx); 1320 1375 jsonParseReset(&x); 1321 1376 } 1377 + 1378 +/* This is the RFC 7396 MergePatch algorithm. 1379 +*/ 1380 +static JsonNode *jsonMergePatch( 1381 + JsonParse *pParse, /* The JSON parser that contains the TARGET */ 1382 + int iTarget, /* Node of the TARGET in pParse */ 1383 + JsonNode *pPatch /* The PATCH */ 1384 +){ 1385 + u32 i, j; 1386 + u32 iRoot; 1387 + JsonNode *pTarget; 1388 + if( pPatch->eType!=JSON_OBJECT ){ 1389 + return pPatch; 1390 + } 1391 + assert( iTarget>=0 && iTarget<pParse->nNode ); 1392 + pTarget = &pParse->aNode[iTarget]; 1393 + assert( (pPatch->jnFlags & JNODE_APPEND)==0 ); 1394 + if( pTarget->eType!=JSON_OBJECT ){ 1395 + jsonRemoveAllNulls(pPatch); 1396 + return pPatch; 1397 + } 1398 + iRoot = iTarget; 1399 + for(i=1; i<pPatch->n; i += jsonNodeSize(&pPatch[i+1])+1){ 1400 + u32 nKey; 1401 + const char *zKey; 1402 + assert( pPatch[i].eType==JSON_STRING ); 1403 + assert( pPatch[i].jnFlags & JNODE_LABEL ); 1404 + nKey = pPatch[i].n; 1405 + zKey = pPatch[i].u.zJContent; 1406 + assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); 1407 + for(j=1; j<pTarget->n; j += jsonNodeSize(&pTarget[j+1])+1 ){ 1408 + assert( pTarget[j].eType==JSON_STRING ); 1409 + assert( pTarget[j].jnFlags & JNODE_LABEL ); 1410 + assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); 1411 + if( pTarget[j].n==nKey && strncmp(pTarget[j].u.zJContent,zKey,nKey)==0 ){ 1412 + if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; 1413 + if( pPatch[i+1].eType==JSON_NULL ){ 1414 + pTarget[j+1].jnFlags |= JNODE_REMOVE; 1415 + }else{ 1416 + JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]); 1417 + if( pNew==0 ) return 0; 1418 + pTarget = &pParse->aNode[iTarget]; 1419 + if( pNew!=&pTarget[j+1] ){ 1420 + pTarget[j+1].u.pPatch = pNew; 1421 + pTarget[j+1].jnFlags |= JNODE_PATCH; 1422 + } 1423 + } 1424 + break; 1425 + } 1426 + } 1427 + if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){ 1428 + int iStart, iPatch; 1429 + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); 1430 + jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); 1431 + iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0); 1432 + if( pParse->oom ) return 0; 1433 + jsonRemoveAllNulls(pPatch); 1434 + pTarget = &pParse->aNode[iTarget]; 1435 + pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; 1436 + pParse->aNode[iRoot].u.iAppend = iStart - iRoot; 1437 + iRoot = iStart; 1438 + pParse->aNode[iPatch].jnFlags |= JNODE_PATCH; 1439 + pParse->aNode[iPatch].u.pPatch = &pPatch[i+1]; 1440 + } 1441 + } 1442 + return pTarget; 1443 +} 1444 + 1445 +/* 1446 +** Implementation of the json_mergepatch(JSON1,JSON2) function. Return a JSON 1447 +** object that is the result of running the RFC 7396 MergePatch() algorithm 1448 +** on the two arguments. 1449 +*/ 1450 +static void jsonPatchFunc( 1451 + sqlite3_context *ctx, 1452 + int argc, 1453 + sqlite3_value **argv 1454 +){ 1455 + JsonParse x; /* The JSON that is being patched */ 1456 + JsonParse y; /* The patch */ 1457 + JsonNode *pResult; /* The result of the merge */ 1458 + 1459 + UNUSED_PARAM(argc); 1460 + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; 1461 + if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){ 1462 + jsonParseReset(&x); 1463 + return; 1464 + } 1465 + pResult = jsonMergePatch(&x, 0, y.aNode); 1466 + assert( pResult!=0 || x.oom ); 1467 + if( pResult ){ 1468 + jsonReturnJson(pResult, ctx, 0); 1469 + }else{ 1470 + sqlite3_result_error_nomem(ctx); 1471 + } 1472 + jsonParseReset(&x); 1473 + jsonParseReset(&y); 1474 +} 1475 + 1322 1476 1323 1477 /* 1324 1478 ** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON 1325 1479 ** object that contains all name/value given in arguments. Or if any name 1326 1480 ** is not a string or if any value is a BLOB, throw an error. 1327 1481 */ 1328 1482 static void jsonObjectFunc( ................................................................................ 1419 1573 assert( x.nNode ); 1420 1574 for(i=1; i<(u32)argc; i+=2){ 1421 1575 zPath = (const char*)sqlite3_value_text(argv[i]); 1422 1576 pNode = jsonLookup(&x, zPath, 0, ctx); 1423 1577 if( x.nErr ) goto replace_err; 1424 1578 if( pNode ){ 1425 1579 pNode->jnFlags |= (u8)JNODE_REPLACE; 1426 - pNode->iVal = (u8)(i+1); 1580 + pNode->u.iReplace = i + 1; 1427 1581 } 1428 1582 } 1429 1583 if( x.aNode[0].jnFlags & JNODE_REPLACE ){ 1430 - sqlite3_result_value(ctx, argv[x.aNode[0].iVal]); 1584 + sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); 1431 1585 }else{ 1432 1586 jsonReturnJson(x.aNode, ctx, argv); 1433 1587 } 1434 1588 replace_err: 1435 1589 jsonParseReset(&x); 1436 1590 } 1437 1591 ................................................................................ 1473 1627 if( x.oom ){ 1474 1628 sqlite3_result_error_nomem(ctx); 1475 1629 goto jsonSetDone; 1476 1630 }else if( x.nErr ){ 1477 1631 goto jsonSetDone; 1478 1632 }else if( pNode && (bApnd || bIsSet) ){ 1479 1633 pNode->jnFlags |= (u8)JNODE_REPLACE; 1480 - pNode->iVal = (u8)(i+1); 1634 + pNode->u.iReplace = i + 1; 1481 1635 } 1482 1636 } 1483 1637 if( x.aNode[0].jnFlags & JNODE_REPLACE ){ 1484 - sqlite3_result_value(ctx, argv[x.aNode[0].iVal]); 1638 + sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); 1485 1639 }else{ 1486 1640 jsonReturnJson(x.aNode, ctx, argv); 1487 1641 } 1488 1642 jsonSetDone: 1489 1643 jsonParseReset(&x); 1490 1644 } 1491 1645 ................................................................................ 1621 1775 } 1622 1776 static void jsonObjectFinal(sqlite3_context *ctx){ 1623 1777 JsonString *pStr; 1624 1778 pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); 1625 1779 if( pStr ){ 1626 1780 jsonAppendChar(pStr, '}'); 1627 1781 if( pStr->bErr ){ 1628 - if( pStr->bErr==0 ) sqlite3_result_error_nomem(ctx); 1782 + if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); 1629 1783 assert( pStr->bStatic ); 1630 1784 }else{ 1631 1785 sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed, 1632 1786 pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); 1633 1787 pStr->bStatic = 1; 1634 1788 } 1635 1789 }else{ ................................................................................ 1899 2053 jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); 1900 2054 jsonResult(&x); 1901 2055 break; 1902 2056 } 1903 2057 /* For json_each() path and root are the same so fall through 1904 2058 ** into the root case */ 1905 2059 } 1906 - case JEACH_ROOT: { 2060 + default: { 1907 2061 const char *zRoot = p->zRoot; 1908 - if( zRoot==0 ) zRoot = "$"; 2062 + if( zRoot==0 ) zRoot = "$"; 1909 2063 sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); 1910 2064 break; 1911 2065 } 1912 2066 case JEACH_JSON: { 1913 2067 assert( i==JEACH_JSON ); 1914 2068 sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); 1915 2069 break; ................................................................................ 2120 2274 { "json", 1, 0, jsonRemoveFunc }, 2121 2275 { "json_array", -1, 0, jsonArrayFunc }, 2122 2276 { "json_array_length", 1, 0, jsonArrayLengthFunc }, 2123 2277 { "json_array_length", 2, 0, jsonArrayLengthFunc }, 2124 2278 { "json_extract", -1, 0, jsonExtractFunc }, 2125 2279 { "json_insert", -1, 0, jsonSetFunc }, 2126 2280 { "json_object", -1, 0, jsonObjectFunc }, 2281 + { "json_patch", 2, 0, jsonPatchFunc }, 2282 + { "json_quote", 1, 0, jsonQuoteFunc }, 2127 2283 { "json_remove", -1, 0, jsonRemoveFunc }, 2128 2284 { "json_replace", -1, 0, jsonReplaceFunc }, 2129 2285 { "json_set", -1, 1, jsonSetFunc }, 2130 2286 { "json_type", 1, 0, jsonTypeFunc }, 2131 2287 { "json_type", 2, 0, jsonTypeFunc }, 2132 2288 { "json_valid", 1, 0, jsonValidFunc }, 2133 2289
Added ext/misc/memvfs.c.
1 +/* 2 +** 2016-09-07 3 +** 4 +** The author disclaims copyright to this source code. In place of 5 +** a legal notice, here is a blessing: 6 +** 7 +** May you do good and not evil. 8 +** May you find forgiveness for yourself and forgive others. 9 +** May you share freely, never taking more than you give. 10 +** 11 +****************************************************************************** 12 +** 13 +** This is an in-memory read-only VFS implementation. The application 14 +** supplies a block of memory which is the database file, and this VFS 15 +** uses that block of memory. 16 +** 17 +** Because there is no place to store journals and no good way to lock 18 +** the "file", this VFS is read-only. 19 +** 20 +** USAGE: 21 +** 22 +** sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336", &db, 23 +** SQLITE_OPEN_READONLY | SQLITE_OPEN_URI, 24 +** "memvfs"); 25 +** 26 +** The ptr= and sz= query parameters are required or the open will fail. 27 +** The ptr= parameter gives the memory address of the buffer holding the 28 +** read-only database and sz= gives the size of the database. The parameter 29 +** values may be in hexadecimal or decimal. The filename is ignored. 30 +*/ 31 +#include <sqlite3ext.h> 32 +SQLITE_EXTENSION_INIT1 33 +#include <string.h> 34 +#include <assert.h> 35 + 36 + 37 +/* 38 +** Forward declaration of objects used by this utility 39 +*/ 40 +typedef struct sqlite3_vfs MemVfs; 41 +typedef struct MemFile MemFile; 42 + 43 +/* Access to a lower-level VFS that (might) implement dynamic loading, 44 +** access to randomness, etc. 45 +*/ 46 +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) 47 + 48 +/* An open file */ 49 +struct MemFile { 50 + sqlite3_file base; /* IO methods */ 51 + sqlite3_int64 sz; /* Size of the file */ 52 + unsigned char *aData; /* content of the file */ 53 +}; 54 + 55 +/* 56 +** Methods for MemFile 57 +*/ 58 +static int memClose(sqlite3_file*); 59 +static int memRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); 60 +static int memWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); 61 +static int memTruncate(sqlite3_file*, sqlite3_int64 size); 62 +static int memSync(sqlite3_file*, int flags); 63 +static int memFileSize(sqlite3_file*, sqlite3_int64 *pSize); 64 +static int memLock(sqlite3_file*, int); 65 +static int memUnlock(sqlite3_file*, int); 66 +static int memCheckReservedLock(sqlite3_file*, int *pResOut); 67 +static int memFileControl(sqlite3_file*, int op, void *pArg); 68 +static int memSectorSize(sqlite3_file*); 69 +static int memDeviceCharacteristics(sqlite3_file*); 70 +static int memShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); 71 +static int memShmLock(sqlite3_file*, int offset, int n, int flags); 72 +static void memShmBarrier(sqlite3_file*); 73 +static int memShmUnmap(sqlite3_file*, int deleteFlag); 74 +static int memFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); 75 +static int memUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); 76 + 77 +/* 78 +** Methods for MemVfs 79 +*/ 80 +static int memOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); 81 +static int memDelete(sqlite3_vfs*, const char *zName, int syncDir); 82 +static int memAccess(sqlite3_vfs*, const char *zName, int flags, int *); 83 +static int memFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); 84 +static void *memDlOpen(sqlite3_vfs*, const char *zFilename); 85 +static void memDlError(sqlite3_vfs*, int nByte, char *zErrMsg); 86 +static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); 87 +static void memDlClose(sqlite3_vfs*, void*); 88 +static int memRandomness(sqlite3_vfs*, int nByte, char *zOut); 89 +static int memSleep(sqlite3_vfs*, int microseconds); 90 +static int memCurrentTime(sqlite3_vfs*, double*); 91 +static int memGetLastError(sqlite3_vfs*, int, char *); 92 +static int memCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); 93 + 94 +static sqlite3_vfs mem_vfs = { 95 + 2, /* iVersion */ 96 + 0, /* szOsFile (set when registered) */ 97 + 1024, /* mxPathname */ 98 + 0, /* pNext */ 99 + "memvfs", /* zName */ 100 + 0, /* pAppData (set when registered) */ 101 + memOpen, /* xOpen */ 102 + memDelete, /* xDelete */ 103 + memAccess, /* xAccess */ 104 + memFullPathname, /* xFullPathname */ 105 + memDlOpen, /* xDlOpen */ 106 + memDlError, /* xDlError */ 107 + memDlSym, /* xDlSym */ 108 + memDlClose, /* xDlClose */ 109 + memRandomness, /* xRandomness */ 110 + memSleep, /* xSleep */ 111 + memCurrentTime, /* xCurrentTime */ 112 + memGetLastError, /* xGetLastError */ 113 + memCurrentTimeInt64 /* xCurrentTimeInt64 */ 114 +}; 115 + 116 +static const sqlite3_io_methods mem_io_methods = { 117 + 3, /* iVersion */ 118 + memClose, /* xClose */ 119 + memRead, /* xRead */ 120 + memWrite, /* xWrite */ 121 + memTruncate, /* xTruncate */ 122 + memSync, /* xSync */ 123 + memFileSize, /* xFileSize */ 124 + memLock, /* xLock */ 125 + memUnlock, /* xUnlock */ 126 + memCheckReservedLock, /* xCheckReservedLock */ 127 + memFileControl, /* xFileControl */ 128 + memSectorSize, /* xSectorSize */ 129 + memDeviceCharacteristics, /* xDeviceCharacteristics */ 130 + memShmMap, /* xShmMap */ 131 + memShmLock, /* xShmLock */ 132 + memShmBarrier, /* xShmBarrier */ 133 + memShmUnmap, /* xShmUnmap */ 134 + memFetch, /* xFetch */ 135 + memUnfetch /* xUnfetch */ 136 +}; 137 + 138 + 139 + 140 +/* 141 +** Close an mem-file. 142 +** 143 +** The pData pointer is owned by the application, so there is nothing 144 +** to free. 145 +*/ 146 +static int memClose(sqlite3_file *pFile){ 147 + return SQLITE_OK; 148 +} 149 + 150 +/* 151 +** Read data from an mem-file. 152 +*/ 153 +static int memRead( 154 + sqlite3_file *pFile, 155 + void *zBuf, 156 + int iAmt, 157 + sqlite_int64 iOfst 158 +){ 159 + MemFile *p = (MemFile *)pFile; 160 + memcpy(zBuf, p->aData+iOfst, iAmt); 161 + return SQLITE_OK; 162 +} 163 + 164 +/* 165 +** Write data to an mem-file. 166 +*/ 167 +static int memWrite( 168 + sqlite3_file *pFile, 169 + const void *z, 170 + int iAmt, 171 + sqlite_int64 iOfst 172 +){ 173 + return SQLITE_READONLY; 174 +} 175 + 176 +/* 177 +** Truncate an mem-file. 178 +*/ 179 +static int memTruncate(sqlite3_file *pFile, sqlite_int64 size){ 180 + return SQLITE_READONLY; 181 +} 182 + 183 +/* 184 +** Sync an mem-file. 185 +*/ 186 +static int memSync(sqlite3_file *pFile, int flags){ 187 + return SQLITE_READONLY; 188 +} 189 + 190 +/* 191 +** Return the current file-size of an mem-file. 192 +*/ 193 +static int memFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ 194 + MemFile *p = (MemFile *)pFile; 195 + *pSize = p->sz; 196 + return SQLITE_OK; 197 +} 198 + 199 +/* 200 +** Lock an mem-file. 201 +*/ 202 +static int memLock(sqlite3_file *pFile, int eLock){ 203 + return SQLITE_READONLY; 204 +} 205 + 206 +/* 207 +** Unlock an mem-file. 208 +*/ 209 +static int memUnlock(sqlite3_file *pFile, int eLock){ 210 + return SQLITE_OK; 211 +} 212 + 213 +/* 214 +** Check if another file-handle holds a RESERVED lock on an mem-file. 215 +*/ 216 +static int memCheckReservedLock(sqlite3_file *pFile, int *pResOut){ 217 + *pResOut = 0; 218 + return SQLITE_OK; 219 +} 220 + 221 +/* 222 +** File control method. For custom operations on an mem-file. 223 +*/ 224 +static int memFileControl(sqlite3_file *pFile, int op, void *pArg){ 225 + MemFile *p = (MemFile *)pFile; 226 + int rc = SQLITE_NOTFOUND; 227 + if( op==SQLITE_FCNTL_VFSNAME ){ 228 + *(char**)pArg = sqlite3_mprintf("mem(%p,%lld)", p->aData, p->sz); 229 + rc = SQLITE_OK; 230 + } 231 + return rc; 232 +} 233 + 234 +/* 235 +** Return the sector-size in bytes for an mem-file. 236 +*/ 237 +static int memSectorSize(sqlite3_file *pFile){ 238 + return 1024; 239 +} 240 + 241 +/* 242 +** Return the device characteristic flags supported by an mem-file. 243 +*/ 244 +static int memDeviceCharacteristics(sqlite3_file *pFile){ 245 + return SQLITE_IOCAP_IMMUTABLE; 246 +} 247 + 248 +/* Create a shared memory file mapping */ 249 +static int memShmMap( 250 + sqlite3_file *pFile, 251 + int iPg, 252 + int pgsz, 253 + int bExtend, 254 + void volatile **pp 255 +){ 256 + return SQLITE_READONLY; 257 +} 258 + 259 +/* Perform locking on a shared-memory segment */ 260 +static int memShmLock(sqlite3_file *pFile, int offset, int n, int flags){ 261 + return SQLITE_READONLY; 262 +} 263 + 264 +/* Memory barrier operation on shared memory */ 265 +static void memShmBarrier(sqlite3_file *pFile){ 266 + return; 267 +} 268 + 269 +/* Unmap a shared memory segment */ 270 +static int memShmUnmap(sqlite3_file *pFile, int deleteFlag){ 271 + return SQLITE_OK; 272 +} 273 + 274 +/* Fetch a page of a memory-mapped file */ 275 +static int memFetch( 276 + sqlite3_file *pFile, 277 + sqlite3_int64 iOfst, 278 + int iAmt, 279 + void **pp 280 +){ 281 + MemFile *p = (MemFile *)pFile; 282 + *pp = (void*)(p->aData + iOfst); 283 + return SQLITE_OK; 284 +} 285 + 286 +/* Release a memory-mapped page */ 287 +static int memUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ 288 + return SQLITE_OK; 289 +} 290 + 291 +/* 292 +** Open an mem file handle. 293 +*/ 294 +static int memOpen( 295 + sqlite3_vfs *pVfs, 296 + const char *zName, 297 + sqlite3_file *pFile, 298 + int flags, 299 + int *pOutFlags 300 +){ 301 + MemFile *p = (MemFile*)pFile; 302 + memset(p, 0, sizeof(*p)); 303 + if( (flags & SQLITE_OPEN_MAIN_DB)==0 ) return SQLITE_CANTOPEN; 304 + p->aData = (unsigned char*)sqlite3_uri_int64(zName,"ptr",0); 305 + if( p->aData==0 ) return SQLITE_CANTOPEN; 306 + p->sz = sqlite3_uri_int64(zName,"sz",0); 307 + if( p->sz<0 ) return SQLITE_CANTOPEN; 308 + pFile->pMethods = &mem_io_methods; 309 + return SQLITE_OK; 310 +} 311 + 312 +/* 313 +** Delete the file located at zPath. If the dirSync argument is true, 314 +** ensure the file-system modifications are synced to disk before 315 +** returning. 316 +*/ 317 +static int memDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ 318 + return SQLITE_READONLY; 319 +} 320 + 321 +/* 322 +** Test for access permissions. Return true if the requested permission 323 +** is available, or false otherwise. 324 +*/ 325 +static int memAccess( 326 + sqlite3_vfs *pVfs, 327 + const char *zPath, 328 + int flags, 329 + int *pResOut 330 +){ 331 + /* The spec says there are three possible values for flags. But only 332 + ** two of them are actually used */ 333 + assert( flags==SQLITE_ACCESS_EXISTS || flags==SQLITE_ACCESS_READWRITE ); 334 + if( flags==SQLITE_ACCESS_READWRITE ){ 335 + *pResOut = 0; 336 + }else{ 337 + *pResOut = 1; 338 + } 339 + return SQLITE_OK; 340 +} 341 + 342 +/* 343 +** Populate buffer zOut with the full canonical pathname corresponding 344 +** to the pathname in zPath. zOut is guaranteed to point to a buffer 345 +** of at least (INST_MAX_PATHNAME+1) bytes. 346 +*/ 347 +static int memFullPathname( 348 + sqlite3_vfs *pVfs, 349 + const char *zPath, 350 + int nOut, 351 + char *zOut 352 +){ 353 + sqlite3_snprintf(nOut, zOut, "%s", zPath); 354 + return SQLITE_OK; 355 +} 356 + 357 +/* 358 +** Open the dynamic library located at zPath and return a handle. 359 +*/ 360 +static void *memDlOpen(sqlite3_vfs *pVfs, const char *zPath){ 361 + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); 362 +} 363 + 364 +/* 365 +** Populate the buffer zErrMsg (size nByte bytes) with a human readable 366 +** utf-8 string describing the most recent error encountered associated 367 +** with dynamic libraries. 368 +*/ 369 +static void memDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ 370 + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); 371 +} 372 + 373 +/* 374 +** Return a pointer to the symbol zSymbol in the dynamic library pHandle. 375 +*/ 376 +static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ 377 + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); 378 +} 379 + 380 +/* 381 +** Close the dynamic library handle pHandle. 382 +*/ 383 +static void memDlClose(sqlite3_vfs *pVfs, void *pHandle){ 384 + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); 385 +} 386 + 387 +/* 388 +** Populate the buffer pointed to by zBufOut with nByte bytes of 389 +** random data. 390 +*/ 391 +static int memRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ 392 + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); 393 +} 394 + 395 +/* 396 +** Sleep for nMicro microseconds. Return the number of microseconds 397 +** actually slept. 398 +*/ 399 +static int memSleep(sqlite3_vfs *pVfs, int nMicro){ 400 + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); 401 +} 402 + 403 +/* 404 +** Return the current time as a Julian Day number in *pTimeOut. 405 +*/ 406 +static int memCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ 407 + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); 408 +} 409 + 410 +static int memGetLastError(sqlite3_vfs *pVfs, int a, char *b){ 411 + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); 412 +} 413 +static int memCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ 414 + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); 415 +} 416 + 417 +#ifdef MEMVFS_TEST 418 +/* 419 +** memload(FILENAME) 420 +** 421 +** This an SQL function used to help in testing the memvfs VFS. The 422 +** function reads the content of a file into memory and then returns 423 +** a string that gives the locate and size of the in-memory buffer. 424 +*/ 425 +#include <stdio.h> 426 +static void memvfsMemloadFunc( 427 + sqlite3_context *context, 428 + int argc, 429 + sqlite3_value **argv 430 +){ 431 + unsigned char *p; 432 + sqlite3_int64 sz; 433 + FILE *in; 434 + const char *zFilename = (const char*)sqlite3_value_text(argv[0]); 435 + char zReturn[100]; 436 + 437 + if( zFilename==0 ) return; 438 + in = fopen(zFilename, "rb"); 439 + if( in==0 ) return; 440 + fseek(in, 0, SEEK_END); 441 + sz = ftell(in); 442 + rewind(in); 443 + p = sqlite3_malloc( sz ); 444 + if( p==0 ){ 445 + fclose(in); 446 + sqlite3_result_error_nomem(context); 447 + return; 448 + } 449 + fread(p, sz, 1, in); 450 + fclose(in); 451 + sqlite3_snprintf(sizeof(zReturn),zReturn,"ptr=%lld&sz=%lld", 452 + (sqlite3_int64)p, sz); 453 + sqlite3_result_text(context, zReturn, -1, SQLITE_TRANSIENT); 454 +} 455 +/* Called for each new database connection */ 456 +static int memvfsRegister( 457 + sqlite3 *db, 458 + const char **pzErrMsg, 459 + const struct sqlite3_api_routines *pThunk 460 +){ 461 + return sqlite3_create_function(db, "memload", 1, SQLITE_UTF8, 0, 462 + memvfsMemloadFunc, 0, 0); 463 +} 464 +#endif /* MEMVFS_TEST */ 465 + 466 + 467 +#ifdef _WIN32 468 +__declspec(dllexport) 469 +#endif 470 +/* 471 +** This routine is called when the extension is loaded. 472 +** Register the new VFS. 473 +*/ 474 +int sqlite3_memvfs_init( 475 + sqlite3 *db, 476 + char **pzErrMsg, 477 + const sqlite3_api_routines *pApi 478 +){ 479 + int rc = SQLITE_OK; 480 + SQLITE_EXTENSION_INIT2(pApi); 481 + mem_vfs.pAppData = sqlite3_vfs_find(0); 482 + mem_vfs.szOsFile = sizeof(MemFile); 483 + rc = sqlite3_vfs_register(&mem_vfs, 1); 484 +#ifdef MEMVFS_TEST 485 + if( rc==SQLITE_OK ){ 486 + rc = sqlite3_auto_extension((void(*)(void))memvfsRegister); 487 + } 488 +#endif 489 + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; 490 + return rc; 491 +}
Changes to ext/misc/percentile.c.
163 163 } 164 164 p->a[p->nUsed++] = y; 165 165 } 166 166 167 167 /* 168 168 ** Compare to doubles for sorting using qsort() 169 169 */ 170 -static int doubleCmp(const void *pA, const void *pB){ 170 +static int SQLITE_CDECL doubleCmp(const void *pA, const void *pB){ 171 171 double a = *(double*)pA; 172 172 double b = *(double*)pB; 173 173 if( a==b ) return 0; 174 174 if( a<b ) return -1; 175 175 return +1; 176 176 } 177 177
Changes to ext/misc/regexp.c.
132 132 unsigned nAlloc; /* Slots allocated for aOp[] and aArg[] */ 133 133 }; 134 134 135 135 /* Add a state to the given state set if it is not already there */ 136 136 static void re_add_state(ReStateSet *pSet, int newState){ 137 137 unsigned i; 138 138 for(i=0; i<pSet->nState; i++) if( pSet->aState[i]==newState ) return; 139 - pSet->aState[pSet->nState++] = newState; 139 + pSet->aState[pSet->nState++] = (ReStateNumber)newState; 140 140 } 141 141 142 142 /* Extract the next unicode character from *pzIn and return it. Advance 143 143 ** *pzIn to the first byte past the end of the character returned. To 144 144 ** be clear: this routine converts utf8 to unicode. This routine is 145 145 ** optimized for the common case where the next character is a single byte. 146 146 */ ................................................................................ 354 354 int i; 355 355 if( p->nAlloc<=p->nState && re_resize(p, p->nAlloc*2) ) return 0; 356 356 for(i=p->nState; i>iBefore; i--){ 357 357 p->aOp[i] = p->aOp[i-1]; 358 358 p->aArg[i] = p->aArg[i-1]; 359 359 } 360 360 p->nState++; 361 - p->aOp[iBefore] = op; 361 + p->aOp[iBefore] = (char)op; 362 362 p->aArg[iBefore] = arg; 363 363 return iBefore; 364 364 } 365 365 366 366 /* Append a new opcode and argument to the end of the RE under construction. 367 367 */ 368 368 static int re_append(ReCompiled *p, int op, int arg){ ................................................................................ 673 673 ** regex engine over the string. Do not worry able trying to match 674 674 ** unicode characters beyond plane 0 - those are very rare and this is 675 675 ** just an optimization. */ 676 676 if( pRe->aOp[0]==RE_OP_ANYSTAR ){ 677 677 for(j=0, i=1; j<sizeof(pRe->zInit)-2 && pRe->aOp[i]==RE_OP_MATCH; i++){ 678 678 unsigned x = pRe->aArg[i]; 679 679 if( x<=127 ){ 680 - pRe->zInit[j++] = x; 680 + pRe->zInit[j++] = (unsigned char)x; 681 681 }else if( x<=0xfff ){ 682 - pRe->zInit[j++] = 0xc0 | (x>>6); 682 + pRe->zInit[j++] = (unsigned char)(0xc0 | (x>>6)); 683 683 pRe->zInit[j++] = 0x80 | (x&0x3f); 684 684 }else if( x<=0xffff ){ 685 - pRe->zInit[j++] = 0xd0 | (x>>12); 685 + pRe->zInit[j++] = (unsigned char)(0xd0 | (x>>12)); 686 686 pRe->zInit[j++] = 0x80 | ((x>>6)&0x3f); 687 687 pRe->zInit[j++] = 0x80 | (x&0x3f); 688 688 }else{ 689 689 break; 690 690 } 691 691 } 692 692 if( j>0 && pRe->zInit[j-1]==0 ) j--;
Added ext/misc/remember.c.
1 +/* 2 +** 2016-08-09 3 +** 4 +** The author disclaims copyright to this source code. In place of 5 +** a legal notice, here is a blessing: 6 +** 7 +** May you do good and not evil. 8 +** May you find forgiveness for yourself and forgive others. 9 +** May you share freely, never taking more than you give. 10 +** 11 +************************************************************************* 12 +** 13 +** This file demonstrates how to create an SQL function that is a pass-through 14 +** for integer values (it returns a copy of its argument) but also saves the 15 +** value that is passed through into a C-language variable. The address of 16 +** the C-language variable is supplied as the second argument. 17 +** 18 +** This allows, for example, a counter to incremented and the original 19 +** value retrieved, atomically, using a single statement: 20 +** 21 +** UPDATE counterTab SET cnt=remember(cnt,$PTR)+1 WHERE id=$ID 22 +** 23 +** Prepare the above statement once. Then to use it, bind the address 24 +** of the output variable to $PTR and the id of the counter to $ID and 25 +** run the prepared statement. 26 +** 27 +** One can imagine doing similar things with floating-point values and 28 +** strings, but this demonstration extension will stick to using just 29 +** integers. 30 +*/ 31 +#include "sqlite3ext.h" 32 +SQLITE_EXTENSION_INIT1 33 +#include <assert.h> 34 + 35 +/* 36 +** remember(V,PTR) 37 +** 38 +** Return the integer value V. Also save the value of V in a 39 +** C-language variable whose address is PTR. 40 +*/ 41 +static void rememberFunc( 42 + sqlite3_context *pCtx, 43 + int argc, 44 + sqlite3_value **argv 45 +){ 46 + sqlite3_int64 v; 47 + sqlite3_int64 ptr; 48 + assert( argc==2 ); 49 + v = sqlite3_value_int64(argv[0]); 50 + ptr = sqlite3_value_int64(argv[1]); 51 + *((sqlite3_int64*)ptr) = v; 52 + sqlite3_result_int64(pCtx, v); 53 +} 54 + 55 +#ifdef _WIN32 56 +__declspec(dllexport) 57 +#endif 58 +int sqlite3_remember_init( 59 + sqlite3 *db, 60 + char **pzErrMsg, 61 + const sqlite3_api_routines *pApi 62 +){ 63 + int rc = SQLITE_OK; 64 + SQLITE_EXTENSION_INIT2(pApi); 65 + rc = sqlite3_create_function(db, "remember", 2, SQLITE_UTF8, 0, 66 + rememberFunc, 0, 0); 67 + return rc; 68 +}
Added ext/misc/scrub.c.
1 +/* 2 +** 2016-05-05 3 +** 4 +** The author disclaims copyright to this source code. In place of 5 +** a legal notice, here is a blessing: 6 +** 7 +** May you do good and not evil. 8 +** May you find forgiveness for yourself and forgive others. 9 +** May you share freely, never taking more than you give. 10 +** 11 +****************************************************************************** 12 +** 13 +** This file implements a utility function (and a utility program) that 14 +** makes a copy of an SQLite database while simultaneously zeroing out all 15 +** deleted content. 16 +** 17 +** Normally (when PRAGMA secure_delete=OFF, which is the default) when SQLite 18 +** deletes content, it does not overwrite the deleted content but rather marks 19 +** the region of the file that held that content as being reusable. This can 20 +** cause deleted content to recoverable from the database file. This stale 21 +** content is removed by the VACUUM command, but VACUUM can be expensive for 22 +** large databases. When in PRAGMA secure_delete=ON mode, the deleted content 23 +** is zeroed, but secure_delete=ON has overhead as well. 24 +** 25 +** This utility attempts to make a copy of a complete SQLite database where 26 +** all of the deleted content is zeroed out in the copy, and it attempts to 27 +** do so while being faster than running VACUUM. 28 +** 29 +** Usage: 30 +** 31 +** int sqlite3_scrub_backup( 32 +** const char *zSourceFile, // Source database filename 33 +** const char *zDestFile, // Destination database filename 34 +** char **pzErrMsg // Write error message here 35 +** ); 36 +** 37 +** Simply call the API above specifying the filename of the source database 38 +** and the name of the backup copy. The source database must already exist 39 +** and can be in active use. (A read lock is held during the backup.) The 40 +** destination file should not previously exist. If the pzErrMsg parameter 41 +** is non-NULL and if an error occurs, then an error message might be written 42 +** into memory obtained from sqlite3_malloc() and *pzErrMsg made to point to 43 +** that error message. But if the error is an OOM, the error might not be 44 +** reported. The routine always returns non-zero if there is an error. 45 +** 46 +** If compiled with -DSCRUB_STANDALONE then a main() procedure is added and 47 +** this file becomes a standalone program that can be run as follows: 48 +** 49 +** ./sqlite3scrub SOURCE DEST 50 +*/ 51 +#include "sqlite3.h" 52 +#include <assert.h> 53 +#include <stdio.h> 54 +#include <stdlib.h> 55 +#include <stdarg.h> 56 +#include <string.h> 57 + 58 +typedef struct ScrubState ScrubState; 59 +typedef unsigned char u8; 60 +typedef unsigned short u16; 61 +typedef unsigned int u32; 62 + 63 + 64 +/* State information for a scrub-and-backup operation */ 65 +struct ScrubState { 66 + const char *zSrcFile; /* Name of the source file */ 67 + const char *zDestFile; /* Name of the destination file */ 68 + int rcErr; /* Error code */ 69 + char *zErr; /* Error message text */ 70 + sqlite3 *dbSrc; /* Source database connection */ 71 + sqlite3_file *pSrc; /* Source file handle */ 72 + sqlite3 *dbDest; /* Destination database connection */ 73 + sqlite3_file *pDest; /* Destination file handle */ 74 + u32 szPage; /* Page size */ 75 + u32 szUsable; /* Usable bytes on each page */ 76 + u32 nPage; /* Number of pages */ 77 + u32 iLastPage; /* Page number of last page written so far*/ 78 + u8 *page1; /* Content of page 1 */ 79 +}; 80 + 81 +/* Store an error message */ 82 +static void scrubBackupErr(ScrubState *p, const char *zFormat, ...){ 83 + va_list ap; 84 + sqlite3_free(p->zErr); 85 + va_start(ap, zFormat); 86 + p->zErr = sqlite3_vmprintf(zFormat, ap); 87 + va_end(ap); 88 + if( p->rcErr==0 ) p->rcErr = SQLITE_ERROR; 89 +} 90 + 91 +/* Allocate memory to hold a single page of content */ 92 +static u8 *scrubBackupAllocPage(ScrubState *p){ 93 + u8 *pPage; 94 + if( p->rcErr ) return 0; 95 + pPage = sqlite3_malloc( p->szPage ); 96 + if( pPage==0 ) p->rcErr = SQLITE_NOMEM; 97 + return pPage; 98 +} 99 + 100 +/* Read a page from the source database into memory. Use the memory 101 +** provided by pBuf if not NULL or allocate a new page if pBuf==NULL. 102 +*/ 103 +static u8 *scrubBackupRead(ScrubState *p, int pgno, u8 *pBuf){ 104 + int rc; 105 + sqlite3_int64 iOff; 106 + u8 *pOut = pBuf; 107 + if( p->rcErr ) return 0; 108 + if( pOut==0 ){ 109 + pOut = scrubBackupAllocPage(p); 110 + if( pOut==0 ) return 0; 111 + } 112 + iOff = (pgno-1)*(sqlite3_int64)p->szPage; 113 + rc = p->pSrc->pMethods->xRead(p->pSrc, pOut, p->szPage, iOff); 114 + if( rc!=SQLITE_OK ){ 115 + if( pBuf==0 ) sqlite3_free(pOut); 116 + pOut = 0; 117 + scrubBackupErr(p, "read failed for page %d", pgno); 118 + p->rcErr = SQLITE_IOERR; 119 + } 120 + return pOut; 121 +} 122 + 123 +/* Write a page to the destination database */ 124 +static void scrubBackupWrite(ScrubState *p, int pgno, const u8 *pData){ 125 + int rc; 126 + sqlite3_int64 iOff; 127 + if( p->rcErr ) return; 128 + iOff = (pgno-1)*(sqlite3_int64)p->szPage; 129 + rc = p->pDest->pMethods->xWrite(p->pDest, pData, p->szPage, iOff); 130 + if( rc!=SQLITE_OK ){ 131 + scrubBackupErr(p, "write failed for page %d", pgno); 132 + p->rcErr = SQLITE_IOERR; 133 + } 134 + if( pgno>p->iLastPage ) p->iLastPage = pgno; 135 +} 136 + 137 +/* Prepare a statement against the "db" database. */ 138 +static sqlite3_stmt *scrubBackupPrepare( 139 + ScrubState *p, /* Backup context */ 140 + sqlite3 *db, /* Database to prepare against */ 141 + const char *zSql /* SQL statement */ 142 +){ 143 + sqlite3_stmt *pStmt; 144 + if( p->rcErr ) return 0; 145 + p->rcErr = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); 146 + if( p->rcErr ){ 147 + scrubBackupErr(p, "SQL error \"%s\" on \"%s\"", 148 + sqlite3_errmsg(db), zSql); 149 + sqlite3_finalize(pStmt); 150 + return 0; 151 + } 152 + return pStmt; 153 +} 154 + 155 + 156 +/* Open the source database file */ 157 +static void scrubBackupOpenSrc(ScrubState *p){ 158 + sqlite3_stmt *pStmt; 159 + int rc; 160 + /* Open the source database file */ 161 + p->rcErr = sqlite3_open_v2(p->zSrcFile, &p->dbSrc, 162 + SQLITE_OPEN_READWRITE | 163 + SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0); 164 + if( p->rcErr ){ 165 + scrubBackupErr(p, "cannot open source database: %s", 166 + sqlite3_errmsg(p->dbSrc)); 167 + return; 168 + } 169 + p->rcErr = sqlite3_exec(p->dbSrc, "SELECT 1 FROM sqlite_master; BEGIN;", 170 + 0, 0, 0); 171 + if( p->rcErr ){ 172 + scrubBackupErr(p, 173 + "cannot start a read transaction on the source database: %s", 174 + sqlite3_errmsg(p->dbSrc)); 175 + return; 176 + } 177 + rc = sqlite3_wal_checkpoint_v2(p->dbSrc, "main", SQLITE_CHECKPOINT_FULL, 178 + 0, 0); 179 + if( rc ){ 180 + scrubBackupErr(p, "cannot checkpoint the source database"); 181 + return; 182 + } 183 + pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_size"); 184 + if( pStmt==0 ) return; 185 + rc = sqlite3_step(pStmt); 186 + if( rc==SQLITE_ROW ){ 187 + p->szPage = sqlite3_column_int(pStmt, 0); 188 + }else{ 189 + scrubBackupErr(p, "unable to determine the page size"); 190 + } 191 + sqlite3_finalize(pStmt); 192 + if( p->rcErr ) return; 193 + pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_count"); 194 + if( pStmt==0 ) return; 195 + rc = sqlite3_step(pStmt); 196 + if( rc==SQLITE_ROW ){ 197 + p->nPage = sqlite3_column_int(pStmt, 0); 198 + }else{ 199 + scrubBackupErr(p, "unable to determine the size of the source database"); 200 + } 201 + sqlite3_finalize(pStmt); 202 + sqlite3_file_control(p->dbSrc, "main", SQLITE_FCNTL_FILE_POINTER, &p->pSrc); 203 + if( p->pSrc==0 || p->pSrc->pMethods==0 ){ 204 + scrubBackupErr(p, "cannot get the source file handle"); 205 + p->rcErr = SQLITE_ERROR; 206 + } 207 +} 208 + 209 +/* Create and open the destination file */ 210 +static void scrubBackupOpenDest(ScrubState *p){ 211 + sqlite3_stmt *pStmt; 212 + int rc; 213 + char *zSql; 214 + if( p->rcErr ) return; 215 + p->rcErr = sqlite3_open_v2(p->zDestFile, &p->dbDest, 216 + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | 217 + SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0); 218 + if( p->rcErr ){ 219 + scrubBackupErr(p, "cannot open destination database: %s", 220 + sqlite3_errmsg(p->dbDest)); 221 + return; 222 + } 223 + zSql = sqlite3_mprintf("PRAGMA page_size(%u);", p->szPage); 224 + if( zSql==0 ){ 225 + p->rcErr = SQLITE_NOMEM; 226 + return; 227 + } 228 + p->rcErr = sqlite3_exec(p->dbDest, zSql, 0, 0, 0); 229 + sqlite3_free(zSql); 230 + if( p->rcErr ){ 231 + scrubBackupErr(p, 232 + "cannot set the page size on the destination database: %s", 233 + sqlite3_errmsg(p->dbDest)); 234 + return; 235 + } 236 + sqlite3_exec(p->dbDest, "PRAGMA journal_mode=OFF;", 0, 0, 0); 237 + p->rcErr = sqlite3_exec(p->dbDest, "BEGIN EXCLUSIVE;", 0, 0, 0); 238 + if( p->rcErr ){ 239 + scrubBackupErr(p, 240 + "cannot start a write transaction on the destination database: %s", 241 + sqlite3_errmsg(p->dbDest)); 242 + return; 243 + } 244 + pStmt = scrubBackupPrepare(p, p->dbDest, "PRAGMA page_count;"); 245 + if( pStmt==0 ) return; 246 + rc = sqlite3_step(pStmt); 247 + if( rc!=SQLITE_ROW ){ 248 + scrubBackupErr(p, "cannot measure the size of the destination"); 249 + }else if( sqlite3_column_int(pStmt, 0)>1 ){ 250 + scrubBackupErr(p, "destination database is not empty - holds %d pages", 251 + sqlite3_column_int(pStmt, 0)); 252 + } 253 + sqlite3_finalize(pStmt); 254 + sqlite3_file_control(p->dbDest, "main", SQLITE_FCNTL_FILE_POINTER, &p->pDest); 255 + if( p->pDest==0 || p->pDest->pMethods==0 ){ 256 + scrubBackupErr(p, "cannot get the destination file handle"); 257 + p->rcErr = SQLITE_ERROR; 258 + } 259 +} 260 + 261 +/* Read a 32-bit big-endian integer */ 262 +static u32 scrubBackupInt32(const u8 *a){ 263 + u32 v = a[3]; 264 + v += ((u32)a[2])<<8; 265 + v += ((u32)a[1])<<16; 266 + v += ((u32)a[0])<<24; 267 + return v; 268 +} 269 + 270 +/* Read a 16-bit big-endian integer */ 271 +static u32 scrubBackupInt16(const u8 *a){ 272 + return (a[0]<<8) + a[1]; 273 +} 274 + 275 +/* 276 +** Read a varint. Put the value in *pVal and return the number of bytes. 277 +*/ 278 +static int scrubBackupVarint(const u8 *z, sqlite3_int64 *pVal){ 279 + sqlite3_int64 v = 0; 280 + int i; 281 + for(i=0; i<8; i++){ 282 + v = (v<<7) + (z[i]&0x7f); 283 + if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; } 284 + } 285 + v = (v<<8) + (z[i]&0xff); 286 + *pVal = v; 287 + return 9; 288 +} 289 + 290 +/* 291 +** Return the number of bytes in a varint. 292 +*/ 293 +static int scrubBackupVarintSize(const u8 *z){ 294 + int i; 295 + for(i=0; i<8; i++){ 296 + if( (z[i]&0x80)==0 ){ return i+1; } 297 + } 298 + return 9; 299 +} 300 + 301 +/* 302 +** Copy the freelist trunk page given, and all its descendents, 303 +** zeroing out as much as possible in the process. 304 +*/ 305 +static void scrubBackupFreelist(ScrubState *p, int pgno, u32 nFree){ 306 + u8 *a, *aBuf; 307 + u32 n, mx; 308 + 309 + if( p->rcErr ) return; 310 + aBuf = scrubBackupAllocPage(p); 311 + if( aBuf==0 ) return; 312 + 313 + while( pgno && nFree){ 314 + a = scrubBackupRead(p, pgno, aBuf); 315 + if( a==0 ) break; 316 + n = scrubBackupInt32(&a[4]); 317 + mx = p->szUsable/4 - 2; 318 + if( n<mx ){ 319 + memset(&a[n*4+8], 0, 4*(mx-n)); 320 + } 321 + scrubBackupWrite(p, pgno, a); 322 + pgno = scrubBackupInt32(a); 323 +#if 0 324 + /* There is really no point in copying the freelist leaf pages. 325 + ** Simply leave them uninitialized in the destination database. The 326 + ** OS filesystem should zero those pages for us automatically. 327 + */ 328 + for(i=0; i<n && nFree; i++){ 329 + u32 iLeaf = scrubBackupInt32(&a[i*4+8]); 330 + if( aZero==0 ){ 331 + aZero = scrubBackupAllocPage(p); 332 + if( aZero==0 ){ pgno = 0; break; } 333 + memset(aZero, 0, p->szPage); 334 + } 335 + scrubBackupWrite(p, iLeaf, aZero); 336 + nFree--; 337 + } 338 +#endif 339 + } 340 + sqlite3_free(aBuf); 341 +} 342 + 343 +/* 344 +** Copy an overflow chain from source to destination. Zero out any