Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Merge all the latest trunk enhancements. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | begin-concurrent |
| Files: | files | file ages | folders |
| SHA3-256: |
213c61cb82d4ee51cc638fc5bb2b54cd |
| User & Date: | drh 2017-07-20 17:47:12 |
Context
|
2017-07-20
| ||
| 18:28 | Get the build working with SQLITE_OMIT_CONCURRENT. (check-in: a29401e9 user: drh tags: begin-concurrent) | |
| 17:47 | Merge all the latest trunk enhancements. (check-in: 213c61cb user: drh tags: begin-concurrent) | |
| 15:08 | Enhance the built-in date/time functions so that they can be used in CHECK constraints, in the WHERE clause or partial indexes, and index expressions, provided that none of the non-deterministic keywords ("now", "localtime", "utc") are used as arguments. (check-in: a90c062d user: drh tags: trunk) | |
|
2017-06-10
| ||
| 17:23 | Remove sqlite3_log() and abort() calls added to this branch to debug the pointer-map problem ([fda22108]). (check-in: 79544fc2 user: dan tags: begin-concurrent) | |
Changes
Changes to Makefile.in.
177 177 icu.lo insert.lo json1.lo legacy.lo loadext.lo \ 178 178 main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ 179 179 memjournal.lo \ 180 180 mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ 181 181 notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ 182 182 pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ 183 183 random.lo resolve.lo rowset.lo rtree.lo \ 184 - sqlite3session.lo select.lo sqlite3rbu.lo status.lo \ 184 + sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \ 185 185 table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ 186 186 update.lo util.lo vacuum.lo \ 187 187 vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ 188 188 vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \ 189 189 utf.lo vtab.lo 190 190 191 191 # Object files for the amalgamation. ................................................................................ 346 346 SRC += \ 347 347 $(TOP)/ext/session/sqlite3session.c \ 348 348 $(TOP)/ext/session/sqlite3session.h 349 349 SRC += \ 350 350 $(TOP)/ext/rbu/sqlite3rbu.h \ 351 351 $(TOP)/ext/rbu/sqlite3rbu.c 352 352 SRC += \ 353 - $(TOP)/ext/misc/json1.c 353 + $(TOP)/ext/misc/json1.c \ 354 + $(TOP)/ext/misc/stmt.c 354 355 355 356 # Generated source code files 356 357 # 357 358 SRC += \ 358 359 keywordhash.h \ 359 360 opcodes.c \ 360 361 opcodes.h \ ................................................................................ 430 431 $(TOP)/ext/misc/nextchar.c \ 431 432 $(TOP)/ext/misc/percentile.c \ 432 433 $(TOP)/ext/misc/regexp.c \ 433 434 $(TOP)/ext/misc/remember.c \ 434 435 $(TOP)/ext/misc/series.c \ 435 436 $(TOP)/ext/misc/spellfix.c \ 436 437 $(TOP)/ext/misc/totype.c \ 438 + $(TOP)/ext/misc/unionvtab.c \ 437 439 $(TOP)/ext/misc/wholenumber.c 438 440 439 441 # Source code to the library files needed by the test fixture 440 442 # 441 443 TESTSRC2 = \ 442 444 $(TOP)/src/attach.c \ 443 445 $(TOP)/src/backup.c \ ................................................................................ 479 481 $(TOP)/ext/fts3/fts3.c \ 480 482 $(TOP)/ext/fts3/fts3_aux.c \ 481 483 $(TOP)/ext/fts3/fts3_expr.c \ 482 484 $(TOP)/ext/fts3/fts3_term.c \ 483 485 $(TOP)/ext/fts3/fts3_tokenizer.c \ 484 486 $(TOP)/ext/fts3/fts3_write.c \ 485 487 $(TOP)/ext/async/sqlite3async.c \ 486 - $(TOP)/ext/session/sqlite3session.c 488 + $(TOP)/ext/session/sqlite3session.c \ 489 + $(TOP)/ext/misc/stmt.c 487 490 488 491 # Header files used by all library source files. 489 492 # 490 493 HDR = \ 491 494 $(TOP)/src/btree.h \ 492 495 $(TOP)/src/btreeInt.h \ 493 496 $(TOP)/src/hash.h \ ................................................................................ 560 563 561 564 # Extra compiler options for various shell tools 562 565 # 563 566 SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 564 567 # SHELL_OPT += -DSQLITE_ENABLE_FTS5 565 568 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS 566 569 SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION 570 +SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB 567 571 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 568 572 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ 569 573 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 570 574 FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c 571 575 DBFUZZ_OPT = 572 576 573 577 # This is the default Makefile target. The objects listed here ................................................................................ 1030 1034 1031 1035 sqlite3session.lo: $(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR) 1032 1036 $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c 1033 1037 1034 1038 json1.lo: $(TOP)/ext/misc/json1.c 1035 1039 $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c 1036 1040 1041 +stmt.lo: $(TOP)/ext/misc/stmt.c 1042 + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/stmt.c 1043 + 1037 1044 # FTS5 things 1038 1045 # 1039 1046 FTS5_SRC = \ 1040 1047 $(TOP)/ext/fts5/fts5.h \ 1041 1048 $(TOP)/ext/fts5/fts5Int.h \ 1042 1049 $(TOP)/ext/fts5/fts5_aux.c \ 1043 1050 $(TOP)/ext/fts5/fts5_buffer.c \ ................................................................................ 1079 1086 # hidden when the library is built via the amalgamation). 1080 1087 # 1081 1088 TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 1082 1089 TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 1083 1090 TESTFIXTURE_FLAGS += -DBUILD_sqlite 1084 1091 TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 1085 1092 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024 1093 +TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB 1086 1094 1087 1095 TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la 1088 1096 TESTFIXTURE_SRC1 = sqlite3.c 1089 1097 TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c 1090 1098 TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION)) 1091 1099 1092 1100 testfixture$(TEXE): $(TESTFIXTURE_SRC)
Changes to Makefile.msc.
1290 1290 $(TOP)\ext\fts3\fts3_unicode.c \ 1291 1291 $(TOP)\ext\fts3\fts3_unicode2.c \ 1292 1292 $(TOP)\ext\fts3\fts3_write.c \ 1293 1293 $(TOP)\ext\icu\icu.c \ 1294 1294 $(TOP)\ext\rtree\rtree.c \ 1295 1295 $(TOP)\ext\session\sqlite3session.c \ 1296 1296 $(TOP)\ext\rbu\sqlite3rbu.c \ 1297 - $(TOP)\ext\misc\json1.c 1297 + $(TOP)\ext\misc\json1.c \ 1298 + $(TOP)\ext\misc\stmt.c 1298 1299 1299 1300 # Extension header files, part 1. 1300 1301 # 1301 1302 SRC08 = \ 1302 1303 $(TOP)\ext\fts1\fts1.h \ 1303 1304 $(TOP)\ext\fts1\fts1_hash.h \ 1304 1305 $(TOP)\ext\fts1\fts1_tokenizer.h \ ................................................................................ 1413 1414 $(TOP)\ext\misc\nextchar.c \ 1414 1415 $(TOP)\ext\misc\percentile.c \ 1415 1416 $(TOP)\ext\misc\regexp.c \ 1416 1417 $(TOP)\ext\misc\remember.c \ 1417 1418 $(TOP)\ext\misc\series.c \ 1418 1419 $(TOP)\ext\misc\spellfix.c \ 1419 1420 $(TOP)\ext\misc\totype.c \ 1421 + $(TOP)\ext\misc\unionvtab.c \ 1420 1422 $(TOP)\ext\misc\wholenumber.c 1421 1423 1422 1424 # Source code to the library files needed by the test fixture 1423 1425 # (non-amalgamation) 1424 1426 # 1425 1427 TESTSRC2 = \ 1426 1428 $(SRC00) \ ................................................................................ 1501 1503 $(TOP)\test\fuzzdata5.db 1502 1504 # <</mark>> 1503 1505 1504 1506 # Additional compiler options for the shell. These are only effective 1505 1507 # when the shell is not being dynamically linked. 1506 1508 # 1507 1509 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0 1508 -SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS 1510 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB 1509 1511 !ENDIF 1510 1512 1511 1513 # <<mark>> 1512 1514 # Extra compiler options for various test tools. 1513 1515 # 1514 1516 MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5 1515 1517 FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 ................................................................................ 2085 2087 # hidden when the library is built via the amalgamation). 2086 2088 # 2087 2089 TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 2088 2090 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" 2089 2091 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN) 2090 2092 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 2091 2093 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024 2094 +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB 2092 2095 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS) 2093 2096 2094 2097 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) 2095 2098 TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C) 2096 2099 !IF $(USE_AMALGAMATION)==0 2097 2100 TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0) 2098 2101 !ELSE ................................................................................ 2238 2241 dbselftest.exe: $(TOP)\test\dbselftest.c $(SQLITE3C) $(SQLITE3H) 2239 2242 $(LTLINK) $(NO_WARN) $(DBSELFTEST_COMPILE_OPTS) $(TOP)\test\dbselftest.c $(SQLITE3C) 2240 2243 2241 2244 rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H) 2242 2245 $(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU \ 2243 2246 $(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 2244 2247 2248 +LSMDIR=$(TOP)\ext\lsm1 2249 +!INCLUDE $(LSMDIR)\Makefile.msc 2250 + 2245 2251 moreclean: clean 2246 2252 del /Q $(SQLITE3C) $(SQLITE3H) 2>NUL 2247 2253 # <</mark>> 2248 2254 2249 2255 clean: 2250 2256 del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL 2251 2257 del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL ................................................................................ 2257 2263 del /Q mkkeywordhash.* keywordhash.h 2>NUL 2258 2264 del /Q notasharedlib.* 2>NUL 2259 2265 -rmdir /Q/S .deps 2>NUL 2260 2266 -rmdir /Q/S .libs 2>NUL 2261 2267 -rmdir /Q/S tsrc 2>NUL 2262 2268 del /Q .target_source 2>NUL 2263 2269 del /Q tclsqlite3.exe $(SQLITETCLH) $(SQLITETCLDECLSH) 2>NUL 2270 + del /Q lsm.dll lsmtest.exe 2>NUL 2264 2271 del /Q testloadext.dll 2>NUL 2265 2272 del /Q testfixture.exe test.db 2>NUL 2266 2273 del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe dbdump.exe 2>NUL 2267 2274 del /Q changeset.exe 2>NUL 2268 2275 del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL 2269 2276 del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL 2270 2277 del /Q sqlite3.c sqlite3-*.c 2>NUL
Changes to README.md.
30 30 to locate the check-in desired, click on its information page link, 31 31 then click on the "Tarball" or "ZIP Archive" links on the information 32 32 page. 33 33 34 34 If you do want to use Fossil to check out the source tree, 35 35 first install Fossil version 2.0 or later. 36 36 (Source tarballs and precompiled binaries available 37 -[here](https://www.fossil-scm.org/fossil/uv/download.html).) 37 +[here](https://www.fossil-scm.org/fossil/uv/download.html). Fossil is 38 +a stand-alone program. To install, simply download or build the single 39 +executable file and put that file someplace on your $PATH.) 38 40 Then run commands like this: 39 41 40 42 mkdir ~/sqlite 41 43 cd ~/sqlite 42 44 fossil clone https://www.sqlite.org/src sqlite.fossil 43 45 fossil open sqlite.fossil 44 46 ................................................................................ 102 104 SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation 103 105 is required by the makefiles (including those for MSVC). SQLite contains 104 106 a lot of generated code and Tcl is used to do much of that code generation. 105 107 The makefiles also require AWK. 106 108 107 109 ## Source Code Tour 108 110 109 -Most of the core source files are in the **src/** subdirectory. But 110 -src/ also contains files used to build the "testfixture" test harness; 111 -those file all begin with "test". And src/ contains the "shell.c" file 112 -which is the main program for the "sqlite3.exe" command-line shell and 113 -the "tclsqlite.c" file which implements the bindings to SQLite from the 114 -Tcl programming language. (Historical note: SQLite began as a Tcl 111 +Most of the core source files are in the **src/** subdirectory. The 112 +**src/** folder also contains files used to build the "testfixture" test 113 +harness. The names of the source files used by "testfixture" all begin 114 +with "test". 115 +The **src/** also contains the "shell.c" file 116 +which is the main program for the "sqlite3.exe" 117 +[command-line shell](https://sqlite.org/cli.html) and 118 +the "tclsqlite.c" file which implements the 119 +[TCL bindings](https://sqlite.org/tclsqlite.html) for SQLite. 120 +(Historical note: SQLite began as a Tcl 115 121 extension and only later escaped to the wild as an independent library.) 116 122 117 123 Test scripts and programs are found in the **test/** subdirectory. 118 -There are other test suites for SQLite (see 119 -[How SQLite Is Tested](http://www.sqlite.org/testing.html)) 120 -but those other test suites are 121 -in separate source repositories. 124 +Addtional test code is found in other source repositories. 125 +See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for 126 +additional information. 122 127 123 128 The **ext/** subdirectory contains code for extensions. The 124 129 Full-text search engine is in **ext/fts3**. The R-Tree engine is in 125 130 **ext/rtree**. The **ext/misc** subdirectory contains a number of 126 131 smaller, single-file extensions, such as a REGEXP operator. 127 132 128 133 The **tool/** subdirectory contains various scripts and programs used ................................................................................ 138 143 The "target_source" make target will create a subdirectory "tsrc/" and 139 144 fill it with all the source files needed to build SQLite, both 140 145 manually-edited files and automatically-generated files. 141 146 142 147 The SQLite interface is defined by the **sqlite3.h** header file, which is 143 148 generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION. The 144 149 [Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion. 145 -The manifest.uuid file contains the SHA1 hash of the particular check-in 150 +The manifest.uuid file contains the SHA3 hash of the particular check-in 146 151 and is used to generate the SQLITE\_SOURCE\_ID macro. The VERSION file 147 152 contains the current SQLite version number. The sqlite3.h header is really 148 153 just a copy of src/sqlite.h.in with the source-id and version number inserted 149 154 at just the right spots. Note that comment text in the sqlite3.h file is 150 155 used to generate much of the SQLite API documentation. The Tcl scripts 151 156 used to generate that documentation are in a separate source repository. 152 157 153 158 The SQL language parser is **parse.c** which is generate from a grammar in 154 159 the src/parse.y file. The conversion of "parse.y" into "parse.c" is done 155 160 by the [lemon](./doc/lemon.html) LALR(1) parser generator. The source code 156 -for lemon is at tool/lemon.c. Lemon uses a 157 -template for generating its parser. A generic template is in tool/lempar.c, 158 -but SQLite uses a slightly modified template found in src/lempar.c. 161 +for lemon is at tool/lemon.c. Lemon uses the tool/lempar.c file as a 162 +template for generating its parser. 159 163 160 164 Lemon also generates the **parse.h** header file, at the same time it 161 165 generates parse.c. But the parse.h header file is 162 166 modified further (to add additional symbols) using the ./addopcodes.awk 163 167 AWK script. 164 168 165 169 The **opcodes.h** header file contains macros that define the numbers ................................................................................ 171 175 opcode-number to opcode-name that is used for EXPLAIN output. 172 176 173 177 The **keywordhash.h** header file contains the definition of a hash table 174 178 that maps SQL language keywords (ex: "CREATE", "SELECT", "INDEX", etc.) into 175 179 the numeric codes used by the parse.c parser. The keywordhash.h file is 176 180 generated by a C-language program at tool mkkeywordhash.c. 177 181 182 +The **pragma.h** header file contains various definitions used to parse 183 +and implement the PRAGMA statements. The header is generated by a 184 +script **tool/mkpragmatab.tcl**. If you want to add a new PRAGMA, edit 185 +the **tool/mkpragmatab.tcl** file to insert the information needed by the 186 +parser for your new PRAGMA, then run the script to regenerate the 187 +**pragma.h** header file. 188 + 178 189 ### The Amalgamation 179 190 180 191 All of the individual C source code and header files (both manually-edited 181 192 and automatically-generated) can be combined into a single big source file 182 193 **sqlite3.c** called "the amalgamation". The amalgamation is the recommended 183 194 way of using SQLite in a larger application. Combining all individual 184 195 source code files into a single big source code file allows the C compiler ................................................................................ 188 199 189 200 The amalgamation is generated from the tool/mksqlite3c.tcl Tcl script. 190 201 First, all of the individual source files must be gathered into the tsrc/ 191 202 subdirectory (using the equivalent of "make target_source") then the 192 203 tool/mksqlite3c.tcl script is run to copy them all together in just the 193 204 right order while resolving internal "#include" references. 194 205 195 -The amalgamation source file is more than 100K lines long. Some symbolic 206 +The amalgamation source file is more than 200K lines long. Some symbolic 196 207 debuggers (most notably MSVC) are unable to deal with files longer than 64K 197 208 lines. To work around this, a separate Tcl script, tool/split-sqlite3c.tcl, 198 209 can be run on the amalgamation to break it up into a single small C file 199 210 called **sqlite3-all.c** that does #include on about five other files 200 211 named **sqlite3-1.c**, **sqlite3-2.c**, ..., **sqlite3-5.c**. In this way, 201 212 all of the source code is contained within a single translation unit so 202 213 that the compiler can do extra cross-procedure optimization, but no ................................................................................ 205 216 ## How It All Fits Together 206 217 207 218 SQLite is modular in design. 208 219 See the [architectural description](http://www.sqlite.org/arch.html) 209 220 for details. Other documents that are useful in 210 221 (helping to understand how SQLite works include the 211 222 [file format](http://www.sqlite.org/fileformat2.html) description, 212 -the [virtual machine](http://www.sqlite.org/vdbe.html) that runs 223 +the [virtual machine](http://www.sqlite.org/opcode.html) that runs 213 224 prepared statements, the description of 214 225 [how transactions work](http://www.sqlite.org/atomiccommit.html), and 215 226 the [overview of the query planner](http://www.sqlite.org/optoverview.html). 216 227 217 -Unfortunately, years of effort have gone into optimizating SQLite, both 228 +Years of effort have gone into optimizating SQLite, both 218 229 for small size and high performance. And optimizations tend to result in 219 -complex code. So there is a lot of complexity in the SQLite implementation. 230 +complex code. So there is a lot of complexity in the current SQLite 231 +implementation. It will not be the easiest library in the world to hack. 220 232 221 233 Key files: 222 234 223 235 * **sqlite.h.in** - This file defines the public interface to the SQLite 224 236 library. Readers will need to be familiar with this interface before 225 237 trying to understand how the library works internally. 226 238 227 239 * **sqliteInt.h** - this header file defines many of the data objects 228 240 used internally by SQLite. 229 241 230 - * **parse.y** - This file describes the LALR(1) grammer that SQLite uses 242 + * **parse.y** - This file describes the LALR(1) grammar that SQLite uses 231 243 to parse SQL statements, and the actions that are taken at each step 232 244 in the parsing process. 233 245 234 246 * **vdbe.c** - This file implements the virtual machine that runs 235 247 prepared statements. There are various helper files whose names 236 248 begin with "vdbe". The VDBE has access to the vdbeInt.h header file 237 249 which defines internal data objects. The rest of SQLite interacts ................................................................................ 256 268 is the file that, when linked against sqlite3.a, generates the 257 269 "sqlite3.exe" command-line shell. 258 270 259 271 * **tclsqlite.c** - This file implements the Tcl bindings for SQLite. It 260 272 is not part of the core SQLite library. But as most of the tests in this 261 273 repository are written in Tcl, the Tcl language bindings are important. 262 274 263 -There are many other source files. Each has a suscinct header comment that 275 +There are many other source files. Each has a succinct header comment that 264 276 describes its purpose and role within the larger system. 265 277 266 278 267 279 ## Contacts 268 280 269 281 The main SQLite webpage is [http://www.sqlite.org/](http://www.sqlite.org/) 270 -with geographically distributed backup servers at 282 +with geographically distributed backups at 271 283 [http://www2.sqlite.org/](http://www2.sqlite.org) and 272 284 [http://www3.sqlite.org/](http://www3.sqlite.org).
Changes to VERSION.
1 -3.19.3 1 +3.20.0
Changes to autoconf/Makefile.am.
8 8 bin_PROGRAMS = sqlite3 9 9 sqlite3_SOURCES = shell.c sqlite3.h 10 10 EXTRA_sqlite3_SOURCES = sqlite3.c 11 11 sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@ 12 12 sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@ 13 13 sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS 14 14 15 -include_HEADERS = sqlite3.h sqlite3ext.h msvc.h 15 +include_HEADERS = sqlite3.h sqlite3ext.h 16 16 17 17 EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs 18 18 pkgconfigdir = ${libdir}/pkgconfig 19 19 pkgconfig_DATA = sqlite3.pc 20 20 21 21 man_MANS = sqlite3.1
Changes to configure.
1 1 #! /bin/sh 2 2 # Guess values for system-dependent variables and create Makefiles. 3 -# Generated by GNU Autoconf 2.69 for sqlite 3.19.3. 3 +# Generated by GNU Autoconf 2.69 for sqlite 3.20.0. 4 4 # 5 5 # 6 6 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. 7 7 # 8 8 # 9 9 # This configure script is free software; the Free Software Foundation 10 10 # gives unlimited permission to copy, distribute and modify it. ................................................................................ 722 722 subdirs= 723 723 MFLAGS= 724 724 MAKEFLAGS= 725 725 726 726 # Identity of this package. 727 727 PACKAGE_NAME='sqlite' 728 728 PACKAGE_TARNAME='sqlite' 729 -PACKAGE_VERSION='3.19.3' 730 -PACKAGE_STRING='sqlite 3.19.3' 729 +PACKAGE_VERSION='3.20.0' 730 +PACKAGE_STRING='sqlite 3.20.0' 731 731 PACKAGE_BUGREPORT='' 732 732 PACKAGE_URL='' 733 733 734 734 # Factoring default headers for most tests. 735 735 ac_includes_default="\ 736 736 #include <stdio.h> 737 737 #ifdef HAVE_SYS_TYPES_H ................................................................................ 1459 1459 # 1460 1460 # Report the --help message. 1461 1461 # 1462 1462 if test "$ac_init_help" = "long"; then 1463 1463 # Omit some internal or obsolete options to make the list less imposing. 1464 1464 # This message is too long to be a string in the A/UX 3.1 sh. 1465 1465 cat <<_ACEOF 1466 -\`configure' configures sqlite 3.19.3 to adapt to many kinds of systems. 1466 +\`configure' configures sqlite 3.20.0 to adapt to many kinds of systems. 1467 1467 1468 1468 Usage: $0 [OPTION]... [VAR=VALUE]... 1469 1469 1470 1470 To assign environment variables (e.g., CC, CFLAGS...), specify them as 1471 1471 VAR=VALUE. See below for descriptions of some of the useful variables. 1472 1472 1473 1473 Defaults for the options are specified in brackets. ................................................................................ 1524 1524 --build=BUILD configure for building on BUILD [guessed] 1525 1525 --host=HOST cross-compile to build programs to run on HOST [BUILD] 1526 1526 _ACEOF 1527 1527 fi 1528 1528 1529 1529 if test -n "$ac_init_help"; then 1530 1530 case $ac_init_help in 1531 - short | recursive ) echo "Configuration of sqlite 3.19.3:";; 1531 + short | recursive ) echo "Configuration of sqlite 3.20.0:";; 1532 1532 esac 1533 1533 cat <<\_ACEOF 1534 1534 1535 1535 Optional Features: 1536 1536 --disable-option-checking ignore unrecognized --enable/--with options 1537 1537 --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) 1538 1538 --enable-FEATURE[=ARG] include FEATURE [ARG=yes] ................................................................................ 1648 1648 cd "$ac_pwd" || { ac_status=$?; break; } 1649 1649 done 1650 1650 fi 1651 1651 1652 1652 test -n "$ac_init_help" && exit $ac_status 1653 1653 if $ac_init_version; then 1654 1654 cat <<\_ACEOF 1655 -sqlite configure 3.19.3 1655 +sqlite configure 3.20.0 1656 1656 generated by GNU Autoconf 2.69 1657 1657 1658 1658 Copyright (C) 2012 Free Software Foundation, Inc. 1659 1659 This configure script is free software; the Free Software Foundation 1660 1660 gives unlimited permission to copy, distribute and modify it. 1661 1661 _ACEOF 1662 1662 exit ................................................................................ 2067 2067 eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno 2068 2068 2069 2069 } # ac_fn_c_check_header_mongrel 2070 2070 cat >config.log <<_ACEOF 2071 2071 This file contains any messages produced by compilers while 2072 2072 running configure, to aid debugging if configure makes a mistake. 2073 2073 2074 -It was created by sqlite $as_me 3.19.3, which was 2074 +It was created by sqlite $as_me 3.20.0, which was 2075 2075 generated by GNU Autoconf 2.69. Invocation command line was 2076 2076 2077 2077 $ $0 $@ 2078 2078 2079 2079 _ACEOF 2080 2080 exec 5>>config.log 2081 2081 { ................................................................................ 12147 12147 test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1 12148 12148 12149 12149 cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 12150 12150 # Save the log message, to keep $0 and so on meaningful, and to 12151 12151 # report actual input values of CONFIG_FILES etc. instead of their 12152 12152 # values after options handling. 12153 12153 ac_log=" 12154 -This file was extended by sqlite $as_me 3.19.3, which was 12154 +This file was extended by sqlite $as_me 3.20.0, which was 12155 12155 generated by GNU Autoconf 2.69. Invocation command line was 12156 12156 12157 12157 CONFIG_FILES = $CONFIG_FILES 12158 12158 CONFIG_HEADERS = $CONFIG_HEADERS 12159 12159 CONFIG_LINKS = $CONFIG_LINKS 12160 12160 CONFIG_COMMANDS = $CONFIG_COMMANDS 12161 12161 $ $0 $@ ................................................................................ 12213 12213 12214 12214 Report bugs to the package provider." 12215 12215 12216 12216 _ACEOF 12217 12217 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 12218 12218 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" 12219 12219 ac_cs_version="\\ 12220 -sqlite config.status 3.19.3 12220 +sqlite config.status 3.20.0 12221 12221 configured by $0, generated by GNU Autoconf 2.69, 12222 12222 with options \\"\$ac_cs_config\\" 12223 12223 12224 12224 Copyright (C) 2012 Free Software Foundation, Inc. 12225 12225 This config.status script is free software; the Free Software Foundation 12226 12226 gives unlimited permission to copy, distribute and modify it." 12227 12227
Changes to doc/lemon.html.
18 18 Lemon also implements features that can be used 19 19 to eliminate resource leaks, making is suitable for use 20 20 in long-running programs such as graphical user interfaces 21 21 or embedded controllers.</p> 22 22 23 23 <p>This document is an introduction to the Lemon 24 24 parser generator.</p> 25 + 26 +<h2>Security Note</h2> 27 + 28 +<p>The language parser code created by Lemon is very robust and 29 +is well-suited for use in internet-facing applications that need to 30 +safely process maliciously crafted inputs. 31 + 32 +<p>The "lemon.exe" command-line tool itself works great when given a valid 33 +input grammar file and almost always gives helpful 34 +error messages for malformed inputs. However, it is possible for 35 +a malicious user to craft a grammar file that will cause 36 +lemon.exe to crash. 37 +We do not see this as a problem, as lemon.exe is not intended to be used 38 +with hostile inputs. 39 +To summarize:</p> 40 + 41 +<ul> 42 +<li>Parser code generated by lemon → Robust and secure 43 +<li>The "lemon.exe" command line tool itself → Not so much 44 +</ul> 25 45 26 46 <h2>Theory of Operation</h2> 27 47 28 48 <p>The main goal of Lemon is to translate a context free grammar (CFG) 29 49 for a particular language into C code that implements a parser for 30 50 that language. 31 51 The program has two inputs:
Changes to ext/fts3/fts3.c.
1702 1702 sqlite3_reset(pCsr->pStmt); 1703 1703 pCsr->pStmt = 0; 1704 1704 } 1705 1705 pCsr->bSeekStmt = 0; 1706 1706 } 1707 1707 sqlite3_finalize(pCsr->pStmt); 1708 1708 } 1709 + 1710 +/* 1711 +** Free all resources currently held by the cursor passed as the only 1712 +** argument. 1713 +*/ 1714 +static void fts3ClearCursor(Fts3Cursor *pCsr){ 1715 + fts3CursorFinalizeStmt(pCsr); 1716 + sqlite3Fts3FreeDeferredTokens(pCsr); 1717 + sqlite3_free(pCsr->aDoclist); 1718 + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); 1719 + sqlite3Fts3ExprFree(pCsr->pExpr); 1720 + memset(&(&pCsr->base)[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); 1721 +} 1709 1722 1710 1723 /* 1711 1724 ** Close the cursor. For additional information see the documentation 1712 1725 ** on the xClose method of the virtual table interface. 1713 1726 */ 1714 1727 static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ 1715 1728 Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; 1716 1729 assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); 1717 - fts3CursorFinalizeStmt(pCsr); 1718 - sqlite3Fts3ExprFree(pCsr->pExpr); 1719 - sqlite3Fts3FreeDeferredTokens(pCsr); 1720 - sqlite3_free(pCsr->aDoclist); 1721 - sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); 1730 + fts3ClearCursor(pCsr); 1722 1731 assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); 1723 1732 sqlite3_free(pCsr); 1724 1733 return SQLITE_OK; 1725 1734 } 1726 1735 1727 1736 /* 1728 1737 ** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then ................................................................................ 1740 1749 char *zSql; 1741 1750 if( p->pSeekStmt ){ 1742 1751 pCsr->pStmt = p->pSeekStmt; 1743 1752 p->pSeekStmt = 0; 1744 1753 }else{ 1745 1754 zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); 1746 1755 if( !zSql ) return SQLITE_NOMEM; 1747 - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); 1756 + rc = sqlite3_prepare_v3(p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0); 1748 1757 sqlite3_free(zSql); 1749 1758 } 1750 1759 if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1; 1751 1760 } 1752 1761 return rc; 1753 1762 } 1754 1763 ................................................................................ 3215 3224 if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++]; 3216 3225 if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++]; 3217 3226 if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++]; 3218 3227 if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++]; 3219 3228 assert( iIdx==nVal ); 3220 3229 3221 3230 /* In case the cursor has been used before, clear it now. */ 3222 - fts3CursorFinalizeStmt(pCsr); 3223 - sqlite3_free(pCsr->aDoclist); 3224 - sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); 3225 - sqlite3Fts3ExprFree(pCsr->pExpr); 3226 - memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); 3231 + fts3ClearCursor(pCsr); 3227 3232 3228 3233 /* Set the lower and upper bounds on docids to return */ 3229 3234 pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); 3230 3235 pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64); 3231 3236 3232 3237 if( idxStr ){ 3233 3238 pCsr->bDesc = (idxStr[0]=='D'); ................................................................................ 3277 3282 ); 3278 3283 }else{ 3279 3284 zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s", 3280 3285 p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") 3281 3286 ); 3282 3287 } 3283 3288 if( zSql ){ 3284 - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); 3289 + rc = sqlite3_prepare_v3(p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0); 3285 3290 sqlite3_free(zSql); 3286 3291 }else{ 3287 3292 rc = SQLITE_NOMEM; 3288 3293 } 3289 3294 }else if( eSearch==FTS3_DOCID_SEARCH ){ 3290 3295 rc = fts3CursorSeekStmt(pCsr); 3291 3296 if( rc==SQLITE_OK ){ ................................................................................ 3298 3303 } 3299 3304 3300 3305 /* 3301 3306 ** This is the xEof method of the virtual table. SQLite calls this 3302 3307 ** routine to find out if it has reached the end of a result set. 3303 3308 */ 3304 3309 static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ 3305 - return ((Fts3Cursor *)pCursor)->isEof; 3310 + Fts3Cursor *pCsr = (Fts3Cursor*)pCursor; 3311 + if( pCsr->isEof ){ 3312 + fts3ClearCursor(pCsr); 3313 + pCsr->isEof = 1; 3314 + } 3315 + return pCsr->isEof; 3306 3316 } 3307 3317 3308 3318 /* 3309 3319 ** This is the xRowid method. The SQLite core calls this routine to 3310 3320 ** retrieve the rowid for the current row of the result set. fts3 3311 3321 ** exposes %_content.docid as the rowid for the virtual table. The 3312 3322 ** rowid should be written to *pRowid. ................................................................................ 3339 3349 3340 3350 /* The column value supplied by SQLite must be in range. */ 3341 3351 assert( iCol>=0 && iCol<=p->nColumn+2 ); 3342 3352 3343 3353 switch( iCol-p->nColumn ){ 3344 3354 case 0: 3345 3355 /* The special 'table-name' column */ 3346 - sqlite3_result_blob(pCtx, &pCsr, sizeof(Fts3Cursor*), SQLITE_TRANSIENT); 3347 - sqlite3_result_subtype(pCtx, SQLITE_BLOB); 3356 + sqlite3_result_pointer(pCtx, pCsr, "fts3cursor"); 3348 3357 break; 3349 3358 3350 3359 case 1: 3351 3360 /* The docid column */ 3352 3361 sqlite3_result_int64(pCtx, pCsr->iPrevId); 3353 3362 break; 3354 3363 ................................................................................ 3558 3567 */ 3559 3568 static int fts3FunctionArg( 3560 3569 sqlite3_context *pContext, /* SQL function call context */ 3561 3570 const char *zFunc, /* Function name */ 3562 3571 sqlite3_value *pVal, /* argv[0] passed to function */ 3563 3572 Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ 3564 3573 ){ 3565 - int rc = SQLITE_OK; 3566 - if( sqlite3_value_subtype(pVal)==SQLITE_BLOB ){ 3567 - *ppCsr = *(Fts3Cursor**)sqlite3_value_blob(pVal); 3574 + int rc; 3575 + *ppCsr = (Fts3Cursor*)sqlite3_value_pointer(pVal, "fts3cursor"); 3576 + if( (*ppCsr)!=0 ){ 3577 + rc = SQLITE_OK; 3568 3578 }else{ 3569 3579 char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); 3570 3580 sqlite3_result_error(pContext, zErr, -1); 3571 3581 sqlite3_free(zErr); 3572 3582 rc = SQLITE_ERROR; 3573 3583 } 3574 3584 return rc;
Changes to ext/fts3/fts3_write.c.
403 403 zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); 404 404 }else{ 405 405 zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); 406 406 } 407 407 if( !zSql ){ 408 408 rc = SQLITE_NOMEM; 409 409 }else{ 410 - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL); 410 + rc = sqlite3_prepare_v3(p->db, zSql, -1, SQLITE_PREPARE_PERSISTENT, 411 + &pStmt, NULL); 411 412 sqlite3_free(zSql); 412 413 assert( rc==SQLITE_OK || pStmt==0 ); 413 414 p->aStmt[eStmt] = pStmt; 414 415 } 415 416 } 416 417 if( apVal ){ 417 418 int i;
Changes to ext/fts5/fts5_hash.c.
167 167 168 168 apNew = (Fts5HashEntry**)sqlite3_malloc(nNew*sizeof(Fts5HashEntry*)); 169 169 if( !apNew ) return SQLITE_NOMEM; 170 170 memset(apNew, 0, nNew*sizeof(Fts5HashEntry*)); 171 171 172 172 for(i=0; i<pHash->nSlot; i++){ 173 173 while( apOld[i] ){ 174 - int iHash; 174 + unsigned int iHash; 175 175 Fts5HashEntry *p = apOld[i]; 176 176 apOld[i] = p->pHashNext; 177 - iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), strlen(fts5EntryKey(p))); 177 + iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), 178 + (int)strlen(fts5EntryKey(p))); 178 179 p->pHashNext = apNew[iHash]; 179 180 apNew[iHash] = p; 180 181 } 181 182 } 182 183 183 184 sqlite3_free(apOld); 184 185 pHash->nSlot = nNew; ................................................................................ 473 474 int sqlite3Fts5HashQuery( 474 475 Fts5Hash *pHash, /* Hash table to query */ 475 476 const char *pTerm, int nTerm, /* Query term */ 476 477 const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */ 477 478 int *pnDoclist /* OUT: Size of doclist in bytes */ 478 479 ){ 479 480 unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm); 480 - char *zKey; 481 + char *zKey = 0; 481 482 Fts5HashEntry *p; 482 483 483 484 for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ 484 485 zKey = fts5EntryKey(p); 485 486 if( memcmp(zKey, pTerm, nTerm)==0 && zKey[nTerm]==0 ) break; 486 487 } 487 488
Changes to ext/fts5/fts5_index.c.
724 724 static int fts5IndexPrepareStmt( 725 725 Fts5Index *p, 726 726 sqlite3_stmt **ppStmt, 727 727 char *zSql 728 728 ){ 729 729 if( p->rc==SQLITE_OK ){ 730 730 if( zSql ){ 731 - p->rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, ppStmt, 0); 731 + p->rc = sqlite3_prepare_v3(p->pConfig->db, zSql, -1, 732 + SQLITE_PREPARE_PERSISTENT, ppStmt, 0); 732 733 }else{ 733 734 p->rc = SQLITE_NOMEM; 734 735 } 735 736 } 736 737 sqlite3_free(zSql); 737 738 return p->rc; 738 739 } ................................................................................ 773 774 char *zSql = sqlite3_mprintf( 774 775 "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?", 775 776 pConfig->zDb, pConfig->zName 776 777 ); 777 778 if( zSql==0 ){ 778 779 rc = SQLITE_NOMEM; 779 780 }else{ 780 - rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->pDeleter, 0); 781 + rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, 782 + SQLITE_PREPARE_PERSISTENT, &p->pDeleter, 0); 781 783 sqlite3_free(zSql); 782 784 } 783 785 if( rc!=SQLITE_OK ){ 784 786 p->rc = rc; 785 787 return; 786 788 } 787 789 }
Changes to ext/fts5/fts5_main.c.
879 879 va_list ap; 880 880 881 881 va_start(ap, zFmt); 882 882 zSql = sqlite3_vmprintf(zFmt, ap); 883 883 if( zSql==0 ){ 884 884 rc = SQLITE_NOMEM; 885 885 }else{ 886 - rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0); 886 + rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, 887 + SQLITE_PREPARE_PERSISTENT, &pRet, 0); 887 888 if( rc!=SQLITE_OK ){ 888 889 *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db)); 889 890 } 890 891 sqlite3_free(zSql); 891 892 } 892 893 893 894 va_end(ap); ................................................................................ 1015 1016 const char *zRank = pCsr->zRank; 1016 1017 const char *zRankArgs = pCsr->zRankArgs; 1017 1018 1018 1019 if( zRankArgs ){ 1019 1020 char *zSql = sqlite3Fts5Mprintf(&rc, "SELECT %s", zRankArgs); 1020 1021 if( zSql ){ 1021 1022 sqlite3_stmt *pStmt = 0; 1022 - rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pStmt, 0); 1023 + rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, 1024 + SQLITE_PREPARE_PERSISTENT, &pStmt, 0); 1023 1025 sqlite3_free(zSql); 1024 1026 assert( rc==SQLITE_OK || pCsr->pRankArgStmt==0 ); 1025 1027 if( rc==SQLITE_OK ){ 1026 1028 if( SQLITE_ROW==sqlite3_step(pStmt) ){ 1027 1029 int nByte; 1028 1030 pCsr->nRankArg = sqlite3_column_count(pStmt); 1029 1031 nByte = sizeof(sqlite3_value*)*pCsr->nRankArg; ................................................................................ 2603 2605 2604 2606 sqlite3_free(pGlobal); 2605 2607 } 2606 2608 2607 2609 static void fts5Fts5Func( 2608 2610 sqlite3_context *pCtx, /* Function call context */ 2609 2611 int nArg, /* Number of args */ 2610 - sqlite3_value **apUnused /* Function arguments */ 2612 + sqlite3_value **apArg /* Function arguments */ 2611 2613 ){ 2612 2614 Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx); 2613 - char buf[8]; 2614 - UNUSED_PARAM2(nArg, apUnused); 2615 - assert( nArg==0 ); 2616 - assert( sizeof(buf)>=sizeof(pGlobal) ); 2617 - memcpy(buf, (void*)&pGlobal, sizeof(pGlobal)); 2618 - sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT); 2615 + fts5_api **ppApi; 2616 + UNUSED_PARAM(nArg); 2617 + assert( nArg==1 ); 2618 + ppApi = (fts5_api**)sqlite3_value_pointer(apArg[0], "fts5_api_ptr"); 2619 + if( ppApi ) *ppApi = &pGlobal->api; 2619 2620 } 2620 2621 2621 2622 /* 2622 2623 ** Implementation of fts5_source_id() function. 2623 2624 */ 2624 2625 static void fts5SourceIdFunc( 2625 2626 sqlite3_context *pCtx, /* Function call context */ ................................................................................ 2676 2677 if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db); 2677 2678 if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db); 2678 2679 if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api); 2679 2680 if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api); 2680 2681 if( rc==SQLITE_OK ) rc = sqlite3Fts5VocabInit(pGlobal, db); 2681 2682 if( rc==SQLITE_OK ){ 2682 2683 rc = sqlite3_create_function( 2683 - db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0 2684 + db, "fts5", 1, SQLITE_UTF8, p, fts5Fts5Func, 0, 0 2684 2685 ); 2685 2686 } 2686 2687 if( rc==SQLITE_OK ){ 2687 2688 rc = sqlite3_create_function( 2688 2689 db, "fts5_source_id", 0, SQLITE_UTF8, p, fts5SourceIdFunc, 0, 0 2689 2690 ); 2690 2691 }
Changes to ext/fts5/fts5_storage.c.
132 132 zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName); 133 133 break; 134 134 } 135 135 136 136 if( zSql==0 ){ 137 137 rc = SQLITE_NOMEM; 138 138 }else{ 139 - rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0); 139 + rc = sqlite3_prepare_v3(pC->db, zSql, -1, 140 + SQLITE_PREPARE_PERSISTENT, &p->aStmt[eStmt], 0); 140 141 sqlite3_free(zSql); 141 142 if( rc!=SQLITE_OK && pzErrMsg ){ 142 143 *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db)); 143 144 } 144 145 } 145 146 } 146 147
Changes to ext/fts5/fts5_tcl.c.
95 95 int rc = f5tDbPointer(interp, pObj, &db); 96 96 if( rc!=TCL_OK ){ 97 97 return TCL_ERROR; 98 98 }else{ 99 99 sqlite3_stmt *pStmt = 0; 100 100 fts5_api *pApi = 0; 101 101 102 - rc = sqlite3_prepare_v2(db, "SELECT fts5()", -1, &pStmt, 0); 102 + rc = sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0); 103 103 if( rc!=SQLITE_OK ){ 104 104 Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0); 105 105 return TCL_ERROR; 106 106 } 107 - 108 - if( SQLITE_ROW==sqlite3_step(pStmt) ){ 109 - const void *pPtr = sqlite3_column_blob(pStmt, 0); 110 - memcpy((void*)&pApi, pPtr, sizeof(pApi)); 111 - } 107 + sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr"); 108 + sqlite3_step(pStmt); 112 109 113 110 if( sqlite3_finalize(pStmt)!=SQLITE_OK ){ 114 111 Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0); 115 112 return TCL_ERROR; 116 113 } 117 114 118 115 *ppDb = db;
Changes to ext/fts5/fts5_test_mi.c.
69 69 ** handle (accessible using sqlite3_errcode()/errmsg()). 70 70 */ 71 71 static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){ 72 72 sqlite3_stmt *pStmt = 0; 73 73 int rc; 74 74 75 75 *ppApi = 0; 76 - rc = sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0); 76 + rc = sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0); 77 77 if( rc==SQLITE_OK ){ 78 - if( SQLITE_ROW==sqlite3_step(pStmt) 79 - && sizeof(fts5_api*)==sqlite3_column_bytes(pStmt, 0) 80 - ){ 81 - memcpy(ppApi, sqlite3_column_blob(pStmt, 0), sizeof(fts5_api*)); 82 - } 78 + sqlite3_bind_pointer(pStmt, 1, (void*)ppApi, "fts5_api_ptr"); 79 + (void)sqlite3_step(pStmt); 83 80 rc = sqlite3_finalize(pStmt); 84 81 } 85 82 86 83 return rc; 87 84 } 88 85 89 86 ................................................................................ 418 415 /* Register the implementation of matchinfo() */ 419 416 rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0); 420 417 421 418 return rc; 422 419 } 423 420 424 421 #endif /* SQLITE_ENABLE_FTS5 */ 425 -
Changes to ext/fts5/fts5_test_tok.c.
178 178 sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ 179 179 char **pzErr /* OUT: sqlite3_malloc'd error message */ 180 180 ){ 181 181 fts5_api *pApi = (fts5_api*)pCtx; 182 182 Fts5tokTable *pTab = 0; 183 183 int rc; 184 184 char **azDequote = 0; 185 - int nDequote; 185 + int nDequote = 0; 186 186 187 187 rc = sqlite3_declare_vtab(db, 188 188 "CREATE TABLE x(input HIDDEN, token, start, end, position)" 189 189 ); 190 190 191 191 if( rc==SQLITE_OK ){ 192 192 nDequote = argc-3;
Changes to ext/fts5/test/fts5aa.test.
437 437 # exception. But since bm25() can now used the cached structure record, 438 438 # it never sees the corruption introduced by funk() and so the following 439 439 # statement no longer fails. 440 440 # 441 441 do_catchsql_test 16.2 { 442 442 SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d' 443 443 } {0 {{} -1e-06 {}}} 444 -# {1 {SQL logic error or missing database}} 444 +# {1 {SQL logic error}} 445 445 446 446 #------------------------------------------------------------------------- 447 447 # 448 448 reset_db 449 449 do_execsql_test 17.1 { 450 450 CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%); 451 451 INSERT INTO b2 VALUES('a'); ................................................................................ 591 591 SELECT rowid FROM t9('a*') 592 592 } {1} 593 593 594 594 } 595 595 596 596 597 597 finish_test 598 - 599 -
Changes to ext/fts5/test/fts5ab.test.
290 290 INSERT INTO x1 VALUES($doc); 291 291 } 292 292 293 293 } ;# foreach_detail_mode... 294 294 295 295 296 296 finish_test 297 -
Changes to ext/fts5/test/fts5ac.test.
272 272 } { 273 273 do_execsql_test 2.3.$tn { 274 274 SELECT fts5_expr_tcl($expr, 'N $x') 275 275 } [list $tclexpr] 276 276 } 277 277 278 278 finish_test 279 -
Changes to ext/fts5/test/fts5ad.test.
227 227 28 {a f*} 29 {a* f*} 30 {a* fghij*} 228 228 } { 229 229 set res [prefix_query $prefix] 230 230 if {$bAsc} { 231 231 set res [lsort -integer -increasing $res] 232 232 } 233 233 set n [llength $res] 234 - if {$T==5} breakpoint 235 234 do_execsql_test $T.$bAsc.$tn.$n $sql $res 236 235 } 237 236 } 238 237 239 238 catchsql COMMIT 240 239 } 241 240 242 241 } 243 242 244 243 finish_test 245 -
Changes to ext/fts5/test/fts5ae.test.
305 305 SELECT fts5_test_phrasecount(t9) FROM t9 WHERE t9 MATCH $q LIMIT 1 306 306 } $cnt 307 307 } 308 308 309 309 } 310 310 311 311 finish_test 312 -
Changes to ext/fts5/test/fts5af.test.
174 174 do_execsql_test 5.1 { 175 175 SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x'); 176 176 } {{[x] a a a a a...}} 177 177 178 178 } ;# foreach_detail_mode 179 179 180 180 finish_test 181 -
Changes to ext/fts5/test/fts5ag.test.
138 138 } 139 139 } 140 140 141 141 } ;# foreach_detail_mode 142 142 143 143 144 144 finish_test 145 -
Changes to ext/fts5/test/fts5ah.test.
163 163 } {10000} 164 164 165 165 } ;# foreach_detail_mode 166 166 167 167 #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} 168 168 169 169 finish_test 170 -
Changes to ext/fts5/test/fts5ai.test.
51 51 do_execsql_test 1.2 { 52 52 INSERT INTO t1(t1) VALUES('integrity-check'); 53 53 } 54 54 } 55 55 56 56 57 57 finish_test 58 -
Changes to ext/fts5/test/fts5aj.test.
62 62 } 63 63 } 64 64 65 65 do_execsql_test 2.0 { INSERT INTO t1(t1) VALUES('integrity-check') } 66 66 67 67 68 68 finish_test 69 -
Changes to ext/fts5/test/fts5ak.test.
143 143 {[a b c] [c d e]} 144 144 {[a b c d e]} 145 145 } 146 146 147 147 } 148 148 149 149 finish_test 150 -
Changes to ext/fts5/test/fts5al.test.
73 73 1 "" 74 74 2 "fname" 75 75 3 "fname(X'234ab')" 76 76 4 "myfunc(-1.,'abc')" 77 77 } { 78 78 do_test 2.2.$tn { 79 79 catchsql { INSERT INTO ft1(ft1, rank) VALUES('rank', $defn) } 80 - } {1 {SQL logic error or missing database}} 80 + } {1 {SQL logic error}} 81 81 } 82 82 83 83 #------------------------------------------------------------------------- 84 84 # Assorted tests of the tcl interface for creating extension functions. 85 85 # 86 86 87 87 do_execsql_test 3.1 { ................................................................................ 293 293 SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL 294 294 } {1 {parse error in rank function: }} 295 295 296 296 } ;# foreach_detail_mode 297 297 298 298 299 299 finish_test 300 -
Changes to ext/fts5/test/fts5alter.test.
85 85 86 86 do_execsql_test 3.1 { 87 87 CREATE VIRTUAL TABLE abc USING fts5(a); 88 88 INSERT INTO abc(rowid, a) VALUES(1, 'a'); 89 89 BEGIN; 90 90 INSERT INTO abc(rowid, a) VALUES(2, 'a'); 91 91 } 92 -breakpoint 93 92 do_execsql_test 3.2 { 94 93 SELECT rowid FROM abc WHERE abc MATCH 'a'; 95 94 } {1 2} 96 95 97 96 do_execsql_test 3.3 { 98 97 COMMIT; 99 98 SELECT rowid FROM abc WHERE abc MATCH 'a'; 100 99 } {1 2} 101 100 102 101 finish_test 103 -
Changes to ext/fts5/test/fts5auto.test.
338 338 } { 339 339 do_auto_test 4.$tn yy $expr 340 340 } 341 341 342 342 343 343 344 344 finish_test 345 -
Changes to ext/fts5/test/fts5aux.test.
236 236 4 {"a a a" "b" "a d"} {"[a] [a] [a]" "[a] d"} 237 237 1 {"b d" "a b"} {"[b] [d]" "[a] b"} 238 238 2 {"d b" "a d"} {"[d] [b]" "[a] d"} 239 239 3 {"a a d"} {"[a] [a] d"} 240 240 } { 241 241 execsql { DELETE FROM x1 } 242 242 foreach row $lRow { execsql { INSERT INTO x1 VALUES($row) } } 243 - breakpoint 244 243 do_execsql_test 8.$tn { 245 244 SELECT highlight(x1, 0, '[', ']') FROM x1 WHERE x1 MATCH 'a OR (b AND d)'; 246 245 } $res 247 246 } 248 247 249 248 #------------------------------------------------------------------------- 250 249 # Test the built-in bm25() demo. ................................................................................ 275 274 } { 276 275 9 10 277 276 } 278 277 279 278 280 279 281 280 finish_test 282 -
Changes to ext/fts5/test/fts5auxdata.test.
108 108 db eval { 109 109 SELECT aux_function_2(f1, 2, 'A'), aux_function_2(f1, 2, 'B') 110 110 FROM f1 WHERE f1 MATCH 'a' 111 111 ORDER BY rowid ASC 112 112 } 113 113 114 114 finish_test 115 -
Changes to ext/fts5/test/fts5bigpl.test.
57 57 set doc [string repeat "$t " 150000000] 58 58 execsql { INSERT INTO t1 VALUES($doc) } 59 59 } 60 60 execsql { INSERT INTO t1(t1) VALUES('integrity-check') } 61 61 } {} 62 62 63 63 finish_test 64 -
Changes to ext/fts5/test/fts5bigtok.test.
60 60 do_execsql_test 2.[string range $v 0 0] { 61 61 SELECT rowid FROM t1($v) ORDER BY rowid DESC 62 62 } [lsort -integer -decr $res] 63 63 } 64 64 } 65 65 66 66 finish_test 67 - 68 -
Changes to ext/fts5/test/fts5colset.test.
80 80 do_catchsql_test 4.1 { 81 81 SELECT * FROM t1 WHERE rowid MATCH 'a' 82 82 } {1 {unable to use function MATCH in the requested context}} 83 83 } 84 84 85 85 86 86 finish_test 87 - 88 -
Changes to ext/fts5/test/fts5columnsize.test.
139 139 # 140 140 do_execsql_test 4.1.1 { 141 141 CREATE VIRTUAL TABLE t5 USING fts5(x, columnsize=0); 142 142 INSERT INTO t5 VALUES('1 2 3 4'); 143 143 INSERT INTO t5 VALUES('2 4 6 8'); 144 144 } 145 145 146 -breakpoint 147 146 do_execsql_test 4.1.2 { 148 147 INSERT INTO t5(t5) VALUES('integrity-check'); 149 148 } 150 149 151 150 finish_test
Changes to ext/fts5/test/fts5config.test.
62 62 5 "f1(x':;')" 63 63 6 "f1(x'[]')" 64 64 7 "f1(x'{}')" 65 65 8 "f1('abc)" 66 66 } { 67 67 do_catchsql_test 3.$tn { 68 68 INSERT INTO t1(t1, rank) VALUES('rank', $val); 69 - } {1 {SQL logic error or missing database}} 69 + } {1 {SQL logic error}} 70 70 } 71 71 72 72 #------------------------------------------------------------------------- 73 73 # The parsing of SQL literals specified as part of 'rank' options. 74 74 # 75 75 do_execsql_test 4.0 { 76 76 CREATE VIRTUAL TABLE zzz USING fts5(one); ................................................................................ 106 106 #------------------------------------------------------------------------- 107 107 # Misquoting in tokenize= and other options. 108 108 # 109 109 do_catchsql_test 5.1 { 110 110 CREATE VIRTUAL TABLE xx USING fts5(x, tokenize="porter 'ascii"); 111 111 } {1 {parse error in tokenize directive}} 112 112 113 -breakpoint 114 113 do_catchsql_test 5.2 { 115 114 CREATE VIRTUAL TABLE xx USING fts5(x, [y[]); 116 115 } {0 {}} 117 116 118 117 do_catchsql_test 5.3 { 119 118 CREATE VIRTUAL TABLE yy USING fts5(x, [y]]); 120 119 } {1 {unrecognized token: "]"}} ................................................................................ 165 164 # 9.5.* 'hashsize' options. 166 165 # 167 166 do_execsql_test 9.0 { 168 167 CREATE VIRTUAL TABLE abc USING fts5(a, b); 169 168 } {} 170 169 do_catchsql_test 9.1.1 { 171 170 INSERT INTO abc(abc, rank) VALUES('pgsz', -5); 172 -} {1 {SQL logic error or missing database}} 171 +} {1 {SQL logic error}} 173 172 do_catchsql_test 9.1.2 { 174 173 INSERT INTO abc(abc, rank) VALUES('pgsz', 50000000); 175 -} {1 {SQL logic error or missing database}} 174 +} {1 {SQL logic error}} 176 175 do_catchsql_test 9.1.3 { 177 176 INSERT INTO abc(abc, rank) VALUES('pgsz', 66.67); 178 -} {1 {SQL logic error or missing database}} 177 +} {1 {SQL logic error}} 179 178 180 179 do_catchsql_test 9.2.1 { 181 180 INSERT INTO abc(abc, rank) VALUES('automerge', -5); 182 -} {1 {SQL logic error or missing database}} 181 +} {1 {SQL logic error}} 183 182 do_catchsql_test 9.2.2 { 184 183 INSERT INTO abc(abc, rank) VALUES('automerge', 50000000); 185 -} {1 {SQL logic error or missing database}} 184 +} {1 {SQL logic error}} 186 185 do_catchsql_test 9.2.3 { 187 186 INSERT INTO abc(abc, rank) VALUES('automerge', 66.67); 188 -} {1 {SQL logic error or missing database}} 187 +} {1 {SQL logic error}} 189 188 do_execsql_test 9.2.4 { 190 189 INSERT INTO abc(abc, rank) VALUES('automerge', 1); 191 190 } {} 192 191 193 192 do_catchsql_test 9.3.1 { 194 193 INSERT INTO abc(abc, rank) VALUES('crisismerge', -5); 195 -} {1 {SQL logic error or missing database}} 194 +} {1 {SQL logic error}} 196 195 do_catchsql_test 9.3.2 { 197 196 INSERT INTO abc(abc, rank) VALUES('crisismerge', 66.67); 198 -} {1 {SQL logic error or missing database}} 197 +} {1 {SQL logic error}} 199 198 do_execsql_test 9.3.3 { 200 199 INSERT INTO abc(abc, rank) VALUES('crisismerge', 1); 201 200 } {} 202 201 do_execsql_test 9.3.4 { 203 202 INSERT INTO abc(abc, rank) VALUES('crisismerge', 50000000); 204 203 } {} 205 204 206 205 do_catchsql_test 9.4.1 { 207 206 INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1); 208 -} {1 {SQL logic error or missing database}} 207 +} {1 {SQL logic error}} 209 208 210 209 do_catchsql_test 9.5.1 { 211 210 INSERT INTO abc(abc, rank) VALUES('hashsize', 'not an integer'); 212 -} {1 {SQL logic error or missing database}} 211 +} {1 {SQL logic error}} 213 212 do_catchsql_test 9.5.2 { 214 213 INSERT INTO abc(abc, rank) VALUES('hashsize', -500000); 215 -} {1 {SQL logic error or missing database}} 214 +} {1 {SQL logic error}} 216 215 do_catchsql_test 9.5.3 { 217 216 INSERT INTO abc(abc, rank) VALUES('hashsize', 500000); 218 217 } {0 {}} 219 218 220 219 #------------------------------------------------------------------------- 221 220 # Too many prefix indexes. Maximum allowed is 31. 222 221 # ................................................................................ 241 240 } { 242 241 set res [list 1 {malformed detail=... directive}] 243 242 do_catchsql_test 11.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res 244 243 } 245 244 246 245 do_catchsql_test 12.1 { 247 246 INSERT INTO t1(t1, rank) VALUES('rank', NULL);; 248 -} {1 {SQL logic error or missing database}} 247 +} {1 {SQL logic error}} 249 248 250 249 #------------------------------------------------------------------------- 251 250 # errors in the 'usermerge' option 252 251 # 253 252 do_execsql_test 13.0 { 254 253 CREATE VIRTUAL TABLE tt USING fts5(ttt); 255 254 } ................................................................................ 256 255 foreach {tn val} { 257 256 1 -1 258 257 2 4.2 259 258 3 17 260 259 4 1 261 260 } { 262 261 set sql "INSERT INTO tt(tt, rank) VALUES('usermerge', $val)" 263 - do_catchsql_test 13.$tn $sql {1 {SQL logic error or missing database}} 262 + do_catchsql_test 13.$tn $sql {1 {SQL logic error}} 264 263 } 265 264 266 265 finish_test 267 -
Changes to ext/fts5/test/fts5conflict.test.
62 62 REPLACE INTO tbl VALUES(1, '4 5 6', '3 2 1'); 63 63 DELETE FROM tbl WHERE a=100; 64 64 65 65 INSERT INTO fts_idx(fts_idx) VALUES('integrity-check'); 66 66 } 67 67 68 68 finish_test 69 - 70 -
Changes to ext/fts5/test/fts5content.test.
251 251 do_execsql_test 6.2 { 252 252 DROP TABLE xx; 253 253 SELECT name FROM sqlite_master; 254 254 } {} 255 255 256 256 257 257 finish_test 258 -
Changes to ext/fts5/test/fts5corrupt.test.
92 92 93 93 do_catchsql_test 3.1 { 94 94 DELETE FROM t3_content WHERE rowid = 3; 95 95 SELECT * FROM t3 WHERE t3 MATCH 'o'; 96 96 } {1 {database disk image is malformed}} 97 97 98 98 finish_test 99 -
Changes to ext/fts5/test/fts5corrupt2.test.
265 265 do_catchsql_test 6.2 { 266 266 SELECT colsize(x5, 0) FROM x5 WHERE x5 MATCH 'a' 267 267 } {1 SQLITE_CORRUPT_VTAB} 268 268 269 269 270 270 sqlite3_fts5_may_be_corrupt 0 271 271 finish_test 272 -
Changes to ext/fts5/test/fts5corrupt3.test.
405 405 } {} 406 406 do_catchsql_test 9.2.2 { 407 407 SELECT * FROM t1('one AND two'); 408 408 } {1 {database disk image is malformed}} 409 409 410 410 sqlite3_fts5_may_be_corrupt 0 411 411 finish_test 412 -
Changes to ext/fts5/test/fts5delete.test.
47 47 for {set i 0} {$i < 5} {incr i} { 48 48 execsql { INSERT INTO t1(t1, rank) VALUES('merge', 1) } 49 49 execsql { INSERT INTO t1(t1) VALUES('integrity-check') } 50 50 } 51 51 } {} 52 52 53 53 finish_test 54 -
Changes to ext/fts5/test/fts5detail.test.
237 237 (SELECT sum(length(block)) from t2_data) < 238 238 (SELECT sum(length(block)) from t3_data) 239 239 } {1} 240 240 241 241 242 242 243 243 finish_test 244 -
Changes to ext/fts5/test/fts5determin.test.
59 59 } {} 60 60 61 61 do_determin_test 1.4 62 62 } 63 63 64 64 65 65 finish_test 66 - 67 -
Changes to ext/fts5/test/fts5dlidx.test.
62 62 append doc " y" 63 63 } 64 64 } 65 65 execsql { INSERT INTO t1(rowid, x) VALUES($rowid, $doc) } 66 66 } 67 67 execsql COMMIT 68 68 69 - breakpoint 70 69 do_test $tn.1 { 71 70 execsql { INSERT INTO t1(t1) VALUES('integrity-check') } 72 71 } {} 73 72 74 73 do_fb_test $tn.3.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND x' } $xdoc 75 74 do_fb_test $tn.3.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND a' } $xdoc 76 75 ................................................................................ 120 119 INSERT INTO t1(rowid,x) SELECT i, $str FROM iii; 121 120 COMMIT; 122 121 } 123 122 124 123 do_execsql_test $tn.1 { 125 124 SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' 126 125 } {1} 127 - breakpoint 128 126 do_execsql_test $tn.2 { 129 127 SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC 130 128 } {1} 131 129 } 132 130 133 131 do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128] 134 132 ................................................................................ 193 191 } 194 192 195 193 } ;# foreach_detail_mode 196 194 197 195 198 196 199 197 finish_test 200 -
Changes to ext/fts5/test/fts5doclist.test.
40 40 41 41 do_execsql_test 1.2 { 42 42 INSERT INTO ccc(ccc) VALUES('integrity-check'); 43 43 } 44 44 45 45 46 46 finish_test 47 -
Changes to ext/fts5/test/fts5eb.test.
77 77 do_execsql_test 3.3 { 78 78 SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank; 79 79 } {1 -1e-06} 80 80 81 81 82 82 83 83 finish_test 84 - 85 - 86 -
Changes to ext/fts5/test/fts5fault1.test.
347 347 if {$ls != "2 0"} { error "fts5_level_segs says {$ls}" } 348 348 } 349 349 } 350 350 351 351 352 352 353 353 finish_test 354 -
Changes to ext/fts5/test/fts5fault2.test.
133 133 ); 134 134 } 135 135 } -test { 136 136 faultsim_test_result {0 {}} 137 137 } 138 138 139 139 finish_test 140 -
Changes to ext/fts5/test/fts5fault3.test.
106 106 } -test { 107 107 faultsim_test_result [list 0 {}] 108 108 } 109 109 110 110 111 111 112 112 finish_test 113 -
Changes to ext/fts5/test/fts5fault4.test.
391 391 } -body { 392 392 db eval { ALTER TABLE "tbl one" RENAME TO "tbl two" } 393 393 } -test { 394 394 faultsim_test_result {0 {}} 395 395 } 396 396 397 397 finish_test 398 -
Changes to ext/fts5/test/fts5fault5.test.
101 101 db eval { 102 102 SELECT term FROM tv WHERE term BETWEEN '1' AND '2'; 103 103 } 104 104 } -test { 105 105 faultsim_test_result {0 {1 10 11 12 13 14 15 16 17 18 19 2}} 106 106 } 107 107 108 -breakpoint 109 108 do_execsql_test 3.3.0 { 110 109 SELECT * FROM tv2; 111 110 } { 112 111 0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {} 113 112 14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19 x 1 {} 114 113 2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {} 115 114 9 x 1 {} ................................................................................ 126 125 9 x 1 {} 127 126 ]] 128 127 } 129 128 130 129 131 130 132 131 finish_test 133 -
Changes to ext/fts5/test/fts5fault6.test.
276 276 } 277 277 } -test { 278 278 faultsim_test_result {0 1} 279 279 } 280 280 281 281 #------------------------------------------------------------------------- 282 282 catch { db close } 283 -breakpoint 284 283 do_faultsim_test 6 -faults oom* -prep { 285 284 sqlite_orig db test.db 286 285 sqlite3_db_config_lookaside db 0 0 0 287 286 } -test { 288 287 faultsim_test_result {0 {}} {1 {initialization of fts5 failed: }} 289 288 if {$testrc==0} { 290 289 db eval { CREATE VIRTUAL TABLE temp.t1 USING fts5(x) } 291 290 } 292 291 db close 293 292 } 294 293 finish_test 295 -
Changes to ext/fts5/test/fts5fault7.test.
112 112 do_faultsim_test 2.2 -faults oom-* -body { 113 113 db eval { SELECT * FROM xy('""') } 114 114 } -test { 115 115 faultsim_test_result {0 {}} 116 116 } 117 117 118 118 finish_test 119 -
Changes to ext/fts5/test/fts5fault8.test.
78 78 execsql { INSERT INTO x2(x2) VALUES('optimize') } 79 79 } -test { 80 80 faultsim_test_result {0 {}} {1 SQLITE_NOMEM} 81 81 } 82 82 83 83 84 84 finish_test 85 -
Changes to ext/fts5/test/fts5fault9.test.
149 149 faultsim_test_result [list 0 {1 3}] 150 150 } 151 151 152 152 153 153 } ;# foreach_detail_mode... 154 154 155 155 finish_test 156 -
Changes to ext/fts5/test/fts5faultA.test.
57 57 sqlite3 db test.db 58 58 } -body { 59 59 execsql { SELECT rowid FROM o2('a+b+c NOT xyz') } 60 60 } -test { 61 61 faultsim_test_result {0 {1 2}} 62 62 } 63 63 finish_test 64 -
Changes to ext/fts5/test/fts5faultB.test.
128 128 execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') } 129 129 } -test { 130 130 faultsim_test_result {0 {2 3}} 131 131 } 132 132 133 133 134 134 finish_test 135 -
Changes to ext/fts5/test/fts5full.test.
36 36 execsql { INSERT INTO x8 VALUES( rnddoc(5) ); } 37 37 } 38 38 } msg] $msg 39 39 } {1 {database or disk is full}} 40 40 41 41 42 42 finish_test 43 -
Changes to ext/fts5/test/fts5fuzz1.test.
86 86 reset_db 87 87 do_catchsql_test 4.1 { 88 88 CREATE VIRTUAL TABLE f2 USING fts5(o, t); 89 89 SELECT * FROM f2('(8 AND 9)`AND 10'); 90 90 } {1 {fts5: syntax error near "`"}} 91 91 92 92 finish_test 93 -
Changes to ext/fts5/test/fts5hash.test.
117 117 set hash [sqlite3_fts5_token_hash 1024 $big] 118 118 while {1} { 119 119 set small [random_token] 120 120 if {[sqlite3_fts5_token_hash 1024 $small]==$hash} break 121 121 } 122 122 123 123 execsql { CREATE VIRTUAL TABLE t2 USING fts5(x, detail=%DETAIL%) } 124 -breakpoint 125 124 execsql { 126 125 INSERT INTO t2 VALUES($small || ' ' || $big); 127 126 } 128 127 } {} 129 128 130 129 } ;# foreach_detail_mode 131 130 132 131 finish_test 133 -
Changes to ext/fts5/test/fts5integrity.test.
206 206 if {$res == [lsort -integer $res2]} { incr ok } 207 207 } 208 208 set ok 209 209 } {1000} 210 210 } 211 211 212 212 finish_test 213 -
Changes to ext/fts5/test/fts5lastrowid.test.
66 66 do_execsql_test 1.6 { 67 67 INSERT INTO t1(rowid, str) SELECT rowid+10, x FROM x1; 68 68 SELECT last_insert_rowid(); 69 69 } {14} 70 70 71 71 72 72 finish_test 73 -
Added ext/fts5/test/fts5leftjoin.test.
1 +# 2014 June 17 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#************************************************************************* 11 +# This file implements regression tests for SQLite library. The 12 +# focus of this script is testing the FTS5 module. 13 +# 14 + 15 +source [file join [file dirname [info script]] fts5_common.tcl] 16 +set testprefix fts5leftjoin 17 + 18 +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. 19 +ifcapable !fts5 { 20 + finish_test 21 + return 22 +} 23 + 24 +do_execsql_test 1.0 { 25 + CREATE VIRTUAL TABLE vt USING fts5(x); 26 + INSERT INTO vt VALUES('abc'); 27 + INSERT INTO vt VALUES('xyz'); 28 + 29 + CREATE TABLE t1(a INTEGER PRIMARY KEY); 30 + INSERT INTO t1 VALUES(1), (2); 31 +} 32 + 33 +do_execsql_test 1.1 { 34 + SELECT * FROM t1 LEFT JOIN ( 35 + SELECT rowid AS rrr, * FROM vt WHERE vt MATCH 'abc' 36 + ) ON t1.a = rrr 37 +} {1 1 abc 2 {} {}} 38 + 39 +do_execsql_test 1.2 { 40 + SELECT * FROM t1 LEFT JOIN vt ON (vt MATCH 'abc') 41 +} {1 abc 2 abc} 42 + 43 +finish_test
Changes to ext/fts5/test/fts5matchinfo.test.
468 468 } ;# foreach_detail_mode 469 469 470 470 #------------------------------------------------------------------------- 471 471 # Test that a bad fts5() return is detected 472 472 # 473 473 reset_db 474 474 proc xyz {} {} 475 -db func fts5 -argcount 0 xyz 475 +db func fts5 -argcount 1 xyz 476 476 do_test 13.1 { 477 477 list [catch { sqlite3_fts5_register_matchinfo db } msg] $msg 478 478 } {1 SQLITE_ERROR} 479 479 480 480 #------------------------------------------------------------------------- 481 481 # Test that an invalid matchinfo() flag is detected 482 482 # ................................................................................ 488 488 } {} 489 489 490 490 do_catchsql_test 14.2 { 491 491 SELECT matchinfo(x1, 'd') FROM x1('a b c'); 492 492 } {1 {unrecognized matchinfo flag: d}} 493 493 494 494 finish_test 495 -
Changes to ext/fts5/test/fts5merge.test.
237 237 do_execsql_test 6.3 { 238 238 INSERT INTO g1(g1) VALUES('integrity-check'); 239 239 } 240 240 241 241 242 242 243 243 finish_test 244 -
Changes to ext/fts5/test/fts5merge2.test.
51 51 do_execsql_test 1.2 { 52 52 INSERT INTO t1(t1) VALUES('integrity-check'); 53 53 } 54 54 55 55 } 56 56 57 57 finish_test 58 -
Changes to ext/fts5/test/fts5multiclient.test.
41 41 sql1 { INSERT INTO t1 VALUES('a b c') } 42 42 sql3 { INSERT INTO t1(t1) VALUES('integrity-check') } 43 43 } {} 44 44 45 45 };# do_multiclient_test 46 46 };# foreach_detail_mode 47 47 finish_test 48 -
Changes to ext/fts5/test/fts5near.test.
64 64 do_near_test 1.23 "a b c d e f g h i" { NEAR(a+b+c+d i b+c, 4) } 0 65 65 66 66 do_near_test 1.24 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 5) } 1 67 67 do_near_test 1.25 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 4) } 0 68 68 69 69 70 70 finish_test 71 -
Changes to ext/fts5/test/fts5onepass.test.
174 174 UPDATE ttt SET x = 'A B C' WHERE rowid = 4; 175 175 INSERT INTO ttt(rowid, x) VALUES(6, 'd e f'); 176 176 COMMIT; 177 177 } {} 178 178 do_test 4.2.2 { fts5_level_segs ttt } {3} 179 179 180 180 finish_test 181 -
Changes to ext/fts5/test/fts5optimize.test.
102 102 do_execsql_test 2.$tn.5 { 103 103 INSERT INTO t1(t1) VALUES('integrity-check'); 104 104 } 105 105 106 106 do_test 2.$tn.6 { fts5_segcount t1 } 1 107 107 } 108 108 finish_test 109 -
Changes to ext/fts5/test/fts5phrase.test.
112 112 FROM t3('a:f+f') 113 113 } { 114 114 31 {h *f f*} {i j g e c} {j j f c a i j} 115 115 50 {*f f* c} {f f b i i} {f f a j e c i} 116 116 } 117 117 118 118 finish_test 119 -
Changes to ext/fts5/test/fts5plan.test.
60 60 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:} 61 61 } 62 62 63 63 64 64 65 65 66 66 finish_test 67 -
Changes to ext/fts5/test/fts5porter.test.
11799 11799 lindex [sqlite3_fts5_tokenize db porter $in] 0 11800 11800 } $out 11801 11801 incr i 11802 11802 } 11803 11803 11804 11804 11805 11805 finish_test 11806 -
Changes to ext/fts5/test/fts5porter2.test.
63 63 lindex [sqlite3_fts5_tokenize db porter $in] 0 64 64 } $out 65 65 incr i 66 66 } 67 67 68 68 69 69 finish_test 70 -
Changes to ext/fts5/test/fts5prefix.test.
337 337 do_execsql_test 7.$tn { 338 338 SELECT md5sum(id, block) FROM tt_data 339 339 } [list $::checksum] 340 340 } 341 341 } 342 342 343 343 finish_test 344 - 345 -
Changes to ext/fts5/test/fts5query.test.
75 75 } {} 76 76 incr ret 77 77 } 78 78 } 79 79 80 80 81 81 finish_test 82 - 83 -
Changes to ext/fts5/test/fts5rank.test.
86 86 execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db2 87 87 } {1 3 2} 88 88 89 89 do_test 2.7 { 90 90 execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db 91 91 } {1 3 2} 92 92 93 +db2 close 93 94 94 95 #-------------------------------------------------------------------------- 95 96 # At one point there was a problem with queries such as: 96 97 # 97 98 # ... MATCH 'x OR y' ORDER BY rank; 98 99 # 99 100 # if there were zero occurrences of token 'y' in the dataset. The ................................................................................ 147 148 VTest MATCH 'wrinkle in time OR a wrinkle in time' ORDER BY rank; 148 149 } {{wrinkle in time} {Bill Smith}} 149 150 150 151 151 152 152 153 153 154 finish_test 154 -
Changes to ext/fts5/test/fts5rebuild.test.
60 60 CREATE VIRTUAL TABLE nc USING fts5(doc, content=); 61 61 } 62 62 63 63 do_catchsql_test 2.2 { 64 64 INSERT INTO nc(nc) VALUES('rebuild'); 65 65 } {1 {'rebuild' may not be used with a contentless fts5 table}} 66 66 finish_test 67 -
Changes to ext/fts5/test/fts5restart.test.
145 145 } 146 146 set res 147 147 } {500 400 300} 148 148 149 149 150 150 151 151 finish_test 152 -
Changes to ext/fts5/test/fts5rowid.test.
212 212 } {36} 213 213 214 214 #db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM x5_data} {puts $r} 215 215 216 216 217 217 218 218 finish_test 219 -
Changes to ext/fts5/test/fts5simple.test.
407 407 408 408 do_catchsql_test 19.2 { 409 409 SELECT * FROM x1 WHERE x1 MATCH 'c0 AND (c1 AND (c2 AND (c3 AND (c4 AND (c5 AND (c6 AND (c7 AND (c8 AND (c9 AND (c10 AND (c11 AND (c12 AND (c13 AND (c14 AND (c15 AND (c16 AND (c17 AND (c18 AND (c19 AND (c20 AND (c21 AND (c22 AND (c23 AND (c24 AND (c25 AND (c26 AND (c27 AND (c28 AND (c29 AND (c30 AND (c31 AND (c32 AND (c33 AND (c34 AND (c35 AND (c36 AND (c37 AND (c38 AND (c39 AND (c40 AND (c41 AND (c42 AND (c43 AND (c44 AND (c45 AND (c46 AND (c47 AND (c48 AND (c49 AND (c50 AND (c51 AND (c52 AND (c53 AND (c54 AND (c55 AND (c56 AND (c57 AND (c58 AND (c59 AND (c60 AND (c61 AND (c62 AND (c63 AND (c64 AND (c65 AND (c66 AND (c67 AND (c68 AND (c69 AND (c70 AND (c71 AND (c72 AND (c73 AND (c74 AND (c75 AND (c76 AND (c77 AND (c78 AND (c79 AND (c80 AND (c81 AND (c82 AND (c83 AND (c84 AND (c85 AND (c86 AND (c87 AND (c88 AND (c89 AND (c90 AND (c91 AND (c92 AND (c93 AND (c94 AND (c95 AND (c96 AND (c97 AND (c98 AND (c99 AND (c100 AND (c101 AND (c102 AND (c103 AND (c104 AND (c105 AND (c106 AND (c107 AND (c108 AND (c109 AND (c110 AND (c111 AND (c112 AND (c113 AND (c114 AND (c115 AND (c116 AND (c117 AND (c118 AND (c119 AND (c120 AND (c121 AND (c122 AND (c123 AND (c124 AND (c125 AND (c126 AND (c127 AND (c128 AND (c129 AND (c130 AND (c131 AND (c132 AND (c133 AND (c134 AND (c135 AND (c136 AND (c137 AND (c138 AND (c139 AND (c140 AND (c141 AND (c142 AND (c143 AND (c144 AND (c145 AND (c146 AND (c147 AND (c148 AND (c149 AND (c150 AND (c151 AND (c152 AND (c153 AND (c154 AND (c155 AND (c156 AND (c157 AND (c158 AND (c159 AND (c160 AND (c161 AND (c162 AND (c163 AND (c164 AND (c165 AND (c166 AND (c167 AND (c168 AND (c169 AND (c170 AND (c171 AND (c172 AND (c173 AND (c174 AND (c175 AND (c176 AND (c177 AND (c178 AND (c179 AND (c180 AND (c181 AND (c182 AND (c183 AND (c184 AND (c185 AND (c186 AND (c187 AND (c188 AND (c189 AND (c190 AND (c191 AND (c192 AND (c193 AND (c194 AND (c195 AND (c196 AND (c197 AND (c198 AND (c199 AND c200)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))'; 410 410 } {1 {fts5: parser stack overflow}} 411 411 412 412 #------------------------------------------------------------------------- 413 413 reset_db 414 -breakpoint 415 414 do_execsql_test 20.0 { 416 415 CREATE VIRTUAL TABLE x1 USING fts5(x); 417 416 INSERT INTO x1(x1, rank) VALUES('pgsz', 32); 418 417 INSERT INTO x1(rowid, x) VALUES(11111, 'onetwothree'); 419 418 } 420 419 do_test 20.1 { 421 420 for {set i 1} {$i <= 200} {incr i} {
Changes to ext/fts5/test/fts5simple2.test.
366 366 do_execsql_test 17.6 { 367 367 SELECT * FROM t2('x:b* OR y:a*') WHERE rowid>55 368 368 } 369 369 370 370 #db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM t2_data} {puts $r} 371 371 372 372 finish_test 373 -
Changes to ext/fts5/test/fts5simple3.test.
112 112 } 113 113 do_execsql_test 4.6 { 114 114 SELECT * FROM t2('ab + xyz'); 115 115 } 116 116 117 117 118 118 finish_test 119 -
Changes to ext/fts5/test/fts5synonym.test.
417 417 do_execsql_test 7.1.2 { 418 418 INSERT INTO t2(t2) VALUES('integrity-check'); 419 419 } 420 420 421 421 } ;# foreach_detail_mode 422 422 423 423 finish_test 424 -
Changes to ext/fts5/test/fts5synonym2.test.
157 157 158 158 } 159 159 160 160 } 161 161 } 162 162 163 163 finish_test 164 -
Changes to ext/fts5/test/fts5tok1.test.
105 105 do_catchsql_test 2.0 { 106 106 CREATE VIRTUAL TABLE tX USING fts5tokenize(nosuchtokenizer); 107 107 } {1 {vtable constructor failed: tX}} 108 108 109 109 do_catchsql_test 2.1 { 110 110 CREATE VIRTUAL TABLE t4 USING fts5tokenize; 111 111 SELECT * FROM t4; 112 -} {1 {SQL logic error or missing database}} 112 +} {1 {SQL logic error}} 113 113 114 114 115 115 finish_test
Changes to ext/fts5/test/fts5tokenizer.test.
298 298 299 299 set ::flags [list] 300 300 do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {} 301 301 do_test 9.5.2 { set ::flags } {query} 302 302 303 303 304 304 finish_test 305 -
Changes to ext/fts5/test/fts5unicode.test.
46 46 CREATE VIRTUAL TABLE t1 USING fts5(x); 47 47 CREATE VIRTUAL TABLE t2 USING fts5(x, tokenize = unicode61); 48 48 CREATE VIRTUAL TABLE t3 USING fts5(x, tokenize = ascii); 49 49 INSERT INTO t1 VALUES('\xC0\xC8\xCC'); 50 50 INSERT INTO t2 VALUES('\xC0\xC8\xCC'); 51 51 INSERT INTO t3 VALUES('\xC0\xC8\xCC'); 52 52 " 53 -breakpoint 54 53 do_execsql_test 2.1 " 55 54 SELECT 't1' FROM t1 WHERE t1 MATCH '\xE0\xE8\xEC'; 56 55 SELECT 't2' FROM t2 WHERE t2 MATCH '\xE0\xE8\xEC'; 57 56 SELECT 't3' FROM t3 WHERE t3 MATCH '\xE0\xE8\xEC'; 58 57 " {t1 t2} 59 58 60 59 61 60 finish_test 62 -
Changes to ext/fts5/test/fts5unicode2.test.
277 277 INSERT INTO t9(a) VALUES('abc%88def %89ghi%90'); 278 278 } 279 279 } {0 {}} 280 280 281 281 282 282 #------------------------------------------------------------------------- 283 283 284 -breakpoint 285 284 do_unicode_token_test3 5.1 {tokenchars {}} { 286 285 sqlite3_reset sqlite3_column_int 287 286 } { 288 287 sqlite3 sqlite3 289 288 reset reset 290 289 sqlite3 sqlite3 291 290 column column
Changes to ext/fts5/test/fts5unicode3.test.
122 122 } 123 123 append str {'");} 124 124 execsql $str 125 125 } {} 126 126 127 127 128 128 finish_test 129 -
Changes to ext/fts5/test/fts5unindexed.test.
72 72 INSERT INTO t4(t4, rowid, a, b, c) VALUES('delete', 20, 'j k l', '', 'p q r'); 73 73 DELETE FROM x4 WHERE rowid=20; 74 74 INSERT INTO t4(t4) VALUES('integrity-check'); 75 75 } {} 76 76 77 77 78 78 finish_test 79 -
Changes to ext/fts5/test/fts5update.test.
113 113 } {} 114 114 do_execsql_test 2.2.integrity { 115 115 INSERT INTO x2(x2) VALUES('integrity-check'); 116 116 } 117 117 118 118 } 119 119 finish_test 120 - 121 -
Changes to ext/fts5/test/fts5version.test.
57 57 db close 58 58 sqlite3 db test.db 59 59 catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' } 60 60 } {1 {invalid fts5 file format (found 0, expected 4) - run 'rebuild'}} 61 61 62 62 63 63 finish_test 64 -
Changes to ext/fts5/test/fts5vocab.test.
206 206 INSERT INTO temp.t1 VALUES('1 5 3'); 207 207 208 208 INSERT INTO aux.t1 VALUES('x y z'); 209 209 INSERT INTO aux.t1 VALUES('m n o'); 210 210 INSERT INTO aux.t1 VALUES('x n z'); 211 211 } 212 212 213 -breakpoint 214 213 do_execsql_test 5.1 { 215 214 CREATE VIRTUAL TABLE temp.vm USING fts5vocab(main, t1, row); 216 215 CREATE VIRTUAL TABLE temp.vt1 USING fts5vocab(t1, row); 217 216 CREATE VIRTUAL TABLE temp.vt2 USING fts5vocab(temp, t1, row); 218 217 CREATE VIRTUAL TABLE temp.va USING fts5vocab(aux, t1, row); 219 218 } 220 219
Added ext/lsm1/Makefile.
1 +# 2 +# This Makefile is designed for use with main.mk in the root directory of 3 +# this project. After including main.mk, the users makefile should contain: 4 +# 5 +# LSMDIR=$(TOP)/ext/lsm1/ 6 +# LSMOPTS=-fPIC 7 +# include $(LSMDIR)/Makefile 8 +# 9 +# The most useful targets are [lsmtest] and [lsm.so]. 10 +# 11 + 12 +LSMOBJ = \ 13 + lsm_ckpt.o \ 14 + lsm_file.o \ 15 + lsm_log.o \ 16 + lsm_main.o \ 17 + lsm_mem.o \ 18 + lsm_mutex.o \ 19 + lsm_shared.o \ 20 + lsm_sorted.o \ 21 + lsm_str.o \ 22 + lsm_tree.o \ 23 + lsm_unix.o \ 24 + lsm_win32.o \ 25 + lsm_varint.o \ 26 + lsm_vtab.o 27 + 28 +LSMHDR = \ 29 + $(LSMDIR)/lsm.h \ 30 + $(LSMDIR)/lsmInt.h 31 + 32 +LSMTESTSRC = $(LSMDIR)/lsm-test/lsmtest1.c $(LSMDIR)/lsm-test/lsmtest2.c \ 33 + $(LSMDIR)/lsm-test/lsmtest3.c $(LSMDIR)/lsm-test/lsmtest4.c \ 34 + $(LSMDIR)/lsm-test/lsmtest5.c $(LSMDIR)/lsm-test/lsmtest6.c \ 35 + $(LSMDIR)/lsm-test/lsmtest7.c $(LSMDIR)/lsm-test/lsmtest8.c \ 36 + $(LSMDIR)/lsm-test/lsmtest9.c \ 37 + $(LSMDIR)/lsm-test/lsmtest_datasource.c \ 38 + $(LSMDIR)/lsm-test/lsmtest_func.c $(LSMDIR)/lsm-test/lsmtest_io.c \ 39 + $(LSMDIR)/lsm-test/lsmtest_main.c $(LSMDIR)/lsm-test/lsmtest_mem.c \ 40 + $(LSMDIR)/lsm-test/lsmtest_tdb.c $(LSMDIR)/lsm-test/lsmtest_tdb3.c \ 41 + $(LSMDIR)/lsm-test/lsmtest_util.c $(LSMDIR)/lsm-test/lsmtest_win32.c 42 + 43 + 44 +# all: lsm.so 45 + 46 +LSMOPTS += -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR) 47 + 48 +lsm.so: $(LSMOBJ) 49 + $(TCCX) -shared -o lsm.so $(LSMOBJ) 50 + 51 +%.o: $(LSMDIR)/%.c $(LSMHDR) sqlite3.h 52 + $(TCCX) $(LSMOPTS) -c $< 53 + 54 +lsmtest$(EXE): $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) sqlite3.o 55 + # $(TCPPX) -c $(TOP)/lsm-test/lsmtest_tdb2.cc 56 + $(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB)
Added ext/lsm1/Makefile.msc.
1 +# 2 +# This Makefile is designed for use with Makefile.msc in the root directory 3 +# of this project. The Makefile.msc should contain: 4 +# 5 +# LSMDIR=$(TOP)\ext\lsm1 6 +# !INCLUDE $(LSMDIR)\Makefile.msc 7 +# 8 +# The most useful targets are [lsmtest.exe] and [lsm.dll]. 9 +# 10 + 11 +LSMOBJ = \ 12 + lsm_ckpt.lo \ 13 + lsm_file.lo \ 14 + lsm_log.lo \ 15 + lsm_main.lo \ 16 + lsm_mem.lo \ 17 + lsm_mutex.lo \ 18 + lsm_shared.lo \ 19 + lsm_sorted.lo \ 20 + lsm_str.lo \ 21 + lsm_tree.lo \ 22 + lsm_unix.lo \ 23 + lsm_win32.lo \ 24 + lsm_varint.lo \ 25 + lsm_vtab.lo 26 + 27 +LSMHDR = \ 28 + $(LSMDIR)\lsm.h \ 29 + $(LSMDIR)\lsmInt.h 30 + 31 +LSMTESTSRC = $(LSMDIR)\lsm-test\lsmtest1.c $(LSMDIR)\lsm-test\lsmtest2.c \ 32 + $(LSMDIR)\lsm-test\lsmtest3.c $(LSMDIR)\lsm-test\lsmtest4.c \ 33 + $(LSMDIR)\lsm-test\lsmtest5.c $(LSMDIR)\lsm-test\lsmtest6.c \ 34 + $(LSMDIR)\lsm-test\lsmtest7.c $(LSMDIR)\lsm-test\lsmtest8.c \ 35 + $(LSMDIR)\lsm-test\lsmtest9.c \ 36 + $(LSMDIR)\lsm-test\lsmtest_datasource.c \ 37 + $(LSMDIR)\lsm-test\lsmtest_func.c $(LSMDIR)\lsm-test\lsmtest_io.c \ 38 + $(LSMDIR)\lsm-test\lsmtest_main.c $(LSMDIR)\lsm-test\lsmtest_mem.c \ 39 + $(LSMDIR)\lsm-test\lsmtest_tdb.c $(LSMDIR)\lsm-test\lsmtest_tdb3.c \ 40 + $(LSMDIR)\lsm-test\lsmtest_util.c $(LSMDIR)\lsm-test\lsmtest_win32.c 41 + 42 +# all: lsm.dll lsmtest.exe 43 + 44 +LSMOPTS = $(NO_WARN) -DLSM_MUTEX_WIN32=1 -I$(LSMDIR) 45 + 46 +!IF $(DEBUG)>2 47 +LSMOPTS = $(LSMOPTS) -DLSM_DEBUG=1 48 +!ENDIF 49 + 50 +!IF $(MEMDEBUG)!=0 51 +LSMOPTS = $(LSMOPTS) -DLSM_DEBUG_MEM=1 52 +!ENDIF 53 + 54 +lsm_ckpt.lo: $(LSMDIR)\lsm_ckpt.c $(LSMHDR) $(SQLITE3H) 55 + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_ckpt.c 56 + 57 +lsm_file.lo: $(LSMDIR)\lsm_file.c $(LSMHDR) $(SQLITE3H) 58 + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_file.c 59 + 60 +lsm_log.lo: $(LSMDIR)\lsm_log.c $(LSMHDR) $(SQLITE3H) 61 + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_log.c 62 + 63 +lsm_main.lo: $(LSMDIR)\lsm_main.c $(LSMHDR) $(SQLITE3H) 64 + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_main.c 65 + 66 +lsm_mem.lo: $(LSMDIR)\lsm_mem.c $(LSMHDR) $(SQLITE3H) 67 + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mem.c 68 + 69 +lsm_mutex.lo: $(LSMDIR)\lsm_mutex.c $(LSMHDR) $(SQLITE3H) 70 + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mutex.c 71 + 72 +lsm_shared.lo: $(LSMDIR)\lsm_shared.c $(LSMHDR) $(SQLITE3H) 73 + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_shared.c 74 + 75 +lsm_sorted.lo: $(LSMDIR)\lsm_sorted.c $(LSMHDR) $(SQLITE3H) 76 + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_sorted.c 77 + 78 +lsm_str.lo: $(LSMDIR)\lsm_str.c $(LSMHDR) $(SQLITE3H) 79 + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_str.c 80 + 81 +lsm_tree.lo: $(LSMDIR)\lsm_tree.c $(LSMHDR) $(SQLITE3H) 82 + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_tree.c 83 + 84 +lsm_unix.lo: $(LSMDIR)\lsm_unix.c $(LSMHDR) $(SQLITE3H) 85 + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_unix.c 86 + 87 +lsm_win32.lo: $(LSMDIR)\lsm_win32.c $(LSMHDR) $(SQLITE3H) 88 + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_win32.c 89 + 90 +lsm_varint.lo: $(LSMDIR)\lsm_varint.c $(LSMHDR) $(SQLITE3H) 91 + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_varint.c 92 + 93 +lsm_vtab.lo: $(LSMDIR)\lsm_vtab.c $(LSMHDR) $(SQLITE3H) 94 + $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_vtab.c 95 + 96 +lsm.dll: $(LSMOBJ) 97 + $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ $(LSMOBJ) 98 + copy /Y $@ $(LSMDIR)\$@ 99 + 100 +lsmtest.exe: $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) $(LIBOBJ) 101 + $(LTLINK) $(LSMOPTS) $(LSMTESTSRC) /link $(LSMOBJ) $(LIBOBJ) 102 + copy /Y $@ $(LSMDIR)\$@
Added ext/lsm1/lsm-test/README.
1 + 2 + 3 +Organization of test case files: 4 + 5 + lsmtest1.c: Data tests. Tests that perform many inserts and deletes on a 6 + database file, then verify that the contents of the database can 7 + be queried. 8 + 9 + lsmtest2.c: Crash tests. Tests that attempt to verify that the database 10 + recovers correctly following an application or system crash. 11 + 12 + lsmtest3.c: Rollback tests. Tests that focus on the explicit rollback of 13 + transactions and sub-transactions. 14 + 15 + lsmtest4.c: Multi-client tests. 16 + 17 + lsmtest5.c: Multi-client tests with a different thread for each client. 18 + 19 + lsmtest6.c: OOM injection tests. 20 + 21 + lsmtest7.c: API tests. 22 + 23 + lsmtest8.c: Writer crash tests. Tests in this file attempt to verify that 24 + the system recovers and other clients proceed unaffected if 25 + a process fails in the middle of a write transaction. 26 + 27 + The difference from lsmtest2.c is that this file tests 28 + live-recovery (recovery from a failure that occurs while other 29 + clients are still running) whereas lsmtest2.c tests recovery 30 + from a system or power failure. 31 + 32 + lsmtest9.c: More data tests. These focus on testing that calling 33 + lsm_work(nMerge=1) to compact the database does not corrupt it. 34 + In other words, that databases containing block-redirects 35 + can be read and written. 36 + 37 + 38 + 39 + 40 +
Added ext/lsm1/lsm-test/lsmtest.h.
1 + 2 +#ifndef __WRAPPER_INT_H_ 3 +#define __WRAPPER_INT_H_ 4 + 5 +#include "lsmtest_tdb.h" 6 +#include "sqlite3.h" 7 +#include "lsm.h" 8 + 9 +#include <assert.h> 10 +#include <stdarg.h> 11 +#include <stdlib.h> 12 +#include <string.h> 13 +#include <stdio.h> 14 +#ifndef _WIN32 15 +# include <unistd.h> 16 +#endif 17 +#include <sys/types.h> 18 +#include <sys/stat.h> 19 +#include <fcntl.h> 20 +#include <ctype.h> 21 +#include <stdlib.h> 22 +#include <errno.h> 23 + 24 +#ifdef __cplusplus 25 +extern "C" { 26 +#endif 27 + 28 +#ifdef _WIN32 29 +# include "windows.h" 30 +# define gettimeofday win32GetTimeOfDay 31 +# define F_OK (0) 32 +# define sleep(sec) Sleep(1000 * (sec)) 33 +# define usleep(usec) Sleep(((usec) + 999) / 1000) 34 +# ifdef _MSC_VER 35 +# include <io.h> 36 +# define snprintf _snprintf 37 +# define fsync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd))) 38 +# define fdatasync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd))) 39 +# define __va_copy(dst,src) ((dst) = (src)) 40 +# define ftruncate(fd,sz) ((_chsize_s((fd), (sz))==0) ? 0 : -1) 41 +# else 42 +# error Unsupported C compiler for Windows. 43 +# endif 44 +int win32GetTimeOfDay(struct timeval *, void *); 45 +#endif 46 + 47 +#ifndef _LSM_INT_H 48 +typedef unsigned int u32; 49 +typedef unsigned char u8; 50 +typedef long long int i64; 51 +typedef unsigned long long int u64; 52 +#endif 53 + 54 + 55 +#define ArraySize(x) ((int)(sizeof(x) / sizeof((x)[0]))) 56 + 57 +#define MIN(x,y) ((x)<(y) ? (x) : (y)) 58 +#define MAX(x,y) ((x)>(y) ? (x) : (y)) 59 + 60 +#define unused_parameter(x) (void)(x) 61 + 62 +#define TESTDB_DEFAULT_PAGE_SIZE 4096 63 +#define TESTDB_DEFAULT_CACHE_SIZE 2048 64 + 65 +#ifndef _O_BINARY 66 +# define _O_BINARY (0) 67 +#endif 68 + 69 +/* 70 +** Ideally, these should be in wrapper.c. But they are here instead so that 71 +** they can be used by the C++ database wrappers in wrapper2.cc. 72 +*/ 73 +typedef struct DatabaseMethods DatabaseMethods; 74 +struct TestDb { 75 + DatabaseMethods const *pMethods; /* Database methods */ 76 + const char *zLibrary; /* Library name for tdb_open() */ 77 +}; 78 +struct DatabaseMethods { 79 + int (*xClose)(TestDb *); 80 + int (*xWrite)(TestDb *, void *, int , void *, int); 81 + int (*xDelete)(TestDb *, void *, int); 82 + int (*xDeleteRange)(TestDb *, void *, int, void *, int); 83 + int (*xFetch)(TestDb *, void *, int, void **, int *); 84 + int (*xScan)(TestDb *, void *, int, void *, int, void *, int, 85 + void (*)(void *, void *, int , void *, int) 86 + ); 87 + int (*xBegin)(TestDb *, int); 88 + int (*xCommit)(TestDb *, int); 89 + int (*xRollback)(TestDb *, int); 90 +}; 91 + 92 +/* 93 +** Functions in wrapper2.cc (a C++ source file). wrapper2.cc contains the 94 +** wrapper for Kyoto Cabinet. Kyoto cabinet has a C API, but 95 +** the primary interface is the C++ API. 96 +*/ 97 +int test_kc_open(const char*, const char *zFilename, int bClear, TestDb **ppDb); 98 +int test_kc_close(TestDb *); 99 +int test_kc_write(TestDb *, void *, int , void *, int); 100 +int test_kc_delete(TestDb *, void *, int); 101 +int test_kc_delete_range(TestDb *, void *, int, void *, int); 102 +int test_kc_fetch(TestDb *, void *, int, void **, int *); 103 +int test_kc_scan(TestDb *, void *, int, void *, int, void *, int, 104 + void (*)(void *, void *, int , void *, int) 105 +); 106 + 107 +int test_mdb_open(const char*, const char *zFile, int bClear, TestDb **ppDb); 108 +int test_mdb_close(TestDb *); 109 +int test_mdb_write(TestDb *, void *, int , void *, int); 110 +int test_mdb_delete(TestDb *, void *, int); 111 +int test_mdb_fetch(TestDb *, void *, int, void **, int *); 112 +int test_mdb_scan(TestDb *, void *, int, void *, int, void *, int, 113 + void (*)(void *, void *, int , void *, int) 114 +); 115 + 116 +/* 117 +** Functions in wrapper3.c. This file contains the tdb wrapper for lsm. 118 +** The wrapper for lsm is a bit more involved than the others, as it 119 +** includes code for a couple of different lsm configurations, and for 120 +** various types of fault injection and robustness testing. 121 +*/ 122 +int test_lsm_open(const char*, const char *zFile, int bClear, TestDb **ppDb); 123 +int test_lsm_lomem_open(const char*, const char*, int bClear, TestDb **ppDb); 124 +int test_lsm_zip_open(const char*, const char*, int bClear, TestDb **ppDb); 125 +int test_lsm_small_open(const char*, const char*, int bClear, TestDb **ppDb); 126 +int test_lsm_mt2(const char*, const char *zFile, int bClear, TestDb **ppDb); 127 +int test_lsm_mt3(const char*, const char *zFile, int bClear, TestDb **ppDb); 128 + 129 +int tdb_lsm_configure(lsm_db *, const char *); 130 + 131 +/* Functions in lsmtest_tdb4.c */ 132 +int test_bt_open(const char*, const char *zFile, int bClear, TestDb **ppDb); 133 +int test_fbt_open(const char*, const char *zFile, int bClear, TestDb **ppDb); 134 +int test_fbts_open(const char*, const char *zFile, int bClear, TestDb **ppDb); 135 + 136 + 137 +/* Functions in testutil.c. */ 138 +int testPrngInit(void); 139 +u32 testPrngValue(u32 iVal); 140 +void testPrngArray(u32 iVal, u32 *aOut, int nOut); 141 +void testPrngString(u32 iVal, char *aOut, int nOut); 142 + 143 +void testErrorInit(int argc, char **); 144 +void testPrintError(const char *zFormat, ...); 145 +void testPrintUsage(const char *zArgs); 146 +void testPrintFUsage(const char *zFormat, ...); 147 +void testTimeInit(void); 148 +int testTimeGet(void); 149 + 150 +/* Functions in testmem.c. */ 151 +void testMallocInstall(lsm_env *pEnv); 152 +void testMallocUninstall(lsm_env *pEnv); 153 +void testMallocCheck(lsm_env *pEnv, int *, int *, FILE *); 154 +void testMallocOom(lsm_env *pEnv, int, int, void(*)(void*), void *); 155 +void testMallocOomEnable(lsm_env *pEnv, int); 156 + 157 +/* lsmtest.c */ 158 +TestDb *testOpen(const char *zSystem, int, int *pRc); 159 +void testReopen(TestDb **ppDb, int *pRc); 160 +void testClose(TestDb **ppDb); 161 + 162 +void testFetch(TestDb *, void *, int, void *, int, int *); 163 +void testWrite(TestDb *, void *, int, void *, int, int *); 164 +void testDelete(TestDb *, void *, int, int *); 165 +void testDeleteRange(TestDb *, void *, int, void *, int, int *); 166 +void testWriteStr(TestDb *, const char *, const char *zVal, int *pRc); 167 +void testFetchStr(TestDb *, const char *, const char *, int *pRc); 168 + 169 +void testBegin(TestDb *pDb, int iTrans, int *pRc); 170 +void testCommit(TestDb *pDb, int iTrans, int *pRc); 171 + 172 +void test_failed(void); 173 + 174 +char *testMallocPrintf(const char *zFormat, ...); 175 +char *testMallocVPrintf(const char *zFormat, va_list ap); 176 +int testGlobMatch(const char *zPattern, const char *zStr); 177 + 178 +void testScanCompare(TestDb *, TestDb *, int, void *, int, void *, int, int *); 179 +void testFetchCompare(TestDb *, TestDb *, void *, int, int *); 180 + 181 +void *testMalloc(int); 182 +void *testMallocCopy(void *pCopy, int nByte); 183 +void *testRealloc(void *, int); 184 +void testFree(void *); 185 + 186 +/* lsmtest_bt.c */ 187 +int do_bt(int nArg, char **azArg); 188 + 189 +/* testio.c */ 190 +int testVfsConfigureDb(TestDb *pDb); 191 + 192 +/* testfunc.c */ 193 +int do_show(int nArg, char **azArg); 194 +int do_work(int nArg, char **azArg); 195 + 196 +/* testio.c */ 197 +int do_io(int nArg, char **azArg); 198 + 199 +/* lsmtest2.c */ 200 +void do_crash_test(const char *zPattern, int *pRc); 201 +int do_rollback_test(int nArg, char **azArg); 202 + 203 +/* test3.c */ 204 +void test_rollback(const char *zSystem, const char *zPattern, int *pRc); 205 + 206 +/* test4.c */ 207 +void test_mc(const char *zSystem, const char *zPattern, int *pRc); 208 + 209 +/* test5.c */ 210 +void test_mt(const char *zSystem, const char *zPattern, int *pRc); 211 + 212 +/* lsmtest6.c */ 213 +void test_oom(const char *zPattern, int *pRc); 214 +void testDeleteLsmdb(const char *zFile); 215 + 216 +void testSaveDb(const char *zFile, const char *zAuxExt); 217 +void testRestoreDb(const char *zFile, const char *zAuxExt); 218 +void testCopyLsmdb(const char *zFrom, const char *zTo); 219 + 220 +/* lsmtest7.c */ 221 +void test_api(const char *zPattern, int *pRc); 222 + 223 +/* lsmtest8.c */ 224 +void do_writer_crash_test(const char *zPattern, int *pRc); 225 + 226 +/************************************************************************* 227 +** Interface to functionality in test_datasource.c. 228 +*/ 229 +typedef struct Datasource Datasource; 230 +typedef struct DatasourceDefn DatasourceDefn; 231 + 232 +struct DatasourceDefn { 233 + int eType; /* A TEST_DATASOURCE_* value */ 234 + int nMinKey; /* Minimum key size */ 235 + int nMaxKey; /* Maximum key size */ 236 + int nMinVal; /* Minimum value size */ 237 + int nMaxVal; /* Maximum value size */ 238 +}; 239 + 240 +#define TEST_DATASOURCE_RANDOM 1 241 +#define TEST_DATASOURCE_SEQUENCE 2 242 + 243 +char *testDatasourceName(const DatasourceDefn *); 244 +Datasource *testDatasourceNew(const DatasourceDefn *); 245 +void testDatasourceFree(Datasource *); 246 +void testDatasourceEntry(Datasource *, int, void **, int *, void **, int *); 247 +/* End of test_datasource.c interface. 248 +*************************************************************************/ 249 +void testDatasourceFetch( 250 + TestDb *pDb, /* Database handle */ 251 + Datasource *pData, 252 + int iKey, 253 + int *pRc /* IN/OUT: Error code */ 254 +); 255 + 256 +void testWriteDatasource(TestDb *, Datasource *, int, int *); 257 +void testWriteDatasourceRange(TestDb *, Datasource *, int, int, int *); 258 +void testDeleteDatasource(TestDb *, Datasource *, int, int *); 259 +void testDeleteDatasourceRange(TestDb *, Datasource *, int, int, int *); 260 + 261 + 262 +/* test1.c */ 263 +void test_data_1(const char *, const char *, int *pRc); 264 +void test_data_2(const char *, const char *, int *pRc); 265 +void test_data_3(const char *, const char *, int *pRc); 266 +void testDbContents(TestDb *, Datasource *, int, int, int, int, int, int *); 267 +void testCaseProgress(int, int, int, int *); 268 +int testCaseNDot(void); 269 + 270 +void testCompareDb(Datasource *, int, int, TestDb *, TestDb *, int *); 271 +int testControlDb(TestDb **ppDb); 272 + 273 +typedef struct CksumDb CksumDb; 274 +CksumDb *testCksumArrayNew(Datasource *, int, int, int); 275 +char *testCksumArrayGet(CksumDb *, int); 276 +void testCksumArrayFree(CksumDb *); 277 +void testCaseStart(int *pRc, char *zFmt, ...); 278 +void testCaseFinish(int rc); 279 +void testCaseSkip(void); 280 +int testCaseBegin(int *, const char *, const char *, ...); 281 + 282 +#define TEST_CKSUM_BYTES 29 283 +int testCksumDatabase(TestDb *pDb, char *zOut); 284 +int testCountDatabase(TestDb *pDb); 285 +void testCompareInt(int, int, int *); 286 +void testCompareStr(const char *z1, const char *z2, int *pRc); 287 + 288 +/* lsmtest9.c */ 289 +void test_data_4(const char *, const char *, int *pRc); 290 + 291 + 292 +/* 293 +** Similar to the Tcl_GetIndexFromObjStruct() Tcl library function. 294 +*/ 295 +#define testArgSelect(w,x,y,z) testArgSelectX(w,x,sizeof(w[0]),y,z) 296 +int testArgSelectX(void *, const char *, int, const char *, int *); 297 + 298 +#ifdef __cplusplus 299 +} /* End of the 'extern "C"' block */ 300 +#endif 301 + 302 +#endif
Added ext/lsm1/lsm-test/lsmtest1.c.
1 + 2 +#include "lsmtest.h" 3 + 4 +#define DATA_SEQUENTIAL TEST_DATASOURCE_SEQUENCE 5 +#define DATA_RANDOM TEST_DATASOURCE_RANDOM 6 + 7 +typedef struct Datatest1 Datatest1; 8 +typedef struct Datatest2 Datatest2; 9 + 10 +/* 11 +** An instance of the following structure contains parameters used to 12 +** customize the test function in this file. Test procedure: 13 +** 14 +** 1. Create a data-source based on the "datasource definition" vars. 15 +** 16 +** 2. Insert nRow key value pairs into the database. 17 +** 18 +** 3. Delete all keys from the database. Deletes are done in the same 19 +** order as the inserts. 20 +** 21 +** During steps 2 and 3 above, after each Datatest1.nVerify inserts or 22 +** deletes, the following: 23 +** 24 +** a. Run Datasource.nTest key lookups and check the results are as expected. 25 +** 26 +** b. If Datasource.bTestScan is true, run a handful (8) of range 27 +** queries (scanning forwards and backwards). Check that the results 28 +** are as expected. 29 +** 30 +** c. Close and reopen the database. Then run (a) and (b) again. 31 +*/ 32 +struct Datatest1 { 33 + /* Datasource definition */ 34 + DatasourceDefn defn; 35 + 36 + /* Test procedure parameters */ 37 + int nRow; /* Number of rows to insert then delete */ 38 + int nVerify; /* How often to verify the db contents */ 39 + int nTest; /* Number of keys to test (0==all) */ 40 + int bTestScan; /* True to do scan tests */ 41 +}; 42 + 43 +/* 44 +** An instance of the following data structure is used to describe the 45 +** second type of test case in this file. The chief difference between 46 +** these tests and those described by Datatest1 is that these tests also 47 +** experiment with range-delete operations. Tests proceed as follows: 48 +** 49 +** 1. Open the datasource described by Datatest2.defn. 50 +** 51 +** 2. Open a connection on an empty database. 52 +** 53 +** 3. Do this Datatest2.nIter times: 54 +** 55 +** a) Insert Datatest2.nWrite key-value pairs from the datasource. 56 +** 57 +** b) Select two pseudo-random keys and use them as the start 58 +** and end points of a range-delete operation. 59 +** 60 +** c) Verify that the contents of the database are as expected (see 61 +** below for details). 62 +** 63 +** d) Close and then reopen the database handle. 64 +** 65 +** e) Verify that the contents of the database are still as expected. 66 +** 67 +** The inserts and range deletes are run twice - once on the database being 68 +** tested and once using a control system (sqlite3, kc etc. - something that 69 +** works). In order to verify that the contents of the db being tested are 70 +** correct, the test runs a bunch of scans and lookups on both the test and 71 +** control databases. If the results are the same, the test passes. 72 +*/ 73 +struct Datatest2 { 74 + DatasourceDefn defn; 75 + int nRange; 76 + int nWrite; /* Number of writes per iteration */ 77 + int nIter; /* Total number of iterations to run */ 78 +}; 79 + 80 +/* 81 +** Generate a unique name for the test case pTest with database system 82 +** zSystem. 83 +*/ 84 +static char *getName(const char *zSystem, int bRecover, Datatest1 *pTest){ 85 + char *zRet; 86 + char *zData; 87 + zData = testDatasourceName(&pTest->defn); 88 + zRet = testMallocPrintf("data.%s.%s.rec=%d.%d.%d", 89 + zSystem, zData, bRecover, pTest->nRow, pTest->nVerify 90 + ); 91 + testFree(zData); 92 + return zRet; 93 +} 94 + 95 +int testControlDb(TestDb **ppDb){ 96 +#ifdef HAVE_KYOTOCABINET 97 + return tdb_open("kyotocabinet", "tmp.db", 1, ppDb); 98 +#else 99 + return tdb_open("sqlite3", "", 1, ppDb); 100 +#endif 101 +} 102 + 103 +void testDatasourceFetch( 104 + TestDb *pDb, /* Database handle */ 105 + Datasource *pData, 106 + int iKey, 107 + int *pRc /* IN/OUT: Error code */ 108 +){ 109 + void *pKey; int nKey; /* Database key to query for */ 110 + void *pVal; int nVal; /* Expected result of query */ 111 + 112 + testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); 113 + testFetch(pDb, pKey, nKey, pVal, nVal, pRc); 114 +} 115 + 116 +/* 117 +** This function is called to test that the contents of database pDb 118 +** are as expected. In this case, expected is defined as containing 119 +** key-value pairs iFirst through iLast, inclusive, from data source 120 +** pData. In other words, a loop like the following could be used to 121 +** construct a database with identical contents from scratch. 122 +** 123 +** for(i=iFirst; i<=iLast; i++){ 124 +** testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal); 125 +** // insert (pKey, nKey) -> (pVal, nVal) into database 126 +** } 127 +** 128 +** The key domain consists of keys 0 to (nRow-1), inclusive, from 129 +** data source pData. For both scan and lookup tests, keys are selected 130 +** pseudo-randomly from within this set. 131 +** 132 +** This function runs nLookupTest lookup tests and nScanTest scan tests. 133 +** 134 +** A lookup test consists of selecting a key from the domain and querying 135 +** pDb for it. The test fails if the presence of the key and, if present, 136 +** the associated value do not match the expectations defined above. 137 +** 138 +** A scan test involves selecting a key from the domain and running 139 +** the following queries: 140 +** 141 +** 1. Scan all keys equal to or greater than the key, in ascending order. 142 +** 2. Scan all keys equal to or smaller than the key, in descending order. 143 +** 144 +** Additionally, if nLookupTest is greater than zero, the following are 145 +** run once: 146 +** 147 +** 1. Scan all keys in the db, in ascending order. 148 +** 2. Scan all keys in the db, in descending order. 149 +** 150 +** As you would assume, the test fails if the returned values do not match 151 +** expectations. 152 +*/ 153 +void testDbContents( 154 + TestDb *pDb, /* Database handle being tested */ 155 + Datasource *pData, /* pDb contains data from here */ 156 + int nRow, /* Size of key domain */ 157 + int iFirst, /* Index of first key from pData in pDb */ 158 + int iLast, /* Index of last key from pData in pDb */ 159 + int nLookupTest, /* Number of lookup tests to run */ 160 + int nScanTest, /* Number of scan tests to run */ 161 + int *pRc /* IN/OUT: Error code */ 162 +){ 163 + int j; 164 + int rc = *pRc; 165 + 166 + if( rc==0 && nScanTest ){ 167 + TestDb *pDb2 = 0; 168 + 169 + /* Open a control db (i.e. one that we assume works) */ 170 + rc = testControlDb(&pDb2); 171 + 172 + for(j=iFirst; rc==0 && j<=iLast; j++){ 173 + void *pKey; int nKey; /* Database key to insert */ 174 + void *pVal; int nVal; /* Database value to insert */ 175 + testDatasourceEntry(pData, j, &pKey, &nKey, &pVal, &nVal); 176 + rc = tdb_write(pDb2, pKey, nKey, pVal, nVal); 177 + } 178 + 179 + if( rc==0 ){ 180 + int iKey1; 181 + int iKey2; 182 + void *pKey1; int nKey1; /* Start key */ 183 + void *pKey2; int nKey2; /* Final key */ 184 + 185 + iKey1 = testPrngValue((iFirst<<8) + (iLast<<16)) % nRow; 186 + iKey2 = testPrngValue((iLast<<8) + (iFirst<<16)) % nRow; 187 + testDatasourceEntry(pData, iKey1, &pKey2, &nKey1, 0, 0); 188 + pKey1 = testMalloc(nKey1+1); 189 + memcpy(pKey1, pKey2, nKey1+1); 190 + testDatasourceEntry(pData, iKey2, &pKey2, &nKey2, 0, 0); 191 + 192 + testScanCompare(pDb2, pDb, 0, 0, 0, 0, 0, &rc); 193 + testScanCompare(pDb2, pDb, 0, 0, 0, pKey2, nKey2, &rc); 194 + testScanCompare(pDb2, pDb, 0, pKey1, nKey1, 0, 0, &rc); 195 + testScanCompare(pDb2, pDb, 0, pKey1, nKey1, pKey2, nKey2, &rc); 196 + testScanCompare(pDb2, pDb, 1, 0, 0, 0, 0, &rc); 197 + testScanCompare(pDb2, pDb, 1, 0, 0, pKey2, nKey2, &rc); 198 + testScanCompare(pDb2, pDb, 1, pKey1, nKey1, 0, 0, &rc); 199 + testScanCompare(pDb2, pDb, 1, pKey1, nKey1, pKey2, nKey2, &rc); 200 + testFree(pKey1); 201 + } 202 + tdb_close(pDb2); 203 + } 204 + 205 + /* Test some lookups. */ 206 + for(j=0; rc==0 && j<nLookupTest; j++){ 207 + int iKey; /* Datasource key to test */ 208 + void *pKey; int nKey; /* Database key to query for */ 209 + void *pVal; int nVal; /* Expected result of query */ 210 + 211 + if( nLookupTest>=nRow ){ 212 + iKey = j; 213 + }else{ 214 + iKey = testPrngValue(j + (iFirst<<8) + (iLast<<16)) % nRow; 215 + } 216 + 217 + testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); 218 + if( iFirst>iKey || iKey>iLast ){ 219 + pVal = 0; 220 + nVal = -1; 221 + } 222 + 223 + testFetch(pDb, pKey, nKey, pVal, nVal, &rc); 224 + } 225 + 226 + *pRc = rc; 227 +} 228 + 229 +/* 230 +** This function should be called during long running test cases to output 231 +** the progress dots (...) to stdout. 232 +*/ 233 +void testCaseProgress(int i, int n, int nDot, int *piDot){ 234 + int iDot = *piDot; 235 + while( iDot < ( ((nDot*2+1) * i) / (n*2) ) ){ 236 + printf("."); 237 + fflush(stdout); 238 + iDot++; 239 + } 240 + *piDot = iDot; 241 +} 242 + 243 +int testCaseNDot(void){ return 20; } 244 + 245 +#if 0 246 +static void printScanCb( 247 + void *pCtx, void *pKey, int nKey, void *pVal, int nVal 248 +){ 249 + printf("%s\n", (char *)pKey); 250 + fflush(stdout); 251 +} 252 +#endif 253 + 254 +void testReopenRecover(TestDb **ppDb, int *pRc){ 255 + if( *pRc==0 ){ 256 + const char *zLib = tdb_library_name(*ppDb); 257 + const char *zDflt = tdb_default_db(zLib); 258 + testCopyLsmdb(zDflt, "bak.db"); 259 + testClose(ppDb); 260 + testCopyLsmdb("bak.db", zDflt); 261 + *pRc = tdb_open(zLib, 0, 0, ppDb); 262 + } 263 +} 264 + 265 + 266 +static void doDataTest1( 267 + const char *zSystem, /* Database system to test */ 268 + int bRecover, 269 + Datatest1 *p, /* Structure containing test parameters */ 270 + int *pRc /* OUT: Error code */ 271 +){ 272 + int i; 273 + int iDot; 274 + int rc = LSM_OK; 275 + Datasource *pData; 276 + TestDb *pDb; 277 + 278 + /* Start the test case, open a database and allocate the datasource. */ 279 + pDb = testOpen(zSystem, 1, &rc); 280 + pData = testDatasourceNew(&p->defn); 281 + 282 + i = 0; 283 + iDot = 0; 284 + while( rc==LSM_OK && i<p->nRow ){ 285 + 286 + /* Insert some data */ 287 + testWriteDatasourceRange(pDb, pData, i, p->nVerify, &rc); 288 + i += p->nVerify; 289 + 290 + /* Check that the db content is correct. */ 291 + testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc); 292 + 293 + if( bRecover ){ 294 + testReopenRecover(&pDb, &rc); 295 + }else{ 296 + testReopen(&pDb, &rc); 297 + } 298 + 299 + /* Check that the db content is still correct. */ 300 + testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc); 301 + 302 + /* Update the progress dots... */ 303 + testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot); 304 + } 305 + 306 + i = 0; 307 + iDot = 0; 308 + while( rc==LSM_OK && i<p->nRow ){ 309 + 310 + /* Delete some entries */ 311 + testDeleteDatasourceRange(pDb, pData, i, p->nVerify, &rc); 312 + i += p->nVerify; 313 + 314 + /* Check that the db content is correct. */ 315 + testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc); 316 + 317 + /* Close and reopen the database. */ 318 + if( bRecover ){ 319 + testReopenRecover(&pDb, &rc); 320 + }else{ 321 + testReopen(&pDb, &rc); 322 + } 323 + 324 + /* Check that the db content is still correct. */ 325 + testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc); 326 + 327 + /* Update the progress dots... */ 328 + testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot); 329 + } 330 + 331 + /* Free the datasource, close the database and finish the test case. */ 332 + testDatasourceFree(pData); 333 + tdb_close(pDb); 334 + testCaseFinish(rc); 335 + *pRc = rc; 336 +} 337 + 338 + 339 +void test_data_1( 340 + const char *zSystem, /* Database system name */ 341 + const char *zPattern, /* Run test cases that match this pattern */ 342 + int *pRc /* IN/OUT: Error code */ 343 +){ 344 + Datatest1 aTest[] = { 345 + { {DATA_RANDOM, 500,600, 1000,2000}, 1000, 100, 10, 0}, 346 + { {DATA_RANDOM, 20,25, 100,200}, 1000, 250, 1000, 1}, 347 + { {DATA_RANDOM, 8,10, 100,200}, 1000, 250, 1000, 1}, 348 + { {DATA_RANDOM, 8,10, 10,20}, 1000, 250, 1000, 1}, 349 + { {DATA_RANDOM, 8,10, 1000,2000}, 1000, 250, 1000, 1}, 350 + { {DATA_RANDOM, 8,100, 10000,20000}, 100, 25, 100, 1}, 351 + { {DATA_RANDOM, 80,100, 10,20}, 1000, 250, 1000, 1}, 352 + { {DATA_RANDOM, 5000,6000, 10,20}, 100, 25, 100, 1}, 353 + { {DATA_SEQUENTIAL, 5,10, 10,20}, 1000, 250, 1000, 1}, 354 + { {DATA_SEQUENTIAL, 5,10, 100,200}, 1000, 250, 1000, 1}, 355 + { {DATA_SEQUENTIAL, 5,10, 1000,2000}, 1000, 250, 1000, 1}, 356 + { {DATA_SEQUENTIAL, 5,100, 10000,20000}, 100, 25, 100, 1}, 357 + { {DATA_RANDOM, 10,10, 100,100}, 100000, 1000, 100, 0}, 358 + { {DATA_SEQUENTIAL, 10,10, 100,100}, 100000, 1000, 100, 0}, 359 + }; 360 + 361 + int i; 362 + int bRecover; 363 + 364 + for(bRecover=0; bRecover<2; bRecover++){ 365 + if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break; 366 + for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){ 367 + char *zName = getName(zSystem, bRecover, &aTest[i]); 368 + if( testCaseBegin(pRc, zPattern, "%s", zName) ){ 369 + doDataTest1(zSystem, bRecover, &aTest[i], pRc); 370 + } 371 + testFree(zName); 372 + } 373 + } 374 +} 375 + 376 +void testCompareDb( 377 + Datasource *pData, 378 + int nData, 379 + int iSeed, 380 + TestDb *pControl, 381 + TestDb *pDb, 382 + int *pRc 383 +){ 384 + int i; 385 + 386 + static int nCall = 0; 387 + nCall++; 388 + 389 + testScanCompare(pControl, pDb, 0, 0, 0, 0, 0, pRc); 390 + testScanCompare(pControl, pDb, 1, 0, 0, 0, 0, pRc); 391 + 392 + if( *pRc==0 ){ 393 + int iKey1; 394 + int iKey2; 395 + void *pKey1; int nKey1; /* Start key */ 396 + void *pKey2; int nKey2; /* Final key */ 397 + 398 + iKey1 = testPrngValue(iSeed) % nData; 399 + iKey2 = testPrngValue(iSeed+1) % nData; 400 + testDatasourceEntry(pData, iKey1, &pKey2, &nKey1, 0, 0); 401 + pKey1 = testMalloc(nKey1+1); 402 + memcpy(pKey1, pKey2, nKey1+1); 403 + testDatasourceEntry(pData, iKey2, &pKey2, &nKey2, 0, 0); 404 + 405 + testScanCompare(pControl, pDb, 0, 0, 0, pKey2, nKey2, pRc); 406 + testScanCompare(pControl, pDb, 0, pKey1, nKey1, 0, 0, pRc); 407 + testScanCompare(pControl, pDb, 0, pKey1, nKey1, pKey2, nKey2, pRc); 408 + testScanCompare(pControl, pDb, 1, 0, 0, pKey2, nKey2, pRc); 409 + testScanCompare(pControl, pDb, 1, pKey1, nKey1, 0, 0, pRc); 410 + testScanCompare(pControl, pDb, 1, pKey1, nKey1, pKey2, nKey2, pRc); 411 + testFree(pKey1); 412 + } 413 + 414 + for(i=0; i<nData && *pRc==0; i++){ 415 + void *pKey; int nKey; 416 + testDatasourceEntry(pData, i, &pKey, &nKey, 0, 0); 417 + testFetchCompare(pControl, pDb, pKey, nKey, pRc); 418 + } 419 +} 420 + 421 +static void doDataTest2( 422 + const char *zSystem, /* Database system to test */ 423 + int bRecover, 424 + Datatest2 *p, /* Structure containing test parameters */ 425 + int *pRc /* OUT: Error code */ 426 +){ 427 + TestDb *pDb; 428 + TestDb *pControl; 429 + Datasource *pData; 430 + int i; 431 + int rc = LSM_OK; 432 + int iDot = 0; 433 + 434 + /* Start the test case, open a database and allocate the datasource. */ 435 + pDb = testOpen(zSystem, 1, &rc); 436 + pData = testDatasourceNew(&p->defn); 437 + rc = testControlDb(&pControl); 438 + 439 + if( tdb_lsm(pDb) ){ 440 + int nBuf = 32 * 1024 * 1024; 441 + lsm_config(tdb_lsm(pDb), LSM_CONFIG_AUTOFLUSH, &nBuf); 442 + } 443 + 444 + for(i=0; rc==0 && i<p->nIter; i++){ 445 + void *pKey1; int nKey1; 446 + void *pKey2; int nKey2; 447 + int ii; 448 + int nRange = MIN(p->nIter*p->nWrite, p->nRange); 449 + 450 + for(ii=0; rc==0 && ii<p->nWrite; ii++){ 451 + int iKey = (i*p->nWrite + ii) % p->nRange; 452 + testWriteDatasource(pControl, pData, iKey, &rc); 453 + testWriteDatasource(pDb, pData, iKey, &rc); 454 + } 455 + 456 + testDatasourceEntry(pData, i+1000000, &pKey1, &nKey1, 0, 0); 457 + pKey1 = testMallocCopy(pKey1, nKey1); 458 + testDatasourceEntry(pData, i+2000000, &pKey2, &nKey2, 0, 0); 459 + 460 + testDeleteRange(pDb, pKey1, nKey1, pKey2, nKey2, &rc); 461 + testDeleteRange(pControl, pKey1, nKey1, pKey2, nKey2, &rc); 462 + testFree(pKey1); 463 + 464 + testCompareDb(pData, nRange, i, pControl, pDb, &rc); 465 + if( bRecover ){ 466 + testReopenRecover(&pDb, &rc); 467 + }else{ 468 + testReopen(&pDb, &rc); 469 + } 470 + testCompareDb(pData, nRange, i, pControl, pDb, &rc); 471 + 472 + /* Update the progress dots... */ 473 + testCaseProgress(i, p->nIter, testCaseNDot(), &iDot); 474 + } 475 + 476 + testClose(&pDb); 477 + testClose(&pControl); 478 + testDatasourceFree(pData); 479 + testCaseFinish(rc); 480 + *pRc = rc; 481 +} 482 + 483 +static char *getName2(const char *zSystem, int bRecover, Datatest2 *pTest){ 484 + char *zRet; 485 + char *zData; 486 + zData = testDatasourceName(&pTest->defn); 487 + zRet = testMallocPrintf("data2.%s.%s.rec=%d.%d.%d.%d", 488 + zSystem, zData, bRecover, pTest->nRange, pTest->nWrite, pTest->nIter 489 + ); 490 + testFree(zData); 491 + return zRet; 492 +} 493 + 494 +void test_data_2( 495 + const char *zSystem, /* Database system name */ 496 + const char *zPattern, /* Run test cases that match this pattern */ 497 + int *pRc /* IN/OUT: Error code */ 498 +){ 499 + Datatest2 aTest[] = { 500 + /* defn, nRange, nWrite, nIter */ 501 + { {DATA_RANDOM, 20,25, 100,200}, 10000, 10, 50 }, 502 + { {DATA_RANDOM, 20,25, 100,200}, 10000, 200, 50 }, 503 + { {DATA_RANDOM, 20,25, 100,200}, 100, 10, 1000 }, 504 + { {DATA_RANDOM, 20,25, 100,200}, 100, 200, 50 }, 505 + }; 506 + 507 + int i; 508 + int bRecover; 509 + 510 + for(bRecover=0; bRecover<2; bRecover++){ 511 + if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break; 512 + for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){ 513 + char *zName = getName2(zSystem, bRecover, &aTest[i]); 514 + if( testCaseBegin(pRc, zPattern, "%s", zName) ){ 515 + doDataTest2(zSystem, bRecover, &aTest[i], pRc); 516 + } 517 + testFree(zName); 518 + } 519 + } 520 +} 521 + 522 +/************************************************************************* 523 +** Test case data3.* 524 +*/ 525 + 526 +typedef struct Datatest3 Datatest3; 527 +struct Datatest3 { 528 + int nRange; /* Keys are between 1 and this value, incl. */ 529 + int nIter; /* Number of iterations */ 530 + int nWrite; /* Number of writes per iteration */ 531 + int nDelete; /* Number of deletes per iteration */ 532 + 533 + int nValMin; /* Minimum value size for writes */ 534 + int nValMax; /* Maximum value size for writes */ 535 +}; 536 + 537 +void testPutU32(u8 *aBuf, u32 iVal){ 538 + aBuf[0] = (iVal >> 24) & 0xFF; 539 + aBuf[1] = (iVal >> 16) & 0xFF; 540 + aBuf[2] = (iVal >> 8) & 0xFF; 541 + aBuf[3] = (iVal >> 0) & 0xFF; 542 +} 543 + 544 +void dt3PutKey(u8 *aBuf, int iKey){ 545 + assert( iKey<100000 && iKey>=0 ); 546 + sprintf((char *)aBuf, "%.5d", iKey); 547 +} 548 + 549 +static void doDataTest3( 550 + const char *zSystem, /* Database system to test */ 551 + Datatest3 *p, /* Structure containing test parameters */ 552 + int *pRc /* OUT: Error code */ 553 +){ 554 + int iDot = 0; 555 + int rc = *pRc; 556 + TestDb *pDb; 557 + u8 *abPresent; /* Array of boolean */ 558 + char *aVal; /* Buffer to hold values */ 559 + int i; 560 + u32 iSeq = 10; /* prng counter */ 561 + 562 + abPresent = (u8 *)testMalloc(p->nRange+1); 563 + aVal = (char *)testMalloc(p->nValMax+1); 564 + pDb = testOpen(zSystem, 1, &rc); 565 + 566 + for(i=0; i<p->nIter && rc==0; i++){ 567 + int ii; 568 + 569 + testCaseProgress(i, p->nIter, testCaseNDot(), &iDot); 570 + 571 + /* Perform nWrite inserts */ 572 + for(ii=0; ii<p->nWrite; ii++){ 573 + u8 aKey[6]; 574 + u32 iKey; 575 + int nVal; 576 + 577 + iKey = (testPrngValue(iSeq++) % p->nRange) + 1; 578 + nVal = (testPrngValue(iSeq++) % (p->nValMax - p->nValMin)) + p->nValMin; 579 + testPrngString(testPrngValue(iSeq++), aVal, nVal); 580 + dt3PutKey(aKey, iKey); 581 + 582 + testWrite(pDb, aKey, sizeof(aKey)-1, aVal, nVal, &rc); 583 + abPresent[iKey] = 1; 584 + } 585 + 586 + /* Perform nDelete deletes */ 587 + for(ii=0; ii<p->nDelete; ii++){ 588 + u8 aKey1[6]; 589 + u8 aKey2[6]; 590 + u32 iKey; 591 + 592 + iKey = (testPrngValue(iSeq++) % p->nRange) + 1; 593 + dt3PutKey(aKey1, iKey-1); 594 + dt3PutKey(aKey2, iKey+1); 595 + 596 + testDeleteRange(pDb, aKey1, sizeof(aKey1)-1, aKey2, sizeof(aKey2)-1, &rc); 597 + abPresent[iKey] = 0; 598 + } 599 + 600 + testReopen(&pDb, &rc); 601 + 602 + for(ii=1; rc==0 && ii<=p->nRange; ii++){ 603 + int nDbVal; 604 + void *pDbVal; 605 + u8 aKey[6]; 606 + int dbrc; 607 + 608 + dt3PutKey(aKey, ii); 609 + dbrc = tdb_fetch(pDb, aKey, sizeof(aKey)-1, &pDbVal, &nDbVal); 610 + testCompareInt(0, dbrc, &rc); 611 + 612 + if( abPresent[ii] ){ 613 + testCompareInt(1, (nDbVal>0), &rc); 614 + }else{ 615 + testCompareInt(1, (nDbVal<0), &rc); 616 + } 617 + } 618 + } 619 + 620 + testClose(&pDb); 621 + testCaseFinish(rc); 622 + *pRc = rc; 623 +} 624 + 625 +static char *getName3(const char *zSystem, Datatest3 *p){ 626 + return testMallocPrintf("data3.%s.%d.%d.%d.%d.(%d..%d)", 627 + zSystem, p->nRange, p->nIter, p->nWrite, p->nDelete, 628 + p->nValMin, p->nValMax 629 + ); 630 +} 631 + 632 +void test_data_3( 633 + const char *zSystem, /* Database system name */ 634 + const char *zPattern, /* Run test cases that match this pattern */ 635 + int *pRc /* IN/OUT: Error code */ 636 +){ 637 + Datatest3 aTest[] = { 638 + /* nRange, nIter, nWrite, nDelete, nValMin, nValMax */ 639 + { 100, 1000, 5, 5, 50, 100 }, 640 + { 100, 1000, 2, 2, 5, 10 }, 641 + }; 642 + 643 + int i; 644 + 645 + for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){ 646 + char *zName = getName3(zSystem, &aTest[i]); 647 + if( testCaseBegin(pRc, zPattern, "%s", zName) ){ 648 + doDataTest3(zSystem, &aTest[i], pRc); 649 + } 650 + testFree(zName); 651 + } 652 +} 653 + 654 +
Added ext/lsm1/lsm-test/lsmtest2.c.
1 + 2 +/* 3 +** This file contains tests related to recovery following application 4 +** and system crashes (power failures) while writing to the database. 5 +*/ 6 + 7 +#include "lsmtest.h" 8 + 9 +/* 10 +** Structure used by testCksumDatabase() to accumulate checksum values in. 11 +*/ 12 +typedef struct Cksum Cksum; 13 +struct Cksum { 14 + int nRow; 15 + int cksum1; 16 + int cksum2; 17 +}; 18 + 19 +/* 20 +** tdb_scan() callback used by testCksumDatabase() 21 +*/ 22 +static void scanCksumDb( 23 + void *pCtx, 24 + void *pKey, int nKey, 25 + void *pVal, int nVal 26 +){ 27 + Cksum *p = (Cksum *)pCtx; 28 + int i; 29 + 30 + p->nRow++; 31 + for(i=0; i<nKey; i++){ 32 + p->cksum1 += ((u8 *)pKey)[i]; 33 + p->cksum2 += p->cksum1; 34 + } 35 + for(i=0; i<nVal; i++){ 36 + p->cksum1 += ((u8 *)pVal)[i]; 37 + p->cksum2 += p->cksum1; 38 + } 39 +} 40 + 41 +/* 42 +** tdb_scan() callback used by testCountDatabase() 43 +*/ 44 +static void scanCountDb( 45 + void *pCtx, 46 + void *pKey, int nKey, 47 + void *pVal, int nVal 48 +){ 49 + Cksum *p = (Cksum *)pCtx; 50 + p->nRow++; 51 + 52 + unused_parameter(pKey); 53 + unused_parameter(nKey); 54 + unused_parameter(pVal); 55 + unused_parameter(nVal); 56 +} 57 + 58 + 59 +/* 60 +** Iterate through the entire contents of database pDb. Write a checksum 61 +** string based on the db contents into buffer zOut before returning. A 62 +** checksum string is at most 29 (TEST_CKSUM_BYTES) bytes in size: 63 +** 64 +** * 32-bit integer (10 bytes) 65 +** * 1 space (1 byte) 66 +** * 32-bit hex (8 bytes) 67 +** * 1 space (1 byte) 68 +** * 32-bit hex (8 bytes) 69 +** * nul-terminator (1 byte) 70 +** 71 +** The number of entries in the database is returned. 72 +*/ 73 +int testCksumDatabase( 74 + TestDb *pDb, /* Database handle */ 75 + char *zOut /* Buffer to write checksum to */ 76 +){ 77 + Cksum cksum; 78 + memset(&cksum, 0, sizeof(Cksum)); 79 + tdb_scan(pDb, (void *)&cksum, 0, 0, 0, 0, 0, scanCksumDb); 80 + sprintf(zOut, "%d %x %x", 81 + cksum.nRow, (u32)cksum.cksum1, (u32)cksum.cksum2 82 + ); 83 + assert( strlen(zOut)<TEST_CKSUM_BYTES ); 84 + return cksum.nRow; 85 +} 86 + 87 +int testCountDatabase(TestDb *pDb){ 88 + Cksum cksum; 89 + memset(&cksum, 0, sizeof(Cksum)); 90 + tdb_scan(pDb, (void *)&cksum, 0, 0, 0, 0, 0, scanCountDb); 91 + return cksum.nRow; 92 +} 93 + 94 +/* 95 +** This function is a no-op if *pRc is not 0 when it is called. 96 +** 97 +** Otherwise, the two nul-terminated strings z1 and z1 are compared. If 98 +** they are the same, the function returns without doing anything. Otherwise, 99 +** an error message is printed, *pRc is set to 1 and the test_failed() 100 +** function called. 101 +*/ 102 +void testCompareStr(const char *z1, const char *z2, int *pRc){ 103 + if( *pRc==0 ){ 104 + if( strcmp(z1, z2) ){ 105 + testPrintError("testCompareStr: \"%s\" != \"%s\"\n", z1, z2); 106 + *pRc = 1; 107 + test_failed(); 108 + } 109 + } 110 +} 111 + 112 +/* 113 +** This function is a no-op if *pRc is not 0 when it is called. 114 +** 115 +** Otherwise, the two integers i1 and i2 are compared. If they are equal, 116 +** the function returns without doing anything. Otherwise, an error message 117 +** is printed, *pRc is set to 1 and the test_failed() function called. 118 +*/ 119 +void testCompareInt(int i1, int i2, int *pRc){ 120 + if( *pRc==0 && i1!=i2 ){ 121 + testPrintError("testCompareInt: %d != %d\n", i1, i2); 122 + *pRc = 1; 123 + test_failed(); 124 + } 125 +} 126 + 127 +void testCaseStart(int *pRc, char *zFmt, ...){ 128 + va_list ap; 129 + va_start(ap, zFmt); 130 + vprintf(zFmt, ap); 131 + printf(" ..."); 132 + va_end(ap); 133 + *pRc = 0; 134 + fflush(stdout); 135 +} 136 + 137 +/* 138 +** This function is a no-op if *pRc is non-zero when it is called. Zero 139 +** is returned in this case. 140 +** 141 +** Otherwise, the zFmt (a printf style format string) and following arguments 142 +** are used to create a test case name. If zPattern is NULL or a glob pattern 143 +** that matches the test case name, 1 is returned and the test case started. 144 +** Otherwise, zero is returned and the test case does not start. 145 +*/ 146 +int testCaseBegin(int *pRc, const char *zPattern, const char *zFmt, ...){ 147 + int res = 0; 148 + if( *pRc==0 ){ 149 + char *zTest; 150 + va_list ap; 151 + 152 + va_start(ap, zFmt); 153 + zTest = testMallocVPrintf(zFmt, ap); 154 + va_end(ap); 155 + if( zPattern==0 || testGlobMatch(zPattern, zTest) ){ 156 + printf("%-50s ...", zTest); 157 + res = 1; 158 + } 159 + testFree(zTest); 160 + fflush(stdout); 161 + } 162 + 163 + return res; 164 +} 165 + 166 +void testCaseFinish(int rc){ 167 + if( rc==0 ){ 168 + printf("Ok\n"); 169 + }else{ 170 + printf("FAILED\n"); 171 + } 172 + fflush(stdout); 173 +} 174 + 175 +void testCaseSkip(){ 176 + printf("Skipped\n"); 177 +} 178 + 179 +void testSetupSavedLsmdb( 180 + const char *zCfg, 181 + const char *zFile, 182 + Datasource *pData, 183 + int nRow, 184 + int *pRc 185 +){ 186 + if( *pRc==0 ){ 187 + int rc; 188 + TestDb *pDb; 189 + rc = tdb_lsm_open(zCfg, zFile, 1, &pDb); 190 + if( rc==0 ){ 191 + testWriteDatasourceRange(pDb, pData, 0, nRow, &rc); 192 + testClose(&pDb); 193 + if( rc==0 ) testSaveDb(zFile, "log"); 194 + } 195 + *pRc = rc; 196 + } 197 +} 198 + 199 +/* 200 +** This function is a no-op if *pRc is non-zero when it is called. 201 +** 202 +** Open the LSM database identified by zFile and compute its checksum 203 +** (a string, as returned by testCksumDatabase()). If the checksum is 204 +** identical to zExpect1 or, if it is not NULL, zExpect2, the test passes. 205 +** Otherwise, print an error message and set *pRc to 1. 206 +*/ 207 +static void testCompareCksumLsmdb( 208 + const char *zFile, /* Path to LSM database */ 209 + int bCompress, /* True if db is compressed */ 210 + const char *zExpect1, /* Expected checksum 1 */ 211 + const char *zExpect2, /* Expected checksum 2 (or NULL) */ 212 + int *pRc /* IN/OUT: Test case error code */ 213 +){ 214 + if( *pRc==0 ){ 215 + char zCksum[TEST_CKSUM_BYTES]; 216 + TestDb *pDb; 217 + 218 + *pRc = tdb_lsm_open((bCompress?"compression=1 mmap=0":""), zFile, 0, &pDb); 219 + testCksumDatabase(pDb, zCksum); 220 + testClose(&pDb); 221 + 222 + if( *pRc==0 ){ 223 + int r1 = 0; 224 + int r2 = -1; 225 + 226 + r1 = strcmp(zCksum, zExpect1); 227 + if( zExpect2 ) r2 = strcmp(zCksum, zExpect2); 228 + if( r1 && r2 ){ 229 + if( zExpect2 ){ 230 + testPrintError("testCompareCksumLsmdb: \"%s\" != (\"%s\" OR \"%s\")", 231 + zCksum, zExpect1, zExpect2 232 + ); 233 + }else{ 234 + testPrintError("testCompareCksumLsmdb: \"%s\" != \"%s\"", 235 + zCksum, zExpect1 236 + ); 237 + } 238 + *pRc = 1; 239 + test_failed(); 240 + } 241 + } 242 + } 243 +} 244 + 245 +#if 0 /* not used */ 246 +static void testCompareCksumBtdb( 247 + const char *zFile, /* Path to LSM database */ 248 + const char *zExpect1, /* Expected checksum 1 */ 249 + const char *zExpect2, /* Expected checksum 2 (or NULL) */ 250 + int *pRc /* IN/OUT: Test case error code */ 251 +){ 252 + if( *pRc==0 ){ 253 + char zCksum[TEST_CKSUM_BYTES]; 254 + TestDb *pDb; 255 + 256 + *pRc = tdb_open("bt", zFile, 0, &pDb); 257 + testCksumDatabase(pDb, zCksum); 258 + testClose(&pDb); 259 + 260 + if( *pRc==0 ){ 261 + int r1 = 0; 262 + int r2 = -1; 263 + 264 + r1 = strcmp(zCksum, zExpect1); 265 + if( zExpect2 ) r2 = strcmp(zCksum, zExpect2); 266 + if( r1 && r2 ){ 267 + if( zExpect2 ){ 268 + testPrintError("testCompareCksumLsmdb: \"%s\" != (\"%s\" OR \"%s\")", 269 + zCksum, zExpect1, zExpect2 270 + ); 271 + }else{ 272 + testPrintError("testCompareCksumLsmdb: \"%s\" != \"%s\"", 273 + zCksum, zExpect1 274 + ); 275 + } 276 + *pRc = 1; 277 + test_failed(); 278 + } 279 + } 280 + } 281 +} 282 +#endif /* not used */ 283 + 284 +/* Above this point are reusable test routines. Not clear that they 285 +** should really be in this file. 286 +*************************************************************************/ 287 + 288 +/* 289 +** This test verifies that if a system crash occurs while doing merge work 290 +** on the db, no data is lost. 291 +*/ 292 +static void crash_test1(int bCompress, int *pRc){ 293 + const char *DBNAME = "testdb.lsm"; 294 + const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 200, 200}; 295 + 296 + const int nRow = 5000; /* Database size */ 297 + const int nIter = 200; /* Number of test iterations */ 298 + const int nWork = 20; /* Maximum lsm_work() calls per iteration */ 299 + const int nPage = 15; /* Pages per lsm_work call */ 300 + 301 + int i; 302 + int iDot = 0; 303 + Datasource *pData; 304 + CksumDb *pCksumDb; 305 + TestDb *pDb; 306 + char *zCfg; 307 + 308 + const char *azConfig[2] = { 309 + "page_size=1024 block_size=65536 autoflush=16384 safety=2 mmap=0", 310 + "page_size=1024 block_size=65536 autoflush=16384 safety=2 " 311 + " compression=1 mmap=0" 312 + }; 313 + assert( bCompress==0 || bCompress==1 ); 314 + 315 + /* Allocate datasource. And calculate the expected checksums. */ 316 + pData = testDatasourceNew(&defn); 317 + pCksumDb = testCksumArrayNew(pData, nRow, nRow, 1); 318 + 319 + /* Setup and save the initial database. */ 320 + 321 + zCfg = testMallocPrintf("%s automerge=7", azConfig[bCompress]); 322 + testSetupSavedLsmdb(zCfg, DBNAME, pData, 5000, pRc); 323 + testFree(zCfg); 324 + 325 + for(i=0; i<nIter && *pRc==0; i++){ 326 + int iWork; 327 + int testrc = 0; 328 + 329 + testCaseProgress(i, nIter, testCaseNDot(), &iDot); 330 + 331 + /* Restore and open the database. */ 332 + testRestoreDb(DBNAME, "log"); 333 + testrc = tdb_lsm_open(azConfig[bCompress], DBNAME, 0, &pDb); 334 + assert( testrc==0 ); 335 + 336 + /* Call lsm_work() on the db */ 337 + tdb_lsm_prepare_sync_crash(pDb, 1 + (i%(nWork*2))); 338 + for(iWork=0; testrc==0 && iWork<nWork; iWork++){ 339 + int nWrite = 0; 340 + lsm_db *db = tdb_lsm(pDb); 341 + testrc = lsm_work(db, 0, nPage, &nWrite); 342 + /* assert( testrc!=0 || nWrite>0 ); */ 343 + if( testrc==0 ) testrc = lsm_checkpoint(db, 0); 344 + } 345 + tdb_close(pDb); 346 + 347 + /* Check that the database content is still correct */ 348 + testCompareCksumLsmdb(DBNAME, 349 + bCompress, testCksumArrayGet(pCksumDb, nRow), 0, pRc); 350 + } 351 + 352 + testCksumArrayFree(pCksumDb); 353 + testDatasourceFree(pData); 354 +} 355 + 356 +/* 357 +** This test verifies that if a system crash occurs while committing a 358 +** transaction to the log file, no earlier transactions are lost or damaged. 359 +*/ 360 +static void crash_test2(int bCompress, int *pRc){ 361 + const char *DBNAME = "testdb.lsm"; 362 + const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 1000, 1000}; 363 + 364 + const int nIter = 200; 365 + const int nInsert = 20; 366 + 367 + int i; 368 + int iDot = 0; 369 + Datasource *pData; 370 + CksumDb *pCksumDb; 371 + TestDb *pDb; 372 + 373 + /* Allocate datasource. And calculate the expected checksums. */ 374 + pData = testDatasourceNew(&defn); 375 + pCksumDb = testCksumArrayNew(pData, 100, 100+nInsert, 1); 376 + 377 + /* Setup and save the initial database. */ 378 + testSetupSavedLsmdb("", DBNAME, pData, 100, pRc); 379 + 380 + for(i=0; i<nIter && *pRc==0; i++){ 381 + int iIns; 382 + int testrc = 0; 383 + 384 + testCaseProgress(i, nIter, testCaseNDot(), &iDot); 385 + 386 + /* Restore and open the database. */ 387 + testRestoreDb(DBNAME, "log"); 388 + testrc = tdb_lsm_open("safety=2", DBNAME, 0, &pDb); 389 + assert( testrc==0 ); 390 + 391 + /* Insert nInsert records into the database. Crash midway through. */ 392 + tdb_lsm_prepare_sync_crash(pDb, 1 + (i%(nInsert+2))); 393 + for(iIns=0; iIns<nInsert; iIns++){ 394 + void *pKey; int nKey; 395 + void *pVal; int nVal; 396 + 397 + testDatasourceEntry(pData, 100+iIns, &pKey, &nKey, &pVal, &nVal); 398 + testrc = tdb_write(pDb, pKey, nKey, pVal, nVal); 399 + if( testrc ) break; 400 + } 401 + tdb_close(pDb); 402 + 403 + /* Check that no data was lost when the system crashed. */ 404 + testCompareCksumLsmdb(DBNAME, bCompress, 405 + testCksumArrayGet(pCksumDb, 100 + iIns), 406 + testCksumArrayGet(pCksumDb, 100 + iIns + 1), 407 + pRc 408 + ); 409 + } 410 + 411 + testDatasourceFree(pData); 412 + testCksumArrayFree(pCksumDb); 413 +} 414 + 415 + 416 +/* 417 +** This test verifies that if a system crash occurs when checkpointing 418 +** the database, data is not lost (assuming that any writes not synced 419 +** to the db have been synced into the log file). 420 +*/ 421 +static void crash_test3(int bCompress, int *pRc){ 422 + const char *DBNAME = "testdb.lsm"; 423 + const int nIter = 100; 424 + const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 1000, 1000}; 425 + 426 + int i; 427 + int iDot = 0; 428 + Datasource *pData; 429 + CksumDb *pCksumDb; 430 + TestDb *pDb; 431 + 432 + /* Allocate datasource. And calculate the expected checksums. */ 433 + pData = testDatasourceNew(&defn); 434 + pCksumDb = testCksumArrayNew(pData, 110, 150, 10); 435 + 436 + /* Setup and save the initial database. */ 437 + testSetupSavedLsmdb("", DBNAME, pData, 100, pRc); 438 + 439 + for(i=0; i<nIter && *pRc==0; i++){ 440 + int iOpen; 441 + testCaseProgress(i, nIter, testCaseNDot(), &iDot); 442 + testRestoreDb(DBNAME, "log"); 443 + 444 + for(iOpen=0; iOpen<5; iOpen++){ 445 + /* Open the database. Insert 10 more records. */ 446 + pDb = testOpen("lsm", 0, pRc); 447 + testWriteDatasourceRange(pDb, pData, 100+iOpen*10, 10, pRc); 448 + 449 + /* Schedule a crash simulation then close the db. */ 450 + tdb_lsm_prepare_sync_crash(pDb, 1 + (i%2)); 451 + tdb_close(pDb); 452 + 453 + /* Open the database and check that the crash did not cause any 454 + ** data loss. */ 455 + testCompareCksumLsmdb(DBNAME, bCompress, 456 + testCksumArrayGet(pCksumDb, 110 + iOpen*10), 0, 457 + pRc 458 + ); 459 + } 460 + } 461 + 462 + testDatasourceFree(pData); 463 + testCksumArrayFree(pCksumDb); 464 +} 465 + 466 +void do_crash_test(const char *zPattern, int *pRc){ 467 + struct Test { 468 + const char *zTest; 469 + void (*x)(int, int *); 470 + int bCompress; 471 + } aTest [] = { 472 + { "crash.lsm.1", crash_test1, 0 }, 473 +#ifdef HAVE_ZLIB 474 + { "crash.lsm_zip.1", crash_test1, 1 }, 475 +#endif 476 + { "crash.lsm.2", crash_test2, 0 }, 477 + { "crash.lsm.3", crash_test3, 0 }, 478 + }; 479 + int i; 480 + 481 + for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){ 482 + struct Test *p = &aTest[i]; 483 + if( testCaseBegin(pRc, zPattern, "%s", p->zTest) ){ 484 + p->x(p->bCompress, pRc); 485 + testCaseFinish(*pRc); 486 + } 487 + } 488 +}
Added ext/lsm1/lsm-test/lsmtest3.c.
1 + 2 + 3 +/* 4 +** This file contains tests related to the explicit rollback of database 5 +** transactions and sub-transactions. 6 +*/ 7 + 8 + 9 +/* 10 +** Repeat 2000 times (until the db contains 100,000 entries): 11 +** 12 +** 1. Open a transaction and insert 500 rows, opening a nested 13 +** sub-transaction each 100 rows. 14 +** 15 +** 2. Roll back to each sub-transaction savepoint. Check the database 16 +** checksum looks Ok. 17 +** 18 +** 3. Every second iteration, roll back the main transaction. Check the 19 +** db checksum is correct. Every other iteration, commit the main 20 +** transaction (increasing the size of the db by 100 rows). 21 +*/ 22 + 23 + 24 +#include "lsmtest.h" 25 + 26 +struct CksumDb { 27 + int nFirst; 28 + int nLast; 29 + int nStep; 30 + char **azCksum; 31 +}; 32 + 33 +CksumDb *testCksumArrayNew( 34 + Datasource *pData, 35 + int nFirst, 36 + int nLast, 37 + int nStep 38 +){ 39 + TestDb *pDb; 40 + CksumDb *pRet; 41 + int i; 42 + int nEntry; 43 + int rc = 0; 44 + 45 + assert( nLast>=nFirst && ((nLast-nFirst)%nStep)==0 ); 46 + 47 + pRet = malloc(sizeof(CksumDb)); 48 + memset(pRet, 0, sizeof(CksumDb)); 49 + pRet->nFirst = nFirst; 50 + pRet->nLast = nLast; 51 + pRet->nStep = nStep; 52 + nEntry = 1 + ((nLast - nFirst) / nStep); 53 + 54 + /* Allocate space so that azCksum is an array of nEntry pointers to 55 + ** buffers each TEST_CKSUM_BYTES in size. */ 56 + pRet->azCksum = (char **)malloc(nEntry * (sizeof(char *) + TEST_CKSUM_BYTES)); 57 + for(i=0; i<nEntry; i++){ 58 + char *pStart = (char *)(&pRet->azCksum[nEntry]); 59 + pRet->azCksum[i] = &pStart[i * TEST_CKSUM_BYTES]; 60 + } 61 + 62 + tdb_open("lsm", "tempdb.lsm", 1, &pDb); 63 + testWriteDatasourceRange(pDb, pData, 0, nFirst, &rc); 64 + for(i=0; i<nEntry; i++){ 65 + testCksumDatabase(pDb, pRet->azCksum[i]); 66 + if( i==nEntry ) break; 67 + testWriteDatasourceRange(pDb, pData, nFirst+i*nStep, nStep, &rc); 68 + } 69 + 70 + tdb_close(pDb); 71 + 72 + return pRet; 73 +} 74 + 75 +char *testCksumArrayGet(CksumDb *p, int nRow){ 76 + int i; 77 + assert( nRow>=p->nFirst ); 78 + assert( nRow<=p->nLast ); 79 + assert( ((nRow-p->nFirst) % p->nStep)==0 ); 80 + 81 + i = (nRow - p->nFirst) / p->nStep; 82 + return p->azCksum[i]; 83 +} 84 + 85 +void testCksumArrayFree(CksumDb *p){ 86 + free(p->azCksum); 87 + memset(p, 0x55, sizeof(*p)); 88 + free(p); 89 +} 90 + 91 +/* End of CksumDb code. 92 +**************************************************************************/ 93 + 94 +/* 95 +** Test utility function. Write key-value pair $i from datasource pData 96 +** into database pDb. 97 +*/ 98 +void testWriteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){ 99 + void *pKey; int nKey; 100 + void *pVal; int nVal; 101 + testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal); 102 + testWrite(pDb, pKey, nKey, pVal, nVal, pRc); 103 +} 104 + 105 +/* 106 +** Test utility function. Delete datasource pData key $i from database pDb. 107 +*/ 108 +void testDeleteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){ 109 + void *pKey; int nKey; 110 + testDatasourceEntry(pData, i, &pKey, &nKey, 0, 0); 111 + testDelete(pDb, pKey, nKey, pRc); 112 +} 113 + 114 +/* 115 +** This function inserts nWrite key/value pairs into database pDb - the 116 +** nWrite key value pairs starting at iFirst from data source pData. 117 +*/ 118 +void testWriteDatasourceRange( 119 + TestDb *pDb, /* Database to write to */ 120 + Datasource *pData, /* Data source to read values from */ 121 + int iFirst, /* Index of first key/value pair */ 122 + int nWrite, /* Number of key/value pairs to write */ 123 + int *pRc /* IN/OUT: Error code */ 124 +){ 125 + int i; 126 + for(i=0; i<nWrite; i++){ 127 + testWriteDatasource(pDb, pData, iFirst+i, pRc); 128 + } 129 +} 130 + 131 +void testDeleteDatasourceRange( 132 + TestDb *pDb, /* Database to write to */ 133 + Datasource *pData, /* Data source to read keys from */ 134 + int iFirst, /* Index of first key */ 135 + int nWrite, /* Number of keys to delete */ 136 + int *pRc /* IN/OUT: Error code */ 137 +){ 138 + int i; 139 + for(i=0; i<nWrite; i++){ 140 + testDeleteDatasource(pDb, pData, iFirst+i, pRc); 141 + } 142 +} 143 + 144 +static char *getName(const char *zSystem){ 145 + char *zRet; 146 + zRet = testMallocPrintf("rollback.%s", zSystem); 147 + return zRet; 148 +} 149 + 150 +static int rollback_test_1( 151 + const char *zSystem, 152 + Datasource *pData 153 +){ 154 + const int nRepeat = 100; 155 + 156 + TestDb *pDb; 157 + int rc; 158 + int i; 159 + CksumDb *pCksum; 160 + char *zName; 161 + 162 + zName = getName(zSystem); 163 + testCaseStart(&rc, zName); 164 + testFree(zName); 165 + 166 + pCksum = testCksumArrayNew(pData, 0, nRepeat*100, 100); 167 + pDb = 0; 168 + rc = tdb_open(zSystem, 0, 1, &pDb); 169 + if( pDb && tdb_transaction_support(pDb)==0 ){ 170 + testCaseSkip(); 171 + goto skip_rollback_test; 172 + } 173 + 174 + for(i=0; i<nRepeat && rc==0; i++){ 175 + char zCksum[TEST_CKSUM_BYTES]; 176 + int nCurrent = (((i+1)/2) * 100); 177 + int nDbRow; 178 + int iTrans; 179 + 180 + /* Check that the database is the expected size. */ 181 + nDbRow = testCountDatabase(pDb); 182 + testCompareInt(nCurrent, nDbRow, &rc); 183 + 184 + for(iTrans=2; iTrans<=6 && rc==0; iTrans++){ 185 + tdb_begin(pDb, iTrans); 186 + testWriteDatasourceRange(pDb, pData, nCurrent, 100, &rc); 187 + nCurrent += 100; 188 + } 189 + 190 + testCksumDatabase(pDb, zCksum); 191 + testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc); 192 + 193 + for(iTrans=6; iTrans>2 && rc==0; iTrans--){ 194 + tdb_rollback(pDb, iTrans); 195 + nCurrent -= 100; 196 + testCksumDatabase(pDb, zCksum); 197 + testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc); 198 + } 199 + 200 + if( i%2 ){ 201 + tdb_rollback(pDb, 0); 202 + nCurrent -= 100; 203 + testCksumDatabase(pDb, zCksum); 204 + testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc); 205 + }else{ 206 + tdb_commit(pDb, 0); 207 + } 208 + } 209 + testCaseFinish(rc); 210 + 211 + skip_rollback_test: 212 + tdb_close(pDb); 213 + testCksumArrayFree(pCksum); 214 + return rc; 215 +} 216 + 217 +void test_rollback( 218 + const char *zSystem, 219 + const char *zPattern, 220 + int *pRc 221 +){ 222 + if( *pRc==0 ){ 223 + int bRun = 1; 224 + 225 + if( zPattern ){ 226 + char *zName = getName(zSystem); 227 + bRun = testGlobMatch(zPattern, zName); 228 + testFree(zName); 229 + } 230 + 231 + if( bRun ){ 232 + DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 50, 100 }; 233 + Datasource *pData = testDatasourceNew(&defn); 234 + *pRc = rollback_test_1(zSystem, pData); 235 + testDatasourceFree(pData); 236 + } 237 + } 238 +}
Added ext/lsm1/lsm-test/lsmtest4.c.
1 + 2 +/* 3 +** This file contains test cases involving multiple database clients. 4 +*/ 5 + 6 +#include "lsmtest.h" 7 + 8 +/* 9 +** The following code implements test cases "mc1.*". 10 +** 11 +** This test case uses one writer and $nReader readers. All connections 12 +** are driven by a single thread. All connections are opened at the start 13 +** of the test and remain open until the test is finished. 14 +** 15 +** The test consists of $nStep steps. Each step the following is performed: 16 +** 17 +** 1. The writer inserts $nWriteStep records into the db. 18 +** 19 +** 2. The writer checks that the contents of the db are as expected. 20 +** 21 +** 3. Each reader that currently has an open read transaction also checks 22 +** that the contents of the db are as expected (according to the snapshot 23 +** the read transaction is reading - see below). 24 +** 25 +** After step 1, reader 1 opens a read transaction. After step 2, reader 26 +** 2 opens a read transaction, and so on. At step ($nReader+1), reader 1 27 +** closes the current read transaction and opens a new one. And so on. 28 +** The result is that at step N (for N > $nReader), there exists a reader 29 +** with an open read transaction reading the snapshot committed following 30 +** steps (N-$nReader-1) to N. 31 +*/ 32 +typedef struct Mctest Mctest; 33 +struct Mctest { 34 + DatasourceDefn defn; /* Datasource to use */ 35 + int nStep; /* Total number of steps in test */ 36 + int nWriteStep; /* Number of rows to insert each step */ 37 + int nReader; /* Number of read connections */ 38 +}; 39 +static void do_mc_test( 40 + const char *zSystem, /* Database system to test */ 41 + Mctest *pTest, 42 + int *pRc /* IN/OUT: return code */ 43 +){ 44 + const int nDomain = pTest->nStep * pTest->nWriteStep; 45 + Datasource *pData; /* Source of data */ 46 + TestDb *pDb; /* First database connection (writer) */ 47 + int iReader; /* Used to iterate through aReader */ 48 + int iStep; /* Current step in test */ 49 + int iDot = 0; /* Current step in test */ 50 + 51 + /* Array of reader connections */ 52 + struct Reader { 53 + TestDb *pDb; /* Connection handle */ 54 + int iLast; /* Current snapshot contains keys 0..iLast */ 55 + } *aReader; 56 + 57 + /* Create a data source */ 58 + pData = testDatasourceNew(&pTest->defn); 59 + 60 + /* Open the writer connection */ 61 + pDb = testOpen(zSystem, 1, pRc); 62 + 63 + /* Allocate aReader */ 64 + aReader = (struct Reader *)testMalloc(sizeof(aReader[0]) * pTest->nReader); 65 + for(iReader=0; iReader<pTest->nReader; iReader++){ 66 + aReader[iReader].pDb = testOpen(zSystem, 0, pRc); 67 + } 68 + 69 + for(iStep=0; iStep<pTest->nStep; iStep++){ 70 + int iLast; 71 + int iBegin; /* Start read trans using aReader[iBegin] */ 72 + 73 + /* Insert nWriteStep more records into the database */ 74 + int iFirst = iStep*pTest->nWriteStep; 75 + testWriteDatasourceRange(pDb, pData, iFirst, pTest->nWriteStep, pRc); 76 + 77 + /* Check that the db is Ok according to the writer */ 78 + iLast = (iStep+1) * pTest->nWriteStep - 1; 79 + testDbContents(pDb, pData, nDomain, 0, iLast, iLast, 1, pRc); 80 + 81 + /* Have reader (iStep % nReader) open a read transaction here. */ 82 + iBegin = (iStep % pTest->nReader); 83 + if( iBegin<iStep ) tdb_commit(aReader[iBegin].pDb, 0); 84 + tdb_begin(aReader[iBegin].pDb, 1); 85 + aReader[iBegin].iLast = iLast; 86 + 87 + /* Check that the db is Ok for each open reader */ 88 + for(iReader=0; iReader<pTest->nReader && aReader[iReader].iLast; iReader++){ 89 + iLast = aReader[iReader].iLast; 90 + testDbContents( 91 + aReader[iReader].pDb, pData, nDomain, 0, iLast, iLast, 1, pRc 92 + ); 93 + } 94 + 95 + /* Report progress */ 96 + testCaseProgress(iStep, pTest->nStep, testCaseNDot(), &iDot); 97 + } 98 + 99 + /* Close all readers */ 100 + for(iReader=0; iReader<pTest->nReader; iReader++){ 101 + testClose(&aReader[iReader].pDb); 102 + } 103 + testFree(aReader); 104 + 105 + /* Close the writer-connection and free the datasource */ 106 + testClose(&pDb); 107 + testDatasourceFree(pData); 108 +} 109 + 110 + 111 +void test_mc( 112 + const char *zSystem, /* Database system name */ 113 + const char *zPattern, /* Run test cases that match this pattern */ 114 + int *pRc /* IN/OUT: Error code */ 115 +){ 116 + int i; 117 + Mctest aTest[] = { 118 + { { TEST_DATASOURCE_RANDOM, 10,10, 100,100 }, 100, 10, 5 }, 119 + }; 120 + 121 + for(i=0; i<ArraySize(aTest); i++){ 122 + if( testCaseBegin(pRc, zPattern, "mc1.%s.%d", zSystem, i) ){ 123 + do_mc_test(zSystem, &aTest[i], pRc); 124 + testCaseFinish(*pRc); 125 + } 126 + } 127 +}
Added ext/lsm1/lsm-test/lsmtest5.c.
1 + 2 +/* 3 +** This file is broken into three semi-autonomous parts: 4 +** 5 +** 1. The database functions. 6 +** 2. The thread wrappers. 7 +** 3. The implementation of the mt1.* tests. 8 +*/ 9 + 10 +/************************************************************************* 11 +** DATABASE CONTENTS: 12 +** 13 +** The database contains up to N key/value pairs, where N is some large 14 +** number (say 10,000,000). Keys are integer values between 0 and (N-1). 15 +** The value associated with each key is a pseudo-random blob of data. 16 +** 17 +** Key/value pair keys are encoded as the two bytes "k." followed by a 18 +** 10-digit decimal number. i.e. key 45 -> "k.0000000045". 19 +** 20 +** As well as the key/value pairs, the database also contains checksum 21 +** entries. The checksums form a hierarchy - for every F key/value 22 +** entries there is one level 1 checksum. And for each F level 1 checksums 23 +** there is one level 2 checksum. And so on. 24 +** 25 +** Checksum keys are encoded as the two byte "c." followed by the 26 +** checksum level, followed by a 10 digit decimal number containing 27 +** the value of the first key that contributes to the checksum value. 28 +** For example, assuming F==10, the level 1 checksum that spans keys 29 +** 10 to 19 is "c.1.0000000010". 30 +** 31 +** Clients may perform one of two operations on the database: a read 32 +** or a write. 33 +** 34 +** READ OPERATIONS: 35 +** 36 +** A read operation scans a range of F key/value pairs. It computes 37 +** the expected checksum and then compares the computed value to the 38 +** actual value stored in the level 1 checksum entry. It then scans 39 +** the group of F level 1 checksums, and compares the computed checksum 40 +** to the associated level 2 checksum value, and so on until the 41 +** highest level checksum value has been verified. 42 +** 43 +** If a checksum ever fails to match the expected value, the test 44 +** has failed. 45 +** 46 +** WRITE OPERATIONS: 47 +** 48 +** A write operation involves writing (possibly clobbering) a single 49 +** key/value pair. The associated level 1 checksum is then recalculated 50 +** updated. Then the level 2 checksum, and so on until the highest 51 +** level checksum has been modified. 52 +** 53 +** All updates occur inside a single transaction. 54 +** 55 +** INTERFACE: 56 +** 57 +** The interface used by test cases to read and write the db consists 58 +** of type DbParameters and the following functions: 59 +** 60 +** dbReadOperation() 61 +** dbWriteOperation() 62 +*/ 63 + 64 +#include "lsmtest.h" 65 + 66 +typedef struct DbParameters DbParameters; 67 +struct DbParameters { 68 + int nFanout; /* Checksum fanout (F) */ 69 + int nKey; /* Size of key space (N) */ 70 +}; 71 + 72 +#define DB_KEY_BYTES (2+5+10+1) 73 + 74 +/* 75 +** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size. 76 +** This function populates the buffer with a nul-terminated key string 77 +** corresponding to key iKey. 78 +*/ 79 +static void dbFormatKey( 80 + DbParameters *pParam, 81 + int iLevel, 82 + int iKey, /* Key value */ 83 + char *aBuf /* Write key string here */ 84 +){ 85 + if( iLevel==0 ){ 86 + snprintf(aBuf, DB_KEY_BYTES, "k.%.10d", iKey); 87 + }else{ 88 + int f = 1; 89 + int i; 90 + for(i=0; i<iLevel; i++) f = f * pParam->nFanout; 91 + snprintf(aBuf, DB_KEY_BYTES, "c.%d.%.10d", iLevel, f*(iKey/f)); 92 + } 93 +} 94 + 95 +/* 96 +** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size. 97 +** This function populates the buffer with the string representation of 98 +** checksum value iVal. 99 +*/ 100 +static void dbFormatCksumValue(u32 iVal, char *aBuf){ 101 + snprintf(aBuf, DB_KEY_BYTES, "%.10u", iVal); 102 +} 103 + 104 +/* 105 +** Return the highest level of checksum in the database described 106 +** by *pParam. 107 +*/ 108 +static int dbMaxLevel(DbParameters *pParam){ 109 + int iMax; 110 + int n = 1; 111 + for(iMax=0; n<pParam->nKey; iMax++){ 112 + n = n * pParam->nFanout; 113 + } 114 + return iMax; 115 +} 116 + 117 +static void dbCksum( 118 + void *pCtx, /* IN/OUT: Pointer to u32 containing cksum */ 119 + void *pKey, int nKey, /* Database key. Unused. */ 120 + void *pVal, int nVal /* Database value. Checksum this. */ 121 +){ 122 + u8 *aVal = (u8 *)pVal; 123 + u32 *pCksum = (u32 *)pCtx; 124 + u32 cksum = *pCksum; 125 + int i; 126 + 127 + unused_parameter(pKey); 128 + unused_parameter(nKey); 129 + 130 + for(i=0; i<nVal; i++){ 131 + cksum += (cksum<<3) + (int)aVal[i]; 132 + } 133 + 134 + *pCksum = cksum; 135 +} 136 + 137 +/* 138 +** Compute the value of the checksum stored on level iLevel that contains 139 +** data from key iKey by scanning the pParam->nFanout entries at level 140 +** iLevel-1. 141 +*/ 142 +static u32 dbComputeCksum( 143 + DbParameters *pParam, /* Database parameters */ 144 + TestDb *pDb, /* Database connection handle */ 145 + int iLevel, /* Level of checksum to compute */ 146 + int iKey, /* Compute checksum for this key */ 147 + int *pRc /* IN/OUT: Error code */ 148 +){ 149 + u32 cksum = 0; 150 + if( *pRc==0 ){ 151 + int nFirst; 152 + int nLast; 153 + int iFirst = 0; 154 + int iLast = 0; 155 + int i; 156 + int f = 1; 157 + char zFirst[DB_KEY_BYTES]; 158 + char zLast[DB_KEY_BYTES]; 159 + 160 + assert( iLevel>=1 ); 161 + for(i=0; i<iLevel; i++) f = f * pParam->nFanout; 162 + 163 + iFirst = f*(iKey/f); 164 + iLast = iFirst + f - 1; 165 + dbFormatKey(pParam, iLevel-1, iFirst, zFirst); 166 + dbFormatKey(pParam, iLevel-1, iLast, zLast); 167 + nFirst = strlen(zFirst); 168 + nLast = strlen(zLast); 169 + 170 + *pRc = tdb_scan(pDb, (u32*)&cksum, 0, zFirst, nFirst, zLast, nLast,dbCksum); 171 + } 172 + 173 + return cksum; 174 +} 175 + 176 +static void dbReadOperation( 177 + DbParameters *pParam, /* Database parameters */ 178 + TestDb *pDb, /* Database connection handle */ 179 + void (*xDelay)(void *), 180 + void *pDelayCtx, 181 + int iKey, /* Key to read */ 182 + int *pRc /* IN/OUT: Error code */ 183 +){ 184 + const int iMax = dbMaxLevel(pParam); 185 + int i; 186 + 187 + if( tdb_transaction_support(pDb) ) testBegin(pDb, 1, pRc); 188 + for(i=1; *pRc==0 && i<=iMax; i++){ 189 + char zCksum[DB_KEY_BYTES]; 190 + char zKey[DB_KEY_BYTES]; 191 + u32 iCksum = 0; 192 + 193 + iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc); 194 + if( iCksum ){ 195 + if( xDelay && i==1 ) xDelay(pDelayCtx); 196 + dbFormatCksumValue(iCksum, zCksum); 197 + dbFormatKey(pParam, i, iKey, zKey); 198 + testFetchStr(pDb, zKey, zCksum, pRc); 199 + } 200 + } 201 + if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc); 202 +} 203 + 204 +static int dbWriteOperation( 205 + DbParameters *pParam, /* Database parameters */ 206 + TestDb *pDb, /* Database connection handle */ 207 + int iKey, /* Key to write to */ 208 + const char *zValue, /* Nul-terminated value to write */ 209 + int *pRc /* IN/OUT: Error code */ 210 +){ 211 + const int iMax = dbMaxLevel(pParam); 212 + char zKey[DB_KEY_BYTES]; 213 + int i; 214 + int rc; 215 + 216 + assert( iKey>=0 && iKey<pParam->nKey ); 217 + dbFormatKey(pParam, 0, iKey, zKey); 218 + 219 + /* Open a write transaction. This may fail - SQLITE4_BUSY */ 220 + if( *pRc==0 && tdb_transaction_support(pDb) ){ 221 + rc = tdb_begin(pDb, 2); 222 + if( rc==5 ) return 0; 223 + *pRc = rc; 224 + } 225 + 226 + testWriteStr(pDb, zKey, zValue, pRc); 227 + for(i=1; i<=iMax; i++){ 228 + char zCksum[DB_KEY_BYTES]; 229 + u32 iCksum = 0; 230 + 231 + iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc); 232 + dbFormatCksumValue(iCksum, zCksum); 233 + dbFormatKey(pParam, i, iKey, zKey); 234 + testWriteStr(pDb, zKey, zCksum, pRc); 235 + } 236 + if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc); 237 + return 1; 238 +} 239 + 240 +/************************************************************************* 241 +** The following block contains testXXX() functions that implement a 242 +** wrapper around the systems native multi-thread support. There are no 243 +** synchronization primitives - just functions to launch and join 244 +** threads. Wrapper functions are: 245 +** 246 +** testThreadSupport() 247 +** 248 +** testThreadInit() 249 +** testThreadShutdown() 250 +** testThreadLaunch() 251 +** testThreadWait() 252 +** 253 +** testThreadSetHalt() 254 +** testThreadGetHalt() 255 +** testThreadSetResult() 256 +** testThreadGetResult() 257 +** 258 +** testThreadEnterMutex() 259 +** testThreadLeaveMutex() 260 +*/ 261 +typedef struct ThreadSet ThreadSet; 262 +#ifdef LSM_MUTEX_PTHREADS 263 + 264 +#include <pthread.h> 265 +#include <unistd.h> 266 + 267 +typedef struct Thread Thread; 268 +struct Thread { 269 + int rc; 270 + char *zMsg; 271 + pthread_t id; 272 + void (*xMain)(ThreadSet *, int, void *); 273 + void *pCtx; 274 + ThreadSet *pThreadSet; 275 +}; 276 + 277 +struct ThreadSet { 278 + int bHalt; /* Halt flag */ 279 + int nThread; /* Number of threads */ 280 + Thread *aThread; /* Array of Thread structures */ 281 + pthread_mutex_t mutex; /* Mutex used for cheating */ 282 +}; 283 + 284 +/* 285 +** Return true if this build supports threads, or false otherwise. If 286 +** this function returns false, no other testThreadXXX() functions should 287 +** be called. 288 +*/ 289 +static int testThreadSupport(){ return 1; } 290 + 291 +/* 292 +** Allocate and return a thread-set handle with enough space allocated 293 +** to handle up to nMax threads. Each call to this function should be 294 +** matched by a call to testThreadShutdown() to delete the object. 295 +*/ 296 +static ThreadSet *testThreadInit(int nMax){ 297 + int nByte; /* Total space to allocate */ 298 + ThreadSet *p; /* Return value */ 299 + 300 + nByte = sizeof(ThreadSet) + sizeof(struct Thread) * nMax; 301 + p = (ThreadSet *)testMalloc(nByte); 302 + p->nThread = nMax; 303 + p->aThread = (Thread *)&p[1]; 304 + pthread_mutex_init(&p->mutex, 0); 305 + 306 + return p; 307 +} 308 + 309 +/* 310 +** Delete a thread-set object and release all resources held by it. 311 +*/ 312 +static void testThreadShutdown(ThreadSet *p){ 313 + int i; 314 + for(i=0; i<p->nThread; i++){ 315 + testFree(p->aThread[i].zMsg); 316 + } 317 + pthread_mutex_destroy(&p->mutex); 318 + testFree(p); 319 +} 320 + 321 +static void *ttMain(void *pArg){ 322 + Thread *pThread = (Thread *)pArg; 323 + int iThread; 324 + iThread = (pThread - pThread->pThreadSet->aThread); 325 + pThread->xMain(pThread->pThreadSet, iThread, pThread->pCtx); 326 + return 0; 327 +} 328 + 329 +/* 330 +** Launch a new thread. 331 +*/ 332 +static int testThreadLaunch( 333 + ThreadSet *p, 334 + int iThread, 335 + void (*xMain)(ThreadSet *, int, void *), 336 + void *pCtx 337 +){ 338 + int rc; 339 + Thread *pThread; 340 + 341 + assert( iThread>=0 && iThread<p->nThread ); 342 + 343 + pThread = &p->aThread[iThread]; 344 + assert( pThread->pThreadSet==0 ); 345 + pThread->xMain = xMain; 346 + pThread->pCtx = pCtx; 347 + pThread->pThreadSet = p; 348 + rc = pthread_create(&pThread->id, 0, ttMain, (void *)pThread); 349 + 350 + return rc; 351 +} 352 + 353 +/* 354 +** Set the thread-set "halt" flag. 355 +*/ 356 +static void testThreadSetHalt(ThreadSet *pThreadSet){ 357 + pThreadSet->bHalt = 1; 358 +} 359 + 360 +/* 361 +** Return the current value of the thread-set "halt" flag. 362 +*/ 363 +static int testThreadGetHalt(ThreadSet *pThreadSet){ 364 + return pThreadSet->bHalt; 365 +} 366 + 367 +static void testThreadSleep(ThreadSet *pThreadSet, int nMs){ 368 + int nRem = nMs; 369 + while( nRem>0 && testThreadGetHalt(pThreadSet)==0 ){ 370 + usleep(50000); 371 + nRem -= 50; 372 + } 373 +} 374 + 375 +/* 376 +** Wait for all threads launched to finish before returning. If nMs 377 +** is greater than zero, set the "halt" flag to tell all threads 378 +** to halt after waiting nMs milliseconds. 379 +*/ 380 +static void testThreadWait(ThreadSet *pThreadSet, int nMs){ 381 + int i; 382 + 383 + testThreadSleep(pThreadSet, nMs); 384 + testThreadSetHalt(pThreadSet); 385 + for(i=0; i<pThreadSet->nThread; i++){ 386 + Thread *pThread = &pThreadSet->aThread[i]; 387 + if( pThread->xMain ){ 388 + pthread_join(pThread->id, 0); 389 + } 390 + } 391 +} 392 + 393 +/* 394 +** Set the result for thread iThread. 395 +*/ 396 +static void testThreadSetResult( 397 + ThreadSet *pThreadSet, /* Thread-set handle */ 398 + int iThread, /* Set result for this thread */ 399 + int rc, /* Result error code */ 400 + char *zFmt, /* Result string format */ 401 + ... /* Result string formatting args... */ 402 +){ 403 + va_list ap; 404 + 405 + testFree(pThreadSet->aThread[iThread].zMsg); 406 + pThreadSet->aThread[iThread].rc = rc; 407 + pThreadSet->aThread[iThread].zMsg = 0; 408 + if( zFmt ){ 409 + va_start(ap, zFmt); 410 + pThreadSet->aThread[iThread].zMsg = testMallocVPrintf(zFmt, ap); 411 + va_end(ap); 412 + } 413 +} 414 + 415 +/* 416 +** Retrieve the result for thread iThread. 417 +*/ 418 +static int testThreadGetResult( 419 + ThreadSet *pThreadSet, /* Thread-set handle */ 420 + int iThread, /* Get result for this thread */ 421 + const char **pzRes /* OUT: Pointer to result string */ 422 +){ 423 + if( pzRes ) *pzRes = pThreadSet->aThread[iThread].zMsg; 424 + return pThreadSet->aThread[iThread].rc; 425 +} 426 + 427 +/* 428 +** Enter and leave the test case mutex. 429 +*/ 430 +#if 0 431 +static void testThreadEnterMutex(ThreadSet *p){ 432 + pthread_mutex_lock(&p->mutex); 433 +} 434 +static void testThreadLeaveMutex(ThreadSet *p){ 435 + pthread_mutex_unlock(&p->mutex); 436 +} 437 +#endif 438 +#endif 439 + 440 +#if !defined(LSM_MUTEX_PTHREADS) 441 +static int testThreadSupport(){ return 0; } 442 + 443 +#define testThreadInit(a) 0 444 +#define testThreadShutdown(a) 445 +#define testThreadLaunch(a,b,c,d) 0 446 +#define testThreadWait(a,b) 447 +#define testThreadSetHalt(a) 448 +#define testThreadGetHalt(a) 0 449 +#define testThreadGetResult(a,b,c) 0 450 +#define testThreadSleep(a,b) 0 451 + 452 +static void testThreadSetResult(ThreadSet *a, int b, int c, char *d, ...){ 453 + unused_parameter(a); 454 + unused_parameter(b); 455 + unused_parameter(c); 456 + unused_parameter(d); 457 +} 458 +#endif 459 +/* End of threads wrapper. 460 +*************************************************************************/ 461 + 462 +/************************************************************************* 463 +** Below this point is the third part of this file - the implementation 464 +** of the mt1.* tests. 465 +*/ 466 +typedef struct Mt1Test Mt1Test; 467 +struct Mt1Test { 468 + DbParameters param; /* Description of database to read/write */ 469 + int nReadwrite; /* Number of read/write threads */ 470 + int nFastReader; /* Number of fast reader threads */ 471 + int nSlowReader; /* Number of slow reader threads */ 472 + int nMs; /* How long to run for */ 473 + const char *zSystem; /* Database system to test */ 474 +}; 475 + 476 +typedef struct Mt1DelayCtx Mt1DelayCtx; 477 +struct Mt1DelayCtx { 478 + ThreadSet *pSet; /* Threadset to sleep within */ 479 + int nMs; /* Sleep in ms */ 480 +}; 481 + 482 +static void xMt1Delay(void *pCtx){ 483 + Mt1DelayCtx *p = (Mt1DelayCtx *)pCtx; 484 + testThreadSleep(p->pSet, p->nMs); 485 +} 486 + 487 +#define MT1_THREAD_RDWR 0 488 +#define MT1_THREAD_SLOW 1 489 +#define MT1_THREAD_FAST 2 490 + 491 +static void xMt1Work(lsm_db *pDb, void *pCtx){ 492 +#if 0 493 + char *z = 0; 494 + lsm_info(pDb, LSM_INFO_DB_STRUCTURE, &z); 495 + printf("%s\n", z); 496 + fflush(stdout); 497 +#endif 498 +} 499 + 500 +/* 501 +** This is the main() proc for all threads in test case "mt1". 502 +*/ 503 +static void mt1Main(ThreadSet *pThreadSet, int iThread, void *pCtx){ 504 + Mt1Test *p = (Mt1Test *)pCtx; /* Test parameters */ 505 + Mt1DelayCtx delay; 506 + int nRead = 0; /* Number of calls to dbReadOperation() */ 507 + int nWrite = 0; /* Number of completed database writes */ 508 + int rc = 0; /* Error code */ 509 + int iPrng; /* Prng argument variable */ 510 + TestDb *pDb; /* Database handle */ 511 + int eType; 512 + 513 + delay.pSet = pThreadSet; 514 + delay.nMs = 0; 515 + if( iThread<p->nReadwrite ){ 516 + eType = MT1_THREAD_RDWR; 517 + }else if( iThread<(p->nReadwrite+p->nFastReader) ){ 518 + eType = MT1_THREAD_FAST; 519 + }else{ 520 + eType = MT1_THREAD_SLOW; 521 + delay.nMs = (p->nMs / 20); 522 + } 523 + 524 + /* Open a new database connection. Initialize the pseudo-random number 525 + ** argument based on the thread number. */ 526 + iPrng = testPrngValue(iThread); 527 + pDb = testOpen(p->zSystem, 0, &rc); 528 + 529 + if( rc==0 ){ 530 + tdb_lsm_config_work_hook(pDb, xMt1Work, 0); 531 + } 532 + 533 + /* Loop until either an error occurs or some other thread sets the 534 + ** halt flag. */ 535 + while( rc==0 && testThreadGetHalt(pThreadSet)==0 ){ 536 + int iKey; 537 + 538 + /* Perform a read operation on an arbitrarily selected key. */ 539 + iKey = (testPrngValue(iPrng++) % p->param.nKey); 540 + dbReadOperation(&p->param, pDb, xMt1Delay, (void *)&delay, iKey, &rc); 541 + if( rc ) continue; 542 + nRead++; 543 + 544 + /* Attempt to write an arbitrary key value pair (and update the associated 545 + ** checksum entries). dbWriteOperation() returns 1 if the write is 546 + ** successful, or 0 if it failed with an LSM_BUSY error. */ 547 + if( eType==MT1_THREAD_RDWR ){ 548 + char aValue[50]; 549 + char aRnd[25]; 550 + 551 + iKey = (testPrngValue(iPrng++) % p->param.nKey); 552 + testPrngString(iPrng, aRnd, sizeof(aRnd)); 553 + iPrng += sizeof(aRnd); 554 + snprintf(aValue, sizeof(aValue), "%d.%s", iThread, aRnd); 555 + nWrite += dbWriteOperation(&p->param, pDb, iKey, aValue, &rc); 556 + } 557 + } 558 + testClose(&pDb); 559 + 560 + /* If an error has occured, set the thread error code and the threadset 561 + ** halt flag to tell the other test threads to halt. Otherwise, set the 562 + ** thread error code to 0 and post a message with the number of read 563 + ** and write operations completed. */ 564 + if( rc ){ 565 + testThreadSetResult(pThreadSet, iThread, rc, 0); 566 + testThreadSetHalt(pThreadSet); 567 + }else{ 568 + testThreadSetResult(pThreadSet, iThread, 0, "r/w: %d/%d", nRead, nWrite); 569 + } 570 +} 571 + 572 +static void do_test_mt1( 573 + const char *zSystem, /* Database system name */ 574 + const char *zPattern, /* Run test cases that match this pattern */ 575 + int *pRc /* IN/OUT: Error code */ 576 +){ 577 + Mt1Test aTest[] = { 578 + /* param, nReadwrite, nFastReader, nSlowReader, nMs, zSystem */ 579 + { {10, 1000}, 4, 0, 0, 10000, 0 }, 580 + { {10, 1000}, 4, 4, 2, 100000, 0 }, 581 + { {10, 100000}, 4, 0, 0, 10000, 0 }, 582 + { {10, 100000}, 4, 4, 2, 100000, 0 }, 583 + }; 584 + int i; 585 + 586 + for(i=0; *pRc==0 && i<ArraySize(aTest); i++){ 587 + Mt1Test *p = &aTest[i]; 588 + int bRun = testCaseBegin(pRc, zPattern, 589 + "mt1.%s.db=%d,%d.ms=%d.rdwr=%d.fast=%d.slow=%d", 590 + zSystem, p->param.nFanout, p->param.nKey, 591 + p->nMs, p->nReadwrite, p->nFastReader, p->nSlowReader 592 + ); 593 + if( bRun ){ 594 + TestDb *pDb; 595 + ThreadSet *pSet; 596 + int iThread; 597 + int nThread; 598 + 599 + p->zSystem = zSystem; 600 + pDb = testOpen(zSystem, 1, pRc); 601 + 602 + nThread = p->nReadwrite + p->nFastReader + p->nSlowReader; 603 + pSet = testThreadInit(nThread); 604 + for(iThread=0; *pRc==0 && iThread<nThread; iThread++){ 605 + testThreadLaunch(pSet, iThread, mt1Main, (void *)p); 606 + } 607 + 608 + testThreadWait(pSet, p->nMs); 609 + for(iThread=0; *pRc==0 && iThread<nThread; iThread++){ 610 + *pRc = testThreadGetResult(pSet, iThread, 0); 611 + } 612 + testCaseFinish(*pRc); 613 + 614 + for(iThread=0; *pRc==0 && iThread<nThread; iThread++){ 615 + const char *zMsg = 0; 616 + *pRc = testThreadGetResult(pSet, iThread, &zMsg); 617 + printf(" Info: thread %d (%d): %s\n", iThread, *pRc, zMsg); 618 + } 619 + 620 + testThreadShutdown(pSet); 621 + testClose(&pDb); 622 + } 623 + } 624 +} 625 + 626 +void test_mt( 627 + const char *zSystem, /* Database system name */ 628 + const char *zPattern, /* Run test cases that match this pattern */ 629 + int *pRc /* IN/OUT: Error code */ 630 +){ 631 + if( testThreadSupport()==0 ) return; 632 + do_test_mt1(zSystem, zPattern, pRc); 633 +}
Added ext/lsm1/lsm-test/lsmtest6.c.
1 + 2 +#include "lsmtest.h" 3 + 4 +typedef struct OomTest OomTest; 5 +struct OomTest { 6 + lsm_env *pEnv; 7 + int iNext; /* Next value to pass to testMallocOom() */ 8 + int nFail; /* Number of OOM events injected */ 9 + int bEnable; 10 + int rc; /* Test case error code */ 11 +}; 12 + 13 +static void testOomStart(OomTest *p){ 14 + memset(p, 0, sizeof(OomTest)); 15 + p->iNext = 1; 16 + p->bEnable = 1; 17 + p->nFail = 1; 18 + p->pEnv = tdb_lsm_env(); 19 +} 20 + 21 +static void xOomHook(OomTest *p){ 22 + p->nFail++; 23 +} 24 + 25 +static int testOomContinue(OomTest *p){ 26 + if( p->rc!=0 || (p->iNext>1 && p->nFail==0) ){ 27 + return 0; 28 + } 29 + p->nFail = 0; 30 + testMallocOom(p->pEnv, p->iNext, 0, (void (*)(void*))xOomHook, (void *)p); 31 + return 1; 32 +} 33 + 34 +static void testOomEnable(OomTest *p, int bEnable){ 35 + p->bEnable = bEnable; 36 + testMallocOomEnable(p->pEnv, bEnable); 37 +} 38 + 39 +static void testOomNext(OomTest *p){ 40 + p->iNext++; 41 +} 42 + 43 +static int testOomHit(OomTest *p){ 44 + return (p->nFail>0); 45 +} 46 + 47 +static int testOomFinish(OomTest *p){ 48 + return p->rc; 49 +} 50 + 51 +static void testOomAssert(OomTest *p, int bVal){ 52 + if( bVal==0 ){ 53 + test_failed(); 54 + p->rc = 1; 55 + } 56 +} 57 + 58 +/* 59 +** Test that the error code matches the state of the OomTest object passed 60 +** as the first argument. Specifically, check that rc is LSM_NOMEM if an 61 +** OOM error has already been injected, or LSM_OK if not. 62 +*/ 63 +static void testOomAssertRc(OomTest *p, int rc){ 64 + testOomAssert(p, rc==LSM_OK || rc==LSM_NOMEM); 65 + testOomAssert(p, testOomHit(p)==(rc==LSM_NOMEM) || p->bEnable==0 ); 66 +} 67 + 68 +static void testOomOpen( 69 + OomTest *pOom, 70 + const char *zName, 71 + lsm_db **ppDb, 72 + int *pRc 73 +){ 74 + if( *pRc==LSM_OK ){ 75 + int rc; 76 + rc = lsm_new(tdb_lsm_env(), ppDb); 77 + if( rc==LSM_OK ) rc = lsm_open(*ppDb, zName); 78 + testOomAssertRc(pOom, rc); 79 + *pRc = rc; 80 + } 81 +} 82 + 83 +static void testOomFetch( 84 + OomTest *pOom, 85 + lsm_db *pDb, 86 + void *pKey, int nKey, 87 + void *pVal, int nVal, 88 + int *pRc 89 +){ 90 + testOomAssertRc(pOom, *pRc); 91 + if( *pRc==LSM_OK ){ 92 + lsm_cursor *pCsr; 93 + int rc; 94 + 95 + rc = lsm_csr_open(pDb, &pCsr); 96 + if( rc==LSM_OK ) rc = lsm_csr_seek(pCsr, pKey, nKey, 0); 97 + testOomAssertRc(pOom, rc); 98 + 99 + if( rc==LSM_OK ){ 100 + const void *p; int n; 101 + testOomAssert(pOom, lsm_csr_valid(pCsr)); 102 + 103 + rc = lsm_csr_key(pCsr, &p, &n); 104 + testOomAssertRc(pOom, rc); 105 + testOomAssert(pOom, rc!=LSM_OK || (n==nKey && memcmp(pKey, p, nKey)==0) ); 106 + } 107 + 108 + if( rc==LSM_OK ){ 109 + const void *p; int n; 110 + testOomAssert(pOom, lsm_csr_valid(pCsr)); 111 + 112 + rc = lsm_csr_value(pCsr, &p, &n); 113 + testOomAssertRc(pOom, rc); 114 + testOomAssert(pOom, rc!=LSM_OK || (n==nVal && memcmp(pVal, p, nVal)==0) ); 115 + } 116 + 117 + lsm_csr_close(pCsr); 118 + *pRc = rc; 119 + } 120 +} 121 + 122 +static void testOomWrite( 123 + OomTest *pOom, 124 + lsm_db *pDb, 125 + void *pKey, int nKey, 126 + void *pVal, int nVal, 127 + int *pRc 128 +){ 129 + testOomAssertRc(pOom, *pRc); 130 + if( *pRc==LSM_OK ){ 131 + int rc; 132 + 133 + rc = lsm_insert(pDb, pKey, nKey, pVal, nVal); 134 + testOomAssertRc(pOom, rc); 135 + 136 + *pRc = rc; 137 + } 138 +} 139 + 140 + 141 +static void testOomFetchStr( 142 + OomTest *pOom, 143 + lsm_db *pDb, 144 + const char *zKey, 145 + const char *zVal, 146 + int *pRc 147 +){ 148 + int nKey = strlen(zKey); 149 + int nVal = strlen(zVal); 150 + testOomFetch(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc); 151 +} 152 + 153 +static void testOomFetchData( 154 + OomTest *pOom, 155 + lsm_db *pDb, 156 + Datasource *pData, 157 + int iKey, 158 + int *pRc 159 +){ 160 + void *pKey; int nKey; 161 + void *pVal; int nVal; 162 + testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); 163 + testOomFetch(pOom, pDb, pKey, nKey, pVal, nVal, pRc); 164 +} 165 + 166 +static void testOomWriteStr( 167 + OomTest *pOom, 168 + lsm_db *pDb, 169 + const char *zKey, 170 + const char *zVal, 171 + int *pRc 172 +){ 173 + int nKey = strlen(zKey); 174 + int nVal = strlen(zVal); 175 + testOomWrite(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc); 176 +} 177 + 178 +static void testOomWriteData( 179 + OomTest *pOom, 180 + lsm_db *pDb, 181 + Datasource *pData, 182 + int iKey, 183 + int *pRc 184 +){ 185 + void *pKey; int nKey; 186 + void *pVal; int nVal; 187 + testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); 188 + testOomWrite(pOom, pDb, pKey, nKey, pVal, nVal, pRc); 189 +} 190 + 191 +static void testOomScan( 192 + OomTest *pOom, 193 + lsm_db *pDb, 194 + int bReverse, 195 + const void *pKey, int nKey, 196 + int nScan, 197 + int *pRc 198 +){ 199 + if( *pRc==0 ){ 200 + int rc; 201 + int iScan = 0; 202 + lsm_cursor *pCsr; 203 + int (*xAdvance)(lsm_cursor *) = 0; 204 + 205 + 206 + rc = lsm_csr_open(pDb, &pCsr); 207 + testOomAssertRc(pOom, rc); 208 + 209 + if( rc==LSM_OK ){ 210 + if( bReverse ){ 211 + rc = lsm_csr_seek(pCsr, pKey, nKey, LSM_SEEK_LE); 212 + xAdvance = lsm_csr_prev; 213 + }else{ 214 + rc = lsm_csr_seek(pCsr, pKey, nKey, LSM_SEEK_GE); 215 + xAdvance = lsm_csr_next; 216 + } 217 + } 218 + testOomAssertRc(pOom, rc); 219 + 220 + while( rc==LSM_OK && lsm_csr_valid(pCsr) && iScan<nScan ){ 221 + const void *p; int n; 222 + 223 + rc = lsm_csr_key(pCsr, &p, &n); 224 + testOomAssertRc(pOom, rc); 225 + if( rc==LSM_OK ){ 226 + rc = lsm_csr_value(pCsr, &p, &n); 227 + testOomAssertRc(pOom, rc); 228 + } 229 + if( rc==LSM_OK ){ 230 + rc = xAdvance(pCsr); 231 + testOomAssertRc(pOom, rc); 232 + } 233 + iScan++; 234 + } 235 + 236 + lsm_csr_close(pCsr); 237 + *pRc = rc; 238 + } 239 +} 240 + 241 +#define LSMTEST6_TESTDB "testdb.lsm" 242 + 243 +void testDeleteLsmdb(const char *zFile){ 244 + char *zLog = testMallocPrintf("%s-log", zFile); 245 + char *zShm = testMallocPrintf("%s-shm", zFile); 246 + unlink(zFile); 247 + unlink(zLog); 248 + unlink(zShm); 249 + testFree(zLog); 250 + testFree(zShm); 251 +} 252 + 253 +static void copy_file(const char *zFrom, const char *zTo, int isDatabase){ 254 + 255 + if( access(zFrom, F_OK) ){ 256 + unlink(zTo); 257 + }else{ 258 + int fd1; 259 + int fd2; 260 + off_t sz; 261 + off_t i; 262 + struct stat buf; 263 + u8 *aBuf; 264 + 265 + fd1 = open(zFrom, O_RDONLY | _O_BINARY, 0644); 266 + fd2 = open(zTo, O_RDWR | O_CREAT | _O_BINARY, 0644); 267 + 268 + fstat(fd1, &buf); 269 + sz = buf.st_size; 270 + ftruncate(fd2, sz); 271 + 272 + aBuf = testMalloc(4096); 273 + for(i=0; i<sz; i+=4096){ 274 + int bLockPage = isDatabase && i == 0; 275 + int nByte = MIN((bLockPage ? 4066 : 4096), sz - i); 276 + memset(aBuf, 0, 4096); 277 + read(fd1, aBuf, nByte); 278 + write(fd2, aBuf, nByte); 279 + if( bLockPage ){ 280 + lseek(fd1, 4096, SEEK_SET); 281 + lseek(fd2, 4096, SEEK_SET); 282 + } 283 + } 284 + testFree(aBuf); 285 + 286 + close(fd1); 287 + close(fd2); 288 + } 289 +} 290 + 291 +void testCopyLsmdb(const char *zFrom, const char *zTo){ 292 + char *zLog1 = testMallocPrintf("%s-log", zFrom); 293 + char *zLog2 = testMallocPrintf("%s-log", zTo); 294 + char *zShm1 = testMallocPrintf("%s-shm", zFrom); 295 + char *zShm2 = testMallocPrintf("%s-shm", zTo); 296 + 297 + unlink(zShm2); 298 + unlink(zLog2); 299 + unlink(zTo); 300 + copy_file(zFrom, zTo, 1); 301 + copy_file(zLog1, zLog2, 0); 302 + copy_file(zShm1, zShm2, 0); 303 + 304 + testFree(zLog1); testFree(zLog2); testFree(zShm1); testFree(zShm2); 305 +} 306 + 307 +/* 308 +** File zFile is the path to a database. This function makes backups 309 +** of the database file and its log as follows: 310 +** 311 +** cp $(zFile) $(zFile)-save 312 +** cp $(zFile)-$(zAux) $(zFile)-save-$(zAux) 313 +** 314 +** Function testRestoreDb() can be used to copy the files back in the 315 +** other direction. 316 +*/ 317 +void testSaveDb(const char *zFile, const char *zAux){ 318 + char *zLog = testMallocPrintf("%s-%s", zFile, zAux); 319 + char *zFileSave = testMallocPrintf("%s-save", zFile); 320 + char *zLogSave = testMallocPrintf("%s-%s-save", zFile, zAux); 321 + 322 + unlink(zFileSave); 323 + unlink(zLogSave); 324 + copy_file(zFile, zFileSave, 1); 325 + copy_file(zLog, zLogSave, 0); 326 + 327 + testFree(zLog); testFree(zFileSave); testFree(zLogSave); 328 +} 329 + 330 +/* 331 +** File zFile is the path to a database. This function restores 332 +** a backup of the database made by a previous call to testSaveDb(). 333 +** Specifically, it does the equivalent of: 334 +** 335 +** cp $(zFile)-save $(zFile) 336 +** cp $(zFile)-save-$(zAux) $(zFile)-$(zAux) 337 +*/ 338 +void testRestoreDb(const char *zFile, const char *zAux){ 339 + char *zLog = testMallocPrintf("%s-%s", zFile, zAux); 340 + char *zFileSave = testMallocPrintf("%s-save", zFile); 341 + char *zLogSave = testMallocPrintf("%s-%s-save", zFile, zAux); 342 + 343 + copy_file(zFileSave, zFile, 1); 344 + copy_file(zLogSave, zLog, 0); 345 + 346 + testFree(zLog); testFree(zFileSave); testFree(zLogSave); 347 +} 348 + 349 + 350 +static int lsmWriteStr(lsm_db *pDb, const char *zKey, const char *zVal){ 351 + int nKey = strlen(zKey); 352 + int nVal = strlen(zVal); 353 + return lsm_insert(pDb, (void *)zKey, nKey, (void *)zVal, nVal); 354 +} 355 + 356 +static void setup_delete_db(void){ 357 + testDeleteLsmdb(LSMTEST6_TESTDB); 358 +} 359 + 360 +/* 361 +** Create a small database. With the following content: 362 +** 363 +** "one" -> "one" 364 +** "two" -> "four" 365 +** "three" -> "nine" 366 +** "four" -> "sixteen" 367 +** "five" -> "twentyfive" 368 +** "six" -> "thirtysix" 369 +** "seven" -> "fourtynine" 370 +** "eight" -> "sixtyfour" 371 +*/ 372 +static void setup_populate_db(void){ 373 + const char *azStr[] = { 374 + "one", "one", 375 + "two", "four", 376 + "three", "nine", 377 + "four", "sixteen", 378 + "five", "twentyfive", 379 + "six", "thirtysix", 380 + "seven", "fourtynine", 381 + "eight", "sixtyfour", 382 + }; 383 + int rc; 384 + int ii; 385 + lsm_db *pDb; 386 + 387 + testDeleteLsmdb(LSMTEST6_TESTDB); 388 + 389 + rc = lsm_new(tdb_lsm_env(), &pDb); 390 + if( rc==LSM_OK ) rc = lsm_open(pDb, LSMTEST6_TESTDB); 391 + 392 + for(ii=0; rc==LSM_OK && ii<ArraySize(azStr); ii+=2){ 393 + rc = lsmWriteStr(pDb, azStr[ii], azStr[ii+1]); 394 + } 395 + lsm_close(pDb); 396 + 397 + testSaveDb(LSMTEST6_TESTDB, "log"); 398 + assert( rc==LSM_OK ); 399 +} 400 + 401 +static Datasource *getDatasource(void){ 402 + const DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 200, 250 }; 403 + return testDatasourceNew(&defn); 404 +} 405 + 406 +/* 407 +** Set up a database file with the following properties: 408 +** 409 +** * Page size is 1024 bytes. 410 +** * Block size is 64 KB. 411 +** * Contains 5000 key-value pairs starting at 0 from the 412 +** datasource returned getDatasource(). 413 +*/ 414 +static void setup_populate_db2(void){ 415 + Datasource *pData; 416 + int ii; 417 + int rc; 418 + int nBlocksize = 64*1024; 419 + int nPagesize = 1024; 420 + int nWritebuffer = 4*1024; 421 + lsm_db *pDb; 422 + 423 + testDeleteLsmdb(LSMTEST6_TESTDB); 424 + rc = lsm_new(tdb_lsm_env(), &pDb); 425 + if( rc==LSM_OK ) rc = lsm_open(pDb, LSMTEST6_TESTDB); 426 + 427 + lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &nBlocksize); 428 + lsm_config(pDb, LSM_CONFIG_PAGE_SIZE, &nPagesize); 429 + lsm_config(pDb, LSM_CONFIG_AUTOFLUSH, &nWritebuffer); 430 + 431 + pData = getDatasource(); 432 + for(ii=0; rc==LSM_OK && ii<5000; ii++){ 433 + void *pKey; int nKey; 434 + void *pVal; int nVal; 435 + testDatasourceEntry(pData, ii, &pKey, &nKey, &pVal, &nVal); 436 + lsm_insert(pDb, pKey, nKey, pVal, nVal); 437 + } 438 + testDatasourceFree(pData); 439 + lsm_close(pDb); 440 + 441 + testSaveDb(LSMTEST6_TESTDB, "log"); 442 + assert( rc==LSM_OK ); 443 +} 444 + 445 +/* 446 +** Test the results of OOM conditions in lsm_new(). 447 +*/ 448 +static void simple_oom_1(OomTest *pOom){ 449 + int rc; 450 + lsm_db *pDb; 451 + 452 + rc = lsm_new(tdb_lsm_env(), &pDb); 453 + testOomAssertRc(pOom, rc); 454 + 455 + lsm_close(pDb); 456 +} 457 + 458 +/* 459 +** Test the results of OOM conditions in lsm_open(). 460 +*/ 461 +static void simple_oom_2(OomTest *pOom){ 462 + int rc; 463 + lsm_db *pDb; 464 + 465 + rc = lsm_new(tdb_lsm_env(), &pDb); 466 + if( rc==LSM_OK ){ 467 + rc = lsm_open(pDb, "testdb.lsm"); 468 + } 469 + testOomAssertRc(pOom, rc); 470 + 471 + lsm_close(pDb); 472 +} 473 + 474 +/* 475 +** Test the results of OOM conditions in simple fetch operations. 476 +*/ 477 +static void simple_oom_3(OomTest *pOom){ 478 + int rc = LSM_OK; 479 + lsm_db *pDb; 480 + 481 + testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc); 482 + 483 + testOomFetchStr(pOom, pDb, "four", "sixteen", &rc); 484 + testOomFetchStr(pOom, pDb, "seven", "fourtynine", &rc); 485 + testOomFetchStr(pOom, pDb, "one", "one", &rc); 486 + testOomFetchStr(pOom, pDb, "eight", "sixtyfour", &rc); 487 + 488 + lsm_close(pDb); 489 +} 490 + 491 +/* 492 +** Test the results of OOM conditions in simple write operations. 493 +*/ 494 +static void simple_oom_4(OomTest *pOom){ 495 + int rc = LSM_OK; 496 + lsm_db *pDb; 497 + 498 + testDeleteLsmdb(LSMTEST6_TESTDB); 499 + testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc); 500 + 501 + testOomWriteStr(pOom, pDb, "123", "onetwothree", &rc); 502 + testOomWriteStr(pOom, pDb, "456", "fourfivesix", &rc); 503 + testOomWriteStr(pOom, pDb, "789", "seveneightnine", &rc); 504 + testOomWriteStr(pOom, pDb, "123", "teneleventwelve", &rc); 505 + testOomWriteStr(pOom, pDb, "456", "fourteenfifteensixteen", &rc); 506 + 507 + lsm_close(pDb); 508 +} 509 + 510 +static void simple_oom_5(OomTest *pOom){ 511 + Datasource *pData = getDatasource(); 512 + int rc = LSM_OK; 513 + lsm_db *pDb; 514 + 515 + testRestoreDb(LSMTEST6_TESTDB, "log"); 516 + testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc); 517 + 518 + testOomFetchData(pOom, pDb, pData, 3333, &rc); 519 + testOomFetchData(pOom, pDb, pData, 0, &rc); 520 + testOomFetchData(pOom, pDb, pData, 4999, &rc); 521 + 522 + lsm_close(pDb); 523 + testDatasourceFree(pData); 524 +} 525 + 526 +static void simple_oom_6(OomTest *pOom){ 527 + Datasource *pData = getDatasource(); 528 + int rc = LSM_OK; 529 + lsm_db *pDb; 530 + 531 + testRestoreDb(LSMTEST6_TESTDB, "log"); 532 + testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc); 533 + 534 + testOomWriteData(pOom, pDb, pData, 5000, &rc); 535 + testOomWriteData(pOom, pDb, pData, 5001, &rc); 536 + testOomWriteData(pOom, pDb, pData, 5002, &rc); 537 + testOomFetchData(pOom, pDb, pData, 5001, &rc); 538 + testOomFetchData(pOom, pDb, pData, 1234, &rc); 539 + 540 + lsm_close(pDb); 541 + testDatasourceFree(pData); 542 +} 543 + 544 +static void simple_oom_7(OomTest *pOom){ 545 + Datasource *pData = getDatasource(); 546 + int rc = LSM_OK; 547 + lsm_db *pDb; 548 + 549 + testRestoreDb(LSMTEST6_TESTDB, "log"); 550 + testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc); 551 + testOomScan(pOom, pDb, 0, "abc", 3, 20, &rc); 552 + lsm_close(pDb); 553 + testDatasourceFree(pData); 554 +} 555 + 556 +static void simple_oom_8(OomTest *pOom){ 557 + Datasource *pData = getDatasource(); 558 + int rc = LSM_OK; 559 + lsm_db *pDb; 560 + testRestoreDb(LSMTEST6_TESTDB, "log"); 561 + testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc); 562 + testOomScan(pOom, pDb, 1, "xyz", 3, 20, &rc); 563 + lsm_close(pDb); 564 + testDatasourceFree(pData); 565 +} 566 + 567 +/* 568 +** This test case has two clients connected to a database. The first client 569 +** hits an OOM while writing to the database. Check that the second 570 +** connection is still able to query the db following the OOM. 571 +*/ 572 +static void simple_oom2_1(OomTest *pOom){ 573 + const int nRecord = 100; /* Number of records initially in db */ 574 + const int nIns = 10; /* Number of records inserted with OOM */ 575 + 576 + Datasource *pData = getDatasource(); 577 + int rc = LSM_OK; 578 + lsm_db *pDb1; 579 + lsm_db *pDb2; 580 + int i; 581 + 582 + testDeleteLsmdb(LSMTEST6_TESTDB); 583 + 584 + /* Open the two connections. Initialize the in-memory tree so that it 585 + ** contains 100 records. Do all this with OOM injection disabled. */ 586 + testOomEnable(pOom, 0); 587 + testOomOpen(pOom, LSMTEST6_TESTDB, &pDb1, &rc); 588 + testOomOpen(pOom, LSMTEST6_TESTDB, &pDb2, &rc); 589 + for(i=0; i<nRecord; i++){ 590 + testOomWriteData(pOom, pDb1, pData, i, &rc); 591 + } 592 + testOomEnable(pOom, 1); 593 + assert( rc==0 ); 594 + 595 + /* Insert 10 more records using pDb1. Stop when an OOM is encountered. */ 596 + for(i=nRecord; i<nRecord+nIns; i++){ 597 + testOomWriteData(pOom, pDb1, pData, i, &rc); 598 + if( rc ) break; 599 + } 600 + testOomAssertRc(pOom, rc); 601 + 602 + /* Switch off OOM injection. Write a few rows using pDb2. Then check 603 + ** that the database may be successfully queried. */ 604 + testOomEnable(pOom, 0); 605 + rc = 0; 606 + for(; i<nRecord+nIns && rc==0; i++){ 607 + testOomWriteData(pOom, pDb2, pData, i, &rc); 608 + } 609 + for(i=0; i<nRecord+nIns; i++) testOomFetchData(pOom, pDb2, pData, i, &rc); 610 + testOomEnable(pOom, 1); 611 + 612 + lsm_close(pDb1); 613 + lsm_close(pDb2); 614 + testDatasourceFree(pData); 615 +} 616 + 617 + 618 +static void do_test_oom1(const char *zPattern, int *pRc){ 619 + struct SimpleOom { 620 + const char *zName; 621 + void (*xSetup)(void); 622 + void (*xFunc)(OomTest *); 623 + } aSimple[] = { 624 + { "oom1.lsm.1", setup_delete_db, simple_oom_1 }, 625 + { "oom1.lsm.2", setup_delete_db, simple_oom_2 }, 626 + { "oom1.lsm.3", setup_populate_db, simple_oom_3 }, 627 + { "oom1.lsm.4", setup_delete_db, simple_oom_4 }, 628 + { "oom1.lsm.5", setup_populate_db2, simple_oom_5 }, 629 + { "oom1.lsm.6", setup_populate_db2, simple_oom_6 }, 630 + { "oom1.lsm.7", setup_populate_db2, simple_oom_7 }, 631 + { "oom1.lsm.8", setup_populate_db2, simple_oom_8 }, 632 + 633 + { "oom2.lsm.1", setup_delete_db, simple_oom2_1 }, 634 + }; 635 + int i; 636 + 637 + for(i=0; i<ArraySize(aSimple); i++){ 638 + if( *pRc==0 && testCaseBegin(pRc, zPattern, "%s", aSimple[i].zName) ){ 639 + OomTest t; 640 + 641 + if( aSimple[i].xSetup ){ 642 + aSimple[i].xSetup(); 643 + } 644 + 645 + for(testOomStart(&t); testOomContinue(&t); testOomNext(&t)){ 646 + aSimple[i].xFunc(&t); 647 + } 648 + 649 + printf("(%d injections).", t.iNext-2); 650 + testCaseFinish( (*pRc = testOomFinish(&t)) ); 651 + testMallocOom(tdb_lsm_env(), 0, 0, 0, 0); 652 + } 653 + } 654 +} 655 + 656 +void test_oom( 657 + const char *zPattern, /* Run test cases that match this pattern */ 658 + int *pRc /* IN/OUT: Error code */ 659 +){ 660 + do_test_oom1(zPattern, pRc); 661 +}
Added ext/lsm1/lsm-test/lsmtest7.c.
1 + 2 + 3 +#include "lsmtest.h" 4 + 5 + 6 +/* 7 +** Test that the rules for when lsm_csr_next() and lsm_csr_prev() are 8 +** enforced. Specifically: 9 +** 10 +** * Both functions always return LSM_MISUSE if the cursor is at EOF 11 +** when they are called. 12 +** 13 +** * lsm_csr_next() may only be used after lsm_csr_seek(LSM_SEEK_GE) or 14 +** lsm_csr_first(). 15 +** 16 +** * lsm_csr_prev() may only be used after lsm_csr_seek(LSM_SEEK_LE) or 17 +** lsm_csr_last(). 18 +*/ 19 +static void do_test_api1_lsm(lsm_db *pDb, int *pRc){ 20 + int ret; 21 + lsm_cursor *pCsr; 22 + lsm_cursor *pCsr2; 23 + int nKey; 24 + const void *pKey; 25 + 26 + ret = lsm_csr_open(pDb, &pCsr); 27 + testCompareInt(LSM_OK, ret, pRc); 28 + 29 + ret = lsm_csr_next(pCsr); 30 + testCompareInt(LSM_MISUSE, ret, pRc); 31 + ret = lsm_csr_prev(pCsr); 32 + testCompareInt(LSM_MISUSE, ret, pRc); 33 + 34 + ret = lsm_csr_seek(pCsr, "jjj", 3, LSM_SEEK_GE); 35 + testCompareInt(LSM_OK, ret, pRc); 36 + ret = lsm_csr_next(pCsr); 37 + testCompareInt(LSM_OK, ret, pRc); 38 + ret = lsm_csr_prev(pCsr); 39 + testCompareInt(LSM_MISUSE, ret, pRc); 40 + 41 + ret = lsm_csr_seek(pCsr, "jjj", 3, LSM_SEEK_LE); 42 + testCompareInt(LSM_OK, ret, pRc); 43 + ret = lsm_csr_next(pCsr); 44 + testCompareInt(LSM_MISUSE, ret, pRc); 45 + ret = lsm_csr_prev(pCsr); 46 + testCompareInt(LSM_OK, ret, pRc); 47 + 48 + ret = lsm_csr_seek(pCsr, "jjj", 3, LSM_SEEK_LEFAST); 49 + testCompareInt(LSM_OK, ret, pRc); 50 + ret = lsm_csr_next(pCsr); 51 + testCompareInt(LSM_MISUSE, ret, pRc); 52 + ret = lsm_csr_prev(pCsr); 53 + testCompareInt(LSM_MISUSE, ret, pRc); 54 + 55 + ret = lsm_csr_key(pCsr, &pKey, &nKey); 56 + testCompareInt(LSM_OK, ret, pRc); 57 + 58 + ret = lsm_csr_open(pDb, &pCsr2); 59 + testCompareInt(LSM_OK, ret, pRc); 60 + 61 + ret = lsm_csr_seek(pCsr2, pKey, nKey, LSM_SEEK_EQ); 62 + testCompareInt(LSM_OK, ret, pRc); 63 + testCompareInt(1, lsm_csr_valid(pCsr2), pRc); 64 + ret = lsm_csr_next(pCsr2); 65 + testCompareInt(LSM_MISUSE, ret, pRc); 66 + ret = lsm_csr_prev(pCsr2); 67 + testCompareInt(LSM_MISUSE, ret, pRc); 68 + 69 + lsm_csr_close(pCsr2); 70 + 71 + ret = lsm_csr_first(pCsr); 72 + testCompareInt(LSM_OK, ret, pRc); 73 + ret = lsm_csr_next(pCsr); 74 + testCompareInt(LSM_OK, ret, pRc); 75 + ret = lsm_csr_prev(pCsr); 76 + testCompareInt(LSM_MISUSE, ret, pRc); 77 + 78 + ret = lsm_csr_last(pCsr); 79 + testCompareInt(LSM_OK, ret, pRc); 80 + ret = lsm_csr_prev(pCsr); 81 + testCompareInt(LSM_OK, ret, pRc); 82 + ret = lsm_csr_next(pCsr); 83 + testCompareInt(LSM_MISUSE, ret, pRc); 84 + 85 + ret = lsm_csr_first(pCsr); 86 + while( lsm_csr_valid(pCsr) ){ 87 + ret = lsm_csr_next(pCsr); 88 + testCompareInt(LSM_OK, ret, pRc); 89 + } 90 + ret = lsm_csr_next(pCsr); 91 + testCompareInt(LSM_OK, ret, pRc); 92 + ret = lsm_csr_prev(pCsr); 93 + testCompareInt(LSM_MISUSE, ret, pRc); 94 + 95 + ret = lsm_csr_last(pCsr); 96 + while( lsm_csr_valid(pCsr) ){ 97 + ret = lsm_csr_prev(pCsr); 98 + testCompareInt(LSM_OK, ret, pRc); 99 + } 100 + ret = lsm_csr_prev(pCsr); 101 + testCompareInt(LSM_OK, ret, pRc); 102 + ret = lsm_csr_next(pCsr); 103 + testCompareInt(LSM_MISUSE, ret, pRc); 104 + 105 + lsm_csr_close(pCsr); 106 +} 107 + 108 +static void do_test_api1(const char *zPattern, int *pRc){ 109 + if( testCaseBegin(pRc, zPattern, "api1.lsm") ){ 110 + const DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 200, 250 }; 111 + Datasource *pData; 112 + TestDb *pDb; 113 + int rc = 0; 114 + 115 + pDb = testOpen("lsm_lomem", 1, &rc); 116 + pData = testDatasourceNew(&defn); 117 + testWriteDatasourceRange(pDb, pData, 0, 1000, pRc); 118 + 119 + do_test_api1_lsm(tdb_lsm(pDb), pRc); 120 + 121 + testDatasourceFree(pData); 122 + testClose(&pDb); 123 + 124 + testCaseFinish(*pRc); 125 + } 126 +} 127 + 128 +static lsm_db *newLsmConnection( 129 + const char *zDb, 130 + int nPgsz, 131 + int nBlksz, 132 + int *pRc 133 +){ 134 + lsm_db *db = 0; 135 + if( *pRc==0 ){ 136 + int n1 = nPgsz; 137 + int n2 = nBlksz; 138 + *pRc = lsm_new(tdb_lsm_env(), &db); 139 + if( *pRc==0 ){ 140 + if( n1 ) lsm_config(db, LSM_CONFIG_PAGE_SIZE, &n1); 141 + if( n2 ) lsm_config(db, LSM_CONFIG_BLOCK_SIZE, &n2); 142 + *pRc = lsm_open(db, "testdb.lsm"); 143 + } 144 + } 145 + return db; 146 +} 147 + 148 +static void testPagesize(lsm_db *db, int nPgsz, int nBlksz, int *pRc){ 149 + if( *pRc==0 ){ 150 + int n1 = 0; 151 + int n2 = 0; 152 + 153 + lsm_config(db, LSM_CONFIG_PAGE_SIZE, &n1); 154 + lsm_config(db, LSM_CONFIG_BLOCK_SIZE, &n2); 155 + 156 + testCompareInt(n1, nPgsz, pRc); 157 + testCompareInt(n2, nBlksz, pRc); 158 + } 159 +} 160 + 161 +/* 162 +** Test case "api2" tests that the default page and block sizes of a 163 +** database may only be modified before lsm_open() is called. And that 164 +** after lsm_open() is called lsm_config() may be used to read the 165 +** actual page and block size of the db. 166 +*/ 167 +static void do_test_api2(const char *zPattern, int *pRc){ 168 + if( *pRc==0 && testCaseBegin(pRc, zPattern, "api2.lsm") ){ 169 + lsm_db *db1 = 0; 170 + lsm_db *db2 = 0; 171 + 172 + testDeleteLsmdb("testdb.lsm"); 173 + db1 = newLsmConnection("testdb.lsm", 0, 0, pRc); 174 + testPagesize(db1, 4096, 1024, pRc); 175 + db2 = newLsmConnection("testdb.lsm", 1024, 64*1024, pRc); 176 + testPagesize(db2, 4096, 1024, pRc); 177 + lsm_close(db1); 178 + lsm_close(db2); 179 + 180 + testDeleteLsmdb("testdb.lsm"); 181 + db1 = newLsmConnection("testdb.lsm", 1024, 64*1024, pRc); 182 + testPagesize(db1, 1024, 64*1024, pRc); 183 + db2 = newLsmConnection("testdb.lsm", 0, 0, pRc); 184 + testPagesize(db2, 1024, 64*1024, pRc); 185 + lsm_close(db1); 186 + lsm_close(db2); 187 + 188 + testDeleteLsmdb("testdb.lsm"); 189 + db1 = newLsmConnection("testdb.lsm", 8192, 2*1024, pRc); 190 + testPagesize(db1, 8192, 2*1024, pRc); 191 + db2 = newLsmConnection("testdb.lsm", 1024, 64*1024, pRc); 192 + testPagesize(db2, 8192, 2*1024, pRc); 193 + lsm_close(db1); 194 + lsm_close(db2); 195 + 196 + testCaseFinish(*pRc); 197 + } 198 +} 199 + 200 +void test_api( 201 + const char *zPattern, /* Run test cases that match this pattern */ 202 + int *pRc /* IN/OUT: Error code */ 203 +){ 204 + do_test_api1(zPattern, pRc); 205 + do_test_api2(zPattern, pRc); 206 +}
Added ext/lsm1/lsm-test/lsmtest8.c.
1 + 2 +/* 3 +** This file contains test cases to verify that "live-recovery" following 4 +** a mid-transaction failure of a writer process. 5 +*/ 6 + 7 + 8 +/* 9 +** This test file includes lsmInt.h to get access to the definition of the 10 +** ShmHeader structure. This is required to cause strategic damage to the 11 +** shared memory header as part of recovery testing. 12 +*/ 13 +#include "lsmInt.h" 14 + 15 +#include "lsmtest.h" 16 + 17 +typedef struct SetupStep SetupStep; 18 +struct SetupStep { 19 + int bFlush; /* Flush to disk and checkpoint */ 20 + int iInsStart; /* First key-value from ds to insert */ 21 + int nIns; /* Number of rows to insert */ 22 + int iDelStart; /* First key from ds to delete */ 23 + int nDel; /* Number of rows to delete */ 24 +}; 25 + 26 +static void doSetupStep( 27 + TestDb *pDb, 28 + Datasource *pData, 29 + const SetupStep *pStep, 30 + int *pRc 31 +){ 32 + testWriteDatasourceRange(pDb, pData, pStep->iInsStart, pStep->nIns, pRc); 33 + testDeleteDatasourceRange(pDb, pData, pStep->iDelStart, pStep->nDel, pRc); 34 + if( *pRc==0 ){ 35 + int nSave = -1; 36 + int nBuf = 64; 37 + lsm_db *db = tdb_lsm(pDb); 38 + 39 + lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nSave); 40 + lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nBuf); 41 + lsm_begin(db, 1); 42 + lsm_commit(db, 0); 43 + lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nSave); 44 + 45 + *pRc = lsm_work(db, 0, 0, 0); 46 + if( *pRc==0 ){ 47 + *pRc = lsm_checkpoint(db, 0); 48 + } 49 + } 50 +} 51 + 52 +static void doSetupStepArray( 53 + TestDb *pDb, 54 + Datasource *pData, 55 + const SetupStep *aStep, 56 + int nStep 57 +){ 58 + int i; 59 + for(i=0; i<nStep; i++){ 60 + int rc = 0; 61 + doSetupStep(pDb, pData, &aStep[i], &rc); 62 + assert( rc==0 ); 63 + } 64 +} 65 + 66 +static void setupDatabase1(TestDb *pDb, Datasource **ppData){ 67 + const SetupStep aStep[] = { 68 + { 0, 1, 2000, 0, 0 }, 69 + { 1, 0, 0, 0, 0 }, 70 + { 0, 10001, 1000, 0, 0 }, 71 + }; 72 + const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 100, 500}; 73 + Datasource *pData; 74 + 75 + pData = testDatasourceNew(&defn); 76 + doSetupStepArray(pDb, pData, aStep, ArraySize(aStep)); 77 + if( ppData ){ 78 + *ppData = pData; 79 + }else{ 80 + testDatasourceFree(pData); 81 + } 82 +} 83 + 84 +#include <stdio.h> 85 +void testReadFile(const char *zFile, int iOff, void *pOut, int nByte, int *pRc){ 86 + if( *pRc==0 ){ 87 + FILE *fd; 88 + fd = fopen(zFile, "rb"); 89 + if( fd==0 ){ 90 + *pRc = 1; 91 + }else{ 92 + if( 0!=fseek(fd, iOff, SEEK_SET) ){ 93 + *pRc = 1; 94 + }else{ 95 + assert( nByte>=0 ); 96 + if( (size_t)nByte!=fread(pOut, 1, nByte, fd) ){ 97 + *pRc = 1; 98 + } 99 + } 100 + fclose(fd); 101 + } 102 + } 103 +} 104 + 105 +void testWriteFile( 106 + const char *zFile, 107 + int iOff, 108 + void *pOut, 109 + int nByte, 110 + int *pRc 111 +){ 112 + if( *pRc==0 ){ 113 + FILE *fd; 114 + fd = fopen(zFile, "r+b"); 115 + if( fd==0 ){ 116 + *pRc = 1; 117 + }else{ 118 + if( 0!=fseek(fd, iOff, SEEK_SET) ){ 119 + *pRc = 1; 120 + }else{ 121 + assert( nByte>=0 ); 122 + if( (size_t)nByte!=fwrite(pOut, 1, nByte, fd) ){ 123 + *pRc = 1; 124 + } 125 + } 126 + fclose(fd); 127 + } 128 + } 129 +} 130 + 131 +static ShmHeader *getShmHeader(const char *zDb){ 132 + int rc = 0; 133 + char *zShm = testMallocPrintf("%s-shm", zDb); 134 + ShmHeader *pHdr; 135 + 136 + pHdr = testMalloc(sizeof(ShmHeader)); 137 + testReadFile(zShm, 0, (void *)pHdr, sizeof(ShmHeader), &rc); 138 + assert( rc==0 ); 139 + 140 + return pHdr; 141 +} 142 + 143 +/* 144 +** This function makes a copy of the three files associated with LSM 145 +** database zDb (i.e. if zDb is "test.db", it makes copies of "test.db", 146 +** "test.db-log" and "test.db-shm"). 147 +** 148 +** It then opens a new database connection to the copy with the xLock() call 149 +** instrumented so that it appears that some other process already connected 150 +** to the db (holding a shared lock on DMS2). This prevents recovery from 151 +** running. Then: 152 +** 153 +** 1) Check that the checksum of the database is zCksum. 154 +** 2) Write a few keys to the database. Then delete the same keys. 155 +** 3) Check that the checksum is zCksum. 156 +** 4) Flush the db to disk and run a checkpoint. 157 +** 5) Check once more that the checksum is still zCksum. 158 +*/ 159 +static void doLiveRecovery(const char *zDb, const char *zCksum, int *pRc){ 160 + if( *pRc==LSM_OK ){ 161 + const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 20, 25, 100, 500}; 162 + Datasource *pData; 163 + const char *zCopy = "testcopy.lsm"; 164 + char zCksum2[TEST_CKSUM_BYTES]; 165 + TestDb *pDb = 0; 166 + int rc; 167 + 168 + pData = testDatasourceNew(&defn); 169 + 170 + testCopyLsmdb(zDb, zCopy); 171 + rc = tdb_lsm_open("test_no_recovery=1", zCopy, 0, &pDb); 172 + if( rc==0 ){ 173 + ShmHeader *pHdr; 174 + lsm_db *db; 175 + testCksumDatabase(pDb, zCksum2); 176 + testCompareStr(zCksum, zCksum2, &rc); 177 + 178 + testWriteDatasourceRange(pDb, pData, 1, 10, &rc); 179 + testDeleteDatasourceRange(pDb, pData, 1, 10, &rc); 180 + 181 + /* Test that the two tree-headers are now consistent. */ 182 + pHdr = getShmHeader(zCopy); 183 + if( rc==0 && memcmp(&pHdr->hdr1, &pHdr->hdr2, sizeof(pHdr->hdr1)) ){ 184 + rc = 1; 185 + } 186 + testFree(pHdr); 187 + 188 + if( rc==0 ){ 189 + int nBuf = 64; 190 + db = tdb_lsm(pDb); 191 + lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nBuf); 192 + lsm_begin(db, 1); 193 + lsm_commit(db, 0); 194 + rc = lsm_work(db, 0, 0, 0); 195 + } 196 + 197 + testCksumDatabase(pDb, zCksum2); 198 + testCompareStr(zCksum, zCksum2, &rc); 199 + } 200 + 201 + testDatasourceFree(pData); 202 + testClose(&pDb); 203 + testDeleteLsmdb(zCopy); 204 + *pRc = rc; 205 + } 206 +} 207 + 208 +static void doWriterCrash1(int *pRc){ 209 + const int nWrite = 2000; 210 + const int nStep = 10; 211 + const int iWriteStart = 20000; 212 + int rc = 0; 213 + TestDb *pDb = 0; 214 + Datasource *pData = 0; 215 + 216 + rc = tdb_lsm_open("autowork=0", "testdb.lsm", 1, &pDb); 217 + if( rc==0 ){ 218 + int iDot = 0; 219 + char zCksum[TEST_CKSUM_BYTES]; 220 + int i; 221 + setupDatabase1(pDb, &pData); 222 + testCksumDatabase(pDb, zCksum); 223 + testBegin(pDb, 2, &rc); 224 + for(i=0; rc==0 && i<nWrite; i+=nStep){ 225 + testCaseProgress(i, nWrite, testCaseNDot(), &iDot); 226 + testWriteDatasourceRange(pDb, pData, iWriteStart+i, nStep, &rc); 227 + doLiveRecovery("testdb.lsm", zCksum, &rc); 228 + } 229 + } 230 + testCommit(pDb, 0, &rc); 231 + testClose(&pDb); 232 + testDatasourceFree(pData); 233 + *pRc = rc; 234 +} 235 + 236 +/* 237 +** This test case verifies that inconsistent tree-headers in shared-memory 238 +** are resolved correctly. 239 +*/ 240 +static void doWriterCrash2(int *pRc){ 241 + int rc = 0; 242 + TestDb *pDb = 0; 243 + Datasource *pData = 0; 244 + 245 + rc = tdb_lsm_open("autowork=0", "testdb.lsm", 1, &pDb); 246 + if( rc==0 ){ 247 + ShmHeader *pHdr1; 248 + ShmHeader *pHdr2; 249 + char zCksum1[TEST_CKSUM_BYTES]; 250 + char zCksum2[TEST_CKSUM_BYTES]; 251 + 252 + pHdr1 = testMalloc(sizeof(ShmHeader)); 253 + pHdr2 = testMalloc(sizeof(ShmHeader)); 254 + setupDatabase1(pDb, &pData); 255 + 256 + /* Grab a copy of the shared-memory header. And the db checksum */ 257 + testReadFile("testdb.lsm-shm", 0, (void *)pHdr1, sizeof(ShmHeader), &rc); 258 + testCksumDatabase(pDb, zCksum1); 259 + 260 + /* Modify the database */ 261 + testBegin(pDb, 2, &rc); 262 + testWriteDatasourceRange(pDb, pData, 30000, 200, &rc); 263 + testCommit(pDb, 0, &rc); 264 + 265 + /* Grab a second copy of the shared-memory header. And the db checksum */ 266 + testReadFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); 267 + testCksumDatabase(pDb, zCksum2); 268 + doLiveRecovery("testdb.lsm", zCksum2, &rc); 269 + 270 + /* If both tree-headers are valid, tree-header-1 is used. */ 271 + memcpy(&pHdr2->hdr1, &pHdr1->hdr1, sizeof(pHdr1->hdr1)); 272 + pHdr2->bWriter = 1; 273 + testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); 274 + doLiveRecovery("testdb.lsm", zCksum1, &rc); 275 + 276 + /* If both tree-headers are valid, tree-header-1 is used. */ 277 + memcpy(&pHdr2->hdr1, &pHdr2->hdr2, sizeof(pHdr1->hdr1)); 278 + memcpy(&pHdr2->hdr2, &pHdr1->hdr1, sizeof(pHdr1->hdr1)); 279 + pHdr2->bWriter = 1; 280 + testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); 281 + doLiveRecovery("testdb.lsm", zCksum2, &rc); 282 + 283 + /* If tree-header 1 is invalid, tree-header-2 is used */ 284 + memcpy(&pHdr2->hdr2, &pHdr2->hdr1, sizeof(pHdr1->hdr1)); 285 + pHdr2->hdr1.aCksum[0] = 5; 286 + pHdr2->hdr1.aCksum[0] = 6; 287 + pHdr2->bWriter = 1; 288 + testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); 289 + doLiveRecovery("testdb.lsm", zCksum2, &rc); 290 + 291 + /* If tree-header 2 is invalid, tree-header-1 is used */ 292 + memcpy(&pHdr2->hdr1, &pHdr2->hdr2, sizeof(pHdr1->hdr1)); 293 + pHdr2->hdr2.aCksum[0] = 5; 294 + pHdr2->hdr2.aCksum[0] = 6; 295 + pHdr2->bWriter = 1; 296 + testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); 297 + doLiveRecovery("testdb.lsm", zCksum2, &rc); 298 + 299 + testFree(pHdr1); 300 + testFree(pHdr2); 301 + testClose(&pDb); 302 + } 303 + 304 + *pRc = rc; 305 +} 306 + 307 +void do_writer_crash_test(const char *zPattern, int *pRc){ 308 + struct Test { 309 + const char *zName; 310 + void (*xFunc)(int *); 311 + } aTest[] = { 312 + { "writercrash1.lsm", doWriterCrash1 }, 313 + { "writercrash2.lsm", doWriterCrash2 }, 314 + }; 315 + int i; 316 + for(i=0; i<ArraySize(aTest); i++){ 317 + struct Test *p = &aTest[i]; 318 + if( testCaseBegin(pRc, zPattern, p->zName) ){ 319 + p->xFunc(pRc); 320 + testCaseFinish(*pRc); 321 + } 322 + } 323 + 324 +} 325 + 326 +
Added ext/lsm1/lsm-test/lsmtest9.c.
1 + 2 +#include "lsmtest.h" 3 + 4 +#define DATA_SEQUENTIAL TEST_DATASOURCE_SEQUENCE 5 +#define DATA_RANDOM TEST_DATASOURCE_RANDOM 6 + 7 +typedef struct Datatest4 Datatest4; 8 + 9 +/* 10 +** Test overview: 11 +** 12 +** 1. Insert (Datatest4.nRec) records into a database. 13 +** 14 +** 2. Repeat (Datatest4.nRepeat) times: 15 +** 16 +** 2a. Delete 2/3 of the records in the database. 17 +** 18 +** 2b. Run lsm_work(nMerge=1). 19 +** 20 +** 2c. Insert as many records as were deleted in 2a. 21 +** 22 +** 2d. Check database content is as expected. 23 +** 24 +** 2e. If (Datatest4.bReopen) is true, close and reopen the database. 25 +*/ 26 +struct Datatest4 { 27 + /* Datasource definition */ 28 + DatasourceDefn defn; 29 + 30 + int nRec; 31 + int nRepeat; 32 + int bReopen; 33 +}; 34 + 35 +static void doDataTest4( 36 + const char *zSystem, /* Database system to test */ 37 + Datatest4 *p, /* Structure containing test parameters */ 38 + int *pRc /* OUT: Error code */ 39 +){ 40 + lsm_db *db = 0; 41 + TestDb *pDb; 42 + TestDb *pControl; 43 + Datasource *pData; 44 + int i; 45 + int rc = 0; 46 + int iDot = 0; 47 + int bMultiThreaded = 0; /* True for MT LSM database */ 48 + 49 + int nRecOn3 = (p->nRec / 3); 50 + int iData = 0; 51 + 52 + /* Start the test case, open a database and allocate the datasource. */ 53 + rc = testControlDb(&pControl); 54 + pDb = testOpen(zSystem, 1, &rc); 55 + pData = testDatasourceNew(&p->defn); 56 + if( rc==0 ){ 57 + db = tdb_lsm(pDb); 58 + bMultiThreaded = tdb_lsm_multithread(pDb); 59 + } 60 + 61 + testWriteDatasourceRange(pControl, pData, iData, nRecOn3*3, &rc); 62 + testWriteDatasourceRange(pDb, pData, iData, nRecOn3*3, &rc); 63 + 64 + for(i=0; rc==0 && i<p->nRepeat; i++){ 65 + 66 + testDeleteDatasourceRange(pControl, pData, iData, nRecOn3*2, &rc); 67 + testDeleteDatasourceRange(pDb, pData, iData, nRecOn3*2, &rc); 68 + 69 + if( db ){ 70 + int nDone; 71 +#if 0 72 + fprintf(stderr, "lsm_work() start...\n"); fflush(stderr); 73 +#endif 74 + do { 75 + nDone = 0; 76 + rc = lsm_work(db, 1, (1<<30), &nDone); 77 + }while( rc==0 && nDone>0 ); 78 + if( bMultiThreaded && rc==LSM_BUSY ) rc = LSM_OK; 79 +#if 0 80 + fprintf(stderr, "lsm_work() done...\n"); fflush(stderr); 81 +#endif 82 + } 83 + 84 +if( i+1<p->nRepeat ){ 85 + iData += (nRecOn3*2); 86 + testWriteDatasourceRange(pControl, pData, iData+nRecOn3, nRecOn3*2, &rc); 87 + testWriteDatasourceRange(pDb, pData, iData+nRecOn3, nRecOn3*2, &rc); 88 + 89 + testCompareDb(pData, nRecOn3*3, iData, pControl, pDb, &rc); 90 + 91 + /* If Datatest4.bReopen is true, close and reopen the database */ 92 + if( p->bReopen ){ 93 + testReopen(&pDb, &rc); 94 + if( rc==0 ) db = tdb_lsm(pDb); 95 + } 96 +} 97 + 98 + /* Update the progress dots... */ 99 + testCaseProgress(i, p->nRepeat, testCaseNDot(), &iDot); 100 + } 101 + 102 + testClose(&pDb); 103 + testClose(&pControl); 104 + testDatasourceFree(pData); 105 + testCaseFinish(rc); 106 + *pRc = rc; 107 +} 108 + 109 +static char *getName4(const char *zSystem, Datatest4 *pTest){ 110 + char *zRet; 111 + char *zData; 112 + zData = testDatasourceName(&pTest->defn); 113 + zRet = testMallocPrintf("data4.%s.%s.%d.%d.%d", 114 + zSystem, zData, pTest->nRec, pTest->nRepeat, pTest->bReopen 115 + ); 116 + testFree(zData); 117 + return zRet; 118 +} 119 + 120 +void test_data_4( 121 + const char *zSystem, /* Database system name */ 122 + const char *zPattern, /* Run test cases that match this pattern */ 123 + int *pRc /* IN/OUT: Error code */ 124 +){ 125 + Datatest4 aTest[] = { 126 + /* defn, nRec, nRepeat, bReopen */ 127 + { {DATA_RANDOM, 20,25, 500,600}, 10000, 10, 0 }, 128 + { {DATA_RANDOM, 20,25, 500,600}, 10000, 10, 1 }, 129 + }; 130 + 131 + int i; 132 + 133 + for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){ 134 + char *zName = getName4(zSystem, &aTest[i]); 135 + if( testCaseBegin(pRc, zPattern, "%s", zName) ){ 136 + doDataTest4(zSystem, &aTest[i], pRc); 137 + } 138 + testFree(zName); 139 + } 140 +} 141 + 142 + 143 +
Added ext/lsm1/lsm-test/lsmtest_bt.c.
1 + 2 +#include "lsmtest.h" 3 +#include "bt.h" 4 + 5 +int do_bt(int nArg, char **azArg){ 6 + struct Option { 7 + const char *zName; 8 + int bPgno; 9 + int eOpt; 10 + } aOpt [] = { 11 + { "dbhdr", 0, BT_INFO_HDRDUMP }, 12 + { "filename", 0, BT_INFO_FILENAME }, 13 + { "block_freelist", 0, BT_INFO_BLOCK_FREELIST }, 14 + { "page_freelist", 0, BT_INFO_PAGE_FREELIST }, 15 + { "filename", 0, BT_INFO_FILENAME }, 16 + { "page", 1, BT_INFO_PAGEDUMP }, 17 + { "page_ascii", 1, BT_INFO_PAGEDUMP_ASCII }, 18 + { "leaks", 0, BT_INFO_PAGE_LEAKS }, 19 + { 0, 0 } 20 + }; 21 + int iOpt; 22 + int rc; 23 + bt_info buf; 24 + char *zOpt; 25 + char *zFile; 26 + 27 + bt_db *db = 0; 28 + 29 + if( nArg<2 ){ 30 + testPrintUsage("FILENAME OPTION ..."); 31 + return -1; 32 + } 33 + zFile = azArg[0]; 34 + zOpt = azArg[1]; 35 + 36 + rc = testArgSelect(aOpt, "option", zOpt, &iOpt); 37 + if( rc!=0 ) return rc; 38 + if( nArg!=2+aOpt[iOpt].bPgno ){ 39 + testPrintFUsage("FILENAME %s %s", zOpt, aOpt[iOpt].bPgno ? "PGNO" : ""); 40 + return -4; 41 + } 42 + 43 + rc = sqlite4BtNew(sqlite4_env_default(), 0, &db); 44 + if( rc!=SQLITE4_OK ){ 45 + testPrintError("sqlite4BtNew() failed: %d", rc); 46 + return -2; 47 + } 48 + rc = sqlite4BtOpen(db, zFile); 49 + if( rc!=SQLITE4_OK ){ 50 + testPrintError("sqlite4BtOpen() failed: %d", rc); 51 + return -3; 52 + } 53 + 54 + buf.eType = aOpt[iOpt].eOpt; 55 + buf.pgno = 0; 56 + sqlite4_buffer_init(&buf.output, 0); 57 + 58 + if( aOpt[iOpt].bPgno ){ 59 + buf.pgno = (u32)atoi(azArg[2]); 60 + } 61 + 62 + rc = sqlite4BtControl(db, BT_CONTROL_INFO, &buf); 63 + if( rc!=SQLITE4_OK ){ 64 + testPrintError("sqlite4BtControl() failed: %d\n", rc); 65 + return -4; 66 + } 67 + 68 + printf("%s\n", (char*)buf.output.p); 69 + sqlite4_buffer_clear(&buf.output); 70 + return 0; 71 +} 72 + 73 + 74 + 75 +
Added ext/lsm1/lsm-test/lsmtest_datasource.c.
1 + 2 + 3 +#include "lsmtest.h" 4 + 5 +struct Datasource { 6 + int eType; 7 + 8 + int nMinKey; 9 + int nMaxKey; 10 + int nMinVal; 11 + int nMaxVal; 12 + 13 + char *aKey; 14 + char *aVal; 15 +}; 16 + 17 +void testDatasourceEntry( 18 + Datasource *p, 19 + int iData, 20 + void **ppKey, int *pnKey, 21 + void **ppVal, int *pnVal 22 +){ 23 + assert( (ppKey==0)==(pnKey==0) ); 24 + assert( (ppVal==0)==(pnVal==0) ); 25 + 26 + if( ppKey ){ 27 + int nKey = 0; 28 + switch( p->eType ){ 29 + case TEST_DATASOURCE_RANDOM: { 30 + int nRange = (1 + p->nMaxKey - p->nMinKey); 31 + nKey = (int)( testPrngValue((u32)iData) % nRange ) + p->nMinKey; 32 + testPrngString((u32)iData, p->aKey, nKey); 33 + break; 34 + } 35 + case TEST_DATASOURCE_SEQUENCE: 36 + nKey = sprintf(p->aKey, "%012d", iData); 37 + break; 38 + } 39 + *ppKey = p->aKey; 40 + *pnKey = nKey; 41 + } 42 + if( ppVal ){ 43 + u32 nVal = testPrngValue((u32)iData)%(1+p->nMaxVal-p->nMinVal)+p->nMinVal; 44 + testPrngString((u32)~iData, p->aVal, (int)nVal); 45 + *ppVal = p->aVal; 46 + *pnVal = (int)nVal; 47 + } 48 +} 49 + 50 +void testDatasourceFree(Datasource *p){ 51 + testFree(p); 52 +} 53 + 54 +/* 55 +** Return a pointer to a nul-terminated string that corresponds to the 56 +** contents of the datasource-definition passed as the first argument. 57 +** The caller should eventually free the returned pointer using testFree(). 58 +*/ 59 +char *testDatasourceName(const DatasourceDefn *p){ 60 + char *zRet; 61 + zRet = testMallocPrintf("%s.(%d-%d).(%d-%d)", 62 + (p->eType==TEST_DATASOURCE_SEQUENCE ? "seq" : "rnd"), 63 + p->nMinKey, p->nMaxKey, 64 + p->nMinVal, p->nMaxVal 65 + ); 66 + return zRet; 67 +} 68 + 69 +Datasource *testDatasourceNew(const DatasourceDefn *pDefn){ 70 + Datasource *p; 71 + int nMinKey; 72 + int nMaxKey; 73 + int nMinVal; 74 + int nMaxVal; 75 + 76 + if( pDefn->eType==TEST_DATASOURCE_SEQUENCE ){ 77 + nMinKey = 128; 78 + nMaxKey = 128; 79 + }else{ 80 + nMinKey = MAX(0, pDefn->nMinKey); 81 + nMaxKey = MAX(nMinKey, pDefn->nMaxKey); 82 + } 83 + nMinVal = MAX(0, pDefn->nMinVal); 84 + nMaxVal = MAX(nMinVal, pDefn->nMaxVal); 85 + 86 + p = (Datasource *)testMalloc(sizeof(Datasource) + nMaxKey + nMaxVal + 1); 87 + p->eType = pDefn->eType; 88 + p->nMinKey = nMinKey; 89 + p->nMinVal = nMinVal; 90 + p->nMaxKey = nMaxKey; 91 + p->nMaxVal = nMaxVal; 92 + 93 + p->aKey = (char *)&p[1]; 94 + p->aVal = &p->aKey[nMaxKey]; 95 + return p; 96 +};
Added ext/lsm1/lsm-test/lsmtest_func.c.
1 + 2 +#include "lsmtest.h" 3 + 4 + 5 +int do_work(int nArg, char **azArg){ 6 + struct Option { 7 + const char *zName; 8 + } aOpt [] = { 9 + { "-nmerge" }, 10 + { "-nkb" }, 11 + { 0 } 12 + }; 13 + 14 + lsm_db *pDb; 15 + int rc; 16 + int i; 17 + const char *zDb; 18 + int nMerge = 1; 19 + int nKB = (1<<30); 20 + 21 + if( nArg==0 ) goto usage; 22 + zDb = azArg[nArg-1]; 23 + for(i=0; i<(nArg-1); i++){ 24 + int iSel; 25 + rc = testArgSelect(aOpt, "option", azArg[i], &iSel); 26 + if( rc ) return rc; 27 + switch( iSel ){ 28 + case 0: 29 + i++; 30 + if( i==(nArg-1) ) goto usage; 31 + nMerge = atoi(azArg[i]); 32 + break; 33 + case 1: 34 + i++; 35 + if( i==(nArg-1) ) goto usage; 36 + nKB = atoi(azArg[i]); 37 + break; 38 + } 39 + } 40 + 41 + rc = lsm_new(0, &pDb); 42 + if( rc!=LSM_OK ){ 43 + testPrintError("lsm_open(): rc=%d\n", rc); 44 + }else{ 45 + rc = lsm_open(pDb, zDb); 46 + if( rc!=LSM_OK ){ 47 + testPrintError("lsm_open(): rc=%d\n", rc); 48 + }else{ 49 + int n = -1; 50 + lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &n); 51 + n = n*2; 52 + lsm_config(pDb, LSM_CONFIG_AUTOCHECKPOINT, &n); 53 + 54 + rc = lsm_work(pDb, nMerge, nKB, 0); 55 + if( rc!=LSM_OK ){ 56 + testPrintError("lsm_work(): rc=%d\n", rc); 57 + } 58 + } 59 + } 60 + if( rc==LSM_OK ){ 61 + rc = lsm_checkpoint(pDb, 0); 62 + } 63 + 64 + lsm_close(pDb); 65 + return rc; 66 + 67 + usage: 68 + testPrintUsage("?-optimize? ?-n N? DATABASE"); 69 + return -1; 70 +} 71 + 72 + 73 +/* 74 +** lsmtest show ?-config LSM-CONFIG? DATABASE ?COMMAND ?PGNO?? 75 +*/ 76 +int do_show(int nArg, char **azArg){ 77 + lsm_db *pDb; 78 + int rc; 79 + const char *zDb; 80 + 81 + int eOpt = LSM_INFO_DB_STRUCTURE; 82 + unsigned int iPg = 0; 83 + int bConfig = 0; 84 + const char *zConfig = ""; 85 + 86 + struct Option { 87 + const char *zName; 88 + int bConfig; 89 + int eOpt; 90 + } aOpt [] = { 91 + { "array", 0, LSM_INFO_ARRAY_STRUCTURE }, 92 + { "array-pages", 0, LSM_INFO_ARRAY_PAGES }, 93 + { "blocksize", 1, LSM_CONFIG_BLOCK_SIZE }, 94 + { "pagesize", 1, LSM_CONFIG_PAGE_SIZE }, 95 + { "freelist", 0, LSM_INFO_FREELIST }, 96 + { "page-ascii", 0, LSM_INFO_PAGE_ASCII_DUMP }, 97 + { "page-hex", 0, LSM_INFO_PAGE_HEX_DUMP }, 98 + { 0, 0 } 99 + }; 100 + 101 + char *z = 0; 102 + int iDb = 0; /* Index of DATABASE in azArg[] */ 103 + 104 + /* Check if there is a "-config" option: */ 105 + if( nArg>2 && strlen(azArg[0])>1 106 + && memcmp(azArg[0], "-config", strlen(azArg[0]))==0 107 + ){ 108 + zConfig = azArg[1]; 109 + iDb = 2; 110 + } 111 + if( nArg<(iDb+1) ) goto usage; 112 + 113 + if( nArg>(iDb+1) ){ 114 + rc = testArgSelect(aOpt, "option", azArg[iDb+1], &eOpt); 115 + if( rc!=0 ) return rc; 116 + bConfig = aOpt[eOpt].bConfig; 117 + eOpt = aOpt[eOpt].eOpt; 118 + if( (bConfig==0 && eOpt==LSM_INFO_FREELIST) 119 + || (bConfig==1 && eOpt==LSM_CONFIG_BLOCK_SIZE) 120 + || (bConfig==1 && eOpt==LSM_CONFIG_PAGE_SIZE) 121 + ){ 122 + if( nArg!=(iDb+2) ) goto usage; 123 + }else{ 124 + if( nArg!=(iDb+3) ) goto usage; 125 + iPg = atoi(azArg[iDb+2]); 126 + } 127 + } 128 + zDb = azArg[iDb]; 129 + 130 + rc = lsm_new(0, &pDb); 131 + tdb_lsm_configure(pDb, zConfig); 132 + if( rc!=LSM_OK ){ 133 + testPrintError("lsm_new(): rc=%d\n", rc); 134 + }else{ 135 + rc = lsm_open(pDb, zDb); 136 + if( rc!=LSM_OK ){ 137 + testPrintError("lsm_open(): rc=%d\n", rc); 138 + } 139 + } 140 + 141 + if( rc==LSM_OK ){ 142 + if( bConfig==0 ){ 143 + switch( eOpt ){ 144 + case LSM_INFO_DB_STRUCTURE: 145 + case LSM_INFO_FREELIST: 146 + rc = lsm_info(pDb, eOpt, &z); 147 + break; 148 + case LSM_INFO_ARRAY_STRUCTURE: 149 + case LSM_INFO_ARRAY_PAGES: 150 + case LSM_INFO_PAGE_ASCII_DUMP: 151 + case LSM_INFO_PAGE_HEX_DUMP: 152 + rc = lsm_info(pDb, eOpt, iPg, &z); 153 + break; 154 + default: 155 + assert( !"no chance" ); 156 + } 157 + 158 + if( rc==LSM_OK ){ 159 + printf("%s\n", z ? z : ""); 160 + fflush(stdout); 161 + } 162 + lsm_free(lsm_get_env(pDb), z); 163 + }else{ 164 + int iRes = -1; 165 + lsm_config(pDb, eOpt, &iRes); 166 + printf("%d\n", iRes); 167 + fflush(stdout); 168 + } 169 + } 170 + 171 + lsm_close(pDb); 172 + return rc; 173 + 174 + usage: 175 + testPrintUsage("DATABASE ?array|page-ascii|page-hex PGNO?"); 176 + return -1; 177 +}
Added ext/lsm1/lsm-test/lsmtest_io.c.
1 + 2 +/* 3 +** SUMMARY 4 +** 5 +** This file implements the 'io' subcommand of the test program. It is used 6 +** for testing the performance of various combinations of write() and fsync() 7 +** system calls. All operations occur on a single file, which may or may not 8 +** exist when a test is started. 9 +** 10 +** A test consists of a series of commands. Each command is either a write 11 +** or an fsync. A write is specified as "<amount>@<offset>", where <amount> 12 +** is the amount of data written, and <offset> is the offset of the file 13 +** to write to. An <amount> or an <offset> is specified as an integer number 14 +** of bytes. Or, if postfixed with a "K", "M" or "G", an integer number of 15 +** KB, MB or GB, respectively. An fsync is simply "S". All commands are 16 +** case-insensitive. 17 +** 18 +** Example test program: 19 +** 20 +** 2M@6M 1492K@4M S 4096@4K S 21 +** 22 +** This program writes 2 MB of data starting at the offset 6MB offset of 23 +** the file, followed by 1492 KB of data written at the 4MB offset of the 24 +** file, followed by a call to fsync(), a write of 4KB of data at byte 25 +** offset 4096, and finally another call to fsync(). 26 +** 27 +** Commands may either be specified on the command line (one command per 28 +** command line argument) or read from stdin. Commands read from stdin 29 +** must be separated by white-space. 30 +** 31 +** COMMAND LINE INVOCATION 32 +** 33 +** The sub-command implemented in this file must be invoked with at least 34 +** two arguments - the path to the file to write to and the page-size to 35 +** use for writing. If there are more than two arguments, then each 36 +** subsequent argument is assumed to be a test command. If there are exactly 37 +** two arguments, the test commands are read from stdin. 38 +** 39 +** A write command does not result in a single call to system call write(). 40 +** Instead, the specified region is written sequentially using one or 41 +** more calls to write(), each of which writes not more than one page of 42 +** data. For example, if the page-size is 4KB, the command "2M@6M" results 43 +** in 512 calls to write(), each of which writes 4KB of data. 44 +** 45 +** EXAMPLES 46 +** 47 +** Two equivalent examples: 48 +** 49 +** $ lsmtest io testfile.db 4KB 2M@6M 1492K@4M S 4096@4K S 50 +** 3544K written in 129 ms 51 +** $ echo "2M@6M 1492K@4M S 4096@4K S" | lsmtest io testfile.db 4096 52 +** 3544K written in 127 ms 53 +** 54 +*/ 55 + 56 +#include "lsmtest.h" 57 + 58 +typedef struct IoContext IoContext; 59 + 60 +struct IoContext { 61 + int fd; 62 + int nWrite; 63 +}; 64 + 65 +/* 66 +** As isspace(3) 67 +*/ 68 +static int safe_isspace(char c){ 69 + if( c&0x80) return 0; 70 + return isspace(c); 71 +} 72 + 73 +/* 74 +** As isdigit(3) 75 +*/ 76 +static int safe_isdigit(char c){ 77 + if( c&0x80) return 0; 78 + return isdigit(c); 79 +} 80 + 81 +static i64 getNextSize(char *zIn, char **pzOut, int *pRc){ 82 + i64 iRet = 0; 83 + if( *pRc==0 ){ 84 + char *z = zIn; 85 + 86 + if( !safe_isdigit(*z) ){ 87 + *pRc = 1; 88 + return 0; 89 + } 90 + 91 + /* Process digits */ 92 + while( safe_isdigit(*z) ){ 93 + iRet = iRet*10 + (*z - '0'); 94 + z++; 95 + } 96 + 97 + /* Process suffix */ 98 + switch( *z ){ 99 + case 'k': case 'K': 100 + iRet = iRet * 1024; 101 + z++; 102 + break; 103 + 104 + case 'm': case 'M': 105 + iRet = iRet * 1024 * 1024; 106 + z++; 107 + break; 108 + 109 + case 'g': case 'G': 110 + iRet = iRet * 1024 * 1024 * 1024; 111 + z++; 112 + break; 113 + } 114 + 115 + if( pzOut ) *pzOut = z; 116 + } 117 + return iRet; 118 +} 119 + 120 +static int doOneCmd( 121 + IoContext *pCtx, 122 + u8 *aData, 123 + int pgsz, 124 + char *zCmd, 125 + char **pzOut 126 +){ 127 + char c; 128 + char *z = zCmd; 129 + 130 + while( safe_isspace(*z) ) z++; 131 + c = *z; 132 + 133 + if( c==0 ){ 134 + if( pzOut ) *pzOut = z; 135 + return 0; 136 + } 137 + 138 + if( c=='s' || c=='S' ){ 139 + if( pzOut ) *pzOut = &z[1]; 140 + return fdatasync(pCtx->fd); 141 + } 142 + 143 + if( safe_isdigit(c) ){ 144 + i64 iOff = 0; 145 + int nByte = 0; 146 + int rc = 0; 147 + int nPg; 148 + int iPg; 149 + 150 + nByte = (int)getNextSize(z, &z, &rc); 151 + if( rc || *z!='@' ) goto bad_command; 152 + z++; 153 + iOff = getNextSize(z, &z, &rc); 154 + if( rc || (safe_isspace(*z)==0 && *z!='\0') ) goto bad_command; 155 + if( pzOut ) *pzOut = z; 156 + 157 + nPg = (nByte+pgsz-1) / pgsz; 158 + lseek(pCtx->fd, (off_t)iOff, SEEK_SET); 159 + for(iPg=0; iPg<nPg; iPg++){ 160 + write(pCtx->fd, aData, pgsz); 161 + } 162 + pCtx->nWrite += nByte/1024; 163 + 164 + return 0; 165 + } 166 + 167 + bad_command: 168 + testPrintError("unrecognized command: %s", zCmd); 169 + return 1; 170 +} 171 + 172 +static int readStdin(char **pzOut){ 173 + int nAlloc = 128; 174 + char *zOut = 0; 175 + int nOut = 0; 176 + 177 + while( !feof(stdin) ){ 178 + int nRead; 179 + 180 + nAlloc = nAlloc*2; 181 + zOut = realloc(zOut, nAlloc); 182 + nRead = fread(&zOut[nOut], 1, nAlloc-nOut-1, stdin); 183 + 184 + if( nRead==0 ) break; 185 + nOut += nRead; 186 + zOut[nOut] = '\0'; 187 + } 188 + 189 + *pzOut = zOut; 190 + return 0; 191 +} 192 + 193 +int do_io(int nArg, char **azArg){ 194 + IoContext ctx; 195 + int pgsz; 196 + char *zFile; 197 + char *zPgsz; 198 + int i; 199 + int rc = 0; 200 + 201 + char *zStdin = 0; 202 + char *z; 203 + 204 + u8 *aData; 205 + 206 + memset(&ctx, 0, sizeof(IoContext)); 207 + if( nArg<2 ){ 208 + testPrintUsage("FILE PGSZ ?CMD-1 ...?"); 209 + return -1; 210 + } 211 + zFile = azArg[0]; 212 + zPgsz = azArg[1]; 213 + 214 + pgsz = (int)getNextSize(zPgsz, 0, &rc); 215 + if( pgsz<=0 ){ 216 + testPrintError("Ridiculous page size: %d", pgsz); 217 + return -1; 218 + } 219 + aData = malloc(pgsz); 220 + memset(aData, 0x77, pgsz); 221 + 222 + ctx.fd = open(zFile, O_RDWR|O_CREAT|_O_BINARY, 0644); 223 + if( ctx.fd<0 ){ 224 + perror("open: "); 225 + return -1; 226 + } 227 + 228 + if( nArg==2 ){ 229 + readStdin(&zStdin); 230 + testTimeInit(); 231 + z = zStdin; 232 + while( *z && rc==0 ){ 233 + rc = doOneCmd(&ctx, aData, pgsz, z, &z); 234 + } 235 + }else{ 236 + testTimeInit(); 237 + for(i=2; i<nArg; i++){ 238 + rc = doOneCmd(&ctx, aData, pgsz, azArg[i], 0); 239 + } 240 + } 241 + 242 + printf("%dK written in %d ms\n", ctx.nWrite, testTimeGet()); 243 + 244 + free(zStdin); 245 + close(ctx.fd); 246 + 247 + return 0; 248 +}
Added ext/lsm1/lsm-test/lsmtest_main.c.
1 + 2 +#include "lsmtest.h" 3 +#include <sqlite3.h> 4 + 5 +void test_failed(){ 6 + assert( 0 ); 7 + return; 8 +} 9 + 10 +#define testSetError(rc) testSetErrorFunc(rc, pRc, __FILE__, __LINE__) 11 +static void testSetErrorFunc(int rc, int *pRc, const char *zFile, int iLine){ 12 + if( rc ){ 13 + *pRc = rc; 14 + fprintf(stderr, "FAILED (%s:%d) rc=%d ", zFile, iLine, rc); 15 + test_failed(); 16 + } 17 +} 18 + 19 +static int lsm_memcmp(u8 *a, u8 *b, int c){ 20 + int i; 21 + for(i=0; i<c; i++){ 22 + if( a[i]!=b[i] ) return a[i] - b[i]; 23 + } 24 + return 0; 25 +} 26 + 27 +/* 28 +** A test utility function. 29 +*/ 30 +void testFetch( 31 + TestDb *pDb, /* Database handle */ 32 + void *pKey, int nKey, /* Key to query database for */ 33 + void *pVal, int nVal, /* Expected value */ 34 + int *pRc /* IN/OUT: Error code */ 35 +){ 36 + if( *pRc==0 ){ 37 + void *pDbVal; 38 + int nDbVal; 39 + int rc; 40 + 41 + static int nCall = 0; nCall++; 42 + 43 + rc = tdb_fetch(pDb, pKey, nKey, &pDbVal, &nDbVal); 44 + testSetError(rc); 45 + if( rc==0 && (nVal!=nDbVal || (nVal>0 && lsm_memcmp(pVal, pDbVal, nVal))) ){ 46 + testSetError(1); 47 + } 48 + } 49 +} 50 + 51 +void testWrite( 52 + TestDb *pDb, /* Database handle */ 53 + void *pKey, int nKey, /* Key to query database for */ 54 + void *pVal, int nVal, /* Value to write */ 55 + int *pRc /* IN/OUT: Error code */ 56 +){ 57 + if( *pRc==0 ){ 58 + int rc; 59 +static int nCall = 0; 60 +nCall++; 61 + rc = tdb_write(pDb, pKey, nKey, pVal, nVal); 62 + testSetError(rc); 63 + } 64 +} 65 +void testDelete( 66 + TestDb *pDb, /* Database handle */ 67 + void *pKey, int nKey, /* Key to query database for */ 68 + int *pRc /* IN/OUT: Error code */ 69 +){ 70 + if( *pRc==0 ){ 71 + int rc; 72 + *pRc = rc = tdb_delete(pDb, pKey, nKey); 73 + testSetError(rc); 74 + } 75 +} 76 +void testDeleteRange( 77 + TestDb *pDb, /* Database handle */ 78 + void *pKey1, int nKey1, 79 + void *pKey2, int nKey2, 80 + int *pRc /* IN/OUT: Error code */ 81 +){ 82 + if( *pRc==0 ){ 83 + int rc; 84 + *pRc = rc = tdb_delete_range(pDb, pKey1, nKey1, pKey2, nKey2); 85 + testSetError(rc); 86 + } 87 +} 88 + 89 +void testBegin(TestDb *pDb, int iTrans, int *pRc){ 90 + if( *pRc==0 ){ 91 + int rc; 92 + rc = tdb_begin(pDb, iTrans); 93 + testSetError(rc); 94 + } 95 +} 96 +void testCommit(TestDb *pDb, int iTrans, int *pRc){ 97 + if( *pRc==0 ){ 98 + int rc; 99 + rc = tdb_commit(pDb, iTrans); 100 + testSetError(rc); 101 + } 102 +} 103 +#if 0 /* unused */ 104 +static void testRollback(TestDb *pDb, int iTrans, int *pRc){ 105 + if( *pRc==0 ){ 106 + int rc; 107 + rc = tdb_rollback(pDb, iTrans); 108 + testSetError(rc); 109 + } 110 +} 111 +#endif 112 + 113 +void testWriteStr( 114 + TestDb *pDb, /* Database handle */ 115 + const char *zKey, /* Key to query database for */ 116 + const char *zVal, /* Value to write */ 117 + int *pRc /* IN/OUT: Error code */ 118 +){ 119 + int nVal = (zVal ? strlen(zVal) : 0); 120 + testWrite(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc); 121 +} 122 + 123 +#if 0 /* unused */ 124 +static void testDeleteStr(TestDb *pDb, const char *zKey, int *pRc){ 125 + testDelete(pDb, (void *)zKey, strlen(zKey), pRc); 126 +} 127 +#endif 128 +void testFetchStr( 129 + TestDb *pDb, /* Database handle */ 130 + const char *zKey, /* Key to query database for */ 131 + const char *zVal, /* Value to write */ 132 + int *pRc /* IN/OUT: Error code */ 133 +){ 134 + int nVal = (zVal ? strlen(zVal) : 0); 135 + testFetch(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc); 136 +} 137 + 138 +void testFetchCompare( 139 + TestDb *pControl, 140 + TestDb *pDb, 141 + void *pKey, int nKey, 142 + int *pRc 143 +){ 144 + int rc; 145 + void *pDbVal1; 146 + void *pDbVal2; 147 + int nDbVal1; 148 + int nDbVal2; 149 + 150 + static int nCall = 0; 151 + nCall++; 152 + 153 + rc = tdb_fetch(pControl, pKey, nKey, &pDbVal1, &nDbVal1); 154 + testSetError(rc); 155 + 156 + rc = tdb_fetch(pDb, pKey, nKey, &pDbVal2, &nDbVal2); 157 + testSetError(rc); 158 + 159 + if( *pRc==0 160 + && (nDbVal1!=nDbVal2 || (nDbVal1>0 && memcmp(pDbVal1, pDbVal2, nDbVal1))) 161 + ){ 162 + testSetError(1); 163 + } 164 +} 165 + 166 +typedef struct ScanResult ScanResult; 167 +struct ScanResult { 168 + TestDb *pDb; 169 + 170 + int nRow; 171 + u32 cksum1; 172 + u32 cksum2; 173 + void *pKey1; int nKey1; 174 + void *pKey2; int nKey2; 175 + 176 + int bReverse; 177 + int nPrevKey; 178 + u8 aPrevKey[256]; 179 +}; 180 + 181 +static int keyCompare(void *pKey1, int nKey1, void *pKey2, int nKey2){ 182 + int res; 183 + res = memcmp(pKey1, pKey2, MIN(nKey1, nKey2)); 184 + if( res==0 ){ 185 + res = nKey1 - nKey2; 186 + } 187 + return res; 188 +} 189 + 190 +int test_scan_debug = 0; 191 + 192 +static void scanCompareCb( 193 + void *pCtx, 194 + void *pKey, int nKey, 195 + void *pVal, int nVal 196 +){ 197 + ScanResult *p = (ScanResult *)pCtx; 198 + u8 *aKey = (u8 *)pKey; 199 + u8 *aVal = (u8 *)pVal; 200 + int i; 201 + 202 + if( test_scan_debug ){ 203 + printf("%d: %.*s\n", p->nRow, nKey, (char *)pKey); 204 + fflush(stdout); 205 + } 206 +#if 0 207 + if( test_scan_debug ) printf("%.20s\n", (char *)pVal); 208 +#endif 209 + 210 +#if 0 211 + /* Check tdb_fetch() matches */ 212 + int rc = 0; 213 + testFetch(p->pDb, pKey, nKey, pVal, nVal, &rc); 214 + assert( rc==0 ); 215 +#endif 216 + 217 + /* Update the checksum data */ 218 + p->nRow++; 219 + for(i=0; i<nKey; i++){ 220 + p->cksum1 += ((int)aKey[i] << (i&0x0F)); 221 + p->cksum2 += p->cksum1; 222 + } 223 + for(i=0; i<nVal; i++){ 224 + p->cksum1 += ((int)aVal[i] << (i&0x0F)); 225 + p->cksum2 += p->cksum1; 226 + } 227 + 228 + /* Check that the delivered row is not out of order. */ 229 + if( nKey<(int)sizeof(p->aPrevKey) ){ 230 + if( p->nPrevKey ){ 231 + int res = keyCompare(p->aPrevKey, p->nPrevKey, pKey, nKey); 232 + if( (res<0 && p->bReverse) || (res>0 && p->bReverse==0) ){ 233 + testPrintError("Returned key out of order at %s:%d\n", 234 + __FILE__, __LINE__ 235 + ); 236 + } 237 + } 238 + 239 + p->nPrevKey = nKey; 240 + memcpy(p->aPrevKey, pKey, MIN(p->nPrevKey, nKey)); 241 + } 242 + 243 + /* Check that the delivered row is within range. */ 244 + if( p->pKey1 && ( 245 + (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))>0) 246 + || (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))==0 && p->nKey1>nKey) 247 + )){ 248 + testPrintError("Returned key too small at %s:%d\n", __FILE__, __LINE__); 249 + } 250 + if( p->pKey2 && ( 251 + (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))<0) 252 + || (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))==0 && p->nKey2<nKey) 253 + )){ 254 + testPrintError("Returned key too large at %s:%d\n", __FILE__, __LINE__); 255 + } 256 + 257 +} 258 + 259 +/* 260 +** Scan the contents of the two databases. Check that they match. 261 +*/ 262 +void testScanCompare( 263 + TestDb *pDb1, /* Control (trusted) database */ 264 + TestDb *pDb2, /* Database being tested */ 265 + int bReverse, 266 + void *pKey1, int nKey1, 267 + void *pKey2, int nKey2, 268 + int *pRc 269 +){ 270 + static int nCall = 0; nCall++; 271 + if( *pRc==0 ){ 272 + ScanResult res1; 273 + ScanResult res2; 274 + void *pRes1 = (void *)&res1; 275 + void *pRes2 = (void *)&res2; 276 + 277 + memset(&res1, 0, sizeof(ScanResult)); 278 + memset(&res2, 0, sizeof(ScanResult)); 279 + 280 + res1.pDb = pDb1; 281 + res1.nKey1 = nKey1; res1.pKey1 = pKey1; 282 + res1.nKey2 = nKey2; res1.pKey2 = pKey2; 283 + res1.bReverse = bReverse; 284 + res2.pDb = pDb2; 285 + res2.nKey1 = nKey1; res2.pKey1 = pKey1; 286 + res2.nKey2 = nKey2; res2.pKey2 = pKey2; 287 + res2.bReverse = bReverse; 288 + 289 + tdb_scan(pDb1, pRes1, bReverse, pKey1, nKey1, pKey2, nKey2, scanCompareCb); 290 +if( test_scan_debug ) printf("\n\n\n"); 291 + tdb_scan(pDb2, pRes2, bReverse, pKey1, nKey1, pKey2, nKey2, scanCompareCb); 292 +if( test_scan_debug ) printf("\n\n\n"); 293 + 294 + if( res1.nRow!=res2.nRow 295 + || res1.cksum1!=res2.cksum1 296 + || res1.cksum2!=res2.cksum2 297 + ){ 298 + printf("expected: %d %X %X\n", res1.nRow, res1.cksum1, res1.cksum2); 299 + printf("got: %d %X %X\n", res2.nRow, res2.cksum1, res2.cksum2); 300 + testSetError(1); 301 + *pRc = 1; 302 + } 303 + } 304 +} 305 + 306 +void testClose(TestDb **ppDb){ 307 + tdb_close(*ppDb); 308 + *ppDb = 0; 309 +} 310 + 311 +TestDb *testOpen(const char *zSystem, int bClear, int *pRc){ 312 + TestDb *pDb = 0; 313 + if( *pRc==0 ){ 314 + int rc; 315 + rc = tdb_open(zSystem, 0, bClear, &pDb); 316 + if( rc!=0 ){ 317 + testSetError(rc); 318 + *pRc = rc; 319 + } 320 + } 321 + return pDb; 322 +} 323 + 324 +void testReopen(TestDb **ppDb, int *pRc){ 325 + if( *pRc==0 ){ 326 + const char *zLib; 327 + zLib = tdb_library_name(*ppDb); 328 + testClose(ppDb); 329 + *pRc = tdb_open(zLib, 0, 0, ppDb); 330 + } 331 +} 332 + 333 + 334 +#if 0 /* unused */ 335 +static void testSystemSelect(const char *zSys, int *piSel, int *pRc){ 336 + if( *pRc==0 ){ 337 + struct SysName { const char *zName; } *aName; 338 + int nSys; 339 + int i; 340 + 341 + for(nSys=0; tdb_system_name(nSys); nSys++); 342 + aName = malloc(sizeof(struct SysName) * (nSys+1)); 343 + for(i=0; i<=nSys; i++){ 344 + aName[i].zName = tdb_system_name(i); 345 + } 346 + 347 + *pRc = testArgSelect(aName, "db", zSys, piSel); 348 + free(aName); 349 + } 350 +} 351 +#endif 352 + 353 +char *testMallocVPrintf(const char *zFormat, va_list ap){ 354 + int nByte; 355 + va_list copy; 356 + char *zRet; 357 + 358 + __va_copy(copy, ap); 359 + nByte = vsnprintf(0, 0, zFormat, copy); 360 + va_end(copy); 361 + 362 + assert( nByte>=0 ); 363 + zRet = (char *)testMalloc(nByte+1); 364 + vsnprintf(zRet, nByte+1, zFormat, ap); 365 + return zRet; 366 +} 367 + 368 +char *testMallocPrintf(const char *zFormat, ...){ 369 + va_list ap; 370 + char *zRet; 371 + 372 + va_start(ap, zFormat); 373 + zRet = testMallocVPrintf(zFormat, ap); 374 + va_end(ap); 375 + 376 + return zRet; 377 +} 378 + 379 + 380 +/* 381 +** A wrapper around malloc(3). 382 +** 383 +** This function should be used for all allocations made by test procedures. 384 +** It has the following properties: 385 +** 386 +** * Test code may assume that allocations may not fail. 387 +** * Returned memory is always zeroed. 388 +** 389 +** Allocations made using testMalloc() should be freed using testFree(). 390 +*/ 391 +void *testMalloc(int n){ 392 + u8 *p = (u8*)malloc(n + 8); 393 + memset(p, 0, n+8); 394 + *(int*)p = n; 395 + return (void*)&p[8]; 396 +} 397 + 398 +void *testMallocCopy(void *pCopy, int nByte){ 399 + void *pRet = testMalloc(nByte); 400 + memcpy(pRet, pCopy, nByte); 401 + return pRet; 402 +} 403 + 404 +void *testRealloc(void *ptr, int n){ 405 + if( ptr ){ 406 + u8 *p = (u8*)ptr - 8; 407 + int nOrig = *(int*)p; 408 + p = (u8*)realloc(p, n+8); 409 + if( nOrig<n ){ 410 + memset(&p[8+nOrig], 0, n-nOrig); 411 + } 412 + *(int*)p = n; 413 + return (void*)&p[8]; 414 + } 415 + return testMalloc(n); 416 +} 417 + 418 +/* 419 +** Free an allocation made by an earlier call to testMalloc(). 420 +*/ 421 +void testFree(void *ptr){ 422 + if( ptr ){ 423 + u8 *p = (u8*)ptr - 8; 424 + memset(p, 0x55, *(int*)p + 8); 425 + free(p); 426 + } 427 +} 428 + 429 +/* 430 +** String zPattern contains a glob pattern. Return true if zStr matches 431 +** the pattern, or false if it does not. 432 +*/ 433 +int testGlobMatch(const char *zPattern, const char *zStr){ 434 + int i = 0; 435 + int j = 0; 436 + 437 + while( zPattern[i] ){ 438 + char p = zPattern[i]; 439 + 440 + if( p=='*' || p=='%' ){ 441 + do { 442 + if( testGlobMatch(&zPattern[i+1], &zStr[j]) ) return 1; 443 + }while( zStr[j++] ); 444 + return 0; 445 + } 446 + 447 + if( zStr[j]==0 || (p!='?' && p!=zStr[j]) ){ 448 + /* Match failed. */ 449 + return 0; 450 + } 451 + 452 + j++; 453 + i++; 454 + } 455 + 456 + return (zPattern[i]==0 && zStr[j]==0); 457 +} 458 + 459 +/* 460 +** End of test utilities 461 +**************************************************************************/ 462 + 463 +int do_test(int nArg, char **azArg){ 464 + int j; 465 + int rc; 466 + int nFail = 0; 467 + const char *zPattern = 0; 468 + 469 + if( nArg>1 ){ 470 + testPrintError("Usage: test ?PATTERN?\n"); 471 + return 1; 472 + } 473 + if( nArg==1 ){ 474 + zPattern = azArg[0]; 475 + } 476 + 477 + for(j=0; tdb_system_name(j); j++){ 478 + rc = 0; 479 + 480 + test_data_1(tdb_system_name(j), zPattern, &rc); 481 + test_data_2(tdb_system_name(j), zPattern, &rc); 482 + test_data_3(tdb_system_name(j), zPattern, &rc); 483 + test_data_4(tdb_system_name(j), zPattern, &rc); 484 + test_rollback(tdb_system_name(j), zPattern, &rc); 485 + test_mc(tdb_system_name(j), zPattern, &rc); 486 + test_mt(tdb_system_name(j), zPattern, &rc); 487 + 488 + if( rc ) nFail++; 489 + } 490 + 491 + rc = 0; 492 + test_oom(zPattern, &rc); 493 + if( rc ) nFail++; 494 + 495 + rc = 0; 496 + test_api(zPattern, &rc); 497 + if( rc ) nFail++; 498 + 499 + rc = 0; 500 + do_crash_test(zPattern, &rc); 501 + if( rc ) nFail++; 502 + 503 + rc = 0; 504 + do_writer_crash_test(zPattern, &rc); 505 + if( rc ) nFail++; 506 + 507 + return (nFail!=0); 508 +} 509 + 510 +static lsm_db *configure_lsm_db(TestDb *pDb){ 511 + lsm_db *pLsm; 512 + pLsm = tdb_lsm(pDb); 513 + if( pLsm ){ 514 + tdb_lsm_config_str(pDb, "mmap=1 autowork=1 automerge=4 worker_automerge=4"); 515 + } 516 + return pLsm; 517 +} 518 + 519 +typedef struct WriteHookEvent WriteHookEvent; 520 +struct WriteHookEvent { 521 + i64 iOff; 522 + int nData; 523 + int nUs; 524 +}; 525 +WriteHookEvent prev = {0, 0, 0}; 526 + 527 +static void flushPrev(FILE *pOut){ 528 + if( prev.nData ){ 529 + fprintf(pOut, "w %s %lld %d %d\n", "d", prev.iOff, prev.nData, prev.nUs); 530 + prev.nData = 0; 531 + } 532 +} 533 + 534 +#if 0 /* unused */ 535 +static void do_speed_write_hook2( 536 + void *pCtx, 537 + int bLog, 538 + i64 iOff, 539 + int nData, 540 + int nUs 541 +){ 542 + FILE *pOut = (FILE *)pCtx; 543 + if( bLog ) return; 544 + 545 + if( prev.nData && nData && iOff==prev.iOff+prev.nData ){ 546 + prev.nData += nData; 547 + prev.nUs += nUs; 548 + }else{ 549 + flushPrev(pOut); 550 + if( nData==0 ){ 551 + fprintf(pOut, "s %s 0 0 %d\n", (bLog ? "l" : "d"), nUs); 552 + }else{ 553 + prev.iOff = iOff; 554 + prev.nData = nData; 555 + prev.nUs = nUs; 556 + } 557 + } 558 +} 559 +#endif 560 + 561 +#define ST_REPEAT 0 562 +#define ST_WRITE 1 563 +#define ST_PAUSE 2 564 +#define ST_FETCH 3 565 +#define ST_SCAN 4 566 +#define ST_NSCAN 5 567 +#define ST_KEYSIZE 6 568 +#define ST_VALSIZE 7 569 +#define ST_TRANS 8 570 + 571 + 572 +static void print_speed_test_help(){ 573 + printf( 574 +"\n" 575 +"Repeat the following $repeat times:\n" 576 +" 1. Insert $write key-value pairs. One transaction for each write op.\n" 577 +" 2. Pause for $pause ms.\n" 578 +" 3. Perform $fetch queries on the database.\n" 579 +"\n" 580 +" Keys are $keysize bytes in size. Values are $valsize bytes in size\n" 581 +" Both keys and values are pseudo-randomly generated\n" 582 +"\n" 583 +"Options are:\n" 584 +" -repeat $repeat (default value 10)\n" 585 +" -write $write (default value 10000)\n" 586 +" -pause $pause (default value 0)\n" 587 +" -fetch $fetch (default value 0)\n" 588 +" -keysize $keysize (default value 12)\n" 589 +" -valsize $valsize (default value 100)\n" 590 +" -system $system (default value \"lsm\")\n" 591 +" -trans $trans (default value 0)\n" 592 +"\n" 593 +); 594 +} 595 + 596 +int do_speed_test2(int nArg, char **azArg){ 597 + struct Option { 598 + const char *zOpt; 599 + int eVal; 600 + int iDefault; 601 + } aOpt[] = { 602 + { "-repeat", ST_REPEAT, 10}, 603 + { "-write", ST_WRITE, 10000}, 604 + { "-pause", ST_PAUSE, 0}, 605 + { "-fetch", ST_FETCH, 0}, 606 + { "-scan", ST_SCAN, 0}, 607 + { "-nscan", ST_NSCAN, 0}, 608 + { "-keysize", ST_KEYSIZE, 12}, 609 + { "-valsize", ST_VALSIZE, 100}, 610 + { "-trans", ST_TRANS, 0}, 611 + { "-system", -1, 0}, 612 + { "help", -2, 0}, 613 + {0, 0, 0} 614 + }; 615 + int i; 616 + int aParam[9]; 617 + int rc = 0; 618 + int bReadonly = 0; 619 + int nContent = 0; 620 + 621 + TestDb *pDb; 622 + Datasource *pData; 623 + DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 0, 0, 0, 0 }; 624 + char *zSystem = ""; 625 + int bLsm = 1; 626 + FILE *pLog = 0; 627 + 628 +#ifdef NDEBUG 629 + /* If NDEBUG is defined, disable the dynamic memory related checks in 630 + ** lsmtest_mem.c. They slow things down. */ 631 + testMallocUninstall(tdb_lsm_env()); 632 +#endif 633 + 634 + /* Initialize aParam[] with default values. */ 635 + for(i=0; i<ArraySize(aOpt); i++){ 636 + if( aOpt[i].zOpt ) aParam[aOpt[i].eVal] = aOpt[i].iDefault; 637 + } 638 + 639 + /* Process the command line switches. */ 640 + for(i=0; i<nArg; i+=2){ 641 + int iSel; 642 + rc = testArgSelect(aOpt, "switch", azArg[i], &iSel); 643 + if( rc ){ 644 + return rc; 645 + } 646 + if( aOpt[iSel].eVal==-2 ){ 647 + print_speed_test_help(); 648 + return 0; 649 + } 650 + if( i+1==nArg ){ 651 + testPrintError("option %s requires an argument\n", aOpt[iSel].zOpt); 652 + return 1; 653 + } 654 + if( aOpt[iSel].eVal>=0 ){ 655 + aParam[aOpt[iSel].eVal] = atoi(azArg[i+1]); 656 + }else{ 657 + zSystem = azArg[i+1]; 658 + bLsm = 0; 659 +#if 0 660 + for(j=0; zSystem[j]; j++){ 661 + if( zSystem[j]=='=' ) bLsm = 1; 662 + } 663 +#endif 664 + } 665 + } 666 + 667 + printf("#"); 668 + for(i=0; i<ArraySize(aOpt); i++){ 669 + if( aOpt[i].zOpt ){ 670 + if( aOpt[i].eVal>=0 ){ 671 + printf(" %s=%d", &aOpt[i].zOpt[1], aParam[aOpt[i].eVal]); 672 + }else if( aOpt[i].eVal==-1 ){ 673 + printf(" %s=\"%s\"", &aOpt[i].zOpt[1], zSystem); 674 + } 675 + } 676 + } 677 + printf("\n"); 678 + 679 + defn.nMinKey = defn.nMaxKey = aParam[ST_KEYSIZE]; 680 + defn.nMinVal = defn.nMaxVal = aParam[ST_VALSIZE]; 681 + pData = testDatasourceNew(&defn); 682 + 683 + if( aParam[ST_WRITE]==0 ){ 684 + bReadonly = 1; 685 + } 686 + 687 + if( bLsm ){ 688 + rc = tdb_lsm_open(zSystem, "testdb.lsm", !bReadonly, &pDb); 689 + }else{ 690 + pDb = testOpen(zSystem, !bReadonly, &rc); 691 + } 692 + if( rc!=0 ) return rc; 693 + if( bReadonly ){ 694 + nContent = testCountDatabase(pDb); 695 + } 696 + 697 +#if 0 698 + pLog = fopen("/tmp/speed.log", "w"); 699 + tdb_lsm_write_hook(pDb, do_speed_write_hook2, (void *)pLog); 700 +#endif 701 + 702 + for(i=0; i<aParam[ST_REPEAT] && rc==0; i++){ 703 + int msWrite, msFetch; 704 + int iFetch; 705 + int nWrite = aParam[ST_WRITE]; 706 + 707 + if( bReadonly ){ 708 + msWrite = 0; 709 + }else{ 710 + testTimeInit(); 711 + 712 + if( aParam[ST_TRANS] ) testBegin(pDb, 2, &rc); 713 + testWriteDatasourceRange(pDb, pData, i*nWrite, nWrite, &rc); 714 + if( aParam[ST_TRANS] ) testCommit(pDb, 0, &rc); 715 + 716 + msWrite = testTimeGet(); 717 + nContent += nWrite; 718 + } 719 + 720 + if( aParam[ST_PAUSE] ){ 721 + if( aParam[ST_PAUSE]/1000 ) sleep(aParam[ST_PAUSE]/1000); 722 + if( aParam[ST_PAUSE]%1000 ) usleep(1000 * (aParam[ST_PAUSE]%1000)); 723 + } 724 + 725 + if( aParam[ST_FETCH] ){ 726 + testTimeInit(); 727 + if( aParam[ST_TRANS] ) testBegin(pDb, 1, &rc); 728 + for(iFetch=0; iFetch<aParam[ST_FETCH]; iFetch++){ 729 + int iKey = testPrngValue(i*nWrite+iFetch) % nContent; 730 +#ifndef NDEBUG 731 + testDatasourceFetch(pDb, pData, iKey, &rc); 732 +#else 733 + void *pKey; int nKey; /* Database key to query for */ 734 + void *pVal; int nVal; /* Result of query */ 735 + 736 + testDatasourceEntry(pData, iKey, &pKey, &nKey, 0, 0); 737 + rc = tdb_fetch(pDb, pKey, nKey, &pVal, &nVal); 738 + if( rc==0 && nVal<0 ) rc = 1; 739 + if( rc ) break; 740 +#endif 741 + } 742 + if( aParam[ST_TRANS] ) testCommit(pDb, 0, &rc); 743 + msFetch = testTimeGet(); 744 + }else{ 745 + msFetch = 0; 746 + } 747 + 748 + if( i==(aParam[ST_REPEAT]-1) ){ 749 + testTimeInit(); 750 + testClose(&pDb); 751 + msWrite += testTimeGet(); 752 + } 753 + 754 + printf("%d %d %d\n", i, msWrite, msFetch); 755 + fflush(stdout); 756 + } 757 + 758 + testClose(&pDb); 759 + testDatasourceFree(pData); 760 + 761 + if( pLog ){ 762 + flushPrev(pLog); 763 + fclose(pLog); 764 + } 765 + return rc; 766 +} 767 + 768 +int do_speed_tests(int nArg, char **azArg){ 769 + 770 + struct DbSystem { 771 + const char *zLibrary; 772 + const char *zColor; 773 + } aSys[] = { 774 + { "sqlite3", "black" }, 775 + { "leveldb", "blue" }, 776 + { "lsm", "red" }, 777 + { "lsm_mt2", "orange" }, 778 + { "lsm_mt3", "purple" }, 779 + { "kyotocabinet", "green" }, 780 + {0, 0} 781 + }; 782 + 783 + int i; 784 + int j; 785 + int rc; 786 + int nSleep = 0; /* ms of rest allowed between INSERT tests */ 787 + int nRow = 0; /* Number of rows to insert into database */ 788 + int nStep; /* Measure INSERT time after this many rows */ 789 + int nSelStep; /* Measure SELECT time after this many rows */ 790 + int nSelTest; /* Number of SELECTs to run for timing */ 791 + int doReadTest = 1; 792 + int doWriteTest = 1; 793 + 794 + int *aTime; /* INSERT timing data */ 795 + int *aWrite; /* Writes per nStep inserts */ 796 + int *aSelTime; /* SELECT timing data */ 797 + int isFirst = 1; 798 + int bSleep = 0; 799 + 800 + /* File to write gnuplot script to. */ 801 + const char *zOut = "lsmtest_speed.gnuplot"; 802 + 803 + u32 sys_mask = 0; 804 + 805 + testMallocUninstall(tdb_lsm_env()); 806 + 807 + for(i=0; i<nArg; i++){ 808 + struct Opt { 809 + const char *zOpt; 810 + int isSwitch; 811 + } aOpt[] = { 812 + { "sqlite3" , 0}, 813 + { "leveldb" , 0}, 814 + { "lsm" , 0}, 815 + { "lsm_mt2" , 0}, 816 + { "lsm_mt3" , 0}, 817 + { "kyotocabinet" , 0}, 818 + { "-rows" , 1}, 819 + { "-sleep" , 2}, 820 + { "-testmode" , 3}, 821 + { "-out" , 4}, 822 + { 0, 0} 823 + }; 824 + int iSel; 825 + 826 + rc = testArgSelect(aOpt, "argument", azArg[i], &iSel); 827 + if( rc ) return rc; 828 + 829 + if( aOpt[iSel].isSwitch ){ 830 + i++; 831 + 832 + if( i>=nArg ){ 833 + testPrintError("option %s requires an argument\n", aOpt[iSel].zOpt); 834 + return 1; 835 + } 836 + if( aOpt[iSel].isSwitch==1 ){ 837 + nRow = atoi(azArg[i]); 838 + } 839 + if( aOpt[iSel].isSwitch==2 ){ 840 + nSleep = atoi(azArg[i]); 841 + } 842 + if( aOpt[iSel].isSwitch==3 ){ 843 + struct Mode { 844 + const char *zMode; 845 + int doReadTest; 846 + int doWriteTest; 847 + } aMode[] = {{"ro", 1, 0} , {"rw", 1, 1}, {"wo", 0, 1}, {0, 0, 0}}; 848 + int iMode; 849 + rc = testArgSelect(aMode, "option", azArg[i], &iMode); 850 + if( rc ) return rc; 851 + doReadTest = aMode[iMode].doReadTest; 852 + doWriteTest = aMode[iMode].doWriteTest; 853 + } 854 + if( aOpt[iSel].isSwitch==4 ){ 855 + /* The "-out FILE" switch. This option is used to specify a file to 856 + ** write the gnuplot script to. */ 857 + zOut = azArg[i]; 858 + } 859 + }else{ 860 + /* A db name */ 861 + rc = testArgSelect(aOpt, "system", azArg[i], &iSel); 862 + if( rc ) return rc; 863 + sys_mask |= (1<<iSel); 864 + } 865 + } 866 + 867 + if( sys_mask==0 ) sys_mask = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3); 868 + nRow = MAX(nRow, 100000); 869 + nStep = nRow/100; 870 + nSelStep = nRow/10; 871 + nSelTest = (nSelStep > 100000) ? 100000 : nSelStep; 872 + 873 + aTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nStep); 874 + aWrite = malloc(sizeof(int) * nRow/nStep); 875 + aSelTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nSelStep); 876 + 877 + /* This loop collects the INSERT speed data. */ 878 + if( doWriteTest ){ 879 + printf("Writing output to file \"%s\".\n", zOut); 880 + 881 + for(j=0; aSys[j].zLibrary; j++){ 882 + FILE *pLog = 0; 883 + TestDb *pDb; /* Database being tested */ 884 + lsm_db *pLsm; 885 + int iDot = 0; 886 + 887 + if( ((1<<j)&sys_mask)==0 ) continue; 888 + if( bSleep && nSleep ) sqlite3_sleep(nSleep); 889 + bSleep = 1; 890 + 891 + testCaseBegin(&rc, 0, "speed.insert.%s", aSys[j].zLibrary); 892 + 893 + rc = tdb_open(aSys[j].zLibrary, 0, 1, &pDb); 894 + if( rc ) return rc; 895 + 896 + pLsm = configure_lsm_db(pDb); 897 +#if 0 898 + pLog = fopen("/tmp/speed.log", "w"); 899 + tdb_lsm_write_hook(pDb, do_speed_write_hook2, (void *)pLog); 900 +#endif 901 + 902 + testTimeInit(); 903 + for(i=0; i<nRow; i+=nStep){ 904 + int iStep; 905 + int nWrite1 = 0, nWrite2 = 0; 906 + testCaseProgress(i, nRow, testCaseNDot(), &iDot); 907 + if( pLsm ) lsm_info(pLsm, LSM_INFO_NWRITE, &nWrite1); 908 + for(iStep=0; iStep<nStep; iStep++){ 909 + u32 aKey[4]; /* 16-byte key */ 910 + u32 aVal[25]; /* 100 byte value */ 911 + testPrngArray(i+iStep, aKey, ArraySize(aKey)); 912 + testPrngArray(i+iStep, aVal, ArraySize(aVal)); 913 + rc = tdb_write(pDb, aKey, sizeof(aKey), aVal, sizeof(aVal)); 914 + } 915 + aTime[(j*nRow+i)/nStep] = testTimeGet(); 916 + if( pLsm ) lsm_info(pLsm, LSM_INFO_NWRITE, &nWrite2); 917 + aWrite[i/nStep] = nWrite2 - nWrite1; 918 + } 919 + 920 + tdb_close(pDb); 921 + if( pLog ) fclose(pLog); 922 + testCaseFinish(rc); 923 + } 924 + } 925 + 926 + /* This loop collects the SELECT speed data. */ 927 + if( doReadTest ){ 928 + for(j=0; aSys[j].zLibrary; j++){ 929 + int iDot = 0; 930 + TestDb *pDb; /* Database being tested */ 931 + 932 + if( ((1<<j)&sys_mask)==0 ) continue; 933 + if( bSleep && nSleep ) sqlite3_sleep(nSleep); 934 + bSleep = 1; 935 + 936 + testCaseBegin(&rc, 0, "speed.select.%s", aSys[j].zLibrary); 937 + 938 + if( doWriteTest ){ 939 + rc = tdb_open(aSys[j].zLibrary, 0, 1, &pDb); 940 + if( rc ) return rc; 941 + configure_lsm_db(pDb); 942 + 943 + for(i=0; i<nRow; i+=nSelStep){ 944 + int iStep; 945 + int iSel; 946 + testCaseProgress(i, nRow, testCaseNDot(), &iDot); 947 + for(iStep=0; iStep<nSelStep; iStep++){ 948 + u32 aKey[4]; /* 16-byte key */ 949 + u32 aVal[25]; /* 100 byte value */ 950 + testPrngArray(i+iStep, aKey, ArraySize(aKey)); 951 + testPrngArray(i+iStep, aVal, ArraySize(aVal)); 952 + rc = tdb_write(pDb, aKey, sizeof(aKey), aVal, sizeof(aVal)); 953 + } 954 + 955 + testTimeInit(); 956 + for(iSel=0; iSel<nSelTest; iSel++){ 957 + void *pDummy; 958 + int nDummy; 959 + u32 iKey; 960 + u32 aKey[4]; /* 16-byte key */ 961 + 962 + iKey = testPrngValue(iSel) % (i+nSelStep); 963 + testPrngArray(iKey, aKey, ArraySize(aKey)); 964 + rc = tdb_fetch(pDb, aKey, sizeof(aKey), &pDummy, &nDummy); 965 + } 966 + aSelTime[(j*nRow+i)/nSelStep] = testTimeGet(); 967 + tdb_fetch(pDb, 0, 0, 0, 0); 968 + } 969 + }else{ 970 + int t; 971 + int iSel; 972 + 973 + rc = tdb_open(aSys[j].zLibrary, 0, 0, &pDb); 974 + configure_lsm_db(pDb); 975 + 976 + testTimeInit(); 977 + for(iSel=0; rc==LSM_OK && iSel<nSelTest; iSel++){ 978 + void *pDummy; 979 + int nDummy; 980 + u32 iKey; 981 + u32 aKey[4]; /* 16-byte key */ 982 +#ifndef NDEBUG 983 + u32 aVal[25]; /* 100 byte value */ 984 +#endif 985 + 986 + testCaseProgress(iSel, nSelTest, testCaseNDot(), &iDot); 987 + 988 + iKey = testPrngValue(iSel) % nRow; 989 + testPrngArray(iKey, aKey, ArraySize(aKey)); 990 + rc = tdb_fetch(pDb, aKey, sizeof(aKey), &pDummy, &nDummy); 991 + 992 +#ifndef NDEBUG 993 + testPrngArray(iKey, aVal, ArraySize(aVal)); 994 + assert( nDummy==100 && memcmp(aVal, pDummy, 100)==0 ); 995 +#endif 996 + } 997 + if( rc!=LSM_OK ) return rc; 998 + 999 + t = testTimeGet(); 1000 + tdb_fetch(pDb, 0, 0, 0, 0); 1001 + 1002 + printf("%s: %d selects/second\n", 1003 + aSys[j].zLibrary, (int)((double)nSelTest*1000.0/t) 1004 + ); 1005 + } 1006 + 1007 + tdb_close(pDb); 1008 + testCaseFinish(rc); 1009 + } 1010 + } 1011 + 1012 + 1013 + if( doWriteTest ){ 1014 + FILE *pOut = fopen(zOut, "w"); 1015 + if( !pOut ){ 1016 + printf("fopen(\"%s\", \"w\"): %s\n", zOut, strerror(errno)); 1017 + return 1; 1018 + } 1019 + 1020 + fprintf(pOut, "set xlabel \"Rows Inserted\"\n"); 1021 + fprintf(pOut, "set ylabel \"Inserts per second\"\n"); 1022 + if( doReadTest ){ 1023 + fprintf(pOut, "set y2label \"Selects per second\"\n"); 1024 + }else if( sys_mask==(1<<2) ){ 1025 + fprintf(pOut, "set y2label \"Page writes per insert\"\n"); 1026 + } 1027 + fprintf(pOut, "set yrange [0:*]\n"); 1028 + fprintf(pOut, "set y2range [0:*]\n"); 1029 + fprintf(pOut, "set xrange [%d:*]\n", MAX(nStep, nRow/20) ); 1030 + fprintf(pOut, "set ytics nomirror\n"); 1031 + fprintf(pOut, "set y2tics nomirror\n"); 1032 + fprintf(pOut, "set key box lw 0.01\n"); 1033 + fprintf(pOut, "plot "); 1034 + 1035 + for(j=0; aSys[j].zLibrary; j++){ 1036 + if( (1<<j)&sys_mask ){ 1037 + const char *zLib = aSys[j].zLibrary; 1038 + fprintf(pOut, "%s\"-\" ti \"%s INSERT\" with lines lc rgb \"%s\" ", 1039 + (isFirst?"":", "), zLib, aSys[j].zColor 1040 + ); 1041 + if( doReadTest ){ 1042 + fprintf(pOut, ", \"-\" ti \"%s SELECT\" " 1043 + "axis x1y2 with points lw 3 lc rgb \"%s\"" 1044 + , zLib, aSys[j].zColor 1045 + ); 1046 + } 1047 + isFirst = 0; 1048 + } 1049 + } 1050 + 1051 + assert( strcmp(aSys[2].zLibrary, "lsm")==0 ); 1052 + if( sys_mask==(1<<2) && !doReadTest ){ 1053 + fprintf(pOut, ", \"-\" ti \"lsm pages written\" " 1054 + "axis x1y2 with boxes lw 1 lc rgb \"grey\"" 1055 + ); 1056 + } 1057 + 1058 + fprintf(pOut, "\n"); 1059 + 1060 + for(j=0; aSys[j].zLibrary; j++){ 1061 + if( ((1<<j)&sys_mask)==0 ) continue; 1062 + fprintf(pOut, "# Rows Inserts per second\n"); 1063 + for(i=0; i<nRow; i+=nStep){ 1064 + int iTime = aTime[(j*nRow+i)/nStep]; 1065 + int ips = (int)((i+nStep)*1000.0 / (double)iTime); 1066 + fprintf(pOut, "%d %d\n", i+nStep, ips); 1067 + } 1068 + fprintf(pOut, "end\n"); 1069 + 1070 + if( doReadTest ){ 1071 + fprintf(pOut, "# Rows Selects per second\n"); 1072 + for(i=0; i<nRow; i+=nSelStep){ 1073 + int sps = (int)(nSelTest*1000.0/(double)aSelTime[(j*nRow+i)/nSelStep]); 1074 + fprintf(pOut, "%d %d\n", i+nSelStep, sps); 1075 + } 1076 + fprintf(pOut, "end\n"); 1077 + }else if( sys_mask==(1<<2) ){ 1078 + for(i=0; i<(nRow/nStep); i++){ 1079 + fprintf(pOut, "%d %f\n", i*nStep, (double)aWrite[i] / (double)nStep); 1080 + } 1081 + fprintf(pOut, "end\n"); 1082 + } 1083 + } 1084 + 1085 + fprintf(pOut, "pause -1\n"); 1086 + fclose(pOut); 1087 + } 1088 + 1089 + free(aTime); 1090 + free(aSelTime); 1091 + free(aWrite); 1092 + testMallocInstall(tdb_lsm_env()); 1093 + return 0; 1094 +} 1095 + 1096 +/* 1097 +** Usage: lsmtest random ?N? 1098 +** 1099 +** This command prints a sequence of zero or more numbers from the PRNG 1100 +** system to stdout. If the "N" argument is missing, values the first 10 1101 +** values (i=0, i=1, ... i=9) are printed. Otherwise, the first N. 1102 +** 1103 +** This was added to verify that the PRNG values do not change between 1104 +** runs of the lsmtest program. 1105 +*/ 1106 +int do_random_tests(int nArg, char **azArg){ 1107 + int i; 1108 + int nRand; 1109 + if( nArg==0 ){ 1110 + nRand = 10; 1111 + }else if( nArg==1 ){ 1112 + nRand = atoi(azArg[0]); 1113 + }else{ 1114 + testPrintError("Usage: random ?N?\n"); 1115 + return -1; 1116 + } 1117 + for(i=0; i<nRand; i++){ 1118 + printf("0x%x\n", testPrngValue(i)); 1119 + } 1120 + return 0; 1121 +} 1122 + 1123 +static int testFormatSize(char *aBuf, int nBuf, i64 nByte){ 1124 + int res; 1125 + if( nByte<(1<<10) ){ 1126 + res = snprintf(aBuf, nBuf, "%d byte", (int)nByte); 1127 + }else if( nByte<(1<<20) ){ 1128 + res = snprintf(aBuf, nBuf, "%dK", (int)(nByte/(1<<10))); 1129 + }else{ 1130 + res = snprintf(aBuf, nBuf, "%dM", (int)(nByte/(1<<20))); 1131 + } 1132 + return res; 1133 +} 1134 + 1135 +static i64 testReadSize(char *z){ 1136 + int n = strlen(z); 1137 + char c = z[n-1]; 1138 + i64 nMul = 1; 1139 + 1140 + switch( c ){ 1141 + case 'g': case 'G': 1142 + nMul = (1<<30); 1143 + break; 1144 + 1145 + case 'm': case 'M': 1146 + nMul = (1<<20); 1147 + break; 1148 + 1149 + case 'k': case 'K': 1150 + nMul = (1<<10); 1151 + break; 1152 + 1153 + default: 1154 + nMul = 1; 1155 + } 1156 + 1157 + return nMul * (i64)atoi(z); 1158 +} 1159 + 1160 +/* 1161 +** Usage: lsmtest writespeed FILESIZE BLOCKSIZE SYNCSIZE 1162 +*/ 1163 +static int do_writer_test(int nArg, char **azArg){ 1164 + int nBlock; 1165 + int nSize; 1166 + int i; 1167 + int fd; 1168 + int ms; 1169 + char aFilesize[32]; 1170 + char aBlockSize[32]; 1171 + 1172 + char *aPage; 1173 + int *aOrder; 1174 + int nSync; 1175 + 1176 + i64 filesize; 1177 + i64 blocksize; 1178 + i64 syncsize; 1179 + int nPage = 4096; 1180 + 1181 + /* How long to sleep before running a trial (in ms). */ 1182 +#if 0 1183 + const int nSleep = 10000; 1184 +#endif 1185 + const int nSleep = 0; 1186 + 1187 + if( nArg!=3 ){ 1188 + testPrintUsage("FILESIZE BLOCKSIZE SYNCSIZE"); 1189 + return -1; 1190 + } 1191 + 1192 + filesize = testReadSize(azArg[0]); 1193 + blocksize = testReadSize(azArg[1]); 1194 + syncsize = testReadSize(azArg[2]); 1195 + 1196 + nBlock = (int)(filesize / blocksize); 1197 + nSize = (int)blocksize; 1198 + nSync = (int)(syncsize / blocksize); 1199 + 1200 + aPage = (char *)malloc(4096); 1201 + aOrder = (int *)malloc(nBlock * sizeof(int)); 1202 + for(i=0; i<nBlock; i++) aOrder[i] = i; 1203 + for(i=0; i<(nBlock*25); i++){ 1204 + int tmp; 1205 + u32 a = testPrngValue(i); 1206 + u32 b = testPrngValue(a); 1207 + a = a % nBlock; 1208 + b = b % nBlock; 1209 + tmp = aOrder[a]; 1210 + aOrder[a] = aOrder[b]; 1211 + aOrder[b] = tmp; 1212 + } 1213 + 1214 + testFormatSize(aFilesize, sizeof(aFilesize), (i64)nBlock * (i64)nSize); 1215 + testFormatSize(aBlockSize, sizeof(aFilesize), nSize); 1216 + 1217 + printf("Testing writing a %s file using %s blocks. ", aFilesize, aBlockSize); 1218 + if( nSync==1 ){ 1219 + printf("Sync after each block.\n"); 1220 + }else{ 1221 + printf("Sync after each %d blocks.\n", nSync); 1222 + } 1223 + 1224 + printf("Preparing file... "); 1225 + fflush(stdout); 1226 + unlink("writer.out"); 1227 + fd = open("writer.out", O_RDWR|O_CREAT|_O_BINARY, 0664); 1228 + if( fd<0 ){ 1229 + testPrintError("open(): %d - %s\n", errno, strerror(errno)); 1230 + return -1; 1231 + } 1232 + testTimeInit(); 1233 + for(i=0; i<nBlock; i++){ 1234 + int iPg; 1235 + memset(aPage, i&0xFF, nPage); 1236 + for(iPg=0; iPg<(nSize/nPage); iPg++){ 1237 + write(fd, aPage, nPage); 1238 + } 1239 + } 1240 + fsync(fd); 1241 + printf("ok (%d ms)\n", testTimeGet()); 1242 + 1243 + for(i=0; i<5; i++){ 1244 + int j; 1245 + 1246 + sqlite3_sleep(nSleep); 1247 + printf("Now writing sequentially... "); 1248 + fflush(stdout); 1249 + 1250 + lseek(fd, 0, SEEK_SET); 1251 + testTimeInit(); 1252 + for(j=0; j<nBlock; j++){ 1253 + int iPg; 1254 + if( ((j+1)%nSync)==0 ) fdatasync(fd); 1255 + memset(aPage, j&0xFF, nPage); 1256 + for(iPg=0; iPg<(nSize/nPage); iPg++){ 1257 + write(fd, aPage, nPage); 1258 + } 1259 + } 1260 + fdatasync(fd); 1261 + ms = testTimeGet(); 1262 + printf("%d ms\n", ms); 1263 + sqlite3_sleep(nSleep); 1264 + printf("Now in an arbitrary order... "); 1265 + 1266 + fflush(stdout); 1267 + testTimeInit(); 1268 + for(j=0; j<nBlock; j++){ 1269 + int iPg; 1270 + if( ((j+1)%nSync)==0 ) fdatasync(fd); 1271 + lseek(fd, aOrder[j]*nSize, SEEK_SET); 1272 + memset(aPage, j&0xFF, nPage); 1273 + for(iPg=0; iPg<(nSize/nPage); iPg++){ 1274 + write(fd, aPage, nPage); 1275 + } 1276 + } 1277 + fdatasync(fd); 1278 + ms = testTimeGet(); 1279 + printf("%d ms\n", ms); 1280 + } 1281 + 1282 + close(fd); 1283 + free(aPage); 1284 + free(aOrder); 1285 + 1286 + return 0; 1287 +} 1288 + 1289 +static void do_insert_work_hook(lsm_db *db, void *p){ 1290 + char *z = 0; 1291 + lsm_info(db, LSM_INFO_DB_STRUCTURE, &z); 1292 + if( z ){ 1293 + printf("%s\n", z); 1294 + fflush(stdout); 1295 + lsm_free(lsm_get_env(db), z); 1296 + } 1297 + 1298 + unused_parameter(p); 1299 +} 1300 + 1301 +typedef struct InsertWriteHook InsertWriteHook; 1302 +struct InsertWriteHook { 1303 + FILE *pOut; 1304 + int bLog; 1305 + i64 iOff; 1306 + int nData; 1307 +}; 1308 + 1309 +static void flushHook(InsertWriteHook *pHook){ 1310 + if( pHook->nData ){ 1311 + fprintf(pHook->pOut, "write %s %d %d\n", 1312 + (pHook->bLog ? "log" : "db"), (int)pHook->iOff, pHook->nData 1313 + ); 1314 + pHook->nData = 0; 1315 + fflush(pHook->pOut); 1316 + } 1317 +} 1318 + 1319 +static void do_insert_write_hook( 1320 + void *pCtx, 1321 + int bLog, 1322 + i64 iOff, 1323 + int nData, 1324 + int nUs 1325 +){ 1326 + InsertWriteHook *pHook = (InsertWriteHook *)pCtx; 1327 + if( bLog ) return; 1328 + 1329 + if( nData==0 ){ 1330 + flushHook(pHook); 1331 + fprintf(pHook->pOut, "sync %s\n", (bLog ? "log" : "db")); 1332 + }else if( pHook->nData 1333 + && bLog==pHook->bLog 1334 + && iOff==(pHook->iOff+pHook->nData) 1335 + ){ 1336 + pHook->nData += nData; 1337 + }else{ 1338 + flushHook(pHook); 1339 + pHook->bLog = bLog; 1340 + pHook->iOff = iOff; 1341 + pHook->nData = nData; 1342 + } 1343 +} 1344 + 1345 +static int do_replay(int nArg, char **azArg){ 1346 + char aBuf[4096]; 1347 + FILE *pInput; 1348 + FILE *pClose = 0; 1349 + const char *zDb; 1350 + 1351 + lsm_env *pEnv; 1352 + lsm_file *pOut; 1353 + int rc; 1354 + 1355 + if( nArg!=2 ){ 1356 + testPrintError("Usage: replay WRITELOG FILE\n"); 1357 + return 1; 1358 + } 1359 + 1360 + if( strcmp(azArg[0], "-")==0 ){ 1361 + pInput = stdin; 1362 + }else{ 1363 + pClose = pInput = fopen(azArg[0], "r"); 1364 + } 1365 + zDb = azArg[1]; 1366 + pEnv = tdb_lsm_env(); 1367 + rc = pEnv->xOpen(pEnv, zDb, 0, &pOut); 1368 + if( rc!=LSM_OK ) return rc; 1369 + 1370 + while( feof(pInput)==0 ){ 1371 + char zLine[80]; 1372 + fgets(zLine, sizeof(zLine)-1, pInput); 1373 + zLine[sizeof(zLine)-1] = '\0'; 1374 + 1375 + if( 0==memcmp("sync db", zLine, 7) ){ 1376 + rc = pEnv->xSync(pOut); 1377 + if( rc!=0 ) break; 1378 + }else{ 1379 + int iOff; 1380 + int nData; 1381 + int nMatch; 1382 + nMatch = sscanf(zLine, "write db %d %d", &iOff, &nData); 1383 + if( nMatch==2 ){ 1384 + int i; 1385 + for(i=0; i<nData; i+=sizeof(aBuf)){ 1386 + memset(aBuf, i&0xFF, sizeof(aBuf)); 1387 + rc = pEnv->xWrite(pOut, iOff+i, aBuf, sizeof(aBuf)); 1388 + if( rc!=0 ) break; 1389 + } 1390 + } 1391 + } 1392 + } 1393 + if( pClose ) fclose(pClose); 1394 + pEnv->xClose(pOut); 1395 + 1396 + return rc; 1397 +} 1398 + 1399 +static int do_insert(int nArg, char **azArg){ 1400 + const char *zDb = "lsm"; 1401 + TestDb *pDb = 0; 1402 + int i; 1403 + int rc; 1404 + const int nRow = 1 * 1000 * 1000; 1405 + 1406 + DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 8, 15, 80, 150 }; 1407 + Datasource *pData = 0; 1408 + 1409 + if( nArg>1 ){ 1410 + testPrintError("Usage: insert ?DATABASE?\n"); 1411 + return 1; 1412 + } 1413 + if( nArg==1 ){ zDb = azArg[0]; } 1414 + 1415 + testMallocUninstall(tdb_lsm_env()); 1416 + for(i=0; zDb[i] && zDb[i]!='='; i++); 1417 + if( zDb[i] ){ 1418 + rc = tdb_lsm_open(zDb, "testdb.lsm", 1, &pDb); 1419 + }else{ 1420 + rc = tdb_open(zDb, 0, 1, &pDb); 1421 + } 1422 + 1423 + if( rc!=0 ){ 1424 + testPrintError("Error opening db \"%s\": %d\n", zDb, rc); 1425 + }else{ 1426 + InsertWriteHook hook; 1427 + memset(&hook, 0, sizeof(hook)); 1428 + hook.pOut = fopen("writelog.txt", "w"); 1429 + 1430 + pData = testDatasourceNew(&defn); 1431 + tdb_lsm_config_work_hook(pDb, do_insert_work_hook, 0); 1432 + tdb_lsm_write_hook(pDb, do_insert_write_hook, (void *)&hook); 1433 + 1434 + if( rc==0 ){ 1435 + for(i=0; i<nRow; i++){ 1436 + void *pKey; int nKey; /* Database key to insert */ 1437 + void *pVal; int nVal; /* Database value to insert */ 1438 + testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal); 1439 + tdb_write(pDb, pKey, nKey, pVal, nVal); 1440 + } 1441 + } 1442 + 1443 + testDatasourceFree(pData); 1444 + tdb_close(pDb); 1445 + flushHook(&hook); 1446 + fclose(hook.pOut); 1447 + } 1448 + testMallocInstall(tdb_lsm_env()); 1449 + 1450 + return rc; 1451 +} 1452 + 1453 +static int st_do_show(int a, char **b) { return do_show(a, b); } 1454 +static int st_do_work(int a, char **b) { return do_work(a, b); } 1455 +static int st_do_io(int a, char **b) { return do_io(a, b); } 1456 + 1457 +#ifdef __linux__ 1458 +#include <sys/time.h> 1459 +#include <sys/resource.h> 1460 + 1461 +static void lsmtest_rusage_report(void){ 1462 + struct rusage r; 1463 + memset(&r, 0, sizeof(r)); 1464 + 1465 + getrusage(RUSAGE_SELF, &r); 1466 + printf("# getrusage: { ru_maxrss %d ru_oublock %d ru_inblock %d }\n", 1467 + (int)r.ru_maxrss, (int)r.ru_oublock, (int)r.ru_inblock 1468 + ); 1469 +} 1470 +#else 1471 +static void lsmtest_rusage_report(void){ 1472 + /* no-op */ 1473 +} 1474 +#endif 1475 + 1476 +int main(int argc, char **argv){ 1477 + struct TestFunc { 1478 + const char *zName; 1479 + int bRusageReport; 1480 + int (*xFunc)(int, char **); 1481 + } aTest[] = { 1482 + {"random", 1, do_random_tests}, 1483 + {"writespeed", 1, do_writer_test}, 1484 + {"io", 1, st_do_io}, 1485 + 1486 + {"insert", 1, do_insert}, 1487 + {"replay", 1, do_replay}, 1488 + 1489 + {"speed", 1, do_speed_tests}, 1490 + {"speed2", 1, do_speed_test2}, 1491 + {"show", 0, st_do_show}, 1492 + {"work", 1, st_do_work}, 1493 + {"test", 1, do_test}, 1494 + 1495 + {0, 0} 1496 + }; 1497 + int rc; /* Return Code */ 1498 + int iFunc; /* Index into aTest[] */ 1499 + 1500 + int nLeakAlloc = 0; /* Allocations leaked by lsm */ 1501 + int nLeakByte = 0; /* Bytes leaked by lsm */ 1502 + 1503 +#ifdef LSM_DEBUG_MEM 1504 + FILE *pReport = 0; /* lsm malloc() report file */ 1505 + const char *zReport = "malloc.txt generated"; 1506 +#else 1507 + const char *zReport = "malloc.txt NOT generated"; 1508 +#endif 1509 + 1510 + testMallocInstall(tdb_lsm_env()); 1511 + 1512 + if( argc<2 ){ 1513 + testPrintError("Usage: %s sub-command ?args...?\n", argv[0]); 1514 + return -1; 1515 + } 1516 + 1517 + /* Initialize error reporting */ 1518 + testErrorInit(argc, argv); 1519 + 1520 + /* Initialize PRNG system */ 1521 + testPrngInit(); 1522 + 1523 + rc = testArgSelect(aTest, "sub-command", argv[1], &iFunc); 1524 + if( rc==0 ){ 1525 + rc = aTest[iFunc].xFunc(argc-2, &argv[2]); 1526 + } 1527 + 1528 +#ifdef LSM_DEBUG_MEM 1529 + pReport = fopen("malloc.txt", "w"); 1530 + testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, pReport); 1531 + fclose(pReport); 1532 +#else 1533 + testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, 0); 1534 +#endif 1535 + 1536 + if( nLeakAlloc ){ 1537 + testPrintError("Leaked %d bytes in %d allocations (%s)\n", 1538 + nLeakByte, nLeakAlloc, zReport 1539 + ); 1540 + if( rc==0 ) rc = -1; 1541 + } 1542 + testMallocUninstall(tdb_lsm_env()); 1543 + 1544 + if( aTest[iFunc].bRusageReport ){ 1545 + lsmtest_rusage_report(); 1546 + } 1547 + return rc; 1548 +}
Added ext/lsm1/lsm-test/lsmtest_mem.c.
1 + 2 +#include <stdio.h> 3 +#include <assert.h> 4 +#include <string.h> 5 + 6 +#define ArraySize(x) ((int)(sizeof(x) / sizeof((x)[0]))) 7 + 8 +#define MIN(x,y) ((x)<(y) ? (x) : (y)) 9 + 10 +typedef unsigned int u32; 11 +typedef unsigned char u8; 12 +typedef long long int i64; 13 +typedef unsigned long long int u64; 14 + 15 +#if defined(__GLIBC__) && defined(LSM_DEBUG_MEM) 16 + extern int backtrace(void**,int); 17 + extern void backtrace_symbols_fd(void*const*,int,int); 18 +# define TM_BACKTRACE 12 19 +#else 20 +# define backtrace(A,B) 1 21 +# define backtrace_symbols_fd(A,B,C) 22 +#endif 23 + 24 + 25 +typedef struct TmBlockHdr TmBlockHdr; 26 +typedef struct TmAgg TmAgg; 27 +typedef struct TmGlobal TmGlobal; 28 + 29 +struct TmGlobal { 30 + /* Linked list of all currently outstanding allocations. And a table of 31 + ** all allocations, past and present, indexed by backtrace() info. */ 32 + TmBlockHdr *pFirst; 33 +#ifdef TM_BACKTRACE 34 + TmAgg *aHash[10000]; 35 +#endif 36 + 37 + /* Underlying malloc/realloc/free functions */ 38 + void *(*xMalloc)(int); /* underlying malloc(3) function */ 39 + void *(*xRealloc)(void *, int); /* underlying realloc(3) function */ 40 + void (*xFree)(void *); /* underlying free(3) function */ 41 + 42 + /* Mutex to protect pFirst and aHash */ 43 + void (*xEnterMutex)(TmGlobal*); /* Call this to enter the mutex */ 44 + void (*xLeaveMutex)(TmGlobal*); /* Call this to leave mutex */ 45 + void (*xDelMutex)(TmGlobal*); /* Call this to delete mutex */ 46 + void *pMutex; /* Mutex handle */ 47 + 48 + void *(*xSaveMalloc)(void *, size_t); 49 + void *(*xSaveRealloc)(void *, void *, size_t); 50 + void (*xSaveFree)(void *, void *); 51 + 52 + /* OOM injection scheduling. If nCountdown is greater than zero when a 53 + ** malloc attempt is made, it is decremented. If this means nCountdown 54 + ** transitions from 1 to 0, then the allocation fails. If bPersist is true 55 + ** when this happens, nCountdown is then incremented back to 1 (so that the 56 + ** next attempt fails too). 57 + */ 58 + int nCountdown; 59 + int bPersist; 60 + int bEnable; 61 + void (*xHook)(void *); 62 + void *pHookCtx; 63 +}; 64 + 65 +struct TmBlockHdr { 66 + TmBlockHdr *pNext; 67 + TmBlockHdr *pPrev; 68 + int nByte; 69 +#ifdef TM_BACKTRACE 70 + TmAgg *pAgg; 71 +#endif 72 + u32 iForeGuard; 73 +}; 74 + 75 +#ifdef TM_BACKTRACE 76 +struct TmAgg { 77 + int nAlloc; /* Number of allocations at this path */ 78 + int nByte; /* Total number of bytes allocated */ 79 + int nOutAlloc; /* Number of outstanding allocations */ 80 + int nOutByte; /* Number of outstanding bytes */ 81 + void *aFrame[TM_BACKTRACE]; /* backtrace() output */ 82 + TmAgg *pNext; /* Next object in hash-table collision */ 83 +}; 84 +#endif 85 + 86 +#define FOREGUARD 0x80F5E153 87 +#define REARGUARD 0xE4676B53 88 +static const u32 rearguard = REARGUARD; 89 + 90 +#define ROUND8(x) (((x)+7)&~7) 91 + 92 +#define BLOCK_HDR_SIZE (ROUND8( sizeof(TmBlockHdr) )) 93 + 94 +static void lsmtest_oom_error(void){ 95 + static int nErr = 0; 96 + nErr++; 97 +} 98 + 99 +static void tmEnterMutex(TmGlobal *pTm){ 100 + pTm->xEnterMutex(pTm); 101 +} 102 +static void tmLeaveMutex(TmGlobal *pTm){ 103 + pTm->xLeaveMutex(pTm); 104 +} 105 + 106 +static void *tmMalloc(TmGlobal *pTm, int nByte){ 107 + TmBlockHdr *pNew; /* New allocation header block */ 108 + u8 *pUser; /* Return value */ 109 + int nReq; /* Total number of bytes requested */ 110 + 111 + assert( sizeof(rearguard)==4 ); 112 + nReq = BLOCK_HDR_SIZE + nByte + 4; 113 + pNew = (TmBlockHdr *)pTm->xMalloc(nReq); 114 + memset(pNew, 0, sizeof(TmBlockHdr)); 115 + 116 + tmEnterMutex(pTm); 117 + assert( pTm->nCountdown>=0 ); 118 + assert( pTm->bPersist==0 || pTm->bPersist==1 ); 119 + 120 + if( pTm->bEnable && pTm->nCountdown==1 ){ 121 + /* Simulate an OOM error. */ 122 + lsmtest_oom_error(); 123 + pTm->xFree(pNew); 124 + pTm->nCountdown = pTm->bPersist; 125 + if( pTm->xHook ) pTm->xHook(pTm->pHookCtx); 126 + pUser = 0; 127 + }else{ 128 + if( pTm->bEnable && pTm->nCountdown ) pTm->nCountdown--; 129 + 130 + pNew->iForeGuard = FOREGUARD; 131 + pNew->nByte = nByte; 132 + pNew->pNext = pTm->pFirst; 133 + 134 + if( pTm->pFirst ){ 135 + pTm->pFirst->pPrev = pNew; 136 + } 137 + pTm->pFirst = pNew; 138 + 139 + pUser = &((u8 *)pNew)[BLOCK_HDR_SIZE]; 140 + memset(pUser, 0x56, nByte); 141 + memcpy(&pUser[nByte], &rearguard, 4); 142 + 143 +#ifdef TM_BACKTRACE 144 + { 145 + TmAgg *pAgg; 146 + int i; 147 + u32 iHash = 0; 148 + void *aFrame[TM_BACKTRACE]; 149 + memset(aFrame, 0, sizeof(aFrame)); 150 + backtrace(aFrame, TM_BACKTRACE); 151 + 152 + for(i=0; i<ArraySize(aFrame); i++){ 153 + iHash += (u64)(aFrame[i]) + (iHash<<3); 154 + } 155 + iHash = iHash % ArraySize(pTm->aHash); 156 + 157 + for(pAgg=pTm->aHash[iHash]; pAgg; pAgg=pAgg->pNext){ 158 + if( memcmp(pAgg->aFrame, aFrame, sizeof(aFrame))==0 ) break; 159 + } 160 + if( !pAgg ){ 161 + pAgg = (TmAgg *)pTm->xMalloc(sizeof(TmAgg)); 162 + memset(pAgg, 0, sizeof(TmAgg)); 163 + memcpy(pAgg->aFrame, aFrame, sizeof(aFrame)); 164 + pAgg->pNext = pTm->aHash[iHash]; 165 + pTm->aHash[iHash] = pAgg; 166 + } 167 + pAgg->nAlloc++; 168 + pAgg->nByte += nByte; 169 + pAgg->nOutAlloc++; 170 + pAgg->nOutByte += nByte; 171 + pNew->pAgg = pAgg; 172 + } 173 +#endif 174 + } 175 + 176 + tmLeaveMutex(pTm); 177 + return pUser; 178 +} 179 + 180 +static void tmFree(TmGlobal *pTm, void *p){ 181 + if( p ){ 182 + TmBlockHdr *pHdr; 183 + u8 *pUser = (u8 *)p; 184 + 185 + tmEnterMutex(pTm); 186 + pHdr = (TmBlockHdr *)(pUser - BLOCK_HDR_SIZE); 187 + assert( pHdr->iForeGuard==FOREGUARD ); 188 + assert( 0==memcmp(&pUser[pHdr->nByte], &rearguard, 4) ); 189 + 190 + if( pHdr->pPrev ){ 191 + assert( pHdr->pPrev->pNext==pHdr ); 192 + pHdr->pPrev->pNext = pHdr->pNext; 193 + }else{ 194 + assert( pHdr==pTm->pFirst ); 195 + pTm->pFirst = pHdr->pNext; 196 + } 197 + if( pHdr->pNext ){ 198 + assert( pHdr->pNext->pPrev==pHdr ); 199 + pHdr->pNext->pPrev = pHdr->pPrev; 200 + } 201 + 202 +#ifdef TM_BACKTRACE 203 + pHdr->pAgg->nOutAlloc--; 204 + pHdr->pAgg->nOutByte -= pHdr->nByte; 205 +#endif 206 + 207 + tmLeaveMutex(pTm); 208 + memset(pUser, 0x58, pHdr->nByte); 209 + memset(pHdr, 0x57, sizeof(TmBlockHdr)); 210 + pTm->xFree(pHdr); 211 + } 212 +} 213 + 214 +static void *tmRealloc(TmGlobal *pTm, void *p, int nByte){ 215 + void *pNew; 216 + 217 + pNew = tmMalloc(pTm, nByte); 218 + if( pNew && p ){ 219 + TmBlockHdr *pHdr; 220 + u8 *pUser = (u8 *)p; 221 + pHdr = (TmBlockHdr *)(pUser - BLOCK_HDR_SIZE); 222 + memcpy(pNew, p, MIN(nByte, pHdr->nByte)); 223 + tmFree(pTm, p); 224 + } 225 + return pNew; 226 +} 227 + 228 +static void tmMallocOom( 229 + TmGlobal *pTm, 230 + int nCountdown, 231 + int bPersist, 232 + void (*xHook)(void *), 233 + void *pHookCtx 234 +){ 235 + assert( nCountdown>=0 ); 236 + assert( bPersist==0 || bPersist==1 ); 237 + pTm->nCountdown = nCountdown; 238 + pTm->bPersist = bPersist; 239 + pTm->xHook = xHook; 240 + pTm->pHookCtx = pHookCtx; 241 + pTm->bEnable = 1; 242 +} 243 + 244 +static void tmMallocOomEnable( 245 + TmGlobal *pTm, 246 + int bEnable 247 +){ 248 + pTm->bEnable = bEnable; 249 +} 250 + 251 +static void tmMallocCheck( 252 + TmGlobal *pTm, 253 + int *pnLeakAlloc, 254 + int *pnLeakByte, 255 + FILE *pFile 256 +){ 257 + TmBlockHdr *pHdr; 258 + int nLeak = 0; 259 + int nByte = 0; 260 + 261 + if( pTm==0 ) return; 262 + 263 + for(pHdr=pTm->pFirst; pHdr; pHdr=pHdr->pNext){ 264 + nLeak++; 265 + nByte += pHdr->nByte; 266 + } 267 + if( pnLeakAlloc ) *pnLeakAlloc = nLeak; 268 + if( pnLeakByte ) *pnLeakByte = nByte; 269 + 270 +#ifdef TM_BACKTRACE 271 + if( pFile ){ 272 + int i; 273 + fprintf(pFile, "LEAKS\n"); 274 + for(i=0; i<ArraySize(pTm->aHash); i++){ 275 + TmAgg *pAgg; 276 + for(pAgg=pTm->aHash[i]; pAgg; pAgg=pAgg->pNext){ 277 + if( pAgg->nOutAlloc ){ 278 + int j; 279 + fprintf(pFile, "%d %d ", pAgg->nOutByte, pAgg->nOutAlloc); 280 + for(j=0; j<TM_BACKTRACE; j++){ 281 + fprintf(pFile, "%p ", pAgg->aFrame[j]); 282 + } 283 + fprintf(pFile, "\n"); 284 + } 285 + } 286 + } 287 + fprintf(pFile, "\nALLOCATIONS\n"); 288 + for(i=0; i<ArraySize(pTm->aHash); i++){ 289 + TmAgg *pAgg; 290 + for(pAgg=pTm->aHash[i]; pAgg; pAgg=pAgg->pNext){ 291 + int j; 292 + fprintf(pFile, "%d %d ", pAgg->nByte, pAgg->nAlloc); 293 + for(j=0; j<TM_BACKTRACE; j++) fprintf(pFile, "%p ", pAgg->aFrame[j]); 294 + fprintf(pFile, "\n"); 295 + } 296 + } 297 + } 298 +#else 299 + (void)pFile; 300 +#endif 301 +} 302 + 303 + 304 +#include "lsm.h" 305 +#include "stdlib.h" 306 + 307 +typedef struct LsmMutex LsmMutex; 308 +struct LsmMutex { 309 + lsm_env *pEnv; 310 + lsm_mutex *pMutex; 311 +}; 312 + 313 +static void tmLsmMutexEnter(TmGlobal *pTm){ 314 + LsmMutex *p = (LsmMutex *)pTm->pMutex; 315 + p->pEnv->xMutexEnter(p->pMutex); 316 +} 317 +static void tmLsmMutexLeave(TmGlobal *pTm){ 318 + LsmMutex *p = (LsmMutex *)(pTm->pMutex); 319 + p->pEnv->xMutexLeave(p->pMutex); 320 +} 321 +static void tmLsmMutexDel(TmGlobal *pTm){ 322 + LsmMutex *p = (LsmMutex *)pTm->pMutex; 323 + pTm->xFree(p); 324 +} 325 +static void *tmLsmMalloc(int n){ return malloc(n); } 326 +static void tmLsmFree(void *ptr){ free(ptr); } 327 +static void *tmLsmRealloc(void *ptr, int n){ return realloc(ptr, n); } 328 + 329 +static void *tmLsmEnvMalloc(lsm_env *p, size_t n){ 330 + return tmMalloc((TmGlobal *)(p->pMemCtx), n); 331 +} 332 +static void tmLsmEnvFree(lsm_env *p, void *ptr){ 333 + tmFree((TmGlobal *)(p->pMemCtx), ptr); 334 +} 335 +static void *tmLsmEnvRealloc(lsm_env *p, void *ptr, size_t n){ 336 + return tmRealloc((TmGlobal *)(p->pMemCtx), ptr, n); 337 +} 338 + 339 +void testMallocInstall(lsm_env *pEnv){ 340 + TmGlobal *pGlobal; 341 + LsmMutex *pMutex; 342 + assert( pEnv->pMemCtx==0 ); 343 + 344 + /* Allocate and populate a TmGlobal structure. */ 345 + pGlobal = (TmGlobal *)tmLsmMalloc(sizeof(TmGlobal)); 346 + memset(pGlobal, 0, sizeof(TmGlobal)); 347 + pGlobal->xMalloc = tmLsmMalloc; 348 + pGlobal->xRealloc = tmLsmRealloc; 349 + pGlobal->xFree = tmLsmFree; 350 + pMutex = (LsmMutex *)pGlobal->xMalloc(sizeof(LsmMutex)); 351 + pMutex->pEnv = pEnv; 352 + pEnv->xMutexStatic(pEnv, LSM_MUTEX_HEAP, &pMutex->pMutex); 353 + pGlobal->xEnterMutex = tmLsmMutexEnter; 354 + pGlobal->xLeaveMutex = tmLsmMutexLeave; 355 + pGlobal->xDelMutex = tmLsmMutexDel; 356 + pGlobal->pMutex = (void *)pMutex; 357 + 358 + pGlobal->xSaveMalloc = pEnv->xMalloc; 359 + pGlobal->xSaveRealloc = pEnv->xRealloc; 360 + pGlobal->xSaveFree = pEnv->xFree; 361 + 362 + /* Set up pEnv to the use the new TmGlobal */ 363 + pEnv->pMemCtx = (void *)pGlobal; 364 + pEnv->xMalloc = tmLsmEnvMalloc; 365 + pEnv->xRealloc = tmLsmEnvRealloc; 366 + pEnv->xFree = tmLsmEnvFree; 367 +} 368 + 369 +void testMallocUninstall(lsm_env *pEnv){ 370 + TmGlobal *p = (TmGlobal *)pEnv->pMemCtx; 371 + pEnv->pMemCtx = 0; 372 + if( p ){ 373 + pEnv->xMalloc = p->xSaveMalloc; 374 + pEnv->xRealloc = p->xSaveRealloc; 375 + pEnv->xFree = p->xSaveFree; 376 + p->xDelMutex(p); 377 + tmLsmFree(p); 378 + } 379 +} 380 + 381 +void testMallocCheck( 382 + lsm_env *pEnv, 383 + int *pnLeakAlloc, 384 + int *pnLeakByte, 385 + FILE *pFile 386 +){ 387 + if( pEnv->pMemCtx==0 ){ 388 + *pnLeakAlloc = 0; 389 + *pnLeakByte = 0; 390 + }else{ 391 + tmMallocCheck((TmGlobal *)(pEnv->pMemCtx), pnLeakAlloc, pnLeakByte, pFile); 392 + } 393 +} 394 + 395 +void testMallocOom( 396 + lsm_env *pEnv, 397 + int nCountdown, 398 + int bPersist, 399 + void (*xHook)(void *), 400 + void *pHookCtx 401 +){ 402 + TmGlobal *pTm = (TmGlobal *)(pEnv->pMemCtx); 403 + tmMallocOom(pTm, nCountdown, bPersist, xHook, pHookCtx); 404 +} 405 + 406 +void testMallocOomEnable(lsm_env *pEnv, int bEnable){ 407 + TmGlobal *pTm = (TmGlobal *)(pEnv->pMemCtx); 408 + tmMallocOomEnable(pTm, bEnable); 409 +}
Added ext/lsm1/lsm-test/lsmtest_tdb.c.
1 + 2 +/* 3 +** This program attempts to test the correctness of some facets of the 4 +** LSM database library. Specifically, that the contents of the database 5 +** are maintained correctly during a series of inserts and deletes. 6 +*/ 7 + 8 + 9 +#include "lsmtest_tdb.h" 10 +#include "lsm.h" 11 + 12 +#include "lsmtest.h" 13 + 14 +#include <stdlib.h> 15 +#include <string.h> 16 +#include <assert.h> 17 +#ifndef _WIN32 18 +# include <unistd.h> 19 +#endif 20 +#include <stdio.h> 21 + 22 + 23 +typedef struct SqlDb SqlDb; 24 + 25 +static int error_transaction_function(TestDb *p, int iLevel){ 26 + unused_parameter(p); 27 + unused_parameter(iLevel); 28 + return -1; 29 +} 30 + 31 + 32 +/************************************************************************* 33 +** Begin wrapper for LevelDB. 34 +*/ 35 +#ifdef HAVE_LEVELDB 36 + 37 +#include <leveldb/c.h> 38 + 39 +typedef struct LevelDb LevelDb; 40 +struct LevelDb { 41 + TestDb base; 42 + leveldb_t *db; 43 + leveldb_options_t *pOpt; 44 + leveldb_writeoptions_t *pWriteOpt; 45 + leveldb_readoptions_t *pReadOpt; 46 + 47 + char *pVal; 48 +}; 49 + 50 +static int test_leveldb_close(TestDb *pTestDb){ 51 + LevelDb *pDb = (LevelDb *)pTestDb; 52 + 53 + leveldb_close(pDb->db); 54 + leveldb_writeoptions_destroy(pDb->pWriteOpt); 55 + leveldb_readoptions_destroy(pDb->pReadOpt); 56 + leveldb_options_destroy(pDb->pOpt); 57 + free(pDb->pVal); 58 + free(pDb); 59 + 60 + return 0; 61 +} 62 + 63 +static int test_leveldb_write( 64 + TestDb *pTestDb, 65 + void *pKey, 66 + int nKey, 67 + void *pVal, 68 + int nVal 69 +){ 70 + LevelDb *pDb = (LevelDb *)pTestDb; 71 + char *zErr = 0; 72 + leveldb_put(pDb->db, pDb->pWriteOpt, pKey, nKey, pVal, nVal, &zErr); 73 + return (zErr!=0); 74 +} 75 + 76 +static int test_leveldb_delete(TestDb *pTestDb, void *pKey, int nKey){ 77 + LevelDb *pDb = (LevelDb *)pTestDb; 78 + char *zErr = 0; 79 + leveldb_delete(pDb->db, pDb->pWriteOpt, pKey, nKey, &zErr); 80 + return (zErr!=0); 81 +} 82 + 83 +static int test_leveldb_fetch( 84 + TestDb *pTestDb, 85 + void *pKey, 86 + int nKey, 87 + void **ppVal, 88 + int *pnVal 89 +){ 90 + LevelDb *pDb = (LevelDb *)pTestDb; 91 + char *zErr = 0; 92 + size_t nVal = 0; 93 + 94 + if( pKey==0 ) return 0; 95 + free(pDb->pVal); 96 + pDb->pVal = leveldb_get(pDb->db, pDb->pReadOpt, pKey, nKey, &nVal, &zErr); 97 + *ppVal = (void *)(pDb->pVal); 98 + if( pDb->pVal==0 ){ 99 + *pnVal = -1; 100 + }else{ 101 + *pnVal = (int)nVal; 102 + } 103 + 104 + return (zErr!=0); 105 +} 106 + 107 +static int test_leveldb_scan( 108 + TestDb *pTestDb, 109 + void *pCtx, 110 + int bReverse, 111 + void *pKey1, int nKey1, /* Start of search */ 112 + void *pKey2, int nKey2, /* End of search */ 113 + void (*xCallback)(void *, void *, int , void *, int) 114 +){ 115 + LevelDb *pDb = (LevelDb *)pTestDb; 116 + leveldb_iterator_t *iter; 117 + 118 + iter = leveldb_create_iterator(pDb->db, pDb->pReadOpt); 119 + 120 + if( bReverse==0 ){ 121 + if( pKey1 ){ 122 + leveldb_iter_seek(iter, pKey1, nKey1); 123 + }else{ 124 + leveldb_iter_seek_to_first(iter); 125 + } 126 + }else{ 127 + if( pKey2 ){ 128 + leveldb_iter_seek(iter, pKey2, nKey2); 129 + 130 + if( leveldb_iter_valid(iter)==0 ){ 131 + leveldb_iter_seek_to_last(iter); 132 + }else{ 133 + const char *k; size_t n; 134 + int res; 135 + k = leveldb_iter_key(iter, &n); 136 + res = memcmp(k, pKey2, MIN(n, nKey2)); 137 + if( res==0 ) res = n - nKey2; 138 + assert( res>=0 ); 139 + if( res>0 ){ 140 + leveldb_iter_prev(iter); 141 + } 142 + } 143 + }else{ 144 + leveldb_iter_seek_to_last(iter); 145 + } 146 + } 147 + 148 + 149 + while( leveldb_iter_valid(iter) ){ 150 + const char *k; size_t n; 151 + const char *v; size_t n2; 152 + int res; 153 + 154 + k = leveldb_iter_key(iter, &n); 155 + if( bReverse==0 && pKey2 ){ 156 + res = memcmp(k, pKey2, MIN(n, nKey2)); 157 + if( res==0 ) res = n - nKey2; 158 + if( res>0 ) break; 159 + } 160 + if( bReverse!=0 && pKey1 ){ 161 + res = memcmp(k, pKey1, MIN(n, nKey1)); 162 + if( res==0 ) res = n - nKey1; 163 + if( res<0 ) break; 164 + } 165 + 166 + v = leveldb_iter_value(iter, &n2); 167 + 168 + xCallback(pCtx, (void *)k, n, (void *)v, n2); 169 + 170 + if( bReverse==0 ){ 171 + leveldb_iter_next(iter); 172 + }else{ 173 + leveldb_iter_prev(iter); 174 + } 175 + } 176 + 177 + leveldb_iter_destroy(iter); 178 + return 0; 179 +} 180 + 181 +static int test_leveldb_open( 182 + const char *zSpec, 183 + const char *zFilename, 184 + int bClear, 185 + TestDb **ppDb 186 +){ 187 + static const DatabaseMethods LeveldbMethods = { 188 + test_leveldb_close, 189 + test_leveldb_write, 190 + test_leveldb_delete, 191 + 0, 192 + test_leveldb_fetch, 193 + test_leveldb_scan, 194 + error_transaction_function, 195 + error_transaction_function, 196 + error_transaction_function 197 + }; 198 + 199 + LevelDb *pLevelDb; 200 + char *zErr = 0; 201 + 202 + if( bClear ){ 203 + char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); 204 + system(zCmd); 205 + sqlite3_free(zCmd); 206 + } 207 + 208 + pLevelDb = (LevelDb *)malloc(sizeof(LevelDb)); 209 + memset(pLevelDb, 0, sizeof(LevelDb)); 210 + 211 + pLevelDb->pOpt = leveldb_options_create(); 212 + leveldb_options_set_create_if_missing(pLevelDb->pOpt, 1); 213 + pLevelDb->pWriteOpt = leveldb_writeoptions_create(); 214 + pLevelDb->pReadOpt = leveldb_readoptions_create(); 215 + 216 + pLevelDb->db = leveldb_open(pLevelDb->pOpt, zFilename, &zErr); 217 + 218 + if( zErr ){ 219 + test_leveldb_close((TestDb *)pLevelDb); 220 + *ppDb = 0; 221 + return 1; 222 + } 223 + 224 + *ppDb = (TestDb *)pLevelDb; 225 + pLevelDb->base.pMethods = &LeveldbMethods; 226 + return 0; 227 +} 228 +#endif /* HAVE_LEVELDB */ 229 +/* 230 +** End wrapper for LevelDB. 231 +*************************************************************************/ 232 + 233 +#ifdef HAVE_KYOTOCABINET 234 +static int kc_close(TestDb *pTestDb){ 235 + return test_kc_close(pTestDb); 236 +} 237 + 238 +static int kc_write( 239 + TestDb *pTestDb, 240 + void *pKey, 241 + int nKey, 242 + void *pVal, 243 + int nVal 244 +){ 245 + return test_kc_write(pTestDb, pKey, nKey, pVal, nVal); 246 +} 247 + 248 +static int kc_delete(TestDb *pTestDb, void *pKey, int nKey){ 249 + return test_kc_delete(pTestDb, pKey, nKey); 250 +} 251 + 252 +static int kc_delete_range( 253 + TestDb *pTestDb, 254 + void *pKey1, int nKey1, 255 + void *pKey2, int nKey2 256 +){ 257 + return test_kc_delete_range(pTestDb, pKey1, nKey1, pKey2, nKey2); 258 +} 259 + 260 +static int kc_fetch( 261 + TestDb *pTestDb, 262 + void *pKey, 263 + int nKey, 264 + void **ppVal, 265 + int *pnVal 266 +){ 267 + if( pKey==0 ) return LSM_OK; 268 + return test_kc_fetch(pTestDb, pKey, nKey, ppVal, pnVal); 269 +} 270 + 271 +static int kc_scan( 272 + TestDb *pTestDb, 273 + void *pCtx, 274 + int bReverse, 275 + void *pFirst, int nFirst, 276 + void *pLast, int nLast, 277 + void (*xCallback)(void *, void *, int , void *, int) 278 +){ 279 + return test_kc_scan( 280 + pTestDb, pCtx, bReverse, pFirst, nFirst, pLast, nLast, xCallback 281 + ); 282 +} 283 + 284 +static int kc_open( 285 + const char *zSpec, 286 + const char *zFilename, 287 + int bClear, 288 + TestDb **ppDb 289 +){ 290 + static const DatabaseMethods KcdbMethods = { 291 + kc_close, 292 + kc_write, 293 + kc_delete, 294 + kc_delete_range, 295 + kc_fetch, 296 + kc_scan, 297 + error_transaction_function, 298 + error_transaction_function, 299 + error_transaction_function 300 + }; 301 + 302 + int rc; 303 + TestDb *pTestDb = 0; 304 + 305 + rc = test_kc_open(zFilename, bClear, &pTestDb); 306 + if( rc!=0 ){ 307 + *ppDb = 0; 308 + return rc; 309 + } 310 + pTestDb->pMethods = &KcdbMethods; 311 + *ppDb = pTestDb; 312 + return 0; 313 +} 314 +#endif /* HAVE_KYOTOCABINET */ 315 +/* 316 +** End wrapper for Kyoto cabinet. 317 +*************************************************************************/ 318 + 319 +#ifdef HAVE_MDB 320 +static int mdb_close(TestDb *pTestDb){ 321 + return test_mdb_close(pTestDb); 322 +} 323 + 324 +static int mdb_write( 325 + TestDb *pTestDb, 326 + void *pKey, 327 + int nKey, 328 + void *pVal, 329 + int nVal 330 +){ 331 + return test_mdb_write(pTestDb, pKey, nKey, pVal, nVal); 332 +} 333 + 334 +static int mdb_delete(TestDb *pTestDb, void *pKey, int nKey){ 335 + return test_mdb_delete(pTestDb, pKey, nKey); 336 +} 337 + 338 +static int mdb_fetch( 339 + TestDb *pTestDb, 340 + void *pKey, 341 + int nKey, 342 + void **ppVal, 343 + int *pnVal 344 +){ 345 + if( pKey==0 ) return LSM_OK; 346 + return test_mdb_fetch(pTestDb, pKey, nKey, ppVal, pnVal); 347 +} 348 + 349 +static int mdb_scan( 350 + TestDb *pTestDb, 351 + void *pCtx, 352 + int bReverse, 353 + void *pFirst, int nFirst, 354 + void *pLast, int nLast, 355 + void (*xCallback)(void *, void *, int , void *, int) 356 +){ 357 + return test_mdb_scan( 358 + pTestDb, pCtx, bReverse, pFirst, nFirst, pLast, nLast, xCallback 359 + ); 360 +} 361 + 362 +static int mdb_open( 363 + const char *zSpec, 364 + const char *zFilename, 365 + int bClear, 366 + TestDb **ppDb 367 +){ 368 + static const DatabaseMethods KcdbMethods = { 369 + mdb_close, 370 + mdb_write, 371 + mdb_delete, 372 + 0, 373 + mdb_fetch, 374 + mdb_scan, 375 + error_transaction_function, 376 + error_transaction_function, 377 + error_transaction_function 378 + }; 379 + 380 + int rc; 381 + TestDb *pTestDb = 0; 382 + 383 + rc = test_mdb_open(zSpec, zFilename, bClear, &pTestDb); 384 + if( rc!=0 ){ 385 + *ppDb = 0; 386 + return rc; 387 + } 388 + pTestDb->pMethods = &KcdbMethods; 389 + *ppDb = pTestDb; 390 + return 0; 391 +} 392 +#endif /* HAVE_MDB */ 393 + 394 +/************************************************************************* 395 +** Begin wrapper for SQLite. 396 +*/ 397 + 398 +/* 399 +** nOpenTrans: 400 +** The number of open nested transactions, in the same sense as used 401 +** by the tdb_begin/commit/rollback and SQLite 4 KV interfaces. If this 402 +** value is 0, there are no transactions open at all. If it is 1, then 403 +** there is a read transaction. If it is 2 or greater, then there are 404 +** (nOpenTrans-1) nested write transactions open. 405 +*/ 406 +struct SqlDb { 407 + TestDb base; 408 + sqlite3 *db; 409 + sqlite3_stmt *pInsert; 410 + sqlite3_stmt *pDelete; 411 + sqlite3_stmt *pDeleteRange; 412 + sqlite3_stmt *pFetch; 413 + sqlite3_stmt *apScan[8]; 414 + 415 + int nOpenTrans; 416 + 417 + /* Used by sql_fetch() to allocate space for results */ 418 + int nAlloc; 419 + u8 *aAlloc; 420 +}; 421 + 422 +static int sql_close(TestDb *pTestDb){ 423 + SqlDb *pDb = (SqlDb *)pTestDb; 424 + sqlite3_finalize(pDb->pInsert); 425 + sqlite3_finalize(pDb->pDelete); 426 + sqlite3_finalize(pDb->pDeleteRange); 427 + sqlite3_finalize(pDb->pFetch); 428 + sqlite3_finalize(pDb->apScan[0]); 429 + sqlite3_finalize(pDb->apScan[1]); 430 + sqlite3_finalize(pDb->apScan[2]); 431 + sqlite3_finalize(pDb->apScan[3]); 432 + sqlite3_finalize(pDb->apScan[4]); 433 + sqlite3_finalize(pDb->apScan[5]); 434 + sqlite3_finalize(pDb->apScan[6]); 435 + sqlite3_finalize(pDb->apScan[7]); 436 + sqlite3_close(pDb->db); 437 + free((char *)pDb->aAlloc); 438 + free((char *)pDb); 439 + return SQLITE_OK; 440 +} 441 + 442 +static int sql_write( 443 + TestDb *pTestDb, 444 + void *pKey, 445 + int nKey, 446 + void *pVal, 447 + int nVal 448 +){ 449 + SqlDb *pDb = (SqlDb *)pTestDb; 450 + sqlite3_bind_blob(pDb->pInsert, 1, pKey, nKey, SQLITE_STATIC); 451 + sqlite3_bind_blob(pDb->pInsert, 2, pVal, nVal, SQLITE_STATIC); 452 + sqlite3_step(pDb->pInsert); 453 + return sqlite3_reset(pDb->pInsert); 454 +} 455 + 456 +static int sql_delete(TestDb *pTestDb, void *pKey, int nKey){ 457 + SqlDb *pDb = (SqlDb *)pTestDb; 458 + sqlite3_bind_blob(pDb->pDelete, 1, pKey, nKey, SQLITE_STATIC); 459 + sqlite3_step(pDb->pDelete); 460 + return sqlite3_reset(pDb->pDelete); 461 +} 462 + 463 +static int sql_delete_range( 464 + TestDb *pTestDb, 465 + void *pKey1, int nKey1, 466 + void *pKey2, int nKey2 467 +){ 468 + SqlDb *pDb = (SqlDb *)pTestDb; 469 + sqlite3_bind_blob(pDb->pDeleteRange, 1, pKey1, nKey1, SQLITE_STATIC); 470 + sqlite3_bind_blob(pDb->pDeleteRange, 2, pKey2, nKey2, SQLITE_STATIC); 471 + sqlite3_step(pDb->pDeleteRange); 472 + return sqlite3_reset(pDb->pDeleteRange); 473 +} 474 + 475 +static int sql_fetch( 476 + TestDb *pTestDb, 477 + void *pKey, 478 + int nKey, 479 + void **ppVal, 480 + int *pnVal 481 +){ 482 + SqlDb *pDb = (SqlDb *)pTestDb; 483 + int rc; 484 + 485 + sqlite3_reset(pDb->pFetch); 486 + if( pKey==0 ){ 487 + assert( ppVal==0 ); 488 + assert( pnVal==0 ); 489 + return LSM_OK; 490 + } 491 + 492 + sqlite3_bind_blob(pDb->pFetch, 1, pKey, nKey, SQLITE_STATIC); 493 + rc = sqlite3_step(pDb->pFetch); 494 + if( rc==SQLITE_ROW ){ 495 + int nVal = sqlite3_column_bytes(pDb->pFetch, 0); 496 + u8 *aVal = (void *)sqlite3_column_blob(pDb->pFetch, 0); 497 + 498 + if( nVal>pDb->nAlloc ){ 499 + free(pDb->aAlloc); 500 + pDb->aAlloc = (u8 *)malloc(nVal*2); 501 + pDb->nAlloc = nVal*2; 502 + } 503 + memcpy(pDb->aAlloc, aVal, nVal); 504 + *pnVal = nVal; 505 + *ppVal = (void *)pDb->aAlloc; 506 + }else{ 507 + *pnVal = -1; 508 + *ppVal = 0; 509 + } 510 + 511 + rc = sqlite3_reset(pDb->pFetch); 512 + return rc; 513 +} 514 + 515 +static int sql_scan( 516 + TestDb *pTestDb, 517 + void *pCtx, 518 + int bReverse, 519 + void *pFirst, int nFirst, 520 + void *pLast, int nLast, 521 + void (*xCallback)(void *, void *, int , void *, int) 522 +){ 523 + SqlDb *pDb = (SqlDb *)pTestDb; 524 + sqlite3_stmt *pScan; 525 + 526 + assert( bReverse==1 || bReverse==0 ); 527 + pScan = pDb->apScan[(pFirst==0) + (pLast==0)*2 + bReverse*4]; 528 + 529 + if( pFirst ) sqlite3_bind_blob(pScan, 1, pFirst, nFirst, SQLITE_STATIC); 530 + if( pLast ) sqlite3_bind_blob(pScan, 2, pLast, nLast, SQLITE_STATIC); 531 + 532 + while( SQLITE_ROW==sqlite3_step(pScan) ){ 533 + void *pKey; int nKey; 534 + void *pVal; int nVal; 535 + 536 + nKey = sqlite3_column_bytes(pScan, 0); 537 + pKey = (void *)sqlite3_column_blob(pScan, 0); 538 + nVal = sqlite3_column_bytes(pScan, 1); 539 + pVal = (void *)sqlite3_column_blob(pScan, 1); 540 + 541 + xCallback(pCtx, pKey, nKey, pVal, nVal); 542 + } 543 + return sqlite3_reset(pScan); 544 +} 545 + 546 +static int sql_begin(TestDb *pTestDb, int iLevel){ 547 + int i; 548 + SqlDb *pDb = (SqlDb *)pTestDb; 549 + 550 + /* iLevel==0 is a no-op */ 551 + if( iLevel==0 ) return 0; 552 + 553 + /* If there are no transactions at all open, open a read transaction. */ 554 + if( pDb->nOpenTrans==0 ){ 555 + int rc = sqlite3_exec(pDb->db, 556 + "BEGIN; SELECT * FROM sqlite_master LIMIT 1;" , 0, 0, 0 557 + ); 558 + if( rc!=0 ) return rc; 559 + pDb->nOpenTrans = 1; 560 + } 561 + 562 + /* Open any required write transactions */ 563 + for(i=pDb->nOpenTrans; i<iLevel; i++){ 564 + char *zSql = sqlite3_mprintf("SAVEPOINT x%d", i); 565 + int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0); 566 + sqlite3_free(zSql); 567 + if( rc!=SQLITE_OK ) return rc; 568 + } 569 + 570 + pDb->nOpenTrans = iLevel; 571 + return 0; 572 +} 573 + 574 +static int sql_commit(TestDb *pTestDb, int iLevel){ 575 + SqlDb *pDb = (SqlDb *)pTestDb; 576 + assert( iLevel>=0 ); 577 + 578 + /* Close the read transaction if requested. */ 579 + if( pDb->nOpenTrans>=1 && iLevel==0 ){ 580 + int rc = sqlite3_exec(pDb->db, "COMMIT", 0, 0, 0); 581 + if( rc!=0 ) return rc; 582 + pDb->nOpenTrans = 0; 583 + } 584 + 585 + /* Close write transactions as required */ 586 + if( pDb->nOpenTrans>iLevel ){ 587 + char *zSql = sqlite3_mprintf("RELEASE x%d", iLevel); 588 + int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0); 589 + sqlite3_free(zSql); 590 + if( rc!=0 ) return rc; 591 + } 592 + 593 + pDb->nOpenTrans = iLevel; 594 + return 0; 595 +} 596 + 597 +static int sql_rollback(TestDb *pTestDb, int iLevel){ 598 + SqlDb *pDb = (SqlDb *)pTestDb; 599 + assert( iLevel>=0 ); 600 + 601 + if( pDb->nOpenTrans>=1 && iLevel==0 ){ 602 + /* Close the read transaction if requested. */ 603 + int rc = sqlite3_exec(pDb->db, "ROLLBACK", 0, 0, 0); 604 + if( rc!=0 ) return rc; 605 + }else if( pDb->nOpenTrans>1 && iLevel==1 ){ 606 + /* Or, rollback and close the top-level write transaction */ 607 + int rc = sqlite3_exec(pDb->db, "ROLLBACK TO x1; RELEASE x1;", 0, 0, 0); 608 + if( rc!=0 ) return rc; 609 + }else{ 610 + /* Or, just roll back some nested transactions */ 611 + char *zSql = sqlite3_mprintf("ROLLBACK TO x%d", iLevel-1); 612 + int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0); 613 + sqlite3_free(zSql); 614 + if( rc!=0 ) return rc; 615 + } 616 + 617 + pDb->nOpenTrans = iLevel; 618 + return 0; 619 +} 620 + 621 +static int sql_open( 622 + const char *zSpec, 623 + const char *zFilename, 624 + int bClear, 625 + TestDb **ppDb 626 +){ 627 + static const DatabaseMethods SqlMethods = { 628 + sql_close, 629 + sql_write, 630 + sql_delete, 631 + sql_delete_range, 632 + sql_fetch, 633 + sql_scan, 634 + sql_begin, 635 + sql_commit, 636 + sql_rollback 637 + }; 638 + const char *zCreate = "CREATE TABLE IF NOT EXISTS t1(k PRIMARY KEY, v)"; 639 + const char *zInsert = "REPLACE INTO t1 VALUES(?, ?)"; 640 + const char *zDelete = "DELETE FROM t1 WHERE k = ?"; 641 + const char *zRange = "DELETE FROM t1 WHERE k>? AND k<?"; 642 + const char *zFetch = "SELECT v FROM t1 WHERE k = ?"; 643 + 644 + const char *zScan0 = "SELECT * FROM t1 WHERE k BETWEEN ?1 AND ?2 ORDER BY k"; 645 + const char *zScan1 = "SELECT * FROM t1 WHERE k <= ?2 ORDER BY k"; 646 + const char *zScan2 = "SELECT * FROM t1 WHERE k >= ?1 ORDER BY k"; 647 + const char *zScan3 = "SELECT * FROM t1 ORDER BY k"; 648 + 649 + const char *zScan4 = 650 + "SELECT * FROM t1 WHERE k BETWEEN ?1 AND ?2 ORDER BY k DESC"; 651 + const char *zScan5 = "SELECT * FROM t1 WHERE k <= ?2 ORDER BY k DESC"; 652 + const char *zScan6 = "SELECT * FROM t1 WHERE k >= ?1 ORDER BY k DESC"; 653 + const char *zScan7 = "SELECT * FROM t1 ORDER BY k DESC"; 654 + 655 + int rc; 656 + SqlDb *pDb; 657 + char *zPragma; 658 + 659 + if( bClear && zFilename && zFilename[0] ){ 660 + unlink(zFilename); 661 + } 662 + 663 + pDb = (SqlDb *)malloc(sizeof(SqlDb)); 664 + memset(pDb, 0, sizeof(SqlDb)); 665 + pDb->base.pMethods = &SqlMethods; 666 + 667 + if( 0!=(rc = sqlite3_open(zFilename, &pDb->db)) 668 + || 0!=(rc = sqlite3_exec(pDb->db, zCreate, 0, 0, 0)) 669 + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zInsert, -1, &pDb->pInsert, 0)) 670 + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zDelete, -1, &pDb->pDelete, 0)) 671 + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zRange, -1, &pDb->pDeleteRange, 0)) 672 + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zFetch, -1, &pDb->pFetch, 0)) 673 + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan0, -1, &pDb->apScan[0], 0)) 674 + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan1, -1, &pDb->apScan[1], 0)) 675 + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan2, -1, &pDb->apScan[2], 0)) 676 + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan3, -1, &pDb->apScan[3], 0)) 677 + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan4, -1, &pDb->apScan[4], 0)) 678 + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan5, -1, &pDb->apScan[5], 0)) 679 + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan6, -1, &pDb->apScan[6], 0)) 680 + || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan7, -1, &pDb->apScan[7], 0)) 681 + ){ 682 + *ppDb = 0; 683 + sql_close((TestDb *)pDb); 684 + return rc; 685 + } 686 + 687 + zPragma = sqlite3_mprintf("PRAGMA page_size=%d", TESTDB_DEFAULT_PAGE_SIZE); 688 + sqlite3_exec(pDb->db, zPragma, 0, 0, 0); 689 + sqlite3_free(zPragma); 690 + zPragma = sqlite3_mprintf("PRAGMA cache_size=%d", TESTDB_DEFAULT_CACHE_SIZE); 691 + sqlite3_exec(pDb->db, zPragma, 0, 0, 0); 692 + sqlite3_free(zPragma); 693 + 694 + /* sqlite3_exec(pDb->db, "PRAGMA locking_mode=EXCLUSIVE", 0, 0, 0); */ 695 + sqlite3_exec(pDb->db, "PRAGMA synchronous=OFF", 0, 0, 0); 696 + sqlite3_exec(pDb->db, "PRAGMA journal_mode=WAL", 0, 0, 0); 697 + sqlite3_exec(pDb->db, "PRAGMA wal_autocheckpoint=4096", 0, 0, 0); 698 + if( zSpec ){ 699 + rc = sqlite3_exec(pDb->db, zSpec, 0, 0, 0); 700 + if( rc!=SQLITE_OK ){ 701 + sql_close((TestDb *)pDb); 702 + return rc; 703 + } 704 + } 705 + 706 + *ppDb = (TestDb *)pDb; 707 + return 0; 708 +} 709 +/* 710 +** End wrapper for SQLite. 711 +*************************************************************************/ 712 + 713 +/************************************************************************* 714 +** Begin exported functions. 715 +*/ 716 +static struct Lib { 717 + const char *zName; 718 + const char *zDefaultDb; 719 + int (*xOpen)(const char *, const char *zFilename, int bClear, TestDb **ppDb); 720 +} aLib[] = { 721 + { "sqlite3", "testdb.sqlite", sql_open }, 722 + { "lsm_small", "testdb.lsm_small", test_lsm_small_open }, 723 + { "lsm_lomem", "testdb.lsm_lomem", test_lsm_lomem_open }, 724 +#ifdef HAVE_ZLIB 725 + { "lsm_zip", "testdb.lsm_zip", test_lsm_zip_open }, 726 +#endif 727 + { "lsm", "testdb.lsm", test_lsm_open }, 728 +#ifdef LSM_MUTEX_PTHREADS 729 + { "lsm_mt2", "testdb.lsm_mt2", test_lsm_mt2 }, 730 + { "lsm_mt3", "testdb.lsm_mt3", test_lsm_mt3 }, 731 +#endif 732 +#ifdef HAVE_LEVELDB 733 + { "leveldb", "testdb.leveldb", test_leveldb_open }, 734 +#endif 735 +#ifdef HAVE_KYOTOCABINET 736 + { "kyotocabinet", "testdb.kc", kc_open }, 737 +#endif 738 +#ifdef HAVE_MDB 739 + { "mdb", "./testdb.mdb", mdb_open } 740 +#endif 741 +}; 742 + 743 +const char *tdb_system_name(int i){ 744 + if( i<0 || i>=ArraySize(aLib) ) return 0; 745 + return aLib[i].zName; 746 +} 747 + 748 +const char *tdb_default_db(const char *zSys){ 749 + int i; 750 + for(i=0; i<ArraySize(aLib); i++){ 751 + if( strcmp(aLib[i].zName, zSys)==0 ) return aLib[i].zDefaultDb; 752 + } 753 + return 0; 754 +} 755 + 756 +int tdb_open(const char *zLib, const char *zDb, int bClear, TestDb **ppDb){ 757 + int i; 758 + int rc = 1; 759 + const char *zSpec = 0; 760 + 761 + int nLib = 0; 762 + while( zLib[nLib] && zLib[nLib]!=' ' ){ 763 + nLib++; 764 + } 765 + zSpec = &zLib[nLib]; 766 + while( *zSpec==' ' ) zSpec++; 767 + if( *zSpec=='\0' ) zSpec = 0; 768 + 769 + for(i=0; i<ArraySize(aLib); i++){ 770 + if( (int)strlen(aLib[i].zName)==nLib 771 + && 0==memcmp(zLib, aLib[i].zName, nLib) ){ 772 + rc = aLib[i].xOpen(zSpec, (zDb ? zDb : aLib[i].zDefaultDb), bClear, ppDb); 773 + if( rc==0 ){ 774 + (*ppDb)->zLibrary = aLib[i].zName; 775 + } 776 + break; 777 + } 778 + } 779 + 780 + if( rc ){ 781 + /* Failed to find the requested database library. Return an error. */ 782 + *ppDb = 0; 783 + } 784 + return rc; 785 +} 786 + 787 +int tdb_close(TestDb *pDb){ 788 + if( pDb ){ 789 + return pDb->pMethods->xClose(pDb); 790 + } 791 + return 0; 792 +} 793 + 794 +int tdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ 795 + return pDb->pMethods->xWrite(pDb, pKey, nKey, pVal, nVal); 796 +} 797 + 798 +int tdb_delete(TestDb *pDb, void *pKey, int nKey){ 799 + return pDb->pMethods->xDelete(pDb, pKey, nKey); 800 +} 801 + 802 +int tdb_delete_range( 803 + TestDb *pDb, void *pKey1, int nKey1, void *pKey2, int nKey2 804 +){ 805 + return pDb->pMethods->xDeleteRange(pDb, pKey1, nKey1, pKey2, nKey2); 806 +} 807 + 808 +int tdb_fetch(TestDb *pDb, void *pKey, int nKey, void **ppVal, int *pnVal){ 809 + return pDb->pMethods->xFetch(pDb, pKey, nKey, ppVal, pnVal); 810 +} 811 + 812 +int tdb_scan( 813 + TestDb *pDb, /* Database handle */ 814 + void *pCtx, /* Context pointer to pass to xCallback */ 815 + int bReverse, /* True to scan in reverse order */ 816 + void *pKey1, int nKey1, /* Start of search */ 817 + void *pKey2, int nKey2, /* End of search */ 818 + void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) 819 +){ 820 + return pDb->pMethods->xScan( 821 + pDb, pCtx, bReverse, pKey1, nKey1, pKey2, nKey2, xCallback 822 + ); 823 +} 824 + 825 +int tdb_begin(TestDb *pDb, int iLevel){ 826 + return pDb->pMethods->xBegin(pDb, iLevel); 827 +} 828 +int tdb_commit(TestDb *pDb, int iLevel){ 829 + return pDb->pMethods->xCommit(pDb, iLevel); 830 +} 831 +int tdb_rollback(TestDb *pDb, int iLevel){ 832 + return pDb->pMethods->xRollback(pDb, iLevel); 833 +} 834 + 835 +int tdb_transaction_support(TestDb *pDb){ 836 + return (pDb->pMethods->xBegin != error_transaction_function); 837 +} 838 + 839 +const char *tdb_library_name(TestDb *pDb){ 840 + return pDb->zLibrary; 841 +} 842 + 843 +/* 844 +** End exported functions. 845 +*************************************************************************/
Added ext/lsm1/lsm-test/lsmtest_tdb.h.
1 + 2 +/* 3 +** This file is the interface to a very simple database library used for 4 +** testing. The interface is similar to that of the LSM. The main virtue 5 +** of this library is that the same API may be used to access a key-value 6 +** store implemented by LSM, SQLite or another database system. Which 7 +** makes it easy to use for correctness and performance tests. 8 +*/ 9 + 10 +#ifndef __WRAPPER_H_ 11 +#define __WRAPPER_H_ 12 + 13 +#ifdef __cplusplus 14 +extern "C" { 15 +#endif 16 + 17 +#include "lsm.h" 18 + 19 +typedef struct TestDb TestDb; 20 + 21 +/* 22 +** Open a new database connection. The first argument is the name of the 23 +** database library to use. e.g. something like: 24 +** 25 +** "sqlite3" 26 +** "lsm" 27 +** 28 +** See function tdb_system_name() for a list of available database systems. 29 +** 30 +** The second argument is the name of the database to open (e.g. a filename). 31 +** 32 +** If the third parameter is non-zero, then any existing database by the 33 +** name of zDb is removed before opening a new one. If it is zero, then an 34 +** existing database may be opened. 35 +*/ 36 +int tdb_open(const char *zLibrary, const char *zDb, int bClear, TestDb **ppDb); 37 + 38 +/* 39 +** Close a database handle. 40 +*/ 41 +int tdb_close(TestDb *pDb); 42 + 43 +/* 44 +** Write a new key/value into the database. 45 +*/ 46 +int tdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal); 47 + 48 +/* 49 +** Delete a key from the database. 50 +*/ 51 +int tdb_delete(TestDb *pDb, void *pKey, int nKey); 52 + 53 +/* 54 +** Delete a range of keys from the database. 55 +*/ 56 +int tdb_delete_range(TestDb *, void *pKey1, int nKey1, void *pKey2, int nKey2); 57 + 58 +/* 59 +** Query the database for key (pKey/nKey). If no entry is found, set *ppVal 60 +** to 0 and *pnVal to -1 before returning. Otherwise, set *ppVal and *pnVal 61 +** to a pointer to and size of the value associated with (pKey/nKey). 62 +*/ 63 +int tdb_fetch(TestDb *pDb, void *pKey, int nKey, void **ppVal, int *pnVal); 64 + 65 +/* 66 +** Open and close nested transactions. Currently, these functions only 67 +** work for SQLite3 and LSM systems. Use the tdb_transaction_support() 68 +** function to determine if a given TestDb handle supports these methods. 69 +** 70 +** These functions and the iLevel parameter follow the same conventions as 71 +** the SQLite 4 transaction interface. Note that this is slightly different 72 +** from the way LSM does things. As follows: 73 +** 74 +** tdb_begin(): 75 +** A successful call to tdb_begin() with (iLevel>1) guarantees that 76 +** there are at least (iLevel-1) write transactions open. If iLevel==1, 77 +** then it guarantees that at least a read-transaction is open. Calling 78 +** tdb_begin() with iLevel==0 is a no-op. 79 +** 80 +** tdb_commit(): 81 +** A successful call to tdb_commit() with (iLevel>1) guarantees that 82 +** there are at most (iLevel-1) write transactions open. If iLevel==1, 83 +** then it guarantees that there are no write transactions open (although 84 +** a read-transaction may remain open). Calling tdb_commit() with 85 +** iLevel==0 ensures that all transactions, read or write, have been 86 +** closed and committed. 87 +** 88 +** tdb_rollback(): 89 +** This call is similar to tdb_commit(), except that instead of committing 90 +** transactions, it reverts them. For example, calling tdb_rollback() with 91 +** iLevel==2 ensures that there is at most one write transaction open, and 92 +** restores the database to the state that it was in when that transaction 93 +** was opened. 94 +** 95 +** In other words, tdb_commit() just closes transactions - tdb_rollback() 96 +** closes transactions and then restores the database to the state it 97 +** was in before those transactions were even opened. 98 +*/ 99 +int tdb_begin(TestDb *pDb, int iLevel); 100 +int tdb_commit(TestDb *pDb, int iLevel); 101 +int tdb_rollback(TestDb *pDb, int iLevel); 102 + 103 +/* 104 +** Return true if transactions are supported, or false otherwise. 105 +*/ 106 +int tdb_transaction_support(TestDb *pDb); 107 + 108 +/* 109 +** Return the name of the database library (as passed to tdb_open()) used 110 +** by the handled passed as the first argument. 111 +*/ 112 +const char *tdb_library_name(TestDb *pDb); 113 + 114 +/* 115 +** Scan a range of database keys. Invoke the callback function for each 116 +** key visited. 117 +*/ 118 +int tdb_scan( 119 + TestDb *pDb, /* Database handle */ 120 + void *pCtx, /* Context pointer to pass to xCallback */ 121 + int bReverse, /* True to scan in reverse order */ 122 + void *pKey1, int nKey1, /* Start of search */ 123 + void *pKey2, int nKey2, /* End of search */ 124 + void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) 125 +); 126 + 127 +const char *tdb_system_name(int i); 128 +const char *tdb_default_db(const char *zSys); 129 + 130 +int tdb_lsm_open(const char *zCfg, const char *zDb, int bClear, TestDb **ppDb); 131 + 132 +/* 133 +** If the TestDb handle passed as an argument is a wrapper around an LSM 134 +** database, return the LSM handle. Otherwise, if the argument is some other 135 +** database system, return NULL. 136 +*/ 137 +lsm_db *tdb_lsm(TestDb *pDb); 138 + 139 +/* 140 +** Return true if the db passed as an argument is a multi-threaded LSM 141 +** connection. 142 +*/ 143 +int tdb_lsm_multithread(TestDb *pDb); 144 + 145 +/* 146 +** Return a pointer to the lsm_env object used by all lsm database 147 +** connections initialized as a copy of the object returned by 148 +** lsm_default_env(). It may be modified (e.g. to override functions) 149 +** if the caller can guarantee that it is not already in use. 150 +*/ 151 +lsm_env *tdb_lsm_env(void); 152 + 153 +/* 154 +** The following functions only work with LSM database handles. It is 155 +** illegal to call them with any other type of database handle specified 156 +** as an argument. 157 +*/ 158 +void tdb_lsm_enable_log(TestDb *pDb, int bEnable); 159 +void tdb_lsm_application_crash(TestDb *pDb); 160 +void tdb_lsm_prepare_system_crash(TestDb *pDb); 161 +void tdb_lsm_system_crash(TestDb *pDb); 162 +void tdb_lsm_prepare_sync_crash(TestDb *pDb, int iSync); 163 + 164 + 165 +void tdb_lsm_safety(TestDb *pDb, int eMode); 166 +void tdb_lsm_config_work_hook(TestDb *pDb, void (*)(lsm_db *, void *), void *); 167 +void tdb_lsm_write_hook(TestDb *, void(*)(void*,int,lsm_i64,int,int), void*); 168 +int tdb_lsm_config_str(TestDb *pDb, const char *zStr); 169 + 170 +#ifdef __cplusplus 171 +} /* End of the 'extern "C"' block */ 172 +#endif 173 + 174 +#endif
Added ext/lsm1/lsm-test/lsmtest_tdb2.cc.
1 + 2 + 3 +#include "lsmtest.h" 4 +#include <stdlib.h> 5 + 6 +#ifdef HAVE_KYOTOCABINET 7 +#include "kcpolydb.h" 8 +extern "C" { 9 + struct KcDb { 10 + TestDb base; 11 + kyotocabinet::TreeDB* db; 12 + char *pVal; 13 + }; 14 +} 15 + 16 +int test_kc_open(const char *zFilename, int bClear, TestDb **ppDb){ 17 + KcDb *pKcDb; 18 + int ok; 19 + int rc = 0; 20 + 21 + if( bClear ){ 22 + char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); 23 + system(zCmd); 24 + sqlite3_free(zCmd); 25 + } 26 + 27 + pKcDb = (KcDb *)malloc(sizeof(KcDb)); 28 + memset(pKcDb, 0, sizeof(KcDb)); 29 + 30 + 31 + pKcDb->db = new kyotocabinet::TreeDB(); 32 + pKcDb->db->tune_page(TESTDB_DEFAULT_PAGE_SIZE); 33 + pKcDb->db->tune_page_cache( 34 + TESTDB_DEFAULT_PAGE_SIZE * TESTDB_DEFAULT_CACHE_SIZE 35 + ); 36 + ok = pKcDb->db->open(zFilename, 37 + kyotocabinet::PolyDB::OWRITER | kyotocabinet::PolyDB::OCREATE 38 + ); 39 + if( ok==0 ){ 40 + free(pKcDb); 41 + pKcDb = 0; 42 + rc = 1; 43 + } 44 + 45 + *ppDb = (TestDb *)pKcDb; 46 + return rc; 47 +} 48 + 49 +int test_kc_close(TestDb *pDb){ 50 + KcDb *pKcDb = (KcDb *)pDb; 51 + if( pKcDb->pVal ){ 52 + delete [] pKcDb->pVal; 53 + } 54 + pKcDb->db->close(); 55 + delete pKcDb->db; 56 + free(pKcDb); 57 + return 0; 58 +} 59 + 60 +int test_kc_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ 61 + KcDb *pKcDb = (KcDb *)pDb; 62 + int ok; 63 + 64 + ok = pKcDb->db->set((const char *)pKey, nKey, (const char *)pVal, nVal); 65 + return (ok ? 0 : 1); 66 +} 67 + 68 +int test_kc_delete(TestDb *pDb, void *pKey, int nKey){ 69 + KcDb *pKcDb = (KcDb *)pDb; 70 + int ok; 71 + 72 + ok = pKcDb->db->remove((const char *)pKey, nKey); 73 + return (ok ? 0 : 1); 74 +} 75 + 76 +int test_kc_delete_range( 77 + TestDb *pDb, 78 + void *pKey1, int nKey1, 79 + void *pKey2, int nKey2 80 +){ 81 + int res; 82 + KcDb *pKcDb = (KcDb *)pDb; 83 + kyotocabinet::DB::Cursor* pCur = pKcDb->db->cursor(); 84 + 85 + if( pKey1 ){ 86 + res = pCur->jump((const char *)pKey1, nKey1); 87 + }else{ 88 + res = pCur->jump(); 89 + } 90 + 91 + while( 1 ){ 92 + const char *pKey; size_t nKey; 93 + const char *pVal; size_t nVal; 94 + 95 + pKey = pCur->get(&nKey, &pVal, &nVal); 96 + if( pKey==0 ) break; 97 + 98 +#ifndef NDEBUG 99 + if( pKey1 ){ 100 + res = memcmp(pKey, pKey1, MIN((size_t)nKey1, nKey)); 101 + assert( res>0 || (res==0 && nKey>nKey1) ); 102 + } 103 +#endif 104 + 105 + if( pKey2 ){ 106 + res = memcmp(pKey, pKey2, MIN((size_t)nKey2, nKey)); 107 + if( res>0 || (res==0 && (size_t)nKey2<nKey) ){ 108 + delete [] pKey; 109 + break; 110 + } 111 + } 112 + pCur->remove(); 113 + delete [] pKey; 114 + } 115 + 116 + delete pCur; 117 + return 0; 118 +} 119 + 120 +int test_kc_fetch( 121 + TestDb *pDb, 122 + void *pKey, 123 + int nKey, 124 + void **ppVal, 125 + int *pnVal 126 +){ 127 + KcDb *pKcDb = (KcDb *)pDb; 128 + size_t nVal; 129 + 130 + if( pKcDb->pVal ){ 131 + delete [] pKcDb->pVal; 132 + pKcDb->pVal = 0; 133 + } 134 + 135 + pKcDb->pVal = pKcDb->db->get((const char *)pKey, nKey, &nVal); 136 + if( pKcDb->pVal ){ 137 + *ppVal = pKcDb->pVal; 138 + *pnVal = nVal; 139 + }else{ 140 + *ppVal = 0; 141 + *pnVal = -1; 142 + } 143 + 144 + return 0; 145 +} 146 + 147 +int test_kc_scan( 148 + TestDb *pDb, /* Database handle */ 149 + void *pCtx, /* Context pointer to pass to xCallback */ 150 + int bReverse, /* True for a reverse order scan */ 151 + void *pKey1, int nKey1, /* Start of search */ 152 + void *pKey2, int nKey2, /* End of search */ 153 + void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) 154 +){ 155 + KcDb *pKcDb = (KcDb *)pDb; 156 + kyotocabinet::DB::Cursor* pCur = pKcDb->db->cursor(); 157 + int res; 158 + 159 + if( bReverse==0 ){ 160 + if( pKey1 ){ 161 + res = pCur->jump((const char *)pKey1, nKey1); 162 + }else{ 163 + res = pCur->jump(); 164 + } 165 + }else{ 166 + if( pKey2 ){ 167 + res = pCur->jump_back((const char *)pKey2, nKey2); 168 + }else{ 169 + res = pCur->jump_back(); 170 + } 171 + } 172 + 173 + while( res ){ 174 + const char *pKey; size_t nKey; 175 + const char *pVal; size_t nVal; 176 + pKey = pCur->get(&nKey, &pVal, &nVal); 177 + 178 + if( bReverse==0 && pKey2 ){ 179 + res = memcmp(pKey, pKey2, MIN((size_t)nKey2, nKey)); 180 + if( res>0 || (res==0 && (size_t)nKey2<nKey) ){ 181 + delete [] pKey; 182 + break; 183 + } 184 + }else if( bReverse!=0 && pKey1 ){ 185 + res = memcmp(pKey, pKey1, MIN((size_t)nKey1, nKey)); 186 + if( res<0 || (res==0 && (size_t)nKey1>nKey) ){ 187 + delete [] pKey; 188 + break; 189 + } 190 + } 191 + 192 + xCallback(pCtx, (void *)pKey, (int)nKey, (void *)pVal, (int)nVal); 193 + delete [] pKey; 194 + 195 + if( bReverse ){ 196 + res = pCur->step_back(); 197 + }else{ 198 + res = pCur->step(); 199 + } 200 + } 201 + 202 + delete pCur; 203 + return 0; 204 +} 205 +#endif /* HAVE_KYOTOCABINET */ 206 + 207 +#ifdef HAVE_MDB 208 +#include "lmdb.h" 209 + 210 +extern "C" { 211 + struct MdbDb { 212 + TestDb base; 213 + MDB_env *env; 214 + MDB_dbi dbi; 215 + }; 216 +} 217 + 218 +int test_mdb_open( 219 + const char *zSpec, 220 + const char *zFilename, 221 + int bClear, 222 + TestDb **ppDb 223 +){ 224 + MDB_txn *txn; 225 + MdbDb *pMdb; 226 + int rc; 227 + 228 + if( bClear ){ 229 + char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); 230 + system(zCmd); 231 + sqlite3_free(zCmd); 232 + } 233 + 234 + pMdb = (MdbDb *)malloc(sizeof(MdbDb)); 235 + memset(pMdb, 0, sizeof(MdbDb)); 236 + 237 + rc = mdb_env_create(&pMdb->env); 238 + if( rc==0 ) rc = mdb_env_set_mapsize(pMdb->env, 1*1024*1024*1024); 239 + if( rc==0 ) rc = mdb_env_open(pMdb->env, zFilename, MDB_NOSYNC|MDB_NOSUBDIR, 0600); 240 + if( rc==0 ) rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); 241 + if( rc==0 ){ 242 + rc = mdb_open(txn, NULL, 0, &pMdb->dbi); 243 + mdb_txn_commit(txn); 244 + } 245 + 246 + *ppDb = (TestDb *)pMdb; 247 + return rc; 248 +} 249 + 250 +int test_mdb_close(TestDb *pDb){ 251 + MdbDb *pMdb = (MdbDb *)pDb; 252 + 253 + mdb_close(pMdb->env, pMdb->dbi); 254 + mdb_env_close(pMdb->env); 255 + free(pMdb); 256 + return 0; 257 +} 258 + 259 +int test_mdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ 260 + int rc; 261 + MdbDb *pMdb = (MdbDb *)pDb; 262 + MDB_val val; 263 + MDB_val key; 264 + MDB_txn *txn; 265 + 266 + val.mv_size = nVal; 267 + val.mv_data = pVal; 268 + key.mv_size = nKey; 269 + key.mv_data = pKey; 270 + 271 + rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); 272 + if( rc==0 ){ 273 + rc = mdb_put(txn, pMdb->dbi, &key, &val, 0); 274 + if( rc==0 ){ 275 + rc = mdb_txn_commit(txn); 276 + }else{ 277 + mdb_txn_abort(txn); 278 + } 279 + } 280 + 281 + return rc; 282 +} 283 + 284 +int test_mdb_delete(TestDb *pDb, void *pKey, int nKey){ 285 + int rc; 286 + MdbDb *pMdb = (MdbDb *)pDb; 287 + MDB_val key; 288 + MDB_txn *txn; 289 + 290 + key.mv_size = nKey; 291 + key.mv_data = pKey; 292 + rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); 293 + if( rc==0 ){ 294 + rc = mdb_del(txn, pMdb->dbi, &key, 0); 295 + if( rc==0 ){ 296 + rc = mdb_txn_commit(txn); 297 + }else{ 298 + mdb_txn_abort(txn); 299 + } 300 + } 301 + 302 + return rc; 303 +} 304 + 305 +int test_mdb_fetch( 306 + TestDb *pDb, 307 + void *pKey, 308 + int nKey, 309 + void **ppVal, 310 + int *pnVal 311 +){ 312 + int rc; 313 + MdbDb *pMdb = (MdbDb *)pDb; 314 + MDB_val key; 315 + MDB_txn *txn; 316 + 317 + key.mv_size = nKey; 318 + key.mv_data = pKey; 319 + 320 + rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn); 321 + if( rc==0 ){ 322 + MDB_val val = {0, 0}; 323 + rc = mdb_get(txn, pMdb->dbi, &key, &val); 324 + if( rc==MDB_NOTFOUND ){ 325 + rc = 0; 326 + *ppVal = 0; 327 + *pnVal = -1; 328 + }else{ 329 + *ppVal = val.mv_data; 330 + *pnVal = val.mv_size; 331 + } 332 + mdb_txn_commit(txn); 333 + } 334 + 335 + return rc; 336 +} 337 + 338 +int test_mdb_scan( 339 + TestDb *pDb, /* Database handle */ 340 + void *pCtx, /* Context pointer to pass to xCallback */ 341 + int bReverse, /* True for a reverse order scan */ 342 + void *pKey1, int nKey1, /* Start of search */ 343 + void *pKey2, int nKey2, /* End of search */ 344 + void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) 345 +){ 346 + MdbDb *pMdb = (MdbDb *)pDb; 347 + int rc; 348 + MDB_cursor_op op = bReverse ? MDB_PREV : MDB_NEXT; 349 + MDB_txn *txn; 350 + 351 + rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn); 352 + if( rc==0 ){ 353 + MDB_cursor *csr; 354 + MDB_val key = {0, 0}; 355 + MDB_val val = {0, 0}; 356 + 357 + rc = mdb_cursor_open(txn, pMdb->dbi, &csr); 358 + if( rc==0 ){ 359 + while( mdb_cursor_get(csr, &key, &val, op)==0 ){ 360 + xCallback(pCtx, key.mv_data, key.mv_size, val.mv_data, val.mv_size); 361 + } 362 + mdb_cursor_close(csr); 363 + } 364 + } 365 + 366 + return rc; 367 +} 368 + 369 +#endif /* HAVE_MDB */ 370 +
Added ext/lsm1/lsm-test/lsmtest_tdb3.c.
1 + 2 +#include "lsmtest_tdb.h" 3 +#include "lsm.h" 4 +#include "lsmtest.h" 5 + 6 +#include <stdlib.h> 7 +#include <string.h> 8 +#include <assert.h> 9 +#ifndef _WIN32 10 +# include <unistd.h> 11 +#endif 12 +#include <stdio.h> 13 + 14 +#ifndef _WIN32 15 +# include <sys/time.h> 16 +#endif 17 + 18 +typedef struct LsmDb LsmDb; 19 +typedef struct LsmWorker LsmWorker; 20 +typedef struct LsmFile LsmFile; 21 + 22 +#define LSMTEST_DFLT_MT_MAX_CKPT (8*1024) 23 +#define LSMTEST_DFLT_MT_MIN_CKPT (2*1024) 24 + 25 +#ifdef LSM_MUTEX_PTHREADS 26 +#include <pthread.h> 27 + 28 +#define LSMTEST_THREAD_CKPT 1 29 +#define LSMTEST_THREAD_WORKER 2 30 +#define LSMTEST_THREAD_WORKER_AC 3 31 + 32 +/* 33 +** There are several different types of worker threads that run in different 34 +** test configurations, depending on the value of LsmWorker.eType. 35 +** 36 +** 1. Checkpointer. 37 +** 2. Worker with auto-checkpoint. 38 +** 3. Worker without auto-checkpoint. 39 +*/ 40 +struct LsmWorker { 41 + LsmDb *pDb; /* Main database structure */ 42 + lsm_db *pWorker; /* Worker database handle */ 43 + pthread_t worker_thread; /* Worker thread */ 44 + pthread_cond_t worker_cond; /* Condition var the worker waits on */ 45 + pthread_mutex_t worker_mutex; /* Mutex used with worker_cond */ 46 + int bDoWork; /* Set to true by client when there is work */ 47 + int worker_rc; /* Store error code here */ 48 + int eType; /* LSMTEST_THREAD_XXX constant */ 49 + int bBlock; 50 +}; 51 +#else 52 +struct LsmWorker { int worker_rc; int bBlock; }; 53 +#endif 54 + 55 +static void mt_shutdown(LsmDb *); 56 + 57 +lsm_env *tdb_lsm_env(void){ 58 + static int bInit = 0; 59 + static lsm_env env; 60 + if( bInit==0 ){ 61 + memcpy(&env, lsm_default_env(), sizeof(env)); 62 + bInit = 1; 63 + } 64 + return &env; 65 +} 66 + 67 +typedef struct FileSector FileSector; 68 +typedef struct FileData FileData; 69 + 70 +struct FileSector { 71 + u8 *aOld; /* Old data for this sector */ 72 +}; 73 + 74 +struct FileData { 75 + int nSector; /* Allocated size of apSector[] array */ 76 + FileSector *aSector; /* Array of file sectors */ 77 +}; 78 + 79 +/* 80 +** bPrepareCrash: 81 +** If non-zero, the file wrappers maintain enough in-memory data to 82 +** simulate the effect of a power-failure on the file-system (i.e. that 83 +** unsynced sectors may be written, not written, or overwritten with 84 +** arbitrary data when the crash occurs). 85 +** 86 +** bCrashed: 87 +** Set to true after a crash is simulated. Once this variable is true, all 88 +** VFS methods other than xClose() return LSM_IOERR as soon as they are 89 +** called (without affecting the contents of the file-system). 90 +** 91 +** env: 92 +** The environment object used by all lsm_db* handles opened by this 93 +** object (i.e. LsmDb.db plus any worker connections). Variable env.pVfsCtx 94 +** always points to the containing LsmDb structure. 95 +*/ 96 +struct LsmDb { 97 + TestDb base; /* Base class - methods table */ 98 + lsm_env env; /* Environment used by connection db */ 99 + char *zName; /* Database file name */ 100 + lsm_db *db; /* LSM database handle */ 101 + 102 + lsm_cursor *pCsr; /* Cursor held open during read transaction */ 103 + void *pBuf; /* Buffer for tdb_fetch() output */ 104 + int nBuf; /* Allocated (not used) size of pBuf */ 105 + 106 + /* Crash testing related state */ 107 + int bCrashed; /* True once a crash has occurred */ 108 + int nAutoCrash; /* Number of syncs until a crash */ 109 + int bPrepareCrash; /* True to store writes in memory */ 110 + 111 + /* Unsynced data (while crash testing) */ 112 + int szSector; /* Assumed size of disk sectors (512B) */ 113 + FileData aFile[2]; /* Database and log file data */ 114 + 115 + /* Other test instrumentation */ 116 + int bNoRecovery; /* If true, assume DMS2 is locked */ 117 + 118 + /* Work hook redirection */ 119 + void (*xWork)(lsm_db *, void *); 120 + void *pWorkCtx; 121 + 122 + /* IO logging hook */ 123 + void (*xWriteHook)(void *, int, lsm_i64, int, int); 124 + void *pWriteCtx; 125 + 126 + /* Worker threads (for lsm_mt) */ 127 + int nMtMinCkpt; 128 + int nMtMaxCkpt; 129 + int eMode; 130 + int nWorker; 131 + LsmWorker *aWorker; 132 +}; 133 + 134 +#define LSMTEST_MODE_SINGLETHREAD 1 135 +#define LSMTEST_MODE_BACKGROUND_CKPT 2 136 +#define LSMTEST_MODE_BACKGROUND_WORK 3 137 +#define LSMTEST_MODE_BACKGROUND_BOTH 4 138 + 139 +/************************************************************************* 140 +************************************************************************** 141 +** Begin test VFS code. 142 +*/ 143 + 144 +struct LsmFile { 145 + lsm_file *pReal; /* Real underlying file */ 146 + int bLog; /* True for log file. False for db file */ 147 + LsmDb *pDb; /* Database handle that uses this file */ 148 +}; 149 + 150 +static int testEnvFullpath( 151 + lsm_env *pEnv, /* Environment for current LsmDb */ 152 + const char *zFile, /* Relative path name */ 153 + char *zOut, /* Output buffer */ 154 + int *pnOut /* IN/OUT: Size of output buffer */ 155 +){ 156 + lsm_env *pRealEnv = tdb_lsm_env(); 157 + return pRealEnv->xFullpath(pRealEnv, zFile, zOut, pnOut); 158 +} 159 + 160 +static int testEnvOpen( 161 + lsm_env *pEnv, /* Environment for current LsmDb */ 162 + const char *zFile, /* Name of file to open */ 163 + int flags, 164 + lsm_file **ppFile /* OUT: New file handle object */ 165 +){ 166 + lsm_env *pRealEnv = tdb_lsm_env(); 167 + LsmDb *pDb = (LsmDb *)pEnv->pVfsCtx; 168 + int rc; /* Return Code */ 169 + LsmFile *pRet; /* The new file handle */ 170 + int nFile; /* Length of string zFile in bytes */ 171 + 172 + nFile = strlen(zFile); 173 + pRet = (LsmFile *)testMalloc(sizeof(LsmFile)); 174 + pRet->pDb = pDb; 175 + pRet->bLog = (nFile > 4 && 0==memcmp("-log", &zFile[nFile-4], 4)); 176 + 177 + rc = pRealEnv->xOpen(pRealEnv, zFile, flags, &pRet->pReal); 178 + if( rc!=LSM_OK ){ 179 + testFree(pRet); 180 + pRet = 0; 181 + } 182 + 183 + *ppFile = (lsm_file *)pRet; 184 + return rc; 185 +} 186 + 187 +static int testEnvRead(lsm_file *pFile, lsm_i64 iOff, void *pData, int nData){ 188 + lsm_env *pRealEnv = tdb_lsm_env(); 189 + LsmFile *p = (LsmFile *)pFile; 190 + if( p->pDb->bCrashed ) return LSM_IOERR; 191 + return pRealEnv->xRead(p->pReal, iOff, pData, nData); 192 +} 193 + 194 +static int testEnvWrite(lsm_file *pFile, lsm_i64 iOff, void *pData, int nData){ 195 + lsm_env *pRealEnv = tdb_lsm_env(); 196 + LsmFile *p = (LsmFile *)pFile; 197 + LsmDb *pDb = p->pDb; 198 + 199 + if( pDb->bCrashed ) return LSM_IOERR; 200 + 201 + if( pDb->bPrepareCrash ){ 202 + FileData *pData2 = &pDb->aFile[p->bLog]; 203 + int iFirst; 204 + int iLast; 205 + int iSector; 206 + 207 + iFirst = (int)(iOff / pDb->szSector); 208 + iLast = (int)((iOff + nData - 1) / pDb->szSector); 209 + 210 + if( pData2->nSector<(iLast+1) ){ 211 + int nNew = ( ((iLast + 1) + 63) / 64 ) * 64; 212 + assert( nNew>iLast ); 213 + pData2->aSector = (FileSector *)testRealloc( 214 + pData2->aSector, nNew*sizeof(FileSector) 215 + ); 216 + memset(&pData2->aSector[pData2->nSector], 217 + 0, (nNew - pData2->nSector) * sizeof(FileSector) 218 + ); 219 + pData2->nSector = nNew; 220 + } 221 + 222 + for(iSector=iFirst; iSector<=iLast; iSector++){ 223 + if( pData2->aSector[iSector].aOld==0 ){ 224 + u8 *aOld = (u8 *)testMalloc(pDb->szSector); 225 + pRealEnv->xRead( 226 + p->pReal, (lsm_i64)iSector*pDb->szSector, aOld, pDb->szSector 227 + ); 228 + pData2->aSector[iSector].aOld = aOld; 229 + } 230 + } 231 + } 232 + 233 + if( pDb->xWriteHook ){ 234 + int rc; 235 + int nUs; 236 + struct timeval t1; 237 + struct timeval t2; 238 + 239 + gettimeofday(&t1, 0); 240 + assert( nData>0 ); 241 + rc = pRealEnv->xWrite(p->pReal, iOff, pData, nData); 242 + gettimeofday(&t2, 0); 243 + 244 + nUs = (t2.tv_sec - t1.tv_sec) * 1000000 + (t2.tv_usec - t1.tv_usec); 245 + pDb->xWriteHook(pDb->pWriteCtx, p->bLog, iOff, nData, nUs); 246 + return rc; 247 + } 248 + 249 + return pRealEnv->xWrite(p->pReal, iOff, pData, nData); 250 +} 251 + 252 +static void doSystemCrash(LsmDb *pDb); 253 + 254 +static int testEnvSync(lsm_file *pFile){ 255 + lsm_env *pRealEnv = tdb_lsm_env(); 256 + LsmFile *p = (LsmFile *)pFile; 257 + LsmDb *pDb = p->pDb; 258 + FileData *pData = &pDb->aFile[p->bLog]; 259 + int i; 260 + 261 + if( pDb->bCrashed ) return LSM_IOERR; 262 + 263 + if( pDb->nAutoCrash ){ 264 + pDb->nAutoCrash--; 265 + if( pDb->nAutoCrash==0 ){ 266 + doSystemCrash(pDb); 267 + pDb->bCrashed = 1; 268 + return LSM_IOERR; 269 + } 270 + } 271 + 272 + if( pDb->bPrepareCrash ){ 273 + for(i=0; i<pData->nSector; i++){ 274 + testFree(pData->aSector[i].aOld); 275 + pData->aSector[i].aOld = 0; 276 + } 277 + } 278 + 279 + if( pDb->xWriteHook ){ 280 + int rc; 281 + int nUs; 282 + struct timeval t1; 283 + struct timeval t2; 284 + 285 + gettimeofday(&t1, 0); 286 + rc = pRealEnv->xSync(p->pReal); 287 + gettimeofday(&t2, 0); 288 + 289 + nUs = (t2.tv_sec - t1.tv_sec) * 1000000 + (t2.tv_usec - t1.tv_usec); 290 + pDb->xWriteHook(pDb->pWriteCtx, p->bLog, 0, 0, nUs); 291 + return rc; 292 + } 293 + 294 + return pRealEnv->xSync(p->pReal); 295 +} 296 + 297 +static int testEnvTruncate(lsm_file *pFile, lsm_i64 iOff){ 298 + lsm_env *pRealEnv = tdb_lsm_env(); 299 + LsmFile *p = (LsmFile *)pFile; 300 + if( p->pDb->bCrashed ) return LSM_IOERR; 301 + return pRealEnv->xTruncate(p->pReal, iOff); 302 +} 303 + 304 +static int testEnvSectorSize(lsm_file *pFile){ 305 + lsm_env *pRealEnv = tdb_lsm_env(); 306 + LsmFile *p = (LsmFile *)pFile; 307 + return pRealEnv->xSectorSize(p->pReal); 308 +} 309 + 310 +static int testEnvRemap( 311 + lsm_file *pFile, 312 + lsm_i64 iMin, 313 + void **ppOut, 314 + lsm_i64 *pnOut 315 +){ 316 + lsm_env *pRealEnv = tdb_lsm_env(); 317 + LsmFile *p = (LsmFile *)pFile; 318 + return pRealEnv->xRemap(p->pReal, iMin, ppOut, pnOut); 319 +} 320 + 321 +static int testEnvFileid( 322 + lsm_file *pFile, 323 + void *ppOut, 324 + int *pnOut 325 +){ 326 + lsm_env *pRealEnv = tdb_lsm_env(); 327 + LsmFile *p = (LsmFile *)pFile; 328 + return pRealEnv->xFileid(p->pReal, ppOut, pnOut); 329 +} 330 + 331 +static int testEnvClose(lsm_file *pFile){ 332 + lsm_env *pRealEnv = tdb_lsm_env(); 333 + LsmFile *p = (LsmFile *)pFile; 334 + 335 + pRealEnv->xClose(p->pReal); 336 + testFree(p); 337 + return LSM_OK; 338 +} 339 + 340 +static int testEnvUnlink(lsm_env *pEnv, const char *zFile){ 341 + lsm_env *pRealEnv = tdb_lsm_env(); 342 + unused_parameter(pEnv); 343 + return pRealEnv->xUnlink(pRealEnv, zFile); 344 +} 345 + 346 +static int testEnvLock(lsm_file *pFile, int iLock, int eType){ 347 + LsmFile *p = (LsmFile *)pFile; 348 + lsm_env *pRealEnv = tdb_lsm_env(); 349 + 350 + if( iLock==2 && eType==LSM_LOCK_EXCL && p->pDb->bNoRecovery ){ 351 + return LSM_BUSY; 352 + } 353 + return pRealEnv->xLock(p->pReal, iLock, eType); 354 +} 355 + 356 +static int testEnvTestLock(lsm_file *pFile, int iLock, int nLock, int eType){ 357 + LsmFile *p = (LsmFile *)pFile; 358 + lsm_env *pRealEnv = tdb_lsm_env(); 359 + 360 + if( iLock==2 && eType==LSM_LOCK_EXCL && p->pDb->bNoRecovery ){ 361 + return LSM_BUSY; 362 + } 363 + return pRealEnv->xTestLock(p->pReal, iLock, nLock, eType); 364 +} 365 + 366 +static int testEnvShmMap(lsm_file *pFile, int iRegion, int sz, void **pp){ 367 + LsmFile *p = (LsmFile *)pFile; 368 + lsm_env *pRealEnv = tdb_lsm_env(); 369 + return pRealEnv->xShmMap(p->pReal, iRegion, sz, pp); 370 +} 371 + 372 +static void testEnvShmBarrier(void){ 373 +} 374 + 375 +static int testEnvShmUnmap(lsm_file *pFile, int bDel){ 376 + LsmFile *p = (LsmFile *)pFile; 377 + lsm_env *pRealEnv = tdb_lsm_env(); 378 + return pRealEnv->xShmUnmap(p->pReal, bDel); 379 +} 380 + 381 +static int testEnvSleep(lsm_env *pEnv, int us){ 382 + lsm_env *pRealEnv = tdb_lsm_env(); 383 + return pRealEnv->xSleep(pRealEnv, us); 384 +} 385 + 386 +static void doSystemCrash(LsmDb *pDb){ 387 + lsm_env *pEnv = tdb_lsm_env(); 388 + int iFile; 389 + int iSeed = pDb->aFile[0].nSector + pDb->aFile[1].nSector; 390 + 391 + char *zFile = pDb->zName; 392 + char *zFree = 0; 393 + 394 + for(iFile=0; iFile<2; iFile++){ 395 + lsm_file *pFile = 0; 396 + int i; 397 + 398 + pEnv->xOpen(pEnv, zFile, 0, &pFile); 399 + for(i=0; i<pDb->aFile[iFile].nSector; i++){ 400 + u8 *aOld = pDb->aFile[iFile].aSector[i].aOld; 401 + if( aOld ){ 402 + int iOpt = testPrngValue(iSeed++) % 3; 403 + switch( iOpt ){ 404 + case 0: 405 + break; 406 + 407 + case 1: 408 + testPrngArray(iSeed++, (u32 *)aOld, pDb->szSector/4); 409 + /* Fall-through */ 410 + 411 + case 2: 412 + pEnv->xWrite( 413 + pFile, (lsm_i64)i * pDb->szSector, aOld, pDb->szSector 414 + ); 415 + break; 416 + } 417 + testFree(aOld); 418 + pDb->aFile[iFile].aSector[i].aOld = 0; 419 + } 420 + } 421 + pEnv->xClose(pFile); 422 + zFree = zFile = sqlite3_mprintf("%s-log", pDb->zName); 423 + } 424 + 425 + sqlite3_free(zFree); 426 +} 427 +/* 428 +** End test VFS code. 429 +************************************************************************** 430 +*************************************************************************/ 431 + 432 +/************************************************************************* 433 +************************************************************************** 434 +** Begin test compression hooks. 435 +*/ 436 + 437 +#ifdef HAVE_ZLIB 438 +#include <zlib.h> 439 + 440 +static int testZipBound(void *pCtx, int nSrc){ 441 + return compressBound(nSrc); 442 +} 443 + 444 +static int testZipCompress( 445 + void *pCtx, /* Context pointer */ 446 + char *aOut, int *pnOut, /* OUT: Buffer containing compressed data */ 447 + const char *aIn, int nIn /* Buffer containing input data */ 448 +){ 449 + uLongf n = *pnOut; /* In/out buffer size for compress() */ 450 + int rc; /* compress() return code */ 451 + 452 + rc = compress((Bytef*)aOut, &n, (Bytef*)aIn, nIn); 453 + *pnOut = n; 454 + return (rc==Z_OK ? 0 : LSM_ERROR); 455 +} 456 + 457 +static int testZipUncompress( 458 + void *pCtx, /* Context pointer */ 459 + char *aOut, int *pnOut, /* OUT: Buffer containing uncompressed data */ 460 + const char *aIn, int nIn /* Buffer containing input data */ 461 +){ 462 + uLongf n = *pnOut; /* In/out buffer size for uncompress() */ 463 + int rc; /* uncompress() return code */ 464 + 465 + rc = uncompress((Bytef*)aOut, &n, (Bytef*)aIn, nIn); 466 + *pnOut = n; 467 + return (rc==Z_OK ? 0 : LSM_ERROR); 468 +} 469 + 470 +static int testConfigureCompression(lsm_db *pDb){ 471 + static lsm_compress zip = { 472 + 0, /* Context pointer (unused) */ 473 + 1, /* Id value */ 474 + testZipBound, /* xBound method */ 475 + testZipCompress, /* xCompress method */ 476 + testZipUncompress /* xUncompress method */ 477 + }; 478 + return lsm_config(pDb, LSM_CONFIG_SET_COMPRESSION, &zip); 479 +} 480 +#endif /* ifdef HAVE_ZLIB */ 481 + 482 +/* 483 +** End test compression hooks. 484 +************************************************************************** 485 +*************************************************************************/ 486 + 487 +static int test_lsm_close(TestDb *pTestDb){ 488 + int i; 489 + int rc = LSM_OK; 490 + LsmDb *pDb = (LsmDb *)pTestDb; 491 + 492 + lsm_csr_close(pDb->pCsr); 493 + lsm_close(pDb->db); 494 + 495 + /* If this is a multi-threaded database, wait on the worker threads. */ 496 + mt_shutdown(pDb); 497 + for(i=0; i<pDb->nWorker && rc==LSM_OK; i++){ 498 + rc = pDb->aWorker[i].worker_rc; 499 + } 500 + 501 + for(i=0; i<pDb->aFile[0].nSector; i++){ 502 + testFree(pDb->aFile[0].aSector[i].aOld); 503 + } 504 + testFree(pDb->aFile[0].aSector); 505 + for(i=0; i<pDb->aFile[1].nSector; i++){ 506 + testFree(pDb->aFile[1].aSector[i].aOld); 507 + } 508 + testFree(pDb->aFile[1].aSector); 509 + 510 + memset(pDb, sizeof(LsmDb), 0x11); 511 + testFree((char *)pDb->pBuf); 512 + testFree((char *)pDb); 513 + return rc; 514 +} 515 + 516 +static void mt_signal_worker(LsmDb*, int); 517 + 518 +static int waitOnCheckpointer(LsmDb *pDb, lsm_db *db){ 519 + int nSleep = 0; 520 + int nKB; 521 + int rc; 522 + 523 + do { 524 + nKB = 0; 525 + rc = lsm_info(db, LSM_INFO_CHECKPOINT_SIZE, &nKB); 526 + if( rc!=LSM_OK || nKB<pDb->nMtMaxCkpt ) break; 527 +#ifdef LSM_MUTEX_PTHREADS 528 + mt_signal_worker(pDb, 529 + (pDb->eMode==LSMTEST_MODE_BACKGROUND_CKPT ? 0 : 1) 530 + ); 531 +#endif 532 + usleep(5000); 533 + nSleep += 5; 534 + }while( 1 ); 535 + 536 +#if 0 537 + if( nSleep ) printf("# waitOnCheckpointer(): nSleep=%d\n", nSleep); 538 +#endif 539 + 540 + return rc; 541 +} 542 + 543 +static int waitOnWorker(LsmDb *pDb){ 544 + int rc; 545 + int nLimit = -1; 546 + int nSleep = 0; 547 + 548 + rc = lsm_config(pDb->db, LSM_CONFIG_AUTOFLUSH, &nLimit); 549 + do { 550 + int nOld, nNew, rc2; 551 + rc2 = lsm_info(pDb->db, LSM_INFO_TREE_SIZE, &nOld, &nNew); 552 + if( rc2!=LSM_OK ) return rc2; 553 + if( nOld==0 || nNew<(nLimit/2) ) break; 554 +#ifdef LSM_MUTEX_PTHREADS 555 + mt_signal_worker(pDb, 0); 556 +#endif 557 + usleep(5000); 558 + nSleep += 5; 559 + }while( 1 ); 560 + 561 +#if 0 562 + if( nSleep ) printf("# waitOnWorker(): nSleep=%d\n", nSleep); 563 +#endif 564 + 565 + return rc; 566 +} 567 + 568 +static int test_lsm_write( 569 + TestDb *pTestDb, 570 + void *pKey, 571 + int nKey, 572 + void *pVal, 573 + int nVal 574 +){ 575 + LsmDb *pDb = (LsmDb *)pTestDb; 576 + int rc = LSM_OK; 577 + 578 + if( pDb->eMode==LSMTEST_MODE_BACKGROUND_CKPT ){ 579 + rc = waitOnCheckpointer(pDb, pDb->db); 580 + }else if( 581 + pDb->eMode==LSMTEST_MODE_BACKGROUND_WORK 582 + || pDb->eMode==LSMTEST_MODE_BACKGROUND_BOTH 583 + ){ 584 + rc = waitOnWorker(pDb); 585 + } 586 + 587 + if( rc==LSM_OK ){ 588 + rc = lsm_insert(pDb->db, pKey, nKey, pVal, nVal); 589 + } 590 + return rc; 591 +} 592 + 593 +static int test_lsm_delete(TestDb *pTestDb, void *pKey, int nKey){ 594 + LsmDb *pDb = (LsmDb *)pTestDb; 595 + return lsm_delete(pDb->db, pKey, nKey); 596 +} 597 + 598 +static int test_lsm_delete_range( 599 + TestDb *pTestDb, 600 + void *pKey1, int nKey1, 601 + void *pKey2, int nKey2 602 +){ 603 + LsmDb *pDb = (LsmDb *)pTestDb; 604 + return lsm_delete_range(pDb->db, pKey1, nKey1, pKey2, nKey2); 605 +} 606 + 607 +static int test_lsm_fetch( 608 + TestDb *pTestDb, 609 + void *pKey, 610 + int nKey, 611 + void **ppVal, 612 + int *pnVal 613 +){ 614 + int rc; 615 + LsmDb *pDb = (LsmDb *)pTestDb; 616 + lsm_cursor *csr; 617 + 618 + if( pKey==0 ) return LSM_OK; 619 + 620