/ Check-in [3b179247]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix some fts3 related issues with the autoconf and amalgamation build systems.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both
Files: files | file ages | folders
SHA1: 3b17924754343c0163464dabf01a9c46ffccef28
User & Date: dan 2009-11-19 15:25:25
Context
2009-11-19
15:26
Merge fts3-refactor into the main trunk. check-in: 551dbe30 user: dan tags: trunk
15:25
Fix some fts3 related issues with the autoconf and amalgamation build systems. Closed-Leaf check-in: 3b179247 user: dan
14:57
Merge with [4bd4330709]. check-in: 7a46d1eb user: dan tags: fts3-refactor
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to Makefile.in.

307
308
309
310
311
312
313
314
315
316
317
318
319

320
321
322

323
324
325
326
327
328
329
...
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
  $(TOP)/ext/fts2/fts2_porter.c \
  $(TOP)/ext/fts2/fts2_tokenizer.h \
  $(TOP)/ext/fts2/fts2_tokenizer.c \
  $(TOP)/ext/fts2/fts2_tokenizer1.c
SRC += \
  $(TOP)/ext/fts3/fts3.c \
  $(TOP)/ext/fts3/fts3.h \
  $(TOP)/ext/fts3/fts3_expr.c \
  $(TOP)/ext/fts3/fts3_expr.h \
  $(TOP)/ext/fts3/fts3_hash.c \
  $(TOP)/ext/fts3/fts3_hash.h \
  $(TOP)/ext/fts3/fts3_icu.c \
  $(TOP)/ext/fts3/fts3_porter.c \

  $(TOP)/ext/fts3/fts3_tokenizer.h \
  $(TOP)/ext/fts3/fts3_tokenizer.c \
  $(TOP)/ext/fts3/fts3_tokenizer1.c

SRC += \
  $(TOP)/ext/icu/sqliteicu.h \
  $(TOP)/ext/icu/icu.c
SRC += \
  $(TOP)/ext/rtree/rtree.h \
  $(TOP)/ext/rtree/rtree.c

................................................................................
  $(TOP)/ext/fts1/fts1_tokenizer.h
HDR += \
  $(TOP)/ext/fts2/fts2.h \
  $(TOP)/ext/fts2/fts2_hash.h \
  $(TOP)/ext/fts2/fts2_tokenizer.h
HDR += \
  $(TOP)/ext/fts3/fts3.h \
  $(TOP)/ext/fts3/fts3_expr.h \
  $(TOP)/ext/fts3/fts3_hash.h \
  $(TOP)/ext/fts3/fts3_tokenizer.h
HDR += \
  $(TOP)/ext/rtree/rtree.h
HDR += \
  $(TOP)/ext/icu/sqliteicu.h








|
|




>


|
>







 







|







307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
...
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
  $(TOP)/ext/fts2/fts2_porter.c \
  $(TOP)/ext/fts2/fts2_tokenizer.h \
  $(TOP)/ext/fts2/fts2_tokenizer.c \
  $(TOP)/ext/fts2/fts2_tokenizer1.c
SRC += \
  $(TOP)/ext/fts3/fts3.c \
  $(TOP)/ext/fts3/fts3.h \
  $(TOP)/ext/fts3/fts3Int.h \
  $(TOP)/ext/fts3/fts3_expr.c \
  $(TOP)/ext/fts3/fts3_hash.c \
  $(TOP)/ext/fts3/fts3_hash.h \
  $(TOP)/ext/fts3/fts3_icu.c \
  $(TOP)/ext/fts3/fts3_porter.c \
  $(TOP)/ext/fts3/fts3_snippet.c \
  $(TOP)/ext/fts3/fts3_tokenizer.h \
  $(TOP)/ext/fts3/fts3_tokenizer.c \
  $(TOP)/ext/fts3/fts3_tokenizer1.c \
  $(TOP)/ext/fts3/fts3_write.c
SRC += \
  $(TOP)/ext/icu/sqliteicu.h \
  $(TOP)/ext/icu/icu.c
SRC += \
  $(TOP)/ext/rtree/rtree.h \
  $(TOP)/ext/rtree/rtree.c

................................................................................
  $(TOP)/ext/fts1/fts1_tokenizer.h
HDR += \
  $(TOP)/ext/fts2/fts2.h \
  $(TOP)/ext/fts2/fts2_hash.h \
  $(TOP)/ext/fts2/fts2_tokenizer.h
HDR += \
  $(TOP)/ext/fts3/fts3.h \
  $(TOP)/ext/fts3/fts3Int.h \
  $(TOP)/ext/fts3/fts3_hash.h \
  $(TOP)/ext/fts3/fts3_tokenizer.h
HDR += \
  $(TOP)/ext/rtree/rtree.h
HDR += \
  $(TOP)/ext/icu/sqliteicu.h

configure became executable.

Changes to ext/fts3/fts3_snippet.c.

7
8
9
10
11
12
13


14
15
16
17
18
19
20
...
744
745
746
747
748
749
750

**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/



#include "fts3Int.h"
#include <string.h>
#include <assert.h>
#include <ctype.h>

typedef struct Snippet Snippet;

................................................................................
  Snippet *p;                     /* Snippet structure */
  int rc = snippetAllOffsets(pCsr, &p);
  snippetText(pCsr, p, zStart, zEnd, zEllipsis);
  sqlite3_result_text(pCtx, p->zSnippet, p->nSnippet, SQLITE_TRANSIENT);
  fts3SnippetFree(p);
}









>
>







 







>
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
...
746
747
748
749
750
751
752
753
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)

#include "fts3Int.h"
#include <string.h>
#include <assert.h>
#include <ctype.h>

typedef struct Snippet Snippet;

................................................................................
  Snippet *p;                     /* Snippet structure */
  int rc = snippetAllOffsets(pCsr, &p);
  snippetText(pCsr, p, zStart, zEnd, zEllipsis);
  sqlite3_result_text(pCtx, p->zSnippet, p->nSnippet, SQLITE_TRANSIENT);
  fts3SnippetFree(p);
}

#endif

Changes to ext/fts3/fts3_tokenizer.c.

31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
...
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
#endif

#include "fts3Int.h"
#include <assert.h>
#include <ctype.h>
#include <string.h>

static int safe_isspace(char c){
  return (c&0x80)==0 ? isspace(c) : 0;
}

/*
** Implementation of the SQL scalar function for accessing the underlying 
** hash table. This function may be called as follows:
**
**   SELECT <function-name>(<key-name>);
**   SELECT <function-name>(<key-name>, <pointer>);
**
................................................................................
  char *zCopy;
  char *zEnd;                     /* Pointer to nul-term of zCopy */
  sqlite3_tokenizer_module *m;

  if( !z ){
    zCopy = sqlite3_mprintf("simple");
  }else{
    while( safe_isspace(*z) ) z++;
    if( sqlite3_strnicmp(z, "tokenize", 8) || fts3IsIdChar(z[8])){
      return SQLITE_OK;
    }
    zCopy = sqlite3_mprintf("%s", &z[8]);
    *pzTokenizer = zArg;
  }
  if( !zCopy ){







<
<
<
<







 







|







31
32
33
34
35
36
37




38
39
40
41
42
43
44
...
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
#endif

#include "fts3Int.h"
#include <assert.h>
#include <ctype.h>
#include <string.h>





/*
** Implementation of the SQL scalar function for accessing the underlying 
** hash table. This function may be called as follows:
**
**   SELECT <function-name>(<key-name>);
**   SELECT <function-name>(<key-name>, <pointer>);
**
................................................................................
  char *zCopy;
  char *zEnd;                     /* Pointer to nul-term of zCopy */
  sqlite3_tokenizer_module *m;

  if( !z ){
    zCopy = sqlite3_mprintf("simple");
  }else{
    while( (*z&0x80) && isspace(*z) ) z++;
    if( sqlite3_strnicmp(z, "tokenize", 8) || fts3IsIdChar(z[8])){
      return SQLITE_OK;
    }
    zCopy = sqlite3_mprintf("%s", &z[8]);
    *pzTokenizer = zArg;
  }
  if( !zCopy ){

Changes to ext/fts3/fts3_write.c.

13
14
15
16
17
18
19


20
21
22
23
24
25
26
....
1943
1944
1945
1946
1947
1948
1949

** This file is part of the SQLite FTS3 extension module. Specifically,
** this file contains code to insert, update and delete rows from FTS3
** tables. It also contains code to merge FTS3 b-tree segments. Some
** of the sub-routines used to merge segments are also used by the query 
** code in fts3.c.
*/



#include "fts3Int.h"
#include <string.h>
#include <assert.h>
#include <stdlib.h>

#define INTERIOR_MAX 2048         /* Soft limit for segment node size */
#define LEAF_MAX 2048             /* Soft limit for segment leaf size */
................................................................................
  int rc = sqlite3Fts3PendingTermsFlush(p);
  if( rc==SQLITE_OK ){
    rc = fts3SegmentMerge(p, -1);
  }
  return rc;
}









>
>







 







>
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
....
1945
1946
1947
1948
1949
1950
1951
1952
** This file is part of the SQLite FTS3 extension module. Specifically,
** this file contains code to insert, update and delete rows from FTS3
** tables. It also contains code to merge FTS3 b-tree segments. Some
** of the sub-routines used to merge segments are also used by the query 
** code in fts3.c.
*/

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)

#include "fts3Int.h"
#include <string.h>
#include <assert.h>
#include <stdlib.h>

#define INTERIOR_MAX 2048         /* Soft limit for segment node size */
#define LEAF_MAX 2048             /* Soft limit for segment leaf size */
................................................................................
  int rc = sqlite3Fts3PendingTermsFlush(p);
  if( rc==SQLITE_OK ){
    rc = fts3SegmentMerge(p, -1);
  }
  return rc;
}

#endif

Changes to main.mk.

183
184
185
186
187
188
189

190
191
192
193
194
195
196
  $(TOP)/ext/fts3/fts3.c \
  $(TOP)/ext/fts3/fts3.h \
  $(TOP)/ext/fts3/fts3_expr.c \
  $(TOP)/ext/fts3/fts3_hash.c \
  $(TOP)/ext/fts3/fts3_hash.h \
  $(TOP)/ext/fts3/fts3_icu.c \
  $(TOP)/ext/fts3/fts3_porter.c \

  $(TOP)/ext/fts3/fts3_tokenizer.h \
  $(TOP)/ext/fts3/fts3_tokenizer.c \
  $(TOP)/ext/fts3/fts3_tokenizer1.c \
  $(TOP)/ext/fts3/fts3_write.c
SRC += \
  $(TOP)/ext/icu/sqliteicu.h \
  $(TOP)/ext/icu/icu.c







>







183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
  $(TOP)/ext/fts3/fts3.c \
  $(TOP)/ext/fts3/fts3.h \
  $(TOP)/ext/fts3/fts3_expr.c \
  $(TOP)/ext/fts3/fts3_hash.c \
  $(TOP)/ext/fts3/fts3_hash.h \
  $(TOP)/ext/fts3/fts3_icu.c \
  $(TOP)/ext/fts3/fts3_porter.c \
  $(TOP)/ext/fts3/fts3_snippet.c \
  $(TOP)/ext/fts3/fts3_tokenizer.h \
  $(TOP)/ext/fts3/fts3_tokenizer.c \
  $(TOP)/ext/fts3/fts3_tokenizer1.c \
  $(TOP)/ext/fts3/fts3_write.c
SRC += \
  $(TOP)/ext/icu/sqliteicu.h \
  $(TOP)/ext/icu/icu.c

Changes to test/fts3malloc.test.

25
26
27
28
29
30
31
32



33
34
35
36
37
38
39
..
50
51
52
53
54
55
56


57





58
59
60
61
62
63
64
65
..
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
sqlite3_db_config_lookaside db 0 0 0

set sqlite_fts3_enable_parentheses 1
set DO_MALLOC_TEST 1

# Test organization:
#
# fts3_malloc-1.*: Test CREATE and DROP table statements.



#
#

#-------------------------------------------------------------------------
# This proc is used to test a single SELECT statement. Parameter $name is
# passed a name for the test case (i.e. "fts3_malloc-1.4.1") and parameter
# $sql is passed the text of the SELECT statement. Parameter $result is
................................................................................
# the SELECT statement succeeds and the results match those specified
# by parameter $result, or (b) TCL throws an "out of memory" error.
#
# If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement
# is executed just once. In this case the test case passes if the results
# match the expected results passed via parameter $result.
#








proc do_passive_test {name sql catchres} {
  if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }

  if {$::DO_MALLOC_TEST} {
    set answers [list {1 {out of memory}} $catchres]
    set modes [list 100000 transient 1 persistent]
  } else {
    set answers [list $catchres]
................................................................................
      do_test $name.$zName.$iFail [list set {} $res] $str
      set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
      if {$nFail==0} break
    }
  }
}

proc do_select_test {name sql result} {
  do_passive_test $name $sql [list 0 $result]
}

proc do_error_test {name sql error} {
  do_passive_test $name $sql [list 1 $error]
}

#-------------------------------------------------------------------------
# Test a single write to the database. In this case a  "write" is a 
# DELETE, UPDATE or INSERT statement.
#
# If OOM testing is performed, there are several acceptable outcomes:
#







|
>
>
>







 







>
>
|
>
>
>
>
>
|







 







<
<
<
<
<
<
<







25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
..
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
..
88
89
90
91
92
93
94







95
96
97
98
99
100
101
sqlite3_db_config_lookaside db 0 0 0

set sqlite_fts3_enable_parentheses 1
set DO_MALLOC_TEST 1

# Test organization:
#
# fts3_malloc-1.*: Test OOM during CREATE and DROP table statements.
# fts3_malloc-2.*: Test OOM during SELECT operations.
# fts3_malloc-3.*: Test OOM during SELECT operations with a larger database.
# fts3_malloc-4.*: Test OOM during database write operations.
#
#

#-------------------------------------------------------------------------
# This proc is used to test a single SELECT statement. Parameter $name is
# passed a name for the test case (i.e. "fts3_malloc-1.4.1") and parameter
# $sql is passed the text of the SELECT statement. Parameter $result is
................................................................................
# the SELECT statement succeeds and the results match those specified
# by parameter $result, or (b) TCL throws an "out of memory" error.
#
# If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement
# is executed just once. In this case the test case passes if the results
# match the expected results passed via parameter $result.
#
proc do_select_test {name sql result} {
  doPassiveTest $name $sql [list 0 $result]
}

proc do_error_test {name sql error} {
  doPassiveTest $name $sql [list 1 $error]
}

proc doPassiveTest {name sql catchres} {
  if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }

  if {$::DO_MALLOC_TEST} {
    set answers [list {1 {out of memory}} $catchres]
    set modes [list 100000 transient 1 persistent]
  } else {
    set answers [list $catchres]
................................................................................
      do_test $name.$zName.$iFail [list set {} $res] $str
      set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
      if {$nFail==0} break
    }
  }
}









#-------------------------------------------------------------------------
# Test a single write to the database. In this case a  "write" is a 
# DELETE, UPDATE or INSERT statement.
#
# If OOM testing is performed, there are several acceptable outcomes:
#

Changes to tool/mksqlite3c.tcl.

83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
...
291
292
293
294
295
296
297


298
299
300
301
302
303
304
305
306
# files are seen in a #include statement in the C code, include the complete
# text of the file in-line.  The file only needs to be included once.
#
foreach hdr {
   btree.h
   btreeInt.h
   fts3.h
   fts3_expr.h
   fts3_hash.h
   fts3_tokenizer.h
   hash.h
   hwtime.h
   keywordhash.h
   mutex.h
   opcodes.h
................................................................................

   fts3.c
   fts3_expr.c
   fts3_hash.c
   fts3_porter.c
   fts3_tokenizer.c
   fts3_tokenizer1.c



   rtree.c
   icu.c
   fts3_icu.c
} {
  copy_file tsrc/$file
}

close $out







<







 







>
>









83
84
85
86
87
88
89

90
91
92
93
94
95
96
...
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
# files are seen in a #include statement in the C code, include the complete
# text of the file in-line.  The file only needs to be included once.
#
foreach hdr {
   btree.h
   btreeInt.h
   fts3.h

   fts3_hash.h
   fts3_tokenizer.h
   hash.h
   hwtime.h
   keywordhash.h
   mutex.h
   opcodes.h
................................................................................

   fts3.c
   fts3_expr.c
   fts3_hash.c
   fts3_porter.c
   fts3_tokenizer.c
   fts3_tokenizer1.c
   fts3_write.c
   fts3_snippet.c

   rtree.c
   icu.c
   fts3_icu.c
} {
  copy_file tsrc/$file
}

close $out