/ Check-in [f9750870]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge the improved incremental doclist loading test from the broken-build branch (which is now fixed) into trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: f9750870ee04935f338e4d808900fee5a8b2b389
User & Date: drh 2011-06-15 13:11:06
Context
2011-06-15
16:07
Fix the wal7.test script so that it works even if secure_delete is engaged. check-in: 68fb7a54 user: drh tags: trunk
13:11
Merge the improved incremental doclist loading test from the broken-build branch (which is now fixed) into trunk. check-in: f9750870 user: drh tags: trunk
13:02
Fix so that the TCL test harness works even if SQLITE_ENABLE_FTS is omitted. Closed-Leaf check-in: 63ebcb52 user: drh tags: broken-build
2011-06-14
14:18
Fix a memory leak that can follow an OOM error in a user-function that uses sqlite3_set_auxdata(). check-in: 0185c4b6 user: dan tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283

3284
3285
3286
3287
3288
3289
3290
        return SQLITE_OK;

      }else if( aPoslist==0 ){
        aPoslist = pList;
        nPoslist = nList;

      }else{
        assert( iPrev>=0 );

        char *aOut = pList;
        char *p1 = aPoslist;
        char *p2 = aOut;


        fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2);
        sqlite3_free(aPoslist);
        aPoslist = pList;
        nPoslist = aOut - aPoslist;
        if( nPoslist==0 ){
          sqlite3_free(aPoslist);
          pPhrase->doclist.pList = 0;







<
<




>







3271
3272
3273
3274
3275
3276
3277


3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
        return SQLITE_OK;

      }else if( aPoslist==0 ){
        aPoslist = pList;
        nPoslist = nList;

      }else{


        char *aOut = pList;
        char *p1 = aPoslist;
        char *p2 = aOut;

        assert( iPrev>=0 );
        fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2);
        sqlite3_free(aPoslist);
        aPoslist = pList;
        nPoslist = aOut - aPoslist;
        if( nPoslist==0 ){
          sqlite3_free(aPoslist);
          pPhrase->doclist.pList = 0;

Changes to ext/fts3/fts3_test.c.

238
239
240
241
242
243
244





































































245
246



247
248
249

 near_match_out: 
  ckfree((char *)aPhrase);
  ckfree((char *)doc.aToken);
  return rc;
}






































































int Sqlitetestfts3_Init(Tcl_Interp *interp){
  Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0);



  return TCL_OK;
}








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


>
>
>


<
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320


 near_match_out: 
  ckfree((char *)aPhrase);
  ckfree((char *)doc.aToken);
  return rc;
}

/*
**   Tclcmd: fts3_configure_incr_load ?CHUNKSIZE THRESHOLD?
**
** Normally, FTS uses hard-coded values to determine the minimum doclist
** size eligible for incremental loading, and the size of the chunks loaded
** when a doclist is incrementally loaded. This command allows the built-in
** values to be overridden for testing purposes.
**
** If present, the first argument is the chunksize in bytes to load doclists
** in. The second argument is the minimum doclist size in bytes to use
** incremental loading with.
**
** Whether or not the arguments are present, this command returns a list of
** two integers - the initial chunksize and threshold when the command is
** invoked. This can be used to restore the default behaviour after running
** tests. For example:
**
**    # Override incr-load settings for testing:
**    set cfg [fts3_configure_incr_load $new_chunksize $new_threshold]
**
**    .... run tests ....
**
**    # Restore initial incr-load settings:
**    eval fts3_configure_incr_load $cfg
*/
static int fts3_configure_incr_load_cmd(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
#ifdef SQLITE_ENABLE_FTS3
  extern int test_fts3_node_chunksize;
  extern int test_fts3_node_chunk_threshold;
  int iArg1;
  int iArg2;
  Tcl_Obj *pRet;

  if( objc!=1 && objc!=3 ){
    Tcl_WrongNumArgs(interp, 1, objv, "?CHUNKSIZE THRESHOLD?");
    return TCL_ERROR;
  }

  pRet = Tcl_NewObj();
  Tcl_IncrRefCount(pRet);
  Tcl_ListObjAppendElement(
      interp, pRet, Tcl_NewIntObj(test_fts3_node_chunksize));
  Tcl_ListObjAppendElement(
      interp, pRet, Tcl_NewIntObj(test_fts3_node_chunk_threshold));

  if( objc==3 ){
    int iArg1;
    int iArg2;
    if( Tcl_GetIntFromObj(interp, objv[1], &iArg1)
     || Tcl_GetIntFromObj(interp, objv[2], &iArg2)
    ){
      Tcl_DecrRefCount(pRet);
      return TCL_ERROR;
    }
    test_fts3_node_chunksize = iArg1;
    test_fts3_node_chunk_threshold = iArg2;
  }

  Tcl_SetObjResult(interp, pRet);
  Tcl_DecrRefCount(pRet);
#endif
  return TCL_OK;
}

int Sqlitetestfts3_Init(Tcl_Interp *interp){
  Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0);
  Tcl_CreateObjCommand(interp, 
      "fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0
  );
  return TCL_OK;
}

Changes to ext/fts3/fts3_write.c.

45
46
47
48
49
50
51
52
53

54



55
56

57
58

59
60
61
62
63
64
65
**
** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD 
** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes.
** The code is written so that the hard lower-limit for each of these values 
** is 1. Clearly such small values would be inefficient, but can be useful 
** for testing purposes.
**
** TODO: Add a test interface to modify these "constants" from a script for
** this purpose.

*/



#define FTS3_NODE_CHUNKSIZE (4*1024) 
#define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4)

/* #define FTS3_NODE_CHUNKSIZE 1 */
/* #define FTS3_NODE_CHUNK_THRESHOLD 1 */


typedef struct PendingList PendingList;
typedef struct SegmentNode SegmentNode;
typedef struct SegmentWriter SegmentWriter;

/*
** An instance of the following data structure is used to build doclists







|
|
>

>
>
>
|
|
>
|
|
>







45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
**
** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD 
** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes.
** The code is written so that the hard lower-limit for each of these values 
** is 1. Clearly such small values would be inefficient, but can be useful 
** for testing purposes.
**
** If this module is built with SQLITE_TEST defined, these constants may
** be overridden at runtime for testing purposes. File fts3_test.c contains
** a Tcl interface to read and write the values.
*/
#ifdef SQLITE_TEST
int test_fts3_node_chunksize = (4*1024);
int test_fts3_node_chunk_threshold = (4*1024)*4;
# define FTS3_NODE_CHUNKSIZE       test_fts3_node_chunksize
# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold
#else
# define FTS3_NODE_CHUNKSIZE (4*1024) 
# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4)
#endif

typedef struct PendingList PendingList;
typedef struct SegmentNode SegmentNode;
typedef struct SegmentWriter SegmentWriter;

/*
** An instance of the following data structure is used to build doclists

Changes to src/func.c.

604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
      }
      return 0;
    }else if( !prevEscape && c==matchOne ){
      if( sqlite3Utf8Read(zString, &zString)==0 ){
        return 0;
      }
    }else if( c==matchSet ){
      int prior_c = 0;
      assert( esc==0 );    /* This only occurs for GLOB, not LIKE */
      seen = 0;
      invert = 0;
      c = sqlite3Utf8Read(zString, &zString);
      if( c==0 ) return 0;
      c2 = sqlite3Utf8Read(zPattern, &zPattern);
      if( c2=='^' ){







|







604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
      }
      return 0;
    }else if( !prevEscape && c==matchOne ){
      if( sqlite3Utf8Read(zString, &zString)==0 ){
        return 0;
      }
    }else if( c==matchSet ){
      u32 prior_c = 0;
      assert( esc==0 );    /* This only occurs for GLOB, not LIKE */
      seen = 0;
      invert = 0;
      c = sqlite3Utf8Read(zString, &zString);
      if( c==0 ) return 0;
      c2 = sqlite3Utf8Read(zPattern, &zPattern);
      if( c2=='^' ){

Changes to test/fts3auto.test.

525
526
527
528
529
530
531
532















































533
534
535

  do_fts3query_test 4.$tn.4.1 -deferred fi* t1 {on* AND fi*}
  do_fts3query_test 4.$tn.4.2 -deferred fi* t1 {on* NEAR fi*}
  do_fts3query_test 4.$tn.4.3 -deferred fi* t1 {on* NEAR/1 fi*}
  do_fts3query_test 4.$tn.4.4 -deferred fi* t1 {on* NEAR/2 fi*}
  do_fts3query_test 4.$tn.4.5 -deferred fi* t1 {on* NEAR/3 fi*}
}
















































set sqlite_fts3_enable_parentheses $sfep
finish_test









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582

  do_fts3query_test 4.$tn.4.1 -deferred fi* t1 {on* AND fi*}
  do_fts3query_test 4.$tn.4.2 -deferred fi* t1 {on* NEAR fi*}
  do_fts3query_test 4.$tn.4.3 -deferred fi* t1 {on* NEAR/1 fi*}
  do_fts3query_test 4.$tn.4.4 -deferred fi* t1 {on* NEAR/2 fi*}
  do_fts3query_test 4.$tn.4.5 -deferred fi* t1 {on* NEAR/3 fi*}
}

#--------------------------------------------------------------------------
# The following test cases - fts3auto-5.* - focus on using prefix indexes.
#
set chunkconfig [fts3_configure_incr_load 1 1]
foreach {tn create} {
  1    "fts4(a, b)"
  2    "fts4(a, b, order=DESC, prefix=1)"
  3    "fts4(a, b, order=ASC,  prefix=1,3)"
  4    "fts4(a, b, order=DESC, prefix=2,4)"
} {

  execsql [subst {
    DROP TABLE t1;
    CREATE VIRTUAL TABLE t1 USING $create;
  }]

  foreach {a b} {
    "the song of songs which is solomons"
    "let him kiss me with the kisses of his mouth for thy love is better than wine"
    "because of the savour of thy good ointments thy name is as ointment poured forth therefore do the virgins love thee"
    "draw me we will run after thee the king hath brought me into his chambers we will be glad and rejoice in thee we will remember thy love more than wine the upright love thee"
    "i am black but comely o ye daughters of jerusalem as the tents of kedar as the curtains of solomon"
    "look not upon me because i am black because the sun hath looked upon me my mothers children were angry with me they made me the keeper of the vineyards but mine own vineyard have i not kept"
    "tell me o thou whom my soul loveth where thou feedest where thou makest thy flock to rest at noon for why should i be as one that turneth aside by the flocks of thy companions?"
    "if thou know not o thou fairest among women go thy way forth by the footsteps of the flock and feed thy kids beside the shepherds tents"
    "i have compared thee o my love to a company of horses in pharaohs chariots"
    "thy cheeks are comely with rows of jewels thy neck with chains of gold"
    "we will make thee borders of gold with studs of silver"
    "while the king sitteth at his table my spikenard sendeth forth the smell thereof"
    "a bundle of myrrh is my wellbeloved unto me he shall lie all night betwixt my breasts"
    "my beloved is unto me as a cluster of camphire in the vineyards of en gedi"
    "behold thou art fair my love behold thou art fair thou hast doves eyes"
    "behold thou art fair my beloved yea pleasant also our bed is green"
    "the beams of our house are cedar and our rafters of fir"
  } {
    execsql {INSERT INTO t1(a, b) VALUES($a, $b)}
  }

  do_fts3query_test 5.$tn.1.1 t1 {s*}
  do_fts3query_test 5.$tn.1.2 t1 {so*}
  do_fts3query_test 5.$tn.1.3 t1 {"s* o*"}
  do_fts3query_test 5.$tn.1.4 t1 {b* NEAR/3 a*}
  do_fts3query_test 5.$tn.1.5 t1 {th* NEAR/5 a* NEAR/5 w*}
  do_fts3query_test 5.$tn.1.6 t1 {"b* th* art* fair*"}
}
eval fts3_configure_incr_load $chunkconfig

set sqlite_fts3_enable_parentheses $sfep
finish_test