/ Check-in [21491a9b]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix various problems to do with segment promotion. Add test file fts4growth2.test, containing tests to check that the FTS index does not grow indefinitely as the table is updated. Allow the user to configure the number of segments merged simultaneously by the automerge option.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts4-experimental
Files: files | file ages | folders
SHA1: 21491a9bc686e63bec32f1a67103622f1160a2f9
User & Date: dan 2014-05-14 15:58:47
Original Comment: Fix various problems to do with segment promotion. Add test file fts4growth2.test, containing tests to check that the FTS index does not grow indefinitely as the table is updated.
Context
2014-05-14
18:45
Fix a problem to do with loading the value of the FTS automerge setting from the database. check-in: 7268119f user: dan tags: fts4-experimental
15:58
Fix various problems to do with segment promotion. Add test file fts4growth2.test, containing tests to check that the FTS index does not grow indefinitely as the table is updated. Allow the user to configure the number of segments merged simultaneously by the automerge option. check-in: 21491a9b user: dan tags: fts4-experimental
2014-05-13
20:11
Fix a problem preventing delete markers from ever being removed from the FTS index. check-in: 7f47ae5c user: dan tags: fts4-experimental
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

  1329   1329     p->azColumn = (char **)&p[1];
  1330   1330     p->pTokenizer = pTokenizer;
  1331   1331     p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
  1332   1332     p->bHasDocsize = (isFts4 && bNoDocsize==0);
  1333   1333     p->bHasStat = isFts4;
  1334   1334     p->bFts4 = isFts4;
  1335   1335     p->bDescIdx = bDescIdx;
  1336         -  p->bAutoincrmerge = 0xff;   /* 0xff means setting unknown */
         1336  +  p->nAutoincrmerge = 0xff;   /* 0xff means setting unknown */
  1337   1337     p->zContentTbl = zContent;
  1338   1338     p->zLanguageid = zLanguageid;
  1339   1339     zContent = 0;
  1340   1340     zLanguageid = 0;
  1341   1341     TESTONLY( p->inTransaction = -1 );
  1342   1342     TESTONLY( p->mxSavepoint = -1 );
  1343   1343   
................................................................................
  3298   3298     ** segments.
  3299   3299     */
  3300   3300     const u32 nMinMerge = 64;       /* Minimum amount of incr-merge work to do */
  3301   3301   
  3302   3302     Fts3Table *p = (Fts3Table*)pVtab;
  3303   3303     int rc = sqlite3Fts3PendingTermsFlush(p);
  3304   3304   
  3305         -  if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>(nMinMerge/16) ){
         3305  +  if( rc==SQLITE_OK 
         3306  +   && p->nLeafAdd>(nMinMerge/16) 
         3307  +   && p->nAutoincrmerge && p->nAutoincrmerge!=0xff
         3308  +  ){
  3306   3309       int mxLevel = 0;              /* Maximum relative level value in db */
  3307   3310       int A;                        /* Incr-merge parameter A */
  3308   3311   
  3309   3312       rc = sqlite3Fts3MaxLevel(p, &mxLevel);
  3310   3313       assert( rc==SQLITE_OK || mxLevel==0 );
  3311   3314       A = p->nLeafAdd * mxLevel;
  3312   3315       A += (A/2);
  3313         -    if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8);
         3316  +    if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge);
  3314   3317     }
  3315   3318     sqlite3Fts3SegmentsClose(p);
  3316   3319     return rc;
  3317   3320   }
  3318   3321   
  3319   3322   /*
  3320   3323   ** If it is currently unknown whether or not the FTS table has an %_stat

Changes to ext/fts3/fts3Int.h.

   206    206     const char *zName;              /* virtual table name */
   207    207     int nColumn;                    /* number of named columns in virtual table */
   208    208     char **azColumn;                /* column names.  malloced */
   209    209     u8 *abNotindexed;               /* True for 'notindexed' columns */
   210    210     sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */
   211    211     char *zContentTbl;              /* content=xxx option, or NULL */
   212    212     char *zLanguageid;              /* languageid=xxx option, or NULL */
   213         -  u8 bAutoincrmerge;              /* True if automerge=1 */
          213  +  int nAutoincrmerge;             /* Value configured by 'automerge' */
   214    214     u32 nLeafAdd;                   /* Number of leaf blocks added this trans */
   215    215   
   216    216     /* Precompiled statements used by the implementation. Each of these 
   217    217     ** statements is run and reset within a single virtual table API call. 
   218    218     */
   219    219     sqlite3_stmt *aStmt[40];
   220    220   

Changes to ext/fts3/fts3_write.c.

  2967   2967       pCsr->aBuffer = 0;
  2968   2968     }
  2969   2969   }
  2970   2970   
  2971   2971   /*
  2972   2972   ** Decode the "end_block" field, selected by column iCol of the SELECT 
  2973   2973   ** statement passed as the first argument. 
         2974  +**
         2975  +** The "end_block" field may contain either an integer, or a text field
         2976  +** containing the text representation of two non-negative integers separated 
         2977  +** by one or more space (0x20) characters. In the first case, set *piEndBlock 
         2978  +** to the integer value and *pnByte to zero before returning. In the second, 
         2979  +** set *piEndBlock to the first value and *pnByte to the second.
  2974   2980   */
  2975   2981   static void fts3ReadEndBlockField(
  2976   2982     sqlite3_stmt *pStmt, 
  2977   2983     int iCol, 
  2978         -  i64 *piEndBlock, 
         2984  +  i64 *piEndBlock,
  2979   2985     i64 *pnByte
  2980   2986   ){
  2981   2987     const unsigned char *zText = sqlite3_column_text(pStmt, iCol);
  2982   2988     if( zText ){
  2983   2989       int i;
         2990  +    int iMul = 1;
  2984   2991       i64 iVal = 0;
  2985   2992       for(i=0; zText[i]>='0' && zText[i]<='9'; i++){
  2986   2993         iVal = iVal*10 + (zText[i] - '0');
  2987   2994       }
  2988   2995       *piEndBlock = iVal;
  2989   2996       while( zText[i]==' ' ) i++;
  2990   2997       iVal = 0;
         2998  +    if( zText[i]=='-' ){
         2999  +      i++;
         3000  +      iMul = -1;
         3001  +    }
  2991   3002       for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){
  2992   3003         iVal = iVal*10 + (zText[i] - '0');
  2993   3004       }
  2994         -    *pnByte = iVal;
         3005  +    *pnByte = (iVal * (i64)iMul);
  2995   3006     }
  2996   3007   }
  2997   3008   
  2998   3009   
  2999   3010   /*
  3000   3011   ** A segment of size nByte bytes has just been written to absolute level
  3001   3012   ** iAbsLevel. Promote any segments that should be promoted as a result.
................................................................................
  3007   3018   ){
  3008   3019     int rc = SQLITE_OK;
  3009   3020     sqlite3_stmt *pRange;
  3010   3021   
  3011   3022     rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0);
  3012   3023   
  3013   3024     if( rc==SQLITE_OK ){
  3014         -    int bOk = 1;
         3025  +    int bOk = 0;
  3015   3026       int iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1;
         3027  +    i64 nLimit = (nByte*3)/2;
  3016   3028   
         3029  +    /* Loop through all entries in the %_segdir table corresponding to 
         3030  +    ** segments in this index on levels greater than iAbsLevel. If there is
         3031  +    ** at least one such segment, and it is possible to determine that all 
         3032  +    ** such segments are smaller than nLimit bytes in size, they will be 
         3033  +    ** promoted to level iAbsLevel.  */
  3017   3034       sqlite3_bind_int(pRange, 1, iAbsLevel+1);
  3018   3035       sqlite3_bind_int(pRange, 2, iLast);
  3019   3036       while( SQLITE_ROW==sqlite3_step(pRange) ){
  3020   3037         i64 nSize, dummy;
  3021   3038         fts3ReadEndBlockField(pRange, 2, &dummy, &nSize);
  3022         -      if( nSize>nByte ){
         3039  +      if( nSize<=0 || nSize>nLimit ){
         3040  +        /* If nSize==0, then the %_segdir.end_block field does not not 
         3041  +        ** contain a size value. This happens if it was written by an
         3042  +        ** old version of FTS. In this case it is not possible to determine
         3043  +        ** the size of the segment, and so segment promotion does not
         3044  +        ** take place.  */
  3023   3045           bOk = 0;
  3024   3046           break;
  3025   3047         }
         3048  +      bOk = 1;
  3026   3049       }
  3027   3050       rc = sqlite3_reset(pRange);
  3028   3051   
  3029   3052       if( bOk ){
  3030   3053         int iIdx = 0;
  3031   3054         sqlite3_stmt *pUpdate1;
  3032   3055         sqlite3_stmt *pUpdate2;
................................................................................
  3035   3058           rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0);
  3036   3059         }
  3037   3060         if( rc==SQLITE_OK ){
  3038   3061           rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0);
  3039   3062         }
  3040   3063   
  3041   3064         if( rc==SQLITE_OK ){
         3065  +
         3066  +        /* Loop through all %_segdir entries for segments in this index with
         3067  +        ** levels equal to or greater than iAbsLevel. As each entry is visited,
         3068  +        ** updated it to set (level = -1) and (idx = N), where N is 0 for the
         3069  +        ** oldest segment in the range, 1 for the next oldest, and so on.
         3070  +        **
         3071  +        ** In other words, move all segments being promoted to level -1,
         3072  +        ** setting the "idx" fields as appropriate to keep them in the same
         3073  +        ** order. The contents of level -1 (which is never used, except
         3074  +        ** transiently here), will be moved back to level iAbsLevel below.  */
  3042   3075           sqlite3_bind_int(pRange, 1, iAbsLevel);
  3043   3076           while( SQLITE_ROW==sqlite3_step(pRange) ){
  3044   3077             sqlite3_bind_int(pUpdate1, 1, iIdx++);
  3045   3078             sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0));
  3046   3079             sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1));
  3047   3080             sqlite3_step(pUpdate1);
  3048   3081             rc = sqlite3_reset(pUpdate1);
................................................................................
  3052   3085             }
  3053   3086           }
  3054   3087         }
  3055   3088         if( rc==SQLITE_OK ){
  3056   3089           rc = sqlite3_reset(pRange);
  3057   3090         }
  3058   3091   
         3092  +      /* Move level -1 to level iAbsLevel */
  3059   3093         if( rc==SQLITE_OK ){
  3060   3094           sqlite3_bind_int(pUpdate2, 1, iAbsLevel);
  3061   3095           sqlite3_step(pUpdate2);
  3062   3096           rc = sqlite3_reset(pUpdate2);
  3063   3097         }
  3064   3098       }
  3065   3099     }
................................................................................
  3184   3218     }
  3185   3219     sqlite3Fts3PendingTermsClear(p);
  3186   3220   
  3187   3221     /* Determine the auto-incr-merge setting if unknown.  If enabled,
  3188   3222     ** estimate the number of leaf blocks of content to be written
  3189   3223     */
  3190   3224     if( rc==SQLITE_OK && p->bHasStat
  3191         -   && p->bAutoincrmerge==0xff && p->nLeafAdd>0
         3225  +   && p->nAutoincrmerge==0xff && p->nLeafAdd>0
  3192   3226     ){
  3193   3227       sqlite3_stmt *pStmt = 0;
  3194   3228       rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0);
  3195   3229       if( rc==SQLITE_OK ){
  3196   3230         sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE);
  3197   3231         rc = sqlite3_step(pStmt);
  3198         -      p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0));
         3232  +      p->nAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0));
         3233  +      if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8;
  3199   3234         rc = sqlite3_reset(pStmt);
  3200   3235       }
  3201   3236     }
  3202   3237     return rc;
  3203   3238   }
  3204   3239   
  3205   3240   /*
................................................................................
  4101   4136       /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */
  4102   4137       sqlite3_bind_int64(pSelect, 1, iAbsLevel+1);
  4103   4138       sqlite3_bind_int(pSelect, 2, iIdx);
  4104   4139       if( sqlite3_step(pSelect)==SQLITE_ROW ){
  4105   4140         iStart = sqlite3_column_int64(pSelect, 1);
  4106   4141         iLeafEnd = sqlite3_column_int64(pSelect, 2);
  4107   4142         fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData);
         4143  +      if( pWriter->nLeafData<0 ){
         4144  +        pWriter->nLeafData = pWriter->nLeafData * -1;
         4145  +      }
  4108   4146         nRoot = sqlite3_column_bytes(pSelect, 4);
  4109   4147         aRoot = sqlite3_column_blob(pSelect, 4);
  4110   4148       }else{
  4111   4149         return sqlite3_reset(pSelect);
  4112   4150       }
  4113   4151   
  4114   4152       /* Check for the zero-length marker in the %_segments table */
................................................................................
  4824   4862             if( nSeg!=0 ){
  4825   4863               bDirtyHint = 1;
  4826   4864               fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc);
  4827   4865             }
  4828   4866           }
  4829   4867         }
  4830   4868   
         4869  +      if( nSeg!=0 ){
         4870  +        pWriter->nLeafData = pWriter->nLeafData * -1;
         4871  +      }
  4831   4872         fts3IncrmergeRelease(p, pWriter, &rc);
  4832   4873         if( nSeg==0 ){
  4833   4874           fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData);
  4834   4875         }
  4835   4876       }
  4836   4877   
  4837   4878       sqlite3Fts3SegReaderFinish(pCsr);
................................................................................
  4914   4955   */
  4915   4956   static int fts3DoAutoincrmerge(
  4916   4957     Fts3Table *p,                   /* FTS3 table handle */
  4917   4958     const char *zParam              /* Nul-terminated string containing boolean */
  4918   4959   ){
  4919   4960     int rc = SQLITE_OK;
  4920   4961     sqlite3_stmt *pStmt = 0;
  4921         -  p->bAutoincrmerge = fts3Getint(&zParam)!=0;
         4962  +  p->nAutoincrmerge = fts3Getint(&zParam);
         4963  +  if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){
         4964  +    p->nAutoincrmerge = 8;
         4965  +  }
  4922   4966     if( !p->bHasStat ){
  4923   4967       assert( p->bFts4==0 );
  4924   4968       sqlite3Fts3CreateStatTable(&rc, p);
  4925   4969       if( rc ) return rc;
  4926   4970     }
  4927   4971     rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0);
  4928   4972     if( rc ) return rc;
  4929   4973     sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE);
  4930         -  sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge);
         4974  +  sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge);
  4931   4975     sqlite3_step(pStmt);
  4932   4976     rc = sqlite3_reset(pStmt);
  4933   4977     return rc;
  4934   4978   }
  4935   4979   
  4936   4980   /*
  4937   4981   ** Return a 64-bit checksum for the FTS index entry specified by the

Changes to test/fts4growth.test.

   127    127   
   128    128   do_execsql_test 2.5 { 
   129    129     SELECT end_block FROM x2_segdir WHERE level=3;
   130    130     INSERT INTO x2(x2) VALUES('merge=4,4');
   131    131     SELECT end_block FROM x2_segdir WHERE level=3;
   132    132     INSERT INTO x2(x2) VALUES('merge=4,4');
   133    133     SELECT end_block FROM x2_segdir WHERE level=3;
   134         -} {{3828 3430} {3828 10191} {3828 14109}}
          134  +} {{3828 -3430} {3828 -10191} {3828 -14109}}
   135    135   
   136    136   do_execsql_test 2.6 {
   137    137     SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE 
   138    138       blockid BETWEEN start_block AND leaves_end_block
   139    139       AND level=3
   140    140   } {14109}
   141    141   
................................................................................
   231    231   
   232    232   do_test 3.3.2 {
   233    233     insert_doc 12
   234    234     execsql { SELECT level, idx, second(end_block) FROM x3_segdir WHERE level=1 }
   235    235   } {1 0 412}
   236    236   
   237    237   #--------------------------------------------------------------------------
          238  +# Check a theory on a bug in fts4 - that segments with idx==0 were not 
          239  +# being incrementally merged correctly. Theory turned out to be false.
          240  +#
   238    241   do_execsql_test 4.1 {
   239    242     DROP TABLE IF EXISTS x4;
   240    243     DROP TABLE IF EXISTS t1;
   241    244     CREATE TABLE t1(docid, words);
   242    245     CREATE VIRTUAL TABLE x4 USING fts4(words);
   243    246   }
   244    247   do_test 4.2 {
................................................................................
   252    255   } {0 0 117483 0 1 118006}
   253    256   
   254    257   do_execsql_test 4.4 {
   255    258     INSERT INTO x4(x4) VALUES('merge=10,2');
   256    259     SELECT count(*) FROM x4_segdir;
   257    260   } {3}
   258    261   
   259         -breakpoint
   260    262   do_execsql_test 4.5 {
   261    263     INSERT INTO x4(x4) VALUES('merge=10,2');
   262    264     SELECT count(*) FROM x4_segdir;
   263    265   } {3}
   264    266   
   265         -if 0 {
   266         -do_execsql_test 3.1 {
          267  +do_execsql_test 4.6 {
          268  +  INSERT INTO x4(x4) VALUES('merge=1000,2');
          269  +  SELECT count(*) FROM x4_segdir;
          270  +} {1}
          271  +
          272  +
          273  +
          274  +#--------------------------------------------------------------------------
          275  +# Check that segments are not promoted if the "end_block" field does not
          276  +# contain a size.
          277  +#
          278  +do_execsql_test 5.1 {
   267    279     DROP TABLE IF EXISTS x2;
   268    280     DROP TABLE IF EXISTS t1;
   269    281     CREATE TABLE t1(docid, words);
   270    282     CREATE VIRTUAL TABLE x2 USING fts4;
   271    283   }
   272    284   fts_kjv_genesis 
   273    285   
          286  +proc first {L} {lindex $L 0}
          287  +db func first first
          288  +
          289  +do_test 5.2 {
          290  +  foreach r [db eval { SELECT rowid FROM t1 }] {
          291  +    execsql {
          292  +      INSERT INTO x2(docid, content) SELECT docid, words FROM t1 WHERE rowid=$r
          293  +    }
          294  +  }
          295  +  foreach d [db eval { SELECT docid FROM t1 LIMIT -1 OFFSET 20 }] {
          296  +    execsql { DELETE FROM x2 WHERE docid = $d }
          297  +  }
          298  +
          299  +  execsql {
          300  +    INSERT INTO x2(x2) VALUES('optimize');
          301  +    SELECT level, idx, end_block FROM x2_segdir
          302  +  }
          303  +} {2 0 {752 1926}}
          304  +
          305  +do_execsql_test 5.3 {
          306  +  UPDATE x2_segdir SET end_block = CAST( first(end_block) AS INTEGER );
          307  +  SELECT end_block, typeof(end_block) FROM x2_segdir;
          308  +} {752 integer}
          309  +
          310  +do_execsql_test 5.4 {
          311  +  INSERT INTO x2 SELECT words FROM t1 LIMIT 50;
          312  +  SELECT level, idx, end_block FROM x2_segdir
          313  +} {2 0 752 0 0 {758 5174}}
          314  +
          315  +do_execsql_test 5.5 {
          316  +  UPDATE x2_segdir SET end_block = end_block || ' 1926' WHERE level=2;
          317  +  INSERT INTO x2 SELECT words FROM t1 LIMIT 40;
          318  +  SELECT level, idx, end_block FROM x2_segdir
          319  +} {0 0 {752 1926} 0 1 {758 5174} 0 2 {763 4170}}
          320  +
   274    321   proc t1_to_x2 {} {
   275    322     foreach id [db eval {SELECT docid FROM t1 LIMIT 2}] {
   276    323       execsql {
   277    324         DELETE FROM x2 WHERE docid=$id;
   278    325         INSERT INTO x2(docid, content) SELECT $id, words FROM t1 WHERE docid=$id;
   279    326       }
   280    327     }
   281    328   }
          329  +
          330  +#--------------------------------------------------------------------------
          331  +# Check that segments created by auto-merge are not promoted until they
          332  +# are completed.
          333  +#
          334  +
          335  +do_execsql_test 6.1 {
          336  +  CREATE VIRTUAL TABLE x5 USING fts4;
          337  +  INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 0;
          338  +  INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 25;
          339  +  INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 50;
          340  +  INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 75;
          341  +  SELECT count(*) FROM x5_segdir
          342  +} {4}
          343  +
          344  +do_execsql_test 6.2 {
          345  +  INSERT INTO x5(x5) VALUES('merge=2,4');
          346  +  SELECT level, idx, end_block FROM x5_segdir;
          347  +} {0 0 {10 9216} 0 1 {21 9330} 0 2 {31 8850} 0 3 {40 8689} 1 0 {1320 -3117}}
          348  +
          349  +do_execsql_test 6.3 {
          350  +  INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 100;
          351  +  SELECT level, idx, end_block FROM x5_segdir;
          352  +} {
          353  +  0 0 {10 9216} 0 1 {21 9330} 0 2 {31 8850} 
          354  +  0 3 {40 8689} 1 0 {1320 -3117} 0 4 {1329 8297}
          355  +}
          356  +
          357  +do_execsql_test 6.4 {
          358  +  INSERT INTO x5(x5) VALUES('merge=200,4');
          359  +  SELECT level, idx, end_block FROM x5_segdir;
          360  +} {0 0 {1329 8297} 1 0 {1320 28009}}
          361  +
          362  +do_execsql_test 6.5 {
          363  +  INSERT INTO x5 SELECT words FROM t1;
          364  +  SELECT level, idx, end_block FROM x5_segdir;
          365  +} {
          366  +  0 1 {1329 8297} 0 0 {1320 28009} 0 2 {1449 118006}
          367  +}
   282    368   
   283    369   #do_test 3.2 {
   284    370     #t1_to_x2
   285    371     #execsql {SELECT level, count(*) FROM x2_segdir GROUP BY level}
   286    372   #} {0 13 1 15 2 5}
   287    373   
   288         -proc second {x} { lindex $x 1 }
   289         -db func second second
   290         -for {set i 0} {$i <1000} {incr i} {
   291         -  t1_to_x2
   292         -  db eval {
   293         -    SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level;
   294         -  } {
   295         -    puts "$i.$level: $c"
   296         -  }
   297         -}
   298         -}
   299         -
          374  +#proc second {x} { lindex $x 1 }
          375  +#db func second second
          376  +#for {set i 0} {$i <1000} {incr i} {
          377  +#  t1_to_x2
          378  +#  db eval {
          379  +#    SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level;
          380  +#  } {
          381  +#    puts "$i.$level: $c"
          382  +#  }
          383  +#}
   300    384   
   301    385   finish_test
   302         -
   303    386   

Added test/fts4growth2.test.

            1  +# 2014 May 12
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS4 module.
           13  +#
           14  +#
           15  +
           16  +set testdir [file dirname $argv0]
           17  +source $testdir/tester.tcl
           18  +set testprefix fts4growth
           19  +
           20  +# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           21  +ifcapable !fts3 {
           22  +  finish_test
           23  +  return
           24  +}
           25  +
           26  +source $testdir/genesis.tcl
           27  +
           28  +do_execsql_test 1.0 { CREATE TABLE t1(docid, words); }
           29  +fts_kjv_genesis 
           30  +
           31  +proc tt {val} {
           32  +  execsql {
           33  +    BEGIN;
           34  +      DELETE FROM x1 
           35  +        WHERE docid IN (SELECT docid FROM t1 WHERE (rowid-1)%4==$val+0);
           36  +      INSERT INTO x1(docid, content) 
           37  +        SELECT docid, words FROM t1 WHERE (rowid%4)==$val+0;
           38  +    COMMIT;
           39  +  }
           40  +}
           41  +
           42  +do_execsql_test 1.1 {
           43  +  CREATE VIRTUAL TABLE x1 USING fts4;
           44  +  INSERT INTO x1(x1) VALUES('automerge=2');
           45  +}
           46  +
           47  +do_test 1.2 {
           48  +  for {set i 0} {$i < 100} {incr i} {
           49  +    tt 0 ; tt 1 ; tt 2 ; tt 3
           50  +  }
           51  +  execsql { 
           52  +    SELECT max(level) FROM x1_segdir; 
           53  +    SELECT count(*) FROM x1_segdir WHERE level=4;
           54  +  }
           55  +} {4 1}
           56  +
           57  +do_test 1.3 {
           58  +  for {set i 0} {$i < 100} {incr i} {
           59  +    tt 0 ; tt 1 ; tt 2 ; tt 3
           60  +  }
           61  +  execsql { 
           62  +    SELECT max(level) FROM x1_segdir; 
           63  +    SELECT count(*) FROM x1_segdir WHERE level=4;
           64  +  }
           65  +} {4 1}
           66  +
           67  +#-------------------------------------------------------------------------
           68  +#
           69  +do_execsql_test 2.1 {
           70  +  DELETE FROM t1 WHERE rowid>16;
           71  +  DROP TABLE IF EXISTS x1;
           72  +  CREATE VIRTUAL TABLE x1 USING fts4;
           73  +}
           74  +
           75  +db func second second
           76  +proc second {L} {lindex $L 1}
           77  +
           78  +for {set tn 0} {$tn < 40} {incr tn} {
           79  +  do_test 2.2.$tn {
           80  +    for {set i 0} {$i < 100} {incr i} {
           81  +      tt 0 ; tt 1 ; tt 2 ; tt 3
           82  +    }
           83  +    execsql { SELECT max(level) FROM x1_segdir }
           84  +  } {1}
           85  +}
           86  +
           87  +
           88  +finish_test
           89  +

Changes to test/permutations.test.

   108    108     savepoint4.test savepoint6.test select9.test 
   109    109     speed1.test speed1p.test speed2.test speed3.test speed4.test 
   110    110     speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test
   111    111     thread003.test thread004.test thread005.test trans2.test vacuum3.test 
   112    112     incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test
   113    113     vtab_err.test walslow.test walcrash.test walcrash3.test
   114    114     walthread.test rtree3.test indexfault.test securedel2.test
          115  +  fts3growth2.test
   115    116   }]
   116    117   if {[info exists ::env(QUICKTEST_INCLUDE)]} {
   117    118     set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)]
   118    119   }
   119    120   
   120    121   #############################################################################
   121    122   # Start of tests
................................................................................
   192    193     fts3fault.test fts3malloc.test fts3matchinfo.test
   193    194     fts3aux1.test fts3comp1.test fts3auto.test
   194    195     fts4aa.test fts4content.test
   195    196     fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test
   196    197     fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test
   197    198     fts4check.test fts4unicode.test fts4noti.test
   198    199     fts3varint.test
          200  +  fts4growth.test fts4growth2.test
   199    201   }
   200    202   
   201    203   test_suite "nofaultsim" -prefix "" -description {
   202    204     "Very" quick test suite. Runs in less than 5 minutes on a workstation. 
   203    205     This test suite is the same as the "quick" tests, except that some files
   204    206     that test malloc and IO errors are omitted.
   205    207   } -files [