/ Check-in [e749be56]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the fts5 'optimize' command.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: e749be563d8e738af113bd301770e2f22763ab77
User & Date: dan 2015-01-07 19:33:11
Context
2015-01-10
20:34
Fix some documentation issues in fts5. check-in: 512e1bdb user: dan tags: fts5
2015-01-07
19:33
Add the fts5 'optimize' command. check-in: e749be56 user: dan tags: fts5
17:11
Add the 'rebuild' and 'delete-all' commands. check-in: 0cb2fed5 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.c.

  1076   1076         fts5SetVtabError(pTab, 
  1077   1077             "'rebuild' may not be used with a contentless fts5 table"
  1078   1078         );
  1079   1079         rc = SQLITE_ERROR;
  1080   1080       }else{
  1081   1081         rc = sqlite3Fts5StorageRebuild(pTab->pStorage);
  1082   1082       }
         1083  +  }else if( 0==sqlite3_stricmp("optimize", z) ){
         1084  +    rc = sqlite3Fts5StorageOptimize(pTab->pStorage);
  1083   1085     }else if( 0==sqlite3_stricmp("integrity-check", z) ){
  1084   1086       rc = sqlite3Fts5StorageIntegrity(pTab->pStorage);
  1085   1087     }else{
  1086   1088       rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, z, pVal, &bError);
  1087   1089       if( rc==SQLITE_OK ){
  1088   1090         if( bError ){
  1089   1091           rc = SQLITE_ERROR;

Changes to ext/fts5/fts5Int.h.

   337    337   /*
   338    338   ** Return the total number of entries read from the %_data table by 
   339    339   ** this connection since it was created.
   340    340   */
   341    341   int sqlite3Fts5IndexReads(Fts5Index *p);
   342    342   
   343    343   int sqlite3Fts5IndexReinit(Fts5Index *p);
          344  +int sqlite3Fts5IndexOptimize(Fts5Index *p);
   344    345   
   345    346   /*
   346    347   ** End of interface to code in fts5_index.c.
   347    348   **************************************************************************/
   348    349   
   349    350   /**************************************************************************
   350    351   ** Interface to code in fts5_hash.c. 
................................................................................
   421    422   
   422    423   int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*);
   423    424   
   424    425   int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**);
   425    426   
   426    427   int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
   427    428   int sqlite3Fts5StorageRebuild(Fts5Storage *p);
          429  +int sqlite3Fts5StorageOptimize(Fts5Storage *p);
   428    430   
   429    431   /*
   430    432   ** End of interface to code in fts5_storage.c.
   431    433   **************************************************************************/
   432    434   
   433    435   
   434    436   /**************************************************************************

Changes to ext/fts5/fts5_index.c.

    37     37   **     large doclists with very small doclists.
    38     38   **
    39     39   **   * extra fields in the "structure record" record the state of ongoing
    40     40   **     incremental merge operations.
    41     41   **
    42     42   */
    43     43   
           44  +#define FTS5_OPT_WORK_UNIT  1000  /* Number of leaf pages per optimize step */
    44     45   #define FTS5_WORK_UNIT      64    /* Number of leaf pages in unit of work */
    45     46   #define FTS5_CRISIS_MERGE   16    /* Maximum number of segments to merge */
    46     47   
    47     48   #define FTS5_MIN_DLIDX_SIZE  4    /* Add dlidx if this many empty pages */
    48     49   
    49     50   /*
    50     51   ** Details:
................................................................................
  3159   3160     ){
  3160   3161       fts5IndexMergeLevel(p, iIdx, &pStruct, iLvl, 0);
  3161   3162       fts5StructurePromote(p, iLvl+1, pStruct);
  3162   3163       iLvl++;
  3163   3164     }
  3164   3165     *ppStruct = pStruct;
  3165   3166   }
         3167  +
         3168  +static int fts5IndexReturn(Fts5Index *p){
         3169  +  int rc = p->rc;
         3170  +  p->rc = SQLITE_OK;
         3171  +  return rc;
         3172  +}
  3166   3173   
  3167   3174   typedef struct Fts5FlushCtx Fts5FlushCtx;
  3168   3175   struct Fts5FlushCtx {
  3169   3176     Fts5Index *pIdx;
  3170   3177     Fts5SegWriter writer; 
  3171   3178   };
  3172   3179   
................................................................................
  3272   3279   
  3273   3280     /* Flush the terms and each prefix index to disk */
  3274   3281     for(i=0; i<=pConfig->nPrefix; i++){
  3275   3282       fts5FlushOneHash(p, i, &nLeaf);
  3276   3283     }
  3277   3284     p->nPendingData = 0;
  3278   3285   }
         3286  +
         3287  +
         3288  +int sqlite3Fts5IndexOptimize(Fts5Index *p){
         3289  +  Fts5Config *pConfig = p->pConfig;
         3290  +  int i;
         3291  +
         3292  +  fts5IndexFlush(p);
         3293  +  for(i=0; i<=pConfig->nPrefix; i++){
         3294  +    Fts5Structure *pStruct = fts5StructureRead(p, i);
         3295  +    Fts5Structure *pNew = 0;
         3296  +    int nSeg = 0;
         3297  +    if( pStruct ){
         3298  +      nSeg = fts5StructureCountSegments(pStruct);
         3299  +      if( nSeg>1 ){
         3300  +        int nByte = sizeof(Fts5Structure);
         3301  +        nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
         3302  +        pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
         3303  +      }
         3304  +    }
         3305  +    if( pNew ){
         3306  +      Fts5StructureLevel *pLvl;
         3307  +      int nByte = nSeg * sizeof(Fts5StructureSegment);
         3308  +      pNew->nLevel = pStruct->nLevel+1;
         3309  +      pNew->nWriteCounter = pStruct->nWriteCounter;
         3310  +      pLvl = &pNew->aLevel[pStruct->nLevel];
         3311  +      pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
         3312  +      if( pLvl->aSeg ){
         3313  +        int iLvl, iSeg;
         3314  +        int iSegOut = 0;
         3315  +        for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
         3316  +          for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
         3317  +            pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg];
         3318  +            iSegOut++;
         3319  +          }
         3320  +        }
         3321  +        pLvl->nSeg = nSeg;
         3322  +      }else{
         3323  +        sqlite3_free(pNew);
         3324  +        pNew = 0;
         3325  +      }
         3326  +    }
         3327  +
         3328  +    if( pNew ){
         3329  +      int iLvl = pNew->nLevel-1;
         3330  +      while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){
         3331  +        int nRem = FTS5_OPT_WORK_UNIT;
         3332  +        fts5IndexMergeLevel(p, i, &pNew, iLvl, &nRem);
         3333  +      }
         3334  +
         3335  +      fts5StructureWrite(p, i, pNew);
         3336  +      fts5StructureRelease(pNew);
         3337  +    }
         3338  +
         3339  +    fts5StructureRelease(pStruct);
         3340  +  }
         3341  +
         3342  +  return fts5IndexReturn(p); 
         3343  +}
         3344  +
         3345  +
  3279   3346   
  3280   3347   /*
  3281   3348   ** Return a simple checksum value based on the arguments.
  3282   3349   */
  3283   3350   static u64 fts5IndexEntryCksum(
  3284   3351     i64 iRowid, 
  3285   3352     int iCol, 
................................................................................
  3784   3851       }
  3785   3852     }
  3786   3853   
  3787   3854     fts5StructureRelease(pStruct);
  3788   3855     sqlite3_free(aBuf);
  3789   3856   }
  3790   3857   
  3791         -static int fts5IndexReturn(Fts5Index *p){
  3792         -  int rc = p->rc;
  3793         -  p->rc = SQLITE_OK;
  3794         -  return rc;
  3795         -}
  3796         -
  3797   3858   /*
  3798   3859   ** Run internal checks to ensure that the FTS index (a) is internally 
  3799   3860   ** consistent and (b) contains entries for which the XOR of the checksums
  3800   3861   ** as calculated by fts5IndexEntryCksum() is cksum.
  3801   3862   **
  3802   3863   ** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
  3803   3864   ** checksum does not match. Return SQLITE_OK if all checks pass without

Changes to ext/fts5/fts5_storage.c.

   580    580   
   581    581     /* Write the averages record */
   582    582     if( rc==SQLITE_OK ){
   583    583       rc = fts5StorageSaveTotals(p);
   584    584     }
   585    585     return rc;
   586    586   }
          587  +
          588  +int sqlite3Fts5StorageOptimize(Fts5Storage *p){
          589  +  return sqlite3Fts5IndexOptimize(p->pIndex);
          590  +}
   587    591   
   588    592   /*
   589    593   ** Allocate a new rowid. This is used for "external content" tables when
   590    594   ** a NULL value is inserted into the rowid column. The new rowid is allocated
   591    595   ** by inserting a dummy row into the %_docsize table. The dummy will be
   592    596   ** overwritten later.
   593    597   */

Added ext/fts5/test/fts5optimize.test.

            1  +# 2014 Dec 20
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +#
           13  +
           14  +source [file join [file dirname [info script]] fts5_common.tcl]
           15  +set testprefix fts5optimize
           16  +
           17  +proc rnddoc {nWord} {
           18  +  set vocab {a b c d e f g h i j k l m n o p q r s t u v w x y z}
           19  +  set nVocab [llength $vocab]
           20  +  set ret [list]
           21  +  for {set i 0} {$i < $nWord} {incr i} {
           22  +    lappend ret [lindex $vocab [expr {int(rand() * $nVocab)}]]
           23  +  }
           24  +  return $ret
           25  +}
           26  +
           27  +
           28  +foreach {tn nStep} {
           29  +  1 2
           30  +  2 10
           31  +  3 50
           32  +  4 500
           33  +} {
           34  +if {$tn!=4} continue
           35  +  reset_db
           36  +  db func rnddoc rnddoc
           37  +  do_execsql_test 1.$tn.1 {
           38  +    CREATE VIRTUAL TABLE t1 USING fts5(x, y);
           39  +  }
           40  +  do_test 1.$tn.2 {
           41  +    for {set i 0} {$i < $nStep} {incr i} {
           42  +      execsql { INSERT INTO t1 VALUES( rnddoc(5), rnddoc(5) ) }
           43  +    }
           44  +  } {}
           45  +
           46  +  do_execsql_test 1.$tn.3 {
           47  +    INSERT INTO t1(t1) VALUES('integrity-check');
           48  +  }
           49  +
           50  +  do_execsql_test 1.$tn.4 {
           51  +    INSERT INTO t1(t1) VALUES('optimize');
           52  +  }
           53  +
           54  +  do_execsql_test 1.$tn.5 {
           55  +    INSERT INTO t1(t1) VALUES('integrity-check');
           56  +  }
           57  +}
           58  +
           59  +finish_test
           60  +