/ Check-in [1fffe51f]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Minor optimizations to fts5 writes.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 1fffe51fa92f1784365140d5b163ab6c690981ae
User & Date: dan 2015-01-31 15:23:44
Context
2015-02-02
09:40
Merge latest trunk changes with this branch. check-in: 76212f2c user: dan tags: fts5
2015-01-31
15:23
Minor optimizations to fts5 writes. check-in: 1fffe51f user: dan tags: fts5
2015-01-29
20:59
Fix some problems with transactions that both read and write an fts5 table. check-in: 0e225b15 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_hash.c.

    72     72   */
    73     73   static void fts5Put4ByteVarint(u8 *a, int iVal){
    74     74     a[0] = (0x80 | (u8)(iVal >> 21));
    75     75     a[1] = (0x80 | (u8)(iVal >> 14));
    76     76     a[2] = (0x80 | (u8)(iVal >>  7));
    77     77     a[3] = (0x7F & (u8)(iVal));
    78     78   }
           79  +
           80  +static int fts5Get4ByteVarint(u8 *a, int *pnVarint){
           81  +  int iRet = ((int)(a[0] & 0x7F) << 21) + ((int)(a[1] & 0x7F) << 14)
           82  +       + ((int)(a[2] & 0x7F) <<  7) + ((int)(a[3]));
           83  +  *pnVarint = (
           84  +      (iRet & 0xFFFFFF80)==0 ? 1 : 
           85  +      (iRet & 0xFFFFC000)==0 ? 2 :
           86  +      (iRet & 0xFFE00000)==0 ? 3 : 4
           87  +  );
           88  +  return iRet;
           89  +}
    79     90   
    80     91   /*
    81     92   ** Allocate a new hash table.
    82     93   */
    83     94   int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){
    84     95     int rc = SQLITE_OK;
    85     96     Fts5Hash *pNew;
................................................................................
   394    405           /* Issue the new-term callback */
   395    406           rc = xTerm(pCtx, pList->zKey, nKey);
   396    407   
   397    408           /* Issue the xEntry callbacks */
   398    409           while( rc==SQLITE_OK && iOff<pList->nData ){
   399    410             i64 iDelta;             /* Rowid delta value */
   400    411             int nPoslist;           /* Size of position list in bytes */
          412  +          int nVarint;
   401    413             iOff += getVarint(&pPtr[iOff], (u64*)&iDelta);
   402    414             iRowid += iDelta;
   403         -          iOff += fts5GetVarint32(&pPtr[iOff], nPoslist);
   404         -          rc = xEntry(pCtx, iRowid, &pPtr[iOff], nPoslist);
          415  +          nPoslist = fts5Get4ByteVarint(&pPtr[iOff], &nVarint);
          416  +          iOff += 4;
          417  +          rc = xEntry(pCtx, iRowid, &pPtr[iOff-nVarint], nPoslist+nVarint);
   405    418             iOff += nPoslist;
   406    419           }
   407    420   
   408    421           /* Issue the term-done callback */
   409    422           if( rc==SQLITE_OK ) rc = xTermDone(pCtx);
   410    423         }
   411    424         sqlite3_free(pList);

Changes to ext/fts5/fts5_index.c.

  3387   3387     void *pCtx, 
  3388   3388     i64 iRowid, 
  3389   3389     const u8 *aPoslist, 
  3390   3390     int nPoslist
  3391   3391   ){
  3392   3392     Fts5FlushCtx *p = (Fts5FlushCtx*)pCtx;
  3393   3393     Fts5Index *pIdx = p->pIdx;
         3394  +
         3395  +#ifdef SQLITE_DEBUG
         3396  +  /* The poslist-size varint should already be at the start of the 
         3397  +  ** aPoslist/nPoslist buffer. This assert verifies that. */
         3398  +  int n, i;
         3399  +  i = fts5GetVarint32(aPoslist, n);
         3400  +  assert( nPoslist==(n+i) );
         3401  +#endif
  3394   3402   
  3395   3403     /* Append the rowid itself */
  3396   3404     fts5WriteAppendRowid(pIdx, &p->writer, iRowid);
  3397   3405   
  3398         -  /* Append the size of the position list in bytes */
  3399         -  fts5WriteAppendPoslistInt(pIdx, &p->writer, nPoslist);
  3400         -
  3401   3406     /* And the poslist data */
  3402   3407     fts5WriteAppendPoslistData(pIdx, &p->writer, aPoslist, nPoslist);
  3403   3408     return pIdx->rc;
  3404   3409   }
  3405   3410   
  3406   3411   /*
  3407   3412   ** Flush the contents of in-memory hash table iHash to a new level-0 

Changes to ext/fts5/tool/loadfts5.tcl.

     4      4     set fd [open $f]
     5      5     set data [read $fd]
     6      6     close $fd
     7      7     return $data
     8      8   }
     9      9   
    10     10   set ::nRow 0
           11  +set ::nRowPerDot 1000
           12  +
    11     13   proc load_hierachy {dir} {
    12     14     foreach f [glob -nocomplain -dir $dir *] {
    13     15       if {$::O(limit) && $::nRow>=$::O(limit)} break
    14     16       if {[file isdir $f]} {
    15     17         load_hierachy $f
    16     18       } else {
    17     19         db eval { INSERT INTO t1 VALUES($f, loadfile($f)) }
    18     20         incr ::nRow
           21  +
           22  +      if {($::nRow % $::nRowPerDot)==0} {
           23  +        puts -nonewline .
           24  +        if {($::nRow % (65*$::nRowPerDot))==0} { puts "" }
           25  +        flush stdout
           26  +      }
           27  +
    19     28       }
    20     29     }
    21     30   }
    22     31   
    23     32   proc usage {} {
    24     33     puts stderr "Usage: $::argv0 ?SWITCHES? DATABASE PATH"
    25     34     puts stderr ""
................................................................................
    77     86     }
    78     87   }
    79     88   
    80     89   sqlite3 db [lindex $argv end-1]
    81     90   db func loadfile loadfile
    82     91   
    83     92   db transaction {
    84         -  db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok))"
           93  +  catch {
           94  +    db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok))"
           95  +  }
    85     96     if {$O(automerge)>=0} {
    86     97       if {$O(vtab) == "fts5"} {
    87     98         db eval { INSERT INTO t1(t1, rank) VALUES('automerge', $O(automerge)) }
    88     99       } else {
    89    100         db eval { INSERT INTO t1(t1) VALUES('automerge=' || $O(automerge)) }
    90    101       }
    91    102     }