Index: ext/fts5/fts5_hash.c ================================================================== --- ext/fts5/fts5_hash.c +++ ext/fts5/fts5_hash.c @@ -74,10 +74,21 @@ a[0] = (0x80 | (u8)(iVal >> 21)); a[1] = (0x80 | (u8)(iVal >> 14)); a[2] = (0x80 | (u8)(iVal >> 7)); a[3] = (0x7F & (u8)(iVal)); } + +static int fts5Get4ByteVarint(u8 *a, int *pnVarint){ + int iRet = ((int)(a[0] & 0x7F) << 21) + ((int)(a[1] & 0x7F) << 14) + + ((int)(a[2] & 0x7F) << 7) + ((int)(a[3])); + *pnVarint = ( + (iRet & 0xFFFFFF80)==0 ? 1 : + (iRet & 0xFFFFC000)==0 ? 2 : + (iRet & 0xFFE00000)==0 ? 3 : 4 + ); + return iRet; +} /* ** Allocate a new hash table. */ int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){ @@ -396,14 +407,16 @@ /* Issue the xEntry callbacks */ while( rc==SQLITE_OK && iOffnData ){ i64 iDelta; /* Rowid delta value */ int nPoslist; /* Size of position list in bytes */ + int nVarint; iOff += getVarint(&pPtr[iOff], (u64*)&iDelta); iRowid += iDelta; - iOff += fts5GetVarint32(&pPtr[iOff], nPoslist); - rc = xEntry(pCtx, iRowid, &pPtr[iOff], nPoslist); + nPoslist = fts5Get4ByteVarint(&pPtr[iOff], &nVarint); + iOff += 4; + rc = xEntry(pCtx, iRowid, &pPtr[iOff-nVarint], nPoslist+nVarint); iOff += nPoslist; } /* Issue the term-done callback */ if( rc==SQLITE_OK ) rc = xTermDone(pCtx); Index: ext/fts5/fts5_index.c ================================================================== --- ext/fts5/fts5_index.c +++ ext/fts5/fts5_index.c @@ -3389,17 +3389,22 @@ const u8 *aPoslist, int nPoslist ){ Fts5FlushCtx *p = (Fts5FlushCtx*)pCtx; Fts5Index *pIdx = p->pIdx; + +#ifdef SQLITE_DEBUG + /* The poslist-size varint should already be at the start of the + ** aPoslist/nPoslist buffer. This assert verifies that. */ + int n, i; + i = fts5GetVarint32(aPoslist, n); + assert( nPoslist==(n+i) ); +#endif /* Append the rowid itself */ fts5WriteAppendRowid(pIdx, &p->writer, iRowid); - /* Append the size of the position list in bytes */ - fts5WriteAppendPoslistInt(pIdx, &p->writer, nPoslist); - /* And the poslist data */ fts5WriteAppendPoslistData(pIdx, &p->writer, aPoslist, nPoslist); return pIdx->rc; } Index: ext/fts5/tool/loadfts5.tcl ================================================================== --- ext/fts5/tool/loadfts5.tcl +++ ext/fts5/tool/loadfts5.tcl @@ -6,18 +6,27 @@ close $fd return $data } set ::nRow 0 +set ::nRowPerDot 1000 + proc load_hierachy {dir} { foreach f [glob -nocomplain -dir $dir *] { if {$::O(limit) && $::nRow>=$::O(limit)} break if {[file isdir $f]} { load_hierachy $f } else { db eval { INSERT INTO t1 VALUES($f, loadfile($f)) } incr ::nRow + + if {($::nRow % $::nRowPerDot)==0} { + puts -nonewline . + if {($::nRow % (65*$::nRowPerDot))==0} { puts "" } + flush stdout + } + } } } proc usage {} { @@ -79,11 +88,13 @@ sqlite3 db [lindex $argv end-1] db func loadfile loadfile db transaction { - db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok))" + catch { + db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok))" + } if {$O(automerge)>=0} { if {$O(vtab) == "fts5"} { db eval { INSERT INTO t1(t1, rank) VALUES('automerge', $O(automerge)) } } else { db eval { INSERT INTO t1(t1) VALUES('automerge=' || $O(automerge)) }