/ Check-in [8358af36]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add a debugging function to print human-readable versions of fts5 detail=none leaf pages.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 8358af3658d888516cdef5f8c8d89e9bdee53f91
User & Date: dan 2016-01-18 09:08:56
Context
2016-01-18
13:18
Avoid unnecessary calls to memset() for a small performance improvement. check-in: 9e8c23ac user: drh tags: trunk
09:08
Add a debugging function to print human-readable versions of fts5 detail=none leaf pages. check-in: 8358af36 user: dan tags: trunk
00:46
Simplification of the VDBE bytecode for incremental blob I/O. check-in: d23849f6 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Changes to ext/fts5/fts5_index.c.

5899
5900
5901
5902
5903
5904
5905









































5906
5907
5908
5909
5910
5911
5912
....
5915
5916
5917
5918
5919
5920
5921

5922
5923
5924
5925
5926
5927
5928
....
5956
5957
5958
5959
5960
5961
5962
















































5963
5964
5965
5966
5967
5968
5969
....
6083
6084
6085
6086
6087
6088
6089








6090
6091
6092
6093
6094
6095
6096
      iDocid += iDelta;
      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid);
    }
  }

  return iOff;
}










































/*
** The implementation of user-defined scalar function fts5_decode().
*/
static void fts5DecodeFunction(
  sqlite3_context *pCtx,          /* Function call context */
  int nArg,                       /* Number of args (always 2) */
................................................................................
  i64 iRowid;                     /* Rowid for record being decoded */
  int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */
  const u8 *aBlob; int n;         /* Record to decode */
  u8 *a = 0;
  Fts5Buffer s;                   /* Build up text to return here */
  int rc = SQLITE_OK;             /* Return code */
  int nSpace = 0;


  assert( nArg==2 );
  memset(&s, 0, sizeof(Fts5Buffer));
  iRowid = sqlite3_value_int64(apVal[0]);

  /* Make a copy of the second argument (a blob) in aBlob[]. The aBlob[]
  ** copy is followed by FTS5_DATA_ZERO_PADDING 0x00 bytes, which prevents
................................................................................
    }
  }else if( iSegid==0 ){
    if( iRowid==FTS5_AVERAGES_ROWID ){
      fts5DecodeAverages(&rc, &s, a, n);
    }else{
      fts5DecodeStructure(&rc, &s, a, n);
    }
















































  }else{
    Fts5Buffer term;              /* Current term read from page */
    int szLeaf;                   /* Offset of pgidx in a[] */
    int iPgidxOff;
    int iPgidxPrev = 0;           /* Previous value read from pgidx */
    int iTermOff = 0;
    int iRowidOff = 0;
................................................................................
** If successful, SQLITE_OK is returned. If an error occurs, some other
** SQLite error code is returned instead.
*/
int sqlite3Fts5IndexInit(sqlite3 *db){
  int rc = sqlite3_create_function(
      db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0
  );








  if( rc==SQLITE_OK ){
    rc = sqlite3_create_function(
        db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
    );
  }
  return rc;
}







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>







5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
....
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
....
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
....
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
      iDocid += iDelta;
      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid);
    }
  }

  return iOff;
}

/*
** This function is part of the fts5_decode() debugging function. It is 
** only ever used with detail=none tables.
**
** Buffer (pData/nData) contains a doclist in the format used by detail=none
** tables. This function appends a human-readable version of that list to
** buffer pBuf.
**
** If *pRc is other than SQLITE_OK when this function is called, it is a
** no-op. If an OOM or other error occurs within this function, *pRc is
** set to an SQLite error code before returning. The final state of buffer
** pBuf is undefined in this case.
*/
static void fts5DecodeRowidList(
  int *pRc,                       /* IN/OUT: Error code */
  Fts5Buffer *pBuf,               /* Buffer to append text to */
  const u8 *pData, int nData      /* Data to decode list-of-rowids from */
){
  int i = 0;
  i64 iRowid = 0;

  while( i<nData ){
    const char *zApp = "";
    u64 iVal;
    i += sqlite3Fts5GetVarint(&pData[i], &iVal);
    iRowid += iVal;

    if( i<nData && pData[i]==0x00 ){
      i++;
      if( i<nData && pData[i]==0x00 ){
        i++;
        zApp = "+";
      }else{
        zApp = "*";
      }
    }

    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %lld%s", iRowid, zApp);
  }
}

/*
** The implementation of user-defined scalar function fts5_decode().
*/
static void fts5DecodeFunction(
  sqlite3_context *pCtx,          /* Function call context */
  int nArg,                       /* Number of args (always 2) */
................................................................................
  i64 iRowid;                     /* Rowid for record being decoded */
  int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */
  const u8 *aBlob; int n;         /* Record to decode */
  u8 *a = 0;
  Fts5Buffer s;                   /* Build up text to return here */
  int rc = SQLITE_OK;             /* Return code */
  int nSpace = 0;
  int eDetailNone = (sqlite3_user_data(pCtx)!=0);

  assert( nArg==2 );
  memset(&s, 0, sizeof(Fts5Buffer));
  iRowid = sqlite3_value_int64(apVal[0]);

  /* Make a copy of the second argument (a blob) in aBlob[]. The aBlob[]
  ** copy is followed by FTS5_DATA_ZERO_PADDING 0x00 bytes, which prevents
................................................................................
    }
  }else if( iSegid==0 ){
    if( iRowid==FTS5_AVERAGES_ROWID ){
      fts5DecodeAverages(&rc, &s, a, n);
    }else{
      fts5DecodeStructure(&rc, &s, a, n);
    }
  }else if( eDetailNone ){
    Fts5Buffer term;              /* Current term read from page */
    int szLeaf;
    int iPgidxOff = szLeaf = fts5GetU16(&a[2]);
    int iTermOff;
    int nKeep = 0;
    int iOff;

    memset(&term, 0, sizeof(Fts5Buffer));

    /* Decode any entries that occur before the first term. */
    if( szLeaf<n ){
      fts5FastGetVarint32(a, iPgidxOff, iTermOff);
    }else{
      iTermOff = szLeaf;
    }
    fts5DecodeRowidList(&rc, &s, &a[4], iTermOff-4);

    iOff = iTermOff;
    while( iOff<szLeaf ){
      int nAppend;

      /* Read the term data for the next term*/
      fts5FastGetVarint32(a, iOff, nAppend);
      term.n = nKeep;
      fts5BufferAppendBlob(&rc, &term, nAppend, &a[iOff]);
      sqlite3Fts5BufferAppendPrintf(
          &rc, &s, " term=%.*s", term.n, (const char*)term.p
      );
      iOff += nAppend;

      /* Figure out where the doclist for this term ends */
      if( iPgidxOff<n ){
        int nIncr;
        fts5FastGetVarint32(a, iPgidxOff, nIncr);
        iTermOff += nIncr;
      }else{
        iTermOff = szLeaf;
      }

      fts5DecodeRowidList(&rc, &s, &a[iOff], iTermOff-iOff);
      iOff = iTermOff;
      if( iOff<szLeaf ){
        fts5FastGetVarint32(a, iOff, nKeep);
      }
    }

    fts5BufferFree(&term);
  }else{
    Fts5Buffer term;              /* Current term read from page */
    int szLeaf;                   /* Offset of pgidx in a[] */
    int iPgidxOff;
    int iPgidxPrev = 0;           /* Previous value read from pgidx */
    int iTermOff = 0;
    int iRowidOff = 0;
................................................................................
** If successful, SQLITE_OK is returned. If an error occurs, some other
** SQLite error code is returned instead.
*/
int sqlite3Fts5IndexInit(sqlite3 *db){
  int rc = sqlite3_create_function(
      db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0
  );

  if( rc==SQLITE_OK ){
    rc = sqlite3_create_function(
        db, "fts5_decode_none", 2, 
        SQLITE_UTF8, (void*)db, fts5DecodeFunction, 0, 0
    );
  }

  if( rc==SQLITE_OK ){
    rc = sqlite3_create_function(
        db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
    );
  }
  return rc;
}

Changes to ext/fts5/test/fts5_common.tcl.

15
16
17
18
19
20
21






22
23
24
25
26
27
28
}
source $testdir/tester.tcl

catch { 
  sqlite3_fts5_may_be_corrupt 0 
  reset_db
}







proc fts5_test_poslist {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
    lappend res [string map {{ } .} [$cmd xInst $i]]
  }
  set res







>
>
>
>
>
>







15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
}
source $testdir/tester.tcl

catch { 
  sqlite3_fts5_may_be_corrupt 0 
  reset_db
}

# If SQLITE_ENABLE_FTS5 is not defined, skip this test.
ifcapable !fts5 {
  finish_test
  return
}

proc fts5_test_poslist {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
    lappend res [string map {{ } .} [$cmd xInst $i]]
  }
  set res

Changes to ext/fts5/test/fts5simple2.test.

15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
...
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
...
318
319
320
321
322
323
324
















325
326

# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}

if 1 {

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
  INSERT INTO t1 VALUES('a b c');
}
do_execsql_test 1.1 {
  SELECT rowid FROM t1('c a b')
} {1}
................................................................................
  SELECT rowid FROM t1('sqlite') ORDER BY rowid DESC;
} {}

do_test 15.4 {
  execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {}

}

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 16.0 {
  CREATE VIRTUAL TABLE t2 USING fts5(x, detail=none);
  BEGIN;
    INSERT INTO t2(rowid, x) VALUES(1, 'a b c');
................................................................................
} {}

do_execsql_test 16.2 {
  SELECT rowid FROM t2('b') ORDER BY rowid DESC
} {1000 456 1}

  
















finish_test








<
<







 







<
<







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


15
16
17
18
19
20
21


22
23
24
25
26
27
28
...
292
293
294
295
296
297
298


299
300
301
302
303
304
305
...
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338

# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}



do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
  INSERT INTO t1 VALUES('a b c');
}
do_execsql_test 1.1 {
  SELECT rowid FROM t1('c a b')
} {1}
................................................................................
  SELECT rowid FROM t1('sqlite') ORDER BY rowid DESC;
} {}

do_test 15.4 {
  execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {}



#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 16.0 {
  CREATE VIRTUAL TABLE t2 USING fts5(x, detail=none);
  BEGIN;
    INSERT INTO t2(rowid, x) VALUES(1, 'a b c');
................................................................................
} {}

do_execsql_test 16.2 {
  SELECT rowid FROM t2('b') ORDER BY rowid DESC
} {1000 456 1}


#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 16.0 {
  CREATE VIRTUAL TABLE t2 USING fts5(x, detail=none);
  BEGIN;
    INSERT INTO t2(rowid, x) VALUES(1, 'a b c');
    INSERT INTO t2(rowid, x) VALUES(456, 'a b c');
    INSERT INTO t2(rowid, x) VALUES(1000, 'a b c');
  COMMIT;
  UPDATE t2 SET x=x;
  DELETE FROM t2;
}

#db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM t2_data} {puts $r}
  
finish_test