/ Check-in [d648ddd9]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Enhance showfts5.tcl so that it can optionally display the number of terms in each segment.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: d648ddd93de039820f5abe064c7bc1318cd9d6b1
User & Date: dan 2015-09-04 11:13:00
Context
2015-09-04
12:54
Continue to support the (broken) legacy syntax of allowing strings for column names in CREATE INDEX statements and in UNIQUE and PRIMARY KEY constraints. check-in: 3d3df79b user: drh tags: trunk
11:13
Enhance showfts5.tcl so that it can optionally display the number of terms in each segment. check-in: d648ddd9 user: dan tags: trunk
10:31
Modify the fts5 custom tokenizer interface to permit synonym support. The fts5_api.iVersion value is now set to 2. Existing fts5 custom tokenizers (if there are such things) will need to be updated to use the new api version. check-in: 0b7e4ab8 user: dan tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Show Whitespace Changes Patch

Changes to ext/fts5/fts5_index.c.

  5100   5100   
  5101   5101   static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
  5102   5102     int iSegid, iHeight, iPgno, bDlidx;       /* Rowid compenents */
  5103   5103     fts5DecodeRowid(iKey, &iSegid, &bDlidx, &iHeight, &iPgno);
  5104   5104   
  5105   5105     if( iSegid==0 ){
  5106   5106       if( iKey==FTS5_AVERAGES_ROWID ){
  5107         -      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(averages) ");
         5107  +      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{averages} ");
  5108   5108       }else{
  5109         -      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(structure)");
         5109  +      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{structure}");
  5110   5110       }
  5111   5111     }
  5112   5112     else{
  5113         -    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(%ssegid=%d h=%d pgno=%d)",
         5113  +    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%ssegid=%d h=%d pgno=%d}",
  5114   5114           bDlidx ? "dlidx " : "", iSegid, iHeight, iPgno
  5115   5115       );
  5116   5116     }
  5117   5117   }
  5118   5118   
  5119   5119   static void fts5DebugStructure(
  5120   5120     int *pRc,                       /* IN/OUT: error code */

Changes to ext/fts5/test/fts5aa.test.

    47     47   }
    48     48   do_execsql_test 2.1 {
    49     49     INSERT INTO t1 VALUES('a b c', 'd e f');
    50     50   }
    51     51   
    52     52   do_test 2.2 {
    53     53     execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 }
    54         -} {/{\(structure\) {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* h=0 leaves=1..1}}}/}
           54  +} {/{{structure} {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* h=0 leaves=1..1}}}/}
    55     55   
    56     56   foreach w {a b c d e f} {
    57     57     do_execsql_test 2.3.$w.asc {
    58     58       SELECT rowid FROM t1 WHERE t1 MATCH $w;
    59     59     } {1}
    60     60     do_execsql_test 2.3.$w.desc {
    61     61       SELECT rowid FROM t1 WHERE t1 MATCH $w ORDER BY rowid DESC;

Changes to ext/fts5/test/fts5ah.test.

    86     86     2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x + w'   }  [list $W]
    87     87     3 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w' }  [list $W]
    88     88     4 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x' }  [list $Y]
    89     89   " {
    90     90   
    91     91     do_test 1.6.$tn.1 {
    92     92       set n [execsql_reads $q]
    93         -    puts -nonewline "(n=$n nReadX=$nReadX)"
           93  +    #puts -nonewline "(n=$n nReadX=$nReadX)"
    94     94       expr {$n < ($nReadX / 8)}
    95     95     } {1}
    96     96   
    97     97     do_test 1.6.$tn.2 {
    98     98       set n [execsql_reads "$q ORDER BY rowid DESC"]
    99         -    puts -nonewline "(n=$n nReadX=$nReadX)"
           99  +    #puts -nonewline "(n=$n nReadX=$nReadX)"
   100    100       expr {$n < ($nReadX / 8)}
   101    101     } {1}
   102    102   
   103    103     do_execsql_test 1.6.$tn.3 $q [lsort -int -incr $res]
   104    104     do_execsql_test 1.6.$tn.4 "$q ORDER BY rowid DESC" [lsort -int -decr $res]
   105    105   }
   106    106   

Changes to ext/fts5/tool/showfts5.tcl.

     1      1   
     2      2   
     3      3   
     4      4   #-------------------------------------------------------------------------
     5      5   # Process command line arguments.
     6      6   #
     7      7   proc usage {} {
     8         -  puts stderr "usage: $::argv0 database table"
            8  +  puts stderr "usage: $::argv0 ?OPTIONS? database table"
            9  +  puts stderr ""
           10  +  puts stderr "  -nterm                (count number of terms in each segment)"
     9     11     puts stderr ""
    10     12     exit 1
    11     13   }
    12         -if {[llength $argv]!=2} usage
    13         -set database [lindex $argv 0]
    14         -set tbl [lindex $argv 1]
           14  +
           15  +set O(nterm) 0
           16  +
           17  +if {[llength $argv]<2} usage
           18  +foreach a [lrange $argv 0 end-2] {
           19  +  switch -- $a {
           20  +    -nterm {
           21  +      set O(nterm) 1
           22  +    }
           23  +
           24  +    default {
           25  +      usage
           26  +    }
           27  +  }
           28  +}
           29  +
           30  +set database [lindex $argv end-1]
           31  +set tbl [lindex $argv end]
           32  +
    15     33   
           34  +#-------------------------------------------------------------------------
           35  +# Count the number of terms in each segment of fts5 table $tbl. Store the
           36  +# counts in the array variable in the parent context named by parameter
           37  +# $arrayname, indexed by segment-id. Example:
           38  +#
           39  +#   count_terms fts_tbl A
           40  +#   foreach {k v} [array get A] { puts "segid=$k nTerm=$v" }
           41  +#
           42  +proc count_terms {tbl arrayname} {
           43  +  upvar A $arrayname
           44  +  array unset A
           45  +  db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data" {
           46  +    set desc [lindex $d 0]
           47  +    if {[regexp {^segid=([0-9]*)} $desc -> id]} {
           48  +      foreach i [lrange $d 1 end] {
           49  +        if {[string match {term=*} $i]} { incr A($id) }
           50  +      }
           51  +    }
           52  +  }
           53  +}
    16     54   
    17     55   
    18     56   #-------------------------------------------------------------------------
    19     57   # Start of main program.
    20     58   #
    21     59   sqlite3 db $database
    22     60   catch { load_static_extension db fts5 }
           61  +
           62  +if {$O(nterm)} { count_terms $tbl A }
    23     63   
    24     64   db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id=10" {
    25     65     foreach lvl [lrange $d 1 end] {
    26     66       puts [lrange $lvl 0 2]
           67  +
    27     68       foreach seg [lrange $lvl 3 end] {
    28         -      puts "        $seg"
           69  +      if {$::O(nterm)} {
           70  +        regexp {^id=([0-9]*)} $seg -> id
           71  +        set nTerm 0
           72  +        catch { set nTerm $A($id) }
           73  +        puts [format "        % -28s    nTerm=%d" $seg $nTerm]
           74  +      } else {
           75  +        puts [format "        % -28s" $seg]
           76  +      }
    29     77       }
    30     78     }
    31     79   }
    32     80   
    33     81   
    34     82   
    35     83   
    36     84