SQLite

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)

#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE)
# define SQLITE_CORE 1
#endif

#include "fts3Int.h"

#include <assert.h>
#include <stdlib.h>
#include <stddef.h>
#include <stdio.h>
#include <string.h>

#include "fts3.h"


#ifndef SQLITE_CORE 
# include "sqlite3ext.h"
  SQLITE_EXTENSION_INIT1
#endif





/* TODO(shess) MAN, this thing needs some refactoring.  At minimum, it
** would be nice to order the file better, perhaps something along the
** lines of:
**
**  - utility functions

    return res;
  }
}

/*
** Values that may be used as the first parameter to fts3DoclistMerge().
*/

#define MERGE_NOT        2        /* D + D -> D */
#define MERGE_AND        3        /* D + D -> D */
#define MERGE_OR         4        /* D + D -> D */
#define MERGE_POS_OR     5        /* P + P -> P */
#define MERGE_PHRASE     6        /* P + P -> D */
#define MERGE_POS_PHRASE 7        /* P + P -> P */
#define MERGE_NEAR       8        /* P + P -> D */
#define MERGE_POS_NEAR   9        /* P + P -> P */

static int fts3DoclistMerge(
  int mergetype,                  /* One of the MERGE_XXX constants */
  int nParam1,
  int nParam2,
  char *aBuffer,                  /* Pre-allocated output buffer */
  int *pnBuffer,                  /* OUT: Bytes written to aBuffer */

*/
static int evalFts3Expr(
  Fts3Table *p,                   /* Virtual table handle */
  Fts3Expr *pExpr,                /* Parsed fts3 expression */
  char **paOut,                   /* OUT: Pointer to malloc'd result buffer */
  int *pnOut                      /* OUT: Size of buffer at *paOut */
){
  int rc = SQLITE_OK;             /* Return code */

  /* Zero the output parameters. */
  *paOut = 0;
  *pnOut = 0;

  if( pExpr ){
    if( pExpr->eType==FTSQUERY_PHRASE ){
      int isReqPos = (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR);
      rc = fts3PhraseSelect(p, pExpr->pPhrase, isReqPos, paOut, pnOut);

            }
            pRight = pExpr->pRight;
            assert( pRight->eType==FTSQUERY_PHRASE );
            assert( pLeft->eType==FTSQUERY_PHRASE );

            nParam1 = pExpr->nNear+1;
            nParam2 = nParam1+pLeft->pPhrase->nToken+pRight->pPhrase->nToken-2;
            aBuffer = sqlite3_malloc(nLeft+nRight+1);
            rc = fts3DoclistMerge(mergetype, nParam1, nParam2, aBuffer,
                pnOut, aLeft, nLeft, aRight, nRight
            );
            if( rc!=SQLITE_OK ){
              sqlite3_free(aBuffer);
            }else{
              *paOut = aBuffer;
            }
            sqlite3_free(aLeft);
            break;
          }

          case FTSQUERY_OR: {
            /* Allocate a buffer for the output. The maximum size is the
            ** sum of the sizes of the two input buffers. The +1 term is




            ** so that a buffer of zero bytes is never allocated - this can

            ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM.





            */

            char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
            rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
                aLeft, nLeft, aRight, nRight
            );
            *paOut = aBuffer;
            sqlite3_free(aLeft);
            break;
          }

          case FTSQUERY_AND:
          case FTSQUERY_NOT: {
            assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
            fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
                aLeft, nLeft, aRight, nRight
            );
            *paOut = aLeft;
            break;
          }
        }
      }
      sqlite3_free(aRight);
    }
  }

  return rc;

**
******************************************************************************
**
*/

#ifndef _FTSINT_H
#define _FTSINT_H

#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
# define NDEBUG 1
#endif

#include <sqlite3.h>
#include "fts3_tokenizer.h"
#include "fts3_hash.h"

/*
** This constant controls how often segments are merged. Once there are

# 2009 November 04
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains common code used the fts3 tests. At one point
# equivalent functionality was implemented in C code. But it is easier
# to use Tcl.
#

#-------------------------------------------------------------------------
# USAGE: fts3_integrity_check TBL
#
# This proc is used to verify that the full-text index is consistent with
# the contents of the fts3 table. In other words, it checks that the
# data in the %_contents table matches that in the %_segdir and %_segments 
# tables.
#
# This is not an efficient procedure. It uses a lot of memory and a lot
# of CPU. But it is better than not checking at all.
#
# The procedure is:
#
#   1) Read the entire full-text index from the %_segdir and %_segments
#      tables into memory. For each entry in the index, the following is
#      done:
#
#          set C($iDocid,$iCol,$iPosition) $zTerm
#
#   2) Iterate through each column of each row of the %_content table. 
#      Tokenize all documents, and check that for each token there is
#      a corresponding entry in the $C array. After checking a token,
#      [unset] the $C array entry.
#
#   3) Check that array $C is now empty.
#      
#
proc fts3_integrity_check {tbl} {

  fts3_read2 $tbl 1 A

  foreach zTerm [array names A] {
    foreach doclist $A($zTerm) {
      set docid 0
      while {[string length $doclist]>0} {
        set iCol 0
        set iPos 0
        set lPos [list]
        set lCol [list]

        # First varint of a doclist-entry is the docid. Delta-compressed
        # with respect to the docid of the previous entry.
        #
        incr docid [gobble_varint doclist]
        if {[info exists D($zTerm,$docid)]} {
          while {[set iDelta [gobble_varint doclist]] != 0} {}
          continue
        }
        set D($zTerm,$docid) 1

        # Gobble varints until the 0x00 that terminates the doclist-entry
        # is found.
        while {[set iDelta [gobble_varint doclist]] > 0} {
          if {$iDelta == 1} {
            set iCol [gobble_varint doclist]
            set iPos 0
          } else {
            incr iPos $iDelta
            incr iPos -2
            set C($docid,$iCol,$iPos) $zTerm
          }
        }
      }
    }
  }


  db eval "SELECT * FROM ${tbl}_content" E {
    set iCol 0
    set iDoc $E(docid)
    foreach col [lrange $E(*) 1 end] {
      set c $E($col)
      set sql {SELECT fts3_tokenizer_test('simple', $c)}

      foreach {pos term dummy} [db one $sql] {
        if {$C($iDoc,$iCol,$pos) != "$term"} {
          set    es "Error at docid=$iDoc col=$iCol pos=$pos. "
          append es "Index has \"$C($iDoc,$iCol,$pos)\", document has \"$term\""
          lappend errors $es
        }
        unset C($iDoc,$iCol,$pos)
      }
      incr iCol
    }
  }

  foreach c [array names C] {
    lappend errors "Bad index entry: $c -> $C($c)"
  }

  if {[info exists errors]} { return [join $errors "\n"] }
  return "ok"
}

# USAGE: fts3_terms TBL WHERE
#
# Argument TBL must be the name of an FTS3 table. Argument WHERE is an
# SQL expression that will be used as the WHERE clause when scanning
# the %_segdir table. As in the following query:
#
#   "SELECT * FROM ${TBL}_segdir WHERE ${WHERE}"
#
# This function returns a list of all terms present in the segments
# selected by the statement above.
#
proc fts3_terms {tbl where} {
  fts3_read $tbl $where a
  return [lsort [array names a]]
}


# USAGE: fts3_doclist TBL TERM WHERE
#
# Argument TBL must be the name of an FTS3 table. TERM is a term that may
# or may not be present in the table. Argument WHERE is used to select a 
# subset of the b-tree segments in the associated full-text index as 
# described above for [fts3_terms].
#
# This function returns the results of merging the doclists associated
# with TERM in the selected segments. Each doclist is an element of the
# returned list. Each doclist is formatted as follows:
#
#   [$docid ?$col[$off1 $off2...]?...]
#
# The formatting is odd for a Tcl command in order to be compatible with
# the original C-language implementation. If argument WHERE is "1", then 
# any empty doclists are omitted from the returned list.
#
proc fts3_doclist {tbl term where} {
  fts3_read $tbl $where a


  foreach doclist $a($term) {
    set docid 0

    while {[string length $doclist]>0} {
      set iCol 0
      set iPos 0
      set lPos [list]
      set lCol [list]
      incr docid [gobble_varint doclist]
  
      while {[set iDelta [gobble_varint doclist]] > 0} {
        if {$iDelta == 1} {
          lappend lCol [list $iCol $lPos]
          set iPos 0
          set lPos [list]
          set iCol [gobble_varint doclist]
        } else {
          incr iPos $iDelta
          incr iPos -2
          lappend lPos $iPos
        }
      }
  
      if {[llength $lPos]>0} {
        lappend lCol [list $iCol $lPos]
      }
  
      if {$where != "1" || [llength $lCol]>0} {
        set ret($docid) $lCol
      } else {
        unset -nocomplain ret($docid)
      }
    }
  }

  set lDoc [list]
  foreach docid [lsort -integer [array names ret]] {
    set lCol [list]
    set cols ""
    foreach col $ret($docid) {
      foreach {iCol lPos} $col {}
      append cols " $iCol\[[join $lPos { }]\]"
    }
    lappend lDoc "\[${docid}${cols}\]"
  }

  join $lDoc " "
}

###########################################################################

proc gobble_varint {varname} {
  upvar $varname blob
  set n [read_varint $blob ret]
  set blob [string range $blob $n end]
  return $ret
}
proc gobble_string {varname nLength} {
  upvar $varname blob
  set ret [string range $blob 0 [expr $nLength-1]]
  set blob [string range $blob $nLength end]
  return $ret
}

# The argument is a blob of data representing an FTS3 segment leaf. 
# Return a list consisting of alternating terms (strings) and doclists
# (blobs of data).
#
proc fts3_readleaf {blob} {
  set zPrev ""
  set terms [list]

  while {[string length $blob] > 0} {
    set nPrefix [gobble_varint blob]
    set nSuffix [gobble_varint blob]

    set zTerm [string range $zPrev 0 [expr $nPrefix-1]]
    append zTerm [gobble_string blob $nSuffix]
    set doclist [gobble_string blob [gobble_varint blob]]

    lappend terms $zTerm $doclist
    set zPrev $zTerm
  }

  return $terms
}

proc fts3_read2 {tbl where varname} {
  upvar $varname a
  array unset a
  db eval " SELECT start_block, leaves_end_block, root 
            FROM ${tbl}_segdir WHERE $where
            ORDER BY level ASC, idx DESC
  " {
    if {$start_block == 0} {
      foreach {t d} [fts3_readleaf $root] { lappend a($t) $d }
    } else {
      db eval " SELECT block 
                FROM ${tbl}_segments 
                WHERE blockid>=$start_block AND blockid<$leaves_end_block
                ORDER BY blockid
      " {
        foreach {t d} [fts3_readleaf $block] { lappend a($t) $d }

      }
    }
  }
}

proc fts3_read {tbl where varname} {
  upvar $varname a
  array unset a
  db eval " SELECT start_block, leaves_end_block, root 
            FROM ${tbl}_segdir WHERE $where
            ORDER BY level DESC, idx ASC
  " {
    if {$start_block == 0} {
      foreach {t d} [fts3_readleaf $root] { lappend a($t) $d }
    } else {
      db eval " SELECT block 
                FROM ${tbl}_segments 
                WHERE blockid>=$start_block AND blockid<$leaves_end_block
                ORDER BY blockid
      " {
        foreach {t d} [fts3_readleaf $block] { lappend a($t) $d }

      }
    }
  }
}

# within the FTS3 module code are handled correctly. 
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
ifcapable !fts3 { finish_test ; return }
source $testdir/malloc_common.tcl
source $testdir/fts3_common.tcl

set sqlite_fts3_enable_parentheses 1

if 0 {
do_malloc_test fts3_malloc-1.1 -sqlbody {
  CREATE VIRTUAL TABLE ft USING fts3(a, b, c);
}

do_malloc_test fts3_malloc-1.2 -sqlprep {
  CREATE VIRTUAL TABLE ft USING fts3(a, b, c);
} -sqlbody {
  DROP TABLE ft;
}
}





set DO_MALLOC_TEST 0

#-------------------------------------------------------------------------
# This proc is used to test a single SELECT statement. Parameter $name is
# passed a name for the test case (i.e. "fts3_malloc-1.4.1") and parameter
# $sql is passed the text of the SELECT statement. Parameter $result is
# set to the expected output if the SELECT statement is successfully
# executed using [db eval].






#

# Example:

#

#   do_select_test testcase-1.1 "SELECT 1+1, 1+2" {1 2}
#
# If global variable DO_MALLOC_TEST is set to a non-zero value, or if
# it is not defined at all, then OOM testing is performed on the SELECT
# statement. Each OOM test case is said to pass if either (a) executing
# the SELECT statement succeeds and the results match those specified
# by parameter $result, or (b) TCL throws an "out of memory" error.
#
# If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement
# is executed just once. In this case the test case passes if the results
# match the expected results passed via parameter $result.
#
proc do_select_test {name sql result} {

  if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }


  if {$::DO_MALLOC_TEST } {
    set answers [list {1 {out of memory}} [list 0 $result]]
    set modes [list 100000 transient 1 persistent]
  } else {
    set answers [list [list 0 $result]]
    set modes [list 0 nofail]
  }
  set str [join $answers " OR "]

  foreach {nRepeat zName} $modes {
    for {set iFail 1} 1 {incr iFail} {
      if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat}

      set res [catchsql $sql]
      if {[lsearch $answers $res]>=0} {
        set res $str
      }
      do_test $name.$zName.$iFail [list set {} $res] $str
      set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
      if {$nFail==0} break
    }
  }
}

#-------------------------------------------------------------------------
# Test a single write to the database. In this case a  "write" is a 
# DELETE, UPDATE or INSERT statement.
#
# If OOM testing is performed, there are several acceptable outcomes:
#
#   1) The write succeeds. No error is returned.
#
#   2) An "out of memory" exception is thrown and:
#
#     a) The statement has no effect, OR
#     b) The current transaction is rolled back, OR
#     c) The statement succeeds. This can only happen if the connection
#        is in auto-commit mode (after the statement is executed, so this
#        includes COMMIT statements).
#
# If the write operation eventually succeeds, zero is returned. If a
# transaction is rolled back, non-zero is returned.
#
# Parameter $name is the name to use for the test case (or test cases).
# The second parameter, $tbl, should be the name of the database table
# being modified. Parameter $sql contains the SQL statement to test.
#
proc do_write_test {name tbl sql} {
  # Figure out an statement to get a checksum for table $tbl.
  db eval "SELECT * FROM $tbl" V break
  set cksumsql "SELECT md5sum([join [concat rowid $V(*)] ,]) FROM $tbl"

  # Calculate the initial table checksum.
  set cksum1 [db one $cksumsql]

  if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }

  if {$::DO_MALLOC_TEST } {
    set answers [list {1 {out of memory}} {0 {}}]
    set modes [list 100000 transient 1 persistent]
  } else {
    set answers [list {0 {}}]
    set modes [list 0 nofail]
  }
  set str [join $answers " OR "]

  foreach {nRepeat zName} $modes {
    for {set iFail 1} 1 {incr iFail} {
      if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat}

      set res [catchsql $sql]
      set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
      if {$nFail==0} {
        do_test $name.$zName.$iFail [list set {} $res] {0 {}}
        return
      } else {
        if {[lsearch $answers $res]>=0} {
          set res $str
        }
        do_test $name.$zName.$iFail [list set {} $res] $str
        set cksum2 [db one $cksumsql]
        if {$cksum1 != $cksum2} return
      }
    }
  }
}

proc normal_list {l} {
  set ret [list]
  foreach elem $l {lappend ret $elem}
  set ret
}

db close
file delete -force test.db test.db-journal
sqlite3 db test.db
sqlite3_db_config_lookaside db 0 0 0



do_test fts3_malloc-2.0 {
  execsql { CREATE VIRTUAL TABLE ft USING fts3(a, b) }
  for {set ii 1} {$ii < 32} {incr ii} {
    set a [list]
    set b [list]
    if {$ii & 0x01} {lappend a one   ; lappend b neung}

  20 {SELECT a FROM ft WHERE ft MATCH 'sahm NOT song'} {
    {three}                   {one three}           {three four}   
    {one three four}          {three five}          {one three five}
    {three four five}         {one three four five}
  }

  21 {SELECT a FROM ft WHERE b MATCH 'neung NEAR song NEAR sahm'} {
    {one two three}           {one two three four}  
    {one two three five}      {one two three four five}
  }

} {
  set result [normal_list $result]
  do_select_test fts3_malloc-2.$tn $sql $result
}

do_test fts3_malloc-3.0 {
  execsql BEGIN
  for {set ii 32} {$ii < 1024} {incr ii} {
    set a [list]
    set b [list]

    if {$ii & 0x0200} {lappend a ten   ; lappend b sip   }
    execsql { INSERT INTO ft VALUES($a, $b) }
  }
  execsql COMMIT
} {}
foreach {tn sql result} {
  1 "SELECT count(*) FROM ft" {1023}

  2 "SELECT a FROM ft WHERE a MATCH 'one two three four five six seven eight'" {
     {one two three four five six seven eight}
     {one two three four five six seven eight nine}
     {one two three four five six seven eight ten}
     {one two three four five six seven eight nine ten}
  }

  3 {SELECT count(*), sum(docid) FROM ft WHERE a MATCH 'o*'} {
    512 262144
  }

  4 {SELECT count(*), sum(docid) FROM ft WHERE a MATCH '"two three four"'} {
    128 66368
  }
} {

  set result [normal_list $result]
  do_select_test fts3_malloc-3.$tn $sql $result
}

do_test fts3_malloc-4.0 {
  execsql { DELETE FROM ft WHERE docid>=32 }
} {}
foreach {tn sql} {
  1 "DELETE FROM ft WHERE ft MATCH 'one'"
  2 "DELETE FROM ft WHERE ft MATCH 'three'"
  3 "DELETE FROM ft WHERE ft MATCH 'five'"
} {
  do_write_test fts3_malloc-4.1.$tn ft_content $sql
}
do_test fts3_malloc-4.2 {
  execsql { SELECT a FROM ft }
} {two four {two four}}


finish_test