/ Check-in [9e3aafe4]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add tests for the matchinfo-like test function. Fix problems found in test and fts5 code by doing so.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 9e3aafe44a0813aa2a0c6172fdba1440b8a973ec
User & Date: dan 2015-08-05 19:35:59
Context
2015-08-06
03:09
Fix duplicate test numbering in the FTS5 matchinfo tests. check-in: 483ebe89 user: mistachkin tags: trunk
2015-08-05
19:35
Add tests for the matchinfo-like test function. Fix problems found in test and fts5 code by doing so. check-in: 9e3aafe4 user: dan tags: trunk
15:29
Update the spellfix virtual table extension so that an explicit "top = ?" constraint works even if there is also a "distance < ?" or "distance <= ?" constraint. check-in: 08888383 user: dan tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_expr.c.

  2008   2008     return rc;
  2009   2009   }
  2010   2010   
  2011   2011   /*
  2012   2012   ** Return the number of phrases in expression pExpr.
  2013   2013   */
  2014   2014   int sqlite3Fts5ExprPhraseCount(Fts5Expr *pExpr){
  2015         -  return pExpr->nPhrase;
         2015  +  return (pExpr ? pExpr->nPhrase : 0);
  2016   2016   }
  2017   2017   
  2018   2018   /*
  2019   2019   ** Return the number of terms in the iPhrase'th phrase in pExpr.
  2020   2020   */
  2021   2021   int sqlite3Fts5ExprPhraseSize(Fts5Expr *pExpr, int iPhrase){
  2022   2022     if( iPhrase<0 || iPhrase>=pExpr->nPhrase ) return 0;

Changes to ext/fts5/fts5_index.c.

  3940   3940         int nSuffix;                /* Size of term suffix */
  3941   3941   
  3942   3942         sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
  3943   3943         nTerm = strlen(zTerm);
  3944   3944   
  3945   3945         /* Decide if the term will fit on the current leaf. If it will not, 
  3946   3946         ** flush the leaf to disk here.  */
  3947         -      if( (pBuf->n + nTerm + 2) > pgsz ){
         3947  +      if( pBuf->n>4 && (pBuf->n + nTerm + 2) > pgsz ){
  3948   3948           fts5WriteFlushLeaf(p, &writer);
  3949   3949           pBuf = &writer.writer.buf;
  3950   3950           if( (nTerm + 32) > pBuf->nSpace ){
  3951   3951             fts5BufferGrow(&p->rc, pBuf, nTerm + 32 - pBuf->n);
  3952   3952             if( p->rc ) break;
  3953   3953           }
  3954   3954         }

Changes to ext/fts5/fts5_main.c.

   165    165   **
   166    166   **   If the cursor iterates in descending order of rowid, iFirstRowid
   167    167   **   is the upper limit (i.e. the "first" rowid visited) and iLastRowid
   168    168   **   the lower.
   169    169   */
   170    170   struct Fts5Cursor {
   171    171     sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
          172  +  Fts5Cursor *pNext;              /* Next cursor in Fts5Cursor.pCsr list */
          173  +  int *aColumnSize;               /* Values for xColumnSize() */
          174  +  i64 iCsrId;                     /* Cursor id */
          175  +
          176  +  /* Zero from this point onwards on cursor reset */
   172    177     int ePlan;                      /* FTS5_PLAN_XXX value */
   173    178     int bDesc;                      /* True for "ORDER BY rowid DESC" queries */
   174    179     i64 iFirstRowid;                /* Return no rowids earlier than this */
   175    180     i64 iLastRowid;                 /* Return no rowids later than this */
   176    181     sqlite3_stmt *pStmt;            /* Statement used to read %_content */
   177    182     Fts5Expr *pExpr;                /* Expression for MATCH queries */
   178    183     Fts5Sorter *pSorter;            /* Sorter for "ORDER BY rank" queries */
   179    184     int csrflags;                   /* Mask of cursor flags (see below) */
   180         -  Fts5Cursor *pNext;              /* Next cursor in Fts5Cursor.pCsr list */
   181    185     i64 iSpecial;                   /* Result of special query */
   182    186   
   183    187     /* "rank" function. Populated on demand from vtab.xColumn(). */
   184    188     char *zRank;                    /* Custom rank function */
   185    189     char *zRankArgs;                /* Custom rank function args */
   186    190     Fts5Auxiliary *pRank;           /* Rank callback (or NULL) */
   187    191     int nRankArg;                   /* Number of trailing arguments for rank() */
   188    192     sqlite3_value **apRankArg;      /* Array of trailing arguments */
   189    193     sqlite3_stmt *pRankArgStmt;     /* Origin of objects in apRankArg[] */
   190    194   
   191         -  /* Variables used by auxiliary functions */
   192         -  i64 iCsrId;                     /* Cursor id */
          195  +  /* Auxiliary data storage */
   193    196     Fts5Auxiliary *pAux;            /* Currently executing extension function */
   194    197     Fts5Auxdata *pAuxdata;          /* First in linked list of saved aux-data */
   195         -  int *aColumnSize;               /* Values for xColumnSize() */
   196    198   
   197    199     /* Cache used by auxiliary functions xInst() and xInstCount() */
   198    200     int nInstCount;                 /* Number of phrase instances */
   199    201     int *aInst;                     /* 3 integers per phrase instance */
   200    202   };
   201    203   
   202    204   /*
................................................................................
   425    427   ){
   426    428     return fts5InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
   427    429   }
   428    430   
   429    431   /*
   430    432   ** The different query plans.
   431    433   */
   432         -#define FTS5_PLAN_MATCH          0       /* (<tbl> MATCH ?) */
   433         -#define FTS5_PLAN_SOURCE         1       /* A source cursor for SORTED_MATCH */
   434         -#define FTS5_PLAN_SPECIAL        2       /* An internal query */
   435         -#define FTS5_PLAN_SORTED_MATCH   3       /* (<tbl> MATCH ? ORDER BY rank) */
   436         -#define FTS5_PLAN_SCAN           4       /* No usable constraint */
   437         -#define FTS5_PLAN_ROWID          5       /* (rowid = ?) */
          434  +#define FTS5_PLAN_MATCH          1       /* (<tbl> MATCH ?) */
          435  +#define FTS5_PLAN_SOURCE         2       /* A source cursor for SORTED_MATCH */
          436  +#define FTS5_PLAN_SPECIAL        3       /* An internal query */
          437  +#define FTS5_PLAN_SORTED_MATCH   4       /* (<tbl> MATCH ? ORDER BY rank) */
          438  +#define FTS5_PLAN_SCAN           5       /* No usable constraint */
          439  +#define FTS5_PLAN_ROWID          6       /* (rowid = ?) */
   438    440   
   439    441   /*
   440    442   ** Implementation of the xBestIndex method for FTS5 tables. Within the 
   441    443   ** WHERE constraint, it searches for the following:
   442    444   **
   443    445   **   1. A MATCH constraint against the special column.
   444    446   **   2. A MATCH constraint against the "rank" column.
................................................................................
   605    607   static void fts5CsrNewrow(Fts5Cursor *pCsr){
   606    608     CsrFlagSet(pCsr, 
   607    609         FTS5CSR_REQUIRE_CONTENT 
   608    610       | FTS5CSR_REQUIRE_DOCSIZE 
   609    611       | FTS5CSR_REQUIRE_INST 
   610    612     );
   611    613   }
          614  +
          615  +static void fts5FreeCursorComponents(Fts5Cursor *pCsr){
          616  +  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
          617  +  Fts5Auxdata *pData;
          618  +  Fts5Auxdata *pNext;
          619  +
          620  +  sqlite3_free(pCsr->aInst);
          621  +  if( pCsr->pStmt ){
          622  +    int eStmt = fts5StmtType(pCsr);
          623  +    sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
          624  +  }
          625  +  if( pCsr->pSorter ){
          626  +    Fts5Sorter *pSorter = pCsr->pSorter;
          627  +    sqlite3_finalize(pSorter->pStmt);
          628  +    sqlite3_free(pSorter);
          629  +  }
          630  +
          631  +  if( pCsr->ePlan!=FTS5_PLAN_SOURCE ){
          632  +    sqlite3Fts5ExprFree(pCsr->pExpr);
          633  +  }
          634  +
          635  +  for(pData=pCsr->pAuxdata; pData; pData=pNext){
          636  +    pNext = pData->pNext;
          637  +    if( pData->xDelete ) pData->xDelete(pData->pPtr);
          638  +    sqlite3_free(pData);
          639  +  }
          640  +
          641  +  sqlite3_finalize(pCsr->pRankArgStmt);
          642  +  sqlite3_free(pCsr->apRankArg);
          643  +
          644  +  if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){
          645  +    sqlite3_free(pCsr->zRank);
          646  +    sqlite3_free(pCsr->zRankArgs);
          647  +  }
          648  +
          649  +  memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan - (u8*)pCsr));
          650  +}
          651  +
   612    652   
   613    653   /*
   614    654   ** Close the cursor.  For additional information see the documentation
   615    655   ** on the xClose method of the virtual table interface.
   616    656   */
   617    657   static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
   618    658     if( pCursor ){
   619    659       Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
   620    660       Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
   621    661       Fts5Cursor **pp;
   622         -    Fts5Auxdata *pData;
   623         -    Fts5Auxdata *pNext;
   624    662   
   625         -    sqlite3_free(pCsr->aInst);
   626         -    if( pCsr->pStmt ){
   627         -      int eStmt = fts5StmtType(pCsr);
   628         -      sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
   629         -    }
   630         -    if( pCsr->pSorter ){
   631         -      Fts5Sorter *pSorter = pCsr->pSorter;
   632         -      sqlite3_finalize(pSorter->pStmt);
   633         -      sqlite3_free(pSorter);
   634         -    }
   635         -
   636         -    if( pCsr->ePlan!=FTS5_PLAN_SOURCE ){
   637         -      sqlite3Fts5ExprFree(pCsr->pExpr);
   638         -    }
   639         -
   640         -    for(pData=pCsr->pAuxdata; pData; pData=pNext){
   641         -      pNext = pData->pNext;
   642         -      if( pData->xDelete ) pData->xDelete(pData->pPtr);
   643         -      sqlite3_free(pData);
   644         -    }
   645         -
          663  +    fts5FreeCursorComponents(pCsr);
   646    664       /* Remove the cursor from the Fts5Global.pCsr list */
   647    665       for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
   648    666       *pp = pCsr->pNext;
   649    667   
   650         -    sqlite3_finalize(pCsr->pRankArgStmt);
   651         -    sqlite3_free(pCsr->apRankArg);
   652         -
   653         -    if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){
   654         -      sqlite3_free(pCsr->zRank);
   655         -      sqlite3_free(pCsr->zRankArgs);
   656         -    }
   657    668       sqlite3_free(pCsr);
   658    669     }
   659    670     return SQLITE_OK;
   660    671   }
   661    672   
   662    673   static int fts5SorterNext(Fts5Cursor *pCsr){
   663    674     Fts5Sorter *pSorter = pCsr->pSorter;
................................................................................
   753    764   ** even if we reach end-of-file.  The fts5EofMethod() will be called
   754    765   ** subsequently to determine whether or not an EOF was hit.
   755    766   */
   756    767   static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
   757    768     Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
   758    769     int rc = SQLITE_OK;
   759    770   
   760         -  assert( (pCsr->ePlan<2)==
          771  +  assert( (pCsr->ePlan<3)==
   761    772             (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) 
   762    773     );
   763    774   
   764         -  if( pCsr->ePlan<2 ){
          775  +  if( pCsr->ePlan<3 ){
   765    776       int bSkip = 0;
   766    777       if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
   767    778       rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid);
   768    779       if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
   769    780         CsrFlagSet(pCsr, FTS5CSR_EOF);
   770    781       }
   771    782       fts5CsrNewrow(pCsr);
................................................................................
  1037   1048     int bOrderByRank;               /* True if ORDER BY rank */
  1038   1049     sqlite3_value *pMatch = 0;      /* <tbl> MATCH ? expression (or NULL) */
  1039   1050     sqlite3_value *pRank = 0;       /* rank MATCH ? expression (or NULL) */
  1040   1051     sqlite3_value *pRowidEq = 0;    /* rowid = ? expression (or NULL) */
  1041   1052     sqlite3_value *pRowidLe = 0;    /* rowid <= ? expression (or NULL) */
  1042   1053     sqlite3_value *pRowidGe = 0;    /* rowid >= ? expression (or NULL) */
  1043   1054     char **pzErrmsg = pConfig->pzErrmsg;
         1055  +
         1056  +  if( pCsr->ePlan ){
         1057  +    fts5FreeCursorComponents(pCsr);
         1058  +    memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr));
         1059  +  }
  1044   1060   
  1045   1061     assert( pCsr->pStmt==0 );
  1046   1062     assert( pCsr->pExpr==0 );
  1047   1063     assert( pCsr->csrflags==0 );
  1048   1064     assert( pCsr->pRank==0 );
  1049   1065     assert( pCsr->zRank==0 );
  1050   1066     assert( pCsr->zRankArgs==0 );

Changes to ext/fts5/fts5_test_mi.c.

    26     26   **         "a OR (b AND c)"
    27     27   **
    28     28   **     In FTS4, if a single row contains instances of tokens "a" and "c", 
    29     29   **     but not "b", all instances of "c" are considered matches. In FTS5,
    30     30   **     they are not (as the "b AND c" sub-tree does not match the current
    31     31   **     row.
    32     32   **
    33         -**  2) ...
           33  +**  2) For the values returned by 'x' that apply to all rows of the table, 
           34  +**     NEAR constraints are not considered. But for the number of hits in
           35  +**     the current row, they are.
    34     36   **     
    35     37   ** This file exports a single function that may be called to register the
    36     38   ** matchinfo() implementation with a database handle:
    37     39   **
    38     40   **   int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db);
    39     41   */
    40     42   
................................................................................
    78     80     sqlite3_finalize(pStmt);
    79     81     return pRet;
    80     82   }
    81     83   
    82     84   
    83     85   /*
    84     86   ** Argument f should be a flag accepted by matchinfo() (a valid character
    85         -** in the string passed as the second argument). If it is not, 0 is 
           87  +** in the string passed as the second argument). If it is not, -1 is 
    86     88   ** returned. Otherwise, if f is a valid matchinfo flag, the value returned
    87     89   ** is the number of 32-bit integers added to the output array if the
    88     90   ** table has nCol columns and the query nPhrase phrases.
    89     91   */
    90     92   static int fts5MatchinfoFlagsize(int nCol, int nPhrase, char f){
    91         -  int ret = 0;
           93  +  int ret = -1;
    92     94     switch( f ){
    93     95       case 'p': ret = 1; break;
    94     96       case 'c': ret = 1; break;
    95     97       case 'x': ret = 3 * nCol * nPhrase; break;
    96     98       case 'y': ret = nCol * nPhrase; break;
    97     99       case 'b': ret = ((nCol + 31) / 32) * nPhrase; break;
    98    100       case 'n': ret = 1; break;
................................................................................
   243    245           int nToken;
   244    246           rc = pApi->xColumnSize(pFts, i, &nToken);
   245    247           aOut[i] = (u32)nToken;
   246    248         }
   247    249         break;
   248    250       }
   249    251   
   250         -    case 's':
          252  +    case 's': {
          253  +      int nInst;
          254  +
   251    255         memset(aOut, 0, sizeof(u32) * p->nCol);
          256  +
          257  +      rc = pApi->xInstCount(pFts, &nInst);
          258  +      for(i=0; rc==SQLITE_OK && i<nInst; i++){
          259  +        int iPhrase, iOff, iCol = 0;
          260  +        int iNextPhrase;
          261  +        int iNextOff;
          262  +        int nSeq = 1;
          263  +        int j;
          264  +
          265  +        rc = pApi->xInst(pFts, i, &iPhrase, &iCol, &iOff);
          266  +        iNextPhrase = iPhrase+1;
          267  +        iNextOff = iOff+pApi->xPhraseSize(pFts, 0);
          268  +        for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
          269  +          int ip, ic, io;
          270  +          rc = pApi->xInst(pFts, j, &ip, &ic, &io);
          271  +          if( ic!=iCol || io>iNextOff ) break;
          272  +          if( ip==iNextPhrase && io==iNextOff ){
          273  +            nSeq++;
          274  +            iNextPhrase = ip+1;
          275  +            iNextOff = io + pApi->xPhraseSize(pFts, ip);
          276  +          }
          277  +        }
          278  +
          279  +        if( nSeq>aOut[iCol] ) aOut[iCol] = nSeq;
          280  +      }
          281  +
   252    282         break;
          283  +    }
   253    284     }
   254    285     return rc;
   255    286   }
   256    287    
   257    288   static Fts5MatchinfoCtx *fts5MatchinfoNew(
   258    289     const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
   259    290     Fts5Context *pFts,              /* First arg to pass to pApi functions */
................................................................................
   270    301   
   271    302     nCol = pApi->xColumnCount(pFts);
   272    303     nPhrase = pApi->xPhraseCount(pFts);
   273    304   
   274    305     nInt = 0;
   275    306     for(i=0; zArg[i]; i++){
   276    307       int n = fts5MatchinfoFlagsize(nCol, nPhrase, zArg[i]);
   277         -    if( n==0 ){
          308  +    if( n<0 ){
   278    309         char *zErr = sqlite3_mprintf("unrecognized matchinfo flag: %c", zArg[i]);
   279    310         sqlite3_result_error(pCtx, zErr, -1);
   280    311         sqlite3_free(zErr);
   281    312         return 0;
   282    313       }
   283    314       nInt += n;
   284    315     }

Changes to ext/fts5/test/fts5_common.tcl.

    88     88       set cnt [list]
    89     89       for {set j 0} {$j < [$cmd xColumnCount]} {incr j} { lappend cnt 0 }
    90     90       $cmd xQueryPhrase $i [list test_queryphrase_cb cnt]
    91     91       lappend res $cnt
    92     92     }
    93     93     set res
    94     94   }
           95  +
           96  +proc fts5_test_phrasecount {cmd} {
           97  +  $cmd xPhraseCount
           98  +}
    95     99   
    96    100   proc fts5_test_all {cmd} {
    97    101     set res [list]
    98    102     lappend res columnsize      [fts5_test_columnsize $cmd]
    99    103     lappend res columntext      [fts5_test_columntext $cmd]
   100    104     lappend res columntotalsize [fts5_test_columntotalsize $cmd]
   101    105     lappend res poslist         [fts5_test_poslist $cmd]
................................................................................
   111    115       fts5_test_columntotalsize
   112    116       fts5_test_poslist
   113    117       fts5_test_tokenize
   114    118       fts5_test_rowcount
   115    119       fts5_test_all
   116    120   
   117    121       fts5_test_queryphrase
          122  +    fts5_test_phrasecount
   118    123     } {
   119    124       sqlite3_fts5_create_function $db $f $f
   120    125     }
   121    126   }
   122    127   
   123    128   proc fts5_level_segs {tbl} {
   124    129     set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"

Changes to ext/fts5/test/fts5ae.test.

   272    272       SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY +rank;
   273    273     } $res
   274    274   
   275    275     do_execsql_test 8.2.$tn.3 {
   276    276       SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY rank;
   277    277     } $res
   278    278   }
          279  +
          280  +#-------------------------------------------------------------------------
          281  +# Test xPhraseCount() for some different queries.
          282  +#
          283  +do_test 9.1 {
          284  +  execsql { CREATE VIRTUAL TABLE t9 USING fts5(x) }
          285  +  foreach x {
          286  +    "a b c" "d e f"
          287  +  } {
          288  +    execsql { INSERT INTO t9 VALUES($x) }
          289  +  }
          290  +} {}
          291  +
          292  +foreach {tn q cnt} {
          293  +  1 {a AND b}      2
          294  +  2 {a OR b}       2
          295  +  3 {a OR b OR c}  3
          296  +  4 {NEAR(a b)}    2
          297  +} {
          298  +  do_execsql_test 9.2.$tn {
          299  +    SELECT fts5_test_phrasecount(t9) FROM t9 WHERE t9 MATCH $q LIMIT 1
          300  +  } $cnt
          301  +}
   279    302   
   280    303   finish_test
   281    304   

Added ext/fts5/test/fts5matchinfo.test.

            1  +# 2015 August 05
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +
           13  +source [file join [file dirname [info script]] fts5_common.tcl]
           14  +set testprefix fts5matchinfo
           15  +
           16  +# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
           17  +ifcapable !fts5 { finish_test ; return }
           18  +
           19  +proc mit {blob} {
           20  +  set scan(littleEndian) i*
           21  +  set scan(bigEndian) I*
           22  +  binary scan $blob $scan($::tcl_platform(byteOrder)) r
           23  +  return $r
           24  +}
           25  +db func mit mit
           26  +
           27  +sqlite3_fts5_register_matchinfo db
           28  +
           29  +do_execsql_test 1.0 {
           30  +  CREATE VIRTUAL TABLE t1 USING fts5(content);
           31  +} 
           32  +
           33  +do_execsql_test 1.1 {
           34  +  INSERT INTO t1(content) VALUES('I wandered lonely as a cloud');
           35  +  INSERT INTO t1(content) VALUES('That floats on high o''er vales and hills,');
           36  +  INSERT INTO t1(content) VALUES('When all at once I saw a crowd,');
           37  +  INSERT INTO t1(content) VALUES('A host, of golden daffodils,');
           38  +  SELECT mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'I';
           39  +} {{1 1 1 2 2} {1 1 1 2 2}}
           40  +
           41  +# Now create an FTS4 table that does not specify matchinfo=fts3.
           42  +#
           43  +do_execsql_test 1.2 {
           44  +  CREATE VIRTUAL TABLE t2 USING fts5(content);
           45  +  INSERT INTO t2 SELECT * FROM t1;
           46  +  SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'I';
           47  +} {{1 1 1 2 2} {1 1 1 2 2}}
           48  +
           49  +
           50  +#--------------------------------------------------------------------------
           51  +# Proc [do_matchinfo_test] is used to test the FTSX matchinfo() function.
           52  +#
           53  +# The first argument - $tn - is a test identifier. This may be either a
           54  +# full identifier (i.e. "fts3matchinfo-1.1") or, if global var $testprefix
           55  +# is set, just the numeric component (i.e. "1.1").
           56  +#
           57  +# The second argument is the name of an FTSX table. The third is the 
           58  +# full text of a WHERE/MATCH expression to query the table for 
           59  +# (i.e. "t1 MATCH 'abc'"). The final argument - $results - should be a
           60  +# key-value list (serialized array) with matchinfo() format specifiers
           61  +# as keys, and the results of executing the statement:
           62  +#
           63  +#   SELECT matchinfo($tbl, '$key') FROM $tbl WHERE $expr
           64  +#
           65  +# For example:
           66  +#
           67  +#   CREATE VIRTUAL TABLE t1 USING fts4;
           68  +#   INSERT INTO t1 VALUES('abc');
           69  +#   INSERT INTO t1 VALUES('def');
           70  +#   INSERT INTO t1 VALUES('abc abc');
           71  +#
           72  +#   do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" {
           73  +#     n {3 3}
           74  +#     p {1 1}
           75  +#     c {1 1}
           76  +#     x {{1 3 2} {2 3 2}}
           77  +#   }
           78  +#
           79  +# If the $results list contains keys mapped to "-" instead of a matchinfo()
           80  +# result, then this command computes the expected results based on other
           81  +# mappings to test the matchinfo() function. For example, the command above
           82  +# could be changed to:
           83  +#
           84  +#   do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" {
           85  +#     n {3 3} p {1 1} c {1 1} x {{1 3 2} {2 3 2}}
           86  +#     pcx -
           87  +#   }
           88  +#
           89  +# And this command would compute the expected results for matchinfo(t1, 'pcx')
           90  +# based on the results of matchinfo(t1, 'p'), matchinfo(t1, 'c') and 
           91  +# matchinfo(t1, 'x') in order to test 'pcx'.
           92  +#
           93  +proc do_matchinfo_test {tn tbl expr results} {
           94  +
           95  +  foreach {fmt res} $results {
           96  +    if {$res == "-"} continue
           97  +    set resarray($fmt) $res
           98  +  }
           99  +
          100  +  set nRow 0
          101  +  foreach {fmt res} [array get resarray] {
          102  +    if {[llength $res]>$nRow} { set nRow [llength $res] }
          103  +  }
          104  +
          105  +  # Construct expected results for any formats for which the caller 
          106  +  # supplied result is "-".
          107  +  #
          108  +  foreach {fmt res} $results {
          109  +    if {$res == "-"} {
          110  +      set res [list]
          111  +      for {set iRow 0} {$iRow<$nRow} {incr iRow} {
          112  +        set rowres [list]
          113  +        foreach c [split $fmt ""] {
          114  +          set rowres [concat $rowres [lindex $resarray($c) $iRow]]
          115  +        }
          116  +        lappend res $rowres
          117  +      }
          118  +      set resarray($fmt) $res
          119  +    }
          120  +  }
          121  +
          122  +  # Test each matchinfo() request individually.
          123  +  #
          124  +  foreach {fmt res} [array get resarray] {
          125  +    set sql "SELECT mit(matchinfo($tbl, '$fmt')) FROM $tbl WHERE $expr"
          126  +    do_execsql_test $tn.$fmt $sql [normalize2 $res]
          127  +  }
          128  +
          129  +  # Test them all executed together (multiple invocations of matchinfo()).
          130  +  #
          131  +  set exprlist [list]
          132  +  foreach {format res} [array get resarray] {
          133  +    lappend exprlist "mit(matchinfo($tbl, '$format'))"
          134  +  }
          135  +  set allres [list]
          136  +  for {set iRow 0} {$iRow<$nRow} {incr iRow} {
          137  +    foreach {format res} [array get resarray] {
          138  +      lappend allres [lindex $res $iRow]
          139  +    }
          140  +  }
          141  +  set sql "SELECT [join $exprlist ,] FROM $tbl WHERE $expr"
          142  +  do_execsql_test $tn.multi $sql [normalize2 $allres]
          143  +}
          144  +proc normalize2 {list_of_lists} {
          145  +  set res [list]
          146  +  foreach elem $list_of_lists {
          147  +    lappend res [list {*}$elem]
          148  +  }
          149  +  return $res
          150  +}
          151  +
          152  +
          153  +do_execsql_test 4.1.0 {
          154  +  CREATE VIRTUAL TABLE t4 USING fts5(x, y);
          155  +  INSERT INTO t4 VALUES('a b c d e', 'f g h i j');
          156  +  INSERT INTO t4 VALUES('f g h i j', 'a b c d e');
          157  +}
          158  +
          159  +do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} {
          160  +  s {{3 0} {0 3}}
          161  +}
          162  +
          163  +do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} {
          164  +  p {3 3}
          165  +  x {
          166  +    {1 1 1   0 1 1   1 1 1   0 1 1   1 1 1   0 1 1}
          167  +    {0 1 1   1 1 1   0 1 1   1 1 1   0 1 1   1 1 1}
          168  +  }
          169  +}
          170  +
          171  +do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} {
          172  +  p {3 3}
          173  +  c {2 2}
          174  +  x {
          175  +    {1 1 1   0 1 1   1 1 1   0 1 1   1 1 1   0 1 1}
          176  +    {0 1 1   1 1 1   0 1 1   1 1 1   0 1 1   1 1 1}
          177  +  }
          178  +  n {2 2}
          179  +  l {{5 5} {5 5}}
          180  +  a {{5 5} {5 5}}
          181  +
          182  +  s {{3 0} {0 3}}
          183  +
          184  +  xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc -
          185  +  xpxsscplax -
          186  +}
          187  +
          188  +do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} {
          189  +  p {1 1}
          190  +  c {2 2}
          191  +  x {
          192  +    {0 1 1   1 1 1}
          193  +    {1 1 1   0 1 1}
          194  +  }
          195  +  n {2 2}
          196  +  l {{5 5} {5 5}}
          197  +  a {{5 5} {5 5}}
          198  +
          199  +  s {{0 1} {1 0}}
          200  +
          201  +  xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc -
          202  +  sxsxs -
          203  +}
          204  +
          205  +do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'}     { s {{2 0} {0 2}} }
          206  +do_matchinfo_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} }
          207  +do_matchinfo_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} }
          208  +do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'}     { s {{1 0} {0 1}} }
          209  +do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} {
          210  +  x { 
          211  +    {0 1 1  1 1 1  0 0 0  0 0 0} 
          212  +    {1 1 1  0 1 1  0 0 0  0 0 0}
          213  +  }
          214  +}
          215  +do_matchinfo_test 4.1.8 t4 {t4 MATCH 'f NOT abcd'} {
          216  +  x { 
          217  +    {0 1 1  1 1 1  0 0 0  0 0 0}
          218  +    {1 1 1  0 1 1  0 0 0  0 0 0}
          219  +  }
          220  +}
          221  +
          222  +do_execsql_test 4.2.0 {
          223  +  CREATE VIRTUAL TABLE t5 USING fts5(content);
          224  +  INSERT INTO t5 VALUES('a a a a a');
          225  +  INSERT INTO t5 VALUES('a b a b a');
          226  +  INSERT INTO t5 VALUES('c b c b c');
          227  +  INSERT INTO t5 VALUES('x x x x x');
          228  +}
          229  +do_matchinfo_test 4.2.1 t5 {t5 MATCH 'a a'}         { 
          230  +  x {{5 8 2   5 8 2} {3 8 2   3 8 2}}
          231  +  s {2 1} 
          232  +}
          233  +do_matchinfo_test 4.2.2 t5 {t5 MATCH 'a b'}         { s {2} }
          234  +do_matchinfo_test 4.2.3 t5 {t5 MATCH 'a b a'}       { s {3} }
          235  +do_matchinfo_test 4.2.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
          236  +do_matchinfo_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
          237  +do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'}      { s {1 2 1} }
          238  +
          239  +do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')";
          240  +
          241  +# It used to be that the second 'a' token would be deferred. That doesn't
          242  +# work any longer.
          243  +if 0 {
          244  +  do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { 
          245  +    x {{5 8 2   5 5 5} {3 8 2   3 5 5}}
          246  +    s {2 1} 
          247  +  }
          248  +}
          249  +
          250  +do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'}         { s {2} }
          251  +do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'}       { s {3} }
          252  +do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
          253  +do_matchinfo_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
          254  +do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'}      { s {1 2 1 1} }
          255  +
          256  +do_execsql_test 4.4.0.1 { INSERT INTO t5(t5) VALUES('optimize') }
          257  +
          258  +do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'}         { s {2} }
          259  +do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'}         { s {2 1} }
          260  +do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'}         { s {2} }
          261  +do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'}       { s {3} }
          262  +do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
          263  +do_matchinfo_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
          264  +
          265  +do_execsql_test 4.5.0 {
          266  +  CREATE VIRTUAL TABLE t6 USING fts5(a, b, c);
          267  +  INSERT INTO t6 VALUES('a', 'b', 'c');
          268  +}
          269  +do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'}       { s {{1 1 1}} }
          270  +
          271  +
          272  +#-------------------------------------------------------------------------
          273  +# Test the outcome of matchinfo() when used within a query that does not
          274  +# use the full-text index (i.e. lookup by rowid or full-table scan).
          275  +#
          276  +do_execsql_test 7.1 {
          277  +  CREATE VIRTUAL TABLE t10 USING fts5(content);
          278  +  INSERT INTO t10 VALUES('first record');
          279  +  INSERT INTO t10 VALUES('second record');
          280  +}
          281  +do_execsql_test 7.2 {
          282  +  SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) FROM t10;
          283  +} {blob 8 blob 8}
          284  +do_execsql_test 7.3 {
          285  +  SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) FROM t10 WHERE rowid=1;
          286  +} {blob 8}
          287  +do_execsql_test 7.4 {
          288  +  SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) 
          289  +  FROM t10 WHERE t10 MATCH 'record'
          290  +} {blob 20 blob 20}
          291  +
          292  +#-------------------------------------------------------------------------
          293  +# Test a special case - matchinfo('nxa') with many zero length documents. 
          294  +# Special because "x" internally uses a statement used by both "n" and "a". 
          295  +# This was causing a problem at one point in the obscure case where the
          296  +# total number of bytes of data stored in an fts3 table was greater than
          297  +# the number of rows. i.e. when the following query returns true:
          298  +#
          299  +#   SELECT sum(length(content)) < count(*) FROM fts4table;
          300  +#
          301  +do_execsql_test 8.1 {
          302  +  CREATE VIRTUAL TABLE t11 USING fts5(content);
          303  +  INSERT INTO t11(t11, rank) VALUES('pgsz', 32);
          304  +  INSERT INTO t11 VALUES('quitealongstringoftext');
          305  +  INSERT INTO t11 VALUES('anotherquitealongstringoftext');
          306  +  INSERT INTO t11 VALUES('athirdlongstringoftext');
          307  +  INSERT INTO t11 VALUES('andonemoreforgoodluck');
          308  +}
          309  +do_test 8.2 {
          310  +  for {set i 0} {$i < 200} {incr i} {
          311  +    execsql { INSERT INTO t11 VALUES('') }
          312  +  }
          313  +  execsql { INSERT INTO t11(t11) VALUES('optimize') }
          314  +} {}
          315  +do_execsql_test 8.3 {
          316  +  SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*'
          317  +} {{204 1 3 3 0} {204 1 3 3 0} {204 1 3 3 0}}
          318  +
          319  +#-------------------------------------------------------------------------
          320  +breakpoint
          321  +do_execsql_test 8.1 {
          322  +  CREATE VIRTUAL TABLE t12 USING fts5(content);
          323  +  INSERT INTO t12 VALUES('a b c d');
          324  +  SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
          325  +} {{0 1 1 0 1 1 1 1 1}}
          326  +do_execsql_test 8.2 {
          327  +  INSERT INTO t12 VALUES('a d c d');
          328  +  SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
          329  +} {
          330  +  {0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2}
          331  +}
          332  +do_execsql_test 8.3 {
          333  +  INSERT INTO t12 VALUES('a d d a');
          334  +  SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
          335  +} {
          336  +  {0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3}
          337  +}
          338  +
          339  +#---------------------------------------------------------------------------
          340  +# Test for a memory leak
          341  +#
          342  +do_execsql_test 10.1 {
          343  +  DROP TABLE t10;
          344  +  CREATE VIRTUAL TABLE t10 USING fts5(idx, value);
          345  +  INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three');
          346  +  SELECT t10.rowid, t10.*
          347  +    FROM t10
          348  +    JOIN (SELECT 1 AS idx UNION SELECT 2 UNION SELECT 3) AS x
          349  +   WHERE t10 MATCH x.idx
          350  +     AND matchinfo(t10) not null
          351  +   GROUP BY t10.rowid
          352  +   ORDER BY 1;
          353  +} {1 1 one 2 2 two 3 3 three}
          354  +  
          355  +#---------------------------------------------------------------------------
          356  +# Test the 'y' matchinfo flag
          357  +#
          358  +set sqlite_fts3_enable_parentheses 1
          359  +reset_db
          360  +do_execsql_test 11.0 {
          361  +  CREATE VIRTUAL TABLE tt USING fts3(x, y);
          362  +  INSERT INTO tt VALUES('c d a c d d', 'e a g b d a');   -- 1
          363  +  INSERT INTO tt VALUES('c c g a e b', 'c g d g e c');   -- 2
          364  +  INSERT INTO tt VALUES('b e f d e g', 'b a c b c g');   -- 3
          365  +  INSERT INTO tt VALUES('a c f f g d', 'd b f d e g');   -- 4
          366  +  INSERT INTO tt VALUES('g a c f c f', 'd g g b c c');   -- 5
          367  +  INSERT INTO tt VALUES('g a c e b b', 'd b f b g g');   -- 6
          368  +  INSERT INTO tt VALUES('f d a a f c', 'e e a d c f');   -- 7
          369  +  INSERT INTO tt VALUES('a c b b g f', 'a b a e d f');   -- 8
          370  +  INSERT INTO tt VALUES('b a f e c c', 'f d b b a b');   -- 9
          371  +  INSERT INTO tt VALUES('f d c e a c', 'f a f a a f');   -- 10
          372  +}
          373  +
          374  +db func mit mit
          375  +foreach {tn expr res} {
          376  +  1 "a" {
          377  +      1 {1 2}   2 {1 0}   3 {0 1}   4 {1 0}   5 {1 0}
          378  +      6 {1 0}   7 {2 1}   8 {1 2}   9 {1 1}  10 {1 3}
          379  +  }
          380  +
          381  +  2 "b" {
          382  +      1 {0 1}   2 {1 0}   3 {1 2}   4 {0 1}   5 {0 1}
          383  +      6 {2 2}             8 {2 1}   9 {1 3}            
          384  +  }
          385  +
          386  +  3 "y:a" {
          387  +      1 {0 2}             3 {0 1}                    
          388  +                7 {0 1}   8 {0 2}   9 {0 1}  10 {0 3}
          389  +  }
          390  +
          391  +  4 "x:a" {
          392  +      1 {1 0}   2 {1 0}             4 {1 0}   5 {1 0}
          393  +      6 {1 0}   7 {2 0}   8 {1 0}   9 {1 0}  10 {1 0}
          394  +  }
          395  +
          396  +  5 "a OR b" {
          397  +      1 {1 2 0 1}   2 {1 0 1 0}   3 {0 1 1 2}   4 {1 0 0 1}   5 {1 0 0 1}
          398  +      6 {1 0 2 2}   7 {2 1 0 0}   8 {1 2 2 1}   9 {1 1 1 3}  10 {1 3 0 0}
          399  +  }
          400  +
          401  +  6 "a AND b" {
          402  +      1 {1 2 0 1}   2 {1 0 1 0}   3 {0 1 1 2}   4 {1 0 0 1}   5 {1 0 0 1}
          403  +      6 {1 0 2 2}                 8 {1 2 2 1}   9 {1 1 1 3}              
          404  +  }
          405  +
          406  +  7 "a OR (a AND b)" {
          407  +      1 {1 2 1 2 0 1}   2 {1 0 1 0 1 0}   3 {0 1 0 1 1 2}   4 {1 0 1 0 0 1}   
          408  +      5 {1 0 1 0 0 1}   6 {1 0 1 0 2 2}   7 {2 1 0 0 0 0}   8 {1 2 1 2 2 1}   
          409  +      9 {1 1 1 1 1 3}  10 {1 3 0 0 0 0}
          410  +  }
          411  +
          412  +} {
          413  +  do_execsql_test 11.1.$tn.1  {
          414  +    SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr
          415  +  } $res
          416  +
          417  +  set r2 [list]
          418  +  foreach {rowid L} $res {
          419  +    lappend r2 $rowid
          420  +    set M [list]
          421  +    foreach {a b} $L {
          422  +      lappend M [expr ($a ? 1 : 0) + ($b ? 2 : 0)]
          423  +    }
          424  +    lappend r2 $M
          425  +  }
          426  +
          427  +  do_execsql_test 11.1.$tn.2  {
          428  +    SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr
          429  +  } $r2
          430  +
          431  +  do_execsql_test 11.1.$tn.2  {
          432  +    SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr
          433  +  } $r2
          434  +}
          435  +set sqlite_fts3_enable_parentheses 0
          436  +
          437  +#---------------------------------------------------------------------------
          438  +# Test the 'b' matchinfo flag
          439  +#
          440  +set sqlite_fts3_enable_parentheses 1
          441  +reset_db
          442  +db func mit mit
          443  +
          444  +do_test 12.0 {
          445  +  set cols [list]
          446  +  for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" }
          447  +  execsql "CREATE VIRTUAL TABLE tt USING fts3([join $cols ,])"
          448  +} {}
          449  +
          450  +do_execsql_test 12.1 {
          451  +  INSERT INTO tt (rowid, c4, c45) VALUES(1, 'abc', 'abc');
          452  +  SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc';
          453  +} [list [list [expr 1<<4] [expr 1<<(45-32)]]]
          454  +
          455  +set sqlite_fts3_enable_parentheses 0
          456  +finish_test
          457  +