SQLite

Check-in [c1f07a3aa9]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Improve fts5 tests.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: c1f07a3aa98eac87e2747527d15e5e5562221ceb
User & Date: dan 2015-04-29 20:54:08.849
Context
2015-05-01
12:14
Improve test coverage of fts5.c. (check-in: add4f4681c user: dan tags: fts5)
2015-04-29
20:54
Improve fts5 tests. (check-in: c1f07a3aa9 user: dan tags: fts5)
2015-04-28
20:24
Fix an fts5 bug in handling writes while there are active cursors. (check-in: 07f7095539 user: dan tags: fts5)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5.c.
525
526
527
528
529
530
531

532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571

572
573
574
575
576
577
578
}

/*
** Close the cursor.  For additional information see the documentation
** on the xClose method of the virtual table interface.
*/
static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){

  Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  Fts5Cursor **pp;
  Fts5Auxdata *pData;
  Fts5Auxdata *pNext;

  fts5CsrNewrow(pCsr);
  if( pCsr->pStmt ){
    int eStmt = fts5StmtType(pCsr->idxNum);
    sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
  }
  if( pCsr->pSorter ){
    Fts5Sorter *pSorter = pCsr->pSorter;
    sqlite3_finalize(pSorter->pStmt);
    sqlite3_free(pSorter);
  }
  
  if( pCsr->idxNum!=FTS5_PLAN_SOURCE ){
    sqlite3Fts5ExprFree(pCsr->pExpr);
  }

  for(pData=pCsr->pAuxdata; pData; pData=pNext){
    pNext = pData->pNext;
    if( pData->xDelete ) pData->xDelete(pData->pPtr);
    sqlite3_free(pData);
  }

  /* Remove the cursor from the Fts5Global.pCsr list */
  for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
  *pp = pCsr->pNext;

  sqlite3_finalize(pCsr->pRankArgStmt);
  sqlite3_free(pCsr->apRankArg);

  sqlite3_free(pCsr->zSpecial);
  if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){
    sqlite3_free(pCsr->zRank);
    sqlite3_free(pCsr->zRankArgs);
  }
  sqlite3_free(pCsr);

  return SQLITE_OK;
}

static int fts5SorterNext(Fts5Cursor *pCsr){
  Fts5Sorter *pSorter = pCsr->pSorter;
  int rc;








>
|
|
|
|
|

|
|
|
|
|
|
|
|
|
|
|
|
|
|

|
|
|
|
|

|
|
|

|
|

|
|
|
|
|
|
>







525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
}

/*
** Close the cursor.  For additional information see the documentation
** on the xClose method of the virtual table interface.
*/
static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
  if( pCursor ){
    Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
    Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
    Fts5Cursor **pp;
    Fts5Auxdata *pData;
    Fts5Auxdata *pNext;

    fts5CsrNewrow(pCsr);
    if( pCsr->pStmt ){
      int eStmt = fts5StmtType(pCsr->idxNum);
      sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
    }
    if( pCsr->pSorter ){
      Fts5Sorter *pSorter = pCsr->pSorter;
      sqlite3_finalize(pSorter->pStmt);
      sqlite3_free(pSorter);
    }

    if( pCsr->idxNum!=FTS5_PLAN_SOURCE ){
      sqlite3Fts5ExprFree(pCsr->pExpr);
    }

    for(pData=pCsr->pAuxdata; pData; pData=pNext){
      pNext = pData->pNext;
      if( pData->xDelete ) pData->xDelete(pData->pPtr);
      sqlite3_free(pData);
    }

    /* Remove the cursor from the Fts5Global.pCsr list */
    for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
    *pp = pCsr->pNext;

    sqlite3_finalize(pCsr->pRankArgStmt);
    sqlite3_free(pCsr->apRankArg);

    sqlite3_free(pCsr->zSpecial);
    if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){
      sqlite3_free(pCsr->zRank);
      sqlite3_free(pCsr->zRankArgs);
    }
    sqlite3_free(pCsr);
  }
  return SQLITE_OK;
}

static int fts5SorterNext(Fts5Cursor *pCsr){
  Fts5Sorter *pSorter = pCsr->pSorter;
  int rc;

889
890
891
892
893
894
895



896

897
898
899
900
901
902
903
){
  int rc = SQLITE_OK;
  if( pRank ){
    const char *z = (const char*)sqlite3_value_text(pRank);
    char *zRank = 0;
    char *zRankArgs = 0;




    rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs);

    if( rc==SQLITE_OK ){
      pCsr->zRank = zRank;
      pCsr->zRankArgs = zRankArgs;
      CsrFlagSet(pCsr, FTS5CSR_FREE_ZRANK);
    }else if( rc==SQLITE_ERROR ){
      pCsr->base.pVtab->zErrMsg = sqlite3_mprintf(
          "parse error in rank function: %s", z







>
>
>
|
>







891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
){
  int rc = SQLITE_OK;
  if( pRank ){
    const char *z = (const char*)sqlite3_value_text(pRank);
    char *zRank = 0;
    char *zRankArgs = 0;

    if( z==0 ){
      if( sqlite3_value_type(pRank)==SQLITE_NULL ) rc = SQLITE_ERROR;
    }else{
      rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs);
    }
    if( rc==SQLITE_OK ){
      pCsr->zRank = zRank;
      pCsr->zRankArgs = zRankArgs;
      CsrFlagSet(pCsr, FTS5CSR_FREE_ZRANK);
    }else if( rc==SQLITE_ERROR ){
      pCsr->base.pVtab->zErrMsg = sqlite3_mprintf(
          "parse error in rank function: %s", z
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216

1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229

1230
1231
1232
1233
1234
1235
1236
1237
  ** Update and insert operations pass:
  **
  **   1. The "old" rowid, or NULL.
  **   2. The "new" rowid.
  **   3. Values for each of the nCol matchable columns.
  **   4. Values for the two hidden columns (<tablename> and "rank").
  */
  assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) );

  eType0 = sqlite3_value_type(apVal[0]);
  eConflict = sqlite3_vtab_on_conflict(pConfig->db);

  assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
  assert( pVtab->zErrMsg==0 );


  fts5TripCursors(pTab);
  if( rc==SQLITE_OK && eType0==SQLITE_INTEGER ){
    if( fts5IsContentless(pTab) ){
      pTab->base.zErrMsg = sqlite3_mprintf(
          "cannot %s contentless fts5 table: %s", 
          (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName
      );
      rc = SQLITE_ERROR;
    }else{
      i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
    }

  }else if( nArg>1 ){
    sqlite3_value *pCmd = apVal[2 + pConfig->nCol];
    if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){
      const char *z = (const char*)sqlite3_value_text(pCmd);
      if( pConfig->eContent!=FTS5_CONTENT_NORMAL 
       && 0==sqlite3_stricmp("delete", z) 
      ){
        return fts5SpecialDelete(pTab, apVal, pRowid);







<






>


|










>
|







1209
1210
1211
1212
1213
1214
1215

1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
  ** Update and insert operations pass:
  **
  **   1. The "old" rowid, or NULL.
  **   2. The "new" rowid.
  **   3. Values for each of the nCol matchable columns.
  **   4. Values for the two hidden columns (<tablename> and "rank").
  */


  eType0 = sqlite3_value_type(apVal[0]);
  eConflict = sqlite3_vtab_on_conflict(pConfig->db);

  assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
  assert( pVtab->zErrMsg==0 );
  assert( (nArg==1 && eType0==SQLITE_INTEGER) || nArg==(2+pConfig->nCol+2) );

  fts5TripCursors(pTab);
  if( eType0==SQLITE_INTEGER ){
    if( fts5IsContentless(pTab) ){
      pTab->base.zErrMsg = sqlite3_mprintf(
          "cannot %s contentless fts5 table: %s", 
          (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName
      );
      rc = SQLITE_ERROR;
    }else{
      i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
    }
  }else{
    assert( nArg>1 );
    sqlite3_value *pCmd = apVal[2 + pConfig->nCol];
    if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){
      const char *z = (const char*)sqlite3_value_text(pCmd);
      if( pConfig->eContent!=FTS5_CONTENT_NORMAL 
       && 0==sqlite3_stricmp("delete", z) 
      ){
        return fts5SpecialDelete(pTab, apVal, pRowid);
1467
1468
1469
1470
1471
1472
1473

1474
1475
1476
1477
1478
1479
1480
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
  int rc = SQLITE_OK;

  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){
    i64 iRowid = fts5CursorRowid(pCsr);
    rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);

  }
  if( iCol<0 ){
    int i;
    *pnToken = 0;
    for(i=0; i<pTab->pConfig->nCol; i++){
      *pnToken += pCsr->aColumnSize[i];
    }







>







1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
  int rc = SQLITE_OK;

  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){
    i64 iRowid = fts5CursorRowid(pCsr);
    rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
    CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE);
  }
  if( iCol<0 ){
    int i;
    *pnToken = 0;
    for(i=0; i<pTab->pConfig->nCol; i++){
      *pnToken += pCsr->aColumnSize[i];
    }
1874
1875
1876
1877
1878
1879
1880

















1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920

1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933

1934

1935
1936
1937



1938
1939
1940
1941
1942
1943
1944
    }
  }else{
    rc = SQLITE_NOMEM;
  }

  return rc;
}


















/*
** Find a tokenizer. This is the implementation of the 
** fts5_api.xFindTokenizer() method.
*/
static int fts5FindTokenizer(
  fts5_api *pApi,                 /* Global context (one per db handle) */
  const char *zName,              /* Name of new function */
  void **ppUserData,
  fts5_tokenizer *pTokenizer      /* Populate this object */
){
  Fts5Global *pGlobal = (Fts5Global*)pApi;
  int rc = SQLITE_OK;
  Fts5TokenizerModule *pTok;

  if( zName==0 ){
    pTok = pGlobal->pDfltTok;
  }else{
    for(pTok=pGlobal->pTok; pTok; pTok=pTok->pNext){
      if( sqlite3_stricmp(zName, pTok->zName)==0 ) break;
    }
  }

  if( pTok ){
    *pTokenizer = pTok->x;
    *ppUserData = pTok->pUserData;
  }else{
    memset(pTokenizer, 0, sizeof(fts5_tokenizer));
    rc = SQLITE_ERROR;
  }

  return rc;
}

int sqlite3Fts5GetTokenizer(
  Fts5Global *pGlobal, 
  const char **azArg,
  int nArg,
  Fts5Tokenizer **ppTok,
  fts5_tokenizer **ppTokApi

){
  Fts5TokenizerModule *pMod = 0;
  int rc = SQLITE_OK;

  if( nArg==0 ){
    pMod = pGlobal->pDfltTok;
  }else{
    for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){
      if( sqlite3_stricmp(azArg[0], pMod->zName)==0 ) break;
    }
  }

  if( pMod==0 ){

    rc = SQLITE_ERROR;

  }else{
    rc = pMod->x.xCreate(pMod->pUserData, &azArg[1], (nArg?nArg-1:0), ppTok);
    *ppTokApi = &pMod->x;



  }

  if( rc!=SQLITE_OK ){
    *ppTokApi = 0;
    *ppTok = 0;
  }








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>











<

|

<
<
<
<
<
<
<
|
|
|
|













|
>

|


<
<
<
<
<
<
<
|

>

>



>
>
>







1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916

1917
1918
1919







1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942







1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
    }
  }else{
    rc = SQLITE_NOMEM;
  }

  return rc;
}

static Fts5TokenizerModule *fts5LocateTokenizer(
  Fts5Global *pGlobal, 
  const char *zName
){
  Fts5TokenizerModule *pMod = 0;

  if( zName==0 ){
    pMod = pGlobal->pDfltTok;
  }else{
    for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){
      if( sqlite3_stricmp(zName, pMod->zName)==0 ) break;
    }
  }

  return pMod;
}

/*
** Find a tokenizer. This is the implementation of the 
** fts5_api.xFindTokenizer() method.
*/
static int fts5FindTokenizer(
  fts5_api *pApi,                 /* Global context (one per db handle) */
  const char *zName,              /* Name of new function */
  void **ppUserData,
  fts5_tokenizer *pTokenizer      /* Populate this object */
){

  int rc = SQLITE_OK;
  Fts5TokenizerModule *pMod;








  pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName);
  if( pMod ){
    *pTokenizer = pMod->x;
    *ppUserData = pMod->pUserData;
  }else{
    memset(pTokenizer, 0, sizeof(fts5_tokenizer));
    rc = SQLITE_ERROR;
  }

  return rc;
}

int sqlite3Fts5GetTokenizer(
  Fts5Global *pGlobal, 
  const char **azArg,
  int nArg,
  Fts5Tokenizer **ppTok,
  fts5_tokenizer **ppTokApi,
  char **pzErr
){
  Fts5TokenizerModule *pMod;
  int rc = SQLITE_OK;








  pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]);
  if( pMod==0 ){
    assert( nArg>0 );
    rc = SQLITE_ERROR;
    *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]);
  }else{
    rc = pMod->x.xCreate(pMod->pUserData, &azArg[1], (nArg?nArg-1:0), ppTok);
    *ppTokApi = &pMod->x;
    if( rc!=SQLITE_OK && pzErr ){
      *pzErr = sqlite3_mprintf("error in tokenizer constructor");
    }
  }

  if( rc!=SQLITE_OK ){
    *ppTokApi = 0;
    *ppTok = 0;
  }

Changes to ext/fts5/fts5Int.h.
57
58
59
60
61
62
63
64

65
66
67
68
69
70
71
typedef struct Fts5Global Fts5Global;

int sqlite3Fts5GetTokenizer(
  Fts5Global*, 
  const char **azArg,
  int nArg,
  Fts5Tokenizer**,
  fts5_tokenizer**

);

/*
** End of interface to code in fts5.c.
**************************************************************************/

/**************************************************************************







|
>







57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
typedef struct Fts5Global Fts5Global;

int sqlite3Fts5GetTokenizer(
  Fts5Global*, 
  const char **azArg,
  int nArg,
  Fts5Tokenizer**,
  fts5_tokenizer**,
  char **pzErr
);

/*
** End of interface to code in fts5.c.
**************************************************************************/

/**************************************************************************
Changes to ext/fts5/fts5_aux.c.
226
227
228
229
230
231
232
233
234
235
236
237



238
239
240
241
242
243
244
    if( rc==SQLITE_OK ){
      rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
    }
    fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);

    if( rc==SQLITE_OK ){
      sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
    }else{
      sqlite3_result_error_code(pCtx, rc);
    }
    sqlite3_free(ctx.zOut);
  }



}
/*
** End of highlight() implementation.
**************************************************************************/

/*
** Implementation of snippet() function.







<
<



>
>
>







226
227
228
229
230
231
232


233
234
235
236
237
238
239
240
241
242
243
244
245
    if( rc==SQLITE_OK ){
      rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
    }
    fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);

    if( rc==SQLITE_OK ){
      sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);


    }
    sqlite3_free(ctx.zOut);
  }
  if( rc!=SQLITE_OK ){
    sqlite3_result_error_code(pCtx, rc);
  }
}
/*
** End of highlight() implementation.
**************************************************************************/

/*
** Implementation of snippet() function.
Changes to ext/fts5/fts5_config.c.
326
327
328
329
330
331
332
333

334
335
336
337
338
339
340
341
342
343
344
          }
        }
        if( p==0 ){
          *pzErr = sqlite3_mprintf("parse error in tokenize directive");
          rc = SQLITE_ERROR;
        }else{
          rc = sqlite3Fts5GetTokenizer(pGlobal, 
              (const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi

          );
          if( rc!=SQLITE_OK ){
            *pzErr = sqlite3_mprintf("error in tokenizer constructor");
          }
        }
      }
    }

    sqlite3_free(azArg);
    sqlite3_free(pDel);
    return rc;







|
>

<
<
<







326
327
328
329
330
331
332
333
334
335



336
337
338
339
340
341
342
          }
        }
        if( p==0 ){
          *pzErr = sqlite3_mprintf("parse error in tokenize directive");
          rc = SQLITE_ERROR;
        }else{
          rc = sqlite3Fts5GetTokenizer(pGlobal, 
              (const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi,
              pzErr
          );



        }
      }
    }

    sqlite3_free(azArg);
    sqlite3_free(pDel);
    return rc;
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
** Allocate an instance of the default tokenizer ("simple") at 
** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
** code if an error occurs.
*/
static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
  assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
  return sqlite3Fts5GetTokenizer(
      pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi
  );
}

/*
** Gobble up the first bareword or quoted word from the input buffer zIn.
** Return a pointer to the character immediately following the last in
** the gobbled word if successful, or a NULL pointer otherwise (failed







|







381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
** Allocate an instance of the default tokenizer ("simple") at 
** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
** code if an error occurs.
*/
static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
  assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
  return sqlite3Fts5GetTokenizer(
      pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi, 0
  );
}

/*
** Gobble up the first bareword or quoted word from the input buffer zIn.
** Return a pointer to the character immediately following the last in
** the gobbled word if successful, or a NULL pointer otherwise (failed
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573

    if( rc==SQLITE_OK ){
      if( z==0 ){
        *pzErr = sqlite3_mprintf("parse error in \"%s\"", zOrig);
        rc = SQLITE_ERROR;
      }else{
        if( bOption ){
          rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo, pzErr);
        }else{
          rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr);
          zOne = 0;
        }
      }
    }








|







557
558
559
560
561
562
563
564
565
566
567
568
569
570
571

    if( rc==SQLITE_OK ){
      if( z==0 ){
        *pzErr = sqlite3_mprintf("parse error in \"%s\"", zOrig);
        rc = SQLITE_ERROR;
      }else{
        if( bOption ){
          rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo?zTwo:"", pzErr);
        }else{
          rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr);
          zOne = 0;
        }
      }
    }

Changes to ext/fts5/fts5_expr.c.
276
277
278
279
280
281
282



283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
  Fts5Expr *pNew;
  Fts5ExprPhrase **apPhrase;
  Fts5ExprNode *pNode;
  Fts5ExprNearset *pNear;
  Fts5ExprPhrase *pCopy;

  pOrig = pExpr->apExprPhrase[iPhrase];



  pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr));
  apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*));
  pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode));
  pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, 
      sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)
  );
  pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, 
      sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm
  );

  for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
    pCopy->aTerm[i].zTerm = fts5ExprStrdup(&rc, pOrig->aTerm[i].zTerm);
    pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
  }

  if( rc==SQLITE_OK ){







>
>
>






<
<
<







276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291



292
293
294
295
296
297
298
  Fts5Expr *pNew;
  Fts5ExprPhrase **apPhrase;
  Fts5ExprNode *pNode;
  Fts5ExprNearset *pNear;
  Fts5ExprPhrase *pCopy;

  pOrig = pExpr->apExprPhrase[iPhrase];
  pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, 
      sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm
  );
  pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr));
  apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*));
  pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode));
  pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, 
      sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)
  );




  for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
    pCopy->aTerm[i].zTerm = fts5ExprStrdup(&rc, pOrig->aTerm[i].zTerm);
    pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
  }

  if( rc==SQLITE_OK ){
Changes to ext/fts5/fts5_tcl.c.
32
33
34
35
36
37
38



39
40
41
42
43
44
45

/*************************************************************************
** This is a copy of the first part of the SqliteDb structure in 
** tclsqlite.c.  We need it here so that the get_sqlite_pointer routine
** can extract the sqlite3* pointer from an existing Tcl SQLite
** connection.
*/



struct SqliteDb {
  sqlite3 *db;
};

/*
** Decode a pointer to an sqlite3 object.
*/







>
>
>







32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

/*************************************************************************
** This is a copy of the first part of the SqliteDb structure in 
** tclsqlite.c.  We need it here so that the get_sqlite_pointer routine
** can extract the sqlite3* pointer from an existing Tcl SQLite
** connection.
*/

extern const char *sqlite3ErrName(int);

struct SqliteDb {
  sqlite3 *db;
};

/*
** Decode a pointer to an sqlite3 object.
*/
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
    default: 
      assert( 0 );
      break;
  }
#undef CASE

  if( rc!=SQLITE_OK ){
    Tcl_AppendResult(interp, "error in api call", 0);
    return TCL_ERROR;
  }

  return TCL_OK;
}

static void xF5tFunction(







|







389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
    default: 
      assert( 0 );
      break;
  }
#undef CASE

  if( rc!=SQLITE_OK ){
    Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
    return TCL_ERROR;
  }

  return TCL_OK;
}

static void xF5tFunction(
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
  Tcl_DecrRefCount(pEval);

  pInst->pContext->pCtx = pOldCtx;
  pInst->pContext->xToken = xOldToken;
  return rc;
}

extern const char *sqlite3ErrName(int);

/*
** sqlite3_fts5_token TEXT START END POS
*/
static int f5tTokenizerReturn(
  void * clientData,
  Tcl_Interp *interp,
  int objc,







<
<







726
727
728
729
730
731
732


733
734
735
736
737
738
739
  Tcl_DecrRefCount(pEval);

  pInst->pContext->pCtx = pOldCtx;
  pInst->pContext->xToken = xOldToken;
  return rc;
}



/*
** sqlite3_fts5_token TEXT START END POS
*/
static int f5tTokenizerReturn(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
Changes to ext/fts5/fts5_tokenize.c.
525
526
527
528
529
530
531





532
533
534
535
536
537
538
539
540
541
542
543
  const char **azArg, int nArg,
  Fts5Tokenizer **ppOut
){
  fts5_api *pApi = (fts5_api*)pCtx;
  int rc = SQLITE_OK;
  PorterTokenizer *pRet;
  void *pUserdata = 0;






  pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer));
  if( pRet ){
    memset(pRet, 0, sizeof(PorterTokenizer));
    rc = pApi->xFindTokenizer(pApi, "unicode61", &pUserdata, &pRet->tokenizer);
  }else{
    rc = SQLITE_NOMEM;
  }
  if( rc==SQLITE_OK ){
    rc = pRet->tokenizer.xCreate(pUserdata, 0, 0, &pRet->pTokenizer);
  }








>
>
>
>
>




|







525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
  const char **azArg, int nArg,
  Fts5Tokenizer **ppOut
){
  fts5_api *pApi = (fts5_api*)pCtx;
  int rc = SQLITE_OK;
  PorterTokenizer *pRet;
  void *pUserdata = 0;
  const char *zBase = "unicode61";

  if( nArg>0 ){
    zBase = azArg[0];
  }

  pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer));
  if( pRet ){
    memset(pRet, 0, sizeof(PorterTokenizer));
    rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer);
  }else{
    rc = SQLITE_NOMEM;
  }
  if( rc==SQLITE_OK ){
    rc = pRet->tokenizer.xCreate(pUserdata, 0, 0, &pRet->pTokenizer);
  }

Changes to ext/fts5/test/fts5aa.test.
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
} {}

do_catchsql_test 12.2 {
  SELECT t2 FROM t2 WHERE t2 MATCH '*stuff'
} {1 {unknown special query: stuff}}

do_test 12.3 {
  set res [db one { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }]
  string is integer $res
} {1}

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 13.1 {







|







293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
} {}

do_catchsql_test 12.2 {
  SELECT t2 FROM t2 WHERE t2 MATCH '*stuff'
} {1 {unknown special query: stuff}}

do_test 12.3 {
  set res [db eval { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }]
  string is integer $res
} {1}

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 13.1 {
Changes to ext/fts5/test/fts5al.test.
263
264
265
266
267
268
269








270
271
272
273
do_execsql_test 4.3.3 {
  SELECT *, rank FROM t3
  WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(3)' 
  ORDER BY rank ASC
} {
  {a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2 
}










finish_test








>
>
>
>
>
>
>
>




263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
do_execsql_test 4.3.3 {
  SELECT *, rank FROM t3
  WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(3)' 
  ORDER BY rank ASC
} {
  {a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2 
}

do_catchsql_test 4.4.3 {
  SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'xyz(3)' 
} {1 {no such function: xyz}}
do_catchsql_test 4.4.4 {
  SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
} {1 {parse error in rank function: }}



finish_test

Added ext/fts5/test/fts5aux.test.


















































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
# 2014 Dec 20
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on the auxiliary function APIs.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5aux

proc inst {cmd i} { 
  $cmd xInst $i
}
sqlite3_fts5_create_function db inst inst

proc colsize {cmd i} { 
  $cmd xColumnSize $i
}
sqlite3_fts5_create_function db colsize colsize

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE f1 USING fts5(a, b);
  INSERT INTO f1 VALUES('one two', 'two one zero');
}

do_catchsql_test 1.1 {
  SELECT inst(f1, -1) FROM f1 WHERE f1 MATCH 'two';
} {1 SQLITE_RANGE}
do_catchsql_test 1.2 {
  SELECT inst(f1, 0) FROM f1 WHERE f1 MATCH 'two';
} {0 {{0 0 1}}}
do_catchsql_test 1.3 {
  SELECT inst(f1, 1) FROM f1 WHERE f1 MATCH 'two';
} {0 {{0 1 0}}}
do_catchsql_test 1.4 {
  SELECT inst(f1, 2) FROM f1 WHERE f1 MATCH 'two';
} {1 SQLITE_RANGE}

do_catchsql_test 2.1 {
  SELECT colsize(f1, 2) FROM f1 WHERE f1 MATCH 'two';
} {1 SQLITE_RANGE}

do_execsql_test 2.2 {
  SELECT colsize(f1, 0), colsize(f1, 1) FROM f1 WHERE f1 MATCH 'zero';
} {2 3}



finish_test

Changes to ext/fts5/test/fts5content.test.
210
211
212
213
214
215
216


















217
218
219
220
221
  DELETE FROM x2 WHERE "key col" = 1;
  INSERT INTO t2(t2, rowid, a, c) VALUES('delete', 1, 'a b', 'e f');
  INSERT INTO t2(t2) VALUES('integrity-check');
}

do_execsql_test 4.8 { SELECT rowid FROM t2 WHERE t2 MATCH 'b'} {}
do_execsql_test 4.9 { SELECT rowid FROM t2 WHERE t2 MATCH 'y'} {-40}




















finish_test









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>





210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
  DELETE FROM x2 WHERE "key col" = 1;
  INSERT INTO t2(t2, rowid, a, c) VALUES('delete', 1, 'a b', 'e f');
  INSERT INTO t2(t2) VALUES('integrity-check');
}

do_execsql_test 4.8 { SELECT rowid FROM t2 WHERE t2 MATCH 'b'} {}
do_execsql_test 4.9 { SELECT rowid FROM t2 WHERE t2 MATCH 'y'} {-40}

#-------------------------------------------------------------------------
# Test that if the 'rowid' field of a 'delete' is not an integer, no
# changes are made to the FTS index.
#
do_execsql_test 5.0 {
  CREATE VIRTUAL TABLE t5 USING fts5(a, b, content=);
  INSERT INTO t5(rowid, a, b) VALUES(-1, 'one',   'two');
  INSERT INTO t5(rowid, a, b) VALUES( 0, 'three', 'four');
  INSERT INTO t5(rowid, a, b) VALUES( 1, 'five',  'six');
}

set ::checksum [execsql {SELECT md5sum(id, block) FROM t5_data}]

do_execsql_test 5.1 {
  INSERT INTO t5(t5, rowid, a, b) VALUES('delete', NULL, 'three', 'four');
  SELECT md5sum(id, block) FROM t5_data;
} $::checksum


finish_test


Changes to ext/fts5/test/fts5corrupt.test.
66
67
68
69
70
71
72

73
















74
75
76
  for {set i 0} {$i < 500} {incr i} {
    execsql { INSERT INTO t2 VALUES(rnddoc(50)) }
  }
  execsql { INSERT INTO t2(t2) VALUES('integrity-check') }
} {}

#--------------------------------------------------------------------

#

















finish_test








>

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
  for {set i 0} {$i < 500} {incr i} {
    execsql { INSERT INTO t2 VALUES(rnddoc(50)) }
  }
  execsql { INSERT INTO t2(t2) VALUES('integrity-check') }
} {}

#--------------------------------------------------------------------
# A mundane test - missing row in the %_content table.
#
do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE t3 USING fts5(x);
  INSERT INTO t3 VALUES('one o');
  INSERT INTO t3 VALUES('two e');
  INSERT INTO t3 VALUES('three o');
  INSERT INTO t3 VALUES('four e');
  INSERT INTO t3 VALUES('five o');
}
do_execsql_test 3.1 {
  SELECT * FROM t3 WHERE t3 MATCH 'o'
} {{one o} {three o} {five o}}

do_catchsql_test 3.1 {
  DELETE FROM t3_content WHERE rowid = 3;
  SELECT * FROM t3 WHERE t3 MATCH 'o';
} {1 {database disk image is malformed}}

finish_test

Added ext/fts5/test/fts5doclist.test.


















































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
# 2015 April 21
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This test is focused on edge cases in the doclist format.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5doclist


#-------------------------------------------------------------------------
# Create a table with 1000 columns. Then add some large documents to it.
# All text is in the right most column of the table.
#
do_test 1.0 {
  set cols [list]
  for {set i 0} {$i < 900} {incr i} { lappend cols "x$i" }
  execsql "CREATE VIRTUAL TABLE ccc USING fts5([join $cols ,])"
} {}

db func rnddoc fts5_rnddoc 
do_execsql_test 1.1 {
  WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
  INSERT INTO ccc(x899) SELECT rnddoc(500) FROM ii;
}

do_execsql_test 1.2 {
  INSERT INTO ccc(ccc) VALUES('integrity-check');
}


finish_test

Changes to ext/fts5/test/fts5fault4.test.
17
18
19
20
21
22
23


24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42


43


















































































































































44
45
set testprefix fts5fault4

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}



#-------------------------------------------------------------------------
# An OOM while dropping an fts5 table.
#
db func rnddoc fts5_rnddoc 
do_test 1.0 {
  execsql { CREATE VIRTUAL TABLE xx USING fts5(x) }
} {}
faultsim_save_and_close

do_faultsim_test 1 -faults oom-* -prep {
  faultsim_restore_and_reopen
  execsql { SELECT * FROM xx }
} -body {
  execsql { DROP TABLE xx }
} -test {
  faultsim_test_result [list 0 {}]
}






















































































































































finish_test








>
>



















>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
set testprefix fts5fault4

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}

if 1 {

#-------------------------------------------------------------------------
# An OOM while dropping an fts5 table.
#
db func rnddoc fts5_rnddoc 
do_test 1.0 {
  execsql { CREATE VIRTUAL TABLE xx USING fts5(x) }
} {}
faultsim_save_and_close

do_faultsim_test 1 -faults oom-* -prep {
  faultsim_restore_and_reopen
  execsql { SELECT * FROM xx }
} -body {
  execsql { DROP TABLE xx }
} -test {
  faultsim_test_result [list 0 {}]
}

#-------------------------------------------------------------------------
# An OOM within an "ORDER BY rank" query.
#
db func rnddoc fts5_rnddoc 
do_execsql_test 2.0 {
  CREATE VIRTUAL TABLE xx USING fts5(x);
  INSERT INTO xx VALUES ('abc ' || rnddoc(10));
  INSERT INTO xx VALUES ('abc abc' || rnddoc(9));
  INSERT INTO xx VALUES ('abc abc abc' || rnddoc(8));
} {}
faultsim_save_and_close

do_faultsim_test 2 -faults oom-* -prep {
  faultsim_restore_and_reopen
  execsql { SELECT * FROM xx }
} -body {
  execsql { SELECT rowid FROM xx WHERE xx MATCH 'abc' ORDER BY rank }
} -test {
  faultsim_test_result [list 0 {3 2 1}]
}

#-------------------------------------------------------------------------
# An OOM while "reseeking" an FTS cursor.
#
do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE jj USING fts5(j);
  INSERT INTO jj(rowid, j) VALUES(101, 'm t w t f s s');
  INSERT INTO jj(rowid, j) VALUES(202, 't w t f s');
  INSERT INTO jj(rowid, j) VALUES(303, 'w t f');
  INSERT INTO jj(rowid, j) VALUES(404, 't');
}
faultsim_save_and_close

do_faultsim_test 3 -faults oom-* -prep {
  faultsim_restore_and_reopen
  execsql { SELECT * FROM jj }
} -body {
  set res [list]
  db eval { SELECT rowid FROM jj WHERE jj MATCH 't' } {
    lappend res $rowid
    if {$rowid==303} {
      execsql { DELETE FROM jj WHERE rowid=404 }
    }
  }
  set res
} -test {
  faultsim_test_result [list 0 {101 202 303}]
}

#-------------------------------------------------------------------------
# An OOM within a special "*reads" query.
#
reset_db
db func rnddoc fts5_rnddoc
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE x1 USING fts5(x);
  INSERT INTO x1(x1, rank) VALUES('pgsz', 32);

  WITH ii(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10 )
  INSERT INTO x1 SELECT rnddoc(5) FROM ii;
}

set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}]

do_faultsim_test 4 -faults oom-* -body {
  db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'}
} -test {
  faultsim_test_result {0 {0 {} 3}}
}

#-------------------------------------------------------------------------
# An OOM within a query that uses a custom rank function.
#
reset_db
do_execsql_test 5.0 {
  PRAGMA encoding='utf16';
  CREATE VIRTUAL TABLE x2 USING fts5(x);
  INSERT INTO x2(rowid, x) VALUES(10, 'a b c'); -- 3
  INSERT INTO x2(rowid, x) VALUES(20, 'a b c'); -- 6
  INSERT INTO x2(rowid, x) VALUES(30, 'a b c'); -- 2
  INSERT INTO x2(rowid, x) VALUES(40, 'a b c'); -- 5
  INSERT INTO x2(rowid, x) VALUES(50, 'a b c'); -- 1
}

proc rowidmod {cmd mod} { 
  set row [$cmd xRowid]
  expr {$row % $mod}
}
sqlite3_fts5_create_function db rowidmod rowidmod

do_faultsim_test 5.1 -faults oom-* -body {
  db eval {
    SELECT rowid || '-' || rank FROM x2 WHERE x2 MATCH 'b' AND 
    rank MATCH "rowidmod('7')" ORDER BY rank
  }
} -test {
  faultsim_test_result {0 {50-1 30-2 10-3 40-5 20-6}}
}

proc rowidprefix {cmd prefix} { 
  set row [$cmd xRowid]
  set {} "${row}-${prefix}"
}
sqlite3_fts5_create_function db rowidprefix rowidprefix

set str [string repeat abcdefghijklmnopqrstuvwxyz 10]
do_faultsim_test 5.2 -faults oom-* -body {
  db eval "
    SELECT rank, x FROM x2 WHERE x2 MATCH 'b' AND 
    rank MATCH 'rowidprefix(''$::str'')'
    LIMIT 1
  "
} -test {
  faultsim_test_result "0 {10-$::str {a b c}}"
}

}


#-------------------------------------------------------------------------
# OOM errors within auxiliary functions.
#
reset_db
do_execsql_test 6.0 {
  CREATE VIRTUAL TABLE x3 USING fts5(xxx);
  INSERT INTO x3 VALUES('a b c d c b a');
}

do_faultsim_test 6.1 -faults oom-t* -body {
  db eval { SELECT highlight(x3, 0, '*', '*') FROM x3 WHERE x3 MATCH 'c' }
} -test {
  faultsim_test_result {0 {{a b *c* d *c* b a}}}
}

proc firstinst {cmd} { 
  foreach {p c o} [$cmd xInst 0] {}
  expr $c*100 + $o
}
sqlite3_fts5_create_function db firstinst firstinst

do_faultsim_test 6.2 -faults oom-t* -body {
  db eval { SELECT firstinst(x3) FROM x3 WHERE x3 MATCH 'c' }
} -test {
  faultsim_test_result {0 2} {1 SQLITE_NOMEM}
}




finish_test

Added ext/fts5/test/fts5plan.test.


























































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
# 2014 Dec 20
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file focuses on testing the planner (xBestIndex function).
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5plan

do_execsql_test 1.0 {
  CREATE TABLE t1(x, y);
  CREATE VIRTUAL TABLE f1 USING fts5(ff);
}

do_eqp_test 1.1 {
  SELECT * FROM t1, f1 WHERE f1 MATCH t1.x
} {
  0 0 0 {SCAN TABLE t1} 
  0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
}

do_eqp_test 1.2 {
  SELECT * FROM t1, f1 WHERE f1 > t1.x
} {
  0 0 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
  0 1 0 {SCAN TABLE t1} 
}

do_eqp_test 1.3 {
  SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff
} {
  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
}

do_eqp_test 1.4 {
  SELECT * FROM f1 ORDER BY rank
} {
  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
}

do_eqp_test 1.5 {
  SELECT * FROM f1 WHERE rank MATCH ?
} {
  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
}




finish_test

Added ext/fts5/test/fts5rank.test.














































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
# 2014 Dec 20
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file focuses on testing queries that use the "rank" column.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5rank


#-------------------------------------------------------------------------
# "ORDER BY rank" + highlight() + large poslists.
#
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE xyz USING fts5(z);
}
do_test 1.1 {
  set doc [string trim [string repeat "x y " 500]]
  execsql { INSERT INTO xyz VALUES($doc) }
} {}
do_execsql_test 1.2 {
  SELECT highlight(xyz, 0, '[', ']') FROM xyz WHERE xyz MATCH 'x' ORDER BY rank
} [list [string map {x [x]} $doc]]

do_execsql_test 1.3 {
  SELECT highlight(xyz, 0, '[', ']') FROM xyz
  WHERE xyz MATCH 'x AND y' ORDER BY rank
} [list [string map {x [x] y [y]} $doc]]

finish_test

Changes to ext/fts5/test/fts5rebuild.test.
42
43
44
45
46
47
48











49
50
} {1 {database disk image is malformed}}

do_execsql_test 1.7 {
  INSERT INTO f1(f1) VALUES('rebuild');
  INSERT INTO f1(f1) VALUES('integrity-check');
} {}












finish_test








>
>
>
>
>
>
>
>
>
>
>


42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
} {1 {database disk image is malformed}}

do_execsql_test 1.7 {
  INSERT INTO f1(f1) VALUES('rebuild');
  INSERT INTO f1(f1) VALUES('integrity-check');
} {}


#-------------------------------------------------------------------------
# Check that 'rebuild' may not be used with a contentless table.
#
do_execsql_test 2.1 {
  CREATE VIRTUAL TABLE nc USING fts5(doc, content=);
}

do_catchsql_test 2.2 {
  INSERT INTO nc(nc) VALUES('rebuild');
} {1 {'rebuild' may not be used with a contentless fts5 table}}
finish_test

Changes to ext/fts5/test/fts5restart.test.
15
16
17
18
19
20
21




22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43































































































44
45
46
47
48
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5restart

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE f1 USING fts5(ff);
}





do_test 1.1 {
  for {set i 1} {$i < 1000} {incr i} {
    execsql { INSERT INTO f1 VALUES('a b c d e') }
    lappend lRowid $i
  }
} {}

do_execsql_test 1.2 {
  SELECT rowid FROM f1 WHERE f1 MATCH 'c';
} $lRowid

breakpoint
do_test 1.3 {
  set res [list]
  db eval { SELECT rowid FROM f1 WHERE f1 MATCH 'c' } {
    if {$rowid == 100} {
      execsql { INSERT INTO f1(f1) VALUES('optimize') }
    }
    lappend res $rowid
  }
  set res
} $lRowid


































































































finish_test








>
>
>
>











<










>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>





15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5restart

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE f1 USING fts5(ff);
}

#-------------------------------------------------------------------------
# Run the 'optimize' command. Check that it does not disturb ongoing
# full-text queries.
#
do_test 1.1 {
  for {set i 1} {$i < 1000} {incr i} {
    execsql { INSERT INTO f1 VALUES('a b c d e') }
    lappend lRowid $i
  }
} {}

do_execsql_test 1.2 {
  SELECT rowid FROM f1 WHERE f1 MATCH 'c';
} $lRowid


do_test 1.3 {
  set res [list]
  db eval { SELECT rowid FROM f1 WHERE f1 MATCH 'c' } {
    if {$rowid == 100} {
      execsql { INSERT INTO f1(f1) VALUES('optimize') }
    }
    lappend res $rowid
  }
  set res
} $lRowid

do_test 1.4.1 {
  sqlite3 db2 test.db
  set res [list]
  db2 eval { SELECT rowid FROM f1 WHERE f1 MATCH 'c' } {
    if {$rowid == 100} {
      set cres [catchsql { INSERT INTO f1(f1) VALUES('optimize') }]
    }
    lappend res $rowid
  }
  set res
} $lRowid

do_test 1.4.2 {
  db2 close
  set cres
} {1 {database is locked}}

#-------------------------------------------------------------------------
# Open a couple of cursors. Then close them in the same order.
#
do_test 2.1 {
  set ::s1 [sqlite3_prepare db "SELECT rowid FROM f1 WHERE f1 MATCH 'b'" -1 X]
  set ::s2 [sqlite3_prepare db "SELECT rowid FROM f1 WHERE f1 MATCH 'c'" -1 X]

  sqlite3_step $::s1
} {SQLITE_ROW}
do_test 2.2 {
  sqlite3_step $::s2
} {SQLITE_ROW}

do_test 2.1 {
  sqlite3_finalize $::s1
  sqlite3_finalize $::s2
} {SQLITE_OK}

#-------------------------------------------------------------------------
# Copy data between two FTS5 tables.
#
do_execsql_test 3.1 {
  CREATE VIRTUAL TABLE f2 USING fts5(gg);
  INSERT INTO f2 SELECT ff FROM f1 WHERE f1 MATCH 'b+c+d';
}
do_execsql_test 3.2 {
  SELECT rowid FROM f2 WHERE f2 MATCH 'a+b+c+d+e'
} $lRowid

#-------------------------------------------------------------------------
# Remove the row that an FTS5 cursor is currently pointing to. And 
# various other similar things. Check that this does not disturb 
# ongoing scans.
#
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE n4 USING fts5(n);
  INSERT INTO n4(rowid, n) VALUES(100, '1 2 3 4 5');
  INSERT INTO n4(rowid, n) VALUES(200, '1 2 3 4');
  INSERT INTO n4(rowid, n) VALUES(300, '2 3 4');
  INSERT INTO n4(rowid, n) VALUES(400, '2 3');
  INSERT INTO n4(rowid, n) VALUES(500, '3');
}

do_test 4.1 {
  set res [list]
  db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' } {
    if {$rowid==300} {
      execsql { DELETE FROM n4 WHERE rowid=300 }
    }
    lappend res $rowid
  }
  set res
} {100 200 300 400 500}

do_test 4.2 {
  execsql { INSERT INTO n4(rowid, n) VALUES(300, '2 3 4') }
  set res [list]
  db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' ORDER BY rowid DESC} {
    if {$rowid==300} {
      execsql { DELETE FROM n4 WHERE rowid=300 }
    }
    lappend res $rowid
  }
  set res
} {500 400 300 200 100}

do_test 4.3 {
  execsql { INSERT INTO n4(rowid, n) VALUES(300, '2 3 4') }
  set res [list]
  db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' ORDER BY rowid DESC} {
    if {$rowid==300} {
      execsql { DELETE FROM n4  }
    }
    lappend res $rowid
  }
  set res
} {500 400 300}



finish_test

Changes to ext/fts5/test/fts5tokenizer.test.
33
34
35
36
37
38
39








40
41
42
43
44
45
46
  DROP TABLE ft1;
}
do_execsql_test 1.4 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter ascii');
  DROP TABLE ft1;
}









do_execsql_test 2.0 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize=porter);
  INSERT INTO ft1 VALUES('embedded databases');
}
do_execsql_test 2.1 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'embedding' } 1
do_execsql_test 2.2 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'database' } 1
do_execsql_test 2.3 { 







>
>
>
>
>
>
>
>







33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
  DROP TABLE ft1;
}
do_execsql_test 1.4 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter ascii');
  DROP TABLE ft1;
}

do_catchsql_test 1.5 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'nosuch');
} {1 {no such tokenizer: nosuch}}

do_catchsql_test 1.6 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter nosuch');
} {1 {error in tokenizer constructor}}

do_execsql_test 2.0 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize=porter);
  INSERT INTO ft1 VALUES('embedded databases');
}
do_execsql_test 2.1 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'embedding' } 1
do_execsql_test 2.2 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'database' } 1
do_execsql_test 2.3 {