/ Check-in [fba0b5fc]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix the customization interfaces so that they match the documentation.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: fba0b5fc7eead07a4853e78e02d788e7c714f6cd
User & Date: dan 2014-11-15 20:07:31
Context
2014-11-24
16:24
Add the auxiliary highlight() function to fts5. check-in: 05909237 user: dan tags: fts5
2014-11-15
20:07
Fix the customization interfaces so that they match the documentation. check-in: fba0b5fc user: dan tags: fts5
2014-08-25
19:58
Add documentation for tokenizer api to fts5.h. Also add a script to extract extension API docs and format them as html. check-in: e240d467 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/extract_api_docs.tcl.

    10     10   #
    11     11   #--------------------------------------------------------------------------
    12     12   #
    13     13   # This script extracts the documentation for the API used by fts5 auxiliary 
    14     14   # functions from header file fts5.h. It outputs html text on stdout that
    15     15   # is included in the documentation on the web.
    16     16   # 
           17  +
           18  +set ::fts5_docs_output ""
           19  +if {[info commands hd_putsnl]==""} {
           20  +  proc output {text} {
           21  +    puts $text
           22  +  }
           23  +} else {
           24  +  proc output {text} {
           25  +    append ::fts5_docs_output $text
           26  +  }
           27  +}
    17     28   
    18     29   set input_file [file join [file dir [info script]] fts5.h]
    19     30   set fd [open $input_file]
    20     31   set data [read $fd]
    21     32   close $fd
    22     33   
    23     34   
................................................................................
    98    109   #
    99    110   set D [get_struct_docs $data [array names M]]
   100    111   
   101    112   foreach {hdr docs} $D {
   102    113     if {[info exists M($hdr)]} {
   103    114       set hdr $M($hdr)
   104    115     }
   105         -  puts "<h3><pre>  $hdr</pre></h3>"
          116  +  output "<h style=\"font-size:1.4em;background-color:#EEEEEE;display:block\"><pre>  $hdr</pre></h>"
   106    117   
   107    118     set mode ""
   108    119     set bEmpty 1
   109    120     foreach line [split [string trim $docs] "\n"] {
   110    121       if {[string trim $line]==""} {
   111         -      if {$mode != ""} {puts "</$mode>"}
          122  +      if {$mode != ""} {output "</$mode>"}
   112    123         set mode ""
   113    124       } elseif {$mode == ""} {
   114    125         if {[regexp {^     } $line]} {
   115         -        set mode code
          126  +        set mode codeblock
   116    127         } else {
   117    128           set mode p
   118    129         }
   119         -      puts "<$mode>"
          130  +      output "<$mode>"
   120    131       }
   121         -    puts $line
          132  +    output $line
   122    133     }
   123         -  if {$mode != ""} {puts "</$mode>"}
          134  +  if {$mode != ""} {output "</$mode>"}
   124    135   }
   125    136   
   126         -
          137  +set ::fts5_docs_output
   127    138   
   128    139   
   129    140   
   130    141   
   131    142   

Changes to ext/fts5/fts5.c.

    19     19   
    20     20   typedef struct Fts5Table Fts5Table;
    21     21   typedef struct Fts5Cursor Fts5Cursor;
    22     22   typedef struct Fts5Global Fts5Global;
    23     23   typedef struct Fts5Auxiliary Fts5Auxiliary;
    24     24   typedef struct Fts5Auxdata Fts5Auxdata;
    25     25   
           26  +typedef struct Fts5TokenizerModule Fts5TokenizerModule;
           27  +
    26     28   /*
    27     29   ** NOTES ON TRANSACTIONS: 
    28     30   **
    29     31   ** SQLite invokes the following virtual table methods as transactions are 
    30     32   ** opened and closed by the user:
    31     33   **
    32     34   **     xBegin():    Start of a new transaction.
................................................................................
    61     63   
    62     64   /*
    63     65   ** A single object of this type is allocated when the FTS5 module is 
    64     66   ** registered with a database handle. It is used to store pointers to
    65     67   ** all registered FTS5 extensions - tokenizers and auxiliary functions.
    66     68   */
    67     69   struct Fts5Global {
           70  +  fts5_api api;                   /* User visible part of object (see fts5.h) */
    68     71     sqlite3 *db;                    /* Associated database connection */ 
    69     72     i64 iNextId;                    /* Used to allocate unique cursor ids */
    70     73     Fts5Auxiliary *pAux;            /* First in list of all aux. functions */
           74  +  Fts5TokenizerModule *pTok;      /* First in list of all tokenizer modules */
    71     75     Fts5Cursor *pCsr;               /* First in list of all open cursors */
    72     76   };
    73     77   
    74     78   /*
    75     79   ** Each auxiliary function registered with the FTS5 module is represented
    76     80   ** by an object of the following type. All such objects are stored as part
    77     81   ** of the Fts5Global.pAux list.
................................................................................
    80     84     Fts5Global *pGlobal;            /* Global context for this function */
    81     85     char *zFunc;                    /* Function name (nul-terminated) */
    82     86     void *pUserData;                /* User-data pointer */
    83     87     fts5_extension_function xFunc;  /* Callback function */
    84     88     void (*xDestroy)(void*);        /* Destructor function */
    85     89     Fts5Auxiliary *pNext;           /* Next registered auxiliary function */
    86     90   };
           91  +
           92  +/*
           93  +** Each tokenizer module registered with the FTS5 module is represented
           94  +** by an object of the following type. All such objects are stored as part
           95  +** of the Fts5Global.pTok list.
           96  +*/
           97  +struct Fts5TokenizerModule {
           98  +  char *zName;                    /* Name of tokenizer */
           99  +  void *pUserData;                /* User pointer passed to xCreate() */
          100  +  fts5_tokenizer x;               /* Tokenizer functions */
          101  +  void (*xDestroy)(void*);        /* Destructor function */
          102  +  Fts5TokenizerModule *pNext;     /* Next registered tokenizer module */
          103  +};
    87    104   
    88    105   /*
    89    106   ** Virtual-table object.
    90    107   */
    91    108   struct Fts5Table {
    92    109     sqlite3_vtab base;              /* Base class used by SQLite core */
    93    110     Fts5Config *pConfig;            /* Virtual table configuration */
................................................................................
   277    294     sqlite3 *db,                    /* The SQLite database connection */
   278    295     void *pAux,                     /* Hash table containing tokenizers */
   279    296     int argc,                       /* Number of elements in argv array */
   280    297     const char * const *argv,       /* xCreate/xConnect argument array */
   281    298     sqlite3_vtab **ppVTab,          /* Write the resulting vtab structure here */
   282    299     char **pzErr                    /* Write any error message here */
   283    300   ){
          301  +  Fts5Global *pGlobal = (Fts5Global*)pAux;
          302  +  const char **azConfig = (const char**)argv;
   284    303     int rc;                         /* Return code */
   285    304     Fts5Config *pConfig;            /* Results of parsing argc/argv */
   286    305     Fts5Table *pTab = 0;            /* New virtual table object */
   287    306   
   288    307     /* Parse the arguments */
   289         -  rc = sqlite3Fts5ConfigParse(db, argc, (const char**)argv, &pConfig, pzErr);
          308  +  rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr);
   290    309     assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 );
   291    310   
   292    311     /* Allocate the new vtab object */
   293    312     if( rc==SQLITE_OK ){
   294    313       pTab = (Fts5Table*)sqlite3_malloc(sizeof(Fts5Table));
   295    314       if( pTab==0 ){
   296    315         rc = SQLITE_NOMEM;
   297    316       }else{
   298    317         memset(pTab, 0, sizeof(Fts5Table));
   299    318         pTab->pConfig = pConfig;
   300         -      pTab->pGlobal = (Fts5Global*)pAux;
          319  +      pTab->pGlobal = pGlobal;
   301    320       }
   302    321     }
   303    322   
   304    323     /* Open the index sub-system */
   305    324     if( rc==SQLITE_OK ){
   306    325       rc = sqlite3Fts5IndexOpen(pConfig, bCreate, &pTab->pIndex, pzErr);
   307    326     }
................................................................................
   853    872   ** an INSERT statement of the form:
   854    873   **
   855    874   **     INSERT INTO fts(fts) VALUES($pVal)
   856    875   **
   857    876   ** Argument pVal is the value assigned to column "fts" by the INSERT 
   858    877   ** statement. This function returns SQLITE_OK if successful, or an SQLite
   859    878   ** error code if an error occurs.
          879  +**
          880  +** The commands implemented by this function are documented in the "Special
          881  +** INSERT Directives" section of the documentation. It should be updated if
          882  +** more commands are added to this function.
   860    883   */
   861    884   static int fts5SpecialCommand(Fts5Table *pTab, sqlite3_value *pVal){
   862    885     const char *z = (const char*)sqlite3_value_text(pVal);
   863    886     int n = sqlite3_value_bytes(pVal);
   864    887     int rc = SQLITE_ERROR;
   865    888   
   866    889     if( 0==sqlite3_stricmp("integrity-check", z) ){
................................................................................
  1383   1406     fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint);
  1384   1407     return sqlite3Fts5StorageRollback(pTab->pStorage);
  1385   1408   }
  1386   1409   
  1387   1410   /*
  1388   1411   ** Register a new auxiliary function with global context pGlobal.
  1389   1412   */
  1390         -int sqlite3Fts5CreateAux(
  1391         -  Fts5Global *pGlobal,            /* Global context (one per db handle) */
         1413  +static int fts5CreateAux(
         1414  +  fts5_api *pApi,                 /* Global context (one per db handle) */
  1392   1415     const char *zName,              /* Name of new function */
  1393   1416     void *pUserData,                /* User data for aux. function */
  1394   1417     fts5_extension_function xFunc,  /* Aux. function implementation */
  1395   1418     void(*xDestroy)(void*)          /* Destructor for pUserData */
  1396   1419   ){
         1420  +  Fts5Global *pGlobal = (Fts5Global*)pApi;
  1397   1421     int rc = sqlite3_overload_function(pGlobal->db, zName, -1);
  1398   1422     if( rc==SQLITE_OK ){
  1399   1423       Fts5Auxiliary *pAux;
  1400   1424       int nByte;                      /* Bytes of space to allocate */
  1401   1425   
  1402   1426       nByte = sizeof(Fts5Auxiliary) + strlen(zName) + 1;
  1403   1427       pAux = (Fts5Auxiliary*)sqlite3_malloc(nByte);
................................................................................
  1415   1439         rc = SQLITE_NOMEM;
  1416   1440       }
  1417   1441     }
  1418   1442   
  1419   1443     return rc;
  1420   1444   }
  1421   1445   
  1422         -static void fts5ModuleDestroy(void *pCtx){
  1423         -  Fts5Auxiliary *pAux;
  1424         -  Fts5Auxiliary *pNext;
  1425         -  Fts5Global *pGlobal = (Fts5Global*)pCtx;
  1426         -  for(pAux=pGlobal->pAux; pAux; pAux=pNext){
  1427         -    pNext = pAux->pNext;
  1428         -    if( pAux->xDestroy ){
  1429         -      pAux->xDestroy(pAux->pUserData);
         1446  +/*
         1447  +** Register a new tokenizer. This is the implementation of the 
         1448  +** fts5_api.xCreateTokenizer() method.
         1449  +*/
         1450  +static int fts5CreateTokenizer(
         1451  +  fts5_api *pApi,                 /* Global context (one per db handle) */
         1452  +  const char *zName,              /* Name of new function */
         1453  +  void *pUserData,                /* User data for aux. function */
         1454  +  fts5_tokenizer *pTokenizer,     /* Tokenizer implementation */
         1455  +  void(*xDestroy)(void*)          /* Destructor for pUserData */
         1456  +){
         1457  +  Fts5Global *pGlobal = (Fts5Global*)pApi;
         1458  +  Fts5TokenizerModule *pNew;
         1459  +  int nByte;                      /* Bytes of space to allocate */
         1460  +  int rc = SQLITE_OK;
         1461  +
         1462  +  nByte = sizeof(Fts5TokenizerModule) + strlen(zName) + 1;
         1463  +  pNew = (Fts5TokenizerModule*)sqlite3_malloc(nByte);
         1464  +  if( pNew ){
         1465  +    memset(pNew, 0, nByte);
         1466  +    pNew->zName = (char*)&pNew[1];
         1467  +    strcpy(pNew->zName, zName);
         1468  +    pNew->pUserData = pUserData;
         1469  +    pNew->x = *pTokenizer;
         1470  +    pNew->xDestroy = xDestroy;
         1471  +    pNew->pNext = pGlobal->pTok;
         1472  +    pGlobal->pTok = pNew;
         1473  +  }else{
         1474  +    rc = SQLITE_NOMEM;
         1475  +  }
         1476  +
         1477  +  return rc;
         1478  +}
         1479  +
         1480  +/*
         1481  +** Find a tokenizer. This is the implementation of the 
         1482  +** fts5_api.xFindTokenizer() method.
         1483  +*/
         1484  +static int fts5FindTokenizer(
         1485  +  fts5_api *pApi,                 /* Global context (one per db handle) */
         1486  +  const char *zName,              /* Name of new function */
         1487  +  fts5_tokenizer *pTokenizer      /* Populate this object */
         1488  +){
         1489  +  Fts5Global *pGlobal = (Fts5Global*)pApi;
         1490  +  int rc = SQLITE_OK;
         1491  +  Fts5TokenizerModule *pTok;
         1492  +
         1493  +  for(pTok=pGlobal->pTok; pTok; pTok=pTok->pNext){
         1494  +    if( sqlite3_stricmp(zName, pTok->zName)==0 ) break;
         1495  +  }
         1496  +
         1497  +  if( pTok ){
         1498  +    *pTokenizer = pTok->x;
         1499  +  }else{
         1500  +    memset(pTokenizer, 0, sizeof(fts5_tokenizer));
         1501  +    rc = SQLITE_ERROR;
         1502  +  }
         1503  +
         1504  +  return rc;
         1505  +}
         1506  +
         1507  +int sqlite3Fts5GetTokenizer(
         1508  +  Fts5Global *pGlobal, 
         1509  +  const char **azArg,
         1510  +  int nArg,
         1511  +  Fts5Tokenizer **ppTok,
         1512  +  fts5_tokenizer **ppTokApi
         1513  +){
         1514  +  Fts5TokenizerModule *pMod = 0;
         1515  +  int rc = SQLITE_OK;
         1516  +  if( nArg==0 ){
         1517  +    pMod = pGlobal->pTok;
         1518  +  }else{
         1519  +    for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){
         1520  +      if( sqlite3_stricmp(azArg[0], pMod->zName)==0 ) break;
  1430   1521       }
         1522  +  }
         1523  +
         1524  +  if( pMod==0 ){
         1525  +    rc = SQLITE_ERROR;
         1526  +  }else{
         1527  +    rc = pMod->x.xCreate(pMod->pUserData, &azArg[1], (nArg?nArg-1:0), ppTok);
         1528  +    *ppTokApi = &pMod->x;
         1529  +  }
         1530  +
         1531  +  if( rc!=SQLITE_OK ){
         1532  +    *ppTokApi = 0;
         1533  +    *ppTok = 0;
         1534  +  }
         1535  +
         1536  +  return rc;
         1537  +}
         1538  +
         1539  +static void fts5ModuleDestroy(void *pCtx){
         1540  +  Fts5TokenizerModule *pTok, *pNextTok;
         1541  +  Fts5Auxiliary *pAux, *pNextAux;
         1542  +  Fts5Global *pGlobal = (Fts5Global*)pCtx;
         1543  +
         1544  +  for(pAux=pGlobal->pAux; pAux; pAux=pNextAux){
         1545  +    pNextAux = pAux->pNext;
         1546  +    if( pAux->xDestroy ) pAux->xDestroy(pAux->pUserData);
  1431   1547       sqlite3_free(pAux);
  1432   1548     }
         1549  +
         1550  +  for(pTok=pGlobal->pTok; pTok; pTok=pNextTok){
         1551  +    pNextTok = pTok->pNext;
         1552  +    if( pTok->xDestroy ) pTok->xDestroy(pTok->pUserData);
         1553  +    sqlite3_free(pTok);
         1554  +  }
         1555  +
  1433   1556     sqlite3_free(pGlobal);
  1434   1557   }
  1435   1558   
         1559  +static void fts5Fts5Func(
         1560  +  sqlite3_context *pCtx,          /* Function call context */
         1561  +  int nArg,                       /* Number of args */
         1562  +  sqlite3_value **apVal           /* Function arguments */
         1563  +){
         1564  +  Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx);
         1565  +  char buf[8];
         1566  +  assert( nArg==0 );
         1567  +  assert( sizeof(buf)>=sizeof(pGlobal) );
         1568  +  memcpy(buf, pGlobal, sizeof(pGlobal));
         1569  +  sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT);
         1570  +}
  1436   1571   
  1437   1572   int sqlite3Fts5Init(sqlite3 *db){
  1438   1573     static const sqlite3_module fts5Mod = {
  1439   1574       /* iVersion      */ 2,
  1440   1575       /* xCreate       */ fts5CreateMethod,
  1441   1576       /* xConnect      */ fts5ConnectMethod,
  1442   1577       /* xBestIndex    */ fts5BestIndexMethod,
................................................................................
  1467   1602   
  1468   1603     if( pGlobal==0 ){
  1469   1604       rc = SQLITE_NOMEM;
  1470   1605     }else{
  1471   1606       void *p = (void*)pGlobal;
  1472   1607       memset(pGlobal, 0, sizeof(Fts5Global));
  1473   1608       pGlobal->db = db;
         1609  +    pGlobal->api.iVersion = 1;
         1610  +    pGlobal->api.xCreateFunction = fts5CreateAux;
         1611  +    pGlobal->api.xCreateTokenizer = fts5CreateTokenizer;
         1612  +    pGlobal->api.xFindTokenizer = fts5FindTokenizer;
  1474   1613       rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy);
  1475   1614       if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db);
  1476         -    if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(db);
  1477         -    if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(pGlobal);
         1615  +    if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db);
         1616  +    if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api);
         1617  +    if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api);
         1618  +    if( rc==SQLITE_OK ){
         1619  +      rc = sqlite3_create_function(
         1620  +          db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
         1621  +      );
         1622  +    }
  1478   1623     }
  1479   1624     return rc;
  1480   1625   }
  1481   1626   
  1482   1627   

Changes to ext/fts5/fts5.h.

   256    256         )
   257    257     );
   258    258   };
   259    259   
   260    260   /*
   261    261   ** END OF CUSTOM TOKENIZERS
   262    262   *************************************************************************/
          263  +
          264  +/*************************************************************************
          265  +** FTS5 EXTENSION REGISTRATION API
          266  +*/
          267  +typedef struct fts5_api fts5_api;
          268  +struct fts5_api {
          269  +  int iVersion;                   /* Currently always set to 1 */
          270  +
          271  +  /* Create a new tokenizer */
          272  +  int (*xCreateTokenizer)(
          273  +    fts5_api *pApi,
          274  +    const char *zName,
          275  +    void *pContext,
          276  +    fts5_tokenizer *pTokenizer,
          277  +    void (*xDestroy)(void*)
          278  +  );
          279  +
          280  +  /* Find an existing tokenizer */
          281  +  int (*xFindTokenizer)(
          282  +    fts5_api *pApi,
          283  +    const char *zName,
          284  +    fts5_tokenizer *pTokenizer
          285  +  );
          286  +
          287  +  /* Create a new auxiliary function */
          288  +  int (*xCreateFunction)(
          289  +    fts5_api *pApi,
          290  +    const char *zName,
          291  +    void *pContext,
          292  +    fts5_extension_function xFunction,
          293  +    void (*xDestroy)(void*)
          294  +  );
          295  +};
          296  +
          297  +/*
          298  +** END OF REGISTRATION API
          299  +*************************************************************************/
   263    300   
   264    301   #endif /* _FTS5_H */
   265    302   

Changes to ext/fts5/fts5Int.h.

    27     27   #define FTS5_MAX_PREFIX_INDEXES 31
    28     28   
    29     29   #define FTS5_DEFAULT_NEARDIST 10
    30     30   
    31     31   /* Name of rank column */
    32     32   #define FTS5_RANK_NAME "rank"
    33     33   
           34  +/**************************************************************************
           35  +** Interface to code in fts5.c. 
           36  +*/
           37  +typedef struct Fts5Global Fts5Global;
           38  +
           39  +int sqlite3Fts5GetTokenizer(
           40  +  Fts5Global*, 
           41  +  const char **azArg,
           42  +  int nArg,
           43  +  Fts5Tokenizer**,
           44  +  fts5_tokenizer**
           45  +);
           46  +
           47  +/*
           48  +** End of interface to code in fts5.c.
           49  +**************************************************************************/
           50  +
    34     51   /**************************************************************************
    35     52   ** Interface to code in fts5_config.c. fts5_config.c contains contains code
    36     53   ** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
    37     54   */
    38     55   
    39     56   typedef struct Fts5Config Fts5Config;
    40     57   
................................................................................
    46     63     sqlite3 *db;                    /* Database handle */
    47     64     char *zDb;                      /* Database holding FTS index (e.g. "main") */
    48     65     char *zName;                    /* Name of FTS index */
    49     66     int nCol;                       /* Number of columns */
    50     67     char **azCol;                   /* Column names */
    51     68     int nPrefix;                    /* Number of prefix indexes */
    52     69     int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
    53         -  sqlite3_tokenizer *pTokenizer;  /* Tokenizer instance for this table */
           70  +  Fts5Tokenizer *pTok;
           71  +  fts5_tokenizer *pTokApi;
    54     72   };
    55     73   
    56         -int sqlite3Fts5ConfigParse(sqlite3*, int, const char**, Fts5Config**, char**);
           74  +int sqlite3Fts5ConfigParse(
           75  +    Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
           76  +);
    57     77   void sqlite3Fts5ConfigFree(Fts5Config*);
    58     78   
    59     79   int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
    60     80   
    61     81   int sqlite3Fts5Tokenize(
    62     82     Fts5Config *pConfig,            /* FTS5 Configuration object */
    63     83     const char *pText, int nText,   /* Text to tokenize */
................................................................................
   399    419   int sqlite3Fts5ExprNext(Fts5Expr*);
   400    420   int sqlite3Fts5ExprEof(Fts5Expr*);
   401    421   i64 sqlite3Fts5ExprRowid(Fts5Expr*);
   402    422   
   403    423   void sqlite3Fts5ExprFree(Fts5Expr*);
   404    424   
   405    425   /* Called during startup to register a UDF with SQLite */
   406         -int sqlite3Fts5ExprInit(sqlite3*);
          426  +int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
   407    427   
   408    428   int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
   409    429   int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
   410    430   int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
   411    431   
   412    432   int sqlite3Fts5ExprPhraseExpr(Fts5Config*, Fts5Expr*, int, Fts5Expr**);
   413    433   
................................................................................
   449    469   void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
   450    470   
   451    471   /*
   452    472   ** End of interface to code in fts5_expr.c.
   453    473   **************************************************************************/
   454    474   
   455    475   
   456         -/**************************************************************************
   457         -** Interface to code in fts5.c. 
   458         -*/
   459         -typedef struct Fts5Global Fts5Global;
   460         -
   461         -int sqlite3Fts5CreateAux(
   462         -    Fts5Global*, 
   463         -    const char*, 
   464         -    void*, 
   465         -    fts5_extension_function, 
   466         -    void(*)(void*)
   467         -);
   468         -/*
   469         -** End of interface to code in fts5.c.
   470         -**************************************************************************/
   471         -
   472    476   
   473    477   /**************************************************************************
   474    478   ** Interface to code in fts5_aux.c. 
   475    479   */
   476    480   
   477         -int sqlite3Fts5AuxInit(Fts5Global*);
          481  +int sqlite3Fts5AuxInit(fts5_api*);
   478    482   /*
   479    483   ** End of interface to code in fts5_aux.c.
   480    484   **************************************************************************/
          485  +
          486  +/**************************************************************************
          487  +** Interface to code in fts5_tokenizer.c. 
          488  +*/
          489  +
          490  +int sqlite3Fts5TokenizerInit(fts5_api*);
          491  +/*
          492  +** End of interface to code in fts5_tokenizer.c.
          493  +**************************************************************************/
   481    494   
   482    495   /**************************************************************************
   483    496   ** Interface to code in fts5_sorter.c. 
   484    497   */
   485    498   typedef struct Fts5Sorter Fts5Sorter;
   486    499   
   487    500   int sqlite3Fts5SorterNew(Fts5Expr *pExpr, Fts5Sorter **pp);
   488    501   
   489    502   /*
   490    503   ** End of interface to code in fts5_sorter.c.
   491    504   **************************************************************************/
   492    505   
   493    506   #endif

Changes to ext/fts5/fts5_aux.c.

   952    952       sqlite3_result_text(pCtx, (const char*)s.p, -1, SQLITE_TRANSIENT);
   953    953     }else{
   954    954       sqlite3_result_error_code(pCtx, rc);
   955    955     }
   956    956     sqlite3Fts5BufferFree(&s);
   957    957   }
   958    958   
   959         -int sqlite3Fts5AuxInit(Fts5Global *pGlobal){
          959  +int sqlite3Fts5AuxInit(fts5_api *pApi){
   960    960     struct Builtin {
   961    961       const char *zFunc;            /* Function name (nul-terminated) */
   962    962       void *pUserData;              /* User-data pointer */
   963    963       fts5_extension_function xFunc;/* Callback function */
   964    964       void (*xDestroy)(void*);      /* Destructor function */
   965    965     } aBuiltin [] = {
   966    966       { "bm25debug", (void*)1, fts5Bm25Function,    0 },
................................................................................
   969    969       { "bm25",      0, fts5Bm25Function,    0 },
   970    970     };
   971    971   
   972    972     int rc = SQLITE_OK;             /* Return code */
   973    973     int i;                          /* To iterate through builtin functions */
   974    974   
   975    975     for(i=0; rc==SQLITE_OK && i<sizeof(aBuiltin)/sizeof(aBuiltin[0]); i++){
   976         -    rc = sqlite3Fts5CreateAux(pGlobal, 
          976  +    rc = pApi->xCreateFunction(pApi,
   977    977           aBuiltin[i].zFunc,
   978    978           aBuiltin[i].pUserData,
   979    979           aBuiltin[i].xFunc,
   980    980           aBuiltin[i].xDestroy
   981    981       );
   982    982     }
   983    983   
   984    984     return rc;
   985    985   }
   986    986   
   987    987   

Changes to ext/fts5/fts5_config.c.

   109    109   **
   110    110   ** Return 0 if an OOM error is encountered.
   111    111   */
   112    112   static char *fts5Strdup(const char *z){
   113    113     return sqlite3_mprintf("%s", z);
   114    114   }
   115    115   
   116         -void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**);
   117         -
   118    116   /*
   119    117   ** Allocate an instance of the default tokenizer ("simple") at 
   120    118   ** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
   121    119   ** code if an error occurs.
   122    120   */
   123         -static int fts5ConfigDefaultTokenizer(Fts5Config *pConfig){
   124         -  const sqlite3_tokenizer_module *pMod; /* Tokenizer module "simple" */
   125         -  sqlite3_tokenizer *pTokenizer;  /* Tokenizer instance */
   126         -  int rc;                         /* Return code */
   127         -
   128         -  sqlite3Fts3SimpleTokenizerModule(&pMod);
   129         -  rc = pMod->xCreate(0, 0, &pTokenizer);
   130         -  if( rc==SQLITE_OK ){
   131         -    pTokenizer->pModule = pMod;
   132         -    pConfig->pTokenizer = pTokenizer;
   133         -  }
   134         -
   135         -  return rc;
          121  +static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
          122  +  return sqlite3Fts5GetTokenizer(
          123  +      pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi
          124  +  );
   136    125   }
   137    126   
   138    127   /*
   139    128   ** Arguments nArg/azArg contain the string arguments passed to the xCreate
   140    129   ** or xConnect method of the virtual table. This function attempts to 
   141    130   ** allocate an instance of Fts5Config containing the results of parsing
   142    131   ** those arguments.
................................................................................
   144    133   ** If successful, SQLITE_OK is returned and *ppOut is set to point to the
   145    134   ** new Fts5Config object. If an error occurs, an SQLite error code is 
   146    135   ** returned, *ppOut is set to NULL and an error message may be left in
   147    136   ** *pzErr. It is the responsibility of the caller to eventually free any 
   148    137   ** such error message using sqlite3_free().
   149    138   */
   150    139   int sqlite3Fts5ConfigParse(
          140  +  Fts5Global *pGlobal,
   151    141     sqlite3 *db,
   152    142     int nArg,                       /* Number of arguments */
   153    143     const char **azArg,             /* Array of nArg CREATE VIRTUAL TABLE args */
   154    144     Fts5Config **ppOut,             /* OUT: Results of parse */
   155    145     char **pzErr                    /* OUT: Error message */
   156    146   ){
   157    147     int rc = SQLITE_OK;             /* Return code */
................................................................................
   202    192               rc = SQLITE_ERROR;
   203    193             }
   204    194           }
   205    195         }
   206    196       }
   207    197     }
   208    198   
   209         -  if( rc==SQLITE_OK && pRet->pTokenizer==0 ){
   210         -    rc = fts5ConfigDefaultTokenizer(pRet);
          199  +  if( rc==SQLITE_OK && pRet->pTok==0 ){
          200  +    rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
   211    201     }
   212    202   
   213    203     if( rc!=SQLITE_OK ){
   214    204       sqlite3Fts5ConfigFree(pRet);
   215    205       *ppOut = 0;
   216    206     }
   217    207     return rc;
................................................................................
   219    209   
   220    210   /*
   221    211   ** Free the configuration object passed as the only argument.
   222    212   */
   223    213   void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
   224    214     if( pConfig ){
   225    215       int i;
   226         -    if( pConfig->pTokenizer ){
   227         -      pConfig->pTokenizer->pModule->xDestroy(pConfig->pTokenizer);
          216  +    if( pConfig->pTok && pConfig->pTokApi->xDelete ){
          217  +      pConfig->pTokApi->xDelete(pConfig->pTok);
   228    218       }
   229    219       sqlite3_free(pConfig->zDb);
   230    220       sqlite3_free(pConfig->zName);
   231    221       for(i=0; i<pConfig->nCol; i++){
   232    222         sqlite3_free(pConfig->azCol[i]);
   233    223       }
   234    224       sqlite3_free(pConfig->azCol);
................................................................................
   298    288   */
   299    289   int sqlite3Fts5Tokenize(
   300    290     Fts5Config *pConfig,            /* FTS5 Configuration object */
   301    291     const char *pText, int nText,   /* Text to tokenize */
   302    292     void *pCtx,                     /* Context passed to xToken() */
   303    293     int (*xToken)(void*, const char*, int, int, int, int)    /* Callback */
   304    294   ){
   305         -  const sqlite3_tokenizer_module *pMod = pConfig->pTokenizer->pModule;
   306         -  sqlite3_tokenizer_cursor *pCsr = 0;
   307         -  int rc;
   308         -
   309         -  rc = pMod->xOpen(pConfig->pTokenizer, pText, nText, &pCsr);
   310         -  assert( rc==SQLITE_OK || pCsr==0 );
   311         -  if( rc==SQLITE_OK ){
   312         -    const char *pToken;           /* Pointer to token buffer */
   313         -    int nToken;                   /* Size of token in bytes */
   314         -    int iStart, iEnd, iPos;       /* Start, end and position of token */
   315         -    pCsr->pTokenizer = pConfig->pTokenizer;
   316         -    for(rc = pMod->xNext(pCsr, &pToken, &nToken, &iStart, &iEnd, &iPos);
   317         -        rc==SQLITE_OK;
   318         -        rc = pMod->xNext(pCsr, &pToken, &nToken, &iStart, &iEnd, &iPos)
   319         -    ){
   320         -      if( (rc = xToken(pCtx, pToken, nToken, iStart, iEnd, iPos)) ) break;
   321         -    }
   322         -    if( rc==SQLITE_DONE ) rc = SQLITE_OK;
   323         -    pMod->xClose(pCsr);
   324         -  }
   325         -  return rc;
          295  +  return pConfig->pTokApi->xTokenize(pConfig->pTok, pCtx, pText, nText, xToken);
   326    296   }
   327    297   
   328    298   

Changes to ext/fts5/fts5_expr.c.

  1516   1516       sqlite3_free(z2);
  1517   1517     }
  1518   1518   
  1519   1519     return zRet;
  1520   1520   }
  1521   1521   
  1522   1522   /*
  1523         -** The implementation of user-defined scalar function fts5_expr().
         1523  +** The implementation of user-defined scalar functions fts5_expr() (bTcl==0)
         1524  +** and fts5_expr_tcl() (bTcl!=0).
  1524   1525   */
  1525   1526   static void fts5ExprFunction(
  1526   1527     sqlite3_context *pCtx,          /* Function call context */
  1527   1528     int nArg,                       /* Number of args */
  1528         -  sqlite3_value **apVal           /* Function arguments */
         1529  +  sqlite3_value **apVal,          /* Function arguments */
         1530  +  int bTcl
  1529   1531   ){
         1532  +  Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx);
  1530   1533     sqlite3 *db = sqlite3_context_db_handle(pCtx);
  1531   1534     const char *zExpr = 0;
  1532   1535     char *zErr = 0;
  1533   1536     Fts5Expr *pExpr = 0;
  1534   1537     int rc;
  1535   1538     int i;
  1536         -  int bTcl = sqlite3_user_data(pCtx)!=0;
  1537   1539   
  1538   1540     const char **azConfig;          /* Array of arguments for Fts5Config */
  1539   1541     const char *zNearsetCmd = "nearset";
  1540   1542     int nConfig;                    /* Size of azConfig[] */
  1541   1543     Fts5Config *pConfig = 0;
  1542   1544   
  1543   1545     if( bTcl && nArg>1 ){
................................................................................
  1554   1556     azConfig[1] = "main";
  1555   1557     azConfig[2] = "tbl";
  1556   1558     for(i=1+bTcl; i<nArg; i++){
  1557   1559       azConfig[i+2-bTcl] = (const char*)sqlite3_value_text(apVal[i]);
  1558   1560     }
  1559   1561     zExpr = (const char*)sqlite3_value_text(apVal[0]);
  1560   1562   
  1561         -  rc = sqlite3Fts5ConfigParse(db, nConfig, azConfig, &pConfig, &zErr);
         1563  +  rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
  1562   1564     if( rc==SQLITE_OK ){
  1563   1565       rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr);
  1564   1566     }
  1565   1567     if( rc==SQLITE_OK ){
  1566   1568       char *zText;
  1567   1569       if( bTcl ){
  1568   1570         zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot);
................................................................................
  1583   1585         sqlite3_result_error_code(pCtx, rc);
  1584   1586       }
  1585   1587     }
  1586   1588     sqlite3_free(azConfig);
  1587   1589     sqlite3Fts5ConfigFree(pConfig);
  1588   1590     sqlite3Fts5ExprFree(pExpr);
  1589   1591   }
         1592  +
         1593  +static void fts5ExprFunctionHr(
         1594  +  sqlite3_context *pCtx,          /* Function call context */
         1595  +  int nArg,                       /* Number of args */
         1596  +  sqlite3_value **apVal           /* Function arguments */
         1597  +){
         1598  +  fts5ExprFunction(pCtx, nArg, apVal, 0);
         1599  +}
         1600  +static void fts5ExprFunctionTcl(
         1601  +  sqlite3_context *pCtx,          /* Function call context */
         1602  +  int nArg,                       /* Number of args */
         1603  +  sqlite3_value **apVal           /* Function arguments */
         1604  +){
         1605  +  fts5ExprFunction(pCtx, nArg, apVal, 1);
         1606  +}
  1590   1607   
  1591   1608   /*
  1592   1609   ** This is called during initialization to register the fts5_expr() scalar
  1593   1610   ** UDF with the SQLite handle passed as the only argument.
  1594   1611   */
  1595         -int sqlite3Fts5ExprInit(sqlite3 *db){
         1612  +int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){
  1596   1613     struct Fts5ExprFunc {
  1597   1614       const char *z;
  1598         -    void *p;
  1599   1615       void (*x)(sqlite3_context*,int,sqlite3_value**);
  1600   1616     } aFunc[] = {
  1601         -    { "fts5_expr", 0, fts5ExprFunction },
  1602         -    { "fts5_expr_tcl", (void*)1, fts5ExprFunction },
         1617  +    { "fts5_expr", fts5ExprFunctionHr },
         1618  +    { "fts5_expr_tcl", fts5ExprFunctionTcl },
  1603   1619     };
  1604   1620     int i;
  1605   1621     int rc = SQLITE_OK;
         1622  +  void *pCtx = (void*)pGlobal;
  1606   1623   
  1607   1624     for(i=0; rc==SQLITE_OK && i<(sizeof(aFunc) / sizeof(aFunc[0])); i++){
  1608   1625       struct Fts5ExprFunc *p = &aFunc[i];
  1609         -    rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, p->p, p->x, 0, 0);
         1626  +    rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0);
  1610   1627     }
  1611   1628   
  1612   1629     return rc;
  1613   1630   }
  1614   1631   
  1615   1632   /*
  1616   1633   ** Return the number of phrases in expression pExpr.

Added ext/fts5/fts5_tokenize.c.

            1  +/*
            2  +** 2014 May 31
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +*/
           13  +
           14  +#include "fts5.h"
           15  +
           16  +
           17  +/*
           18  +** Create a "simple" tokenizer.
           19  +*/
           20  +static int fts5SimpleCreate(
           21  +  void *pCtx, 
           22  +  const char **azArg, int nArg,
           23  +  Fts5Tokenizer **ppOut
           24  +){
           25  +  *ppOut = 0;
           26  +  return SQLITE_OK;
           27  +}
           28  +
           29  +/*
           30  +** Delete a "simple" tokenizer.
           31  +*/
           32  +static void fts5SimpleDelete(Fts5Tokenizer *p){
           33  +  return;
           34  +}
           35  +
           36  +/*
           37  +** For tokenizers with no "unicode" modifier, the set of token characters
           38  +** is the same as the set of ASCII range alphanumeric characters. 
           39  +*/
           40  +static unsigned char aSimpleTokenChar[128] = {
           41  +  0, 0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0, 0, 0, 0, 0,   /* 0x00..0x0F */
           42  +  0, 0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0, 0, 0, 0, 0,   /* 0x10..0x1F */
           43  +  0, 0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0, 0, 0, 0, 0,   /* 0x20..0x2F */
           44  +  1, 1, 1, 1, 1, 1, 1, 1,   1, 1, 0, 0, 0, 0, 0, 0,   /* 0x30..0x3F */
           45  +  0, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 1, 1, 1, 1, 1,   /* 0x40..0x4F */
           46  +  1, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 0, 0, 0, 0, 0,   /* 0x50..0x5F */
           47  +  0, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 1, 1, 1, 1, 1,   /* 0x60..0x6F */
           48  +  1, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 0, 0, 0, 0, 0,   /* 0x70..0x7F */
           49  +};
           50  +
           51  +
           52  +static void simpleFold(char *aOut, const char *aIn, int nByte){
           53  +  int i;
           54  +  for(i=0; i<nByte; i++){
           55  +    char c = aIn[i];
           56  +    if( c>='A' && c<='Z' ) c += 32;
           57  +    aOut[i] = c;
           58  +  }
           59  +}
           60  +
           61  +/*
           62  +** Tokenize some text using the simple tokenizer.
           63  +*/
           64  +static int fts5SimpleTokenize(
           65  +  Fts5Tokenizer *pTokenizer,
           66  +  void *pCtx,
           67  +  const char *pText, int nText,
           68  +  int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos)
           69  +){
           70  +  int rc;
           71  +  int ie;
           72  +  int is = 0;
           73  +  int iPos = 0;
           74  +
           75  +  char aFold[64];
           76  +  int nFold = sizeof(aFold);
           77  +  char *pFold = aFold;
           78  +
           79  +  do {
           80  +    int nByte;
           81  +
           82  +    /* Skip any leading divider characters. */
           83  +    while( is<nText && ((pText[is]&0x80) || aSimpleTokenChar[pText[is]]==0 ) ){
           84  +      is++;
           85  +    }
           86  +    if( is==nText ) break;
           87  +
           88  +    /* Count the token characters */
           89  +    ie = is+1;
           90  +    while( ie<nText && ((pText[ie]&0x80)==0 && aSimpleTokenChar[pText[ie]] ) ){
           91  +      ie++;
           92  +    }
           93  +
           94  +    /* Fold to lower case */
           95  +    nByte = ie-is;
           96  +    if( nByte>nFold ){
           97  +      if( pFold!=aFold ) sqlite3_free(pFold);
           98  +      pFold = sqlite3_malloc(nByte*2);
           99  +      if( pFold==0 ){
          100  +        rc = SQLITE_NOMEM;
          101  +        break;
          102  +      }
          103  +      nFold = nByte*2;
          104  +    }
          105  +    simpleFold(pFold, &pText[is], nByte);
          106  +
          107  +    /* Invoke the token callback */
          108  +    rc = xToken(pCtx, pFold, nByte, is, ie, iPos);
          109  +    iPos++;
          110  +    is = ie+1;
          111  +  }while( is<nText && rc==SQLITE_OK );
          112  +  
          113  +  if( pFold!=aFold ) sqlite3_free(pFold);
          114  +  if( rc==SQLITE_DONE ) rc = SQLITE_OK;
          115  +  return rc;
          116  +}
          117  +
          118  +/*
          119  +** Register all built-in tokenizers with FTS5.
          120  +*/
          121  +int sqlite3Fts5TokenizerInit(fts5_api *pApi){
          122  +  struct BuiltinTokenizer {
          123  +    const char *zName;
          124  +    void *pUserData;
          125  +    fts5_tokenizer x;
          126  +  } aBuiltin[] = {
          127  +    { "simple", 0, { fts5SimpleCreate, fts5SimpleDelete, fts5SimpleTokenize } }
          128  +  };
          129  +  
          130  +  int rc = SQLITE_OK;             /* Return code */
          131  +  int i;                          /* To iterate through builtin functions */
          132  +
          133  +  for(i=0; rc==SQLITE_OK && i<sizeof(aBuiltin)/sizeof(aBuiltin[0]); i++){
          134  +    rc = pApi->xCreateTokenizer(pApi,
          135  +        aBuiltin[i].zName,
          136  +        &aBuiltin[i].pUserData,
          137  +        &aBuiltin[i].x,
          138  +        0
          139  +    );
          140  +  }
          141  +
          142  +  return SQLITE_OK;
          143  +}
          144  +
          145  +

Changes to main.mk.

    76     76   LIBOBJ += fts5_aux.o
    77     77   LIBOBJ += fts5_buffer.o
    78     78   LIBOBJ += fts5_config.o
    79     79   LIBOBJ += fts5_expr.o
    80     80   LIBOBJ += fts5_hash.o
    81     81   LIBOBJ += fts5_index.o
    82     82   LIBOBJ += fts5_storage.o
           83  +LIBOBJ += fts5_tokenize.o
    83     84   LIBOBJ += fts5parse.o
    84     85   
    85     86   
    86     87   
    87     88   # All of the source code files.
    88     89   #
    89     90   SRC = \
................................................................................
   232    233      $(TOP)/ext/fts5/fts5_buffer.c \
   233    234      $(TOP)/ext/fts5/fts5.c \
   234    235      $(TOP)/ext/fts5/fts5_config.c \
   235    236      $(TOP)/ext/fts5/fts5_expr.c \
   236    237      $(TOP)/ext/fts5/fts5_hash.c \
   237    238      $(TOP)/ext/fts5/fts5_index.c \
   238    239      fts5parse.c \
   239         -   $(TOP)/ext/fts5/fts5_storage.c 
          240  +   $(TOP)/ext/fts5/fts5_storage.c \
          241  +   $(TOP)/ext/fts5/fts5_tokenize.c 
   240    242   
   241    243   
   242    244   # Generated source code files
   243    245   #
   244    246   SRC += \
   245    247     keywordhash.h \
   246    248     opcodes.c \
................................................................................
   606    608   
   607    609   fts5_index.o:	$(TOP)/ext/fts5/fts5_index.c $(HDR) $(EXTHDR)
   608    610   	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_index.c
   609    611   
   610    612   fts5_storage.o:	$(TOP)/ext/fts5/fts5_storage.c $(HDR) $(EXTHDR)
   611    613   	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_storage.c
   612    614   
          615  +fts5_tokenize.o:	$(TOP)/ext/fts5/fts5_tokenize.c $(HDR) $(EXTHDR)
          616  +	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_tokenize.c
          617  +
   613    618   fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
   614    619   	cp $(TOP)/ext/fts5/fts5parse.y .
   615    620   	rm -f fts5parse.h
   616    621   	./lemon $(OPTS) fts5parse.y
          622  +	mv fts5parse.c fts5parse.c.orig
          623  +	cat fts5parse.c.orig | sed 's/yy/fts5yy/g' | sed 's/YY/fts5YY/g' > fts5parse.c
   617    624   
   618    625   
   619    626   # Rules for building test programs and for running tests
   620    627   #
   621    628   tclsqlite3:	$(TOP)/src/tclsqlite.c libsqlite3.a
   622    629   	$(TCCX) $(TCL_FLAGS) -DTCLSH=1 -o tclsqlite3 \
   623    630   		$(TOP)/src/tclsqlite.c libsqlite3.a $(LIBTCL) $(THREADLIB)

Changes to src/main.c.

    15     15   ** accessed by users of the library.
    16     16   */
    17     17   #include "sqliteInt.h"
    18     18   
    19     19   #ifdef SQLITE_ENABLE_FTS3
    20     20   # include "fts3.h"
    21     21   #endif
           22  +#ifdef SQLITE_ENABLE_FTS5
           23  +int sqlite3Fts5Init(sqlite3*);
           24  +#endif
    22     25   #ifdef SQLITE_ENABLE_RTREE
    23     26   # include "rtree.h"
    24     27   #endif
    25     28   #ifdef SQLITE_ENABLE_ICU
    26     29   # include "sqliteicu.h"
    27     30   #endif
    28     31   
................................................................................
  2605   2608       rc = sqlite3Fts2Init(db);
  2606   2609     }
  2607   2610   #endif
  2608   2611   
  2609   2612   #ifdef SQLITE_ENABLE_FTS3
  2610   2613     if( !db->mallocFailed && rc==SQLITE_OK ){
  2611   2614       rc = sqlite3Fts3Init(db);
  2612         -    if( rc==SQLITE_OK ) rc = sqlite3Fts5Init(db);
         2615  +  }
         2616  +#endif
         2617  +
         2618  +#ifdef SQLITE_ENABLE_FTS5
         2619  +  if( !db->mallocFailed && rc==SQLITE_OK ){
         2620  +    rc = sqlite3Fts5Init(db);
  2613   2621     }
  2614   2622   #endif
  2615   2623   
  2616   2624   #ifdef SQLITE_ENABLE_ICU
  2617   2625     if( !db->mallocFailed && rc==SQLITE_OK ){
  2618   2626       rc = sqlite3IcuInit(db);
  2619   2627     }

Changes to src/test_config.c.

   325    325   #endif
   326    326   
   327    327   #ifdef SQLITE_ENABLE_FTS3
   328    328     Tcl_SetVar2(interp, "sqlite_options", "fts3", "1", TCL_GLOBAL_ONLY);
   329    329   #else
   330    330     Tcl_SetVar2(interp, "sqlite_options", "fts3", "0", TCL_GLOBAL_ONLY);
   331    331   #endif
          332  +
          333  +#ifdef SQLITE_ENABLE_FTS5
          334  +  Tcl_SetVar2(interp, "sqlite_options", "fts5", "1", TCL_GLOBAL_ONLY);
          335  +#else
          336  +  Tcl_SetVar2(interp, "sqlite_options", "fts5", "0", TCL_GLOBAL_ONLY);
          337  +#endif
   332    338   
   333    339   #if defined(SQLITE_ENABLE_FTS3) && defined(SQLITE_ENABLE_FTS4_UNICODE61)
   334    340     Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "1", TCL_GLOBAL_ONLY);
   335    341   #else
   336    342     Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "0", TCL_GLOBAL_ONLY);
   337    343   #endif
   338    344   

Changes to test/fts5aa.test.

    13     13   #
    14     14   
    15     15   set testdir [file dirname $argv0]
    16     16   source $testdir/tester.tcl
    17     17   set testprefix fts5aa
    18     18   
    19     19   # If SQLITE_ENABLE_FTS3 is defined, omit this file.
    20         -ifcapable !fts3 {
           20  +ifcapable !fts5 {
    21     21     finish_test
    22     22     return
    23     23   }
    24     24   
    25     25   do_execsql_test 1.0 {
    26     26     CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
    27     27     SELECT name, sql FROM sqlite_master;

Changes to test/fts5ab.test.

    13     13   #
    14     14   #
    15     15   
    16     16   set testdir [file dirname $argv0]
    17     17   source $testdir/tester.tcl
    18     18   set testprefix fts5ab
    19     19   
    20         -# If SQLITE_ENABLE_FTS3 is defined, omit this file.
    21         -ifcapable !fts3 {
           20  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           21  +ifcapable !fts5 {
    22     22     finish_test
    23     23     return
    24     24   }
    25     25   
    26     26   do_execsql_test 1.0 {
    27     27     CREATE VIRTUAL TABLE t1 USING fts5(a, b);
    28     28     INSERT INTO t1 VALUES('hello', 'world');

Changes to test/fts5ac.test.

    13     13   #
    14     14   #
    15     15   
    16     16   set testdir [file dirname $argv0]
    17     17   source $testdir/tester.tcl
    18     18   set testprefix fts5ac
    19     19   
    20         -# If SQLITE_ENABLE_FTS3 is defined, omit this file.
    21         -ifcapable !fts3 {
           20  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           21  +ifcapable !fts5 {
    22     22     finish_test
    23     23     return
    24     24   }
    25     25   
    26     26   do_execsql_test 1.0 {
    27     27     CREATE VIRTUAL TABLE xx USING fts5(x,y);
    28     28     INSERT INTO xx(xx) VALUES('pgsz=32');

Changes to test/fts5ad.test.

    13     13   #
    14     14   #
    15     15   
    16     16   set testdir [file dirname $argv0]
    17     17   source $testdir/tester.tcl
    18     18   set testprefix fts5ad
    19     19   
    20         -# If SQLITE_ENABLE_FTS3 is defined, omit this file.
    21         -ifcapable !fts3 {
           20  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           21  +ifcapable !fts5 {
    22     22     finish_test
    23     23     return
    24     24   }
    25     25   
    26     26   do_execsql_test 1.0 {
    27     27     CREATE VIRTUAL TABLE yy USING fts5(x, y);
    28     28     INSERT INTO yy VALUES('Changes the result to be', 'the list of all matching');

Changes to test/fts5ae.test.

    13     13   #
    14     14   #
    15     15   
    16     16   set testdir [file dirname $argv0]
    17     17   source $testdir/tester.tcl
    18     18   set testprefix fts5ae
    19     19   
    20         -# If SQLITE_ENABLE_FTS3 is defined, omit this file.
    21         -ifcapable !fts3 {
           20  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           21  +ifcapable !fts5 {
    22     22     finish_test
    23     23     return
    24     24   }
    25     25   
    26     26   do_execsql_test 1.0 {
    27     27     CREATE VIRTUAL TABLE t1 USING fts5(a, b);
    28     28     INSERT INTO t1(t1) VALUES('pgsz=32');

Changes to test/fts5af.test.

    15     15   # snippet() function.
    16     16   #
    17     17   
    18     18   set testdir [file dirname $argv0]
    19     19   source $testdir/tester.tcl
    20     20   set testprefix fts5af
    21     21   
    22         -# If SQLITE_ENABLE_FTS3 is defined, omit this file.
    23         -ifcapable !fts3 {
           22  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           23  +ifcapable !fts5 {
    24     24     finish_test
    25     25     return
    26     26   }
    27     27   
    28     28   
    29     29   do_execsql_test 1.0 {
    30     30     CREATE VIRTUAL TABLE t1 USING fts5(x, y);
................................................................................
   134    134     3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
   135    135     3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
   136    136     3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
   137    137     3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
   138    138     3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
   139    139   
   140    140   } {
   141         -  do_snippet_test 1.$tn $doc "X + Y" $res
          141  +  do_snippet_test 2.$tn $doc "X + Y" $res
   142    142   }
   143    143   
   144    144   finish_test
   145    145   

Changes to test/fts5ag.test.

    12     12   # focus of this script is testing the FTS5 module.
    13     13   #
    14     14   
    15     15   set testdir [file dirname $argv0]
    16     16   source $testdir/tester.tcl
    17     17   set testprefix fts5ag
    18     18   
    19         -# If SQLITE_ENABLE_FTS3 is defined, omit this file.
    20         -ifcapable !fts3 {
           19  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           20  +ifcapable !fts5 {
    21     21     finish_test
    22     22     return
    23     23   }
    24     24   
    25     25   #-------------------------------------------------------------------------
    26     26   # This file attempts to verify that the extension APIs work with 
    27     27   # "ORDER BY rank" queries. This is done by comparing the results of

Changes to test/fts5ah.test.

    12     12   # focus of this script is testing the FTS5 module.
    13     13   #
    14     14   
    15     15   set testdir [file dirname $argv0]
    16     16   source $testdir/tester.tcl
    17     17   set testprefix fts5ah
    18     18   
    19         -# If SQLITE_ENABLE_FTS3 is defined, omit this file.
    20         -ifcapable !fts3 {
           19  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           20  +ifcapable !fts5 {
    21     21     finish_test
    22     22     return
    23     23   }
    24     24   
    25     25   #-------------------------------------------------------------------------
    26     26   # This file contains tests for very large doclists.
    27     27   #

Changes to test/fts5ai.test.

    14     14   # Specifically, it tests transactions and savepoints
    15     15   #
    16     16   
    17     17   set testdir [file dirname $argv0]
    18     18   source $testdir/tester.tcl
    19     19   set testprefix fts5ai
    20     20   
    21         -# If SQLITE_ENABLE_FTS3 is defined, omit this file.
    22         -ifcapable !fts3 {
           21  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           22  +ifcapable !fts5 {
    23     23     finish_test
    24     24     return
    25     25   }
    26     26   
    27     27   do_execsql_test 1.0 {
    28     28     CREATE VIRTUAL TABLE t1 USING fts5(a);
    29     29   } {}

Changes to test/fts5aj.test.

    16     16   # and deleted,
    17     17   #
    18     18   
    19     19   set testdir [file dirname $argv0]
    20     20   source $testdir/tester.tcl
    21     21   set testprefix fts5aj
    22     22   
    23         -# If SQLITE_ENABLE_FTS3 is defined, omit this file.
    24         -ifcapable !fts3 {
           23  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           24  +ifcapable !fts5 {
    25     25     finish_test
    26     26     return
    27     27   }
    28     28   
    29     29   proc doc {} {
    30     30     set dict [list a b c d e f g h i j k l m n o p q r s t u v w x y z]
    31     31     set res [list]

Changes to test/fts5ea.test.

    10     10   #*************************************************************************
    11     11   #
    12     12   
    13     13   set testdir [file dirname $argv0]
    14     14   source $testdir/tester.tcl
    15     15   set testprefix fts5ea
    16     16   
    17         -# If SQLITE_ENABLE_FTS3 is defined, omit this file.
    18         -ifcapable !fts3 {
           17  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           18  +ifcapable !fts5 {
    19     19     finish_test
    20     20     return
    21     21   }
    22     22   
    23     23   proc do_syntax_error_test {tn expr err} {
    24     24     set ::se_expr $expr
    25     25     do_catchsql_test $tn {SELECT fts5_expr($se_expr)} [list 1 $err]

Changes to tool/mksqlite3c.tcl.

   331    331      fts3_unicode2.c
   332    332   
   333    333      fts5_aux.c
   334    334      fts5_buffer.c
   335    335      fts5.c
   336    336      fts5_config.c
   337    337      fts5_expr.c
          338  +   fts5_hash.c
   338    339      fts5_index.c
   339    340      fts5parse.c
   340    341      fts5_storage.c
          342  +   fts5_tokenize.c
   341    343   
   342    344      rtree.c
   343    345      icu.c
   344    346      fts3_icu.c
   345    347   } {
   346    348     copy_file tsrc/$file
   347    349   }
   348    350   
   349    351   close $out