Index: ext/fts5/extract_api_docs.tcl
==================================================================
--- ext/fts5/extract_api_docs.tcl
+++ ext/fts5/extract_api_docs.tcl
@@ -12,10 +12,21 @@
 #
 # This script extracts the documentation for the API used by fts5 auxiliary 
 # functions from header file fts5.h. It outputs html text on stdout that
 # is included in the documentation on the web.
 # 
+
+set ::fts5_docs_output ""
+if {[info commands hd_putsnl]==""} {
+  proc output {text} {
+    puts $text
+  }
+} else {
+  proc output {text} {
+    append ::fts5_docs_output $text
+  }
+}
 
 set input_file [file join [file dir [info script]] fts5.h]
 set fd [open $input_file]
 set data [read $fd]
 close $fd
@@ -100,32 +111,32 @@
 
 foreach {hdr docs} $D {
   if {[info exists M($hdr)]} {
     set hdr $M($hdr)
   }
-  puts "<h3><pre>  $hdr</pre></h3>"
+  output "<h style=\"font-size:1.4em;background-color:#EEEEEE;display:block\"><pre>  $hdr</pre></h>"
 
   set mode ""
   set bEmpty 1
   foreach line [split [string trim $docs] "\n"] {
     if {[string trim $line]==""} {
-      if {$mode != ""} {puts "</$mode>"}
+      if {$mode != ""} {output "</$mode>"}
       set mode ""
     } elseif {$mode == ""} {
       if {[regexp {^     } $line]} {
-        set mode code
+        set mode codeblock
       } else {
         set mode p
       }
-      puts "<$mode>"
+      output "<$mode>"
     }
-    puts $line
+    output $line
   }
-  if {$mode != ""} {puts "</$mode>"}
+  if {$mode != ""} {output "</$mode>"}
 }
 
-
+set ::fts5_docs_output
 
 
 
 
 

Index: ext/fts5/fts5.c
==================================================================
--- ext/fts5/fts5.c
+++ ext/fts5/fts5.c
@@ -21,10 +21,12 @@
 typedef struct Fts5Cursor Fts5Cursor;
 typedef struct Fts5Global Fts5Global;
 typedef struct Fts5Auxiliary Fts5Auxiliary;
 typedef struct Fts5Auxdata Fts5Auxdata;
 
+typedef struct Fts5TokenizerModule Fts5TokenizerModule;
+
 /*
 ** NOTES ON TRANSACTIONS: 
 **
 ** SQLite invokes the following virtual table methods as transactions are 
 ** opened and closed by the user:
@@ -63,13 +65,15 @@
 ** A single object of this type is allocated when the FTS5 module is 
 ** registered with a database handle. It is used to store pointers to
 ** all registered FTS5 extensions - tokenizers and auxiliary functions.
 */
 struct Fts5Global {
+  fts5_api api;                   /* User visible part of object (see fts5.h) */
   sqlite3 *db;                    /* Associated database connection */ 
   i64 iNextId;                    /* Used to allocate unique cursor ids */
   Fts5Auxiliary *pAux;            /* First in list of all aux. functions */
+  Fts5TokenizerModule *pTok;      /* First in list of all tokenizer modules */
   Fts5Cursor *pCsr;               /* First in list of all open cursors */
 };
 
 /*
 ** Each auxiliary function registered with the FTS5 module is represented
@@ -82,10 +86,23 @@
   void *pUserData;                /* User-data pointer */
   fts5_extension_function xFunc;  /* Callback function */
   void (*xDestroy)(void*);        /* Destructor function */
   Fts5Auxiliary *pNext;           /* Next registered auxiliary function */
 };
+
+/*
+** Each tokenizer module registered with the FTS5 module is represented
+** by an object of the following type. All such objects are stored as part
+** of the Fts5Global.pTok list.
+*/
+struct Fts5TokenizerModule {
+  char *zName;                    /* Name of tokenizer */
+  void *pUserData;                /* User pointer passed to xCreate() */
+  fts5_tokenizer x;               /* Tokenizer functions */
+  void (*xDestroy)(void*);        /* Destructor function */
+  Fts5TokenizerModule *pNext;     /* Next registered tokenizer module */
+};
 
 /*
 ** Virtual-table object.
 */
 struct Fts5Table {
@@ -279,16 +296,18 @@
   int argc,                       /* Number of elements in argv array */
   const char * const *argv,       /* xCreate/xConnect argument array */
   sqlite3_vtab **ppVTab,          /* Write the resulting vtab structure here */
   char **pzErr                    /* Write any error message here */
 ){
+  Fts5Global *pGlobal = (Fts5Global*)pAux;
+  const char **azConfig = (const char**)argv;
   int rc;                         /* Return code */
   Fts5Config *pConfig;            /* Results of parsing argc/argv */
   Fts5Table *pTab = 0;            /* New virtual table object */
 
   /* Parse the arguments */
-  rc = sqlite3Fts5ConfigParse(db, argc, (const char**)argv, &pConfig, pzErr);
+  rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr);
   assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 );
 
   /* Allocate the new vtab object */
   if( rc==SQLITE_OK ){
     pTab = (Fts5Table*)sqlite3_malloc(sizeof(Fts5Table));
@@ -295,11 +314,11 @@
     if( pTab==0 ){
       rc = SQLITE_NOMEM;
     }else{
       memset(pTab, 0, sizeof(Fts5Table));
       pTab->pConfig = pConfig;
-      pTab->pGlobal = (Fts5Global*)pAux;
+      pTab->pGlobal = pGlobal;
     }
   }
 
   /* Open the index sub-system */
   if( rc==SQLITE_OK ){
@@ -855,10 +874,14 @@
 **     INSERT INTO fts(fts) VALUES($pVal)
 **
 ** Argument pVal is the value assigned to column "fts" by the INSERT 
 ** statement. This function returns SQLITE_OK if successful, or an SQLite
 ** error code if an error occurs.
+**
+** The commands implemented by this function are documented in the "Special
+** INSERT Directives" section of the documentation. It should be updated if
+** more commands are added to this function.
 */
 static int fts5SpecialCommand(Fts5Table *pTab, sqlite3_value *pVal){
   const char *z = (const char*)sqlite3_value_text(pVal);
   int n = sqlite3_value_bytes(pVal);
   int rc = SQLITE_ERROR;
@@ -1385,17 +1408,18 @@
 }
 
 /*
 ** Register a new auxiliary function with global context pGlobal.
 */
-int sqlite3Fts5CreateAux(
-  Fts5Global *pGlobal,            /* Global context (one per db handle) */
+static int fts5CreateAux(
+  fts5_api *pApi,                 /* Global context (one per db handle) */
   const char *zName,              /* Name of new function */
   void *pUserData,                /* User data for aux. function */
   fts5_extension_function xFunc,  /* Aux. function implementation */
   void(*xDestroy)(void*)          /* Destructor for pUserData */
 ){
+  Fts5Global *pGlobal = (Fts5Global*)pApi;
   int rc = sqlite3_overload_function(pGlobal->db, zName, -1);
   if( rc==SQLITE_OK ){
     Fts5Auxiliary *pAux;
     int nByte;                      /* Bytes of space to allocate */
 
@@ -1417,24 +1441,135 @@
   }
 
   return rc;
 }
 
-static void fts5ModuleDestroy(void *pCtx){
-  Fts5Auxiliary *pAux;
-  Fts5Auxiliary *pNext;
-  Fts5Global *pGlobal = (Fts5Global*)pCtx;
-  for(pAux=pGlobal->pAux; pAux; pAux=pNext){
-    pNext = pAux->pNext;
-    if( pAux->xDestroy ){
-      pAux->xDestroy(pAux->pUserData);
+/*
+** Register a new tokenizer. This is the implementation of the 
+** fts5_api.xCreateTokenizer() method.
+*/
+static int fts5CreateTokenizer(
+  fts5_api *pApi,                 /* Global context (one per db handle) */
+  const char *zName,              /* Name of new function */
+  void *pUserData,                /* User data for aux. function */
+  fts5_tokenizer *pTokenizer,     /* Tokenizer implementation */
+  void(*xDestroy)(void*)          /* Destructor for pUserData */
+){
+  Fts5Global *pGlobal = (Fts5Global*)pApi;
+  Fts5TokenizerModule *pNew;
+  int nByte;                      /* Bytes of space to allocate */
+  int rc = SQLITE_OK;
+
+  nByte = sizeof(Fts5TokenizerModule) + strlen(zName) + 1;
+  pNew = (Fts5TokenizerModule*)sqlite3_malloc(nByte);
+  if( pNew ){
+    memset(pNew, 0, nByte);
+    pNew->zName = (char*)&pNew[1];
+    strcpy(pNew->zName, zName);
+    pNew->pUserData = pUserData;
+    pNew->x = *pTokenizer;
+    pNew->xDestroy = xDestroy;
+    pNew->pNext = pGlobal->pTok;
+    pGlobal->pTok = pNew;
+  }else{
+    rc = SQLITE_NOMEM;
+  }
+
+  return rc;
+}
+
+/*
+** Find a tokenizer. This is the implementation of the 
+** fts5_api.xFindTokenizer() method.
+*/
+static int fts5FindTokenizer(
+  fts5_api *pApi,                 /* Global context (one per db handle) */
+  const char *zName,              /* Name of new function */
+  fts5_tokenizer *pTokenizer      /* Populate this object */
+){
+  Fts5Global *pGlobal = (Fts5Global*)pApi;
+  int rc = SQLITE_OK;
+  Fts5TokenizerModule *pTok;
+
+  for(pTok=pGlobal->pTok; pTok; pTok=pTok->pNext){
+    if( sqlite3_stricmp(zName, pTok->zName)==0 ) break;
+  }
+
+  if( pTok ){
+    *pTokenizer = pTok->x;
+  }else{
+    memset(pTokenizer, 0, sizeof(fts5_tokenizer));
+    rc = SQLITE_ERROR;
+  }
+
+  return rc;
+}
+
+int sqlite3Fts5GetTokenizer(
+  Fts5Global *pGlobal, 
+  const char **azArg,
+  int nArg,
+  Fts5Tokenizer **ppTok,
+  fts5_tokenizer **ppTokApi
+){
+  Fts5TokenizerModule *pMod = 0;
+  int rc = SQLITE_OK;
+  if( nArg==0 ){
+    pMod = pGlobal->pTok;
+  }else{
+    for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){
+      if( sqlite3_stricmp(azArg[0], pMod->zName)==0 ) break;
     }
+  }
+
+  if( pMod==0 ){
+    rc = SQLITE_ERROR;
+  }else{
+    rc = pMod->x.xCreate(pMod->pUserData, &azArg[1], (nArg?nArg-1:0), ppTok);
+    *ppTokApi = &pMod->x;
+  }
+
+  if( rc!=SQLITE_OK ){
+    *ppTokApi = 0;
+    *ppTok = 0;
+  }
+
+  return rc;
+}
+
+static void fts5ModuleDestroy(void *pCtx){
+  Fts5TokenizerModule *pTok, *pNextTok;
+  Fts5Auxiliary *pAux, *pNextAux;
+  Fts5Global *pGlobal = (Fts5Global*)pCtx;
+
+  for(pAux=pGlobal->pAux; pAux; pAux=pNextAux){
+    pNextAux = pAux->pNext;
+    if( pAux->xDestroy ) pAux->xDestroy(pAux->pUserData);
     sqlite3_free(pAux);
   }
+
+  for(pTok=pGlobal->pTok; pTok; pTok=pNextTok){
+    pNextTok = pTok->pNext;
+    if( pTok->xDestroy ) pTok->xDestroy(pTok->pUserData);
+    sqlite3_free(pTok);
+  }
+
   sqlite3_free(pGlobal);
 }
 
+static void fts5Fts5Func(
+  sqlite3_context *pCtx,          /* Function call context */
+  int nArg,                       /* Number of args */
+  sqlite3_value **apVal           /* Function arguments */
+){
+  Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx);
+  char buf[8];
+  assert( nArg==0 );
+  assert( sizeof(buf)>=sizeof(pGlobal) );
+  memcpy(buf, pGlobal, sizeof(pGlobal));
+  sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT);
+}
 
 int sqlite3Fts5Init(sqlite3 *db){
   static const sqlite3_module fts5Mod = {
     /* iVersion      */ 2,
     /* xCreate       */ fts5CreateMethod,
@@ -1469,14 +1604,24 @@
     rc = SQLITE_NOMEM;
   }else{
     void *p = (void*)pGlobal;
     memset(pGlobal, 0, sizeof(Fts5Global));
     pGlobal->db = db;
+    pGlobal->api.iVersion = 1;
+    pGlobal->api.xCreateFunction = fts5CreateAux;
+    pGlobal->api.xCreateTokenizer = fts5CreateTokenizer;
+    pGlobal->api.xFindTokenizer = fts5FindTokenizer;
     rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy);
     if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db);
-    if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(db);
-    if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(pGlobal);
+    if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db);
+    if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api);
+    if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api);
+    if( rc==SQLITE_OK ){
+      rc = sqlite3_create_function(
+          db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
+      );
+    }
   }
   return rc;
 }
 
 

Index: ext/fts5/fts5.h
==================================================================
--- ext/fts5/fts5.h
+++ ext/fts5/fts5.h
@@ -258,8 +258,45 @@
 };
 
 /*
 ** END OF CUSTOM TOKENIZERS
 *************************************************************************/
+
+/*************************************************************************
+** FTS5 EXTENSION REGISTRATION API
+*/
+typedef struct fts5_api fts5_api;
+struct fts5_api {
+  int iVersion;                   /* Currently always set to 1 */
+
+  /* Create a new tokenizer */
+  int (*xCreateTokenizer)(
+    fts5_api *pApi,
+    const char *zName,
+    void *pContext,
+    fts5_tokenizer *pTokenizer,
+    void (*xDestroy)(void*)
+  );
+
+  /* Find an existing tokenizer */
+  int (*xFindTokenizer)(
+    fts5_api *pApi,
+    const char *zName,
+    fts5_tokenizer *pTokenizer
+  );
+
+  /* Create a new auxiliary function */
+  int (*xCreateFunction)(
+    fts5_api *pApi,
+    const char *zName,
+    void *pContext,
+    fts5_extension_function xFunction,
+    void (*xDestroy)(void*)
+  );
+};
+
+/*
+** END OF REGISTRATION API
+*************************************************************************/
 
 #endif /* _FTS5_H */
 

Index: ext/fts5/fts5Int.h
==================================================================
--- ext/fts5/fts5Int.h
+++ ext/fts5/fts5Int.h
@@ -29,10 +29,27 @@
 #define FTS5_DEFAULT_NEARDIST 10
 
 /* Name of rank column */
 #define FTS5_RANK_NAME "rank"
 
+/**************************************************************************
+** Interface to code in fts5.c. 
+*/
+typedef struct Fts5Global Fts5Global;
+
+int sqlite3Fts5GetTokenizer(
+  Fts5Global*, 
+  const char **azArg,
+  int nArg,
+  Fts5Tokenizer**,
+  fts5_tokenizer**
+);
+
+/*
+** End of interface to code in fts5.c.
+**************************************************************************/
+
 /**************************************************************************
 ** Interface to code in fts5_config.c. fts5_config.c contains contains code
 ** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
 */
 
@@ -48,14 +65,17 @@
   char *zName;                    /* Name of FTS index */
   int nCol;                       /* Number of columns */
   char **azCol;                   /* Column names */
   int nPrefix;                    /* Number of prefix indexes */
   int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
-  sqlite3_tokenizer *pTokenizer;  /* Tokenizer instance for this table */
+  Fts5Tokenizer *pTok;
+  fts5_tokenizer *pTokApi;
 };
 
-int sqlite3Fts5ConfigParse(sqlite3*, int, const char**, Fts5Config**, char**);
+int sqlite3Fts5ConfigParse(
+    Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
+);
 void sqlite3Fts5ConfigFree(Fts5Config*);
 
 int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
 
 int sqlite3Fts5Tokenize(
@@ -401,11 +421,11 @@
 i64 sqlite3Fts5ExprRowid(Fts5Expr*);
 
 void sqlite3Fts5ExprFree(Fts5Expr*);
 
 /* Called during startup to register a UDF with SQLite */
-int sqlite3Fts5ExprInit(sqlite3*);
+int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
 
 int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
 int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
 int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
 
@@ -451,35 +471,28 @@
 /*
 ** End of interface to code in fts5_expr.c.
 **************************************************************************/
 
 
-/**************************************************************************
-** Interface to code in fts5.c. 
-*/
-typedef struct Fts5Global Fts5Global;
-
-int sqlite3Fts5CreateAux(
-    Fts5Global*, 
-    const char*, 
-    void*, 
-    fts5_extension_function, 
-    void(*)(void*)
-);
-/*
-** End of interface to code in fts5.c.
-**************************************************************************/
-
 
 /**************************************************************************
 ** Interface to code in fts5_aux.c. 
 */
 
-int sqlite3Fts5AuxInit(Fts5Global*);
+int sqlite3Fts5AuxInit(fts5_api*);
 /*
 ** End of interface to code in fts5_aux.c.
 **************************************************************************/
+
+/**************************************************************************
+** Interface to code in fts5_tokenizer.c. 
+*/
+
+int sqlite3Fts5TokenizerInit(fts5_api*);
+/*
+** End of interface to code in fts5_tokenizer.c.
+**************************************************************************/
 
 /**************************************************************************
 ** Interface to code in fts5_sorter.c. 
 */
 typedef struct Fts5Sorter Fts5Sorter;

Index: ext/fts5/fts5_aux.c
==================================================================
--- ext/fts5/fts5_aux.c
+++ ext/fts5/fts5_aux.c
@@ -954,11 +954,11 @@
     sqlite3_result_error_code(pCtx, rc);
   }
   sqlite3Fts5BufferFree(&s);
 }
 
-int sqlite3Fts5AuxInit(Fts5Global *pGlobal){
+int sqlite3Fts5AuxInit(fts5_api *pApi){
   struct Builtin {
     const char *zFunc;            /* Function name (nul-terminated) */
     void *pUserData;              /* User-data pointer */
     fts5_extension_function xFunc;/* Callback function */
     void (*xDestroy)(void*);      /* Destructor function */
@@ -971,11 +971,11 @@
 
   int rc = SQLITE_OK;             /* Return code */
   int i;                          /* To iterate through builtin functions */
 
   for(i=0; rc==SQLITE_OK && i<sizeof(aBuiltin)/sizeof(aBuiltin[0]); i++){
-    rc = sqlite3Fts5CreateAux(pGlobal, 
+    rc = pApi->xCreateFunction(pApi,
         aBuiltin[i].zFunc,
         aBuiltin[i].pUserData,
         aBuiltin[i].xFunc,
         aBuiltin[i].xDestroy
     );

Index: ext/fts5/fts5_config.c
==================================================================
--- ext/fts5/fts5_config.c
+++ ext/fts5/fts5_config.c
@@ -111,30 +111,19 @@
 */
 static char *fts5Strdup(const char *z){
   return sqlite3_mprintf("%s", z);
 }
 
-void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**);
-
 /*
 ** Allocate an instance of the default tokenizer ("simple") at 
 ** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
 ** code if an error occurs.
 */
-static int fts5ConfigDefaultTokenizer(Fts5Config *pConfig){
-  const sqlite3_tokenizer_module *pMod; /* Tokenizer module "simple" */
-  sqlite3_tokenizer *pTokenizer;  /* Tokenizer instance */
-  int rc;                         /* Return code */
-
-  sqlite3Fts3SimpleTokenizerModule(&pMod);
-  rc = pMod->xCreate(0, 0, &pTokenizer);
-  if( rc==SQLITE_OK ){
-    pTokenizer->pModule = pMod;
-    pConfig->pTokenizer = pTokenizer;
-  }
-
-  return rc;
+static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
+  return sqlite3Fts5GetTokenizer(
+      pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi
+  );
 }
 
 /*
 ** Arguments nArg/azArg contain the string arguments passed to the xCreate
 ** or xConnect method of the virtual table. This function attempts to 
@@ -146,10 +135,11 @@
 ** returned, *ppOut is set to NULL and an error message may be left in
 ** *pzErr. It is the responsibility of the caller to eventually free any 
 ** such error message using sqlite3_free().
 */
 int sqlite3Fts5ConfigParse(
+  Fts5Global *pGlobal,
   sqlite3 *db,
   int nArg,                       /* Number of arguments */
   const char **azArg,             /* Array of nArg CREATE VIRTUAL TABLE args */
   Fts5Config **ppOut,             /* OUT: Results of parse */
   char **pzErr                    /* OUT: Error message */
@@ -204,12 +194,12 @@
         }
       }
     }
   }
 
-  if( rc==SQLITE_OK && pRet->pTokenizer==0 ){
-    rc = fts5ConfigDefaultTokenizer(pRet);
+  if( rc==SQLITE_OK && pRet->pTok==0 ){
+    rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
   }
 
   if( rc!=SQLITE_OK ){
     sqlite3Fts5ConfigFree(pRet);
     *ppOut = 0;
@@ -221,12 +211,12 @@
 ** Free the configuration object passed as the only argument.
 */
 void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
   if( pConfig ){
     int i;
-    if( pConfig->pTokenizer ){
-      pConfig->pTokenizer->pModule->xDestroy(pConfig->pTokenizer);
+    if( pConfig->pTok && pConfig->pTokApi->xDelete ){
+      pConfig->pTokApi->xDelete(pConfig->pTok);
     }
     sqlite3_free(pConfig->zDb);
     sqlite3_free(pConfig->zName);
     for(i=0; i<pConfig->nCol; i++){
       sqlite3_free(pConfig->azCol[i]);
@@ -300,29 +290,9 @@
   Fts5Config *pConfig,            /* FTS5 Configuration object */
   const char *pText, int nText,   /* Text to tokenize */
   void *pCtx,                     /* Context passed to xToken() */
   int (*xToken)(void*, const char*, int, int, int, int)    /* Callback */
 ){
-  const sqlite3_tokenizer_module *pMod = pConfig->pTokenizer->pModule;
-  sqlite3_tokenizer_cursor *pCsr = 0;
-  int rc;
-
-  rc = pMod->xOpen(pConfig->pTokenizer, pText, nText, &pCsr);
-  assert( rc==SQLITE_OK || pCsr==0 );
-  if( rc==SQLITE_OK ){
-    const char *pToken;           /* Pointer to token buffer */
-    int nToken;                   /* Size of token in bytes */
-    int iStart, iEnd, iPos;       /* Start, end and position of token */
-    pCsr->pTokenizer = pConfig->pTokenizer;
-    for(rc = pMod->xNext(pCsr, &pToken, &nToken, &iStart, &iEnd, &iPos);
-        rc==SQLITE_OK;
-        rc = pMod->xNext(pCsr, &pToken, &nToken, &iStart, &iEnd, &iPos)
-    ){
-      if( (rc = xToken(pCtx, pToken, nToken, iStart, iEnd, iPos)) ) break;
-    }
-    if( rc==SQLITE_DONE ) rc = SQLITE_OK;
-    pMod->xClose(pCsr);
-  }
-  return rc;
+  return pConfig->pTokApi->xTokenize(pConfig->pTok, pCtx, pText, nText, xToken);
 }
 
 

Index: ext/fts5/fts5_expr.c
==================================================================
--- ext/fts5/fts5_expr.c
+++ ext/fts5/fts5_expr.c
@@ -1518,24 +1518,26 @@
 
   return zRet;
 }
 
 /*
-** The implementation of user-defined scalar function fts5_expr().
+** The implementation of user-defined scalar functions fts5_expr() (bTcl==0)
+** and fts5_expr_tcl() (bTcl!=0).
 */
 static void fts5ExprFunction(
   sqlite3_context *pCtx,          /* Function call context */
   int nArg,                       /* Number of args */
-  sqlite3_value **apVal           /* Function arguments */
+  sqlite3_value **apVal,          /* Function arguments */
+  int bTcl
 ){
+  Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx);
   sqlite3 *db = sqlite3_context_db_handle(pCtx);
   const char *zExpr = 0;
   char *zErr = 0;
   Fts5Expr *pExpr = 0;
   int rc;
   int i;
-  int bTcl = sqlite3_user_data(pCtx)!=0;
 
   const char **azConfig;          /* Array of arguments for Fts5Config */
   const char *zNearsetCmd = "nearset";
   int nConfig;                    /* Size of azConfig[] */
   Fts5Config *pConfig = 0;
@@ -1556,11 +1558,11 @@
   for(i=1+bTcl; i<nArg; i++){
     azConfig[i+2-bTcl] = (const char*)sqlite3_value_text(apVal[i]);
   }
   zExpr = (const char*)sqlite3_value_text(apVal[0]);
 
-  rc = sqlite3Fts5ConfigParse(db, nConfig, azConfig, &pConfig, &zErr);
+  rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
   if( rc==SQLITE_OK ){
     rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr);
   }
   if( rc==SQLITE_OK ){
     char *zText;
@@ -1585,30 +1587,45 @@
   }
   sqlite3_free(azConfig);
   sqlite3Fts5ConfigFree(pConfig);
   sqlite3Fts5ExprFree(pExpr);
 }
+
+static void fts5ExprFunctionHr(
+  sqlite3_context *pCtx,          /* Function call context */
+  int nArg,                       /* Number of args */
+  sqlite3_value **apVal           /* Function arguments */
+){
+  fts5ExprFunction(pCtx, nArg, apVal, 0);
+}
+static void fts5ExprFunctionTcl(
+  sqlite3_context *pCtx,          /* Function call context */
+  int nArg,                       /* Number of args */
+  sqlite3_value **apVal           /* Function arguments */
+){
+  fts5ExprFunction(pCtx, nArg, apVal, 1);
+}
 
 /*
 ** This is called during initialization to register the fts5_expr() scalar
 ** UDF with the SQLite handle passed as the only argument.
 */
-int sqlite3Fts5ExprInit(sqlite3 *db){
+int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){
   struct Fts5ExprFunc {
     const char *z;
-    void *p;
     void (*x)(sqlite3_context*,int,sqlite3_value**);
   } aFunc[] = {
-    { "fts5_expr", 0, fts5ExprFunction },
-    { "fts5_expr_tcl", (void*)1, fts5ExprFunction },
+    { "fts5_expr", fts5ExprFunctionHr },
+    { "fts5_expr_tcl", fts5ExprFunctionTcl },
   };
   int i;
   int rc = SQLITE_OK;
+  void *pCtx = (void*)pGlobal;
 
   for(i=0; rc==SQLITE_OK && i<(sizeof(aFunc) / sizeof(aFunc[0])); i++){
     struct Fts5ExprFunc *p = &aFunc[i];
-    rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, p->p, p->x, 0, 0);
+    rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0);
   }
 
   return rc;
 }
 

ADDED   ext/fts5/fts5_tokenize.c
Index: ext/fts5/fts5_tokenize.c
==================================================================
--- ext/fts5/fts5_tokenize.c
+++ ext/fts5/fts5_tokenize.c
@@ -0,0 +1,145 @@
+/*
+** 2014 May 31
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+*/
+
+#include "fts5.h"
+
+
+/*
+** Create a "simple" tokenizer.
+*/
+static int fts5SimpleCreate(
+  void *pCtx, 
+  const char **azArg, int nArg,
+  Fts5Tokenizer **ppOut
+){
+  *ppOut = 0;
+  return SQLITE_OK;
+}
+
+/*
+** Delete a "simple" tokenizer.
+*/
+static void fts5SimpleDelete(Fts5Tokenizer *p){
+  return;
+}
+
+/*
+** For tokenizers with no "unicode" modifier, the set of token characters
+** is the same as the set of ASCII range alphanumeric characters. 
+*/
+static unsigned char aSimpleTokenChar[128] = {
+  0, 0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0, 0, 0, 0, 0,   /* 0x00..0x0F */
+  0, 0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0, 0, 0, 0, 0,   /* 0x10..0x1F */
+  0, 0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0, 0, 0, 0, 0,   /* 0x20..0x2F */
+  1, 1, 1, 1, 1, 1, 1, 1,   1, 1, 0, 0, 0, 0, 0, 0,   /* 0x30..0x3F */
+  0, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 1, 1, 1, 1, 1,   /* 0x40..0x4F */
+  1, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 0, 0, 0, 0, 0,   /* 0x50..0x5F */
+  0, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 1, 1, 1, 1, 1,   /* 0x60..0x6F */
+  1, 1, 1, 1, 1, 1, 1, 1,   1, 1, 1, 0, 0, 0, 0, 0,   /* 0x70..0x7F */
+};
+
+
+static void simpleFold(char *aOut, const char *aIn, int nByte){
+  int i;
+  for(i=0; i<nByte; i++){
+    char c = aIn[i];
+    if( c>='A' && c<='Z' ) c += 32;
+    aOut[i] = c;
+  }
+}
+
+/*
+** Tokenize some text using the simple tokenizer.
+*/
+static int fts5SimpleTokenize(
+  Fts5Tokenizer *pTokenizer,
+  void *pCtx,
+  const char *pText, int nText,
+  int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos)
+){
+  int rc;
+  int ie;
+  int is = 0;
+  int iPos = 0;
+
+  char aFold[64];
+  int nFold = sizeof(aFold);
+  char *pFold = aFold;
+
+  do {
+    int nByte;
+
+    /* Skip any leading divider characters. */
+    while( is<nText && ((pText[is]&0x80) || aSimpleTokenChar[pText[is]]==0 ) ){
+      is++;
+    }
+    if( is==nText ) break;
+
+    /* Count the token characters */
+    ie = is+1;
+    while( ie<nText && ((pText[ie]&0x80)==0 && aSimpleTokenChar[pText[ie]] ) ){
+      ie++;
+    }
+
+    /* Fold to lower case */
+    nByte = ie-is;
+    if( nByte>nFold ){
+      if( pFold!=aFold ) sqlite3_free(pFold);
+      pFold = sqlite3_malloc(nByte*2);
+      if( pFold==0 ){
+        rc = SQLITE_NOMEM;
+        break;
+      }
+      nFold = nByte*2;
+    }
+    simpleFold(pFold, &pText[is], nByte);
+
+    /* Invoke the token callback */
+    rc = xToken(pCtx, pFold, nByte, is, ie, iPos);
+    iPos++;
+    is = ie+1;
+  }while( is<nText && rc==SQLITE_OK );
+  
+  if( pFold!=aFold ) sqlite3_free(pFold);
+  if( rc==SQLITE_DONE ) rc = SQLITE_OK;
+  return rc;
+}
+
+/*
+** Register all built-in tokenizers with FTS5.
+*/
+int sqlite3Fts5TokenizerInit(fts5_api *pApi){
+  struct BuiltinTokenizer {
+    const char *zName;
+    void *pUserData;
+    fts5_tokenizer x;
+  } aBuiltin[] = {
+    { "simple", 0, { fts5SimpleCreate, fts5SimpleDelete, fts5SimpleTokenize } }
+  };
+  
+  int rc = SQLITE_OK;             /* Return code */
+  int i;                          /* To iterate through builtin functions */
+
+  for(i=0; rc==SQLITE_OK && i<sizeof(aBuiltin)/sizeof(aBuiltin[0]); i++){
+    rc = pApi->xCreateTokenizer(pApi,
+        aBuiltin[i].zName,
+        &aBuiltin[i].pUserData,
+        &aBuiltin[i].x,
+        0
+    );
+  }
+
+  return SQLITE_OK;
+}
+
+

Index: main.mk
==================================================================
--- main.mk
+++ main.mk
@@ -78,10 +78,11 @@
 LIBOBJ += fts5_config.o
 LIBOBJ += fts5_expr.o
 LIBOBJ += fts5_hash.o
 LIBOBJ += fts5_index.o
 LIBOBJ += fts5_storage.o
+LIBOBJ += fts5_tokenize.o
 LIBOBJ += fts5parse.o
 
 
 
 # All of the source code files.
@@ -234,11 +235,12 @@
    $(TOP)/ext/fts5/fts5_config.c \
    $(TOP)/ext/fts5/fts5_expr.c \
    $(TOP)/ext/fts5/fts5_hash.c \
    $(TOP)/ext/fts5/fts5_index.c \
    fts5parse.c \
-   $(TOP)/ext/fts5/fts5_storage.c 
+   $(TOP)/ext/fts5/fts5_storage.c \
+   $(TOP)/ext/fts5/fts5_tokenize.c 
 
 
 # Generated source code files
 #
 SRC += \
@@ -608,14 +610,19 @@
 	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_index.c
 
 fts5_storage.o:	$(TOP)/ext/fts5/fts5_storage.c $(HDR) $(EXTHDR)
 	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_storage.c
 
+fts5_tokenize.o:	$(TOP)/ext/fts5/fts5_tokenize.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_tokenize.c
+
 fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
 	cp $(TOP)/ext/fts5/fts5parse.y .
 	rm -f fts5parse.h
 	./lemon $(OPTS) fts5parse.y
+	mv fts5parse.c fts5parse.c.orig
+	cat fts5parse.c.orig | sed 's/yy/fts5yy/g' | sed 's/YY/fts5YY/g' > fts5parse.c
 
 
 # Rules for building test programs and for running tests
 #
 tclsqlite3:	$(TOP)/src/tclsqlite.c libsqlite3.a

Index: src/main.c
==================================================================
--- src/main.c
+++ src/main.c
@@ -17,10 +17,13 @@
 #include "sqliteInt.h"
 
 #ifdef SQLITE_ENABLE_FTS3
 # include "fts3.h"
 #endif
+#ifdef SQLITE_ENABLE_FTS5
+int sqlite3Fts5Init(sqlite3*);
+#endif
 #ifdef SQLITE_ENABLE_RTREE
 # include "rtree.h"
 #endif
 #ifdef SQLITE_ENABLE_ICU
 # include "sqliteicu.h"
@@ -2607,11 +2610,16 @@
 #endif
 
 #ifdef SQLITE_ENABLE_FTS3
   if( !db->mallocFailed && rc==SQLITE_OK ){
     rc = sqlite3Fts3Init(db);
-    if( rc==SQLITE_OK ) rc = sqlite3Fts5Init(db);
+  }
+#endif
+
+#ifdef SQLITE_ENABLE_FTS5
+  if( !db->mallocFailed && rc==SQLITE_OK ){
+    rc = sqlite3Fts5Init(db);
   }
 #endif
 
 #ifdef SQLITE_ENABLE_ICU
   if( !db->mallocFailed && rc==SQLITE_OK ){

Index: src/test_config.c
==================================================================
--- src/test_config.c
+++ src/test_config.c
@@ -327,10 +327,16 @@
 #ifdef SQLITE_ENABLE_FTS3
   Tcl_SetVar2(interp, "sqlite_options", "fts3", "1", TCL_GLOBAL_ONLY);
 #else
   Tcl_SetVar2(interp, "sqlite_options", "fts3", "0", TCL_GLOBAL_ONLY);
 #endif
+
+#ifdef SQLITE_ENABLE_FTS5
+  Tcl_SetVar2(interp, "sqlite_options", "fts5", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "fts5", "0", TCL_GLOBAL_ONLY);
+#endif
 
 #if defined(SQLITE_ENABLE_FTS3) && defined(SQLITE_ENABLE_FTS4_UNICODE61)
   Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "1", TCL_GLOBAL_ONLY);
 #else
   Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "0", TCL_GLOBAL_ONLY);

Index: test/fts5aa.test
==================================================================
--- test/fts5aa.test
+++ test/fts5aa.test
@@ -15,11 +15,11 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix fts5aa
 
 # If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+ifcapable !fts5 {
   finish_test
   return
 }
 
 do_execsql_test 1.0 {

Index: test/fts5ab.test
==================================================================
--- test/fts5ab.test
+++ test/fts5ab.test
@@ -15,12 +15,12 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix fts5ab
 
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
   finish_test
   return
 }
 
 do_execsql_test 1.0 {

Index: test/fts5ac.test
==================================================================
--- test/fts5ac.test
+++ test/fts5ac.test
@@ -15,12 +15,12 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix fts5ac
 
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
   finish_test
   return
 }
 
 do_execsql_test 1.0 {

Index: test/fts5ad.test
==================================================================
--- test/fts5ad.test
+++ test/fts5ad.test
@@ -15,12 +15,12 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix fts5ad
 
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
   finish_test
   return
 }
 
 do_execsql_test 1.0 {

Index: test/fts5ae.test
==================================================================
--- test/fts5ae.test
+++ test/fts5ae.test
@@ -15,12 +15,12 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix fts5ae
 
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
   finish_test
   return
 }
 
 do_execsql_test 1.0 {

Index: test/fts5af.test
==================================================================
--- test/fts5af.test
+++ test/fts5af.test
@@ -17,12 +17,12 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix fts5af
 
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
   finish_test
   return
 }
 
 
@@ -136,10 +136,10 @@
   3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
   3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
   3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
 
 } {
-  do_snippet_test 1.$tn $doc "X + Y" $res
+  do_snippet_test 2.$tn $doc "X + Y" $res
 }
 
 finish_test
 

Index: test/fts5ag.test
==================================================================
--- test/fts5ag.test
+++ test/fts5ag.test
@@ -14,12 +14,12 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix fts5ag
 
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
   finish_test
   return
 }
 
 #-------------------------------------------------------------------------

Index: test/fts5ah.test
==================================================================
--- test/fts5ah.test
+++ test/fts5ah.test
@@ -14,12 +14,12 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix fts5ah
 
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
   finish_test
   return
 }
 
 #-------------------------------------------------------------------------

Index: test/fts5ai.test
==================================================================
--- test/fts5ai.test
+++ test/fts5ai.test
@@ -16,12 +16,12 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix fts5ai
 
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
   finish_test
   return
 }
 
 do_execsql_test 1.0 {

Index: test/fts5aj.test
==================================================================
--- test/fts5aj.test
+++ test/fts5aj.test
@@ -18,12 +18,12 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix fts5aj
 
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
   finish_test
   return
 }
 
 proc doc {} {

Index: test/fts5ea.test
==================================================================
--- test/fts5ea.test
+++ test/fts5ea.test
@@ -12,12 +12,12 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix fts5ea
 
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
-ifcapable !fts3 {
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
   finish_test
   return
 }
 
 proc do_syntax_error_test {tn expr err} {

Index: tool/mksqlite3c.tcl
==================================================================
--- tool/mksqlite3c.tcl
+++ tool/mksqlite3c.tcl
@@ -333,17 +333,19 @@
    fts5_aux.c
    fts5_buffer.c
    fts5.c
    fts5_config.c
    fts5_expr.c
+   fts5_hash.c
    fts5_index.c
    fts5parse.c
    fts5_storage.c
+   fts5_tokenize.c
 
    rtree.c
    icu.c
    fts3_icu.c
 } {
   copy_file tsrc/$file
 }
 
 close $out