SQLite

  int bContent,                   /* Loaded from content table */
  int *pbResetTokenizer,          /* OUT: True if ClearLocale() required */
  const char **ppText,            /* OUT: Pointer to text buffer */
  int *pnText                     /* OUT: Size of (*ppText) in bytes */
);

void sqlite3Fts5ClearLocale(Fts5Config *pConfig);

int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal);

/*
** End of interface to code in fts5.c.
**************************************************************************/

/**************************************************************************
** Interface to code in fts5_hash.c. 

  fts5_api api;                   /* User visible part of object (see fts5.h) */
  sqlite3 *db;                    /* Associated database connection */ 
  i64 iNextId;                    /* Used to allocate unique cursor ids */
  Fts5Auxiliary *pAux;            /* First in list of all aux. functions */
  Fts5TokenizerModule *pTok;      /* First in list of all tokenizer modules */
  Fts5TokenizerModule *pDfltTok;  /* Default tokenizer module */
  Fts5Cursor *pCsr;               /* First in list of all open cursors */
  u32 aLocaleHdr[4];
};

/*
** Size of header on fts5_locale() values. And macro to access a buffer
** containing a copy of the header from an Fts5Config pointer.
*/
#define FTS5_LOCALE_HDR_SIZE sizeof( ((Fts5Global*)0)->aLocaleHdr )
#define FTS5_LOCALE_HDR(pConfig) ((const u8*)(pConfig->pGlobal->aLocaleHdr))


/*
** Each auxiliary function registered with the FTS5 module is represented
** by an object of the following type. All such objects are stored as part
** of the Fts5Global.pAux list.
*/
struct Fts5Auxiliary {

#define FTS5CSR_FREE_ZRANK        0x10
#define FTS5CSR_REQUIRE_RESEEK    0x20
#define FTS5CSR_REQUIRE_POSLIST   0x40

#define BitFlagAllTest(x,y) (((x) & (y))==(y))
#define BitFlagTest(x,y)    (((x) & (y))!=0)








/*
** Macros to Set(), Clear() and Test() cursor flags.
*/
#define CsrFlagSet(pCsr, flag)   ((pCsr)->csrflags |= (flag))
#define CsrFlagClear(pCsr, flag) ((pCsr)->csrflags &= ~(flag))
#define CsrFlagTest(pCsr, flag)  ((pCsr)->csrflags & (flag))

/*
** Clear any locale configured by an earlier call to fts5SetLocale() or
** sqlite3Fts5ExtractText().
*/
void sqlite3Fts5ClearLocale(Fts5Config *pConfig){
  fts5SetLocale(pConfig, 0, 0);
}

/*
** Return true if the value passed as the only argument is an 
** fts5_locale() value.  
*/
int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal){
  int ret = 0;
  if( sqlite3_value_type(pVal)==SQLITE_BLOB ){
    if( sqlite3_value_bytes(pVal)>FTS5_LOCALE_HDR_SIZE
     && 0==memcmp(sqlite3_value_blob(pVal), FTS5_LOCALE_HDR(pConfig), 4)
    ){
      ret = 1;
    }
  }
  return ret;
}

/*
** This function is used to extract utf-8 text from an sqlite3_value. This
** is usually done in order to tokenize it. For example, when:
**
**     * a value is written to an fts5 table,
**     * a value is deleted from an FTS5_CONTENT_NORMAL table,
**     * a value containing a query expression is passed to xFilter()
**
** and so on.
**
** This function handles 2 cases:
**
**   1) Ordinary values. The text can be extracted from these using
**      sqlite3_value_text().
**
**   2) Combination text/locale blobs created by fts5_locale(). There
**      are several cases for these:
**
**        * Blobs that have the 16-byte header, and


**        * Blobs read from the content table of a locale=1 regular 
**          content table.
**
**      The first case above has the 16 byte FTS5_LOCALE_HDR(pConfig)
**      header. It is an error if a blob read from the content table of 
**      an external content table does not have the required header. A blob 
**      read from the content table of a regular locale=1 table does not 
**      have the header. This is to save space.
**
** If successful, SQLITE_OK is returned and output parameters (*ppText)
** and (*pnText) are set to point to a buffer containing the extracted utf-8 
** text and its length in bytes, respectively. The buffer is not 
** nul-terminated. It has the same lifetime as the sqlite3_value object
** from which it is extracted.
**

  int *pbResetTokenizer,          /* OUT: True if xSetLocale(NULL) required */
  const char **ppText,            /* OUT: Pointer to text buffer */
  int *pnText                     /* OUT: Size of (*ppText) in bytes */
){
  const char *pText = 0;
  int nText = 0;
  int rc = SQLITE_OK;

  /* 0: Do not decode blob 
  ** 1: Decode blob, expect fts5_locale() header
  ** 2: Decode blob, expect no fts5_locale() header
  */
  int eDecodeBlob = 0;

  assert( pbResetTokenizer==0 || *pbResetTokenizer==0 );
  assert( bContent==0 || pConfig->eContent!=FTS5_CONTENT_NONE );


  if( sqlite3_value_type(pVal)==SQLITE_BLOB ){
    if( bContent 
     && pConfig->bLocale 
     && pConfig->eContent==FTS5_CONTENT_NORMAL 
    ){
      eDecodeBlob = 2;
    }else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
      eDecodeBlob = 1;
    }else if( bContent && pConfig->bLocale ){
      return SQLITE_ERROR;
    }
  }

  if( eDecodeBlob ){

    const u8 *pBlob = sqlite3_value_blob(pVal);
    int nBlob = sqlite3_value_bytes(pVal);
    int nLocale = 0;

    /* Unless this blob was read from the %_content table of an 
    ** FTS5_CONTENT_NORMAL table, it should have the 4 byte fts5_locale() 
    ** header. Check for this. If it is not found, return an error.  */
    if( eDecodeBlob==1 ){



      pBlob += FTS5_LOCALE_HDR_SIZE;
      nBlob -= FTS5_LOCALE_HDR_SIZE;
    }





    for(nLocale=0; nLocale<nBlob; nLocale++){
      if( pBlob[nLocale]==0x00 ) break;
    }
    if( nLocale==nBlob || nLocale==0 ){
      rc = SQLITE_ERROR;
    }else{
      pText = (const char*)&pBlob[nLocale+1];
      nText = nBlob-nLocale-1;

      if( pbResetTokenizer ){
        fts5SetLocale(pConfig, (const char*)pBlob, nLocale);
        *pbResetTokenizer = 1;

      }
    }

  }else{
    pText = (const char*)sqlite3_value_text(pVal);
    nText = sqlite3_value_bytes(pVal);
  }

      bUpdateOrDelete = 1;
    }

    /* INSERT or UPDATE */
    else{
      int eType1 = sqlite3_value_numeric_type(apVal[1]);




      int ii;
      for(ii=0; ii<pConfig->nCol; ii++){
        sqlite3_value *pVal = apVal[ii+2];
        if( sqlite3_value_type(pVal)==SQLITE_BLOB ){


          int isLocale = sqlite3Fts5IsLocaleValue(pConfig, pVal);

          if( pConfig->bLocale ){
            if( isLocale==0 && pConfig->abUnindexed[ii]==0 ){
              rc = SQLITE_MISMATCH;
              goto update_out;
            }
          }else{
            if( isLocale ){
              fts5SetVtabError(pTab, "fts5_locale() requires locale=1");
              rc = SQLITE_MISMATCH;
              goto update_out;
            }
          }
        }
      }

      if( eType0!=SQLITE_INTEGER ){
        /* An INSERT statement. If the conflict-mode is REPLACE, first remove
        ** the current entry (if any). */

   && pConfig->bLocale
  ){
    rc = fts5SeekCursor(pCsr, 0);
    if( rc==SQLITE_OK ){
      /* Load the value into pVal. pVal is a locale/text pair iff:
      **
      **   1) It is an SQLITE_BLOB, and
      **   2) Either the FTS5_LOCALE_HDR header is present, or else the
      **      value was loaded from an FTS5_CONTENT_NORMAL table.
      **
      ** If condition (1) is met but condition (2) is not, it is an error.
      */ 
      sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1);
      if( sqlite3_value_type(pVal)==SQLITE_BLOB ){
        const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal);
        int nBlob = sqlite3_value_bytes(pVal);
        if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){
          if( sqlite3Fts5IsLocaleValue(pConfig, pVal)==0 ){

            rc = SQLITE_ERROR;
          }
          pBlob += FTS5_LOCALE_HDR_SIZE;
          nBlob -= FTS5_LOCALE_HDR_SIZE;
        }
        if( rc==SQLITE_OK ){
          int nLocale = 0;
          for(nLocale=0; nLocale<nBlob && pBlob[nLocale]!=0x00; nLocale++);
          if( nLocale==nBlob || nLocale==0 ){
            rc = SQLITE_ERROR;
          }else{

){
  assert( pConfig->eContent!=FTS5_CONTENT_NONE );

  if( pConfig->bLocale 
   && sqlite3_value_type(pVal)==SQLITE_BLOB 
   && pConfig->abUnindexed[iCol]==0
  ){

    const u8 *pBlob = sqlite3_value_blob(pVal);
    int nBlob = sqlite3_value_bytes(pVal);
    int ii;

    if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){
      if( nBlob<FTS5_LOCALE_HDR_SIZE 
       || memcmp(pBlob, FTS5_LOCALE_HDR(pConfig), FTS5_LOCALE_HDR_SIZE) 
      ){
        sqlite3_result_error_code(pCtx, SQLITE_ERROR);
        return;
      }else{
        pBlob += FTS5_LOCALE_HDR_SIZE;
        nBlob -= FTS5_LOCALE_HDR_SIZE;
      }
    }

    for(ii=0; ii<nBlob && pBlob[ii]; ii++);
    if( ii==0 || ii==nBlob ){
      sqlite3_result_error_code(pCtx, SQLITE_ERROR);
    }else{

  zText = (const char*)sqlite3_value_text(apArg[1]);
  nText = sqlite3_value_bytes(apArg[1]);

  if( zLocale==0 || zLocale[0]=='\0' ){
    sqlite3_result_text(pCtx, zText, nText, SQLITE_TRANSIENT);
  }else{
    Fts5Global *p = (Fts5Global*)sqlite3_user_data(pCtx);
    u8 *pBlob = 0;
    u8 *pCsr = 0;
    int nBlob = 0;



    nBlob = FTS5_LOCALE_HDR_SIZE + nLocale + 1 + nText;
    pBlob = (u8*)sqlite3_malloc(nBlob);
    if( pBlob==0 ){
      sqlite3_result_error_nomem(pCtx);
      return;
    }

    pCsr = pBlob;
    memcpy(pCsr, (const u8*)p->aLocaleHdr, FTS5_LOCALE_HDR_SIZE);
    pCsr += FTS5_LOCALE_HDR_SIZE;
    memcpy(pCsr, zLocale, nLocale);
    pCsr += nLocale;
    (*pCsr++) = 0x00;
    if( zText ) memcpy(pCsr, zText, nText);
    assert( &pCsr[nText]==&pBlob[nBlob] );

    sqlite3_result_blob(pCtx, pBlob, nBlob, sqlite3_free);

  }
}

/*
** Return true if zName is the extension on one of the shadow tables used
** by this module.
*/

    pGlobal->db = db;
    pGlobal->api.iVersion = 3;
    pGlobal->api.xCreateFunction = fts5CreateAux;
    pGlobal->api.xCreateTokenizer = fts5CreateTokenizer;
    pGlobal->api.xFindTokenizer = fts5FindTokenizer;
    pGlobal->api.xCreateTokenizer_v2 = fts5CreateTokenizer_v2;
    pGlobal->api.xFindTokenizer_v2 = fts5FindTokenizer_v2;

    /* Initialize pGlobal->aLocaleHdr[] to a 128-bit pseudo-random vector.
    ** The constants below were generated randomly.  */
    sqlite3_randomness(sizeof(pGlobal->aLocaleHdr), pGlobal->aLocaleHdr);
    pGlobal->aLocaleHdr[0] ^= 0xF924976D;
    pGlobal->aLocaleHdr[1] ^= 0x16596E13;
    pGlobal->aLocaleHdr[2] ^= 0x7C80BEAA;
    pGlobal->aLocaleHdr[3] ^= 0x9B03A67F;
    assert( sizeof(pGlobal->aLocaleHdr)==16 );

    rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy);
    if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db);
    if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db);
    if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api);
    if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api);
    if( rc==SQLITE_OK ) rc = sqlite3Fts5VocabInit(pGlobal, db);
    if( rc==SQLITE_OK ){

    rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
    for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
      sqlite3_value *pVal = apVal[i];
      if( sqlite3_value_nochange(pVal) && p->pSavedRow ){
        /* This is an UPDATE statement, and column (i-2) was not modified.
        ** Retrieve the value from Fts5Storage.pSavedRow instead. */
        pVal = sqlite3_column_value(p->pSavedRow, i-1);
      }else if( sqlite3_value_type(pVal)==SQLITE_BLOB && pConfig->bLocale ){
        assert( pConfig->bLocale );
        assert( i>1 );
        if( pConfig->abUnindexed[i-2] ){
          if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
            /* At attempt to insert an fts5_locale() value into an UNINDEXED
            ** column. Strip the locale away and just bind the text.  */
            const char *pText = 0;
            int nText = 0;
            rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText);
            sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT);
            continue;
          }
        }else{
          const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal);
          int nBlob = sqlite3_value_bytes(pVal);
          assert( nBlob>4 );
          sqlite3_bind_blob(pInsert, i, pBlob+16, nBlob-16, SQLITE_TRANSIENT);

          continue;
        }
      }

      rc = sqlite3_bind_value(pInsert, i, pVal);
    }
    if( rc==SQLITE_OK ){
      sqlite3_step(pInsert);
      rc = sqlite3_reset(pInsert);

  
    do_catchsql_test 10.2.$tn.3 {
      INSERT INTO ft(ft) VALUES('rebuild');
    } {1 {SQL logic error}}
  
    do_catchsql_test 10.2.$tn.4 "
      SELECT * FROM ft( test_setsubtype($v, 76) );
    " {1 {fts5: syntax error near ""}}
  
    do_execsql_test 10.2.$tn.5 {
      INSERT INTO ft(rowid, x) VALUES(1, 'hello world');
    }
  
    if {"%DETAIL%"!="full"} {
      do_catchsql_test 10.2.$tn.6 {

    do_execsql_test 10.2.$tn.10 {
      DELETE FROM x1;
      INSERT INTO x1(ii, x) VALUES(1, 'hello world');
    }

    do_catchsql_test 10.2.$tn.11 "
      INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, test_setsubtype($v,76) )
    " {0 {}}

    do_catchsql_test 10.2.$tn.12 "
      INSERT INTO ft(rowid, x) VALUES(2, test_setsubtype($v,76) )
    " {1 {datatype mismatch}}

    do_execsql_test 10.2.$tn.13 {
      INSERT INTO ft2(rowid, x) VALUES(1, 'hello world');
    }
    do_execsql_test 10.2.$tn.14 "UPDATE ft2_content SET c0=$v"

    do_catchsql_test 10.2.$tn.15 {

  FROM ft('one AND three') ORDER BY rowid
} {1 {non-integer argument passed to function fts5_get_locale()}}
do_catchsql_test 13.2.7 {
  SELECT 
    quote(fts5_get_locale(ft, 0.0)), quote(fts5_get_locale(ft, 1)) 
  FROM ft('one AND three') ORDER BY rowid
} {1 {non-integer argument passed to function fts5_get_locale()}}

#-------------------------------------------------------------------------
# Check that UPDATE statements that may affect more than one row work.
#
reset_db
do_execsql_test 14.1 {
  CREATE VIRTUAL TABLE ft USING fts5(a, b, locale=1);
}

do_execsql_test 14.2 {
  INSERT INTO ft VALUES('hello', 'world');
}

do_execsql_test 14.3 {
  UPDATE ft SET b = fts5_locale('en_AU', 'world');
}

do_catchsql_test 14.4 {
  INSERT INTO ft VALUES(X'abcd', X'1234');
} {1 {datatype mismatch}}

do_execsql_test 14.4 {
  SELECT * FROM ft
} {hello world}

finish_test

  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x%1 FROM c WHERE x<599237)
    INSERT INTO t1( a ) SELECT randomblob(3000) FROM t2 ;
} {0 {}}

do_catchsql_test 25.6 {
  INSERT INTO t1(t1) SELECT x FROM t2;
  INSERT INTO t1(t1) SELECT x FROM t2;
} {0 {}}

#-------------------------------------------------------------------------
reset_db
do_test 26.0 {
  sqlite3 db {}
  db deserialize [decode_hexdb {
.open --hexdb