SQLite4
Check-in [49eff5d82d]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add update function to fts5.c.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 49eff5d82d9869c0027f165822dfacb8e9ab0ff1
User & Date: dan 2012-12-21 19:58:58
Context
2012-12-22
19:59
More work on updating an fts5 index. Add a pragma that checks if the index and table contents match. check-in: 4693eb7bcc user: dan tags: trunk
2012-12-21
19:58
Add update function to fts5.c. check-in: 49eff5d82d user: dan tags: trunk
2012-12-20
18:41
Add "tokenizer=xxx" syntax to fts5. check-in: e0748900db user: dan tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/build.c.

292
293
294
295
296
297
298

299
300
301
302
303
304
305
/*
** Reclaim the memory used by an index
*/
static void freeIndex(sqlite4 *db, Index *p){
#ifndef SQLITE4_OMIT_ANALYZE
  sqlite4DeleteIndexSamples(db, p);
#endif

  sqlite4DbFree(db, p->zColAff);
  sqlite4DbFree(db, p);
}

/*
** For the index called zIdxName which is found in the database iDb,
** unlike that index from its Table then remove the index from







>







292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
/*
** Reclaim the memory used by an index
*/
static void freeIndex(sqlite4 *db, Index *p){
#ifndef SQLITE4_OMIT_ANALYZE
  sqlite4DeleteIndexSamples(db, p);
#endif
  sqlite4Fts5IndexFree(db, p);
  sqlite4DbFree(db, p->zColAff);
  sqlite4DbFree(db, p);
}

/*
** For the index called zIdxName which is found in the database iDb,
** unlike that index from its Table then remove the index from

Changes to src/fts5.c.

8
9
10
11
12
13
14

15
16
17
18
19
20
21
..
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
...
312
313
314
315
316
317
318
319






320
321
322
323
324
325
326
...
341
342
343
344
345
346
347
348






349
350
351
352
353
354
355
...
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
...
869
870
871
872
873
874
875
876





























































































































































































877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
*/

#include "sqliteInt.h"


/*
** Default distance value for NEAR operators.
*/
#define FTS5_DEFAULT_NEAR 10

/*
................................................................................
** sqlite4.pTokenizer and connected by Fts5Tokenizer.pNext.
*/
struct Fts5Tokenizer {
  char *zName;                   /* Name of tokenizer (nul-terminated) */
  void *pCtx;
  int (*xCreate)(void*, const char**, int, sqlite4_tokenizer**);
  int (*xTokenize)(void*, sqlite4_tokenizer*,
      const char*, int, int(*x)(void*, int, const char*, int)
  );
  int (*xDestroy)(sqlite4_tokenizer *);
  Fts5Tokenizer *pNext;
};

/*
** FTS5 specific index data.
................................................................................
  pPhrase->nStr++;
  return SQLITE4_OK;
}

/*
** Callback for fts5CountTokens().
*/
static int fts5CountTokensCb(void *pCtx, int iWeight, const char *z, int n){






  (*((int *)pCtx))++;
  return 0;
}

/*
** Count the number of tokens in document zDoc/nDoc using the tokenizer and
** tokenizer instance supplied as the first two arguments. Set *pnToken to
................................................................................
}

struct AppendTokensCtx {
  Fts5Parser *pParse;
  Fts5Str *pStr;
};

static int fts5AppendTokensCb(void *pCtx, int iWeight, const char *z, int n){






  struct AppendTokensCtx *p = (struct AppendTokensCtx *)pCtx;
  Fts5Token *pToken;
  char *zSpace;

  zSpace = &p->pParse->aSpace[p->pParse->iSpace];
  p->pParse->iSpace += (n+1);
  memcpy(zSpace, z, n);
................................................................................
*/
int sqlite4_create_tokenizer(
  sqlite4 *db,
  const char *zName,
  void *pCtx,
  int (*xCreate)(void*, const char**, int, sqlite4_tokenizer**),
  int (*xTokenize)(void*, sqlite4_tokenizer*,
      const char*, int, int(*x)(void*, int, const char*, int)
  ),
  int (*xDestroy)(sqlite4_tokenizer *)
){
  int rc = SQLITE4_OK;
  sqlite4_mutex_enter(db->mutex);

  /* It is not possible to override an existing tokenizer */
................................................................................
      }
      else{
        sqlite4ErrorMsg(pParse,"unrecognized argument: \"%s\"", zArg?zArg:zVal);
      }
    }
  }
}






























































































































































































/**************************************************************************
***************************************************************************
** Below this point is test code.
*/
#ifdef SQLITE4_TEST
static int fts5PrintExprNode(sqlite4 *, const char **, Fts5ExprNode *, char **);
static int fts5PrintExprNodeParen(
  sqlite4 *db, 
  const char **azCol,
  Fts5ExprNode *pNode, 
  char **pzRet
){
  int bParen = (pNode->eType!=TOKEN_PRIMITIVE || pNode->pPhrase->nStr>1);
  sqlite4_env *pEnv = sqlite4_db_env(db);
  char *zRet = *pzRet;








>







 







|







 







|
>
>
>
>
>
>







 







|
>
>
>
>
>
>







 







|







 








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







<
|







8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
..
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
...
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
...
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
...
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
...
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085

1086
1087
1088
1089
1090
1091
1092
1093
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
*/

#include "sqliteInt.h"
#include "vdbeInt.h"

/*
** Default distance value for NEAR operators.
*/
#define FTS5_DEFAULT_NEAR 10

/*
................................................................................
** sqlite4.pTokenizer and connected by Fts5Tokenizer.pNext.
*/
struct Fts5Tokenizer {
  char *zName;                   /* Name of tokenizer (nul-terminated) */
  void *pCtx;
  int (*xCreate)(void*, const char**, int, sqlite4_tokenizer**);
  int (*xTokenize)(void*, sqlite4_tokenizer*,
      const char*, int, int(*x)(void*, int, int, const char*, int, int, int)
  );
  int (*xDestroy)(sqlite4_tokenizer *);
  Fts5Tokenizer *pNext;
};

/*
** FTS5 specific index data.
................................................................................
  pPhrase->nStr++;
  return SQLITE4_OK;
}

/*
** Callback for fts5CountTokens().
*/
static int fts5CountTokensCb(
  void *pCtx, 
  int iWeight, 
  int iOff, 
  const char *z, int n,
  int iSrc, int nSrc
){
  (*((int *)pCtx))++;
  return 0;
}

/*
** Count the number of tokens in document zDoc/nDoc using the tokenizer and
** tokenizer instance supplied as the first two arguments. Set *pnToken to
................................................................................
}

struct AppendTokensCtx {
  Fts5Parser *pParse;
  Fts5Str *pStr;
};

static int fts5AppendTokensCb(
  void *pCtx, 
  int iWeight, 
  int iOff, 
  const char *z, int n, 
  int iSrc, int nSrc
){
  struct AppendTokensCtx *p = (struct AppendTokensCtx *)pCtx;
  Fts5Token *pToken;
  char *zSpace;

  zSpace = &p->pParse->aSpace[p->pParse->iSpace];
  p->pParse->iSpace += (n+1);
  memcpy(zSpace, z, n);
................................................................................
*/
int sqlite4_create_tokenizer(
  sqlite4 *db,
  const char *zName,
  void *pCtx,
  int (*xCreate)(void*, const char**, int, sqlite4_tokenizer**),
  int (*xTokenize)(void*, sqlite4_tokenizer*,
      const char*, int, int(*x)(void*, int, int, const char*, int, int, int)
  ),
  int (*xDestroy)(sqlite4_tokenizer *)
){
  int rc = SQLITE4_OK;
  sqlite4_mutex_enter(db->mutex);

  /* It is not possible to override an existing tokenizer */
................................................................................
      }
      else{
        sqlite4ErrorMsg(pParse,"unrecognized argument: \"%s\"", zArg?zArg:zVal);
      }
    }
  }
}

void sqlite4Fts5IndexFree(sqlite4 *db, Index *pIdx){
  if( pIdx->pFts ){
    sqlite4DbFree(db, pIdx->pFts->azTokenizer);
  }
}


/*
** Context structure passed to tokenizer callback when tokenizing a document.
**
** The hash table maps between tokens and TokenizeTerm structures.
*/
typedef struct TokenizeCtx TokenizeCtx;
typedef struct TokenizeTerm TokenizeTerm;
struct TokenizeCtx {
  int rc;
  int iCol;
  sqlite4 *db;
  int nMax;
  Hash hash;
};
struct TokenizeTerm {
  int iWeight;                    /* Weight of previous entry */
  int iCol;                       /* Column containing previous entry */
  int iOff;                       /* Token offset of previous entry */
  int nData;                      /* Bytes of data in value */
  int nAlloc;                     /* Bytes of data allocated */
};

TokenizeTerm *fts5TokenizeAppendInt(
  TokenizeCtx *p, 
  TokenizeTerm *pTerm, 
  int iVal
){
  unsigned char *a;
  if( (pTerm->nAlloc - pTerm->nData) < 5 ){
    int nAlloc = (pTerm->nAlloc<256) ? 256 : pTerm->nAlloc * 2;
    pTerm = sqlite4DbReallocOrFree(p->db, pTerm, nAlloc);
    if( !pTerm ) return 0;
    pTerm->nAlloc = nAlloc - sizeof(TokenizeTerm);
  }

  a = &(((unsigned char *)&pTerm[1])[pTerm->nData]);
  pTerm->nData += sqlite4PutVarint32(a, iVal);
  return pTerm;
}


static int fts5TokenizeCb(
  void *pCtx, 
  int iWeight, 
  int iOff,
  const char *zToken, 
  int nToken, 
  int iSrc, 
  int nSrc
){
  TokenizeCtx *p = (TokenizeCtx *)pCtx;
  TokenizeTerm *pTerm = 0;
  TokenizeTerm *pOrig = 0;

  if( nToken>p->nMax ) p->nMax = nToken;

  pTerm = (TokenizeTerm *)sqlite4HashFind(&p->hash, zToken, nToken);
  if( pTerm==0 ){
    const int nAlloc = 100-sizeof(TokenizeTerm);
    pTerm = sqlite4DbMallocZero(p->db, sizeof(TokenizeTerm) + nAlloc);
    if( pTerm ){
      void *pFree;
      pTerm->nAlloc = nAlloc;
      pFree = sqlite4HashInsert(&p->hash, zToken, nToken, pTerm);
      if( pFree ){
        sqlite4DbFree(p->db, pFree);
        pTerm = 0;
      }
      if( pTerm==0 ) goto tokenize_cb_out;
    }
  }
  pOrig = pTerm;

  if( iWeight!=pTerm->iWeight ){
    pTerm = fts5TokenizeAppendInt(p, pTerm, (iWeight << 2) | 0x00000003);
    if( !pTerm ) goto tokenize_cb_out;
    pTerm->iWeight = iWeight;
  }

  if( pTerm && p->iCol!=pTerm->iCol ){
    pTerm = fts5TokenizeAppendInt(p, pTerm, (p->iCol << 2) | 0x00000001);
    if( !pTerm ) goto tokenize_cb_out;
    pTerm->iCol = p->iCol;
    pTerm->iOff = 0;
  }

  pTerm = fts5TokenizeAppendInt(p, pTerm, (iOff-pTerm->iOff) << 1);
  if( !pTerm ) goto tokenize_cb_out;
  pTerm->iOff = iOff;

tokenize_cb_out:
  if( pTerm!=pOrig ){
    sqlite4HashInsert(&p->hash, zToken, nToken, 0);
  }
  if( !pTerm ){
    p->rc = SQLITE4_NOMEM;
    return 1;
  }

  return 0;
}

/*
** Update an fts index.
*/
int sqlite4Fts5Update(
  sqlite4 *db,                    /* Database handle */
  Fts5Info *pInfo,                /* Description of fts index to update */
  Mem *aArg,                      /* Array of arguments (see above) */
  int bDel,                       /* True for a delete, false for insert */
  char **pzErr                    /* OUT: Error message */
){
  int i;
  int rc = SQLITE4_OK;
  KVStore *pStore;
  TokenizeCtx sCtx;
  u8 *aKey = 0;
  int nKey = 0;

  const void *pPK;
  int nPK;
  HashElem *pElem;

  pStore = db->aDb[pInfo->iDb].pKV;
  sCtx.rc = SQLITE4_OK;
  sCtx.db = db;
  sCtx.nMax = 0;
  sqlite4HashInit(db->pEnv, &sCtx.hash);

  pPK = sqlite4_value_blob(&aArg[0]);
  nPK = sqlite4_value_bytes(&aArg[0]);

  for(i=0; rc==SQLITE4_OK && i<pInfo->nCol; i++){
    sqlite4_value *pArg = (sqlite4_value *)(&aArg[i+1]);
    if( sqlite4_value_type(pArg)==SQLITE4_TEXT ){
      const char *zText;
      int nText;

      zText = (const char *)sqlite4_value_text(pArg);
      nText = sqlite4_value_bytes(pArg);
      sCtx.iCol = i;
      rc = pInfo->pTokenizer->xTokenize(
          &sCtx, pInfo->p, zText, nText, fts5TokenizeCb
      );
    }
  }

  nKey = sqlite4VarintLen(pInfo->iRoot) + 2 + sCtx.nMax + nPK;
  aKey = sqlite4DbMallocRaw(db, nKey);
  if( aKey==0 ) rc = SQLITE4_NOMEM;

  for(pElem=sqliteHashFirst(&sCtx.hash); pElem; pElem=sqliteHashNext(pElem)){
    TokenizeTerm *pTerm = (TokenizeTerm *)sqliteHashData(pElem);
    if( rc==SQLITE4_OK ){
      int nToken = sqliteHashKeysize(pElem);
      char *zToken = (char *)sqliteHashKey(pElem);

      nKey = sqlite4PutVarint32(aKey, pInfo->iRoot);
      aKey[nKey++] = 0x24;
      memcpy(&aKey[nKey], zToken, nToken);
      nKey += nToken;
      aKey[nKey++] = 0x00;
      memcpy(&aKey[nKey], pPK, nPK);
      nKey += nPK;

      if( bDel ){
        /* delete key aKey/nKey... */
        assert( 0 );
      }else{
        const KVByteArray *aData = (const KVByteArray *)&pTerm[1];
        rc = sqlite4KVStoreReplace(pStore, aKey, nKey, aData, pTerm->nData);
      }
    }
    sqlite4DbFree(db, pTerm);
  }
  
 fts5_update_out:
  sqlite4DbFree(db, aKey);
  sqlite4HashClear(&sCtx.hash);
  return rc;
}

/**************************************************************************
***************************************************************************
** Below this point is test code.
*/
#ifdef SQLITE4_TEST
static int fts5PrintExprNode(sqlite4 *, const char **, Fts5ExprNode *, char **);
static int fts5PrintExprNodeParen(

  sqlite4 *db, const char **azCol,
  Fts5ExprNode *pNode, 
  char **pzRet
){
  int bParen = (pNode->eType!=TOKEN_PRIMITIVE || pNode->pPhrase->nStr>1);
  sqlite4_env *pEnv = sqlite4_db_env(db);
  char *zRet = *pzRet;

Changes to src/fts5func.c.

33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
..
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
}

static int fts5SimpleTokenize(
  void *pCtx, 
  sqlite4_tokenizer *p,
  const char *zDoc,
  int nDoc,
  int(*x)(void*, int, const char*, int)
){
  sqlite4_env *pEnv = (sqlite4_env *)p;
  char *aBuf;
  int nBuf;
  int iBuf;
  int i;
  int brk = 0;
................................................................................
  if( !aBuf ) return SQLITE4_NOMEM;

  iBuf = 0;
  for(i=0; brk==0 && i<nDoc; i++){
    if( sqlite4Isalnum(zDoc[i]) ){
      aBuf[iBuf++] = fts5Tolower(zDoc[i]);
    }else if( iBuf>0 ){
      brk = x(pCtx, 0, aBuf, iBuf);
      iBuf = 0;
    }
  }
  if( iBuf>0 ) x(pCtx, 0, aBuf, iBuf);

  sqlite4_free(pEnv, aBuf);
  return SQLITE4_OK;
}

int sqlite4InitFts5Func(sqlite4 *db){
  int rc;







|







 







|



|







33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
..
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
}

static int fts5SimpleTokenize(
  void *pCtx, 
  sqlite4_tokenizer *p,
  const char *zDoc,
  int nDoc,
  int(*x)(void*, int, const char*, int, int, int)
){
  sqlite4_env *pEnv = (sqlite4_env *)p;
  char *aBuf;
  int nBuf;
  int iBuf;
  int i;
  int brk = 0;
................................................................................
  if( !aBuf ) return SQLITE4_NOMEM;

  iBuf = 0;
  for(i=0; brk==0 && i<nDoc; i++){
    if( sqlite4Isalnum(zDoc[i]) ){
      aBuf[iBuf++] = fts5Tolower(zDoc[i]);
    }else if( iBuf>0 ){
      brk = x(pCtx, 0, aBuf, iBuf, i-iBuf, iBuf);
      iBuf = 0;
    }
  }
  if( iBuf>0 ) x(pCtx, 0, aBuf, iBuf, i-iBuf, iBuf);

  sqlite4_free(pEnv, aBuf);
  return SQLITE4_OK;
}

int sqlite4InitFts5Func(sqlite4 *db){
  int rc;

Changes to src/hash.h.

82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
**     SomeStructure *pData = sqliteHashData(p);
**     // do something with pData
**   }
*/
#define sqliteHashFirst(H)  ((H)->first)
#define sqliteHashNext(E)   ((E)->next)
#define sqliteHashData(E)   ((E)->data)
/* #define sqliteHashKey(E)    ((E)->pKey) // NOT USED */
/* #define sqliteHashKeysize(E) ((E)->nKey)  // NOT USED */

/*
** Number of entries in a hash table
*/
/* #define sqliteHashCount(H)  ((H)->count) // NOT USED */

#endif /* _SQLITE4_HASH_H_ */







|
|







82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
**     SomeStructure *pData = sqliteHashData(p);
**     // do something with pData
**   }
*/
#define sqliteHashFirst(H)  ((H)->first)
#define sqliteHashNext(E)   ((E)->next)
#define sqliteHashData(E)   ((E)->data)
#define sqliteHashKey(E)    ((E)->pKey)
#define sqliteHashKeysize(E) ((E)->nKey)

/*
** Number of entries in a hash table
*/
/* #define sqliteHashCount(H)  ((H)->count) // NOT USED */

#endif /* _SQLITE4_HASH_H_ */

Changes to src/sqlite.h.in.

4373
4374
4375
4376
4377
4378
4379
4380












4381
4382
4383
4384
4385
4386
4387
4388
4389

4390
4391
4392
4393
4394
4395
4396
typedef struct sqlite4_tokenizer sqlite4_tokenizer;

/*
** CAPI4REF: Register an FTS tokenizer implementation
**
** xTokenize:
**   This function does the actual tokenization of an input string. For
**   each token in the input, the callback function is invoked once.












*/
int sqlite4_create_tokenizer(
  sqlite4 *db,
  const char *zName,
  void *pCtx,
  int (*xCreate)(void*, const char**, int, sqlite4_tokenizer**),
  int (*xTokenize)(void*, sqlite4_tokenizer*,
      const char*, int, 
      int(*x)(void *ctx, int iWeight, const char *zToken, int nToken)

  ),
  int (*xDestroy)(sqlite4_tokenizer *)
);

/*
** Undo the hack that converts floating point types to integer for
** builds on processors without floating point support.







|
>
>
>
>
>
>
>
>
>
>
>
>








|
>







4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
typedef struct sqlite4_tokenizer sqlite4_tokenizer;

/*
** CAPI4REF: Register an FTS tokenizer implementation
**
** xTokenize:
**   This function does the actual tokenization of an input string. For
**   each token in the input, the callback function is invoked once. The
**   arguments passed to the callback by the tokenizer must be as follows:
**
**     xCallback(ctx, iWeight, zToken, nToken, iSrc, nSrc);
**
**   ctx     - Copy of the first argument passed to xTokenize.
**   iOff    - Offset of token in document.
**   iWeight - Weight assigned to the token by the tokenizer. Larger values
**             indicate more important tokens. 
**   zToken  - Pointer to buffer containing token text. 
**   nToken  - Number of bytes in zToken.
**   iSrc    - Byte offset in source document of the start of this token.
**   nSrc    - Bytes of text in source document tokenized to this token.
*/
int sqlite4_create_tokenizer(
  sqlite4 *db,
  const char *zName,
  void *pCtx,
  int (*xCreate)(void*, const char**, int, sqlite4_tokenizer**),
  int (*xTokenize)(void*, sqlite4_tokenizer*,
      const char*, int, 
      int(*x)(void *ctx, int iWeight, int iOff, 
              const char *zToken, int nToken, int iSrc, int nSrc)
  ),
  int (*xDestroy)(sqlite4_tokenizer *)
);

/*
** Undo the hack that converts floating point types to integer for
** builds on processors without floating point support.

Changes to src/sqliteInt.h.

560
561
562
563
564
565
566

567
568
569
570
571
572
573
....
2474
2475
2476
2477
2478
2479
2480
2481












2482
2483
2484
2485
2486
2487
2488
....
3249
3250
3251
3252
3253
3254
3255
3256

3257


3258
typedef struct ExprSpan ExprSpan;
typedef struct FKey FKey;
typedef struct FuncDestructor FuncDestructor;
typedef struct FuncDef FuncDef;
typedef struct FuncDefTable FuncDefTable;
typedef struct Fts5Tokenizer Fts5Tokenizer;
typedef struct Fts5Index Fts5Index;

typedef struct IdList IdList;
typedef struct IdListItem IdListItem;
typedef struct Index Index;
typedef struct IndexSample IndexSample;
typedef struct KeyClass KeyClass;
typedef struct KeyInfo KeyInfo;
typedef struct Lookaside Lookaside;
................................................................................
  Parse *pParse;                            /* Parser context.  */
  union {                                   /* Extra data for callback */
    NameContext *pNC;                          /* Naming context */
    int i;                                     /* Integer value */
  } u;
};

/* Forward declarations */












int sqlite4WalkExpr(Walker*, Expr*);
int sqlite4WalkExprList(Walker*, ExprList*);
int sqlite4WalkSelect(Walker*, Select*);
int sqlite4WalkSelectExpr(Walker*, Select*);
int sqlite4WalkSelectFrom(Walker*, Select*);

/*
................................................................................

int sqlite4InitFts5(sqlite4 *db);
int sqlite4InitFts5Func(sqlite4 *db);
void sqlite4ShutdownFts5(sqlite4 *db);
void sqlite4CreateUsingIndex(Parse*, CreateIndex*, ExprList*, Token*, Token*);

int sqlite4Fts5IndexSz(void);
void sqlite4Fts5IndexInit(Parse *pParse, Index *pIdx, ExprList *pArgs);




#endif /* _SQLITEINT_H_ */







>







 







|
>
>
>
>
>
>
>
>
>
>
>
>







 







|
>

>
>

560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
....
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
....
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
typedef struct ExprSpan ExprSpan;
typedef struct FKey FKey;
typedef struct FuncDestructor FuncDestructor;
typedef struct FuncDef FuncDef;
typedef struct FuncDefTable FuncDefTable;
typedef struct Fts5Tokenizer Fts5Tokenizer;
typedef struct Fts5Index Fts5Index;
typedef struct Fts5Info Fts5Info;
typedef struct IdList IdList;
typedef struct IdListItem IdListItem;
typedef struct Index Index;
typedef struct IndexSample IndexSample;
typedef struct KeyClass KeyClass;
typedef struct KeyInfo KeyInfo;
typedef struct Lookaside Lookaside;
................................................................................
  Parse *pParse;                            /* Parser context.  */
  union {                                   /* Extra data for callback */
    NameContext *pNC;                          /* Naming context */
    int i;                                     /* Integer value */
  } u;
};

/*
** An instance of this structure is used as the p4 argument to some fts5
** related vdbe opcodes.
*/
struct Fts5Info {
  int iDb;                        /* Database containing this index */
  int iRoot;                      /* Root page number of index */
  int nCol;                       /* Number of columns in indexed table */
  char **azCol;                   /* Column names for table */
  Fts5Tokenizer *pTokenizer;      /* Tokenizer module */
  sqlite4_tokenizer *p;           /* Tokenizer instance */
};

int sqlite4WalkExpr(Walker*, Expr*);
int sqlite4WalkExprList(Walker*, ExprList*);
int sqlite4WalkSelect(Walker*, Select*);
int sqlite4WalkSelectExpr(Walker*, Select*);
int sqlite4WalkSelectFrom(Walker*, Select*);

/*
................................................................................

int sqlite4InitFts5(sqlite4 *db);
int sqlite4InitFts5Func(sqlite4 *db);
void sqlite4ShutdownFts5(sqlite4 *db);
void sqlite4CreateUsingIndex(Parse*, CreateIndex*, ExprList*, Token*, Token*);

int sqlite4Fts5IndexSz(void);
void sqlite4Fts5IndexInit(Parse *, Index *, ExprList *);
void sqlite4Fts5IndexFree(sqlite4 *, Index *);

int sqlite4Fts5Update(sqlite4 *, Fts5Info *, Mem *aArg, int, char **);

#endif /* _SQLITEINT_H_ */

Changes to src/vdbe.c.

2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
....
4823
4824
4825
4826
4827
4828
4829





























4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
    p->expired = 0;
  }
  break;
}

/* Opcode: VerifyCookie P1 P2 P3 * *
**
** Check the value of global database parameter number 0 (the
** schema version) and make sure it is equal to P2 and that the
** generation counter on the local schema parse equals P3.
**
** P1 is the database number which is 0 for the main database file
** and 1 for the file holding temporary tables and some higher number
** for auxiliary databases.
**
................................................................................
    sqlite4DebugPrintf("SQL-trace: %s\n", zTrace);
  }
#endif /* SQLITE4_DEBUG */
  break;
}
#endif































/* Opcode: Noop * * * * *
**
** Do nothing.  This instruction is often useful as a jump
** destination.
*/
/*
** The magic Explain opcode are only inserted when explain==2 (which
** is to say when the EXPLAIN QUERY PLAN syntax is used.)
** This opcode records information from the optimizer.  It is the
** the same as a no-op.  This opcodesnever appears in a real VM program.
*/
default: {          /* This is really OP_Noop and OP_Explain */
  assert( pOp->opcode==OP_Noop || pOp->opcode==OP_Explain );
  break;
}

/*****************************************************************************







|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>










|







2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
....
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
    p->expired = 0;
  }
  break;
}

/* Opcode: VerifyCookie P1 P2 P3 * *
**
** cHECK THe value of global database parameter number 0 (the
** schema version) and make sure it is equal to P2 and that the
** generation counter on the local schema parse equals P3.
**
** P1 is the database number which is 0 for the main database file
** and 1 for the file holding temporary tables and some higher number
** for auxiliary databases.
**
................................................................................
    sqlite4DebugPrintf("SQL-trace: %s\n", zTrace);
  }
#endif /* SQLITE4_DEBUG */
  break;
}
#endif

/* Opcode: FtsUpdate * * P3 P4 P5
**
** This opcode is used to write to an FTS index. P4 points to an Fts5Info 
** object describing the index.
**
** If argument P5 is non-zero, then entries are removed from the FTS index.
** If it is zero, then entries are inserted. In other words, when a row
** is deleted from a table with an FTS index, this opcode is invoked with
** P5==1. When a row is inserted, it is invoked with P5==0. If an existing
** row is updated, this opcode is invoked twice - once with P5==1 and then
** again with P5==0.
**
** P3 is the first in an array of N+1 registers, where N is the number of
** columns in the indexed table. The first register in the array contains
** the PK (a blob in key-format) of the affected row. The following N
** registers contain the column values.
*/
case OP_FtsUpdate: {
  Fts5Info *pInfo;                /* Description of fts5 index to update */
  Mem *aArg;                      /* Pointer to array of N+1 arguments */

  assert( pOp->p4type==P4_FTS5INFO );
  pInfo = pOp->p4.pFtsInfo;
  aArg = &aMem[pOp->p3];

  rc = sqlite4Fts5Update(db, pInfo, aArg, pOp->p5, &p->zErrMsg);
  break;
}


/* Opcode: Noop * * * * *
**
** Do nothing.  This instruction is often useful as a jump
** destination.
*/
/*
** The magic Explain opcode are only inserted when explain==2 (which
** is to say when the EXPLAIN QUERY PLAN syntax is used.)
** This opcode records information from the optimizer.  It is the
** the same as a no-op.  This opcode never appears in a real VM program.
*/
default: {          /* This is really OP_Noop and OP_Explain */
  assert( pOp->opcode==OP_Noop || pOp->opcode==OP_Explain );
  break;
}

/*****************************************************************************

Changes to src/vdbe.h.

58
59
60
61
62
63
64

65
66
67
68
69
70
71
...
116
117
118
119
120
121
122

123
124
125
126
127
128
129
    VdbeFunc *pVdbeFunc;   /* Used when p4type is P4_VDBEFUNC */
    CollSeq *pColl;        /* Used when p4type is P4_COLLSEQ */
    Mem *pMem;             /* Used when p4type is P4_MEM */
    VTable *pVtab;         /* Used when p4type is P4_VTAB */
    KeyInfo *pKeyInfo;     /* Used when p4type is P4_KEYINFO */
    int *ai;               /* Used when p4type is P4_INTARRAY */
    SubProgram *pProgram;  /* Used when p4type is P4_SUBPROGRAM */

    int (*xAdvance)(VdbeCursor*);
  } p4;
#ifdef SQLITE4_DEBUG
  char *zComment;          /* Comment to improve readability */
#endif
#ifdef VDBE_PROFILE
  int cnt;                 /* Number of times this instruction was executed */
................................................................................
#define P4_MPRINTF  (-11) /* P4 is a string obtained from sqlite4_mprintf() */
#define P4_REAL     (-12) /* P4 is a 64-bit floating point value */
#define P4_INT64    (-13) /* P4 is a 64-bit signed integer */
#define P4_INT32    (-14) /* P4 is a 32-bit signed integer */
#define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */
#define P4_SUBPROGRAM  (-18) /* P4 is a pointer to a SubProgram structure */
#define P4_ADVANCE  (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */


/* When adding a P4 argument using P4_KEYINFO, a copy of the KeyInfo structure
** is made.  That copy is freed when the Vdbe is finalized.  But if the
** argument is P4_KEYINFO_HANDOFF, the passed in pointer is used.  It still
** gets freed when the Vdbe is finalized so it still should be obtained
** from a single sqliteMalloc().  But no copy is made and the calling
** function should *not* try to free the KeyInfo.







>







 







>







58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
...
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
    VdbeFunc *pVdbeFunc;   /* Used when p4type is P4_VDBEFUNC */
    CollSeq *pColl;        /* Used when p4type is P4_COLLSEQ */
    Mem *pMem;             /* Used when p4type is P4_MEM */
    VTable *pVtab;         /* Used when p4type is P4_VTAB */
    KeyInfo *pKeyInfo;     /* Used when p4type is P4_KEYINFO */
    int *ai;               /* Used when p4type is P4_INTARRAY */
    SubProgram *pProgram;  /* Used when p4type is P4_SUBPROGRAM */
    Fts5Info *pFtsInfo;    /* Used when p4type is P4_FTS5INDEXINFO */
    int (*xAdvance)(VdbeCursor*);
  } p4;
#ifdef SQLITE4_DEBUG
  char *zComment;          /* Comment to improve readability */
#endif
#ifdef VDBE_PROFILE
  int cnt;                 /* Number of times this instruction was executed */
................................................................................
#define P4_MPRINTF  (-11) /* P4 is a string obtained from sqlite4_mprintf() */
#define P4_REAL     (-12) /* P4 is a 64-bit floating point value */
#define P4_INT64    (-13) /* P4 is a 64-bit signed integer */
#define P4_INT32    (-14) /* P4 is a 32-bit signed integer */
#define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */
#define P4_SUBPROGRAM  (-18) /* P4 is a pointer to a SubProgram structure */
#define P4_ADVANCE  (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */
#define P4_FTS5INFO (-20) /* P4 points to an Fts5Info structure */

/* When adding a P4 argument using P4_KEYINFO, a copy of the KeyInfo structure
** is made.  That copy is freed when the Vdbe is finalized.  But if the
** argument is P4_KEYINFO_HANDOFF, the passed in pointer is used.  It still
** gets freed when the Vdbe is finalized so it still should be obtained
** from a single sqliteMalloc().  But no copy is made and the calling
** function should *not* try to free the KeyInfo.