SQLite

Check-in [9cf0f2b76b]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Experimental changes to fts3 function matchinfo().
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts3-experimental
Files: files | file ages | folders
SHA1: 9cf0f2b76bc68c168e3fa861b7235f384db21d38
User & Date: dan 2010-11-23 19:16:48.000
Context
2010-11-24
11:51
Remove some unused code from fts3. Add tests to fts3matchinfo.test. (check-in: ae40b34cf7 user: dan tags: fts3-experimental)
2010-11-23
19:16
Experimental changes to fts3 function matchinfo(). (check-in: 9cf0f2b76b user: dan tags: fts3-experimental)
2010-11-22
17:26
Fix a typo in unixCurrentTimeInt64() preventing compilation with NO_GETTOD defined. (check-in: 3df3e79b56 user: dan tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3.c.
3269
3270
3271
3272
3273
3274
3275
3276
3277

3278
3279
3280
3281
3282
3283
3284
3285
*/
static void fts3MatchinfoFunc(
  sqlite3_context *pContext,      /* SQLite function call context */
  int nVal,                       /* Size of argument array */
  sqlite3_value **apVal           /* Array of arguments */
){
  Fts3Cursor *pCsr;               /* Cursor handle passed through apVal[0] */
  assert( nVal==1 );
  if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){

    sqlite3Fts3Matchinfo(pContext, pCsr);
  }
}

/*
** This routine implements the xFindFunction method for the FTS3
** virtual table.
*/







|

>
|







3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
*/
static void fts3MatchinfoFunc(
  sqlite3_context *pContext,      /* SQLite function call context */
  int nVal,                       /* Size of argument array */
  sqlite3_value **apVal           /* Array of arguments */
){
  Fts3Cursor *pCsr;               /* Cursor handle passed through apVal[0] */
  assert( nVal==1 || nVal==2 );
  if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){
    const char *zArg = (nVal>1 ? sqlite3_value_text(apVal[1]) : 0);
    sqlite3Fts3Matchinfo(pContext, pCsr, zArg);
  }
}

/*
** This routine implements the xFindFunction method for the FTS3
** virtual table.
*/
3460
3461
3462
3463
3464
3465
3466

3467
3468
3469
3470
3471
3472
3473
  ** module with sqlite.
  */
  if( SQLITE_OK==rc 
   && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
   && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
   && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
   && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1))

   && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1))
  ){
    rc = sqlite3_create_module_v2(
        db, "fts3", &fts3Module, (void *)pHash, hashDestroy
    );
    if( rc==SQLITE_OK ){
      rc = sqlite3_create_module_v2(







>







3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
  ** module with sqlite.
  */
  if( SQLITE_OK==rc 
   && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
   && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
   && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
   && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1))
   && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2))
   && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1))
  ){
    rc = sqlite3_create_module_v2(
        db, "fts3", &fts3Module, (void *)pHash, hashDestroy
    );
    if( rc==SQLITE_OK ){
      rc = sqlite3_create_module_v2(
Changes to ext/fts3/fts3Int.h.
158
159
160
161
162
163
164

165
166
167
168
169



170
171
172
173
174
175
176
177
178
179
180
struct Fts3Cursor {
  sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
  i16 eSearch;                    /* Search strategy (see below) */
  u8 isEof;                       /* True if at End Of Results */
  u8 isRequireSeek;               /* True if must seek pStmt to %_content row */
  sqlite3_stmt *pStmt;            /* Prepared statement in use by the cursor */
  Fts3Expr *pExpr;                /* Parsed MATCH query string */

  Fts3DeferredToken *pDeferred;   /* Deferred search tokens, if any */
  sqlite3_int64 iPrevId;          /* Previous id read from aDoclist */
  char *pNextId;                  /* Pointer into the body of aDoclist */
  char *aDoclist;                 /* List of docids for full-text queries */
  int nDoclist;                   /* Size of buffer at aDoclist */



  int isMatchinfoNeeded;          /* True when aMatchinfo[] needs filling in */
  u32 *aMatchinfo;                /* Information about most recent match */
  int eEvalmode;                  /* An FTS3_EVAL_XX constant */
  int nRowAvg;                    /* Average size of database rows, in pages */
};

#define FTS3_EVAL_FILTER    0
#define FTS3_EVAL_NEXT      1
#define FTS3_EVAL_MATCHINFO 2

/*







>





>
>
>


|
|







158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
struct Fts3Cursor {
  sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
  i16 eSearch;                    /* Search strategy (see below) */
  u8 isEof;                       /* True if at End Of Results */
  u8 isRequireSeek;               /* True if must seek pStmt to %_content row */
  sqlite3_stmt *pStmt;            /* Prepared statement in use by the cursor */
  Fts3Expr *pExpr;                /* Parsed MATCH query string */
  int nPhrase;                    /* Number of matchable phrases in query */
  Fts3DeferredToken *pDeferred;   /* Deferred search tokens, if any */
  sqlite3_int64 iPrevId;          /* Previous id read from aDoclist */
  char *pNextId;                  /* Pointer into the body of aDoclist */
  char *aDoclist;                 /* List of docids for full-text queries */
  int nDoclist;                   /* Size of buffer at aDoclist */
  int eEvalmode;                  /* An FTS3_EVAL_XX constant */
  int nRowAvg;                    /* Average size of database rows, in pages */

  int isMatchinfoNeeded;          /* True when aMatchinfo[] needs filling in */
  u32 *aMatchinfo;                /* Information about most recent match */
  int nMatchinfo;                 /* Number of elements in aMatchinfo[] */
  char *zMatchinfo;               /* Matchinfo specification */
};

#define FTS3_EVAL_FILTER    0
#define FTS3_EVAL_NEXT      1
#define FTS3_EVAL_MATCHINFO 2

/*
288
289
290
291
292
293
294



295
296
297
298
299
300
301
int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **);
int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*);
int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*);
int sqlite3Fts3ReadLock(Fts3Table *);
int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);




void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *);

void sqlite3Fts3SegmentsClose(Fts3Table *);







>
>
>







292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **);
int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*);
int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*);
int sqlite3Fts3ReadLock(Fts3Table *);
int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);

int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);

void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *);

void sqlite3Fts3SegmentsClose(Fts3Table *);
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
int sqlite3Fts3IsIdChar(char);

/* fts3_snippet.c */
void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*);
void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
  const char *, const char *, int, int
);
void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *);

/* fts3_expr.c */
int sqlite3Fts3ExprParse(sqlite3_tokenizer *, 
  char **, int, int, const char *, int, Fts3Expr **
);
void sqlite3Fts3ExprFree(Fts3Expr *);
#ifdef SQLITE_TEST
int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
#endif

#endif /* _FTSINT_H */







|











342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
int sqlite3Fts3IsIdChar(char);

/* fts3_snippet.c */
void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*);
void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
  const char *, const char *, int, int
);
void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);

/* fts3_expr.c */
int sqlite3Fts3ExprParse(sqlite3_tokenizer *, 
  char **, int, int, const char *, int, Fts3Expr **
);
void sqlite3Fts3ExprFree(Fts3Expr *);
#ifdef SQLITE_TEST
int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
#endif

#endif /* _FTSINT_H */
Changes to ext/fts3/fts3_snippet.c.
13
14
15
16
17
18
19
















20
21
22
23
24
25
26

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)

#include "fts3Int.h"
#include <string.h>
#include <assert.h>


















/*
** Used as an fts3ExprIterate() context when loading phrase doclists to
** Fts3Expr.aDoclist[]/nDoclist.
*/
typedef struct LoadDoclistCtx LoadDoclistCtx;
struct LoadDoclistCtx {







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)

#include "fts3Int.h"
#include <string.h>
#include <assert.h>

/*
** Characters that may appear in the second argument to matchinfo().
*/
#define FTS3_MATCHINFO_NPHRASE   'p'        /* 1 value */
#define FTS3_MATCHINFO_NCOL      'c'        /* 1 value */
#define FTS3_MATCHINFO_NDOC      'n'        /* 1 value */
#define FTS3_MATCHINFO_AVGLENGTH 'a'        /* nCol values */
#define FTS3_MATCHINFO_LENGTH    'l'        /* nCol values */
#define FTS3_MATCHINFO_LCS       's'        /* nCol values */
#define FTS3_MATCHINFO_HITS      'x'        /* 3*nCol*nPhrase values */

/*
** The default value for the second argument to matchinfo(). 
*/
#define FTS3_MATCHINFO_DEFAULT   "pcx"


/*
** Used as an fts3ExprIterate() context when loading phrase doclists to
** Fts3Expr.aDoclist[]/nDoclist.
*/
typedef struct LoadDoclistCtx LoadDoclistCtx;
struct LoadDoclistCtx {
66
67
68
69
70
71
72


73
74
75
76
77
78
79
** This type is used as an fts3ExprIterate() context object while 
** accumulating the data returned by the matchinfo() function.
*/
typedef struct MatchInfo MatchInfo;
struct MatchInfo {
  Fts3Cursor *pCursor;            /* FTS3 Cursor */
  int nCol;                       /* Number of columns in table */


  u32 *aMatchinfo;                /* Pre-allocated buffer */
};



/*
** The snippet() and offsets() functions both return text values. An instance







>
>







82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
** This type is used as an fts3ExprIterate() context object while 
** accumulating the data returned by the matchinfo() function.
*/
typedef struct MatchInfo MatchInfo;
struct MatchInfo {
  Fts3Cursor *pCursor;            /* FTS3 Cursor */
  int nCol;                       /* Number of columns in table */
  int nPhrase;                    /* Number of matchable phrases in query */
  sqlite3_int64 nDoc;             /* Number of docs in database */
  u32 *aMatchinfo;                /* Pre-allocated buffer */
};



/*
** The snippet() and offsets() functions both return text values. An instance
779
780
781
782
783
784
785
786



787













788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820

821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866





























































































































































867
868
869
870
871



872
873
874

875

876
877
878







879


880
881
882
883
884


885
886
887
888
889
890
891
892
893
894
895
896
897


898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913

914
915
916

917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
  }
  pCsr++;
  *pp = pCsr;
}

/*
** fts3ExprIterate() callback used to collect the "global" matchinfo stats
** for a single query. The "global" stats are those elements of the matchinfo



** array that are constant for all rows returned by the current query.













*/
static int fts3ExprGlobalMatchinfoCb(
  Fts3Expr *pExpr,                /* Phrase expression node */
  int iPhrase,                    /* Phrase number (numbered from zero) */
  void *pCtx                      /* Pointer to MatchInfo structure */
){
  MatchInfo *p = (MatchInfo *)pCtx;
  Fts3Cursor *pCsr = p->pCursor;
  char *pIter;
  char *pEnd;
  char *pFree = 0;
  const int iStart = 2 + (iPhrase * p->nCol * 3) + 1;

  assert( pExpr->isLoaded );
  assert( pExpr->eType==FTSQUERY_PHRASE );

  if( pCsr->pDeferred ){
    Fts3Phrase *pPhrase = pExpr->pPhrase;
    int ii;
    for(ii=0; ii<pPhrase->nToken; ii++){
      if( pPhrase->aToken[ii].bFulltext ) break;
    }
    if( ii<pPhrase->nToken ){
      int nFree = 0;
      int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree);
      if( rc!=SQLITE_OK ) return rc;
      pIter = pFree;
      pEnd = &pFree[nFree];
    }else{
      int nDoc = p->aMatchinfo[2 + 3*p->nCol*p->aMatchinfo[0]];
      for(ii=0; ii<p->nCol; ii++){
        p->aMatchinfo[iStart + ii*3] = nDoc;
        p->aMatchinfo[iStart + ii*3 + 1] = nDoc;

      }
      return SQLITE_OK;
    }
  }else{
    pIter = pExpr->aDoclist;
    pEnd = &pExpr->aDoclist[pExpr->nDoclist];
  }

  /* Fill in the global hit count matrix row for this phrase. */
  while( pIter<pEnd ){
    while( *pIter++ & 0x80 );      /* Skip past docid. */
    fts3LoadColumnlistCounts(&pIter, &p->aMatchinfo[iStart], 1);
  }

  sqlite3_free(pFree);
  return SQLITE_OK;
}

/*
** fts3ExprIterate() callback used to collect the "local" matchinfo stats
** for a single query. The "local" stats are those elements of the matchinfo
** array that are different for each row returned by the query.
*/
static int fts3ExprLocalMatchinfoCb(
  Fts3Expr *pExpr,                /* Phrase expression node */
  int iPhrase,                    /* Phrase number */
  void *pCtx                      /* Pointer to MatchInfo structure */
){
  MatchInfo *p = (MatchInfo *)pCtx;

  if( pExpr->aDoclist ){
    char *pCsr;
    int iStart = 2 + (iPhrase * p->nCol * 3);
    int i;

    for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0;

    pCsr = sqlite3Fts3FindPositions(pExpr, p->pCursor->iPrevId, -1);
    if( pCsr ){
      fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0);
    }
  }

  return SQLITE_OK;
}






























































































































































/*
** Populate pCsr->aMatchinfo[] with data for the current row. The 
** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32).
*/
static int fts3GetMatchinfo(Fts3Cursor *pCsr){



  MatchInfo sInfo;
  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  int rc = SQLITE_OK;



  sInfo.pCursor = pCsr;
  sInfo.nCol = pTab->nColumn;








  if( pCsr->aMatchinfo==0 ){


    /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the
    ** matchinfo function has been called for this query. In this case 
    ** allocate the array used to accumulate the matchinfo data and
    ** initialize those elements that are constant for every row.
    */


    int nPhrase;                  /* Number of phrases */
    int nMatchinfo;               /* Number of u32 elements in match-info */

    /* Load doclists for each phrase in the query. */
    rc = fts3ExprLoadDoclists(pCsr, &nPhrase, 0);
    if( rc!=SQLITE_OK ){
      return rc;
    }
    nMatchinfo = 2 + 3*sInfo.nCol*nPhrase;
    if( pTab->bHasDocsize ){
      nMatchinfo += 1 + 2*pTab->nColumn;
    }



    sInfo.aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo);
    if( !sInfo.aMatchinfo ){ 
      return SQLITE_NOMEM;
    }
    memset(sInfo.aMatchinfo, 0, sizeof(u32)*nMatchinfo);

    /* First element of match-info is the number of phrases in the query */
    sInfo.aMatchinfo[0] = nPhrase;
    sInfo.aMatchinfo[1] = sInfo.nCol;
    if( pTab->bHasDocsize ){
      int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1];
      rc = sqlite3Fts3MatchinfoDocsizeGlobal(pCsr, &sInfo.aMatchinfo[ofst]);
    }
    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprGlobalMatchinfoCb,(void*)&sInfo);
    pCsr->aMatchinfo = sInfo.aMatchinfo;
    pCsr->isMatchinfoNeeded = 1;

  }

  sInfo.aMatchinfo = pCsr->aMatchinfo;

  if( rc==SQLITE_OK && pCsr->isMatchinfoNeeded ){
    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLocalMatchinfoCb, (void*)&sInfo);
    if( pTab->bHasDocsize ){
      int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1];
      rc = sqlite3Fts3MatchinfoDocsizeLocal(pCsr, &sInfo.aMatchinfo[ofst]);
    }
    pCsr->isMatchinfoNeeded = 0;
  }

  return SQLITE_OK;
}

/*
** Implementation of snippet() function.
*/
void sqlite3Fts3Snippet(
  sqlite3_context *pCtx,          /* SQLite function call context */







|
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>

|









|

















|
|
<
|
>











|







|
|


|








|













>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>




|
>
>
>



>

>



>
>
>
>
>
>
>
|
>
>
|
|
|
|
|
>
>
|
|


|
|
|
|
|
<
|


>
>
|
<
|
|
<
|
<
|
|
<
<
<
<
<
|

>



>

<
<
<
|
<



|







797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852

853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100

1101
1102
1103
1104
1105
1106

1107
1108

1109

1110
1111





1112
1113
1114
1115
1116
1117
1118
1119



1120

1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
  }
  pCsr++;
  *pp = pCsr;
}

/*
** fts3ExprIterate() callback used to collect the "global" matchinfo stats
** for a single query. 
**
** fts3ExprIterate() callback to load the 'global' elements of a
** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements 
** of the matchinfo array that are constant for all rows returned by the 
** current query.
**
** Argument pCtx is actually a pointer to a struct of type MatchInfo. This
** function populates Matchinfo.aMatchinfo[] as follows:
**
**   for(iCol=0; iCol<nCol; iCol++){
**     aMatchinfo[3*iPhrase*nCol + 3*iCol + 1] = X;
**     aMatchinfo[3*iPhrase*nCol + 3*iCol + 2] = Y;
**   }
**
** where X is the number of matches for phrase iPhrase is column iCol of all
** rows of the table. Y is the number of rows for which column iCol contains
** at least one instance of phrase iPhrase.
*/
static int fts3ExprGlobalHitsCb(
  Fts3Expr *pExpr,                /* Phrase expression node */
  int iPhrase,                    /* Phrase number (numbered from zero) */
  void *pCtx                      /* Pointer to MatchInfo structure */
){
  MatchInfo *p = (MatchInfo *)pCtx;
  Fts3Cursor *pCsr = p->pCursor;
  char *pIter;
  char *pEnd;
  char *pFree = 0;
  u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol];

  assert( pExpr->isLoaded );
  assert( pExpr->eType==FTSQUERY_PHRASE );

  if( pCsr->pDeferred ){
    Fts3Phrase *pPhrase = pExpr->pPhrase;
    int ii;
    for(ii=0; ii<pPhrase->nToken; ii++){
      if( pPhrase->aToken[ii].bFulltext ) break;
    }
    if( ii<pPhrase->nToken ){
      int nFree = 0;
      int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree);
      if( rc!=SQLITE_OK ) return rc;
      pIter = pFree;
      pEnd = &pFree[nFree];
    }else{
      int iCol;                   /* Column index */
      for(iCol=0; iCol<p->nCol; iCol++){

        aOut[iCol*3 + 1] = (u32)p->nDoc;
        aOut[iCol*3 + 2] = (u32)p->nDoc;
      }
      return SQLITE_OK;
    }
  }else{
    pIter = pExpr->aDoclist;
    pEnd = &pExpr->aDoclist[pExpr->nDoclist];
  }

  /* Fill in the global hit count matrix row for this phrase. */
  while( pIter<pEnd ){
    while( *pIter++ & 0x80 );      /* Skip past docid. */
    fts3LoadColumnlistCounts(&pIter, &aOut[1], 1);
  }

  sqlite3_free(pFree);
  return SQLITE_OK;
}

/*
** fts3ExprIterate() callback used to collect the "local" part of the
** FTS3_MATCHINFO_HITS array. The local stats are those elements of the 
** array that are different for each row returned by the query.
*/
static int fts3ExprLocalHitsCb(
  Fts3Expr *pExpr,                /* Phrase expression node */
  int iPhrase,                    /* Phrase number */
  void *pCtx                      /* Pointer to MatchInfo structure */
){
  MatchInfo *p = (MatchInfo *)pCtx;

  if( pExpr->aDoclist ){
    char *pCsr;
    int iStart = iPhrase * p->nCol * 3;
    int i;

    for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0;

    pCsr = sqlite3Fts3FindPositions(pExpr, p->pCursor->iPrevId, -1);
    if( pCsr ){
      fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0);
    }
  }

  return SQLITE_OK;
}

static int fts3MatchinfoCheck(
  Fts3Table *pTab, 
  char cArg,
  char **pzErr
){
  if( cArg==FTS3_MATCHINFO_NPHRASE
   || cArg==FTS3_MATCHINFO_NCOL
   || cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat
   || cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat
   || cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize
   || cArg==FTS3_MATCHINFO_LCS
   || cArg==FTS3_MATCHINFO_HITS
  ){
    return SQLITE_OK;
  }
  *pzErr = sqlite3_mprintf("unrecognized matchinfo request: %c", cArg);
  return SQLITE_ERROR;
}

static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){
  int nVal;                       /* Number of integers output by cArg */

  switch( cArg ){
    case FTS3_MATCHINFO_NDOC:
    case FTS3_MATCHINFO_NPHRASE: 
    case FTS3_MATCHINFO_NCOL: 
      nVal = 1;
      break;

    case FTS3_MATCHINFO_AVGLENGTH:
    case FTS3_MATCHINFO_LENGTH:
    case FTS3_MATCHINFO_LCS:
      nVal = pInfo->nCol;
      break;

    case FTS3_MATCHINFO_HITS:
      nVal = pInfo->nCol * pInfo->nPhrase * 3;
      break;
  }

  return nVal;
}

static int fts3MatchinfoSelectDoctotal(
  Fts3Table *pTab,
  sqlite3_stmt **ppStmt,
  sqlite3_int64 *pnDoc,
  const char **paLen
){
  sqlite3_stmt *pStmt;
  const char *a;
  sqlite3_int64 nDoc;

  if( !*ppStmt ){
    int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt);
    if( rc!=SQLITE_OK ) return rc;
  }
  pStmt = *ppStmt;

  a = sqlite3_column_blob(pStmt, 0);
  a += sqlite3Fts3GetVarint(a, &nDoc);
  *pnDoc = (u32)nDoc;

  if( paLen ) *paLen = a;
  return SQLITE_OK;
}
  

static int fts3MatchinfoValues(
  Fts3Cursor *pCsr,               /* FTS3 cursor object */
  int bGlobal,                    /* True to grab the global stats */
  MatchInfo *pInfo,               /* Matchinfo context object */
  const char *zArg                /* Matchinfo format string */
){
  int rc = SQLITE_OK;
  int i;
  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;

  sqlite3_stmt *pSelect = 0;

  for(i=0; zArg[i]; i++){

    switch( zArg[i] ){
      case FTS3_MATCHINFO_NPHRASE: 
        if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase;
        break;

      case FTS3_MATCHINFO_NCOL: 
        if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol;
        break;
        
      case FTS3_MATCHINFO_NDOC:
        if( bGlobal ){
          sqlite3_int64 nDoc;
          rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0);
          pInfo->aMatchinfo[0] = (u32)nDoc;
        }
        break;

      case FTS3_MATCHINFO_AVGLENGTH: 
        if( bGlobal ){
          sqlite3_int64 nDoc;     /* Number of rows in table */
          const char *a;          /* Aggregate column length array */

          rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a);
          if( rc==SQLITE_OK ){
            int iCol;
            for(iCol=0; iCol<pInfo->nCol; iCol++){
              sqlite3_int64 nToken;
              a += sqlite3Fts3GetVarint(a, &nToken);
              pInfo->aMatchinfo[iCol] = ((u32)(nToken&0xffffffff)+nDoc/2)/nDoc;
            }
          }
        }
        break;

      case FTS3_MATCHINFO_LENGTH: {
        sqlite3_stmt *pSelectDocsize = 0;
        rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize);
        if( rc==SQLITE_OK ){
          int iCol;
          const char *a = sqlite3_column_blob(pSelectDocsize, 0);
          for(iCol=0; iCol<pInfo->nCol; iCol++){
            sqlite3_int64 nToken;
            a += sqlite3Fts3GetVarint(a, &nToken);
            pInfo->aMatchinfo[iCol] = (u32)nToken;
          }
        }
        sqlite3_reset(pSelectDocsize);
        break;
      }

      case FTS3_MATCHINFO_HITS: {
        Fts3Expr *pExpr = pCsr->pExpr;
        if( bGlobal ){
          if( pCsr->pDeferred ){
            rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0);
          }
          (void)fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
        }
        (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
        break;
      }


      default: 
        assert( zArg[i]==FTS3_MATCHINFO_LCS );
    }

    pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]);
  }

  sqlite3_reset(pSelect);
  return rc;
}


/*
** Populate pCsr->aMatchinfo[] with data for the current row. The 
** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32).
*/
static int fts3GetMatchinfo(
  Fts3Cursor *pCsr,               /* FTS3 Cursor object */
  const char *zArg                /* Second argument to matchinfo() function */
){
  MatchInfo sInfo;
  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  int rc = SQLITE_OK;
  int bGlobal = 0;                /* Collect 'global' stats as well as local */

  memset(&sInfo, 0, sizeof(MatchInfo));
  sInfo.pCursor = pCsr;
  sInfo.nCol = pTab->nColumn;

  /* If there is cached matchinfo() data, but the format string for the 
  ** cache does not match the format string for this request, discard 
  ** the cached data. */
  if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){
    assert( pCsr->aMatchinfo );
    sqlite3_free(pCsr->aMatchinfo);
    pCsr->zMatchinfo = 0;
    pCsr->aMatchinfo = 0;
  }

  /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the
  ** matchinfo function has been called for this query. In this case 
  ** allocate the array used to accumulate the matchinfo data and
  ** initialize those elements that are constant for every row.
  */
  if( pCsr->aMatchinfo==0 ){
    int nMatchinfo = 0;           /* Number of u32 elements in match-info */
    int nArg;                     /* Bytes in zArg */
    int i;                        /* Used to iterate through zArg */

    /* Load doclists for each phrase in the query. */
    rc = fts3ExprLoadDoclists(pCsr, &pCsr->nPhrase, 0);
    if( rc!=SQLITE_OK ) return rc;
    sInfo.nPhrase = pCsr->nPhrase;

    for(i=0; zArg[i]; i++){

      nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]);
    }

    /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */
    nArg = strlen(zArg);
    pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1);

    if( !pCsr->aMatchinfo ) return SQLITE_NOMEM;


    pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo];

    pCsr->nMatchinfo = nMatchinfo;
    memcpy(pCsr->zMatchinfo, zArg, nArg+1);





    memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo);
    pCsr->isMatchinfoNeeded = 1;
    bGlobal = 1;
  }

  sInfo.aMatchinfo = pCsr->aMatchinfo;
  sInfo.nPhrase = pCsr->nPhrase;
  if( rc==SQLITE_OK && pCsr->isMatchinfoNeeded ){



    rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg);

    pCsr->isMatchinfoNeeded = 0;
  }

  return rc;
}

/*
** Implementation of snippet() function.
*/
void sqlite3Fts3Snippet(
  sqlite3_context *pCtx,          /* SQLite function call context */
1207
1208
1209
1210
1211
1212
1213
1214





1215

















1216
1217
1218
1219


1220
1221

1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
  }
  return;
}

/*
** Implementation of matchinfo() function.
*/
void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor *pCsr){





  int rc;

















  if( !pCsr->pExpr ){
    sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
    return;
  }


  rc = fts3GetMatchinfo(pCsr);
  sqlite3Fts3SegmentsClose((Fts3Table *)pCsr->base.pVtab );

  if( rc!=SQLITE_OK ){
    sqlite3_result_error_code(pContext, rc);
  }else{
    Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab;
    int n = sizeof(u32)*(2+pCsr->aMatchinfo[0]*pCsr->aMatchinfo[1]*3);
    if( pTab->bHasDocsize ){
      n += sizeof(u32)*(1 + 2*pTab->nColumn);
    }
    sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT);
  }
}

#endif







|
>
>
>
>
>

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>




>
>
|
|
>



<
<
<
|
<





1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447



1448

1449
1450
1451
1452
1453
  }
  return;
}

/*
** Implementation of matchinfo() function.
*/
void sqlite3Fts3Matchinfo(
  sqlite3_context *pContext,      /* Function call context */
  Fts3Cursor *pCsr,               /* FTS3 table cursor */
  const char *zArg                /* Second arg to matchinfo() function */
){
  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  int rc;
  int i;
  const char *zFormat;

  if( zArg ){
    for(i=0; zArg[i]; i++){
      char *zErr = 0;
      if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){
        sqlite3_result_error(pContext, zErr, -1);
        sqlite3_free(zErr);
        return;
      }
    }
    zFormat = zArg;
  }else{
    zFormat = FTS3_MATCHINFO_DEFAULT;
  }

  if( !pCsr->pExpr ){
    sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
    return;
  }

  /* Retrieve matchinfo() data. */
  rc = fts3GetMatchinfo(pCsr, zFormat);
  sqlite3Fts3SegmentsClose(pTab);

  if( rc!=SQLITE_OK ){
    sqlite3_result_error_code(pContext, rc);
  }else{



    int n = pCsr->nMatchinfo * sizeof(u32);

    sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT);
  }
}

#endif
Changes to ext/fts3/fts3_write.c.
278
279
280
281
282
283
284













































285
286
287
288
289
290
291
    for(i=0; rc==SQLITE_OK && i<nParam; i++){
      rc = sqlite3_bind_value(pStmt, i+1, apVal[i]);
    }
  }
  *pp = pStmt;
  return rc;
}














































/*
** Similar to fts3SqlStmt(). Except, after binding the parameters in
** array apVal[] to the SQL statement identified by eStmt, the statement
** is executed.
**
** Returns SQLITE_OK if the statement is successfully executed, or an







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
    for(i=0; rc==SQLITE_OK && i<nParam; i++){
      rc = sqlite3_bind_value(pStmt, i+1, apVal[i]);
    }
  }
  *pp = pStmt;
  return rc;
}

static int fts3SelectDocsize(
  Fts3Table *pTab,                /* FTS3 table handle */
  int eStmt,                      /* Either SQL_SELECT_DOCSIZE or DOCTOTAL */
  sqlite3_int64 iDocid,           /* Docid to bind for SQL_SELECT_DOCSIZE */
  sqlite3_stmt **ppStmt           /* OUT: Statement handle */
){
  sqlite3_stmt *pStmt = 0;        /* Statement requested from fts3SqlStmt() */
  int rc;                         /* Return code */

  assert( eStmt==SQL_SELECT_DOCSIZE || eStmt==SQL_SELECT_DOCTOTAL );

  rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0);
  if( rc==SQLITE_OK ){
    if( eStmt==SQL_SELECT_DOCSIZE ){
      sqlite3_bind_int64(pStmt, 1, iDocid);
    }
    rc = sqlite3_step(pStmt);
    if( rc!=SQLITE_ROW ){
      rc = sqlite3_reset(pStmt);
      if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT;
      pStmt = 0;
    }else{
      rc = SQLITE_OK;
    }
  }

  *ppStmt = pStmt;
  return rc;
}

int sqlite3Fts3SelectDoctotal(
  Fts3Table *pTab,                /* Fts3 table handle */
  sqlite3_stmt **ppStmt           /* OUT: Statement handle */
){
  return fts3SelectDocsize(pTab, SQL_SELECT_DOCTOTAL, 0, ppStmt);
}

int sqlite3Fts3SelectDocsize(
  Fts3Table *pTab,                /* Fts3 table handle */
  sqlite3_int64 iDocid,           /* Docid to read size data for */
  sqlite3_stmt **ppStmt           /* OUT: Statement handle */
){
  return fts3SelectDocsize(pTab, SQL_SELECT_DOCSIZE, iDocid, ppStmt);
}

/*
** Similar to fts3SqlStmt(). Except, after binding the parameters in
** array apVal[] to the SQL statement identified by eStmt, the statement
** is executed.
**
** Returns SQLITE_OK if the statement is successfully executed, or an
Changes to test/fts3defer2.test.
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
} {2}

do_execsql_test 1.2.1 {
  SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
} {{a b c d e f a x y}}

do_execsql_test 1.2.2 {
  SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1))
  FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
} [list                              \
   {a b c d [e] [f] [a] x y}         \
   {0 1 8 1 0 0 10 1 0 2 12 1}       \
   [list 3 1   1 1 1   1 8 8   1 8 8   8 5001 9]
]

do_execsql_test 1.2.3 {
  SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1))
  FROM t1 WHERE t1 MATCH 'f (e NEAR/3 a)';
} [list                                 \
   {[a] b c d [e] [f] [a] x y}          \
   {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1}  \
   [list 3 1   1 1 1   1 8 8   2 8 8   8 5001 9]
]








|








|







49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
} {2}

do_execsql_test 1.2.1 {
  SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
} {{a b c d e f a x y}}

do_execsql_test 1.2.2 {
  SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
  FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
} [list                              \
   {a b c d [e] [f] [a] x y}         \
   {0 1 8 1 0 0 10 1 0 2 12 1}       \
   [list 3 1   1 1 1   1 8 8   1 8 8   8 5001 9]
]

do_execsql_test 1.2.3 {
  SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
  FROM t1 WHERE t1 MATCH 'f (e NEAR/3 a)';
} [list                                 \
   {[a] b c d [e] [f] [a] x y}          \
   {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1}  \
   [list 3 1   1 1 1   1 8 8   2 8 8   8 5001 9]
]

88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
  2 { INSERT INTO t2(t2) VALUES('optimize') }
  3 { UPDATE t2_segments SET block = zeroblob(length(block)) 
      WHERE length(block)>10000;
  }
} {
  execsql $sql
  do_execsql_test 2.2.$tn {
    SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'a b';
  } [list                                          \
    [list 2 1  1 54 54  1 3 3  54 372 7]        \
    [list 2 1  1 54 54  1 3 3  54 372 7]        \
  ]
}

do_execsql_test 2.3.1 {







|







88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
  2 { INSERT INTO t2(t2) VALUES('optimize') }
  3 { UPDATE t2_segments SET block = zeroblob(length(block)) 
      WHERE length(block)>10000;
  }
} {
  execsql $sql
  do_execsql_test 2.2.$tn {
    SELECT mit(matchinfo(t2, 'pcxnal')) FROM t2 WHERE t2 MATCH 'a b';
  } [list                                          \
    [list 2 1  1 54 54  1 3 3  54 372 7]        \
    [list 2 1  1 54 54  1 3 3  54 372 7]        \
  ]
}

do_execsql_test 2.3.1 {
120
121
122
123
124
125
126
127
128
129
130
131
132
133
  2 { INSERT INTO t3(t3) VALUES('optimize') }
  3 { UPDATE t3_segments SET block = zeroblob(length(block)) 
      WHERE length(block)>10000;
  }
} {
  execsql $sql
  do_execsql_test 2.4.$tn {
    SELECT docid, mit(matchinfo(t3)) FROM t3 WHERE t3 MATCH '"a b c"';
  } {1 {1 1 1 4 4 11 912 6} 3 {1 1 1 4 4 11 912 6}}
}


finish_test








|






120
121
122
123
124
125
126
127
128
129
130
131
132
133
  2 { INSERT INTO t3(t3) VALUES('optimize') }
  3 { UPDATE t3_segments SET block = zeroblob(length(block)) 
      WHERE length(block)>10000;
  }
} {
  execsql $sql
  do_execsql_test 2.4.$tn {
    SELECT docid, mit(matchinfo(t3, 'pcxnal')) FROM t3 WHERE t3 MATCH '"a b c"';
  } {1 {1 1 1 4 4 11 912 6} 3 {1 1 1 4 4 11 912 6}}
}


finish_test

Changes to test/fts3matchinfo.test.
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
  INSERT INTO t1(content) VALUES('I wandered lonely as a cloud');
  INSERT INTO t1(content) VALUES('That floats on high o''er vales and hills,');
  INSERT INTO t1(content) VALUES('When all at once I saw a crowd,');
  INSERT INTO t1(content) VALUES('A host, of golden daffodils,');
  SELECT mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'I';
} {{1 1 1 2 2} {1 1 1 2 2}}

# Now create an FTS4 table that does not specify matchinfo=fts3. The 
# %_docsize table is created in this case and the array of integers returned
# by matchinfo() includes the extra data.
#
do_execsql_test 1.2 {
  CREATE VIRTUAL TABLE t2 USING fts4;
  INSERT INTO t2 SELECT * FROM t1;
  SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'I';
} {{1 1 1 2 2 4 7 6} {1 1 1 2 2 4 7 8}}

# Test some syntax-error handling.
#
do_catchsql_test 2.0 {
  CREATE VIRTUAL TABLE x1 USING fts4(matchinfo=fs3);
} {1 {unrecognized matchinfo: fs3}}
do_catchsql_test 2.1 {







|
<
<





|







37
38
39
40
41
42
43
44


45
46
47
48
49
50
51
52
53
54
55
56
57
  INSERT INTO t1(content) VALUES('I wandered lonely as a cloud');
  INSERT INTO t1(content) VALUES('That floats on high o''er vales and hills,');
  INSERT INTO t1(content) VALUES('When all at once I saw a crowd,');
  INSERT INTO t1(content) VALUES('A host, of golden daffodils,');
  SELECT mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'I';
} {{1 1 1 2 2} {1 1 1 2 2}}

# Now create an FTS4 table that does not specify matchinfo=fts3.


#
do_execsql_test 1.2 {
  CREATE VIRTUAL TABLE t2 USING fts4;
  INSERT INTO t2 SELECT * FROM t1;
  SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'I';
} {{1 1 1 2 2} {1 1 1 2 2}}

# Test some syntax-error handling.
#
do_catchsql_test 2.0 {
  CREATE VIRTUAL TABLE x1 USING fts4(matchinfo=fs3);
} {1 {unrecognized matchinfo: fs3}}
do_catchsql_test 2.1 {
Changes to test/fts3query.test.
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176




177

178
179
180
181
182
183
184
  CREATE VIRTUAL TABLE t2 USING FTS4;
  INSERT INTO t2 VALUES('it was the first time in history');
}
do_select_tests 5.2 -errorformat {
  wrong number of arguments to function %s()
} {
  1 "SELECT matchinfo() FROM t2 WHERE t2 MATCH 'history'"       matchinfo
  2 "SELECT matchinfo(t2, t2) FROM t2 WHERE t2 MATCH 'history'" matchinfo

  3 "SELECT snippet(t2, 1, 2, 3, 4, 5, 6) FROM t2 WHERE t2 MATCH 'history'" 
    snippet
}
do_select_tests 5.3 -errorformat {
  illegal first argument to %s
} {
  1 "SELECT matchinfo(content) FROM t2 WHERE t2 MATCH 'history'" matchinfo
  2 "SELECT offsets(content) FROM t2 WHERE t2 MATCH 'history'"   offsets
  3 "SELECT snippet(content) FROM t2 WHERE t2 MATCH 'history'"   snippet
  4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'"  optimize
}
do_execsql_test 5.4.0 { UPDATE t2_content SET c0content = X'1234' }
do_select_tests 5.4 -errorformat {
  illegal first argument to %s
} {
  1 "SELECT matchinfo(content) FROM t2 WHERE t2 MATCH 'history'" matchinfo
  2 "SELECT offsets(content) FROM t2 WHERE t2 MATCH 'history'"   offsets
  3 "SELECT snippet(content) FROM t2 WHERE t2 MATCH 'history'"   snippet
  4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'"  optimize
}




do_execsql_test 5.5 { DROP TABLE t2 }


# Test the snippet() function with 1 to 6 arguments.
# 
do_execsql_test 6.1 {
  CREATE VIRTUAL TABLE t3 USING FTS4(a, b);
  INSERT INTO t3 VALUES('no gestures', 'another intriguing discovery by observing the hand gestures (called beats) people make while speaking. Research has shown that such gestures do more than add visual emphasis to our words (many people gesture while they''re on the telephone, for example); it seems they actually help our brains find words');
}







<
<




















>
>
>
>

>







148
149
150
151
152
153
154


155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
  CREATE VIRTUAL TABLE t2 USING FTS4;
  INSERT INTO t2 VALUES('it was the first time in history');
}
do_select_tests 5.2 -errorformat {
  wrong number of arguments to function %s()
} {
  1 "SELECT matchinfo() FROM t2 WHERE t2 MATCH 'history'"       matchinfo


  3 "SELECT snippet(t2, 1, 2, 3, 4, 5, 6) FROM t2 WHERE t2 MATCH 'history'" 
    snippet
}
do_select_tests 5.3 -errorformat {
  illegal first argument to %s
} {
  1 "SELECT matchinfo(content) FROM t2 WHERE t2 MATCH 'history'" matchinfo
  2 "SELECT offsets(content) FROM t2 WHERE t2 MATCH 'history'"   offsets
  3 "SELECT snippet(content) FROM t2 WHERE t2 MATCH 'history'"   snippet
  4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'"  optimize
}
do_execsql_test 5.4.0 { UPDATE t2_content SET c0content = X'1234' }
do_select_tests 5.4 -errorformat {
  illegal first argument to %s
} {
  1 "SELECT matchinfo(content) FROM t2 WHERE t2 MATCH 'history'" matchinfo
  2 "SELECT offsets(content) FROM t2 WHERE t2 MATCH 'history'"   offsets
  3 "SELECT snippet(content) FROM t2 WHERE t2 MATCH 'history'"   snippet
  4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'"  optimize
}
do_catchsql_test 5.5.1 {
  SELECT matchinfo(t2, 'abc') FROM t2 WHERE t2 MATCH 'history'
} {1 {unrecognized matchinfo request: b}}

do_execsql_test 5.5 { DROP TABLE t2 }


# Test the snippet() function with 1 to 6 arguments.
# 
do_execsql_test 6.1 {
  CREATE VIRTUAL TABLE t3 USING FTS4(a, b);
  INSERT INTO t3 VALUES('no gestures', 'another intriguing discovery by observing the hand gestures (called beats) people make while speaking. Research has shown that such gestures do more than add visual emphasis to our words (many people gesture while they''re on the telephone, for example); it seems they actually help our brains find words');
}