SQLite

Check-in [f6a0193f5a]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Allow the "order=DESC" and "order=ASC" parameters in FTS4 "CREATE VIRTUAL TABLE" statements. Tables created with "order=DESC" store all doclists in descending order, which allows optimizations normally applied to "ORDER BY docid ASC" queries to be used with "ORDER BY docid DESC" queries instead.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts3-prefix-search
Files: files | file ages | folders
SHA1: f6a0193f5a32603eb48bddc6297042dbd2ffe96e
User & Date: dan 2011-06-04 20:04:35.492
Context
2011-06-04
20:13
Remove some unreachable code. (check-in: 650e1a79ed user: dan tags: fts3-prefix-search)
20:04
Allow the "order=DESC" and "order=ASC" parameters in FTS4 "CREATE VIRTUAL TABLE" statements. Tables created with "order=DESC" store all doclists in descending order, which allows optimizations normally applied to "ORDER BY docid ASC" queries to be used with "ORDER BY docid DESC" queries instead. (check-in: f6a0193f5a user: dan tags: fts3-prefix-search)
2011-06-03
18:00
FTS changes: Remove unreachable code. Fix bugs. When processing a large doclist incrementally, read from disk incrementally too. (check-in: a4c7e28208 user: dan tags: fts3-prefix-search)
Changes
Unified Diff Show Whitespace Changes Patch
Changes to ext/fts3/fts3.c.
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
  *pVal += iVal;
}

/*
** When this function is called, *pp points to the first byte following a
** varint that is part of a doclist (or position-list, or any other list
** of varints). This function moves *pp to point to the start of that varint,
** and decrements the value stored in *pVal by the varint value.
**
** Argument pStart points to the first byte of the doclist that the
** varint is part of.
*/
static void fts3GetReverseDeltaVarint(
  char **pp, 
  char *pStart, 
  sqlite3_int64 *pVal
){
  sqlite3_int64 iVal;
  char *p = *pp;

  /* Pointer p now points at the first byte past the varint we are 
  ** interested in. So, unless the doclist is corrupt, the 0x80 bit is
  ** clear on character p[-1]. */
  for(p = (*pp)-2; p>=pStart && *p&0x80; p--);
  p++;
  *pp = p;

  sqlite3Fts3GetVarint(p, &iVal);
  *pVal -= iVal;
}

/*
** As long as *pp has not reached its end (pEnd), then do the same
** as fts3GetDeltaVarint(): read a single varint and add it to *pVal.
** But if we have reached the end of the varint, just set *pp=0 and
** leave *pVal unchanged.







|




|















|







419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
  *pVal += iVal;
}

/*
** When this function is called, *pp points to the first byte following a
** varint that is part of a doclist (or position-list, or any other list
** of varints). This function moves *pp to point to the start of that varint,
** and sets *pVal by the varint value.
**
** Argument pStart points to the first byte of the doclist that the
** varint is part of.
*/
static void fts3GetReverseVarint(
  char **pp, 
  char *pStart, 
  sqlite3_int64 *pVal
){
  sqlite3_int64 iVal;
  char *p = *pp;

  /* Pointer p now points at the first byte past the varint we are 
  ** interested in. So, unless the doclist is corrupt, the 0x80 bit is
  ** clear on character p[-1]. */
  for(p = (*pp)-2; p>=pStart && *p&0x80; p--);
  p++;
  *pp = p;

  sqlite3Fts3GetVarint(p, &iVal);
  *pVal = iVal;
}

/*
** As long as *pp has not reached its end (pEnd), then do the same
** as fts3GetDeltaVarint(): read a single varint and add it to *pVal.
** But if we have reached the end of the varint, just set *pp=0 and
** leave *pVal unchanged.
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927





928
929

















930
931
932
933
934
935
936
  int iCol;                       /* Column index */
  int nString = 0;                /* Bytes required to hold all column names */
  int nCol = 0;                   /* Number of columns in the FTS table */
  char *zCsr;                     /* Space for holding column names */
  int nDb;                        /* Bytes required to hold database name */
  int nName;                      /* Bytes required to hold table name */
  int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
  int bNoDocsize = 0;             /* True to omit %_docsize table */
  const char **aCol;              /* Array of column names */
  sqlite3_tokenizer *pTokenizer = 0;        /* Tokenizer for this table */

  char *zPrefix = 0;              /* Prefix parameter value (or NULL) */
  int nIndex;                     /* Size of aIndex[] array */
  struct Fts3Index *aIndex;       /* Array of indexes for this table */
  struct Fts3Index *aFree = 0;    /* Free this before returning */






  char *zCompress = 0;
  char *zUncompress = 0;


















  assert( strlen(argv[0])==4 );
  assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
       || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
  );

  nDb = (int)strlen(argv[1]) + 1;







<



<




>
>
>
>
>
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







912
913
914
915
916
917
918

919
920
921

922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
  int iCol;                       /* Column index */
  int nString = 0;                /* Bytes required to hold all column names */
  int nCol = 0;                   /* Number of columns in the FTS table */
  char *zCsr;                     /* Space for holding column names */
  int nDb;                        /* Bytes required to hold database name */
  int nName;                      /* Bytes required to hold table name */
  int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */

  const char **aCol;              /* Array of column names */
  sqlite3_tokenizer *pTokenizer = 0;        /* Tokenizer for this table */


  int nIndex;                     /* Size of aIndex[] array */
  struct Fts3Index *aIndex;       /* Array of indexes for this table */
  struct Fts3Index *aFree = 0;    /* Free this before returning */

  int bNoDocsize = 0;             /* True to omit %_docsize table */
  int bDescIdx = 0;               /* True to store descending indexes */

  char *zMatchinfo = 0;           /* Prefix parameter value (or NULL) */
  char *zPrefix = 0;              /* Prefix parameter value (or NULL) */
  char *zCompress = 0;            /* compress=? parameter (or NULL) */
  char *zUncompress = 0;          /* uncompress=? parameter (or NULL) */
  char *zOrder = 0;               /* order=? parameter (or NULL) */
  struct Fts4Option {
    const char *zOpt;
    int nOpt;
    char **pzVar;
  } aFts4Opt[] = {
    { "matchinfo",   9, 0 },
    { "prefix",      6, 0 },
    { "compress",    8, 0 },
    { "uncompress", 10, 0 },
    { "order",       5, 0 }
  };
  aFts4Opt[0].pzVar = &zMatchinfo;
  aFts4Opt[1].pzVar = &zPrefix;
  aFts4Opt[2].pzVar = &zCompress;
  aFts4Opt[3].pzVar = &zUncompress;
  aFts4Opt[4].pzVar = &zOrder;

  assert( strlen(argv[0])==4 );
  assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
       || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
  );

  nDb = (int)strlen(argv[1]) + 1;
963
964
965
966
967
968
969

970
971
972
973
974
975
976
977
978
979
980

981
982
983
984
985

986
987
988
989
990
991



992
993
994
995
996


997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011




















1012
1013
1014
1015
1016
1017
1018
     && 0==sqlite3Fts3IsIdChar(z[8])
    ){
      rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr);
    }

    /* Check if it is an FTS4 special argument. */
    else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){

      if( !zVal ){
        rc = SQLITE_NOMEM;
        goto fts3_init_out;
      }
      if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){
        if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){
          bNoDocsize = 1;
        }else{
          *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal);
          rc = SQLITE_ERROR;
        }

      }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){
        zCompress = zVal;
        zVal = 0;
      }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){
        zUncompress = zVal;

        zVal = 0;
      }else if( nKey==6 && 0==sqlite3_strnicmp(z, "prefix", 6) ){
        sqlite3_free(zPrefix);
        zPrefix = zVal;
        zVal = 0;
      }else{



        *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
        rc = SQLITE_ERROR;
      }
      sqlite3_free(zVal);
    }



    /* Otherwise, the argument is a column name. */
    else {
      nString += (int)(strlen(z) + 1);
      aCol[nCol++] = z;
    }
  }
  if( rc!=SQLITE_OK ) goto fts3_init_out;

  if( nCol==0 ){
    assert( nString==0 );
    aCol[0] = "content";
    nString = 8;
    nCol = 1;
  }





















  if( pTokenizer==0 ){
    rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
    if( rc!=SQLITE_OK ) goto fts3_init_out;
  }
  assert( pTokenizer );








>


<
<
<
<
<

<
<
<
>
|
<
<
|
<
>
|
<
|
|

|
>
>
>


<


>
>















>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







983
984
985
986
987
988
989
990
991
992





993



994
995


996

997
998

999
1000
1001
1002
1003
1004
1005
1006
1007

1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
     && 0==sqlite3Fts3IsIdChar(z[8])
    ){
      rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr);
    }

    /* Check if it is an FTS4 special argument. */
    else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){
      int iOpt;
      if( !zVal ){
        rc = SQLITE_NOMEM;





        }else{



        for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){
          if( nKey==aFts4Opt[iOpt].nOpt 


           && !sqlite3_strnicmp(z, aFts4Opt[iOpt].zOpt, aFts4Opt[iOpt].nOpt) 

          ){
            char **pzVar = aFts4Opt[iOpt].pzVar;

            sqlite3_free(*pzVar);
            *pzVar = zVal;
        zVal = 0;
            break;
          }
        }
        if( zVal ){
        *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
        rc = SQLITE_ERROR;

      sqlite3_free(zVal);
    }
      }
    }

    /* Otherwise, the argument is a column name. */
    else {
      nString += (int)(strlen(z) + 1);
      aCol[nCol++] = z;
    }
  }
  if( rc!=SQLITE_OK ) goto fts3_init_out;

  if( nCol==0 ){
    assert( nString==0 );
    aCol[0] = "content";
    nString = 8;
    nCol = 1;
  }

  if( zMatchinfo ){
    if( strlen(zMatchinfo)!=4 || sqlite3_strnicmp(zMatchinfo, "fts3", 4) ){
      *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zMatchinfo);
      rc = SQLITE_ERROR;
      goto fts3_init_out;
    }
    bNoDocsize = 1;
  }

  if( zOrder ){
    if( (strlen(zOrder)!=3 || sqlite3_strnicmp(zOrder, "asc", 3)) 
     && (strlen(zOrder)!=4 || sqlite3_strnicmp(zOrder, "desc", 3)) 
    ){
      *pzErr = sqlite3_mprintf("unrecognized order: %s", zOrder);
      rc = SQLITE_ERROR;
      goto fts3_init_out;
    }
    bDescIdx = (zOrder[0]=='d' || zOrder[0]=='D');
  }

  if( pTokenizer==0 ){
    rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
    if( rc!=SQLITE_OK ) goto fts3_init_out;
  }
  assert( pTokenizer );

1042
1043
1044
1045
1046
1047
1048

1049
1050
1051
1052
1053
1054
1055
  p->nPendingData = 0;
  p->azColumn = (char **)&p[1];
  p->pTokenizer = pTokenizer;
  p->nNodeSize = 1000;
  p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
  p->bHasDocsize = (isFts4 && bNoDocsize==0);
  p->bHasStat = isFts4;

  TESTONLY( p->inTransaction = -1 );
  TESTONLY( p->mxSavepoint = -1 );

  p->aIndex = (struct Fts3Index *)&p->azColumn[nCol];
  memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex);
  p->nIndex = nIndex;
  for(i=0; i<nIndex; i++){







>







1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
  p->nPendingData = 0;
  p->azColumn = (char **)&p[1];
  p->pTokenizer = pTokenizer;
  p->nNodeSize = 1000;
  p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
  p->bHasDocsize = (isFts4 && bNoDocsize==0);
  p->bHasStat = isFts4;
  p->bDescIdx = bDescIdx;
  TESTONLY( p->inTransaction = -1 );
  TESTONLY( p->mxSavepoint = -1 );

  p->aIndex = (struct Fts3Index *)&p->azColumn[nCol];
  memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex);
  p->nIndex = nIndex;
  for(i=0; i<nIndex; i++){
1104
1105
1106
1107
1108
1109
1110


1111
1112
1113
1114
1115
1116
1117
  fts3DeclareVtab(&rc, p);

fts3_init_out:
  sqlite3_free(zPrefix);
  sqlite3_free(aFree);
  sqlite3_free(zCompress);
  sqlite3_free(zUncompress);


  sqlite3_free((void *)aCol);
  if( rc!=SQLITE_OK ){
    if( p ){
      fts3DisconnectMethod((sqlite3_vtab *)p);
    }else if( pTokenizer ){
      pTokenizer->pModule->xDestroy(pTokenizer);
    }







>
>







1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
  fts3DeclareVtab(&rc, p);

fts3_init_out:
  sqlite3_free(zPrefix);
  sqlite3_free(aFree);
  sqlite3_free(zCompress);
  sqlite3_free(zUncompress);
  sqlite3_free(zOrder);
  sqlite3_free(zMatchinfo);
  sqlite3_free((void *)aCol);
  if( rc!=SQLITE_OK ){
    if( p ){
      fts3DisconnectMethod((sqlite3_vtab *)p);
    }else if( pTokenizer ){
      pTokenizer->pModule->xDestroy(pTokenizer);
    }
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886



1887
1888
1889
1890
1891
1892
1893

/*
** Values that may be used as the first parameter to fts3DoclistMerge().
*/
#define MERGE_NOT        2        /* D + D -> D */
#define MERGE_AND        3        /* D + D -> D */
#define MERGE_OR         4        /* D + D -> D */
#define MERGE_POS_OR     5        /* P + P -> P */
#define MERGE_PHRASE     6        /* P + P -> D */
#define MERGE_POS_PHRASE 7        /* P + P -> P */
#define MERGE_NEAR       8        /* P + P -> D */



#define MERGE_POS_NEAR   9        /* P + P -> P */

/*
** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2
** (size n2 bytes). The output is written to pre-allocated buffer aBuffer,
** which is guaranteed to be large enough to hold the results. The number
** of bytes written to aBuffer is stored in *pnBuffer before returning.







<

<

>
>
>







1914
1915
1916
1917
1918
1919
1920

1921

1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932

/*
** Values that may be used as the first parameter to fts3DoclistMerge().
*/
#define MERGE_NOT        2        /* D + D -> D */
#define MERGE_AND        3        /* D + D -> D */
#define MERGE_OR         4        /* D + D -> D */

#define MERGE_PHRASE     6        /* P + P -> D */

#define MERGE_NEAR       8        /* P + P -> D */

#define MERGE_POS_OR     5        /* P + P -> P */
#define MERGE_POS_PHRASE 7        /* P + P -> P */
#define MERGE_POS_NEAR   9        /* P + P -> P */

/*
** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2
** (size n2 bytes). The output is written to pre-allocated buffer aBuffer,
** which is guaranteed to be large enough to hold the results. The number
** of bytes written to aBuffer is stored in *pnBuffer before returning.
1920
1921
1922
1923
1924
1925
1926

1927
1928
1929
1930
1931
1932
1933
  int nDoc = 0;

  assert( mergetype==MERGE_OR     || mergetype==MERGE_POS_OR 
       || mergetype==MERGE_AND    || mergetype==MERGE_NOT
       || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE
       || mergetype==MERGE_NEAR   || mergetype==MERGE_POS_NEAR
  );


  if( !aBuffer ){
    *pnBuffer = 0;
    return SQLITE_NOMEM;
  }

  /* Read the first docid from each doclist */







>







1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
  int nDoc = 0;

  assert( mergetype==MERGE_OR     || mergetype==MERGE_POS_OR 
       || mergetype==MERGE_AND    || mergetype==MERGE_NOT
       || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE
       || mergetype==MERGE_NEAR   || mergetype==MERGE_POS_NEAR
  );
  assert( mergetype==MERGE_POS_PHRASE || mergetype==MERGE_POS_NEAR );

  if( !aBuffer ){
    *pnBuffer = 0;
    return SQLITE_NOMEM;
  }

  /* Read the first docid from each doclist */
2060
2061
2062
2063
2064
2065
2066





























































































































































































































2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095




2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
*/
typedef struct TermSelect TermSelect;
struct TermSelect {
  int isReqPos;
  char *aaOutput[16];             /* Malloc'd output buffer */
  int anOutput[16];               /* Size of output in bytes */
};






























































































































































































































/*
** Merge all doclists in the TermSelect.aaOutput[] array into a single
** doclist stored in TermSelect.aaOutput[0]. If successful, delete all
** other doclists (except the aaOutput[0] one) and return SQLITE_OK.
**
** If an OOM error occurs, return SQLITE_NOMEM. In this case it is
** the responsibility of the caller to free any doclists left in the
** TermSelect.aaOutput[] array.
*/
static int fts3TermSelectMerge(TermSelect *pTS){
  int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
  char *aOut = 0;
  int nOut = 0;
  int i;

  /* Loop through the doclists in the aaOutput[] array. Merge them all
  ** into a single doclist.
  */
  for(i=0; i<SizeofArray(pTS->aaOutput); i++){
    if( pTS->aaOutput[i] ){
      if( !aOut ){
        aOut = pTS->aaOutput[i];
        nOut = pTS->anOutput[i];
        pTS->aaOutput[i] = 0;
      }else{
        int nNew = nOut + pTS->anOutput[i];
        char *aNew = sqlite3_malloc(nNew);
        if( !aNew ){




          sqlite3_free(aOut);
          return SQLITE_NOMEM;
        }
        fts3DoclistMerge(mergetype, 0, 0,
            aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0
        );
        sqlite3_free(pTS->aaOutput[i]);
        sqlite3_free(aOut);
        pTS->aaOutput[i] = 0;
        aOut = aNew;
        nOut = nNew;
      }
    }







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>










|















|
|
|
>
>
>
>

|

|
<
<







2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364


2365
2366
2367
2368
2369
2370
2371
*/
typedef struct TermSelect TermSelect;
struct TermSelect {
  int isReqPos;
  char *aaOutput[16];             /* Malloc'd output buffer */
  int anOutput[16];               /* Size of output in bytes */
};


static void fts3GetDeltaVarint3(
  char **pp, 
  char *pEnd, 
  int bDescIdx,
  sqlite3_int64 *pVal
){
  if( *pp>=pEnd ){
    *pp = 0;
  }else{
    sqlite3_int64 iVal;
    *pp += sqlite3Fts3GetVarint(*pp, &iVal);
    if( bDescIdx ){
      *pVal -= iVal;
    }else{
      *pVal += iVal;
    }
  }
}

static void fts3PutDeltaVarint3(
  char **pp,                      /* IN/OUT: Output pointer */
  int bDescIdx,                   /* True for descending docids */
  sqlite3_int64 *piPrev,          /* IN/OUT: Previous value written to list */
  int *pbFirst,                   /* IN/OUT: True after first int written */
  sqlite3_int64 iVal              /* Write this value to the list */
){
  sqlite3_int64 iWrite;
  if( bDescIdx==0 || *pbFirst==0 ){
    iWrite = iVal - *piPrev;
  }else{
    iWrite = *piPrev - iVal;
  }
  assert( *pbFirst || *piPrev==0 );
  assert( *pbFirst==0 || iWrite>0 );
  *pp += sqlite3Fts3PutVarint(*pp, iWrite);
  *piPrev = iVal;
  *pbFirst = 1;
}

#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2))

static int fts3DoclistOrMerge(
  int bDescIdx,                   /* True if arguments are desc */
  u8 *a1, int n1,                 /* First doclist */
  u8 *a2, int n2,                 /* Second doclist */
  u8 **paOut, int *pnOut          /* OUT: Malloc'd doclist */
){
  sqlite3_int64 i1 = 0;
  sqlite3_int64 i2 = 0;
  sqlite3_int64 iPrev = 0;
  char *pEnd1 = &a1[n1];
  char *pEnd2 = &a2[n2];
  char *p1 = a1;
  char *p2 = a2;
  char *p;
  int nOut;
  char *aOut;
  int bFirstOut = 0;

  *paOut = 0;
  *pnOut = 0;
  aOut = sqlite3_malloc(n1+n2);
  if( !aOut ) return SQLITE_NOMEM;

  p = aOut;
  fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
  fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
  while( p1 || p2 ){
    sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);

    if( p2 && p1 && iDiff==0 ){
      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
      fts3PoslistMerge(&p, &p1, &p2);
      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
    }else if( !p2 || (p1 && iDiff<0) ){
      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
      fts3PoslistCopy(&p, &p1);
      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
    }else{
      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2);
      fts3PoslistCopy(&p, &p2);
      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
    }
  }

  *paOut = aOut;
  *pnOut = (p-aOut);
  return SQLITE_OK;
}

static void fts3DoclistPhraseMerge(
  int bDescIdx,                   /* True if arguments are desc */
  int nDist,                      /* Distance from left to right (1=adjacent) */
  u8 *aLeft, int nLeft,           /* Left doclist */
  u8 *aRight, int *pnRight        /* IN/OUT: Right/output doclist */
){
  sqlite3_int64 i1 = 0;
  sqlite3_int64 i2 = 0;
  sqlite3_int64 iPrev = 0;
  char *pEnd1 = &aLeft[nLeft];
  char *pEnd2 = &aRight[*pnRight];
  char *p1 = aLeft;
  char *p2 = aRight;
  char *p;
  int bFirstOut = 0;
  char *aOut = aRight;

  assert( nDist>0 );

  p = aOut;
  fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
  fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);

  while( p1 && p2 ){
    sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
    if( iDiff==0 ){
      char *pSave = p;
      sqlite3_int64 iPrevSave = iPrev;
      int bFirstOutSave = bFirstOut;

      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
      if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){
        p = pSave;
        iPrev = iPrevSave;
        bFirstOut = bFirstOutSave;
      }
      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
    }else if( iDiff<0 ){
      fts3PoslistCopy(0, &p1);
      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
    }else{
      fts3PoslistCopy(0, &p2);
      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
    }
  }

  *pnRight = p - aOut;
}

/*
** This function merges two doclists according to the requirements of a
** NEAR operator.
*/
static int fts3DoclistNearMerge(
  int bDescIdx,
  int mergetype,                  /* MERGE_POS_NEAR or MERGE_NEAR */
  int nNear,                      /* Parameter to NEAR operator */
  int nTokenLeft,                 /* Number of tokens in LHS phrase arg */
  char *aLeft,                    /* Doclist for LHS (incl. positions) */
  int nLeft,                      /* Size of LHS doclist in bytes */
  int nTokenRight,                /* As nTokenLeft */
  char *aRight,                   /* As aLeft */
  int nRight,                     /* As nRight */
  char **paOut,                   /* OUT: Results of merge (malloced) */
  int *pnOut                      /* OUT: Sized of output buffer */
){
  char *aOut;                     /* Buffer to write output doclist to */
  char *aTmp;                     /* Temp buffer used by PoslistNearMerge() */

  sqlite3_int64 i1 = 0;
  sqlite3_int64 i2 = 0;
  sqlite3_int64 iPrev = 0;
  int bFirstOut = 0;

  char *pEnd1 = &aLeft[nLeft];
  char *pEnd2 = &aRight[nRight];
  char *p1 = aLeft;
  char *p2 = aRight;
  char *p;

  int nParam1 = nNear+nTokenRight;
  int nParam2 = nNear+nTokenLeft;

  p = aOut = sqlite3_malloc(nLeft+nRight+1);
  aTmp = sqlite3_malloc(2*(nLeft+nRight+1));
  if( !aOut || !aTmp ){
    sqlite3_free(aOut);
    sqlite3_free(aTmp);
    *paOut = 0;
    *pnOut = 0;
    return SQLITE_NOMEM;
  }

  fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
  fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);

  while( p1 && p2 ){
    sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
    if( iDiff==0 ){
      char *pSave = p;
      sqlite3_int64 iPrevSave = iPrev;
      int bFirstOutSave = bFirstOut;
      fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
      if( !fts3PoslistNearMerge(&p, aTmp, nParam1, nParam2, &p1, &p2) ){
        p = pSave;
        iPrev = iPrevSave;
        bFirstOut = bFirstOutSave;
      }

      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
    }else if( iDiff<0 ){
      fts3PoslistCopy(0, &p1);
      fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
    }else{
      fts3PoslistCopy(0, &p2);
      fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
    }
  }

  sqlite3_free(aTmp);
  *paOut = aOut;
  *pnOut = p - aOut;
  return SQLITE_OK;
}



/*
** Merge all doclists in the TermSelect.aaOutput[] array into a single
** doclist stored in TermSelect.aaOutput[0]. If successful, delete all
** other doclists (except the aaOutput[0] one) and return SQLITE_OK.
**
** If an OOM error occurs, return SQLITE_NOMEM. In this case it is
** the responsibility of the caller to free any doclists left in the
** TermSelect.aaOutput[] array.
*/
static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){
  int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
  char *aOut = 0;
  int nOut = 0;
  int i;

  /* Loop through the doclists in the aaOutput[] array. Merge them all
  ** into a single doclist.
  */
  for(i=0; i<SizeofArray(pTS->aaOutput); i++){
    if( pTS->aaOutput[i] ){
      if( !aOut ){
        aOut = pTS->aaOutput[i];
        nOut = pTS->anOutput[i];
        pTS->aaOutput[i] = 0;
      }else{
        int nNew;
        u8 *aNew;

        int rc = fts3DoclistOrMerge(p->bDescIdx, 
            pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew
        );
        if( rc!=SQLITE_OK ){
          sqlite3_free(aOut);
          return rc;
        }



        sqlite3_free(pTS->aaOutput[i]);
        sqlite3_free(aOut);
        pTS->aaOutput[i] = 0;
        aOut = aNew;
        nOut = nNew;
      }
    }
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161


2162

2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186

2187
2188
2189
2190
2191
2192
2193

  UNUSED_PARAMETER(p);
  UNUSED_PARAMETER(zTerm);
  UNUSED_PARAMETER(nTerm);

  if( pTS->aaOutput[0]==0 ){
    /* If this is the first term selected, copy the doclist to the output
    ** buffer using memcpy(). TODO: Add a way to transfer control of the
    ** aDoclist buffer from the caller so as to avoid the memcpy().
    */
    pTS->aaOutput[0] = sqlite3_malloc(nDoclist);
    pTS->anOutput[0] = nDoclist;
    if( pTS->aaOutput[0] ){
      memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
    }else{
      return SQLITE_NOMEM;
    }
  }else{
    int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
    char *aMerge = aDoclist;
    int nMerge = nDoclist;
    int iOut;

    for(iOut=0; iOut<SizeofArray(pTS->aaOutput); iOut++){
      char *aNew;
      int nNew;
      if( pTS->aaOutput[iOut]==0 ){
        assert( iOut>0 );
        pTS->aaOutput[iOut] = aMerge;
        pTS->anOutput[iOut] = nMerge;
        break;
      }




      nNew = nMerge + pTS->anOutput[iOut];
      aNew = sqlite3_malloc(nNew);
      if( !aNew ){
        if( aMerge!=aDoclist ){
          sqlite3_free(aMerge);
        }
        return SQLITE_NOMEM;
      }
      fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, 
          pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0
      );

      if( iOut>0 ) sqlite3_free(aMerge);
      sqlite3_free(pTS->aaOutput[iOut]);
      pTS->aaOutput[iOut] = 0;

      aMerge = aNew;
      nMerge = nNew;
      if( (iOut+1)==SizeofArray(pTS->aaOutput) ){
        pTS->aaOutput[iOut] = aMerge;
        pTS->anOutput[iOut] = nMerge;
      }
    }
  }

  return SQLITE_OK;
}

/*
** Append SegReader object pNew to the end of the pCsr->apSegment[] array.
*/
static int fts3SegReaderCursorAppend(







|
<
<














<
<





|
>
>

>
|
|
|
|
<
<
|

<
<
<

|











>







2393
2394
2395
2396
2397
2398
2399
2400


2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414


2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428


2429
2430



2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451

  UNUSED_PARAMETER(p);
  UNUSED_PARAMETER(zTerm);
  UNUSED_PARAMETER(nTerm);

  if( pTS->aaOutput[0]==0 ){
    /* If this is the first term selected, copy the doclist to the output
    ** buffer using memcpy(). */


    pTS->aaOutput[0] = sqlite3_malloc(nDoclist);
    pTS->anOutput[0] = nDoclist;
    if( pTS->aaOutput[0] ){
      memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
    }else{
      return SQLITE_NOMEM;
    }
  }else{
    int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
    char *aMerge = aDoclist;
    int nMerge = nDoclist;
    int iOut;

    for(iOut=0; iOut<SizeofArray(pTS->aaOutput); iOut++){


      if( pTS->aaOutput[iOut]==0 ){
        assert( iOut>0 );
        pTS->aaOutput[iOut] = aMerge;
        pTS->anOutput[iOut] = nMerge;
        break;
      }else{
        u8 *aNew;
        int nNew;

        int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, 
            pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew
        );
        if( rc!=SQLITE_OK ){
          if( aMerge!=aDoclist ) sqlite3_free(aMerge);


          return rc;
      }




        if( aMerge!=aDoclist ) sqlite3_free(aMerge);
      sqlite3_free(pTS->aaOutput[iOut]);
      pTS->aaOutput[iOut] = 0;

      aMerge = aNew;
      nMerge = nNew;
      if( (iOut+1)==SizeofArray(pTS->aaOutput) ){
        pTS->aaOutput[iOut] = aMerge;
        pTS->anOutput[iOut] = nMerge;
      }
    }
  }
  }
  return SQLITE_OK;
}

/*
** Append SegReader object pNew to the end of the pCsr->apSegment[] array.
*/
static int fts3SegReaderCursorAppend(
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
  ){
    rc = fts3TermSelectCb(p, (void *)&tsc, 
        pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist
    );
  }

  if( rc==SQLITE_OK ){
    rc = fts3TermSelectMerge(&tsc);
  }
  if( rc==SQLITE_OK ){
    *ppOut = tsc.aaOutput[0];
    *pnOut = tsc.anOutput[0];
  }else{
    int i;
    for(i=0; i<SizeofArray(tsc.aaOutput); i++){







|







2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
  ){
    rc = fts3TermSelectCb(p, (void *)&tsc, 
        pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist
    );
  }

  if( rc==SQLITE_OK ){
    rc = fts3TermSelectMerge(p, &tsc);
  }
  if( rc==SQLITE_OK ){
    *ppOut = tsc.aaOutput[0];
    *pnOut = tsc.anOutput[0];
  }else{
    int i;
    for(i=0; i<SizeofArray(tsc.aaOutput); i++){
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
      }
    }
  }

  return nDoc;
}

/*
** This function merges two doclists according to the requirements of a
** NEAR operator.
**
** Both input doclists must include position information. The output doclist 
** includes position information if the first argument to this function
** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR.
*/
static int fts3NearMerge(
  int mergetype,                  /* MERGE_POS_NEAR or MERGE_NEAR */
  int nNear,                      /* Parameter to NEAR operator */
  int nTokenLeft,                 /* Number of tokens in LHS phrase arg */
  char *aLeft,                    /* Doclist for LHS (incl. positions) */
  int nLeft,                      /* Size of LHS doclist in bytes */
  int nTokenRight,                /* As nTokenLeft */
  char *aRight,                   /* As aLeft */
  int nRight,                     /* As nRight */
  char **paOut,                   /* OUT: Results of merge (malloced) */
  int *pnOut                      /* OUT: Sized of output buffer */
){
  char *aOut;                     /* Buffer to write output doclist to */
  int rc;                         /* Return code */

  assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR );

  aOut = sqlite3_malloc(nLeft+nRight+1);
  if( aOut==0 ){
    rc = SQLITE_NOMEM;
  }else{
    rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, 
      aOut, pnOut, aLeft, nLeft, aRight, nRight, 0
    );
    if( rc!=SQLITE_OK ){
      sqlite3_free(aOut);
      aOut = 0;
    }
  }

  *paOut = aOut;
  return rc;
}

/*
** Advance the cursor to the next row in the %_content table that
** matches the search criteria.  For a MATCH search, this will be
** the next row that matches. For a full-table scan, this will be
** simply the next row in the %_content table.  For a docid lookup,
** this routine simply sets the EOF flag.
**







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







2730
2731
2732
2733
2734
2735
2736










































2737
2738
2739
2740
2741
2742
2743
      }
    }
  }

  return nDoc;
}











































/*
** Advance the cursor to the next row in the %_content table that
** matches the search criteria.  For a MATCH search, this will be
** the next row that matches. For a full-table scan, this will be
** simply the next row in the %_content table.  For a docid lookup,
** this routine simply sets the EOF flag.
**
2584
2585
2586
2587
2588
2589
2590

2591



2592
2593
2594
2595
2596
2597
2598

  /* In case the cursor has been used before, clear it now. */
  sqlite3_finalize(pCsr->pStmt);
  sqlite3_free(pCsr->aDoclist);
  sqlite3Fts3ExprFree(pCsr->pExpr);
  memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));


  pCsr->bDesc = (idxStr && idxStr[0]=='D');



  pCsr->eSearch = (i16)idxNum;

  if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){
    int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
    const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);

    if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){







>
|
>
>
>







2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818

  /* In case the cursor has been used before, clear it now. */
  sqlite3_finalize(pCsr->pStmt);
  sqlite3_free(pCsr->aDoclist);
  sqlite3Fts3ExprFree(pCsr->pExpr);
  memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));

  if( idxStr ){
    pCsr->bDesc = (idxStr[0]=='D');
  }else{
    pCsr->bDesc = p->bDescIdx;
  }
  pCsr->eSearch = (i16)idxNum;

  if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){
    int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
    const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);

    if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637

  /* Compile a SELECT statement for this cursor. For a full-table-scan, the
  ** statement loops through all rows of the %_content table. For a
  ** full-text query or docid lookup, the statement retrieves a single
  ** row by docid.
  */
  if( idxNum==FTS3_FULLSCAN_SEARCH ){
    const char *zSort = (idxStr ? idxStr : "ASC");
    const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s";
    zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort);
  }else{
    const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?";
    zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName);
  }
  if( !zSql ) return SQLITE_NOMEM;







|







2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857

  /* Compile a SELECT statement for this cursor. For a full-table-scan, the
  ** statement loops through all rows of the %_content table. For a
  ** full-text query or docid lookup, the statement retrieves a single
  ** row by docid.
  */
  if( idxNum==FTS3_FULLSCAN_SEARCH ){
    const char *zSort = (pCsr->bDesc ? "DESC" : "ASC");
    const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s";
    zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort);
  }else{
    const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?";
    zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName);
  }
  if( !zSql ) return SQLITE_NOMEM;
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
          nDoclist = 0;
          break;
        }else if( aDoclist==0 ){
          aDoclist = pThis;
          nDoclist = nThis;
        }else{
          assert( iPrev>=0 );
          fts3DoclistMerge(MERGE_POS_PHRASE, iToken-iPrev, 
              0, pThis, &nThis, aDoclist, nDoclist, pThis, nThis, 0
          );
          sqlite3_free(aDoclist);
          aDoclist = pThis;
          nDoclist = nThis;
        }
        iPrev = iToken;
      }







|
|







3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
          nDoclist = 0;
          break;
        }else if( aDoclist==0 ){
          aDoclist = pThis;
          nDoclist = nThis;
        }else{
          assert( iPrev>=0 );
          fts3DoclistPhraseMerge(pTab->bDescIdx,
              iToken-iPrev, aDoclist, nDoclist, pThis, &nThis
          );
          sqlite3_free(aDoclist);
          aDoclist = pThis;
          nDoclist = nThis;
        }
        iPrev = iToken;
      }
3407
3408
3409
3410
3411
3412
3413

3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435




















































3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452

3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509



3510



3511
3512
3513
3514
3515
3516
3517
** function has been called successfully on an Fts3Phrase, it may be
** used with fts3EvalPhraseNext() to iterate through the matching docids.
*/
static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){
  int rc;
  Fts3Doclist *pList = &p->doclist;
  Fts3PhraseToken *pFirst = &p->aToken[0];


  assert( pList->aAll==0 );

  if( pCsr->bDesc==0 && bOptOk==1 && p->nToken==1 
   && pFirst->pSegcsr && pFirst->pSegcsr->bLookup 
  ){
    /* Use the incremental approach. */
    Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
    int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn);
    rc = sqlite3Fts3MsrIncrStart(
        pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n);
    p->bIncr = 1;

  }else{
    /* Load the full doclist for the phrase into memory. */
    rc = fts3EvalPhraseLoad(pCsr, p);
    p->bIncr = 0;
  }

  assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr );
  return rc;
}





















































/*
** Attempt to move the phrase iterator to point to the next matching docid. 
** If an error occurs, return an SQLite error code. Otherwise, return 
** SQLITE_OK.
**
** If there is no "next" entry and no error occurs, then *pbEof is set to
** 1 before returning. Otherwise, if no error occurs and the iterator is
** successfully advanced, *pbEof is set to 0.
*/
static int fts3EvalPhraseNext(
  Fts3Cursor *pCsr, 
  Fts3Phrase *p, 
  u8 *pbEof
){
  int rc = SQLITE_OK;
  Fts3Doclist *pDL = &p->doclist;


  if( p->bIncr ){
    Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
    assert( p->nToken==1 );
    rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, 
        &pDL->iDocid, &pDL->pList, &pDL->nList
    );
    if( rc==SQLITE_OK && !pDL->pList ){
      *pbEof = 1;
    }
  }else if( pCsr->bDesc && pDL->aAll ){

    if( pDL->pNextDocid==0 ){
      sqlite3_int64 iDocid = 0;
      char *pNext;
      char *pDocid = pDL->aAll;
      char *pEnd = &pDocid[pDL->nAll];

      while( pDocid<pEnd ){
        fts3GetDeltaVarint(&pDocid, &iDocid);
        pDL->pNextDocid = pDocid;
        pDL->pList = pDocid;
        fts3PoslistCopy(0, &pDocid);
      }
      pDL->nList = (pEnd - pDL->pList);
      pDL->iDocid = iDocid;
    }else{

      assert( *pbEof==0 );
      assert( pDL->pNextDocid>pDL->aAll );

      fts3GetReverseDeltaVarint(
          &pDL->pNextDocid, pDL->aAll, &pDL->iDocid
      );
      if( pDL->pNextDocid==pDL->aAll ){
        *pbEof = 1;
      }else{
        char *pSave = pDL->pNextDocid;
        fts3ReversePoslist(pDL->aAll, &pDL->pNextDocid);
        pDL->pList = pDL->pNextDocid;
        pDL->nList = pSave - pDL->pNextDocid;
      }
    }

  }else{

    char *pIter;
    if( pDL->pNextDocid ){
      pIter = pDL->pNextDocid;
    }else{
      pIter = pDL->aAll;
    }

    if( pIter>=&pDL->aAll[pDL->nAll] ){
      /* We have already reached the end of this doclist. EOF. */
      *pbEof = 1;
    }else{



      fts3GetDeltaVarint(&pIter, &pDL->iDocid);



      pDL->pList = pIter;
      fts3PoslistCopy(0, &pIter);
      pDL->nList = (pIter - pDL->pList);
      pDL->pNextDocid = pIter;
      *pbEof = 0;
    }
  }







>



|



<














>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

















>


<







|
|
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<

<
<
<
<
<

<
<
<
<

<











>
>
>
|
>
>
>







3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641

3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727

3728
3729
3730
3731
3732
3733
3734
3735
3736
3737




















3738





3739




3740

3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
** function has been called successfully on an Fts3Phrase, it may be
** used with fts3EvalPhraseNext() to iterate through the matching docids.
*/
static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){
  int rc;
  Fts3Doclist *pList = &p->doclist;
  Fts3PhraseToken *pFirst = &p->aToken[0];
  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;

  assert( pList->aAll==0 );

  if( pCsr->bDesc==pTab->bDescIdx && bOptOk==1 && p->nToken==1 
   && pFirst->pSegcsr && pFirst->pSegcsr->bLookup 
  ){
    /* Use the incremental approach. */

    int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn);
    rc = sqlite3Fts3MsrIncrStart(
        pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n);
    p->bIncr = 1;

  }else{
    /* Load the full doclist for the phrase into memory. */
    rc = fts3EvalPhraseLoad(pCsr, p);
    p->bIncr = 0;
  }

  assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr );
  return rc;
}

void sqlite3Fts3DoclistPrev(
  int bDescIdx,                   /* True if the doclist is desc */
  char *aDoclist,                 /* Pointer to entire doclist */
  int nDoclist,                   /* Length of aDoclist in bytes */
  char **ppIter,                  /* IN/OUT: Iterator pointer */
  sqlite3_int64 *piDocid,         /* IN/OUT: Docid pointer */
  int *pnList,                    /* IN/OUT: List length pointer */
  u8 *pbEof                       /* OUT: End-of-file flag */
){
  char *p = *ppIter;
  int iMul = (bDescIdx ? -1 : 1);

  assert( *pbEof==0 );
  assert( p || *piDocid==0 );
  assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) );

  if( p==0 ){
    sqlite3_int64 iDocid = 0;
    char *pNext = 0;
    char *pDocid = aDoclist;
    char *pEnd = &aDoclist[nDoclist];

    pDocid += sqlite3Fts3GetVarint(pDocid, &iDocid);
    pNext = pDocid;
    fts3PoslistCopy(0, &pDocid);
    while( pDocid<pEnd ){
      sqlite3_int64 iDelta;
      pDocid += sqlite3Fts3GetVarint(pDocid, &iDelta);
      iDocid += (iMul * iDelta);
      pNext = pDocid;
      fts3PoslistCopy(0, &pDocid);
    }

    *pnList = pEnd - pNext;
    *ppIter = pNext;
    *piDocid = iDocid;
  }else{
    sqlite3_int64 iDelta;
    fts3GetReverseVarint(&p, aDoclist, &iDelta);
    *piDocid -= (iMul * iDelta);

    if( p==aDoclist ){
      *pbEof = 1;
    }else{
      char *pSave = p;
      fts3ReversePoslist(aDoclist, &p);
      *pnList = (pSave - p);
    }
    *ppIter = p;
  }
}

/*
** Attempt to move the phrase iterator to point to the next matching docid. 
** If an error occurs, return an SQLite error code. Otherwise, return 
** SQLITE_OK.
**
** If there is no "next" entry and no error occurs, then *pbEof is set to
** 1 before returning. Otherwise, if no error occurs and the iterator is
** successfully advanced, *pbEof is set to 0.
*/
static int fts3EvalPhraseNext(
  Fts3Cursor *pCsr, 
  Fts3Phrase *p, 
  u8 *pbEof
){
  int rc = SQLITE_OK;
  Fts3Doclist *pDL = &p->doclist;
  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;

  if( p->bIncr ){

    assert( p->nToken==1 );
    rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, 
        &pDL->iDocid, &pDL->pList, &pDL->nList
    );
    if( rc==SQLITE_OK && !pDL->pList ){
      *pbEof = 1;
    }
  }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->aAll ){
    sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, 
        &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof




















      );





        pDL->pList = pDL->pNextDocid;




  }else{

    char *pIter;
    if( pDL->pNextDocid ){
      pIter = pDL->pNextDocid;
    }else{
      pIter = pDL->aAll;
    }

    if( pIter>=&pDL->aAll[pDL->nAll] ){
      /* We have already reached the end of this doclist. EOF. */
      *pbEof = 1;
    }else{
      sqlite3_int64 iDelta;
      pIter += sqlite3Fts3GetVarint(pIter, &iDelta);
      if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){
        pDL->iDocid += iDelta;
      }else{
        pDL->iDocid -= iDelta;
      }
      pDL->pList = pIter;
      fts3PoslistCopy(0, &pIter);
      pDL->nList = (pIter - pDL->pList);
      pDL->pNextDocid = pIter;
      *pbEof = 0;
    }
  }
3542
3543
3544
3545
3546
3547
3548

3549
3550
3551
3552
3553
3554
3555
      fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc);
      pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred);
    }
  }
}

static void fts3EvalNearMerge(

  Fts3Expr *p1,
  Fts3Expr *p2,
  int nNear,
  int *pRc
){
  if( *pRc==SQLITE_OK ){
    int rc;                         /* Return code */







>







3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
      fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc);
      pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred);
    }
  }
}

static void fts3EvalNearMerge(
  int bDescIdx,
  Fts3Expr *p1,
  Fts3Expr *p2,
  int nNear,
  int *pRc
){
  if( *pRc==SQLITE_OK ){
    int rc;                         /* Return code */
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
      sqlite3_free(pRight->doclist.aAll);
      pRight->doclist.aAll = 0;
      pRight->doclist.nAll = 0;
    }else if( pRight->doclist.aAll ){
      char *aOut;                 /* Buffer in which to assemble new doclist */
      int nOut;                   /* Size of buffer aOut in bytes */
  
      *pRc = fts3NearMerge(MERGE_POS_NEAR, nNear, 
          pLeft->nToken, pLeft->doclist.aAll, pLeft->doclist.nAll,
          pRight->nToken, pRight->doclist.aAll, pRight->doclist.nAll,
          &aOut, &nOut
      );
      sqlite3_free(pRight->doclist.aAll);
      pRight->doclist.aAll = aOut;
      pRight->doclist.nAll = nOut;







|







3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
      sqlite3_free(pRight->doclist.aAll);
      pRight->doclist.aAll = 0;
      pRight->doclist.nAll = 0;
    }else if( pRight->doclist.aAll ){
      char *aOut;                 /* Buffer in which to assemble new doclist */
      int nOut;                   /* Size of buffer aOut in bytes */
  
      *pRc = fts3DoclistNearMerge(bDescIdx, MERGE_POS_NEAR, nNear, 
          pLeft->nToken, pLeft->doclist.aAll, pLeft->doclist.nAll,
          pRight->nToken, pRight->doclist.aAll, pRight->doclist.nAll,
          &aOut, &nOut
      );
      sqlite3_free(pRight->doclist.aAll);
      pRight->doclist.aAll = aOut;
      pRight->doclist.nAll = nOut;
3598
3599
3600
3601
3602
3603
3604

3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
        nPhrase++;
      }

      aPhrase = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nPhrase);
      if( !aPhrase ){
        *pRc = SQLITE_NOMEM;
      }else{

        int i = 1;
        aPhrase[0] = pLeft;
        do {
          pLeft = pLeft->pParent;
          aPhrase[i++] = pLeft->pRight;
        }while( pLeft!=pExpr );

        for(i=0; i<(nPhrase-1); i++){
          int nNear = aPhrase[i+1]->pParent->nNear;
          fts3EvalNearMerge(aPhrase[i], aPhrase[i+1], nNear, pRc);
        }
        for(i=nPhrase-2; i>=0; i--){
          int nNear = aPhrase[i+1]->pParent->nNear;
          fts3EvalNearMerge(aPhrase[i+1], aPhrase[i], nNear, pRc);
        }

        sqlite3_free(aPhrase);
      }

    }else{
      fts3EvalNearTrim(pCsr, pExpr->pLeft, pRc);







>









|



|







3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
        nPhrase++;
      }

      aPhrase = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nPhrase);
      if( !aPhrase ){
        *pRc = SQLITE_NOMEM;
      }else{
        Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
        int i = 1;
        aPhrase[0] = pLeft;
        do {
          pLeft = pLeft->pParent;
          aPhrase[i++] = pLeft->pRight;
        }while( pLeft!=pExpr );

        for(i=0; i<(nPhrase-1); i++){
          int nNear = aPhrase[i+1]->pParent->nNear;
          fts3EvalNearMerge(p->bDescIdx, aPhrase[i], aPhrase[i+1], nNear, pRc);
        }
        for(i=nPhrase-2; i>=0; i--){
          int nNear = aPhrase[i+1]->pParent->nNear;
          fts3EvalNearMerge(p->bDescIdx, aPhrase[i+1], aPhrase[i], nNear, pRc);
        }

        sqlite3_free(aPhrase);
      }

    }else{
      fts3EvalNearTrim(pCsr, pExpr->pLeft, pRc);
Changes to ext/fts3/fts3Int.h.
176
177
178
179
180
181
182

183
184
185
186
187
188
189

  char *zReadExprlist;
  char *zWriteExprlist;

  int nNodeSize;                  /* Soft limit for node size */
  u8 bHasStat;                    /* True if %_stat table exists */
  u8 bHasDocsize;                 /* True if %_docsize table exists */

  int nPgsz;                      /* Page size for host database */
  char *zSegmentsTbl;             /* Name of %_segments table */
  sqlite3_blob *pSegments;        /* Blob handle open on %_segments table */

  /* TODO: Fix the first paragraph of this comment.
  **
  ** The following hash table is used to buffer pending index updates during







>







176
177
178
179
180
181
182
183
184
185
186
187
188
189
190

  char *zReadExprlist;
  char *zWriteExprlist;

  int nNodeSize;                  /* Soft limit for node size */
  u8 bHasStat;                    /* True if %_stat table exists */
  u8 bHasDocsize;                 /* True if %_docsize table exists */
  u8 bDescIdx;                    /* True if doclists are in reverse order */
  int nPgsz;                      /* Page size for host database */
  char *zSegmentsTbl;             /* Name of %_segments table */
  sqlite3_blob *pSegments;        /* Blob handle open on %_segments table */

  /* TODO: Fix the first paragraph of this comment.
  **
  ** The following hash table is used to buffer pending index updates during
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
  Fts3Expr *pExpr;                /* Parsed MATCH query string */
  int nPhrase;                    /* Number of matchable phrases in query */
  Fts3DeferredToken *pDeferred;   /* Deferred search tokens, if any */
  sqlite3_int64 iPrevId;          /* Previous id read from aDoclist */
  char *pNextId;                  /* Pointer into the body of aDoclist */
  char *aDoclist;                 /* List of docids for full-text queries */
  int nDoclist;                   /* Size of buffer at aDoclist */
  int bDesc;                      /* True to sort in descending order */
  int eEvalmode;                  /* An FTS3_EVAL_XX constant */
  int nRowAvg;                    /* Average size of database rows, in pages */

  int isMatchinfoNeeded;          /* True when aMatchinfo[] needs filling in */
  u32 *aMatchinfo;                /* Information about most recent match */
  int nMatchinfo;                 /* Number of elements in aMatchinfo[] */
  char *zMatchinfo;               /* Matchinfo specification */







|







233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
  Fts3Expr *pExpr;                /* Parsed MATCH query string */
  int nPhrase;                    /* Number of matchable phrases in query */
  Fts3DeferredToken *pDeferred;   /* Deferred search tokens, if any */
  sqlite3_int64 iPrevId;          /* Previous id read from aDoclist */
  char *pNextId;                  /* Pointer into the body of aDoclist */
  char *aDoclist;                 /* List of docids for full-text queries */
  int nDoclist;                   /* Size of buffer at aDoclist */
  u8 bDesc;                       /* True to sort in descending order */
  int eEvalmode;                  /* An FTS3_EVAL_XX constant */
  int nRowAvg;                    /* Average size of database rows, in pages */

  int isMatchinfoNeeded;          /* True when aMatchinfo[] needs filling in */
  u32 *aMatchinfo;                /* Information about most recent match */
  int nMatchinfo;                 /* Number of elements in aMatchinfo[] */
  char *zMatchinfo;               /* Matchinfo specification */
434
435
436
437
438
439
440

441
442
443
444
445
446
447

/* fts3.c */
int sqlite3Fts3PutVarint(char *, sqlite3_int64);
int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
int sqlite3Fts3GetVarint32(const char *, int *);
int sqlite3Fts3VarintLen(sqlite3_uint64);
void sqlite3Fts3Dequote(char *);


int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *);
int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);

/* fts3_tokenizer.c */
const char *sqlite3Fts3NextToken(const char *, int *);
int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);







>







435
436
437
438
439
440
441
442
443
444
445
446
447
448
449

/* fts3.c */
int sqlite3Fts3PutVarint(char *, sqlite3_int64);
int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
int sqlite3Fts3GetVarint32(const char *, int *);
int sqlite3Fts3VarintLen(sqlite3_uint64);
void sqlite3Fts3Dequote(char *);
void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);

int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *);
int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);

/* fts3_tokenizer.c */
const char *sqlite3Fts3NextToken(const char *, int *);
int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
490
491
492
493
494
495
496
497

498
    Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
int sqlite3Fts3MsrIncrNext(
    Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); 
int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);

int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *);


#endif /* _FTSINT_H */








>

492
493
494
495
496
497
498
499
500
501
    Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
int sqlite3Fts3MsrIncrNext(
    Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); 
int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);

int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *);


#endif /* _FTSINT_H */
Changes to ext/fts3/fts3_write.c.
58
59
60
61
62
63
64
65
66
67

68
69
70
71
72
73
74
/* #define FTS3_NODE_CHUNK_THRESHOLD 1 */

typedef struct PendingList PendingList;
typedef struct SegmentNode SegmentNode;
typedef struct SegmentWriter SegmentWriter;

/*
** Data structure used while accumulating terms in the pending-terms hash
** table. The hash table entry maps from term (a string) to a malloc'd
** instance of this structure.

*/
struct PendingList {
  int nData;
  char *aData;
  int nSpace;
  sqlite3_int64 iLastDocid;
  sqlite3_int64 iLastCol;







<
<
|
>







58
59
60
61
62
63
64


65
66
67
68
69
70
71
72
73
/* #define FTS3_NODE_CHUNK_THRESHOLD 1 */

typedef struct PendingList PendingList;
typedef struct SegmentNode SegmentNode;
typedef struct SegmentWriter SegmentWriter;

/*


** An instance of the following data structure is used to build doclists
** incrementally. See function fts3PendingListAppend() for details.
*/
struct PendingList {
  int nData;
  char *aData;
  int nSpace;
  sqlite3_int64 iLastDocid;
  sqlite3_int64 iLastCol;
126
127
128
129
130
131
132
133


134

135
136
137
138
139
140
141
  */
  int nTerm;                      /* Number of bytes in current term */
  char *zTerm;                    /* Pointer to current term */
  int nTermAlloc;                 /* Allocated size of zTerm buffer */
  char *aDoclist;                 /* Pointer to doclist of current entry */
  int nDoclist;                   /* Size of doclist in current entry */

  /* The following variables are used to iterate through the current doclist */


  char *pOffsetList;

  sqlite3_int64 iDocid;
};

#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0)
#define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1])

/*







|
>
>

>







125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
  */
  int nTerm;                      /* Number of bytes in current term */
  char *zTerm;                    /* Pointer to current term */
  int nTermAlloc;                 /* Allocated size of zTerm buffer */
  char *aDoclist;                 /* Pointer to doclist of current entry */
  int nDoclist;                   /* Size of doclist in current entry */

  /* The following variables are used by fts3SegReaderNextDocid() to iterate 
  ** through the current doclist (aDoclist/nDoclist).
  */
  char *pOffsetList;
  int nOffsetList;                /* For descending pending seg-readers only */
  sqlite3_int64 iDocid;
};

#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0)
#define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1])

/*
569
570
571
572
573
574
575










576
577
578
579
580
581
582
583
584
585
586
587
  if( p!=*pp ){
    *pp = p;
    return 1;
  }
  return 0;
}











static int fts3PendingTermsAddOne(
  Fts3Table *p,
  int iCol,
  int iPos,
  Fts3Hash *pHash,
  const char *zToken,
  int nToken
){
  PendingList *pList;
  int rc = SQLITE_OK;

  pList = (PendingList *)fts3HashFind(pHash, zToken, nToken);







>
>
>
>
>
>
>
>
>
>




|







571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
  if( p!=*pp ){
    *pp = p;
    return 1;
  }
  return 0;
}

/*
** Free a PendingList object allocated by fts3PendingListAppend().
*/
static void fts3PendingListDelete(PendingList *pList){
  sqlite3_free(pList);
}

/*
** Add an entry to one of the pending-terms hash tables.
*/
static int fts3PendingTermsAddOne(
  Fts3Table *p,
  int iCol,
  int iPos,
  Fts3Hash *pHash,                /* Pending terms hash table to add entry to */
  const char *zToken,
  int nToken
){
  PendingList *pList;
  int rc = SQLITE_OK;

  pList = (PendingList *)fts3HashFind(pHash, zToken, nToken);
709
710
711
712
713
714
715
716

717
718
719
720
721
722
723
*/
void sqlite3Fts3PendingTermsClear(Fts3Table *p){
  int i;
  for(i=0; i<p->nIndex; i++){
    Fts3HashElem *pElem;
    Fts3Hash *pHash = &p->aIndex[i].hPending;
    for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){
      sqlite3_free(fts3HashData(pElem));

    }
    fts3HashClear(pHash);
  }
  p->nPendingData = 0;
}

/*







|
>







721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
*/
void sqlite3Fts3PendingTermsClear(Fts3Table *p){
  int i;
  for(i=0; i<p->nIndex; i++){
    Fts3HashElem *pElem;
    Fts3Hash *pHash = &p->aIndex[i].hPending;
    for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){
      PendingList *pList = (PendingList *)fts3HashData(pElem);
      fts3PendingListDelete(pList);
    }
    fts3HashClear(pHash);
  }
  p->nPendingData = 0;
}

/*
1110
1111
1112
1113
1114
1115
1116


1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
    assert( pReader->pBlob==0 );
    if( bIncr && pReader->nPopulate<pReader->nNode ){
      pReader->pBlob = p->pSegments;
      p->pSegments = 0;
    }
    pNext = pReader->aNode;
  }



  rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2);
  if( rc!=SQLITE_OK ) return rc;
  
  /* Because of the FTS3_NODE_PADDING bytes of padding, the following is 
  ** safe (no risk of overread) even if the node data is corrupted.  
  */
  pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
  pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
  if( nPrefix<0 || nSuffix<=0 
   || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 
  ){
    return SQLITE_CORRUPT_VTAB;
  }







>
>





|
<







1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137

1138
1139
1140
1141
1142
1143
1144
    assert( pReader->pBlob==0 );
    if( bIncr && pReader->nPopulate<pReader->nNode ){
      pReader->pBlob = p->pSegments;
      p->pSegments = 0;
    }
    pNext = pReader->aNode;
  }

  assert( !fts3SegReaderIsPending(pReader) );

  rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2);
  if( rc!=SQLITE_OK ) return rc;
  
  /* Because of the FTS3_NODE_PADDING bytes of padding, the following is 
  ** safe (no risk of overread) even if the node data is corrupted. */

  pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
  pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
  if( nPrefix<0 || nSuffix<=0 
   || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 
  ){
    return SQLITE_CORRUPT_VTAB;
  }
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171









1172
1173
1174
1175
1176

1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190

1191
1192
1193
1194
1195
1196
1197
1198






















1199
1200
1201
1202
1203
1204
1205
1206






1207
1208

1209
1210
1211
1212
1213
1214
1215
1216
  return SQLITE_OK;
}

/*
** Set the SegReader to point to the first docid in the doclist associated
** with the current term.
*/
static int fts3SegReaderFirstDocid(Fts3SegReader *pReader){
  int rc;
  assert( pReader->aDoclist );
  assert( !pReader->pOffsetList );









  rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX);
  if( rc==SQLITE_OK ){
    int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
    pReader->pOffsetList = &pReader->aDoclist[n];
  }

  return rc;
}

/*
** Advance the SegReader to point to the next docid in the doclist
** associated with the current term.
** 
** If arguments ppOffsetList and pnOffsetList are not NULL, then 
** *ppOffsetList is set to point to the first column-offset list
** in the doclist entry (i.e. immediately past the docid varint).
** *pnOffsetList is set to the length of the set of column-offset
** lists, not including the nul-terminator byte. For example:
*/
static int fts3SegReaderNextDocid(

  Fts3SegReader *pReader,
  char **ppOffsetList,
  int *pnOffsetList
){
  int rc = SQLITE_OK;
  char *p = pReader->pOffsetList;
  char c = 0;























  /* Pointer p currently points at the first byte of an offset list. The
  ** following two lines advance it to point one byte past the end of
  ** the same offset list.
  */
  while( 1 ){
    int nRead;
    int rc;







    while( *p | c ) c = *p++ & 0x80;
    assert( *p==0 );

    if( pReader->pBlob==0 || (p - pReader->aNode)!=pReader->nPopulate ) break;
    rc = fts3SegReaderIncrRead(pReader);
    if( rc!=SQLITE_OK ) return rc;
  }
  p++;

  /* If required, populate the output variables with a pointer to and the
  ** size of the previous offset-list.







|
|


>
>
>
>
>
>
>
>
>





>














>
|
|
|





>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

|
|
<

<
<

>
>
>
>
>
>


>
|







1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248

1249


1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
  return SQLITE_OK;
}

/*
** Set the SegReader to point to the first docid in the doclist associated
** with the current term.
*/
static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){
  int rc = SQLITE_OK;
  assert( pReader->aDoclist );
  assert( !pReader->pOffsetList );
  if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){
    u8 bEof = 0;
    pReader->iDocid = 0;
    pReader->nOffsetList = 0;
    sqlite3Fts3DoclistPrev(0,
        pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, 
        &pReader->iDocid, &pReader->nOffsetList, &bEof
    );
  }else{
  rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX);
  if( rc==SQLITE_OK ){
    int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
    pReader->pOffsetList = &pReader->aDoclist[n];
  }
  }
  return rc;
}

/*
** Advance the SegReader to point to the next docid in the doclist
** associated with the current term.
** 
** If arguments ppOffsetList and pnOffsetList are not NULL, then 
** *ppOffsetList is set to point to the first column-offset list
** in the doclist entry (i.e. immediately past the docid varint).
** *pnOffsetList is set to the length of the set of column-offset
** lists, not including the nul-terminator byte. For example:
*/
static int fts3SegReaderNextDocid(
  Fts3Table *pTab,
  Fts3SegReader *pReader,         /* Reader to advance to next docid */
  char **ppOffsetList,            /* OUT: Pointer to current position-list */
  int *pnOffsetList               /* OUT: Length of *ppOffsetList in bytes */
){
  int rc = SQLITE_OK;
  char *p = pReader->pOffsetList;
  char c = 0;

  assert( p );

  if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){
    /* A pending-terms seg-reader for an FTS4 table that uses order=desc.
    ** Pending-terms doclists are always built up in ascending order, so
    ** we have to iterate through them backwards here. */
    u8 bEof = 0;
    if( ppOffsetList ){
      *ppOffsetList = pReader->pOffsetList;
      *pnOffsetList = pReader->nOffsetList - 1;
    }
    sqlite3Fts3DoclistPrev(0,
        pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid,
        &pReader->nOffsetList, &bEof
    );
    if( bEof ){
      pReader->pOffsetList = 0;
    }else{
      pReader->pOffsetList = p;
    }
  }else{

  /* Pointer p currently points at the first byte of an offset list. The
    ** following block advances it to point one byte past the end of
    ** the same offset list. */

  while( 1 ){



      /* The following line of code (and the "p++" below the while() loop) is
      ** normally all that is required to move pointer p to the desired 
      ** position. The exception is if this node is being loaded from disk
      ** incrementally and pointer "p" now points to the first byte passed
      ** the populated part of pReader->aNode[].
      */
    while( *p | c ) c = *p++ & 0x80;
    assert( *p==0 );
  
      if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break;
    rc = fts3SegReaderIncrRead(pReader);
    if( rc!=SQLITE_OK ) return rc;
  }
  p++;

  /* If required, populate the output variables with a pointer to and the
  ** size of the previous offset-list.
1228
1229
1230
1231
1232
1233
1234



1235
1236
1237


1238
1239
1240
1241
1242
1243
1244
  if( p>=&pReader->aDoclist[pReader->nDoclist] ){
    pReader->pOffsetList = 0;
  }else{
    rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX);
    if( rc==SQLITE_OK ){
      sqlite3_int64 iDelta;
      pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);



      pReader->iDocid += iDelta;
    }
  }



  return SQLITE_OK;
}

/*
** This function is called to estimate the amount of data that will be 
** loaded from the disk If SegReaderIterate() is called on this seg-reader,







>
>
>



>
>







1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
  if( p>=&pReader->aDoclist[pReader->nDoclist] ){
    pReader->pOffsetList = 0;
  }else{
    rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX);
    if( rc==SQLITE_OK ){
      sqlite3_int64 iDelta;
      pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
        if( pTab->bDescIdx ){
          pReader->iDocid -= iDelta;
        }else{
      pReader->iDocid += iDelta;
    }
  }
    }
  }

  return SQLITE_OK;
}

/*
** This function is called to estimate the amount of data that will be 
** loaded from the disk If SegReaderIterate() is called on this seg-reader,
1593
1594
1595
1596
1597
1598
1599












1600
1601
1602
1603
1604
1605
1606
  int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0);
  if( rc==0 ){
    if( pLhs->iDocid==pRhs->iDocid ){
      rc = pRhs->iIdx - pLhs->iIdx;
    }else{
      rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1;
    }












  }
  assert( pLhs->aNode && pRhs->aNode );
  return rc;
}

/*
** Compare the term that the Fts3SegReader object passed as the first argument







>
>
>
>
>
>
>
>
>
>
>
>







1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
  int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0);
  if( rc==0 ){
    if( pLhs->iDocid==pRhs->iDocid ){
      rc = pRhs->iIdx - pLhs->iIdx;
    }else{
      rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1;
    }
  }
  assert( pLhs->aNode && pRhs->aNode );
  return rc;
}
static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){
  int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0);
  if( rc==0 ){
    if( pLhs->iDocid==pRhs->iDocid ){
      rc = pRhs->iIdx - pLhs->iIdx;
    }else{
      rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1;
    }
  }
  assert( pLhs->aNode && pRhs->aNode );
  return rc;
}

/*
** Compare the term that the Fts3SegReader object passed as the first argument
2286
2287
2288
2289
2290
2291
2292



2293
2294
2295
2296
2297
2298
2299
  Fts3MultiSegReader *pCsr,       /* Cursor object */
  int iCol,                       /* Column to match on. */
  const char *zTerm,              /* Term to iterate through a doclist for */
  int nTerm                       /* Number of bytes in zTerm */
){
  int i;
  int nSegment = pCsr->nSegment;




  assert( pCsr->pFilter==0 );
  assert( zTerm && nTerm>0 );

  /* Advance each segment iterator until it points to the term zTerm/nTerm. */
  for(i=0; i<nSegment; i++){
    Fts3SegReader *pSeg = pCsr->apSegment[i];







>
>
>







2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
  Fts3MultiSegReader *pCsr,       /* Cursor object */
  int iCol,                       /* Column to match on. */
  const char *zTerm,              /* Term to iterate through a doclist for */
  int nTerm                       /* Number of bytes in zTerm */
){
  int i;
  int nSegment = pCsr->nSegment;
  int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
    p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
  );

  assert( pCsr->pFilter==0 );
  assert( zTerm && nTerm>0 );

  /* Advance each segment iterator until it points to the term zTerm/nTerm. */
  for(i=0; i<nSegment; i++){
    Fts3SegReader *pSeg = pCsr->apSegment[i];
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337



2338
2339
2340
2341
2342
2343
2344
      break;
    }
  }
  pCsr->nAdvance = i;

  /* Advance each of the segments to point to the first docid. */
  for(i=0; i<pCsr->nAdvance; i++){
    int rc = fts3SegReaderFirstDocid(pCsr->apSegment[i]);
    if( rc!=SQLITE_OK ) return rc;
  }
  fts3SegReaderSort(pCsr->apSegment, i, i, fts3SegReaderDoclistCmp);

  assert( iCol<0 || iCol<p->nColumn );
  pCsr->iColFilter = iCol;

  return SQLITE_OK;
}

int sqlite3Fts3MsrIncrNext(
  Fts3Table *p,                   /* Virtual table handle */
  Fts3MultiSegReader *pMsr,       /* Multi-segment-reader handle */
  sqlite3_int64 *piDocid,         /* OUT: Docid value */
  char **paPoslist,               /* OUT: Pointer to position list */
  int *pnPoslist                  /* OUT: Size of position list in bytes */
){
  int nMerge = pMsr->nAdvance;
  Fts3SegReader **apSegment = pMsr->apSegment;




  if( nMerge==0 ){
    *paPoslist = 0;
    return SQLITE_OK;
  }

  while( 1 ){







|


|
















>
>
>







2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
      break;
    }
  }
  pCsr->nAdvance = i;

  /* Advance each of the segments to point to the first docid. */
  for(i=0; i<pCsr->nAdvance; i++){
    int rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]);
    if( rc!=SQLITE_OK ) return rc;
  }
  fts3SegReaderSort(pCsr->apSegment, i, i, xCmp);

  assert( iCol<0 || iCol<p->nColumn );
  pCsr->iColFilter = iCol;

  return SQLITE_OK;
}

int sqlite3Fts3MsrIncrNext(
  Fts3Table *p,                   /* Virtual table handle */
  Fts3MultiSegReader *pMsr,       /* Multi-segment-reader handle */
  sqlite3_int64 *piDocid,         /* OUT: Docid value */
  char **paPoslist,               /* OUT: Pointer to position list */
  int *pnPoslist                  /* OUT: Size of position list in bytes */
){
  int nMerge = pMsr->nAdvance;
  Fts3SegReader **apSegment = pMsr->apSegment;
  int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
    p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
  );

  if( nMerge==0 ){
    *paPoslist = 0;
    return SQLITE_OK;
  }

  while( 1 ){
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
    }else{
      int rc;
      char *pList;
      int nList;
      int j;
      sqlite3_int64 iDocid = apSegment[0]->iDocid;

      rc = fts3SegReaderNextDocid(apSegment[0], &pList, &nList);
      j = 1;
      while( rc==SQLITE_OK 
        && j<nMerge
        && apSegment[j]->pOffsetList
        && apSegment[j]->iDocid==iDocid
      ){
        fts3SegReaderNextDocid(apSegment[j], 0, 0);
        j++;
      }
      if( rc!=SQLITE_OK ) return rc;

      fts3SegReaderSort(pMsr->apSegment, nMerge, j, fts3SegReaderDoclistCmp);

      if( pMsr->iColFilter>=0 ){
        fts3ColumnFilter(pMsr->iColFilter, &pList, &nList);
      }

      if( nList>0 ){
        *piDocid = iDocid;







|






|



<
|







2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443

2444
2445
2446
2447
2448
2449
2450
2451
    }else{
      int rc;
      char *pList;
      int nList;
      int j;
      sqlite3_int64 iDocid = apSegment[0]->iDocid;

      rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList);
      j = 1;
      while( rc==SQLITE_OK 
        && j<nMerge
        && apSegment[j]->pOffsetList
        && apSegment[j]->iDocid==iDocid
      ){
        rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0);
        j++;
      }
      if( rc!=SQLITE_OK ) return rc;

      fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp);

      if( pMsr->iColFilter>=0 ){
        fts3ColumnFilter(pMsr->iColFilter, &pList, &nList);
      }

      if( nList>0 ){
        *piDocid = iDocid;
2429
2430
2431
2432
2433
2434
2435



2436
2437
2438
2439
2440
2441
2442
  int isColFilter =    (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER);
  int isPrefix =       (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX);
  int isScan =         (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN);

  Fts3SegReader **apSegment = pCsr->apSegment;
  int nSegment = pCsr->nSegment;
  Fts3SegFilter *pFilter = pCsr->pFilter;




  if( pCsr->nSegment==0 ) return SQLITE_OK;

  do {
    int nMerge;
    int i;
  







>
>
>







2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
  int isColFilter =    (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER);
  int isPrefix =       (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX);
  int isScan =         (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN);

  Fts3SegReader **apSegment = pCsr->apSegment;
  int nSegment = pCsr->nSegment;
  Fts3SegFilter *pFilter = pCsr->pFilter;
  int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
    p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
  );

  if( pCsr->nSegment==0 ) return SQLITE_OK;

  do {
    int nMerge;
    int i;
  
2479
2480
2481
2482
2483
2484
2485
2486



2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522












2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
        && apSegment[nMerge]->nTerm==pCsr->nTerm 
        && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm)
    ){
      nMerge++;
    }

    assert( isIgnoreEmpty || (isRequirePos && !isColFilter) );
    if( nMerge==1 && !isIgnoreEmpty ){



      pCsr->aDoclist = apSegment[0]->aDoclist;
      pCsr->nDoclist = apSegment[0]->nDoclist;
      rc = SQLITE_ROW;
    }else{
      int nDoclist = 0;           /* Size of doclist */
      sqlite3_int64 iPrev = 0;    /* Previous docid stored in doclist */

      /* The current term of the first nMerge entries in the array
      ** of Fts3SegReader objects is the same. The doclists must be merged
      ** and a single term returned with the merged doclist.
      */
      for(i=0; i<nMerge; i++){
        fts3SegReaderFirstDocid(apSegment[i]);
      }
      fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp);
      while( apSegment[0]->pOffsetList ){
        int j;                    /* Number of segments that share a docid */
        char *pList;
        int nList;
        int nByte;
        sqlite3_int64 iDocid = apSegment[0]->iDocid;
        fts3SegReaderNextDocid(apSegment[0], &pList, &nList);
        j = 1;
        while( j<nMerge
            && apSegment[j]->pOffsetList
            && apSegment[j]->iDocid==iDocid
        ){
          fts3SegReaderNextDocid(apSegment[j], 0, 0);
          j++;
        }

        if( isColFilter ){
          fts3ColumnFilter(pFilter->iCol, &pList, &nList);
        }

        if( !isIgnoreEmpty || nList>0 ){












          nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0);
          if( nDoclist+nByte>pCsr->nBuffer ){
            char *aNew;
            pCsr->nBuffer = (nDoclist+nByte)*2;
            aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer);
            if( !aNew ){
              return SQLITE_NOMEM;
            }
            pCsr->aBuffer = aNew;
          }
          nDoclist += sqlite3Fts3PutVarint(
              &pCsr->aBuffer[nDoclist], iDocid-iPrev
          );
          iPrev = iDocid;
          if( isRequirePos ){
            memcpy(&pCsr->aBuffer[nDoclist], pList, nList);
            nDoclist += nList;
            pCsr->aBuffer[nDoclist++] = '\0';
          }
        }

        fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp);
      }
      if( nDoclist>0 ){
        pCsr->aDoclist = pCsr->aBuffer;
        pCsr->nDoclist = nDoclist;
        rc = SQLITE_ROW;
      }
    }







|
>
>
>












|

|






|





|








>
>
>
>
>
>
>
>
>
>
>
>
|









|
<
<








|







2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624


2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
        && apSegment[nMerge]->nTerm==pCsr->nTerm 
        && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm)
    ){
      nMerge++;
    }

    assert( isIgnoreEmpty || (isRequirePos && !isColFilter) );
    if( nMerge==1 
     && !isIgnoreEmpty 
     && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0)
    ){
      pCsr->aDoclist = apSegment[0]->aDoclist;
      pCsr->nDoclist = apSegment[0]->nDoclist;
      rc = SQLITE_ROW;
    }else{
      int nDoclist = 0;           /* Size of doclist */
      sqlite3_int64 iPrev = 0;    /* Previous docid stored in doclist */

      /* The current term of the first nMerge entries in the array
      ** of Fts3SegReader objects is the same. The doclists must be merged
      ** and a single term returned with the merged doclist.
      */
      for(i=0; i<nMerge; i++){
        fts3SegReaderFirstDocid(p, apSegment[i]);
      }
      fts3SegReaderSort(apSegment, nMerge, nMerge, xCmp);
      while( apSegment[0]->pOffsetList ){
        int j;                    /* Number of segments that share a docid */
        char *pList;
        int nList;
        int nByte;
        sqlite3_int64 iDocid = apSegment[0]->iDocid;
        fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList);
        j = 1;
        while( j<nMerge
            && apSegment[j]->pOffsetList
            && apSegment[j]->iDocid==iDocid
        ){
          fts3SegReaderNextDocid(p, apSegment[j], 0, 0);
          j++;
        }

        if( isColFilter ){
          fts3ColumnFilter(pFilter->iCol, &pList, &nList);
        }

        if( !isIgnoreEmpty || nList>0 ){

          /* Calculate the 'docid' delta value to write into the merged 
          ** doclist. */
          sqlite3_int64 iDelta;
          if( p->bDescIdx && nDoclist>0 ){
            iDelta = iPrev - iDocid;
          }else{
            iDelta = iDocid - iPrev;
          }
          assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) );
          assert( nDoclist>0 || iDelta==iDocid );

          nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0);
          if( nDoclist+nByte>pCsr->nBuffer ){
            char *aNew;
            pCsr->nBuffer = (nDoclist+nByte)*2;
            aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer);
            if( !aNew ){
              return SQLITE_NOMEM;
            }
            pCsr->aBuffer = aNew;
          }
          nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta);


          iPrev = iDocid;
          if( isRequirePos ){
            memcpy(&pCsr->aBuffer[nDoclist], pList, nList);
            nDoclist += nList;
            pCsr->aBuffer[nDoclist++] = '\0';
          }
        }

        fts3SegReaderSort(apSegment, nMerge, j, xCmp);
      }
      if( nDoclist>0 ){
        pCsr->aDoclist = pCsr->aBuffer;
        pCsr->nDoclist = nDoclist;
        rc = SQLITE_ROW;
      }
    }
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
    *pnByte = pDeferred->pList->nData;
    return pDeferred->pList->aData;
  }
  *pnByte = 0;
  return 0;
}

/*
** Helper fucntion for FreeDeferredDoclists(). This function removes all
** references to deferred doclists from within the tree of Fts3Expr 
** structures headed by 
*/
static void fts3DeferredDoclistClear(Fts3Expr *pExpr){
  if( pExpr ){
    Fts3Phrase *pPhrase = pExpr->pPhrase;
    fts3DeferredDoclistClear(pExpr->pLeft);
    fts3DeferredDoclistClear(pExpr->pRight);
  }
}

/*
** Delete all cached deferred doclists. Deferred doclists are cached
** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function.
*/
void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){
  Fts3DeferredToken *pDef;
  for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){
    sqlite3_free(pDef->pList);
    pDef->pList = 0;
  }
  if( pCsr->pDeferred ){
    fts3DeferredDoclistClear(pCsr->pExpr);
  }
}

/*
** Free all entries in the pCsr->pDeffered list. Entries are added to 
** this list using sqlite3Fts3DeferToken().
*/
void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){
  Fts3DeferredToken *pDef;
  Fts3DeferredToken *pNext;
  for(pDef=pCsr->pDeferred; pDef; pDef=pNext){
    pNext = pDef->pNext;
    sqlite3_free(pDef->pList);
    sqlite3_free(pDef);
  }
  pCsr->pDeferred = 0;
}

/*
** Generate deferred-doclists for all tokens in the pCsr->pDeferred list







<
<
<
<
<
<
<
<
<
<
<
<
<







|


<
<
<











|







2968
2969
2970
2971
2972
2973
2974













2975
2976
2977
2978
2979
2980
2981
2982
2983
2984



2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
    *pnByte = pDeferred->pList->nData;
    return pDeferred->pList->aData;
  }
  *pnByte = 0;
  return 0;
}














/*
** Delete all cached deferred doclists. Deferred doclists are cached
** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function.
*/
void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){
  Fts3DeferredToken *pDef;
  for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){
    fts3PendingListDelete(pDef->pList);
    pDef->pList = 0;
  }



}

/*
** Free all entries in the pCsr->pDeffered list. Entries are added to 
** this list using sqlite3Fts3DeferToken().
*/
void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){
  Fts3DeferredToken *pDef;
  Fts3DeferredToken *pNext;
  for(pDef=pCsr->pDeferred; pDef; pDef=pNext){
    pNext = pDef->pNext;
    fts3PendingListDelete(pDef->pList);
    sqlite3_free(pDef);
  }
  pCsr->pDeferred = 0;
}

/*
** Generate deferred-doclists for all tokens in the pCsr->pDeferred list
Changes to test/fts3sort.test.
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44











45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61




62
63
64
65
66
67
68

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}

set testprefix fts3sort

proc build_database {nRow} {
  db close
  forcedelete test.db
  sqlite3 db test.db

  set vocab [list    aa ab ac   ba bb bc    ca cb cc   da]
  expr srand(0)

  execsql { CREATE VIRTUAL TABLE t1 USING fts4 }
  for {set i 0} {$i < $nRow} {incr i} {
    set v [expr int(rand()*1000000)]
    set doc [list]
    for {set div 1} {$div < 1000000} {set div [expr $div*10]} {
      lappend doc [lindex $vocab [expr ($v/$div) % 10]]
    }
    execsql { INSERT INTO t1 VALUES($doc) }
  }
}












set nRow 1000
do_test 1.0 {
  build_database $nRow
  execsql { SELECT count(*) FROM t1 }
} $nRow

foreach {tn query} {
  1   "SELECT docid, * FROM t1"
  2   "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa'"
  3   "SELECT docid, * FROM t1 WHERE t1 MATCH 'a*'"
  4   "SELECT docid, quote(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'a*'"
  5   "SELECT docid, quote(matchinfo(t1,'pcnxals')) FROM t1 WHERE t1 MATCH 'b*'"
  6   "SELECT docid, * FROM t1 WHERE t1 MATCH 'a* b* c*'"
  7   "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa OR da'"
  8   "SELECT docid, * FROM t1 WHERE t1 MATCH 'nosuchtoken'"
  9   "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR da'"
  10  "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR nosuchtoken'"




} {

  unset -nocomplain A B C D
  set A_list [list]
  set B_list [list]
  set C_list [list]
  set D_list [list]







<

|







|










>
>
>
>
>
>
>
>
>
>
>


|














>
>
>
>







17
18
19
20
21
22
23

24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}



proc build_database {nRow param} {
  db close
  forcedelete test.db
  sqlite3 db test.db

  set vocab [list    aa ab ac   ba bb bc    ca cb cc   da]
  expr srand(0)

  execsql "CREATE VIRTUAL TABLE t1 USING fts4($param)"
  for {set i 0} {$i < $nRow} {incr i} {
    set v [expr int(rand()*1000000)]
    set doc [list]
    for {set div 1} {$div < 1000000} {set div [expr $div*10]} {
      lappend doc [lindex $vocab [expr ($v/$div) % 10]]
    }
    execsql { INSERT INTO t1 VALUES($doc) }
  }
}

set testprefix fts3sort

unset -nocomplain CONTROL
foreach {t param} {
  1     ""
  2     "order=asc"
  3     "order=desc"
} {

  set testprefix fts3sort-1.$t

set nRow 1000
do_test 1.0 {
    build_database $nRow $param
  execsql { SELECT count(*) FROM t1 }
} $nRow

foreach {tn query} {
  1   "SELECT docid, * FROM t1"
  2   "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa'"
  3   "SELECT docid, * FROM t1 WHERE t1 MATCH 'a*'"
  4   "SELECT docid, quote(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'a*'"
  5   "SELECT docid, quote(matchinfo(t1,'pcnxals')) FROM t1 WHERE t1 MATCH 'b*'"
  6   "SELECT docid, * FROM t1 WHERE t1 MATCH 'a* b* c*'"
  7   "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa OR da'"
  8   "SELECT docid, * FROM t1 WHERE t1 MATCH 'nosuchtoken'"
  9   "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR da'"
  10  "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR nosuchtoken'"
  11  "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa NEAR bb'"
  12  "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH '\"aa bb\"'"
  13  "SELECT docid, content FROM t1 WHERE t1 MATCH 'aa NEAR/2 bb NEAR/3 cc'"
  14  "SELECT docid, content FROM t1 WHERE t1 MATCH '\"aa bb cc\"'"
} {

  unset -nocomplain A B C D
  set A_list [list]
  set B_list [list]
  set C_list [list]
  set D_list [list]
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106




107


































108

  }
  unset -nocomplain X
  db eval "$query ORDER BY docid DESC" X  { 
    set D($X(docid)) [array get X] 
    lappend D_list $X(docid)
  }

  do_test 1.$tn.1 { set A_list } [lsort -integer -increasing $A_list]
  do_test 1.$tn.2 { set B_list } [lsort -integer -decreasing $B_list]
  do_test 1.$tn.3 { set C_list } [lsort -integer -increasing $C_list]
  do_test 1.$tn.4 { set D_list } [lsort -integer -decreasing $D_list]

  unset -nocomplain DATA
  unset -nocomplain X
  db eval "$query" X  { 
    set DATA($X(docid)) [array get X] 
  }

  do_test 1.$tn.5 { lsort [array get A] } [lsort [array get DATA]]
  do_test 1.$tn.6 { lsort [array get B] } [lsort [array get DATA]]
  do_test 1.$tn.7 { lsort [array get C] } [lsort [array get DATA]]
  do_test 1.$tn.8 { lsort [array get D] } [lsort [array get DATA]]
}







































finish_test








|
|
|
|







|
|
|
|
|
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

>
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
  }
  unset -nocomplain X
  db eval "$query ORDER BY docid DESC" X  { 
    set D($X(docid)) [array get X] 
    lappend D_list $X(docid)
  }

    do_test $tn.1 { set A_list } [lsort -integer -increasing $A_list]
    do_test $tn.2 { set B_list } [lsort -integer -decreasing $B_list]
    do_test $tn.3 { set C_list } [lsort -integer -increasing $C_list]
    do_test $tn.4 { set D_list } [lsort -integer -decreasing $D_list]

  unset -nocomplain DATA
  unset -nocomplain X
  db eval "$query" X  { 
    set DATA($X(docid)) [array get X] 
  }

    do_test $tn.5 { lsort [array get A] } [lsort [array get DATA]]
    do_test $tn.6 { lsort [array get B] } [lsort [array get DATA]]
    do_test $tn.7 { lsort [array get C] } [lsort [array get DATA]]
    do_test $tn.8 { lsort [array get D] } [lsort [array get DATA]]

    if {[info exists CONTROL($tn)]} {
      do_test $tn.9 { set CONTROL($tn) } [lsort [array get DATA]]
    } else {
      set CONTROL($tn) [lsort [array get DATA]]
    }
  }
}
unset -nocomplain CONTROL

set testprefix fts3sort

#-------------------------------------------------------------------------
# Tests for parsing the "order=asc" and "order=desc" directives.
#
foreach {tn param res} {
  1 "order=asc"             {0 {}}
  2 "order=desc"            {0 {}}
  3 "order=dec"             {1 {unrecognized order: dec}}
  4 "order=xxx, order=asc"  {0 {}}
} {
  execsql { DROP TABLE IF EXISTS t1 }
  do_catchsql_test 2.1.$tn "
    CREATE VIRTUAL TABLE t1 USING fts4(a, b, $param)
  " $res
}

do_execsql_test 2.2 {
  BEGIN;
    CREATE VIRTUAL TABLE t2 USING fts4(order=desc);
    INSERT INTO t2 VALUES('aa bb');
    INSERT INTO t2 VALUES('bb cc');
    INSERT INTO t2 VALUES('cc aa');
    SELECT docid FROM t2 WHERE t2 MATCH 'aa';
  END;
} {3 1}
do_execsql_test 2.3 {
  SELECT docid FROM t2 WHERE t2 MATCH 'aa';
} {3 1}

finish_test