SQLite4
Check-in [739cd82fca]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix problems with the matchinfo functions used by the bm25(f) implementation.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 739cd82fca631bc54a5cf99e8746c0ac358d57d5
User & Date: dan 2013-01-14 17:05:09
Context
2013-01-15
17:39
Fix a typo in lsmusr.wiki. check-in: ace47a5829 user: dan tags: trunk
2013-01-14
17:05
Fix problems with the matchinfo functions used by the bm25(f) implementation. check-in: 739cd82fca user: dan tags: trunk
2013-01-13
05:30
Avoid using keyword "near" as a variable name in fts5.c. check-in: 278cfaeb70 user: dan tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/fts5.c.

1278
1279
1280
1281
1282
1283
1284

1285
1286


1287
1288
1289
1290










1291
1292
1293

1294
1295
1296
1297
1298
1299
1300
....
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
....
1613
1614
1615
1616
1617
1618
1619

1620
1621
1622
1623
1624
1625
1626
....
2866
2867
2868
2869
2870
2871
2872
2873




2874
2875
2876
2877


2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
....
2892
2893
2894
2895
2896
2897
2898

2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909

2910
2911
2912
2913
2914
2915
2916
....
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
  TokenizeTerm *pTerm = 0;

  /* TODO: Error here if iStream is out of range */

  if( nToken>p->nMax ) p->nMax = nToken;

  if( iStream>=p->nStream ){

    int nOld = p->nStream;
    int nNew = 4;


    while( nNew<=iStream ) nNew = nNew*2;
    p->aSz = (i64*)sqlite4DbReallocOrFree(db, p->aSz, nNew*p->nCol*sizeof(i64));
    if( p->aSz==0 ) goto tokenize_cb_out;
    memset(&p->aSz[nOld * p->nCol], 0, (nNew-nOld)*p->nCol*sizeof(i64));










    p->nStream = nNew;
  }
  p->aSz[iStream*p->nCol + p->iCol]++;


  pTerm = (TokenizeTerm *)sqlite4HashFind(&p->hash, zToken, nToken);
  if( pTerm==0 ){
    /* Size the initial allocation so that it fits in the lookaside buffer */
    int nAlloc = sizeof(TokenizeTerm) + nToken + 32;

    pTerm = sqlite4DbMallocZero(p->db, nAlloc);
................................................................................
    iOff += sqlite4PutVarint(&a[iOff], nRow);
  }
  iOff += sqlite4PutVarint(&a[iOff], pSz->nStream);

  for(iCol=0; iCol<pSz->nCol; iCol++){
    int i;
    for(i=0; i<pSz->nStream; i++){
      iOff += sqlite4PutVarint(&a[iOff], pSz->aSz[i*pSz->nCol+iCol]);
    }
  }

  return sqlite4KVStoreReplace(p, aKey, nKey, a, iOff);
}

static int fts5CsrLoadGlobal(Fts5Cursor *pCsr){
................................................................................
      int iCol;
      for(iCol=0; iCol<pSz->nCol; iCol++){
        int iStr;
        i64 *aIn = &sCtx.aSz[iCol * sCtx.nStream];
        i64 *aOut = &pSz->aSz[iCol * pSz->nStream];
        for(iStr=0; iStr<sCtx.nStream; iStr++){
          aOut[iStr] += (aIn[iStr] * (bDel?-1:1));

        }
      }
      nRow += (bDel?-1:1);
      rc = fts5StoreSizeRecord(pStore, aKey, nKey, pSz, nRow, &aKey[nKey]);
    }

    sqlite4DbFree(db, pSz);
................................................................................
  }else{
    Fts5Info *pInfo = pCsr->pInfo;
    rc = pInfo->pTokenizer->xTokenize(p, pInfo->p, zText, nText, x);
  }
  return rc;
}

static Fts5Str *fts5FindStr(Fts5ExprNode *p, int *piStr){




  Fts5Str *pRet = 0;
  if( p->eType==TOKEN_PRIMITIVE ){
    int iStr = *piStr;
    if( iStr<p->pPhrase->nStr ){


      pRet = &p->pPhrase->aStr[iStr];
    }else{
      *piStr = iStr - p->pPhrase->nStr;
    }
  }else{
    pRet = fts5FindStr(p->pLeft, piStr);
    if( pRet==0 ) pRet = fts5FindStr(p->pRight, piStr);
  }
  return pRet;
}

int sqlite4_mi_match_count(
  sqlite4_context *pCtx, 
  int iC,
................................................................................
  int iS,
  int iPhrase,
  int *pnMatch
){
  int rc = SQLITE4_OK;
  Fts5Cursor *pCsr = pCtx->pFts;
  if( pCsr ){

    int nMatch = 0;
    Fts5Str *pStr;
    int iCopy = iPhrase;
    InstanceList sList;

    pStr = fts5FindStr(pCsr->pExpr->pRoot, &iCopy);
    assert( pStr );

    fts5InstanceListInit(pStr->aList, pStr->nList, &sList);
    while( 0==fts5InstanceListNext(&sList) ){
      if( (iC<0 || sList.iCol==iC) && (iS<0 || sList.iStream==iS) ) nMatch++;

    }
    *pnMatch = nMatch;
  }else{
    rc = SQLITE4_MISUSE;
  }
  return rc;
}
................................................................................

    if( pNode->eType==TOKEN_PRIMITIVE ){
      Fts5Phrase *pPhrase = pNode->pPhrase;
      int iStr = *piStr;

      rc = fts5ExprAdvance(db, pNode, 1);
      while( rc==SQLITE4_OK && pNode->aPk ){
        int nIncr =  pInfo->nCol * nStream;      /* Values for each Fts5Str */
        int i;
        for(i=0; i<pPhrase->nStr; i++){
          int *anRow = &pCsr->anRow[(iStr+i) * pInfo->nCol * nStream];
          int *anRowC = &pCsr->anRowC[(iStr+i) * pInfo->nCol];
          int *anRowS = &pCsr->anRowS[(iStr+i) * nStream];
          int *pnRowCS = &pCsr->anRowCS[iStr+i];
          fts5StrLoadRowcounts(







>


>
>

|
|
<
>
>
>
>
>
>
>
>
>
>


<
>







 







|







 







>







 







|
>
>
>
>



|
>
>





|
|







 







>





|
|
<
|
|
|
>







 







<







1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292

1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304

1305
1306
1307
1308
1309
1310
1311
1312
....
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
....
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
....
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
....
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925

2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
....
2997
2998
2999
3000
3001
3002
3003

3004
3005
3006
3007
3008
3009
3010
  TokenizeTerm *pTerm = 0;

  /* TODO: Error here if iStream is out of range */

  if( nToken>p->nMax ) p->nMax = nToken;

  if( iStream>=p->nStream ){
    int iCol;
    int nOld = p->nStream;
    int nNew = 4;
    i64 *aNew;
    
    while( nNew<=iStream ) nNew = nNew*2;
    aNew = sqlite4DbMallocZero(db, nNew*p->nCol*sizeof(i64));
    if( aNew==0 ) goto tokenize_cb_out;


    for(iCol=0; iCol<p->nCol; iCol++){
      int iStr;
      for(iStr=0; iStr<nOld; iStr++){
        aNew[nNew*iCol + iStr] = p->aSz[nOld*iCol + iStr];
      }
    }

    sqlite4DbFree(db, p->aSz);
    p->aSz = aNew;
    p->nStream = nNew;
  }

  p->aSz[p->iCol * p->nStream + iStream]++;

  pTerm = (TokenizeTerm *)sqlite4HashFind(&p->hash, zToken, nToken);
  if( pTerm==0 ){
    /* Size the initial allocation so that it fits in the lookaside buffer */
    int nAlloc = sizeof(TokenizeTerm) + nToken + 32;

    pTerm = sqlite4DbMallocZero(p->db, nAlloc);
................................................................................
    iOff += sqlite4PutVarint(&a[iOff], nRow);
  }
  iOff += sqlite4PutVarint(&a[iOff], pSz->nStream);

  for(iCol=0; iCol<pSz->nCol; iCol++){
    int i;
    for(i=0; i<pSz->nStream; i++){
      iOff += sqlite4PutVarint(&a[iOff], pSz->aSz[iCol*pSz->nStream+i]);
    }
  }

  return sqlite4KVStoreReplace(p, aKey, nKey, a, iOff);
}

static int fts5CsrLoadGlobal(Fts5Cursor *pCsr){
................................................................................
      int iCol;
      for(iCol=0; iCol<pSz->nCol; iCol++){
        int iStr;
        i64 *aIn = &sCtx.aSz[iCol * sCtx.nStream];
        i64 *aOut = &pSz->aSz[iCol * pSz->nStream];
        for(iStr=0; iStr<sCtx.nStream; iStr++){
          aOut[iStr] += (aIn[iStr] * (bDel?-1:1));
          assert( iStr==0 || aOut[iStr]==0 );
        }
      }
      nRow += (bDel?-1:1);
      rc = fts5StoreSizeRecord(pStore, aKey, nKey, pSz, nRow, &aKey[nKey]);
    }

    sqlite4DbFree(db, pSz);
................................................................................
  }else{
    Fts5Info *pInfo = pCsr->pInfo;
    rc = pInfo->pTokenizer->xTokenize(p, pInfo->p, zText, nText, x);
  }
  return rc;
}

static Fts5Str *fts5FindStr(
  const u8 *aPk, int nPk,
  Fts5ExprNode *p,
  int *piStr
){
  Fts5Str *pRet = 0;
  if( p->eType==TOKEN_PRIMITIVE ){
    int iStr = *piStr;
    if( iStr<p->pPhrase->nStr && iStr>=0 
     && p->nPk==nPk && 0==memcmp(p->aPk, aPk, nPk) 
    ){
      pRet = &p->pPhrase->aStr[iStr];
    }else{
      *piStr = iStr - p->pPhrase->nStr;
    }
  }else{
    pRet = fts5FindStr(aPk, nPk, p->pLeft, piStr);
    if( pRet==0 ) pRet = fts5FindStr(aPk, nPk, p->pRight, piStr);
  }
  return pRet;
}

int sqlite4_mi_match_count(
  sqlite4_context *pCtx, 
  int iC,
................................................................................
  int iS,
  int iPhrase,
  int *pnMatch
){
  int rc = SQLITE4_OK;
  Fts5Cursor *pCsr = pCtx->pFts;
  if( pCsr ){
    Fts5ExprNode *pRoot = pCsr->pExpr->pRoot;
    int nMatch = 0;
    Fts5Str *pStr;
    int iCopy = iPhrase;
    InstanceList sList;

    pStr = fts5FindStr(pRoot->aPk, pRoot->nPk, pRoot, &iCopy);
    if( pStr ){

      fts5InstanceListInit(pStr->aList, pStr->nList, &sList);
      while( 0==fts5InstanceListNext(&sList) ){
        if( (iC<0 || sList.iCol==iC) && (iS<0 || sList.iStream==iS) ) nMatch++;
      }
    }
    *pnMatch = nMatch;
  }else{
    rc = SQLITE4_MISUSE;
  }
  return rc;
}
................................................................................

    if( pNode->eType==TOKEN_PRIMITIVE ){
      Fts5Phrase *pPhrase = pNode->pPhrase;
      int iStr = *piStr;

      rc = fts5ExprAdvance(db, pNode, 1);
      while( rc==SQLITE4_OK && pNode->aPk ){

        int i;
        for(i=0; i<pPhrase->nStr; i++){
          int *anRow = &pCsr->anRow[(iStr+i) * pInfo->nCol * nStream];
          int *anRowC = &pCsr->anRowC[(iStr+i) * pInfo->nCol];
          int *anRowS = &pCsr->anRowS[(iStr+i) * nStream];
          int *pnRowCS = &pCsr->anRowCS[iStr+i];
          fts5StrLoadRowcounts(

Changes to src/fts5func.c.

40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58






































59
60
61
62
63
64
65
66
67
68
69
70
71
72
..
76
77
78
79
80
81
82
83
84

85
86




87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107

108
109
110
111
112
113
114
115
116
117
118
119

120


121





122

123
124
125
126
127
128
129






















130































131

132
133
134

135
136
137
138
139
140




141
142
143



144
145
146
147
148
149


150



151
152
153
154
155
156
157
158
159


160
161
162
163
164
165
166

167
168
169
170
171
172
173
174
175
176

177
178
179
180
181
182
183
...
531
532
533
534
535
536
537

538












539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
static int fts5SimpleDestroy(sqlite4_tokenizer *p){
  return SQLITE4_OK;
}

typedef struct Fts5RankCtx Fts5RankCtx;
struct Fts5RankCtx {
  sqlite4 *db;
  double avgdl;                   /* Average document size in tokens */
  int nPhrase;                    /* Number of phrases in query */
  double *aIdf;                   /* IDF weights for each phrase in query */
};

static void fts5RankFreeCtx(void *pCtx){
  if( pCtx ){
    Fts5RankCtx *p = (Fts5RankCtx *)pCtx;
    sqlite4DbFree(p->db, p);
  }
}







































/*
** A BM25 based ranking function for fts5.
**
** This is based on the information in the Robertson/Zaragoza paper 
** referenced above. As there is no way to provide relevance feedback 
** IDF weights (equation 3.3 in R/Z) are used instead of RSJ for each phrase.
** The rest of the implementation is as presented in equation 3.15.
**
** R and Z observe that the experimental evidence suggests that reasonable
** values for free parameters "b" and "k1" are often in the ranges 
** (0.5 < b < 0.8) and (1.2 < k1 < 2), although the optimal values depend
** on the nature of both the documents and queries. The implementation
** below sets each parameter to the midpoint of the suggested range.
*/
................................................................................

  sqlite4 *db = sqlite4_context_db_handle(pCtx);
  int rc = SQLITE4_OK;            /* Error code */
  Fts5RankCtx *p;                 /* Structure to store reusable values */
  int i;                          /* Used to iterate through phrases */
  double rank = 0.0;              /* UDF return value */

  int bExplain = 0;
  char *zExplain = 0;


  if( sqlite4_user_data(pCtx) ) bExplain = 1;





  p = sqlite4_get_auxdata(pCtx, 0);
  if( p==0 ){
    int nPhrase;                  /* Number of phrases in query expression */
    int nByte;                    /* Number of bytes of data to allocate */

    sqlite4_mi_phrase_count(pCtx, &nPhrase);
    nByte = sizeof(Fts5RankCtx) + nPhrase * sizeof(double);
    p = (Fts5RankCtx *)sqlite4DbMallocZero(db, nByte);
    sqlite4_set_auxdata(pCtx, 0, (void *)p, fts5RankFreeCtx);
    p = sqlite4_get_auxdata(pCtx, 0);

    if( !p ){
      rc = SQLITE4_NOMEM;
    }else{
      int N;                      /* Total number of docs in collection */
      int ni;                     /* Number of docs with phrase i */

      p->db = db;
      p->nPhrase = nPhrase;
      p->aIdf = (double *)&p[1];


      /* Determine the IDF weight for each phrase in the query. */
      rc = sqlite4_mi_total_rows(pCtx, &N);
      for(i=0; rc==SQLITE4_OK && i<nPhrase; i++){
        rc = sqlite4_mi_row_count(pCtx, -1, -1, i, &ni);
        if( rc==SQLITE4_OK ){
          assert( ni<=N );
          p->aIdf[i] = log((0.5 + N - ni) / (0.5 + ni));
        }
      }

      /* Determine the average document length */

      if( rc==SQLITE4_OK ){


        int nTotal;





        rc = sqlite4_mi_total_size(pCtx, -1, -1, &nTotal);

        if( rc==SQLITE4_OK ){
          p->avgdl = (double)nTotal / (double)N;
        }
      }
    }
  }























  for(i=0; rc==SQLITE4_OK && i<p->nPhrase; i++){































    int tf;                     /* Occurences of phrase i in row (term freq.) */

    int dl;                     /* Tokens in this row (document length) */
    double L;                   /* Normalized document length */
    double prank;               /* Contribution to rank of this phrase */


    /* Set variable tf to the total number of occurrences of phrase iPhrase
    ** in this row (within any column). And dl to the number of tokens in
    ** the current row (again, in any column).  */
    rc = sqlite4_mi_match_count(pCtx, -1, -1, i, &tf); 
    if( rc==SQLITE4_OK ) rc = sqlite4_mi_size(pCtx, -1, -1, &dl); 





    /* Calculate the normalized document length */
    L = (double)dl / p->avgdl;





    /* Calculate the contribution to the rank made by this phrase. Then
    ** add it to variable rank.  */
    prank = (p->aIdf[i] * tf) / (k1 * ( (1.0 - b) + b * L) + tf);
    rank += prank;






    if( bExplain ){
      zExplain = sqlite4MAppendf(
          db, zExplain, "%s(idf=%.2f L=%.2f tf=%d) rank=%.2f", zExplain,
          p->aIdf[i], L, tf, prank
      );
      if( (i+1)<p->nPhrase ){
        zExplain = sqlite4MAppendf(db, zExplain, "%s<br>", zExplain);
      }
    }


  }

  if( rc==SQLITE4_OK ){
    if( bExplain ){
      if( p->nPhrase>1 ){
        zExplain = sqlite4MAppendf(
            db, zExplain, "%s<br>total=%.2f", zExplain, rank

        );
      }
      sqlite4_result_text(pCtx, zExplain, -1, SQLITE4_TRANSIENT);
      sqlite4DbFree(db, zExplain);
    }else{
      sqlite4_result_double(pCtx, rank);
    }
  }else{
    sqlite4_result_error_code(pCtx, rc);
  }

}

typedef struct Snippet Snippet;
typedef struct SnippetText SnippetText;

struct Snippet {
  int iCol;
................................................................................

  sqlite4_free(pEnv, aBuf);
  return SQLITE4_OK;
}

int sqlite4InitFts5Func(sqlite4 *db){
  int rc;

  sqlite4_env *pEnv = sqlite4_db_env(db);













  rc = sqlite4_create_tokenizer(db, "simple", (void *)pEnv, 
      fts5SimpleCreate, fts5SimpleTokenize, fts5SimpleDestroy
  );
  if( rc!=SQLITE4_OK ) return rc;

  rc = sqlite4_create_mi_function(db, "rank", 0, SQLITE4_UTF8, 0, fts5Rank, 0);
  if( rc!=SQLITE4_OK ) return rc;
  rc = sqlite4_create_mi_function(
      db, "erank", 0, SQLITE4_UTF8, (void *)1, fts5Rank, 0
  );
  if( rc!=SQLITE4_OK ) return rc;

  rc = sqlite4_create_mi_function(
      db, "snippet", -1, SQLITE4_UTF8, 0, fts5Snippet, 0
  );
  return rc;
}








|











>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

|




|







 







|
|
>

|
>
>
>
>







|













>











|
>

>
>
|
>
>
>
>
>
|
>
|
|
|
|
|
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
|
<
<
>

|
|
<
<
<
>
>
>
>

<
<
>
>
>


<
<
<
<
>
>
|
>
>
>

<
<
<
<
<
|
|
|
>
>




<
|
<
>
|
<

<






>







 







>

>
>
>
>
>
>
>
>
>
>
>
>




|
|
|
|
|
|
|
|
|
|
|
<



40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
...
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239


240
241
242
243



244
245
246
247
248


249
250
251
252
253




254
255
256
257
258
259
260





261
262
263
264
265
266
267
268
269

270

271
272

273

274
275
276
277
278
279
280
281
282
283
284
285
286
287
...
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670

671
672
673
static int fts5SimpleDestroy(sqlite4_tokenizer *p){
  return SQLITE4_OK;
}

typedef struct Fts5RankCtx Fts5RankCtx;
struct Fts5RankCtx {
  sqlite4 *db;
  double *aAvgdl;                 /* Average document size of each field */
  int nPhrase;                    /* Number of phrases in query */
  double *aIdf;                   /* IDF weights for each phrase in query */
};

static void fts5RankFreeCtx(void *pCtx){
  if( pCtx ){
    Fts5RankCtx *p = (Fts5RankCtx *)pCtx;
    sqlite4DbFree(p->db, p);
  }
}

#define BM25_EXPLAIN  0x01
#define BM25_FCOLUMNS 0x02
#define BM25_FSTREAMS 0x04

static int fts5GetSizeFreqScale(
  sqlite4_context *pCtx,
  int nArg, sqlite4_value **apArg,/* Function arguments */
  int bm25mask,                   /* bm25 configuration mask */
  int iPhrase,                    /* Phrase number */
  int iField,                     /* Field number */
  int *pnSize,                    /* OUT: Size of field in tokens */
  int *pnFreq,                    /* OUT: Occurences of phrase in field */
  double *pdScale                 /* OUT: Scale to use with this field */
){
  int rc;
  double scale = 1.0;
  int nSize = 0;
  int nFreq = 0;

  if( bm25mask & BM25_FCOLUMNS ){
    rc = sqlite4_mi_match_count(pCtx, iField, -1, iPhrase, &nFreq); 
    if( rc==SQLITE4_OK ) rc = sqlite4_mi_size(pCtx, iField, -1, &nSize); 
    if( nArg>iField ) scale = sqlite4_value_double(apArg[iField]);
  }else if( bm25mask & BM25_FSTREAMS ){
    rc = sqlite4_mi_match_count(pCtx, -1, iField, iPhrase, &nFreq); 
    if( rc==SQLITE4_OK ) rc = sqlite4_mi_size(pCtx, -1, iField, &nSize); 
    if( nArg>iField ) scale = sqlite4_value_double(apArg[iField]);
  }else{
    rc = sqlite4_mi_match_count(pCtx, -1, -1, iPhrase, &nFreq); 
    if( rc==SQLITE4_OK ) rc = sqlite4_mi_size(pCtx, -1, -1, &nSize); 
  }

  *pnSize = nSize;
  *pnFreq = nFreq;
  *pdScale = scale;
  return rc;
}

/*
** A BM25(F) based ranking function for fts5.
**
** This is based on the information in the Robertson/Zaragoza paper 
** referenced above. As there is no way to provide relevance feedback 
** IDF weights (equation 3.3 in R/Z) are used instead of RSJ for each phrase.
** The rest of the implementation is as presented in equations 3.19-21.
**
** R and Z observe that the experimental evidence suggests that reasonable
** values for free parameters "b" and "k1" are often in the ranges 
** (0.5 < b < 0.8) and (1.2 < k1 < 2), although the optimal values depend
** on the nature of both the documents and queries. The implementation
** below sets each parameter to the midpoint of the suggested range.
*/
................................................................................

  sqlite4 *db = sqlite4_context_db_handle(pCtx);
  int rc = SQLITE4_OK;            /* Error code */
  Fts5RankCtx *p;                 /* Structure to store reusable values */
  int i;                          /* Used to iterate through phrases */
  double rank = 0.0;              /* UDF return value */

  int bExplain;                   /* True to run in explain mode */
  char *zExplain = 0;             /* String to return in explain mode */
  int nField = 1;                 /* Number of fields in collection */

  int bm25mask = SQLITE4_PTR_TO_INT(sqlite4_user_data(pCtx));
  bExplain = (bm25mask & BM25_EXPLAIN);

  if( bm25mask & BM25_FCOLUMNS ) sqlite4_mi_column_count(pCtx, &nField);
  if( bm25mask & BM25_FSTREAMS ) sqlite4_mi_stream_count(pCtx, &nField);

  p = sqlite4_get_auxdata(pCtx, 0);
  if( p==0 ){
    int nPhrase;                  /* Number of phrases in query expression */
    int nByte;                    /* Number of bytes of data to allocate */

    sqlite4_mi_phrase_count(pCtx, &nPhrase);
    nByte = sizeof(Fts5RankCtx) + (nPhrase+nField) * sizeof(double);
    p = (Fts5RankCtx *)sqlite4DbMallocZero(db, nByte);
    sqlite4_set_auxdata(pCtx, 0, (void *)p, fts5RankFreeCtx);
    p = sqlite4_get_auxdata(pCtx, 0);

    if( !p ){
      rc = SQLITE4_NOMEM;
    }else{
      int N;                      /* Total number of docs in collection */
      int ni;                     /* Number of docs with phrase i */

      p->db = db;
      p->nPhrase = nPhrase;
      p->aIdf = (double *)&p[1];
      p->aAvgdl = &p->aIdf[nPhrase];

      /* Determine the IDF weight for each phrase in the query. */
      rc = sqlite4_mi_total_rows(pCtx, &N);
      for(i=0; rc==SQLITE4_OK && i<nPhrase; i++){
        rc = sqlite4_mi_row_count(pCtx, -1, -1, i, &ni);
        if( rc==SQLITE4_OK ){
          assert( ni<=N );
          p->aIdf[i] = log((0.5 + N - ni) / (0.5 + ni));
        }
      }

      /* Determine the average document length. For bm25f, determine the
      ** average length of each field.  */
      if( rc==SQLITE4_OK ){
        int iField;
        for(iField=0; iField<nField; iField++){
          int nTotal;
          if( bm25mask & BM25_FCOLUMNS ){
            rc = sqlite4_mi_total_size(pCtx, iField, -1, &nTotal);
          }else if( bm25mask & BM25_FSTREAMS ){
            rc = sqlite4_mi_total_size(pCtx, -1, iField, &nTotal);
          }else{
            rc = sqlite4_mi_total_size(pCtx, -1, -1, &nTotal);
          }
          if( rc==SQLITE4_OK ){
            p->aAvgdl[iField] = (double)nTotal / (double)N;
          }
        }
      }
    }
  }

  if( bExplain ){
    int iField;
    zExplain = sqlite4MAppendf(
        db, zExplain, "%s<table><tr><th>Stream<th>Scale<th>avgsl<th>sl", 
        zExplain
    );
    for(i=0; i<p->nPhrase; i++){
      zExplain = sqlite4MAppendf(
        db, zExplain, "%s<th>tf</span><sub>%d</sub>", zExplain, i
      );
    }
    for(iField=0; rc==SQLITE4_OK && iField<nField; iField++){
      int dl, tf;
      double scale;
      rc = fts5GetSizeFreqScale(
          pCtx, nArg, apArg, bm25mask, 0, iField, &dl, &tf, &scale
      );
      zExplain = sqlite4MAppendf(
          db, zExplain, "%s<tr><td>%d<td>%.2f<td>%.2f<td>%d", 
          zExplain, iField, scale, p->aAvgdl[iField], dl
      );
      for(i=0; rc==SQLITE4_OK && i<p->nPhrase; i++){
        rc = fts5GetSizeFreqScale(
            pCtx, nArg, apArg, bm25mask, i, iField, &dl, &tf, &scale
        );
        zExplain = sqlite4MAppendf(db, zExplain, "%s<td>%d", zExplain, tf);
      }
    }
    zExplain = sqlite4MAppendf(
        db, zExplain, "%s</table><table><tr><th>Phrase<th>IDF", zExplain
    );
    for(i=0; i<nField; i++){
      zExplain = sqlite4MAppendf(
        db, zExplain, "%s<th><span style=text-decoration:overline>"
        "tf</span><sub>s%d</sub>", zExplain, i
      );
    }
    zExplain = sqlite4MAppendf(db, zExplain, "%s<th>rank", zExplain);
  }

  for(i=0; rc==SQLITE4_OK && i<p->nPhrase; i++){
    int iField;
    double tfns = 0.0;            /* Sum of tfn for all fields */
    double prank;                 /* Contribution to rank of this phrase */

    if( bExplain ){
      zExplain = sqlite4MAppendf(
        db, zExplain, "%s<tr><td>%d<td>%.2f", zExplain, i, p->aIdf[i]
      );
    }

    for(iField = 0; iField<nField; iField++){
      double scale = 1.0;
      int tf;                       /* Count of phrase i in row (term freq.) */
      double tfn;                   /* Normalized term frequency */
      int dl;                       /* Tokens in this row (document length) */


      double B;                     /* B from formula 3.20 */

      /* Set variable tf to the total number of occurrences of phrase iPhrase
      ** in this row/field. And dl to the number of tokens in the current 



      ** row/field. */
      rc = fts5GetSizeFreqScale(
          pCtx, nArg, apArg, bm25mask, i, iField, &dl, &tf, &scale
      );



      B = (1.0 - b) + b * (double)dl / p->aAvgdl[iField];    /* 3.20 */
      tfn = scale * (double)tf / B;
      tfns += tfn;                                           /* 3.19 */






      if( bExplain ){
        zExplain = sqlite4MAppendf(db, zExplain, "%s<td>%.2f", zExplain, tfn);
      }
    }

    prank = p->aIdf[i] * tfns / (k1 + tfns);                 /* 3.21 */
    if( bExplain ){





      zExplain = sqlite4MAppendf(db, zExplain, "%s<td>%.2f", zExplain, prank);
    }

    /* Add it to the overall rank */
    rank += prank;
  }

  if( rc==SQLITE4_OK ){
    if( bExplain ){

      zExplain = sqlite4MAppendf(

          db, zExplain, "%s</table><b>overall rank=%.2f</b>", zExplain, rank
      );

      sqlite4_result_text(pCtx, zExplain, -1, SQLITE4_TRANSIENT);

    }else{
      sqlite4_result_double(pCtx, rank);
    }
  }else{
    sqlite4_result_error_code(pCtx, rc);
  }
  sqlite4DbFree(db, zExplain);
}

typedef struct Snippet Snippet;
typedef struct SnippetText SnippetText;

struct Snippet {
  int iCol;
................................................................................

  sqlite4_free(pEnv, aBuf);
  return SQLITE4_OK;
}

int sqlite4InitFts5Func(sqlite4 *db){
  int rc;
  int i;
  sqlite4_env *pEnv = sqlite4_db_env(db);

  struct RankFunction {
    const char *zName;
    int mask;
  } aRank[] = {
    { "rank",  0 },
    { "erank",  BM25_EXPLAIN },
    { "rankc",  BM25_FCOLUMNS },
    { "erankc", BM25_FCOLUMNS|BM25_EXPLAIN },
    { "ranks",  BM25_FSTREAMS },
    { "eranks", BM25_FSTREAMS|BM25_EXPLAIN }
  };

  rc = sqlite4_create_tokenizer(db, "simple", (void *)pEnv, 
      fts5SimpleCreate, fts5SimpleTokenize, fts5SimpleDestroy
  );
  if( rc==SQLITE4_OK ){
    rc = sqlite4_create_mi_function(
        db, "snippet", -1, SQLITE4_UTF8, 0, fts5Snippet, 0);
  }

  for(i=0; rc==SQLITE4_OK && i<ArraySize(aRank); i++){
    void *p = SQLITE4_INT_TO_PTR(aRank[i].mask);
    const char *z = aRank[i].zName;
    rc = sqlite4_create_mi_function(db, z, -1, SQLITE4_UTF8, p, fts5Rank, 0);
  }


  return rc;
}