/ Check-in [4cc048c3]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add hidden column "rank". Currently this always returns the same value as the bm25() function.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 4cc048c3651e830a6aeded924c7f3a60b634e133
User & Date: dan 2014-07-30 19:41:58
Context
2014-07-30
20:26
Fix things so that the fts5 extension API works with "ORDER BY rank" queries. check-in: f1b4e1a9 user: dan tags: fts5
19:41
Add hidden column "rank". Currently this always returns the same value as the bm25() function. check-in: 4cc048c3 user: dan tags: fts5
2014-07-28
20:14
Add the "loadfts" program, for performance testing the loading of data into fts3/fts4/fts5 tables. check-in: 770b9540 user: dan tags: fts5
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.c.

52
53
54
55
56
57
58

59
60
61
62
63
64














65
66
67
68
69
70
71
72
73

74
75

76
77
78
79
80
81
82
83
84
85
86
87
...
223
224
225
226
227
228
229
230
231

232

233
234
235
236
237
238
239
...
280
281
282
283
284
285
286
287



288







289

290
291
292
293
294
295
296
...
337
338
339
340
341
342
343







344

345
346
347
348
349
350
351
...
353
354
355
356
357
358
359















360
361
362
363
364
365
366
...
367
368
369
370
371
372
373

374














375
376
377
378
379
380
381


























382
383
384
385


386
387
388






389







390
391
392
393
394
395
396
...
410
411
412
413
414
415
416
417
418
419
420

421
422
423
424
425

426









427
428
429
430
431

432
433

434


435

436
437
438
439
440

441
442
443
444
445
446
447
...
460
461
462
463
464
465
466


467
468
469
470









471
472
473
474
475
476
477
...
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
...
569
570
571
572
573
574
575








576
577
578
579
580
581
582
583
...
835
836
837
838
839
840
841













842
843
844
845
846
847
848
...
857
858
859
860
861
862
863
864
865
866






























867

868







869
870
871
872
873
874
875
*/
struct Fts5Table {
  sqlite3_vtab base;              /* Base class used by SQLite core */
  Fts5Config *pConfig;            /* Virtual table configuration */
  Fts5Index *pIndex;              /* Full-text index */
  Fts5Storage *pStorage;          /* Document store */
  Fts5Global *pGlobal;            /* Global (connection wide) data */

};

struct Fts5MatchPhrase {
  Fts5Buffer *pPoslist;           /* Pointer to current poslist */
  int nTerm;                      /* Size of phrase in terms */
};















/*
** Virtual-table cursor object.
*/
struct Fts5Cursor {
  sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
  int idxNum;                     /* idxNum passed to xFilter() */
  sqlite3_stmt *pStmt;            /* Statement used to read %_content */
  Fts5Expr *pExpr;                /* Expression for MATCH queries */

  int csrflags;                   /* Mask of cursor flags (see below) */
  Fts5Cursor *pNext;              /* Next cursor in Fts5Cursor.pCsr list */


  /* Variables used by auxiliary functions */
  i64 iCsrId;                     /* Cursor id */
  Fts5Auxiliary *pAux;            /* Currently executing extension function */
  Fts5Auxdata *pAuxdata;          /* First in linked list of aux-data */
  int *aColumnSize;               /* Values for xColumnSize() */
};

/*
** Values for Fts5Cursor.csrflags
*/
#define FTS5CSR_REQUIRE_CONTENT   0x01
................................................................................
){
  return fts5InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
}

/*
** The three query plans xBestIndex may choose between.
*/
#define FTS5_PLAN_SCAN    1       /* No usable constraint */
#define FTS5_PLAN_MATCH   2       /* (<tbl> MATCH ?) */

#define FTS5_PLAN_ROWID   3       /* (rowid = ?) */


#define FTS5_PLAN(idxNum) ((idxNum) & 0x7)

#define FTS5_ORDER_DESC   8       /* ORDER BY rowid DESC */
#define FTS5_ORDER_ASC   16       /* ORDER BY rowid ASC */

/*
................................................................................
  if( iCons>=0 ){
    pInfo->aConstraintUsage[iCons].argvIndex = 1;
    pInfo->aConstraintUsage[iCons].omit = 1;
  }else{
    pInfo->estimatedCost = 10000000.0;
  }

  if( pInfo->nOrderBy==1 && pInfo->aOrderBy[0].iColumn<0 ){



    pInfo->orderByConsumed = 1;







    ePlan |= pInfo->aOrderBy[0].desc ? FTS5_ORDER_DESC : FTS5_ORDER_ASC;

  }
   
  pInfo->idxNum = ePlan;
  return SQLITE_OK;
}

/*
................................................................................
  Fts5Auxdata *pData;
  Fts5Auxdata *pNext;

  if( pCsr->pStmt ){
    int eStmt = fts5StmtType(pCsr->idxNum);
    sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
  }







  sqlite3Fts5ExprFree(pCsr->pExpr);


  for(pData=pCsr->pAuxdata; pData; pData=pNext){
    pNext = pData->pNext;
    if( pData->xDelete ) pData->xDelete(pData->pPtr);
    sqlite3_free(pData);
  }

................................................................................
  for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
  *pp = pCsr->pNext;

  sqlite3_free(pCsr);
  return SQLITE_OK;
}

















/*
** Advance the cursor to the next row in the table that matches the 
** search criteria.
**
** Return SQLITE_OK if nothing goes wrong.  SQLITE_OK is returned
** even if we reach end-of-file.  The fts5EofMethod() will be called
................................................................................
** subsequently to determine whether or not an EOF was hit.
*/
static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int ePlan = FTS5_PLAN(pCsr->idxNum);
  int rc = SQLITE_OK;


  if( ePlan!=FTS5_PLAN_MATCH ){














    rc = sqlite3_step(pCsr->pStmt);
    if( rc!=SQLITE_ROW ){
      CsrFlagSet(pCsr, FTS5CSR_EOF);
      rc = sqlite3_reset(pCsr->pStmt);
    }else{
      rc = SQLITE_OK;
    }


























  }else{
    rc = sqlite3Fts5ExprNext(pCsr->pExpr);
    if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
      CsrFlagSet(pCsr, FTS5CSR_EOF);


    }
    CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
  }






  







  return rc;
}

static int fts5CursorFirst(Fts5Table *pTab, Fts5Cursor *pCsr, int bAsc){
  int rc;
  rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, bAsc);
  if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
................................................................................
  int idxNum,                     /* Strategy index */
  const char *idxStr,             /* Unused */
  int nVal,                       /* Number of elements in apVal */
  sqlite3_value **apVal           /* Arguments for the indexing scheme */
){
  Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int rc = SQLITE_OK;
  int ePlan = FTS5_PLAN(idxNum);
  int eStmt = fts5StmtType(idxNum);
  int bAsc = ((idxNum & FTS5_ORDER_ASC) ? 1 : 0);


  pCsr->idxNum = idxNum;
  assert( pCsr->pStmt==0 );
  assert( pCsr->pExpr==0 );
  assert( pCsr->csrflags==0 );











  rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt);
  if( rc==SQLITE_OK ){
    if( ePlan==FTS5_PLAN_MATCH ){
      char **pzErr = &pTab->base.zErrMsg;
      const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);

      rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr);
      if( rc==SQLITE_OK ){

        rc = fts5CursorFirst(pTab, pCsr, bAsc);


      }

    }else{
      if( ePlan==FTS5_PLAN_ROWID ){
        sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
      }
      rc = fts5NextMethod(pCursor);

    }
  }

  return rc;
}

/* 
................................................................................
** rowid should be written to *pRowid.
*/
static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int ePlan = FTS5_PLAN(pCsr->idxNum);
  
  assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );


  if( ePlan!=FTS5_PLAN_MATCH ){
    *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
  }else{
    *pRowid = sqlite3Fts5ExprRowid(pCsr->pExpr);









  }

  return SQLITE_OK;
}

/*
** If the cursor requires seeking (bSeekRequired flag is set), seek it.
................................................................................
    }else{
      rc = sqlite3_reset(pCsr->pStmt);
      if( rc==SQLITE_OK ){
        rc = SQLITE_CORRUPT_VTAB;
      }
    }
  }
  return rc;
}

/* 
** This is the xColumn method, called by SQLite to request a value from
** the row that the supplied cursor currently points to.
*/
static int fts5ColumnMethod(
  sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
  sqlite3_context *pCtx,          /* Context for sqlite3_result_xxx() calls */
  int iCol                        /* Index of column to read value from */
){
  Fts5Config *pConfig = ((Fts5Table*)(pCursor->pVtab))->pConfig;
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int rc = SQLITE_OK;
  
  assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );

  if( iCol==pConfig->nCol ){
    /* User is requesting the value of the special column with the same name
    ** as the table. Return the cursor integer id number. This value is only
    ** useful in that it may be passed as the first argument to an FTS5
    ** auxiliary function.  */
    sqlite3_result_int64(pCtx, pCsr->iCsrId);
  }else{
    rc = fts5SeekCursor(pCsr);
    if( rc==SQLITE_OK ){
      sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1));
    }
  }
  return rc;
}

/*
** This function is called to handle an FTS INSERT command. In other words,
** an INSERT statement of the form:
**
................................................................................
){
  Fts5Table *pTab = (Fts5Table*)pVtab;
  Fts5Config *pConfig = pTab->pConfig;
  int eType0;                     /* value_type() of apVal[0] */
  int eConflict;                  /* ON CONFLICT for this DML */
  int rc = SQLITE_OK;             /* Return code */









  assert( nArg==1 || nArg==(2 + pConfig->nCol + 1) );

  if( nArg>1 && SQLITE_NULL!=sqlite3_value_type(apVal[2 + pConfig->nCol]) ){
    return fts5SpecialCommand(pTab, apVal[2 + pConfig->nCol]);
  }

  eType0 = sqlite3_value_type(apVal[0]);
  eConflict = sqlite3_vtab_on_conflict(pConfig->db);
................................................................................
      }
    }
  }

  fts5CloseMethod((sqlite3_vtab_cursor*)pNew);
  return rc;
}














static void fts5ApiCallback(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){

................................................................................
  for(pCsr=pAux->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
    if( pCsr->iCsrId==iCsrId ) break;
  }
  if( pCsr==0 ){
    char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId);
    sqlite3_result_error(context, zErr, -1);
  }else{
    assert( pCsr->pAux==0 );
    pCsr->pAux = pAux;
    pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc-1, &argv[1]);






























    pCsr->pAux = 0;

  }







}


/*
** This routine implements the xFindFunction method for the FTS3
** virtual table.
*/







>






>
>
>
>
>
>
>
>
>
>
>
>
>
>









>


>




|







 







|
|
>
|
>







 







|
>
>
>
|
>
>
>
>
>
>
>
|
>







 







>
>
>
>
>
>
>
|
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
|
|
|
|
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

<
<
<
>
>
|
<
|
>
>
>
>
>
>
|
>
>
>
>
>
>
>







 







<
<
<

>

<



>

>
>
>
>
>
>
>
>
>
|
|
|
|
|
>
|
|
>
|
>
>
|
>
|
|
|
|
|
>







 







>
>
|
<
<
|
>
>
>
>
>
>
>
>
>







 







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







>
>
>
>
>
>
>
>
|







 







>
>
>
>
>
>
>
>
>
>
>
>
>







 







<
<
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
|
>
>
>
>
>
>
>







52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
...
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
...
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
...
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
...
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
...
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476



477
478
479

480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
...
515
516
517
518
519
520
521



522
523
524

525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
...
578
579
580
581
582
583
584
585
586
587


588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
...
617
618
619
620
621
622
623






























624
625
626
627
628
629
630
...
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
...
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
...
975
976
977
978
979
980
981


982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
*/
struct Fts5Table {
  sqlite3_vtab base;              /* Base class used by SQLite core */
  Fts5Config *pConfig;            /* Virtual table configuration */
  Fts5Index *pIndex;              /* Full-text index */
  Fts5Storage *pStorage;          /* Document store */
  Fts5Global *pGlobal;            /* Global (connection wide) data */
  Fts5Cursor *pSortCsr;           /* Sort data from this cursor */
};

struct Fts5MatchPhrase {
  Fts5Buffer *pPoslist;           /* Pointer to current poslist */
  int nTerm;                      /* Size of phrase in terms */
};

/*
** Variable pStmt is set to a compiled SQL statement of the form:
**
**   SELECT rowid, <fts> FROM <fts> ORDER BY +rank;
**
*/
struct Fts5Sorter {
  sqlite3_stmt *pStmt;
  i64 iRowid;                     /* Current rowid */
  u8 *aPoslist;                   /* Position lists for current row */
  int nIdx;                       /* Number of entries in aIdx[] */
  int aIdx[0];                    /* Offsets into aPoslist for current row */
};

/*
** Virtual-table cursor object.
*/
struct Fts5Cursor {
  sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
  int idxNum;                     /* idxNum passed to xFilter() */
  sqlite3_stmt *pStmt;            /* Statement used to read %_content */
  Fts5Expr *pExpr;                /* Expression for MATCH queries */
  Fts5Sorter *pSorter;            /* Sorter for "ORDER BY rank" queries */
  int csrflags;                   /* Mask of cursor flags (see below) */
  Fts5Cursor *pNext;              /* Next cursor in Fts5Cursor.pCsr list */
  Fts5Auxiliary *pRank;           /* Rank callback (or NULL) */

  /* Variables used by auxiliary functions */
  i64 iCsrId;                     /* Cursor id */
  Fts5Auxiliary *pAux;            /* Currently executing extension function */
  Fts5Auxdata *pAuxdata;          /* First in linked list of saved aux-data */
  int *aColumnSize;               /* Values for xColumnSize() */
};

/*
** Values for Fts5Cursor.csrflags
*/
#define FTS5CSR_REQUIRE_CONTENT   0x01
................................................................................
){
  return fts5InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
}

/*
** The three query plans xBestIndex may choose between.
*/
#define FTS5_PLAN_SCAN           1       /* No usable constraint */
#define FTS5_PLAN_MATCH          2       /* (<tbl> MATCH ?) */
#define FTS5_PLAN_SORTED_MATCH   3       /* (<tbl> MATCH ? ORDER BY rank) */
#define FTS5_PLAN_ROWID          4       /* (rowid = ?) */
#define FTS5_PLAN_SOURCE         5       /* A source cursor for SORTED_MATCH */

#define FTS5_PLAN(idxNum) ((idxNum) & 0x7)

#define FTS5_ORDER_DESC   8       /* ORDER BY rowid DESC */
#define FTS5_ORDER_ASC   16       /* ORDER BY rowid ASC */

/*
................................................................................
  if( iCons>=0 ){
    pInfo->aConstraintUsage[iCons].argvIndex = 1;
    pInfo->aConstraintUsage[iCons].omit = 1;
  }else{
    pInfo->estimatedCost = 10000000.0;
  }

  if( pInfo->nOrderBy==1 ){
    int iSort = pInfo->aOrderBy[0].iColumn;
    if( iSort<0 ){
      /* ORDER BY rowid [ASC|DESC] */
      pInfo->orderByConsumed = 1;
    }else if( iSort==(pConfig->nCol+1) && ePlan==FTS5_PLAN_MATCH ){
      /* ORDER BY rank [ASC|DESC] */
      pInfo->orderByConsumed = 1;
      ePlan = FTS5_PLAN_SORTED_MATCH;
    }

    if( pInfo->orderByConsumed ){
      ePlan |= pInfo->aOrderBy[0].desc ? FTS5_ORDER_DESC : FTS5_ORDER_ASC;
    }
  }
   
  pInfo->idxNum = ePlan;
  return SQLITE_OK;
}

/*
................................................................................
  Fts5Auxdata *pData;
  Fts5Auxdata *pNext;

  if( pCsr->pStmt ){
    int eStmt = fts5StmtType(pCsr->idxNum);
    sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
  }
  if( pCsr->pSorter ){
    Fts5Sorter *pSorter = pCsr->pSorter;
    sqlite3_finalize(pSorter->pStmt);
    sqlite3_free(pSorter);
  }
  
  if( pCsr->idxNum!=FTS5_PLAN_SOURCE ){
    sqlite3Fts5ExprFree(pCsr->pExpr);
  }

  for(pData=pCsr->pAuxdata; pData; pData=pNext){
    pNext = pData->pNext;
    if( pData->xDelete ) pData->xDelete(pData->pPtr);
    sqlite3_free(pData);
  }

................................................................................
  for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
  *pp = pCsr->pNext;

  sqlite3_free(pCsr);
  return SQLITE_OK;
}

static int fts5SorterNext(Fts5Cursor *pCsr){
  Fts5Sorter *pSorter = pCsr->pSorter;
  int rc;

  rc = sqlite3_step(pSorter->pStmt);
  if( rc==SQLITE_DONE ){
    rc = SQLITE_OK;
    CsrFlagSet(pCsr, FTS5CSR_EOF);
  }else if( rc==SQLITE_ROW ){
    rc = SQLITE_OK;
    pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0);
  }

  return rc;
}

/*
** Advance the cursor to the next row in the table that matches the 
** search criteria.
**
** Return SQLITE_OK if nothing goes wrong.  SQLITE_OK is returned
** even if we reach end-of-file.  The fts5EofMethod() will be called
................................................................................
** subsequently to determine whether or not an EOF was hit.
*/
static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int ePlan = FTS5_PLAN(pCsr->idxNum);
  int rc = SQLITE_OK;

  switch( ePlan ){
    case FTS5_PLAN_MATCH:
    case FTS5_PLAN_SOURCE:
      rc = sqlite3Fts5ExprNext(pCsr->pExpr);
      if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
        CsrFlagSet(pCsr, FTS5CSR_EOF);
      }
      CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
      break;

    case FTS5_PLAN_SORTED_MATCH: {
      rc = fts5SorterNext(pCsr);
      break;
    }

    default:
      rc = sqlite3_step(pCsr->pStmt);
      if( rc!=SQLITE_ROW ){
        CsrFlagSet(pCsr, FTS5CSR_EOF);
        rc = sqlite3_reset(pCsr->pStmt);
      }else{
        rc = SQLITE_OK;
      }
      break;
  }
  
  return rc;
}

static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bAsc){
  Fts5Config *pConfig = pTab->pConfig;
  Fts5Sorter *pSorter;
  int nPhrase;
  int nByte;
  int rc = SQLITE_OK;
  char *zSql;
  
  nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
  nByte = sizeof(Fts5Sorter) + sizeof(int) * nPhrase;
  pSorter = (Fts5Sorter*)sqlite3_malloc(nByte);
  if( pSorter==0 ) return SQLITE_NOMEM;
  memset(pSorter, 0, nByte);

  zSql = sqlite3_mprintf("SELECT rowid, %Q FROM %Q.%Q ORDER BY +%s %s",
      pConfig->zName, pConfig->zDb, pConfig->zName, FTS5_RANK_NAME,
      bAsc ? "ASC" : "DESC"
  );
  if( zSql==0 ){
    rc = SQLITE_NOMEM;
  }else{



    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pSorter->pStmt, 0);
    sqlite3_free(zSql);
  }


  pCsr->pSorter = pSorter;
  if( rc==SQLITE_OK ){
    assert( pTab->pSortCsr==0 );
    pTab->pSortCsr = pCsr;
    rc = fts5SorterNext(pCsr);
    pTab->pSortCsr = 0;
  }

  if( rc!=SQLITE_OK ){
    sqlite3_finalize(pSorter->pStmt);
    sqlite3_free(pSorter);
    pCsr->pSorter = 0;
  }

  return rc;
}

static int fts5CursorFirst(Fts5Table *pTab, Fts5Cursor *pCsr, int bAsc){
  int rc;
  rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, bAsc);
  if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
................................................................................
  int idxNum,                     /* Strategy index */
  const char *idxStr,             /* Unused */
  int nVal,                       /* Number of elements in apVal */
  sqlite3_value **apVal           /* Arguments for the indexing scheme */
){
  Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;



  int bAsc = ((idxNum & FTS5_ORDER_ASC) ? 1 : 0);
  int rc = SQLITE_OK;


  assert( pCsr->pStmt==0 );
  assert( pCsr->pExpr==0 );
  assert( pCsr->csrflags==0 );
  assert( pCsr->pRank==0 );

  if( pTab->pSortCsr ){
    pCsr->idxNum = FTS5_PLAN_SOURCE;
    pCsr->pRank = pTab->pSortCsr->pRank;
    pCsr->pExpr = pTab->pSortCsr->pExpr;
    rc = fts5CursorFirst(pTab, pCsr, bAsc);
  }else{
    int ePlan = FTS5_PLAN(idxNum);
    int eStmt = fts5StmtType(idxNum);
    pCsr->idxNum = idxNum;
    rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt);
    if( rc==SQLITE_OK ){
      if( ePlan==FTS5_PLAN_MATCH || ePlan==FTS5_PLAN_SORTED_MATCH ){
        char **pzErr = &pTab->base.zErrMsg;
        const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
        pCsr->pRank = pTab->pGlobal->pAux;
        rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr);
        if( rc==SQLITE_OK ){
          if( ePlan==FTS5_PLAN_MATCH ){
            rc = fts5CursorFirst(pTab, pCsr, bAsc);
          }else{
            rc = fts5CursorFirstSorted(pTab, pCsr, bAsc);
          }
        }
      }else{
        if( ePlan==FTS5_PLAN_ROWID ){
          sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
        }
        rc = fts5NextMethod(pCursor);
      }
    }
  }

  return rc;
}

/* 
................................................................................
** rowid should be written to *pRowid.
*/
static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int ePlan = FTS5_PLAN(pCsr->idxNum);
  
  assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
  switch( ePlan ){
    case FTS5_PLAN_SOURCE:
    case FTS5_PLAN_MATCH:


      *pRowid = sqlite3Fts5ExprRowid(pCsr->pExpr);
      break;

    case FTS5_PLAN_SORTED_MATCH:
      *pRowid = pCsr->pSorter->iRowid;
      break;

    default:
      *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
      break;
  }

  return SQLITE_OK;
}

/*
** If the cursor requires seeking (bSeekRequired flag is set), seek it.
................................................................................
    }else{
      rc = sqlite3_reset(pCsr->pStmt);
      if( rc==SQLITE_OK ){
        rc = SQLITE_CORRUPT_VTAB;
      }
    }
  }






























  return rc;
}

/*
** This function is called to handle an FTS INSERT command. In other words,
** an INSERT statement of the form:
**
................................................................................
){
  Fts5Table *pTab = (Fts5Table*)pVtab;
  Fts5Config *pConfig = pTab->pConfig;
  int eType0;                     /* value_type() of apVal[0] */
  int eConflict;                  /* ON CONFLICT for this DML */
  int rc = SQLITE_OK;             /* Return code */

  /* A delete specifies a single argument - the rowid of the row to remove.
  ** Update and insert operations pass:
  **
  **   1. The "old" rowid, or NULL.
  **   2. The "new" rowid.
  **   3. Values for each of the nCol matchable columns.
  **   4. Values for the two hidden columns (<tablename> and "rank").
  */
  assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) );

  if( nArg>1 && SQLITE_NULL!=sqlite3_value_type(apVal[2 + pConfig->nCol]) ){
    return fts5SpecialCommand(pTab, apVal[2 + pConfig->nCol]);
  }

  eType0 = sqlite3_value_type(apVal[0]);
  eConflict = sqlite3_vtab_on_conflict(pConfig->db);
................................................................................
      }
    }
  }

  fts5CloseMethod((sqlite3_vtab_cursor*)pNew);
  return rc;
}

static void fts5ApiInvoke(
  Fts5Auxiliary *pAux,
  Fts5Cursor *pCsr,
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  assert( pCsr->pAux==0 );
  pCsr->pAux = pAux;
  pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv);
  pCsr->pAux = 0;
}

static void fts5ApiCallback(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){

................................................................................
  for(pCsr=pAux->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
    if( pCsr->iCsrId==iCsrId ) break;
  }
  if( pCsr==0 ){
    char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId);
    sqlite3_result_error(context, zErr, -1);
  }else{


    fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]);
  }
}

/* 
** This is the xColumn method, called by SQLite to request a value from
** the row that the supplied cursor currently points to.
*/
static int fts5ColumnMethod(
  sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
  sqlite3_context *pCtx,          /* Context for sqlite3_result_xxx() calls */
  int iCol                        /* Index of column to read value from */
){
  Fts5Config *pConfig = ((Fts5Table*)(pCursor->pVtab))->pConfig;
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int rc = SQLITE_OK;
  
  assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );

  if( iCol==pConfig->nCol ){
    if( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SOURCE ){
      /* todo */
    }else{
      /* User is requesting the value of the special column with the same name
      ** as the table. Return the cursor integer id number. This value is only
      ** useful in that it may be passed as the first argument to an FTS5
      ** auxiliary function.  */
      sqlite3_result_int64(pCtx, pCsr->iCsrId);
    }
  }else if( iCol==pConfig->nCol+1 ){
    /* The value of the "rank" column. */
    if( pCsr->pRank ){
      fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, 0, 0);
    }
  }else{
    rc = fts5SeekCursor(pCsr);
    if( rc==SQLITE_OK ){
      sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1));
    }
  }
  return rc;
}


/*
** This routine implements the xFindFunction method for the FTS3
** virtual table.
*/

Changes to ext/fts5/fts5Int.h.

24
25
26
27
28
29
30



31
32
33
34
35
36
37
...
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
...
419
420
421
422
423
424
425
426











427
428
429
** less than 32. If it is set to anything large than that, an #error
** directive in fts5_index.c will cause the build to fail.
*/
#define FTS5_MAX_PREFIX_INDEXES 31

#define FTS5_DEFAULT_NEARDIST 10




/**************************************************************************
** Interface to code in fts5_config.c. fts5_config.c contains contains code
** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
*/

typedef struct Fts5Config Fts5Config;

................................................................................
void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);

void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseSetColumn(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);


/*
** End of interface to code in fts5_expr.c.
**************************************************************************/


/**************************************************************************
** Interface to code in fts5.c. 
................................................................................

/**************************************************************************
** Interface to code in fts5_aux.c. 
*/

int sqlite3Fts5AuxInit(Fts5Global*);
/*
** End of interface to code in fts5_expr.c.











**************************************************************************/

#endif







>
>
>







 







<







 







|
>
>
>
>
>
>
>
>
>
>
>



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
...
393
394
395
396
397
398
399

400
401
402
403
404
405
406
...
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
** less than 32. If it is set to anything large than that, an #error
** directive in fts5_index.c will cause the build to fail.
*/
#define FTS5_MAX_PREFIX_INDEXES 31

#define FTS5_DEFAULT_NEARDIST 10

/* Name of rank column */
#define FTS5_RANK_NAME "rank"

/**************************************************************************
** Interface to code in fts5_config.c. fts5_config.c contains contains code
** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
*/

typedef struct Fts5Config Fts5Config;

................................................................................
void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);

void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseSetColumn(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);


/*
** End of interface to code in fts5_expr.c.
**************************************************************************/


/**************************************************************************
** Interface to code in fts5.c. 
................................................................................

/**************************************************************************
** Interface to code in fts5_aux.c. 
*/

int sqlite3Fts5AuxInit(Fts5Global*);
/*
** End of interface to code in fts5_aux.c.
**************************************************************************/

/**************************************************************************
** Interface to code in fts5_sorter.c. 
*/
typedef struct Fts5Sorter Fts5Sorter;

int sqlite3Fts5SorterNew(Fts5Expr *pExpr, Fts5Sorter **pp);

/*
** End of interface to code in fts5_sorter.c.
**************************************************************************/

#endif

Changes to ext/fts5/fts5_aux.c.

952
953
954
955
956
957
958
959
960
961
962

963
964
965
966
967
968
969
int sqlite3Fts5AuxInit(Fts5Global *pGlobal){
  struct Builtin {
    const char *zFunc;            /* Function name (nul-terminated) */
    void *pUserData;              /* User-data pointer */
    fts5_extension_function xFunc;/* Callback function */
    void (*xDestroy)(void*);      /* Destructor function */
  } aBuiltin [] = {
    { "bm25",      0, fts5Bm25Function,    0 },
    { "bm25debug", (void*)1, fts5Bm25Function,    0 },
    { "snippet",   0, fts5SnippetFunction, 0 },
    { "fts5_test", 0, fts5TestFunction,    0 },

  };

  int rc = SQLITE_OK;             /* Return code */
  int i;                          /* To iterate through builtin functions */

  for(i=0; rc==SQLITE_OK && i<sizeof(aBuiltin)/sizeof(aBuiltin[0]); i++){
    rc = sqlite3Fts5CreateAux(pGlobal, 







<



>







952
953
954
955
956
957
958

959
960
961
962
963
964
965
966
967
968
969
int sqlite3Fts5AuxInit(Fts5Global *pGlobal){
  struct Builtin {
    const char *zFunc;            /* Function name (nul-terminated) */
    void *pUserData;              /* User-data pointer */
    fts5_extension_function xFunc;/* Callback function */
    void (*xDestroy)(void*);      /* Destructor function */
  } aBuiltin [] = {

    { "bm25debug", (void*)1, fts5Bm25Function,    0 },
    { "snippet",   0, fts5SnippetFunction, 0 },
    { "fts5_test", 0, fts5TestFunction,    0 },
    { "bm25",      0, fts5Bm25Function,    0 },
  };

  int rc = SQLITE_OK;             /* Return code */
  int i;                          /* To iterate through builtin functions */

  for(i=0; rc==SQLITE_OK && i<sizeof(aBuiltin)/sizeof(aBuiltin[0]); i++){
    rc = sqlite3Fts5CreateAux(pGlobal, 

Changes to ext/fts5/fts5_config.c.

161
162
163
164
165
166
167



168
169
170
171
172
173
174
175
...
185
186
187
188
189
190
191
192

193
194
195




196
197
198
199
200
201
202
...
245
246
247
248
249
250
251
252


253
254
255
256
257
258
259
  if( pRet==0 ) return SQLITE_NOMEM;
  memset(pRet, 0, sizeof(Fts5Config));
  pRet->db = db;

  pRet->azCol = (char**)sqlite3_malloc(sizeof(char*) * nArg);
  pRet->zDb = fts5Strdup(azArg[1]);
  pRet->zName = fts5Strdup(azArg[2]);



  if( pRet->azCol==0 || pRet->zDb==0 || pRet->zName==0 ){
    rc = SQLITE_NOMEM;
  }else{
    int i;
    for(i=3; rc==SQLITE_OK && i<nArg; i++){
      char *zDup = fts5Strdup(azArg[i]);
      if( zDup==0 ){
        rc = SQLITE_NOMEM;
................................................................................
            sqlite3Fts5Dequote(zArg);
            rc = fts5ConfigParseSpecial(pRet, zDup, zArg, pzErr);
            sqlite3_free(zDup);
            zDup = 0;
          }
        }

        /* If it is not a special directive, it must be a column name */

        if( zDup ){
          sqlite3Fts5Dequote(zDup);
          pRet->azCol[pRet->nCol++] = zDup;




        }
      }
    }
  }

  if( rc==SQLITE_OK && pRet->pTokenizer==0 ){
    rc = fts5ConfigDefaultTokenizer(pRet);
................................................................................
    zOld = zSql;
    zSql = sqlite3_mprintf("%s%s%Q", zOld, (i==0?"":", "), pConfig->azCol[i]);
    sqlite3_free(zOld);
  }

  if( zSql ){
    zOld = zSql;
    zSql = sqlite3_mprintf("%s, %Q HIDDEN)", zOld, pConfig->zName);


    sqlite3_free(zOld);
  }

  if( zSql==0 ){
    rc = SQLITE_NOMEM;
  }else{
    rc = sqlite3_declare_vtab(pConfig->db, zSql);







>
>
>
|







 







|
>



>
>
>
>







 







|
>
>







161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
...
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
...
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
  if( pRet==0 ) return SQLITE_NOMEM;
  memset(pRet, 0, sizeof(Fts5Config));
  pRet->db = db;

  pRet->azCol = (char**)sqlite3_malloc(sizeof(char*) * nArg);
  pRet->zDb = fts5Strdup(azArg[1]);
  pRet->zName = fts5Strdup(azArg[2]);
  if( sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
    *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
    rc = SQLITE_ERROR;
  }else if( pRet->azCol==0 || pRet->zDb==0 || pRet->zName==0 ){
    rc = SQLITE_NOMEM;
  }else{
    int i;
    for(i=3; rc==SQLITE_OK && i<nArg; i++){
      char *zDup = fts5Strdup(azArg[i]);
      if( zDup==0 ){
        rc = SQLITE_NOMEM;
................................................................................
            sqlite3Fts5Dequote(zArg);
            rc = fts5ConfigParseSpecial(pRet, zDup, zArg, pzErr);
            sqlite3_free(zDup);
            zDup = 0;
          }
        }

        /* If it is not a special directive, it must be a column name. In
        ** this case, check that it is not the reserved column name "rank". */
        if( zDup ){
          sqlite3Fts5Dequote(zDup);
          pRet->azCol[pRet->nCol++] = zDup;
          if( sqlite3_stricmp(zDup, FTS5_RANK_NAME)==0 ){
            *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zDup);
            rc = SQLITE_ERROR;
          }
        }
      }
    }
  }

  if( rc==SQLITE_OK && pRet->pTokenizer==0 ){
    rc = fts5ConfigDefaultTokenizer(pRet);
................................................................................
    zOld = zSql;
    zSql = sqlite3_mprintf("%s%s%Q", zOld, (i==0?"":", "), pConfig->azCol[i]);
    sqlite3_free(zOld);
  }

  if( zSql ){
    zOld = zSql;
    zSql = sqlite3_mprintf("%s, %Q HIDDEN, %s HIDDEN)", 
        zOld, pConfig->zName, FTS5_RANK_NAME
    );
    sqlite3_free(zOld);
  }

  if( zSql==0 ){
    rc = SQLITE_NOMEM;
  }else{
    rc = sqlite3_declare_vtab(pConfig->db, zSql);

Changes to ext/fts5/fts5_expr.c.

30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
...
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
...
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
...
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
...
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
....
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*);

struct Fts5Expr {
  Fts5Index *pIndex;
  Fts5ExprNode *pRoot;
  int bAsc;
  int nPhrase;                    /* Number of phrases in expression */
  Fts5ExprPhrase **apPhrase;      /* Pointers to phrase objects */
};

/*
** eType:
**   Expression node type. Always one of:
**
**       FTS5_AND                 (pLeft, pRight valid)
................................................................................
  if( sParse.rc==SQLITE_OK ){
    *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
    if( pNew==0 ){
      sParse.rc = SQLITE_NOMEM;
    }else{
      pNew->pRoot = sParse.pExpr;
      pNew->pIndex = 0;
      pNew->apPhrase = sParse.apPhrase;
      pNew->nPhrase = sParse.nPhrase;
      sParse.apPhrase = 0;
    }
  }

  sqlite3_free(sParse.apPhrase);
  *pzErr = sParse.zErr;
................................................................................
  /* Components of the new expression object */
  Fts5Expr *pNew;
  Fts5ExprPhrase **apPhrase;
  Fts5ExprNode *pNode;
  Fts5ExprNearset *pNear;
  Fts5ExprPhrase *pCopy;

  pOrig = pExpr->apPhrase[iPhrase];
  pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr));
  apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*));
  pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode));
  pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, 
      sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)
  );
  pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, 
................................................................................
  }

  if( rc==SQLITE_OK ){
    /* All the allocations succeeded. Put the expression object together. */
    pNew->pIndex = pExpr->pIndex;
    pNew->pRoot = pNode;
    pNew->nPhrase = 1;
    pNew->apPhrase = apPhrase;
    pNew->apPhrase[0] = pCopy;

    pNode->eType = FTS5_STRING;
    pNode->pNear = pNear;

    pNear->iCol = -1;
    pNear->nPhrase = 1;
    pNear->apPhrase[0] = pCopy;
................................................................................

/*
** Free the expression object passed as the only argument.
*/
void sqlite3Fts5ExprFree(Fts5Expr *p){
  if( p ){
    sqlite3Fts5ParseNodeFree(p->pRoot);
    sqlite3_free(p->apPhrase);
    sqlite3_free(p);
  }
}

/*
** All individual term iterators in pPhrase are guaranteed to be valid and
** pointing to the same rowid when this function is called. This function 
................................................................................
}

/*
** Return the number of terms in the iPhrase'th phrase in pExpr.
*/
int sqlite3Fts5ExprPhraseSize(Fts5Expr *pExpr, int iPhrase){
  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ) return 0;
  return pExpr->apPhrase[iPhrase]->nTerm;
}

/*
** This function is used to access the current position list for phrase
** iPhrase.
*/
int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){
  if( iPhrase>=0 && iPhrase<pExpr->nPhrase ){
    Fts5ExprPhrase *pPhrase = pExpr->apPhrase[iPhrase];
    Fts5ExprNode *pNode = pPhrase->pNode;
    if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid ){
      *pa = pPhrase->poslist.p;
      return pPhrase->poslist.n;
    }
  }
  *pa = 0;
  return 0;
}








|







 







|







 







|







 







|
|







 







|







 







|








|










30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
...
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
...
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
...
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
...
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
....
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*);

struct Fts5Expr {
  Fts5Index *pIndex;
  Fts5ExprNode *pRoot;
  int bAsc;
  int nPhrase;                    /* Number of phrases in expression */
  Fts5ExprPhrase **apExprPhrase;  /* Pointers to phrase objects */
};

/*
** eType:
**   Expression node type. Always one of:
**
**       FTS5_AND                 (pLeft, pRight valid)
................................................................................
  if( sParse.rc==SQLITE_OK ){
    *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
    if( pNew==0 ){
      sParse.rc = SQLITE_NOMEM;
    }else{
      pNew->pRoot = sParse.pExpr;
      pNew->pIndex = 0;
      pNew->apExprPhrase = sParse.apPhrase;
      pNew->nPhrase = sParse.nPhrase;
      sParse.apPhrase = 0;
    }
  }

  sqlite3_free(sParse.apPhrase);
  *pzErr = sParse.zErr;
................................................................................
  /* Components of the new expression object */
  Fts5Expr *pNew;
  Fts5ExprPhrase **apPhrase;
  Fts5ExprNode *pNode;
  Fts5ExprNearset *pNear;
  Fts5ExprPhrase *pCopy;

  pOrig = pExpr->apExprPhrase[iPhrase];
  pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr));
  apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*));
  pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode));
  pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, 
      sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)
  );
  pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, 
................................................................................
  }

  if( rc==SQLITE_OK ){
    /* All the allocations succeeded. Put the expression object together. */
    pNew->pIndex = pExpr->pIndex;
    pNew->pRoot = pNode;
    pNew->nPhrase = 1;
    pNew->apExprPhrase = apPhrase;
    pNew->apExprPhrase[0] = pCopy;

    pNode->eType = FTS5_STRING;
    pNode->pNear = pNear;

    pNear->iCol = -1;
    pNear->nPhrase = 1;
    pNear->apPhrase[0] = pCopy;
................................................................................

/*
** Free the expression object passed as the only argument.
*/
void sqlite3Fts5ExprFree(Fts5Expr *p){
  if( p ){
    sqlite3Fts5ParseNodeFree(p->pRoot);
    sqlite3_free(p->apExprPhrase);
    sqlite3_free(p);
  }
}

/*
** All individual term iterators in pPhrase are guaranteed to be valid and
** pointing to the same rowid when this function is called. This function 
................................................................................
}

/*
** Return the number of terms in the iPhrase'th phrase in pExpr.
*/
int sqlite3Fts5ExprPhraseSize(Fts5Expr *pExpr, int iPhrase){
  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ) return 0;
  return pExpr->apExprPhrase[iPhrase]->nTerm;
}

/*
** This function is used to access the current position list for phrase
** iPhrase.
*/
int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){
  if( iPhrase>=0 && iPhrase<pExpr->nPhrase ){
    Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase];
    Fts5ExprNode *pNode = pPhrase->pNode;
    if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid ){
      *pa = pPhrase->poslist.p;
      return pPhrase->poslist.n;
    }
  }
  *pa = 0;
  return 0;
}

Changes to test/fts5aa.test.

271
272
273
274
275
276
277









278
279
280
foreach {rowid x y} $d10 {
  do_execsql_test 10.3.$rowid.1 { INSERT INTO t1 VALUES($x, $y) }
  do_execsql_test 10.3.$rowid.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
}

do_execsql_test 10.4.1 { DELETE FROM t1 }
do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') }










finish_test








>
>
>
>
>
>
>
>
>



271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
foreach {rowid x y} $d10 {
  do_execsql_test 10.3.$rowid.1 { INSERT INTO t1 VALUES($x, $y) }
  do_execsql_test 10.3.$rowid.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
}

do_execsql_test 10.4.1 { DELETE FROM t1 }
do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') }

#-------------------------------------------------------------------------
#
do_catchsql_test 11.1 {
  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank);
} {1 {reserved fts5 column name: rank}}
do_catchsql_test 11.2 {
  CREATE VIRTUAL TABLE rank USING fts5(a, b, c);
} {1 {reserved fts5 table name: rank}}

finish_test

Changes to test/fts5ae.test.

257
258
259
260
261
262
263
264
265
266








267
268
269
270
271
  1 {a} {0 2}
  2 {b} {3 1}
  3 {c} {1 0}
  4 {d} {2 3}
  5 {g AND (e OR f)} {5 4}
  6 {j AND (h OR i)} {5 6}
} {
  do_execsql_test 8.2.$tn {
    SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY bm25(t8) DESC;
  } $res








}


finish_test








|


>
>
>
>
>
>
>
>





257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
  1 {a} {0 2}
  2 {b} {3 1}
  3 {c} {1 0}
  4 {d} {2 3}
  5 {g AND (e OR f)} {5 4}
  6 {j AND (h OR i)} {5 6}
} {
  do_execsql_test 8.2.$tn.1 {
    SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY bm25(t8) DESC;
  } $res

  do_execsql_test 8.2.$tn.2 {
    SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY +rank DESC;
  } $res

  do_execsql_test 8.3.$tn.3 {
    SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY rank DESC;
  } $res
}


finish_test