/ Check-in [f1b4e1a9]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix things so that the fts5 extension API works with "ORDER BY rank" queries.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: f1b4e1a98d49ecaba962beba16f8224175e4ba59
User & Date: dan 2014-07-30 20:26:24
Context
2014-07-31
11:57
Add further tests for the extension APIs with "ORDER BY rank" queries. check-in: 37a417d2 user: dan tags: fts5
2014-07-30
20:26
Fix things so that the fts5 extension API works with "ORDER BY rank" queries. check-in: f1b4e1a9 user: dan tags: fts5
19:41
Add hidden column "rank". Currently this always returns the same value as the bm25() function. check-in: 4cc048c3 user: dan tags: fts5
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.c.

69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
...
400
401
402
403
404
405
406




407

408









409
410
411
412
413
414
415
...
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
...
566
567
568
569
570
571
572















573
574
575
576
577
578
579
...
581
582
583
584
585
586
587
588
589
590
591
592

593
594
595
596
597
598
599
...
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
...
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
...
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
...
825
826
827
828
829
830
831






832

833
834
835
836
837
838
839
...
978
979
980
981
982
983
984

























985
986
987
988
989
990
991
...
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
**
**   SELECT rowid, <fts> FROM <fts> ORDER BY +rank;
**
*/
struct Fts5Sorter {
  sqlite3_stmt *pStmt;
  i64 iRowid;                     /* Current rowid */
  u8 *aPoslist;                   /* Position lists for current row */
  int nIdx;                       /* Number of entries in aIdx[] */
  int aIdx[0];                    /* Offsets into aPoslist for current row */
};

/*
** Virtual-table cursor object.
*/
................................................................................
  int rc;

  rc = sqlite3_step(pSorter->pStmt);
  if( rc==SQLITE_DONE ){
    rc = SQLITE_OK;
    CsrFlagSet(pCsr, FTS5CSR_EOF);
  }else if( rc==SQLITE_ROW ){




    rc = SQLITE_OK;

    pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0);









  }

  return rc;
}

/*
** Advance the cursor to the next row in the table that matches the 
................................................................................
  
  nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
  nByte = sizeof(Fts5Sorter) + sizeof(int) * nPhrase;
  pSorter = (Fts5Sorter*)sqlite3_malloc(nByte);
  if( pSorter==0 ) return SQLITE_NOMEM;
  memset(pSorter, 0, nByte);

  zSql = sqlite3_mprintf("SELECT rowid, %Q FROM %Q.%Q ORDER BY +%s %s",
      pConfig->zName, pConfig->zDb, pConfig->zName, FTS5_RANK_NAME,
      bAsc ? "ASC" : "DESC"
  );
  if( zSql==0 ){
    rc = SQLITE_NOMEM;
  }else{
    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pSorter->pStmt, 0);
................................................................................
** This is the xEof method of the virtual table. SQLite calls this 
** routine to find out if it has reached the end of a result set.
*/
static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  return (CsrFlagTest(pCsr, FTS5CSR_EOF) ? 1 : 0);
}
















/* 
** This is the xRowid method. The SQLite core calls this routine to
** retrieve the rowid for the current row of the result set. fts5
** exposes %_content.docid as the rowid for the virtual table. The
** rowid should be written to *pRowid.
*/
................................................................................
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int ePlan = FTS5_PLAN(pCsr->idxNum);
  
  assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
  switch( ePlan ){
    case FTS5_PLAN_SOURCE:
    case FTS5_PLAN_MATCH:
      *pRowid = sqlite3Fts5ExprRowid(pCsr->pExpr);
      break;

    case FTS5_PLAN_SORTED_MATCH:
      *pRowid = pCsr->pSorter->iRowid;

      break;

    default:
      *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
      break;
  }

................................................................................
** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise.
*/
static int fts5SeekCursor(Fts5Cursor *pCsr){
  int rc = SQLITE_OK;
  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){
    assert( pCsr->pExpr );
    sqlite3_reset(pCsr->pStmt);
    sqlite3_bind_int64(pCsr->pStmt, 1, sqlite3Fts5ExprRowid(pCsr->pExpr));
    rc = sqlite3_step(pCsr->pStmt);
    if( rc==SQLITE_ROW ){
      rc = SQLITE_OK;
      CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT);
    }else{
      rc = sqlite3_reset(pCsr->pStmt);
      if( rc==SQLITE_OK ){
................................................................................
static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase);
}

static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  return sqlite3Fts5ExprRowid(pCsr->pExpr);
}

static int fts5ApiColumnText(
  Fts5Context *pCtx, 
  int iCol, 
  const char **pz, 
  int *pn
................................................................................

static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
  int rc = SQLITE_OK;

  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){
    i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr);
    rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
  }
  if( iCol>=0 && iCol<pTab->pConfig->nCol ){
    *pnToken = pCsr->aColumnSize[iCol];
  }else{
    *pnToken = 0;
  }
................................................................................
  Fts5Context *pCtx, 
  int iPhrase, 
  int *pi, 
  i64 *piPos 
){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  const u8 *a; int n;             /* Poslist for phrase iPhrase */






  n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, &a);

  return sqlite3Fts5PoslistNext64(a, n, pi, piPos);
}

static int fts5ApiSetAuxdata(
  Fts5Context *pCtx,              /* Fts5 context */
  void *pPtr,                     /* Pointer to save as auxdata */
  void(*xDelete)(void*)           /* Destructor for pPtr (or NULL) */
................................................................................
  if( pCsr==0 ){
    char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId);
    sqlite3_result_error(context, zErr, -1);
  }else{
    fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]);
  }
}


























/* 
** This is the xColumn method, called by SQLite to request a value from
** the row that the supplied cursor currently points to.
*/
static int fts5ColumnMethod(
  sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
................................................................................
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int rc = SQLITE_OK;
  
  assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );

  if( iCol==pConfig->nCol ){
    if( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SOURCE ){
      /* todo */
    }else{
      /* User is requesting the value of the special column with the same name
      ** as the table. Return the cursor integer id number. This value is only
      ** useful in that it may be passed as the first argument to an FTS5
      ** auxiliary function.  */
      sqlite3_result_int64(pCtx, pCsr->iCsrId);
    }







|







 







>
>
>
>

>

>
>
>
>
>
>
>
>
>







 







|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







<
<
<

<
>







 







|







 







|







 







|







 







>
>
>
>
>
>
|
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|







69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
...
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
...
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
...
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
...
610
611
612
613
614
615
616



617

618
619
620
621
622
623
624
625
...
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
...
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
...
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
...
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
....
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
....
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
**
**   SELECT rowid, <fts> FROM <fts> ORDER BY +rank;
**
*/
struct Fts5Sorter {
  sqlite3_stmt *pStmt;
  i64 iRowid;                     /* Current rowid */
  const u8 *aPoslist;             /* Position lists for current row */
  int nIdx;                       /* Number of entries in aIdx[] */
  int aIdx[0];                    /* Offsets into aPoslist for current row */
};

/*
** Virtual-table cursor object.
*/
................................................................................
  int rc;

  rc = sqlite3_step(pSorter->pStmt);
  if( rc==SQLITE_DONE ){
    rc = SQLITE_OK;
    CsrFlagSet(pCsr, FTS5CSR_EOF);
  }else if( rc==SQLITE_ROW ){
    const u8 *a;
    const u8 *aBlob;
    int nBlob;
    int i;
    rc = SQLITE_OK;

    pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0);
    nBlob = sqlite3_column_bytes(pSorter->pStmt, 1);
    aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1);

    for(i=0; i<(pSorter->nIdx-1); i++){
      a += getVarint32(a, pSorter->aIdx[i]);
    }
    pSorter->aIdx[i] = &aBlob[nBlob] - a;
    pSorter->aPoslist = a;
    CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
  }

  return rc;
}

/*
** Advance the cursor to the next row in the table that matches the 
................................................................................
  
  nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
  nByte = sizeof(Fts5Sorter) + sizeof(int) * nPhrase;
  pSorter = (Fts5Sorter*)sqlite3_malloc(nByte);
  if( pSorter==0 ) return SQLITE_NOMEM;
  memset(pSorter, 0, nByte);

  zSql = sqlite3_mprintf("SELECT rowid, %s FROM %Q.%Q ORDER BY +%s %s",
      pConfig->zName, pConfig->zDb, pConfig->zName, FTS5_RANK_NAME,
      bAsc ? "ASC" : "DESC"
  );
  if( zSql==0 ){
    rc = SQLITE_NOMEM;
  }else{
    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pSorter->pStmt, 0);
................................................................................
** This is the xEof method of the virtual table. SQLite calls this 
** routine to find out if it has reached the end of a result set.
*/
static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  return (CsrFlagTest(pCsr, FTS5CSR_EOF) ? 1 : 0);
}

/*
** Return the rowid that the cursor currently points to.
*/
static i64 fts5CursorRowid(Fts5Cursor *pCsr){
  assert( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_MATCH 
       || FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SORTED_MATCH 
       || FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SOURCE 
  );
  if( pCsr->pSorter ){
    return pCsr->pSorter->iRowid;
  }else{
    return sqlite3Fts5ExprRowid(pCsr->pExpr);
  }
}

/* 
** This is the xRowid method. The SQLite core calls this routine to
** retrieve the rowid for the current row of the result set. fts5
** exposes %_content.docid as the rowid for the virtual table. The
** rowid should be written to *pRowid.
*/
................................................................................
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int ePlan = FTS5_PLAN(pCsr->idxNum);
  
  assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
  switch( ePlan ){
    case FTS5_PLAN_SOURCE:
    case FTS5_PLAN_MATCH:



    case FTS5_PLAN_SORTED_MATCH:

      *pRowid = fts5CursorRowid(pCsr);
      break;

    default:
      *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
      break;
  }

................................................................................
** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise.
*/
static int fts5SeekCursor(Fts5Cursor *pCsr){
  int rc = SQLITE_OK;
  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){
    assert( pCsr->pExpr );
    sqlite3_reset(pCsr->pStmt);
    sqlite3_bind_int64(pCsr->pStmt, 1, fts5CursorRowid(pCsr));
    rc = sqlite3_step(pCsr->pStmt);
    if( rc==SQLITE_ROW ){
      rc = SQLITE_OK;
      CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT);
    }else{
      rc = sqlite3_reset(pCsr->pStmt);
      if( rc==SQLITE_OK ){
................................................................................
static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase);
}

static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  return fts5CursorRowid((Fts5Cursor*)pCtx);
}

static int fts5ApiColumnText(
  Fts5Context *pCtx, 
  int iCol, 
  const char **pz, 
  int *pn
................................................................................

static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
  int rc = SQLITE_OK;

  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){
    i64 iRowid = fts5CursorRowid(pCsr);
    rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
  }
  if( iCol>=0 && iCol<pTab->pConfig->nCol ){
    *pnToken = pCsr->aColumnSize[iCol];
  }else{
    *pnToken = 0;
  }
................................................................................
  Fts5Context *pCtx, 
  int iPhrase, 
  int *pi, 
  i64 *piPos 
){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  const u8 *a; int n;             /* Poslist for phrase iPhrase */
  if( pCsr->pSorter ){
    Fts5Sorter *pSorter = pCsr->pSorter;
    int i1 = (iPhrase ? 0 : pSorter->aIdx[iPhrase-1]);
    n = pSorter->aIdx[iPhrase] - i1;
    a = &pSorter->aPoslist[i1];
  }else{
    n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, &a);
  }
  return sqlite3Fts5PoslistNext64(a, n, pi, piPos);
}

static int fts5ApiSetAuxdata(
  Fts5Context *pCtx,              /* Fts5 context */
  void *pPtr,                     /* Pointer to save as auxdata */
  void(*xDelete)(void*)           /* Destructor for pPtr (or NULL) */
................................................................................
  if( pCsr==0 ){
    char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId);
    sqlite3_result_error(context, zErr, -1);
  }else{
    fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]);
  }
}

static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){
  int i;
  int rc = SQLITE_OK;
  int nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
  Fts5Buffer val;
  int iOff = 0;

  memset(&val, 0, sizeof(Fts5Buffer));
  for(i=0; i<nPhrase; i++){
    const u8 *dummy;
    if( i ) sqlite3Fts5BufferAppendVarint(&rc, &val, iOff);
    iOff += sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy);
  }

  for(i=0; i<nPhrase; i++){
    const u8 *pPoslist;
    int nPoslist;
    nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist);
    sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
  }

  sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free);
  return rc;
}

/* 
** This is the xColumn method, called by SQLite to request a value from
** the row that the supplied cursor currently points to.
*/
static int fts5ColumnMethod(
  sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
................................................................................
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int rc = SQLITE_OK;
  
  assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );

  if( iCol==pConfig->nCol ){
    if( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SOURCE ){
      fts5PoslistBlob(pCtx, pCsr);
    }else{
      /* User is requesting the value of the special column with the same name
      ** as the table. Return the cursor integer id number. This value is only
      ** useful in that it may be passed as the first argument to an FTS5
      ** auxiliary function.  */
      sqlite3_result_int64(pCtx, pCsr->iCsrId);
    }

Changes to test/fts5af.test.

42
43
44
45
46
47
48








49
50
51
52
53
54
55
  } [list $res]

  do_execsql_test $tn.2 {
    DELETE FROM t1;
    INSERT INTO t1 VALUES(NULL, $v1);
    SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2;
  } [list $res]









}


foreach {tn doc res} {

  1.1 {X o o o o o o} {[X] o o o o o o}







>
>
>
>
>
>
>
>







42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
  } [list $res]

  do_execsql_test $tn.2 {
    DELETE FROM t1;
    INSERT INTO t1 VALUES(NULL, $v1);
    SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2;
  } [list $res]

  do_execsql_test $tn.3 {
    DELETE FROM t1;
    INSERT INTO t1 VALUES($v1, NULL);
    SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2
    ORDER BY rank DESC;
  } [list $res]


}


foreach {tn doc res} {

  1.1 {X o o o o o o} {[X] o o o o o o}