SQLite4
Check-in [fb07003744]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fill in some functions so that a tiny subset of fts5 queries work.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: fb070037445f92b5f402b6d9835a6025408a0c0a
User & Date: dan 2012-12-27 18:01:10
Context
2012-12-28
18:57
Add support for phrase queries to fts5. check-in: 0780ef9305 user: dan tags: trunk
2012-12-27
18:01
Fill in some functions so that a tiny subset of fts5 queries work. check-in: fb07003744 user: dan tags: trunk
2012-12-26
19:40
Modify where.c and so on to handle fts scans. Opcodes do not work yet. check-in: 58a5617da3 user: dan tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/fts5.c.

105
106
107
108
109
110
111
112
113

114
115
116
117

118
119
120


121
122
123




124
125
126
127
128
129
130
...
147
148
149
150
151
152
153


154



155
156
157
158
159
160
161
...
364
365
366
367
368
369
370

371
372

373
374
375
376
377
378

379
380
381
382
383


384








385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402



403
404
405
406
407
408
409
...
428
429
430
431
432
433
434




435
436
437
438
439
440
441
...
590
591
592
593
594
595
596

597
598
599
600
601
602
603
604
...
614
615
616
617
618
619
620

621
622
623
624

625
626
627
628
629
630
631
....
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
....
1383
1384
1385
1386
1387
1388
1389





































1390
1391
1392
1393
1394
1395
1396
....
1406
1407
1408
1409
1410
1411
1412


1413
1414
1415
1416
1417




1418
1419

1420

1421
1422
1423
1424












1425



1426
1427



1428
1429
1430
1431
1432
















































1433
1434
1435
1436
1437
1438
1439
1440
....
1587
1588
1589
1590
1591
1592
1593
1594

1595
1596
1597
1598
1599
1600
1601

  char *zErr;                     /* Error message (or NULL) */

  const char *zExpr;              /* Pointer to expression text (nul-term) */
  int iExpr;                      /* Current offset in zExpr */
  Fts5ParserToken next;           /* Next token */

  const char **azCol;             /* Column names of indexed table */
  int nCol;                       /* Size of azCol[] in bytes */


  /* Space for dequoted copies of strings */
  char *aSpace;
  int iSpace;

};

struct Fts5Token {


  int bPrefix;                    /* True for a prefix search */
  int n;                          /* Size of z[] in bytes */
  const char *z;                  /* Token value */




};

struct Fts5Str {
  Fts5Token *aToken;
  int nToken;
};

................................................................................
};


/*
** FTS5 specific cursor data.
*/
struct Fts5Cursor {


  Fts5Expr *pExpr;                /* MATCH expression for this cursor */



};

/*
** Return true if argument c is one of the special non-whitespace 
** characters that ends an unquoted expression token. 
*/
static int fts5IsSpecial(char c){
................................................................................
  void *pCtx, 
  int iWeight, 
  int iOff, 
  const char *z, int n, 
  int iSrc, int nSrc
){
  struct AppendTokensCtx *p = (struct AppendTokensCtx *)pCtx;

  Fts5Token *pToken;
  char *zSpace;


  zSpace = &p->pParse->aSpace[p->pParse->iSpace];
  p->pParse->iSpace += (n+1);
  memcpy(zSpace, z, n);
  zSpace[n] = '\0';


  pToken = &p->pStr->aToken[p->pStr->nToken];
  p->pStr->nToken++;
  pToken->bPrefix = 0;
  pToken->z = zSpace;
  pToken->n = n;











  return 0;
}

static int fts5AppendTokens(
  Fts5Parser *pParse,
  Fts5Str *pStr,
  const char *zPrim,
  int nPrim
){
  struct AppendTokensCtx ctx;
  ctx.pParse = pParse;
  ctx.pStr = pStr;

  return pParse->pTokenizer->xTokenize(
      (void *)&ctx, pParse->p , zPrim, nPrim, fts5AppendTokensCb
  );
}




static int fts5PhraseAppend(
  Fts5Parser *pParse,
  Fts5Phrase *pPhrase,
  const char *zPrim,
  int nPrim
){
  Fts5Tokenizer *pTok = pParse->pTokenizer;
................................................................................
  return rc;
}

static void fts5PhraseFree(sqlite4 *db, Fts5Phrase *p){
  if( p ){
    int i;
    for(i=0; i<p->nStr; i++){




      sqlite4DbFree(db, p->aStr[i].aToken);
    }
    sqlite4DbFree(db, p->aiNear);
    sqlite4DbFree(db, p->aStr);
    sqlite4DbFree(db, p);
  }
}
................................................................................
  return SQLITE4_OK;
}

static int fts5ParseExpression(
  sqlite4 *db,                    /* Database handle */
  Fts5Tokenizer *pTokenizer,      /* Tokenizer module */
  sqlite4_tokenizer *p,           /* Tokenizer instance */

  const char **azCol,             /* Array of column names (nul-term'd) */
  int nCol,                       /* Size of array azCol[] */
  const char *zExpr,              /* FTS expression text */
  Fts5Expr **ppExpr,              /* OUT: Expression object */
  char **pzErr                    /* OUT: Error message */
){
  int rc = SQLITE4_OK;
  Fts5Parser sParse;
................................................................................
  memset(&sParse, 0, sizeof(Fts5Parser));
  sParse.zExpr = zExpr;
  sParse.azCol = azCol;
  sParse.nCol = nCol;
  sParse.pTokenizer = pTokenizer;
  sParse.p = p;
  sParse.db = db;


  pExpr = sqlite4DbMallocZero(db, sizeof(Fts5Expr) + nExpr*2);
  if( !pExpr ) return SQLITE4_NOMEM;
  sParse.aSpace = (char *)&pExpr[1];


  rc = fts5GrowExprHier(db, &nHierAlloc, &aHier, 1);
  if( rc==SQLITE4_OK ){
    aHier[0].ppNode = &pExpr->pRoot;
    aHier[0].nOpen = 0;
    nHier = 1;
  }
................................................................................
      pInfo = 0;
    }
    else if( bCol ){
      int i;
      char *p;
      int nCol = pIdx->pTable->nCol;

      pInfo->azCol = (char **)&pInfo[1];
      p = (char *)(&pInfo->azCol[nCol]);
      for(i=0; i<nCol; i++){
        const char *zCol = pIdx->pTable->aCol[i].zName;
        int n = sqlite4Strlen30(zCol) + 1;
        pInfo->azCol[i] = p;
        memcpy(p, zCol, n);
        p += n;
................................................................................
      );
    }
  }

  *piCksum = sCtx.cksum;
  return rc;
}






































void sqlite4Fts5Close(sqlite4 *db, Fts5Cursor *pCsr){
  if( pCsr ){
    fts5ExpressionFree(db, pCsr->pExpr);
    sqlite4DbFree(db, pCsr);
  }
}
................................................................................
  int rc = SQLITE4_OK;
  Fts5Cursor *pCsr;

  pCsr = sqlite4DbMallocZero(db, sizeof(Fts5Cursor));
  if( !pCsr ){
    rc = SQLITE4_NOMEM;
  }else{


    rc = fts5ParseExpression(db, pInfo->pTokenizer, pInfo->p, 
        pInfo->azCol, pInfo->nCol, zMatch, &pCsr->pExpr, pzErr
    );
  }





  if( rc!=SQLITE4_OK ){
    sqlite4Fts5Close(db, pCsr);

  }

  return rc;
}

int sqlite4Fts5Next(sqlite4 *db, Fts5Cursor *pCsr){












  return SQLITE4_OK;



}




/*
** Return true if the cursor passed as the second argument currently points
** to a valid entry, or false otherwise.
*/
int sqlite4Fts5Valid(Fts5Cursor *pCsr){
















































  return 0;
}

/**************************************************************************
***************************************************************************
** Below this point is test code.
*/
#ifdef SQLITE4_TEST
................................................................................
    rc = pTok->xCreate(pTok->pCtx, 0, 0, &p);
    if( rc!=SQLITE4_OK ){
      zErr = sqlite4MPrintf(db, "error creating tokenizer: %d", rc);
      goto fts5_parse_expr_out;
    }
  }

  rc = fts5ParseExpression(db, pTok, p, azCol, nCol, zExpr, &pExpr, &zErr);

  if( rc!=SQLITE4_OK ){
    if( zErr==0 ){
      zErr = sqlite4MPrintf(db, "error parsing expression: %d", rc);
    }
    goto fts5_parse_expr_out;
  }








|

>




>



>
>


|
>
>
>
>







 







>
>

>
>
>







 







>


>

<
<
|
<

>
|
|

|

>
>

>
>
>
>
>
>
>
>



|

<
|











>
>
>







 







>
>
>
>







 







>
|







 







>

|


>







 







|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>

|



>
>
>
>


>

>



|
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
|
|
>
>
>





>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|







 







|
>







105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
...
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
...
377
378
379
380
381
382
383
384
385
386
387
388


389

390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412

413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
...
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
...
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
...
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
....
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
....
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
....
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
....
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745

  char *zErr;                     /* Error message (or NULL) */

  const char *zExpr;              /* Pointer to expression text (nul-term) */
  int iExpr;                      /* Current offset in zExpr */
  Fts5ParserToken next;           /* Next token */

  char **azCol;                   /* Column names of indexed table */
  int nCol;                       /* Size of azCol[] in bytes */
  int iRoot;                      /* Root page number of FTS index */

  /* Space for dequoted copies of strings */
  char *aSpace;
  int iSpace;
  int nSpace;                     /* Total size of aSpace in bytes */
};

struct Fts5Token {
  /* TODO: The first three members are redundant, since they can be encoded
  ** in the aPrefix[]/nPrefix key.  */
  int bPrefix;                    /* True for a prefix search */
  int n;                          /* Size of z[] in bytes */
  char *z;                        /* Token value */

  KVCursor *pCsr;                 /* Cursor to iterate thru entries for token */
  KVByteArray *aPrefix;           /* KV prefix to iterate through */
  KVSize nPrefix;                 /* Size of aPrefix in bytes */
};

struct Fts5Str {
  Fts5Token *aToken;
  int nToken;
};

................................................................................
};


/*
** FTS5 specific cursor data.
*/
struct Fts5Cursor {
  sqlite4 *db;
  Fts5Info *pInfo;
  Fts5Expr *pExpr;                /* MATCH expression for this cursor */

  KVByteArray *aKey;              /* Buffer for primary key */
  int nKeyAlloc;                  /* Bytes allocated at aKey[] */
};

/*
** Return true if argument c is one of the special non-whitespace 
** characters that ends an unquoted expression token. 
*/
static int fts5IsSpecial(char c){
................................................................................
  void *pCtx, 
  int iWeight, 
  int iOff, 
  const char *z, int n, 
  int iSrc, int nSrc
){
  struct AppendTokensCtx *p = (struct AppendTokensCtx *)pCtx;
  Fts5Parser *pParse = p->pParse;
  Fts5Token *pToken;
  char *zSpace;
  int nUsed;



  pToken = &p->pStr->aToken[p->pStr->nToken];


  zSpace = &pParse->aSpace[pParse->iSpace];
  nUsed = putVarint32((u8 *)zSpace, pParse->iRoot);
  zSpace[nUsed++] = 0x24;
  pToken->bPrefix = 0;
  pToken->z = &zSpace[nUsed];
  pToken->n = n;
  memcpy(pToken->z, z, n);
  pToken->z[n] = '\0';

  nUsed += (n+1);
  pToken->aPrefix = (u8 *)zSpace;
  pToken->nPrefix = nUsed;
  pToken->pCsr = 0;
  pParse->iSpace += nUsed;
  p->pStr->nToken++;

  assert( pParse->iSpace<=pParse->nSpace );
  return 0;
}

static int fts5AppendTokens( 
  Fts5Parser *pParse,

  Fts5Str *pStr, const char *zPrim,
  int nPrim
){
  struct AppendTokensCtx ctx;
  ctx.pParse = pParse;
  ctx.pStr = pStr;

  return pParse->pTokenizer->xTokenize(
      (void *)&ctx, pParse->p , zPrim, nPrim, fts5AppendTokensCb
  );
}

/*
** Append a new token to the current phrase.
*/
static int fts5PhraseAppend(
  Fts5Parser *pParse,
  Fts5Phrase *pPhrase,
  const char *zPrim,
  int nPrim
){
  Fts5Tokenizer *pTok = pParse->pTokenizer;
................................................................................
  return rc;
}

static void fts5PhraseFree(sqlite4 *db, Fts5Phrase *p){
  if( p ){
    int i;
    for(i=0; i<p->nStr; i++){
      int iTok;
      for(iTok=0; iTok<p->aStr[i].nToken; iTok++){
        sqlite4KVCursorClose(p->aStr[i].aToken[iTok].pCsr);
      }
      sqlite4DbFree(db, p->aStr[i].aToken);
    }
    sqlite4DbFree(db, p->aiNear);
    sqlite4DbFree(db, p->aStr);
    sqlite4DbFree(db, p);
  }
}
................................................................................
  return SQLITE4_OK;
}

static int fts5ParseExpression(
  sqlite4 *db,                    /* Database handle */
  Fts5Tokenizer *pTokenizer,      /* Tokenizer module */
  sqlite4_tokenizer *p,           /* Tokenizer instance */
  int iRoot,                      /* Root page number of FTS index */
  char **azCol,                   /* Array of column names (nul-term'd) */
  int nCol,                       /* Size of array azCol[] */
  const char *zExpr,              /* FTS expression text */
  Fts5Expr **ppExpr,              /* OUT: Expression object */
  char **pzErr                    /* OUT: Error message */
){
  int rc = SQLITE4_OK;
  Fts5Parser sParse;
................................................................................
  memset(&sParse, 0, sizeof(Fts5Parser));
  sParse.zExpr = zExpr;
  sParse.azCol = azCol;
  sParse.nCol = nCol;
  sParse.pTokenizer = pTokenizer;
  sParse.p = p;
  sParse.db = db;
  sParse.iRoot = iRoot;

  pExpr = sqlite4DbMallocZero(db, sizeof(Fts5Expr) + nExpr*4);
  if( !pExpr ) return SQLITE4_NOMEM;
  sParse.aSpace = (char *)&pExpr[1];
  sParse.nSpace = nExpr*4;

  rc = fts5GrowExprHier(db, &nHierAlloc, &aHier, 1);
  if( rc==SQLITE4_OK ){
    aHier[0].ppNode = &pExpr->pRoot;
    aHier[0].nOpen = 0;
    nHier = 1;
  }
................................................................................
      pInfo = 0;
    }
    else if( bCol ){
      int i;
      char *p;
      int nCol = pIdx->pTable->nCol;

      pInfo->azCol = (char **)(&pInfo[1]);
      p = (char *)(&pInfo->azCol[nCol]);
      for(i=0; i<nCol; i++){
        const char *zCol = pIdx->pTable->aCol[i].zName;
        int n = sqlite4Strlen30(zCol) + 1;
        pInfo->azCol[i] = p;
        memcpy(p, zCol, n);
        p += n;
................................................................................
      );
    }
  }

  *piCksum = sCtx.cksum;
  return rc;
}

static int fts5OpenExprCursors(sqlite4 *db, Fts5Info *pInfo, Fts5ExprNode *p){
  int rc = SQLITE4_OK;
  if( p ){
    if( p->eType==TOKEN_PRIMITIVE ){
      KVStore *pStore = db->aDb[pInfo->iDb].pKV;
      Fts5Phrase *pPhrase = p->pPhrase;
      int iStr;

      for(iStr=0; rc==SQLITE4_OK && iStr<pPhrase->nStr; iStr++){
        Fts5Str *pStr = &pPhrase->aStr[iStr];
        int i;
        for(i=0; rc==SQLITE4_OK && i<pStr->nToken; i++){
          Fts5Token *pToken = &pStr->aToken[i];
          rc = sqlite4KVStoreOpenCursor(pStore, &pToken->pCsr);
          if( rc==SQLITE4_OK ){
            rc = sqlite4KVCursorSeek(
                pToken->pCsr, pToken->aPrefix, pToken->nPrefix, 1
            );
            if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK;
          }
        }
      }
    }
    if( rc==SQLITE4_OK ) rc = fts5OpenExprCursors(db, pInfo, p->pLeft);
    if( rc==SQLITE4_OK ) rc = fts5OpenExprCursors(db, pInfo, p->pRight);
  }

  return rc;
}

/*
** Open a cursor for each token in the expression.
*/
static int fts5OpenCursors(sqlite4 *db, Fts5Info *pInfo, Fts5Cursor *pCsr){
  return fts5OpenExprCursors(db, pInfo, pCsr->pExpr->pRoot);
}

void sqlite4Fts5Close(sqlite4 *db, Fts5Cursor *pCsr){
  if( pCsr ){
    fts5ExpressionFree(db, pCsr->pExpr);
    sqlite4DbFree(db, pCsr);
  }
}
................................................................................
  int rc = SQLITE4_OK;
  Fts5Cursor *pCsr;

  pCsr = sqlite4DbMallocZero(db, sizeof(Fts5Cursor));
  if( !pCsr ){
    rc = SQLITE4_NOMEM;
  }else{
    pCsr->pInfo = pInfo;
    pCsr->db = db;
    rc = fts5ParseExpression(db, pInfo->pTokenizer, pInfo->p, 
        pInfo->iRoot, pInfo->azCol, pInfo->nCol, zMatch, &pCsr->pExpr, pzErr
    );
  }

  if( rc==SQLITE4_OK ){
    /* Open a KV cursor for each term in the expression. */
    rc = fts5OpenCursors(db, pInfo, pCsr);
  }
  if( rc!=SQLITE4_OK ){
    sqlite4Fts5Close(db, pCsr);
    pCsr = 0;
  }
  *ppCsr = pCsr;
  return rc;
}

int sqlite4Fts5Next(Fts5Cursor *pCsr){
  Fts5Token *pToken;
  KVCursor *pKVCsr;
  int rc;

  assert( pCsr->pExpr->pRoot->eType==TOKEN_PRIMITIVE );
  pToken = &pCsr->pExpr->pRoot->pPhrase->aStr[0].aToken[0];

  rc = sqlite4KVCursorNext(pToken->pCsr);
  if( rc==SQLITE4_OK ){
    const KVByteArray *aKey;
    KVSize nKey;
    rc = sqlite4KVCursorKey(pToken->pCsr, &aKey, &nKey);
    if( rc==SQLITE4_OK 
     && (nKey<pToken->nPrefix || memcmp(pToken->aPrefix, aKey, pToken->nPrefix))
    ){
      rc = SQLITE4_NOTFOUND;
    }
  }
  return rc;
}

/*
** Return true if the cursor passed as the second argument currently points
** to a valid entry, or false otherwise.
*/
int sqlite4Fts5Valid(Fts5Cursor *pCsr){
  const KVByteArray *aKey;
  KVSize nKey;
  KVCursor *pKVCsr;
  int rc;

  assert( pCsr->pExpr->pRoot->eType==TOKEN_PRIMITIVE );
  pKVCsr = pCsr->pExpr->pRoot->pPhrase->aStr[0].aToken[0].pCsr;

  rc = sqlite4KVCursorKey(pKVCsr, &aKey, &nKey);
  return (rc==SQLITE4_OK);
}

int sqlite4Fts5Pk(
  Fts5Cursor *pCsr, 
  int iTbl, 
  KVByteArray **paKey, 
  KVSize *pnKey
){
  const KVByteArray *aKey;
  KVSize nKey;
  KVCursor *pKVCsr;
  int rc;
  int i;
  int i2;
  int nReq;

  assert( pCsr->pExpr->pRoot->eType==TOKEN_PRIMITIVE );

  pKVCsr = pCsr->pExpr->pRoot->pPhrase->aStr[0].aToken[0].pCsr;
  rc = sqlite4KVCursorKey(pKVCsr, &aKey, &nKey);
  if( rc!=SQLITE4_OK ) return rc;

  i = sqlite4VarintLen(pCsr->pInfo->iRoot);
  while( aKey[i] ) i++;
  i++;

  nReq = sqlite4VarintLen(iTbl) + (nKey-i);
  if( nReq>pCsr->nKeyAlloc ){
    pCsr->aKey = sqlite4DbReallocOrFree(pCsr->db, pCsr->aKey, nReq*2);
    if( !pCsr->aKey ) return SQLITE4_NOMEM;
    pCsr->nKeyAlloc = nReq*2;
  }

  i2 = putVarint32(pCsr->aKey, iTbl);
  memcpy(&pCsr->aKey[i2], &aKey[i], nKey-i);

  *paKey = pCsr->aKey;
  *pnKey = nReq;
  return SQLITE4_OK;
}

/**************************************************************************
***************************************************************************
** Below this point is test code.
*/
#ifdef SQLITE4_TEST
................................................................................
    rc = pTok->xCreate(pTok->pCtx, 0, 0, &p);
    if( rc!=SQLITE4_OK ){
      zErr = sqlite4MPrintf(db, "error creating tokenizer: %d", rc);
      goto fts5_parse_expr_out;
    }
  }

  rc = fts5ParseExpression(
      db, pTok, p, 0, (char **)azCol, nCol, zExpr, &pExpr, &zErr);
  if( rc!=SQLITE4_OK ){
    if( zErr==0 ){
      zErr = sqlite4MPrintf(db, "error parsing expression: %d", rc);
    }
    goto fts5_parse_expr_out;
  }

Changes to src/kvlsm.c.

305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
*/
static int kvlsmKey(
  KVCursor *pKVCursor,         /* The cursor whose key is desired */
  const KVByteArray **paKey,   /* Make this point to the key */
  KVSize *pN                   /* Make this point to the size of the key */
){
  KVLsmCsr *pCsr = (KVLsmCsr *)pKVCursor;

  assert( lsm_csr_valid(pCsr->pCsr) );
  return lsm_csr_key(pCsr->pCsr, (const void **)paKey, (int *)pN);
}

/*
** Return the data of the node the cursor is pointing to.
*/
static int kvlsmData(







<
|







305
306
307
308
309
310
311

312
313
314
315
316
317
318
319
*/
static int kvlsmKey(
  KVCursor *pKVCursor,         /* The cursor whose key is desired */
  const KVByteArray **paKey,   /* Make this point to the key */
  KVSize *pN                   /* Make this point to the size of the key */
){
  KVLsmCsr *pCsr = (KVLsmCsr *)pKVCursor;

  if( 0==lsm_csr_valid(pCsr->pCsr) ) return SQLITE4_DONE;
  return lsm_csr_key(pCsr->pCsr, (const void **)paKey, (int *)pN);
}

/*
** Return the data of the node the cursor is pointing to.
*/
static int kvlsmData(

Changes to src/sqliteInt.h.

2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
....
3272
3273
3274
3275
3276
3277
3278
3279



3280
** An instance of this structure is used as the p4 argument to some fts5
** related vdbe opcodes.
*/
struct Fts5Info {
  int iDb;                        /* Database containing this index */
  int iRoot;                      /* Root page number of index */
  int nCol;                       /* Number of columns in indexed table */
  const char **azCol;             /* Column names for table */
  Fts5Tokenizer *pTokenizer;      /* Tokenizer module */
  sqlite4_tokenizer *p;           /* Tokenizer instance */
};

int sqlite4WalkExpr(Walker*, Expr*);
int sqlite4WalkExprList(Walker*, ExprList*);
int sqlite4WalkSelect(Walker*, Select*);
................................................................................
void sqlite4Fts5IndexFree(sqlite4 *, Index *);

int sqlite4Fts5Update(sqlite4 *, Fts5Info *, Mem *pPk, Mem *aArg, int, char **);
void sqlite4Fts5FreeInfo(sqlite4 *db, Fts5Info *);
void sqlite4Fts5CodeUpdate(Parse *, Index *pIdx, int iRegPk, int iRegData, int);
void sqlite4Fts5CodeCksum(Parse *, Index *, int, int, int);
void sqlite4Fts5CodeQuery(Parse *, Index *, int, int, int);




#endif /* _SQLITEINT_H_ */







|







 








>
>
>

2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
....
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
** An instance of this structure is used as the p4 argument to some fts5
** related vdbe opcodes.
*/
struct Fts5Info {
  int iDb;                        /* Database containing this index */
  int iRoot;                      /* Root page number of index */
  int nCol;                       /* Number of columns in indexed table */
  char **azCol;                   /* Column names for table */
  Fts5Tokenizer *pTokenizer;      /* Tokenizer module */
  sqlite4_tokenizer *p;           /* Tokenizer instance */
};

int sqlite4WalkExpr(Walker*, Expr*);
int sqlite4WalkExprList(Walker*, ExprList*);
int sqlite4WalkSelect(Walker*, Select*);
................................................................................
void sqlite4Fts5IndexFree(sqlite4 *, Index *);

int sqlite4Fts5Update(sqlite4 *, Fts5Info *, Mem *pPk, Mem *aArg, int, char **);
void sqlite4Fts5FreeInfo(sqlite4 *db, Fts5Info *);
void sqlite4Fts5CodeUpdate(Parse *, Index *pIdx, int iRegPk, int iRegData, int);
void sqlite4Fts5CodeCksum(Parse *, Index *, int, int, int);
void sqlite4Fts5CodeQuery(Parse *, Index *, int, int, int);

int sqlite4Fts5Pk(Fts5Cursor *, int, KVByteArray **, KVSize *);
int sqlite4Fts5Next(Fts5Cursor *pCsr);

#endif /* _SQLITEINT_H_ */

Changes to src/vdbe.c.

2842
2843
2844
2845
2846
2847
2848









2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869

2870
2871
2872
2873
2874
2875
2876
....
4914
4915
4916
4917
4918
4919
4920

4921

4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932






4933
4934
4935
4936
4937
4938
4939
  KVSize nKey;                    /* Size of aKey[] in bytes */
  int nShort;                     /* Size of aKey[] without PK fields */
  KVByteArray *aPkKey;
  KVSize nPkKey;

  pPk = p->apCsr[pOp->p1];
  pIdx = p->apCsr[pOp->p3];









  assert( pIdx->pKeyInfo->nPK>0 );
  assert( pPk->pKeyInfo->nPK==0 );

  rc = sqlite4KVCursorKey(pIdx->pKVCur, &aKey, &nKey);
  if( rc==SQLITE4_OK ){
    nShort = sqlite4VdbeShortKey(aKey, nKey, 
        pIdx->pKeyInfo->nField - pIdx->pKeyInfo->nPK
    );

    nPkKey = sqlite4VarintLen(pPk->iRoot) + nKey - nShort;
    aPkKey = sqlite4DbMallocRaw(db, nPkKey);

    if( aPkKey ){
      putVarint32(aPkKey, pPk->iRoot);
      memcpy(&aPkKey[nPkKey - (nKey-nShort)], &aKey[nShort], nKey-nShort);
      rc = sqlite4KVCursorSeek(pPk->pKVCur, aPkKey, nPkKey, 0);
      if( rc==SQLITE4_NOTFOUND ){
        rc = SQLITE4_CORRUPT_BKPT;
      }
      pPk->nullRow = 0;
      sqlite4DbFree(db, aPkKey);

    }
  }

  break;
}

/* Opcode: SeekGe P1 P2 P3 P4 *
................................................................................
  assert( pOp->p4type==P4_FTS5INFO );
  pInfo = pOp->p4.pFtsInfo;
  pCur = allocateCursor(p, pOp->p1, 0, pInfo->iDb, 0);
  if( pCur ){
    rc = sqlite4Fts5Open(db, pInfo, zMatch, pOp->p5, &pCur->pFts, &p->zErrMsg);
  }


  pc = pOp->p2-1;

  break;
}

/* Opcode: FtsNext P1 P2 * * *
**
** Advance FTS cursor P1 to the next entry and jump to instruction P2. Or,
** if there is no next entry, set the cursor to point to EOF and fall through
** to the next instruction.
*/
case OP_FtsNext: {
  assert( 0 );






  break;
}

/* Opcode: FtsPk P1 P2 * * * 
**
** P1 is an FTS cursor that points to a valid entry (not EOF). Copy the PK 
** blob for the current entry to register P2.







>
>
>
>
>
>
>
>
>
|
|
<
|
|
|
|
|

|
|

|
|
|
|
|
|
|
|
|
>







 







>
|
>










|
>
>
>
>
>
>







2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859

2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
....
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
  KVSize nKey;                    /* Size of aKey[] in bytes */
  int nShort;                     /* Size of aKey[] without PK fields */
  KVByteArray *aPkKey;
  KVSize nPkKey;

  pPk = p->apCsr[pOp->p1];
  pIdx = p->apCsr[pOp->p3];

  if( pIdx->pFts ){
    rc = sqlite4Fts5Pk(pIdx->pFts, pPk->iRoot, &aPkKey, &nPkKey);
    if( rc==SQLITE4_OK ){
      rc = sqlite4KVCursorSeek(pPk->pKVCur, aPkKey, nPkKey, 0);
      if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_CORRUPT_BKPT;
      pPk->nullRow = 0;
    }
  }else{
    assert( pIdx->pKeyInfo->nPK>0 );
    assert( pPk->pKeyInfo->nPK==0 );

    rc = sqlite4KVCursorKey(pIdx->pKVCur, &aKey, &nKey);
    if( rc==SQLITE4_OK ){
      nShort = sqlite4VdbeShortKey(aKey, nKey, 
          pIdx->pKeyInfo->nField - pIdx->pKeyInfo->nPK
      );

      nPkKey = sqlite4VarintLen(pPk->iRoot) + nKey - nShort;
      aPkKey = sqlite4DbMallocRaw(db, nPkKey);

      if( aPkKey ){
        putVarint32(aPkKey, pPk->iRoot);
        memcpy(&aPkKey[nPkKey - (nKey-nShort)], &aKey[nShort], nKey-nShort);
        rc = sqlite4KVCursorSeek(pPk->pKVCur, aPkKey, nPkKey, 0);
        if( rc==SQLITE4_NOTFOUND ){
          rc = SQLITE4_CORRUPT_BKPT;
        }
        pPk->nullRow = 0;
        sqlite4DbFree(db, aPkKey);
      }
    }
  }

  break;
}

/* Opcode: SeekGe P1 P2 P3 P4 *
................................................................................
  assert( pOp->p4type==P4_FTS5INFO );
  pInfo = pOp->p4.pFtsInfo;
  pCur = allocateCursor(p, pOp->p1, 0, pInfo->iDb, 0);
  if( pCur ){
    rc = sqlite4Fts5Open(db, pInfo, zMatch, pOp->p5, &pCur->pFts, &p->zErrMsg);
  }

  if( rc==SQLITE4_OK && 0==sqlite4Fts5Valid(pCur->pFts) ){
    pc = pOp->p2-1;
  }
  break;
}

/* Opcode: FtsNext P1 P2 * * *
**
** Advance FTS cursor P1 to the next entry and jump to instruction P2. Or,
** if there is no next entry, set the cursor to point to EOF and fall through
** to the next instruction.
*/
case OP_FtsNext: {
  VdbeCursor *pCsr;

  pCsr = p->apCsr[pOp->p1];
  rc = sqlite4Fts5Next(pCsr->pFts);
  if( rc==SQLITE4_OK ) pc = pOp->p2-1;
  if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK;

  break;
}

/* Opcode: FtsPk P1 P2 * * * 
**
** P1 is an FTS cursor that points to a valid entry (not EOF). Copy the PK 
** blob for the current entry to register P2.

Changes to test/fts5query1.test.

42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

  INSERT INTO t1 VALUES(1, 'o n e');
  INSERT INTO t1 VALUES(2, 't w o');
  INSERT INTO t1 VALUES(3, 't h r e e');
  INSERT INTO t1 VALUES(4, 'f o u r');
}

breakpoint
  explain {SELECT x FROM t1 WHERE t1 MATCH 't'} 
foreach {tn stmt res} {
  1 {SELECT x FROM t1 WHERE t1 MATCH 't'} {2 3}
} {
  do_execsql_test 2.$tn $stmt $res
}

finish_test








<
<








42
43
44
45
46
47
48


49
50
51
52
53
54
55
56

  INSERT INTO t1 VALUES(1, 'o n e');
  INSERT INTO t1 VALUES(2, 't w o');
  INSERT INTO t1 VALUES(3, 't h r e e');
  INSERT INTO t1 VALUES(4, 'f o u r');
}



foreach {tn stmt res} {
  1 {SELECT x FROM t1 WHERE t1 MATCH 't'} {2 3}
} {
  do_execsql_test 2.$tn $stmt $res
}

finish_test