/ Check-in [e358c3de]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix various problems in fts5 revealed by fault-injection tests.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: e358c3de5c916f2c851ab9324ceaae4e4e7a0fbd
User & Date: dan 2014-12-18 18:25:48
Context
2014-12-18
20:01
Fix a problem with prefix queries and the AND operator. check-in: 38b3c65e user: dan tags: fts5
18:25
Fix various problems in fts5 revealed by fault-injection tests. check-in: e358c3de user: dan tags: fts5
2014-12-03
17:27
Begin testing fts5 OOM and IO error handling. check-in: 2037dba6 user: dan tags: fts5
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Changes to ext/fts5/fts5.c.

219
220
221
222
223
224
225
226
227
228
229
230
231
232
233

    case FTS5_COMMIT:
      assert( p->ts.eState==2 );
      p->ts.eState = 0;
      break;

    case FTS5_ROLLBACK:
      assert( p->ts.eState==1 || p->ts.eState==2 );
      p->ts.eState = 0;
      break;

    case FTS5_SAVEPOINT:
      assert( p->ts.eState==1 );
      assert( iSavepoint>=0 );
      assert( iSavepoint>p->ts.iSavepoint );







|







219
220
221
222
223
224
225
226
227
228
229
230
231
232
233

    case FTS5_COMMIT:
      assert( p->ts.eState==2 );
      p->ts.eState = 0;
      break;

    case FTS5_ROLLBACK:
      assert( p->ts.eState==1 || p->ts.eState==2 || p->ts.eState==0 );
      p->ts.eState = 0;
      break;

    case FTS5_SAVEPOINT:
      assert( p->ts.eState==1 );
      assert( iSavepoint>=0 );
      assert( iSavepoint>p->ts.iSavepoint );

Changes to ext/fts5/fts5_config.c.

70
71
72
73
74
75
76


77
78
79
80


81
82
83
84

85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
static int fts5ConfigParseSpecial(
  Fts5Config *pConfig,            /* Configuration object to update */
  char *zCmd,                     /* Special command to parse */
  char *zArg,                     /* Argument to parse */
  char **pzErr                    /* OUT: Error message */
){
  if( sqlite3_stricmp(zCmd, "prefix")==0 ){


    char *p;
    if( pConfig->aPrefix ){
      *pzErr = sqlite3_mprintf("multiple prefix=... directives");
      return SQLITE_ERROR;


    }
    pConfig->aPrefix = sqlite3_malloc(sizeof(int) * FTS5_MAX_PREFIX_INDEXES);
    p = zArg;
    while( p[0] ){

      int nPre = 0;
      while( p[0]==' ' ) p++;
      while( p[0]>='0' && p[0]<='9' && nPre<1000 ){
        nPre = nPre*10 + (p[0] - '0');
        p++;
      }
      while( p[0]==' ' ) p++;
      if( p[0]==',' ){
        p++;
      }else if( p[0] ){
        *pzErr = sqlite3_mprintf("malformed prefix=... directive");
        return SQLITE_ERROR;
      }
      if( nPre==0 || nPre>=1000 ){
        *pzErr = sqlite3_mprintf("prefix length out of range: %d", nPre);
        return SQLITE_ERROR;
      }
      pConfig->aPrefix[pConfig->nPrefix] = nPre;
      pConfig->nPrefix++;
    }
    return SQLITE_OK;
  }

  *pzErr = sqlite3_mprintf("unrecognized directive: \"%s\"", zCmd);
  return SQLITE_ERROR;
}

/*







>
>



|
>
>

<

<
>











|

|

|




|







70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

86

87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
static int fts5ConfigParseSpecial(
  Fts5Config *pConfig,            /* Configuration object to update */
  char *zCmd,                     /* Special command to parse */
  char *zArg,                     /* Argument to parse */
  char **pzErr                    /* OUT: Error message */
){
  if( sqlite3_stricmp(zCmd, "prefix")==0 ){
    const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES;
    int rc = SQLITE_OK;
    char *p;
    if( pConfig->aPrefix ){
      *pzErr = sqlite3_mprintf("multiple prefix=... directives");
      rc = SQLITE_ERROR;
    }else{
      pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte);
    }

    p = zArg;

    while( rc==SQLITE_OK && p[0] ){
      int nPre = 0;
      while( p[0]==' ' ) p++;
      while( p[0]>='0' && p[0]<='9' && nPre<1000 ){
        nPre = nPre*10 + (p[0] - '0');
        p++;
      }
      while( p[0]==' ' ) p++;
      if( p[0]==',' ){
        p++;
      }else if( p[0] ){
        *pzErr = sqlite3_mprintf("malformed prefix=... directive");
        rc = SQLITE_ERROR;
      }
      if( rc==SQLITE_OK && (nPre==0 || nPre>=1000) ){
        *pzErr = sqlite3_mprintf("prefix length out of range: %d", nPre);
        rc = SQLITE_ERROR;
      }
      pConfig->aPrefix[pConfig->nPrefix] = nPre;
      pConfig->nPrefix++;
    }
    return rc;
  }

  *pzErr = sqlite3_mprintf("unrecognized directive: \"%s\"", zCmd);
  return SQLITE_ERROR;
}

/*

Changes to ext/fts5/fts5_expr.c.

209
210
211
212
213
214
215

216
217
218
219
220
221
222
...
767
768
769
770
771
772
773

774
775
776
777
778
779
780
781
....
1200
1201
1202
1203
1204
1205
1206



1207
1208
1209
1210
1211
1212




1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224

1225
1226
1227
1228
1229
1230
1231
  sqlite3Fts5ParserFree(pEngine, fts5ParseFree);

  assert( sParse.pExpr==0 || (sParse.rc==SQLITE_OK && sParse.zErr==0) );
  if( sParse.rc==SQLITE_OK ){
    *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
    if( pNew==0 ){
      sParse.rc = SQLITE_NOMEM;

    }else{
      pNew->pRoot = sParse.pExpr;
      pNew->pIndex = 0;
      pNew->apExprPhrase = sParse.apPhrase;
      pNew->nPhrase = sParse.nPhrase;
      sParse.apPhrase = 0;
    }
................................................................................
      pTerm = &pPhrase->aTerm[j];
      rc = sqlite3Fts5IndexQuery(
          pExpr->pIndex, pTerm->zTerm, strlen(pTerm->zTerm),
          (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
          (pExpr->bAsc ? FTS5INDEX_QUERY_ASC : 0),
          &pTerm->pIter
      );

      if( pTerm->pIter && sqlite3Fts5IterEof(pTerm->pIter) ){
        pNode->bEof = 1;
        break;
      }
    }
  }

  return rc;
................................................................................
  int bPrefix                     /* True if there is a trailing "*" */
){
  Fts5Config *pConfig = pParse->pConfig;
  TokenCtx sCtx;                  /* Context object passed to callback */
  int rc;                         /* Tokenize return code */
  char *z = 0;




  if( pPhrase==0 ){
    if( (pParse->nPhrase % 8)==0 ){
      int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8);
      Fts5ExprPhrase **apNew;
      apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte);
      if( apNew==0 ) return 0;




      pParse->apPhrase = apNew;
    }
    pParse->nPhrase++;
  }

  pParse->rc = fts5ParseStringFromToken(pToken, &z);
  if( z==0 ) return 0;
  sqlite3Fts5Dequote(z);

  memset(&sCtx, 0, sizeof(TokenCtx));
  sCtx.pPhrase = pPhrase;
  rc = sqlite3Fts5Tokenize(pConfig, z, strlen(z), &sCtx, fts5ParseTokenize);

  if( rc ){
    pParse->rc = rc;
    fts5ExprPhraseFree(sCtx.pPhrase);
    sCtx.pPhrase = 0;
  }else if( sCtx.pPhrase->nTerm>0 ){
    sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
  }







>







 







>
|







 







>
>
>





|
>
>
>
>





|
|
|
<
<
<
|
>







209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
...
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
....
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229



1230
1231
1232
1233
1234
1235
1236
1237
1238
  sqlite3Fts5ParserFree(pEngine, fts5ParseFree);

  assert( sParse.pExpr==0 || (sParse.rc==SQLITE_OK && sParse.zErr==0) );
  if( sParse.rc==SQLITE_OK ){
    *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
    if( pNew==0 ){
      sParse.rc = SQLITE_NOMEM;
      sqlite3Fts5ParseNodeFree(sParse.pExpr);
    }else{
      pNew->pRoot = sParse.pExpr;
      pNew->pIndex = 0;
      pNew->apExprPhrase = sParse.apPhrase;
      pNew->nPhrase = sParse.nPhrase;
      sParse.apPhrase = 0;
    }
................................................................................
      pTerm = &pPhrase->aTerm[j];
      rc = sqlite3Fts5IndexQuery(
          pExpr->pIndex, pTerm->zTerm, strlen(pTerm->zTerm),
          (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
          (pExpr->bAsc ? FTS5INDEX_QUERY_ASC : 0),
          &pTerm->pIter
      );
      assert( rc==SQLITE_OK || pTerm->pIter==0 );
      if( pTerm->pIter==0 || sqlite3Fts5IterEof(pTerm->pIter) ){
        pNode->bEof = 1;
        break;
      }
    }
  }

  return rc;
................................................................................
  int bPrefix                     /* True if there is a trailing "*" */
){
  Fts5Config *pConfig = pParse->pConfig;
  TokenCtx sCtx;                  /* Context object passed to callback */
  int rc;                         /* Tokenize return code */
  char *z = 0;

  memset(&sCtx, 0, sizeof(TokenCtx));
  sCtx.pPhrase = pPhrase;

  if( pPhrase==0 ){
    if( (pParse->nPhrase % 8)==0 ){
      int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8);
      Fts5ExprPhrase **apNew;
      apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte);
      if( apNew==0 ){
        pParse->rc = SQLITE_NOMEM;
        fts5ExprPhraseFree(pPhrase);
        return 0;
      }
      pParse->apPhrase = apNew;
    }
    pParse->nPhrase++;
  }

  rc = fts5ParseStringFromToken(pToken, &z);
  if( rc==SQLITE_OK ){
    sqlite3Fts5Dequote(z);



    rc = sqlite3Fts5Tokenize(pConfig, z, strlen(z), &sCtx, fts5ParseTokenize);
  }
  if( rc ){
    pParse->rc = rc;
    fts5ExprPhraseFree(sCtx.pPhrase);
    sCtx.pPhrase = 0;
  }else if( sCtx.pPhrase->nTerm>0 ){
    sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
  }

Changes to ext/fts5/fts5_index.c.

598
599
600
601
602
603
604
605
606
607
608
609
610
611
612

613
614
615
616
617
618
619
...
658
659
660
661
662
663
664
665
666

667
668
669
670
671
672
673
...
703
704
705
706
707
708
709
710
711
712

713
714
715
716
717
718
719
...
849
850
851
852
853
854
855














856
857
858
859
860
861
862
...
914
915
916
917
918
919
920



921
922
923
924
925
926
927
....
1005
1006
1007
1008
1009
1010
1011




1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
....
1041
1042
1043
1044
1045
1046
1047

1048
1049
1050
1051
1052
1053
1054
....
1076
1077
1078
1079
1080
1081
1082

1083
1084
1085
1086
1087
1088
1089
....
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
....
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
....
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
....
2656
2657
2658
2659
2660
2661
2662

2663
2664
2665
2666
2667
2668
2669
....
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688

2689
2690
2691
2692
2693
2694

2695
2696
2697
2698
2699
2700
2701

2702
2703
2704
2705
2706
2707
2708
....
2740
2741
2742
2743
2744
2745
2746

2747
2748
2749
2750
2751
2752
2753
....
2762
2763
2764
2765
2766
2767
2768

2769
2770


2771
2772
2773

2774
2775
2776
2777
2778
2779
2780
....
3021
3022
3023
3024
3025
3026
3027

3028
3029
3030
3031
3032
3033
3034
....
3072
3073
3074
3075
3076
3077
3078

3079
3080
3081
3082
3083
3084
3085
....
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
....
3481
3482
3483
3484
3485
3486
3487

3488
3489
3490
3491
3492
3493
3494
....
3496
3497
3498
3499
3500
3501
3502

3503
3504
3505
3506
3507
3508
3509
/*
** Allocate and return a buffer at least nByte bytes in size.
**
** If an OOM error is encountered, return NULL and set the error code in
** the Fts5Index handle passed as the first argument.
*/
static void *fts5IdxMalloc(Fts5Index *p, int nByte){
  void *pRet;
  assert( p->rc==SQLITE_OK );
  pRet = sqlite3_malloc(nByte);
  if( pRet==0 ){
    p->rc = SQLITE_NOMEM;
  }else{
    memset(pRet, 0, nByte);
  }

  return pRet;
}

/*
** Compare the contents of the pLeft buffer with the pRight/nRight blob.
**
** Return -ve if pLeft is smaller than pRight, 0 if they are equal or
................................................................................


/*
** Close the read-only blob handle, if it is open.
*/
static void fts5CloseReader(Fts5Index *p){
  if( p->pReader ){
    sqlite3_blob_close(p->pReader);
    p->pReader = 0;

  }
}

static Fts5Data *fts5DataReadOrBuffer(
  Fts5Index *p, 
  Fts5Buffer *pBuf, 
  i64 iRowid
................................................................................
      );
    }

    if( rc==SQLITE_OK ){
      int nByte = sqlite3_blob_bytes(p->pReader);
      if( pBuf ){
        fts5BufferZero(pBuf);
        fts5BufferGrow(&rc, pBuf, nByte);
        rc = sqlite3_blob_read(p->pReader, pBuf->p, nByte, 0);
        if( rc==SQLITE_OK ) pBuf->n = nByte;

      }else{
        pRet = (Fts5Data*)fts5IdxMalloc(p, sizeof(Fts5Data) + nByte);
        if( !pRet ) return 0;

        pRet->n = nByte;
        pRet->p = (u8*)&pRet[1];
        pRet->nRef = 1;
................................................................................
** Remove all records associated with segment iSegid in index iIdx.
*/
static void fts5DataRemoveSegment(Fts5Index *p, int iIdx, int iSegid){
  i64 iFirst = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, 0);
  i64 iLast = FTS5_SEGMENT_ROWID(iIdx, iSegid+1, 0, 0)-1;
  fts5DataDelete(p, iFirst, iLast);
}















/*
** Deserialize and return the structure record currently stored in serialized
** form within buffer pData/nData.
**
** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array
** are over-allocated by one slot. This allows the structure contents
................................................................................
        pLvl->nSeg = nTotal;
        for(iSeg=0; iSeg<nTotal; iSeg++){
          i += getVarint32(&pData[i], pLvl->aSeg[iSeg].iSegid);
          i += getVarint32(&pData[i], pLvl->aSeg[iSeg].nHeight);
          i += getVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst);
          i += getVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast);
        }



      }
    }
  }

  *ppOut = pRet;
  return rc;
}
................................................................................
  p->rc = fts5StructureDecode(pData->p, pData->n, &iCookie, &pRet);

  if( p->rc==SQLITE_OK && p->pConfig->iCookie!=iCookie ){
    p->rc = sqlite3Fts5ConfigLoad(p->pConfig, iCookie);
  }

  fts5DataRelease(pData);




  return pRet;
}

/*
** Release a reference to an Fts5Structure object returned by an earlier 
** call to fts5StructureRead() or fts5StructureDecode().
*/
static void fts5StructureRelease(Fts5Structure *pStruct){
  int i;
  for(i=0; i<pStruct->nLevel; i++){
    sqlite3_free(pStruct->aLevel[i].aSeg);
  }
  sqlite3_free(pStruct);
}

/*
** Return the total number of segments in index structure pStruct.
*/
static int fts5StructureCountSegments(Fts5Structure *pStruct){
  int nSegment = 0;               /* Total number of segments */
  int iLvl;                       /* Used to iterate through levels */

................................................................................
/*
** Serialize and store the "structure" record for index iIdx.
**
** If an error occurs, leave an error code in the Fts5Index object. If an
** error has already occurred, this function is a no-op.
*/
static void fts5StructureWrite(Fts5Index *p, int iIdx, Fts5Structure *pStruct){

  int nSegment;                   /* Total number of segments */
  Fts5Buffer buf;                 /* Buffer to serialize record into */
  int iLvl;                       /* Used to iterate through levels */
  int iCookie;                    /* Cookie value to store */

  nSegment = fts5StructureCountSegments(pStruct);
  memset(&buf, 0, sizeof(Fts5Buffer));
................................................................................
      fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast);
    }
  }

  fts5DataWrite(p, FTS5_STRUCTURE_ROWID(iIdx), buf.p, buf.n);
  fts5BufferFree(&buf);
}


#if 0
static void fts5PrintStructure(const char *zCaption, Fts5Structure *pStruct){
  int rc = SQLITE_OK;
  Fts5Buffer buf;
  memset(&buf, 0, sizeof(buf));
  fts5DebugStructure(&rc, &buf, pStruct);
................................................................................
    int res;
    pIter->iLeafOffset = fts5GetU16(&pIter->pLeaf->p[2]);
    fts5SegIterLoadTerm(p, pIter, 0);
    do {
      res = fts5BufferCompareBlob(&pIter->term, pTerm, nTerm);
      if( res>=0 ) break;
      fts5SegIterNext(p, pIter);
    }while( pIter->pLeaf );

    if( bGe==0 && res ){
      /* Set iterator to point to EOF */
      fts5DataRelease(pIter->pLeaf);
      pIter->pLeaf = 0;
    }
  }

  if( bGe==0 ){
    pIter->flags |= FTS5_SEGITER_ONETERM;
    if( pIter->pLeaf ){
      if( flags & FTS5INDEX_QUERY_ASC ){
        pIter->flags |= FTS5_SEGITER_REVERSE;
      }
      if( bDlidx ){
        fts5SegIterLoadDlidx(p, iIdx, pIter);
................................................................................
*/
static void fts5IndexDiscardData(Fts5Index *p){
  assert( p->apHash || p->nPendingData==0 );
  if( p->apHash ){
    Fts5Config *pConfig = p->pConfig;
    int i;
    for(i=0; i<=pConfig->nPrefix; i++){
      sqlite3Fts5HashClear(p->apHash[i]);
    }
    p->nPendingData = 0;
  }
}

/*
** Return the size of the prefix, in bytes, that buffer (nNew/pNew) shares
................................................................................
  Fts5Index *p, 
  Fts5SegWriter *pWriter,
  int nTerm, const u8 *pTerm 
){
  int nPrefix;                    /* Bytes of prefix compression for term */
  Fts5PageWriter *pPage = &pWriter->aWriter[0];

  assert( pPage->buf.n==0 || pPage->buf.n>4 );
  if( pPage->buf.n==0 ){
    /* Zero the first term and first docid fields */
    static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 };
    fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero);
    assert( pPage->term.n==0 );
  }
  if( p->rc ) return;
  
................................................................................
** Append a docid to the writers output. 
*/
static void fts5WriteAppendRowid(
  Fts5Index *p, 
  Fts5SegWriter *pWriter,
  i64 iRowid
){

  Fts5PageWriter *pPage = &pWriter->aWriter[0];

  /* If this is to be the first docid written to the page, set the 
  ** docid-pointer in the page-header. Also append a value to the dlidx
  ** buffer, in case a doclist-index is required.  */
  if( pWriter->bFirstRowidInPage ){
    fts5PutU16(pPage->buf.p, pPage->buf.n);
................................................................................
    fts5WriteDlidxAppend(p, pWriter, iRowid);
  }

  /* Write the docid. */
  if( pWriter->bFirstRowidInDoclist || pWriter->bFirstRowidInPage ){
    fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid);
  }else{
    assert( iRowid<pWriter->iPrevRowid );
    fts5BufferAppendVarint(&p->rc, &pPage->buf, pWriter->iPrevRowid - iRowid);
  }
  pWriter->iPrevRowid = iRowid;
  pWriter->bFirstRowidInDoclist = 0;
  pWriter->bFirstRowidInPage = 0;

  if( pPage->buf.n>=p->pConfig->pgsz ){
    fts5WriteFlushLeaf(p, pWriter);
    pWriter->bFirstRowidInPage = 1;
  }
}


static void fts5WriteAppendPoslistInt(
  Fts5Index *p, 
  Fts5SegWriter *pWriter,
  int iVal
){

  Fts5PageWriter *pPage = &pWriter->aWriter[0];
  fts5BufferAppendVarint(&p->rc, &pPage->buf, iVal);
  if( pPage->buf.n>=p->pConfig->pgsz ){
    fts5WriteFlushLeaf(p, pWriter);
    pWriter->bFirstRowidInPage = 1;
  }
}


static void fts5WriteAppendPoslistData(
  Fts5Index *p, 
  Fts5SegWriter *pWriter, 
  const u8 *aData, 
  int nData
){
................................................................................
static void fts5WriteFinish(
  Fts5Index *p, 
  Fts5SegWriter *pWriter,         /* Writer object */
  int *pnHeight,                  /* OUT: Height of the b-tree */
  int *pnLeaf                     /* OUT: Number of leaf pages in b-tree */
){
  int i;

  *pnLeaf = pWriter->aWriter[0].pgno;
  if( *pnLeaf==1 && pWriter->aWriter[0].buf.n==0 ){
    *pnLeaf = 0;
    *pnHeight = 0;
  }else{
    fts5WriteFlushLeaf(p, pWriter);
    if( pWriter->nWriter==1 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){
................................................................................
      Fts5PageWriter *pPg = &pWriter->aWriter[i];
      fts5DataWrite(p, 
          FTS5_SEGMENT_ROWID(pWriter->iIdx, pWriter->iSegid, i, pPg->pgno), 
          pPg->buf.p, pPg->buf.n
      );
    }
  }

  for(i=0; i<pWriter->nWriter; i++){
    Fts5PageWriter *pPg = &pWriter->aWriter[i];


    fts5BufferFree(&pPg->term);
    fts5BufferFree(&pPg->buf);
  }

  sqlite3_free(pWriter->aWriter);
  sqlite3Fts5BufferFree(&pWriter->dlidx);
}

static void fts5WriteInit(
  Fts5Index *p, 
  Fts5SegWriter *pWriter, 
................................................................................
*/
static void fts5IndexWork(
  Fts5Index *p,                   /* FTS5 backend object */
  int iIdx,                       /* Index to work on */
  Fts5Structure **ppStruct,       /* IN/OUT: Current structure of index */
  int nLeaf                       /* Number of output leaves just written */
){

  Fts5Structure *pStruct = *ppStruct;
  i64 nWrite;                     /* Initial value of write-counter */
  int nWork;                      /* Number of work-quanta to perform */
  int nRem;                       /* Number of leaf pages left to write */

  /* Update the write-counter. While doing so, set nWork. */
  nWrite = pStruct->nWriteCounter;
................................................................................
    }
    fts5IndexMergeLevel(p, iIdx, &pStruct, iBestLvl, &nRem);
    fts5StructurePromote(p, iBestLvl+1, pStruct);
    assert( nRem==0 || p->rc==SQLITE_OK );
    *ppStruct = pStruct;
  }
}


static void fts5IndexCrisisMerge(
  Fts5Index *p,                   /* FTS5 backend object */
  int iIdx,                       /* Index to work on */
  Fts5Structure **ppStruct        /* IN/OUT: Current structure of index */
){
  Fts5Structure *pStruct = *ppStruct;
................................................................................
static int fts5FlushNewEntry(
  void *pCtx, 
  i64 iRowid, 
  const u8 *aPoslist, 
  int nPoslist
){
  Fts5FlushCtx *p = (Fts5FlushCtx*)pCtx;
  int rc = SQLITE_OK;

  /* Append the rowid itself */
  fts5WriteAppendRowid(p->pIdx, &p->writer, iRowid);

  /* Append the size of the position list in bytes */
  fts5WriteAppendPoslistInt(p->pIdx, &p->writer, nPoslist);

  /* And the poslist data */
  fts5WriteAppendPoslistData(p->pIdx, &p->writer, aPoslist, nPoslist);
  return rc;
}

/*
** Flush the contents of in-memory hash table iHash to a new level-0 
** segment on disk. Also update the corresponding structure record.
**
** If an error occurs, set the Fts5Index.rc error code. If an error has 
................................................................................
*/
static void fts5MultiIterPoslist(
  Fts5Index *p,
  Fts5MultiSegIter *pMulti,
  int bSz,
  Fts5Buffer *pBuf
){

  Fts5ChunkIter iter;
  Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1] ];
  assert( fts5MultiIterEof(p, pMulti)==0 );
  fts5ChunkIterInit(p, pSeg, &iter);
  if( fts5ChunkIterEof(p, &iter)==0 ){
    if( bSz ){
      fts5BufferAppendVarint(&p->rc, pBuf, iter.nRem);
................................................................................
    while( fts5ChunkIterEof(p, &iter)==0 ){
      fts5BufferAppendBlob(&p->rc, pBuf, iter.n, iter.p);
      fts5ChunkIterNext(p, &iter);
    }
  }
  fts5ChunkIterRelease(&iter);
}


static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
  if( pIter->i<pIter->n ){
    if( pIter->i ){
      i64 iDelta;
      pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&iDelta);
      if( pIter->bAsc ){







|
|






>







 







|

>







 







|
|
|
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>







 







>
>
>
>



<
<
<
<
<
<
<
<
<
<
<
<







 







>







 







>







 







|








|







 







|







 







|
|







 







>







 







|











>






>







>







 







>







 







>


>
>



>







 







>







 







>







 







|


|


|


|
|







 







>







 







>







598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
...
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
...
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
...
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
...
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
....
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038












1039
1040
1041
1042
1043
1044
1045
....
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
....
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
....
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
....
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
....
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
....
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
....
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
....
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
....
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
....
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
....
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
....
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
....
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
....
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
/*
** Allocate and return a buffer at least nByte bytes in size.
**
** If an OOM error is encountered, return NULL and set the error code in
** the Fts5Index handle passed as the first argument.
*/
static void *fts5IdxMalloc(Fts5Index *p, int nByte){
  void *pRet = 0;
  if( p->rc==SQLITE_OK ){
    pRet = sqlite3_malloc(nByte);
    if( pRet==0 ){
      p->rc = SQLITE_NOMEM;
    }else{
      memset(pRet, 0, nByte);
    }
  }
  return pRet;
}

/*
** Compare the contents of the pLeft buffer with the pRight/nRight blob.
**
** Return -ve if pLeft is smaller than pRight, 0 if they are equal or
................................................................................


/*
** Close the read-only blob handle, if it is open.
*/
static void fts5CloseReader(Fts5Index *p){
  if( p->pReader ){
    sqlite3_blob *pReader = p->pReader;
    p->pReader = 0;
    sqlite3_blob_close(pReader);
  }
}

static Fts5Data *fts5DataReadOrBuffer(
  Fts5Index *p, 
  Fts5Buffer *pBuf, 
  i64 iRowid
................................................................................
      );
    }

    if( rc==SQLITE_OK ){
      int nByte = sqlite3_blob_bytes(p->pReader);
      if( pBuf ){
        fts5BufferZero(pBuf);
        if( SQLITE_OK==fts5BufferGrow(&rc, pBuf, nByte) ){
          rc = sqlite3_blob_read(p->pReader, pBuf->p, nByte, 0);
          if( rc==SQLITE_OK ) pBuf->n = nByte;
        }
      }else{
        pRet = (Fts5Data*)fts5IdxMalloc(p, sizeof(Fts5Data) + nByte);
        if( !pRet ) return 0;

        pRet->n = nByte;
        pRet->p = (u8*)&pRet[1];
        pRet->nRef = 1;
................................................................................
** Remove all records associated with segment iSegid in index iIdx.
*/
static void fts5DataRemoveSegment(Fts5Index *p, int iIdx, int iSegid){
  i64 iFirst = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, 0);
  i64 iLast = FTS5_SEGMENT_ROWID(iIdx, iSegid+1, 0, 0)-1;
  fts5DataDelete(p, iFirst, iLast);
}

/*
** Release a reference to an Fts5Structure object returned by an earlier 
** call to fts5StructureRead() or fts5StructureDecode().
*/
static void fts5StructureRelease(Fts5Structure *pStruct){
  if( pStruct ){
    int i;
    for(i=0; i<pStruct->nLevel; i++){
      sqlite3_free(pStruct->aLevel[i].aSeg);
    }
    sqlite3_free(pStruct);
  }
}

/*
** Deserialize and return the structure record currently stored in serialized
** form within buffer pData/nData.
**
** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array
** are over-allocated by one slot. This allows the structure contents
................................................................................
        pLvl->nSeg = nTotal;
        for(iSeg=0; iSeg<nTotal; iSeg++){
          i += getVarint32(&pData[i], pLvl->aSeg[iSeg].iSegid);
          i += getVarint32(&pData[i], pLvl->aSeg[iSeg].nHeight);
          i += getVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst);
          i += getVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast);
        }
      }else{
        fts5StructureRelease(pRet);
        pRet = 0;
      }
    }
  }

  *ppOut = pRet;
  return rc;
}
................................................................................
  p->rc = fts5StructureDecode(pData->p, pData->n, &iCookie, &pRet);

  if( p->rc==SQLITE_OK && p->pConfig->iCookie!=iCookie ){
    p->rc = sqlite3Fts5ConfigLoad(p->pConfig, iCookie);
  }

  fts5DataRelease(pData);
  if( p->rc!=SQLITE_OK ){
    fts5StructureRelease(pRet);
    pRet = 0;
  }
  return pRet;
}













/*
** Return the total number of segments in index structure pStruct.
*/
static int fts5StructureCountSegments(Fts5Structure *pStruct){
  int nSegment = 0;               /* Total number of segments */
  int iLvl;                       /* Used to iterate through levels */

................................................................................
/*
** Serialize and store the "structure" record for index iIdx.
**
** If an error occurs, leave an error code in the Fts5Index object. If an
** error has already occurred, this function is a no-op.
*/
static void fts5StructureWrite(Fts5Index *p, int iIdx, Fts5Structure *pStruct){
  if( p->rc==SQLITE_OK ){
    int nSegment;                 /* Total number of segments */
    Fts5Buffer buf;               /* Buffer to serialize record into */
    int iLvl;                     /* Used to iterate through levels */
    int iCookie;                  /* Cookie value to store */

    nSegment = fts5StructureCountSegments(pStruct);
    memset(&buf, 0, sizeof(Fts5Buffer));
................................................................................
        fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast);
      }
    }

    fts5DataWrite(p, FTS5_STRUCTURE_ROWID(iIdx), buf.p, buf.n);
    fts5BufferFree(&buf);
  }
}

#if 0
static void fts5PrintStructure(const char *zCaption, Fts5Structure *pStruct){
  int rc = SQLITE_OK;
  Fts5Buffer buf;
  memset(&buf, 0, sizeof(buf));
  fts5DebugStructure(&rc, &buf, pStruct);
................................................................................
    int res;
    pIter->iLeafOffset = fts5GetU16(&pIter->pLeaf->p[2]);
    fts5SegIterLoadTerm(p, pIter, 0);
    do {
      res = fts5BufferCompareBlob(&pIter->term, pTerm, nTerm);
      if( res>=0 ) break;
      fts5SegIterNext(p, pIter);
    }while( pIter->pLeaf && p->rc==SQLITE_OK );

    if( bGe==0 && res ){
      /* Set iterator to point to EOF */
      fts5DataRelease(pIter->pLeaf);
      pIter->pLeaf = 0;
    }
  }

  if( p->rc==SQLITE_OK && bGe==0 ){
    pIter->flags |= FTS5_SEGITER_ONETERM;
    if( pIter->pLeaf ){
      if( flags & FTS5INDEX_QUERY_ASC ){
        pIter->flags |= FTS5_SEGITER_REVERSE;
      }
      if( bDlidx ){
        fts5SegIterLoadDlidx(p, iIdx, pIter);
................................................................................
*/
static void fts5IndexDiscardData(Fts5Index *p){
  assert( p->apHash || p->nPendingData==0 );
  if( p->apHash ){
    Fts5Config *pConfig = p->pConfig;
    int i;
    for(i=0; i<=pConfig->nPrefix; i++){
      if( p->apHash[i] ) sqlite3Fts5HashClear(p->apHash[i]);
    }
    p->nPendingData = 0;
  }
}

/*
** Return the size of the prefix, in bytes, that buffer (nNew/pNew) shares
................................................................................
  Fts5Index *p, 
  Fts5SegWriter *pWriter,
  int nTerm, const u8 *pTerm 
){
  int nPrefix;                    /* Bytes of prefix compression for term */
  Fts5PageWriter *pPage = &pWriter->aWriter[0];

  assert( pPage==0 || pPage->buf.n==0 || pPage->buf.n>4 );
  if( pPage && pPage->buf.n==0 ){
    /* Zero the first term and first docid fields */
    static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 };
    fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero);
    assert( pPage->term.n==0 );
  }
  if( p->rc ) return;
  
................................................................................
** Append a docid to the writers output. 
*/
static void fts5WriteAppendRowid(
  Fts5Index *p, 
  Fts5SegWriter *pWriter,
  i64 iRowid
){
  if( p->rc==SQLITE_OK ){
    Fts5PageWriter *pPage = &pWriter->aWriter[0];

    /* If this is to be the first docid written to the page, set the 
    ** docid-pointer in the page-header. Also append a value to the dlidx
    ** buffer, in case a doclist-index is required.  */
    if( pWriter->bFirstRowidInPage ){
      fts5PutU16(pPage->buf.p, pPage->buf.n);
................................................................................
      fts5WriteDlidxAppend(p, pWriter, iRowid);
    }

    /* Write the docid. */
    if( pWriter->bFirstRowidInDoclist || pWriter->bFirstRowidInPage ){
      fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid);
    }else{
      assert( p->rc || iRowid<pWriter->iPrevRowid );
      fts5BufferAppendVarint(&p->rc, &pPage->buf, pWriter->iPrevRowid - iRowid);
    }
    pWriter->iPrevRowid = iRowid;
    pWriter->bFirstRowidInDoclist = 0;
    pWriter->bFirstRowidInPage = 0;

    if( pPage->buf.n>=p->pConfig->pgsz ){
      fts5WriteFlushLeaf(p, pWriter);
      pWriter->bFirstRowidInPage = 1;
    }
  }
}

static void fts5WriteAppendPoslistInt(
  Fts5Index *p, 
  Fts5SegWriter *pWriter,
  int iVal
){
  if( p->rc==SQLITE_OK ){
    Fts5PageWriter *pPage = &pWriter->aWriter[0];
    fts5BufferAppendVarint(&p->rc, &pPage->buf, iVal);
    if( pPage->buf.n>=p->pConfig->pgsz ){
      fts5WriteFlushLeaf(p, pWriter);
      pWriter->bFirstRowidInPage = 1;
    }
  }
}

static void fts5WriteAppendPoslistData(
  Fts5Index *p, 
  Fts5SegWriter *pWriter, 
  const u8 *aData, 
  int nData
){
................................................................................
static void fts5WriteFinish(
  Fts5Index *p, 
  Fts5SegWriter *pWriter,         /* Writer object */
  int *pnHeight,                  /* OUT: Height of the b-tree */
  int *pnLeaf                     /* OUT: Number of leaf pages in b-tree */
){
  int i;
  if( p->rc==SQLITE_OK ){
    *pnLeaf = pWriter->aWriter[0].pgno;
    if( *pnLeaf==1 && pWriter->aWriter[0].buf.n==0 ){
      *pnLeaf = 0;
      *pnHeight = 0;
    }else{
      fts5WriteFlushLeaf(p, pWriter);
      if( pWriter->nWriter==1 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){
................................................................................
        Fts5PageWriter *pPg = &pWriter->aWriter[i];
        fts5DataWrite(p, 
            FTS5_SEGMENT_ROWID(pWriter->iIdx, pWriter->iSegid, i, pPg->pgno), 
            pPg->buf.p, pPg->buf.n
        );
      }
    }
  }
  for(i=0; i<pWriter->nWriter; i++){
    Fts5PageWriter *pPg = &pWriter->aWriter[i];
    assert( pPg || p->rc!=SQLITE_OK );
    if( pPg ){
      fts5BufferFree(&pPg->term);
      fts5BufferFree(&pPg->buf);
    }
  }
  sqlite3_free(pWriter->aWriter);
  sqlite3Fts5BufferFree(&pWriter->dlidx);
}

static void fts5WriteInit(
  Fts5Index *p, 
  Fts5SegWriter *pWriter, 
................................................................................
*/
static void fts5IndexWork(
  Fts5Index *p,                   /* FTS5 backend object */
  int iIdx,                       /* Index to work on */
  Fts5Structure **ppStruct,       /* IN/OUT: Current structure of index */
  int nLeaf                       /* Number of output leaves just written */
){
  if( p->rc==SQLITE_OK ){
    Fts5Structure *pStruct = *ppStruct;
    i64 nWrite;                   /* Initial value of write-counter */
    int nWork;                    /* Number of work-quanta to perform */
    int nRem;                     /* Number of leaf pages left to write */

    /* Update the write-counter. While doing so, set nWork. */
    nWrite = pStruct->nWriteCounter;
................................................................................
      }
      fts5IndexMergeLevel(p, iIdx, &pStruct, iBestLvl, &nRem);
      fts5StructurePromote(p, iBestLvl+1, pStruct);
      assert( nRem==0 || p->rc==SQLITE_OK );
      *ppStruct = pStruct;
    }
  }
}

static void fts5IndexCrisisMerge(
  Fts5Index *p,                   /* FTS5 backend object */
  int iIdx,                       /* Index to work on */
  Fts5Structure **ppStruct        /* IN/OUT: Current structure of index */
){
  Fts5Structure *pStruct = *ppStruct;
................................................................................
static int fts5FlushNewEntry(
  void *pCtx, 
  i64 iRowid, 
  const u8 *aPoslist, 
  int nPoslist
){
  Fts5FlushCtx *p = (Fts5FlushCtx*)pCtx;
  Fts5Index *pIdx = p->pIdx;

  /* Append the rowid itself */
  fts5WriteAppendRowid(pIdx, &p->writer, iRowid);

  /* Append the size of the position list in bytes */
  fts5WriteAppendPoslistInt(pIdx, &p->writer, nPoslist);

  /* And the poslist data */
  fts5WriteAppendPoslistData(pIdx, &p->writer, aPoslist, nPoslist);
  return pIdx->rc;
}

/*
** Flush the contents of in-memory hash table iHash to a new level-0 
** segment on disk. Also update the corresponding structure record.
**
** If an error occurs, set the Fts5Index.rc error code. If an error has 
................................................................................
*/
static void fts5MultiIterPoslist(
  Fts5Index *p,
  Fts5MultiSegIter *pMulti,
  int bSz,
  Fts5Buffer *pBuf
){
  if( p->rc==SQLITE_OK ){
    Fts5ChunkIter iter;
    Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1] ];
    assert( fts5MultiIterEof(p, pMulti)==0 );
    fts5ChunkIterInit(p, pSeg, &iter);
    if( fts5ChunkIterEof(p, &iter)==0 ){
      if( bSz ){
        fts5BufferAppendVarint(&p->rc, pBuf, iter.nRem);
................................................................................
      while( fts5ChunkIterEof(p, &iter)==0 ){
        fts5BufferAppendBlob(&p->rc, pBuf, iter.n, iter.p);
        fts5ChunkIterNext(p, &iter);
      }
    }
    fts5ChunkIterRelease(&iter);
  }
}

static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
  if( pIter->i<pIter->n ){
    if( pIter->i ){
      i64 iDelta;
      pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&iDelta);
      if( pIter->bAsc ){

Changes to src/vtab.c.

785
786
787
788
789
790
791


792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
** the offset of the method to call in the sqlite3_module structure.
**
** The array is cleared after invoking the callbacks. 
*/
static void callFinaliser(sqlite3 *db, int offset){
  int i;
  if( db->aVTrans ){


    for(i=0; i<db->nVTrans; i++){
      VTable *pVTab = db->aVTrans[i];
      sqlite3_vtab *p = pVTab->pVtab;
      if( p ){
        int (*x)(sqlite3_vtab *);
        x = *(int (**)(sqlite3_vtab *))((char *)p->pModule + offset);
        if( x ) x(p);
      }
      pVTab->iSavepoint = 0;
      sqlite3VtabUnlock(pVTab);
    }
    sqlite3DbFree(db, db->aVTrans);
    db->nVTrans = 0;
    db->aVTrans = 0;
  }
}

/*
** Invoke the xSync method of all virtual tables in the sqlite3.aVTrans
** array. Return the error code for the first error that occurs, or
** SQLITE_OK if all xSync operations are successful.







>
>

|









|

<







785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806

807
808
809
810
811
812
813
** the offset of the method to call in the sqlite3_module structure.
**
** The array is cleared after invoking the callbacks. 
*/
static void callFinaliser(sqlite3 *db, int offset){
  int i;
  if( db->aVTrans ){
    VTable **aVTrans = db->aVTrans;
    db->aVTrans = 0;
    for(i=0; i<db->nVTrans; i++){
      VTable *pVTab = aVTrans[i];
      sqlite3_vtab *p = pVTab->pVtab;
      if( p ){
        int (*x)(sqlite3_vtab *);
        x = *(int (**)(sqlite3_vtab *))((char *)p->pModule + offset);
        if( x ) x(p);
      }
      pVTab->iSavepoint = 0;
      sqlite3VtabUnlock(pVTab);
    }
    sqlite3DbFree(db, aVTrans);
    db->nVTrans = 0;

  }
}

/*
** Invoke the xSync method of all virtual tables in the sqlite3.aVTrans
** array. Return the error code for the first error that occurs, or
** SQLITE_OK if all xSync operations are successful.

Changes to test/fts5fault1.test.

18
19
20
21
22
23
24
25










26
27
28
29
30
31
32
33
34



35






36






















































37

set testprefix fts5fault1

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}











faultsim_save_and_close
do_faultsim_test 1 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { CREATE VIRTUAL TABLE t1 USING fts5(a) }
} -test {
  faultsim_test_result {0 {}} 
}


































































finish_test









>
>
>
>
>
>
>
>
>
>




|




>
>
>
|
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

>
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
set testprefix fts5fault1

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}

# Simple tests:
#
#   1: CREATE VIRTUAL TABLE
#   2: INSERT statement
#   3: DELETE statement
#   4: MATCH expressions
#

if 1 {

faultsim_save_and_close
do_faultsim_test 1 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3') }
} -test {
  faultsim_test_result {0 {}} 
}

reset_db
do_execsql_test 2.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3');
}
faultsim_save_and_close
do_faultsim_test 2 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { 
    INSERT INTO t1 VALUES('a b c', 'a bc def ghij klmno');
  }
} -test {
  faultsim_test_result {0 {}} 
}

reset_db
do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3');
  INSERT INTO t1 VALUES('a b c', 'a bc def ghij klmno');
}
faultsim_save_and_close
do_faultsim_test 3 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { DELETE FROM t1 }
} -test {
  faultsim_test_result {0 {}} 
}

}

reset_db
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE t2 USING fts5(a, b);
  INSERT INTO t2 VALUES('m f a jj th q jr ar',   'hj n h h sg j i m');
  INSERT INTO t2 VALUES('nr s t g od j kf h',    'sb h aq rg op rb n nl');
  INSERT INTO t2 VALUES('do h h pb p p q fr',    'c rj qs or cr a l i');
  INSERT INTO t2 VALUES('lk gp t i lq mq qm p',  'h mr g f op ld aj h');
  INSERT INTO t2 VALUES('ct d sq kc qi k f j',   'sn gh c of g s qt q');
  INSERT INTO t2 VALUES('d ea d d om mp s ab',   'dm hg l df cm ft pa c');
  INSERT INTO t2 VALUES('tc dk c jn n t sr ge',  'a a kn bc n i af h');
  INSERT INTO t2 VALUES('ie ii d i b sa qo rf',  'a h m aq i b m fn');
  INSERT INTO t2 VALUES('gs r fo a er m h li',   'tm c p gl eb ml q r');
  INSERT INTO t2 VALUES('k fe fd rd a gi ho kk', 'ng m c r d ml rm r');
}
faultsim_save_and_close

foreach {tn expr res} {
  1 { dk  }           7
  2 { m f }           1
  3 { f*  }           {10 9 8 6 5 4 3 1}
  4 { m OR f }        {10 9 8 5 4 1}
  5 { sn + gh }       {5}
  6 { "sn gh" }       {5}
  7 { NEAR(r a, 5) }  {9}
} {
  do_faultsim_test 4.$tn -prep {
    faultsim_restore_and_reopen
  } -body "
    execsql { SELECT rowid FROM t2 WHERE t2 MATCH '$expr' }
  " -test "
    faultsim_test_result {[list 0 $res]}
  "
}

finish_test

Changes to test/malloc_common.tcl.

125
126
127
128
129
130
131


132
133
134
135
136
137
138
...
142
143
144
145
146
147
148
149

150
151
152
153
154
155
156
...
314
315
316
317
318
319
320


321
322
323
324
325
326
327
328
329
330
331
332


333
334
335
336
337
338
339
...
342
343
344
345
346
347
348
349



350
351
352
353
354
355
356
    if {$n != "interrupt"} {lappend DEFAULT(-faults) $n}
  }
  set DEFAULT(-prep)          ""
  set DEFAULT(-body)          ""
  set DEFAULT(-test)          ""
  set DEFAULT(-install)       ""
  set DEFAULT(-uninstall)     ""



  fix_testname name

  array set O [array get DEFAULT]
  array set O $args
  foreach o [array names O] {
    if {[info exists DEFAULT($o)]==0} { error "unknown option: $o" }
................................................................................
  foreach f $O(-faults) {
    set flist [array names FAULTSIM $f]
    if {[llength $flist]==0} { error "unknown fault: $f" }
    set faultlist [concat $faultlist $flist]
  }

  set testspec [list -prep $O(-prep) -body $O(-body) \
      -test $O(-test) -install $O(-install) -uninstall $O(-uninstall)

  ]
  foreach f [lsort -unique $faultlist] {
    eval do_one_faultsim_test "$name-$f" $FAULTSIM($f) $testspec
  }
}


................................................................................
#
#     -prep             Script to execute before -body.
#
#     -body             Script to execute (with fault injection).
#
#     -test             Script to execute after -body.
#


proc do_one_faultsim_test {testname args} {

  set DEFAULT(-injectstart)     "expr"
  set DEFAULT(-injectstop)      "expr 0"
  set DEFAULT(-injecterrlist)   [list]
  set DEFAULT(-injectinstall)   ""
  set DEFAULT(-injectuninstall) ""
  set DEFAULT(-prep)            ""
  set DEFAULT(-body)            ""
  set DEFAULT(-test)            ""
  set DEFAULT(-install)         ""
  set DEFAULT(-uninstall)       ""



  array set O [array get DEFAULT]
  array set O $args
  foreach o [array names O] {
    if {[info exists DEFAULT($o)]==0} { error "unknown option: $o" }
  }

................................................................................
    uplevel faultsim_test_result_int \$args [list $O(-injecterrlist)]
  "

  eval $O(-injectinstall)
  eval $O(-install)

  set stop 0
  for {set iFail 1} {!$stop} {incr iFail} {




    # Evaluate the -prep script.
    #
    eval $O(-prep)

    # Start the fault-injection. Run the -body script. Stop the fault
    # injection. Local var $nfail is set to the total number of faults 







>
>







 







|
>







 







>
>












>
>







 







|
>
>
>







125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
...
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
...
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
...
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
    if {$n != "interrupt"} {lappend DEFAULT(-faults) $n}
  }
  set DEFAULT(-prep)          ""
  set DEFAULT(-body)          ""
  set DEFAULT(-test)          ""
  set DEFAULT(-install)       ""
  set DEFAULT(-uninstall)     ""
  set DEFAULT(-start)          1
  set DEFAULT(-end)            0

  fix_testname name

  array set O [array get DEFAULT]
  array set O $args
  foreach o [array names O] {
    if {[info exists DEFAULT($o)]==0} { error "unknown option: $o" }
................................................................................
  foreach f $O(-faults) {
    set flist [array names FAULTSIM $f]
    if {[llength $flist]==0} { error "unknown fault: $f" }
    set faultlist [concat $faultlist $flist]
  }

  set testspec [list -prep $O(-prep) -body $O(-body) \
      -test $O(-test) -install $O(-install) -uninstall $O(-uninstall) \
      -start $O(-start) -end $O(-end)
  ]
  foreach f [lsort -unique $faultlist] {
    eval do_one_faultsim_test "$name-$f" $FAULTSIM($f) $testspec
  }
}


................................................................................
#
#     -prep             Script to execute before -body.
#
#     -body             Script to execute (with fault injection).
#
#     -test             Script to execute after -body.
#
#     -start            Index of first fault to inject (default 1)
#
proc do_one_faultsim_test {testname args} {

  set DEFAULT(-injectstart)     "expr"
  set DEFAULT(-injectstop)      "expr 0"
  set DEFAULT(-injecterrlist)   [list]
  set DEFAULT(-injectinstall)   ""
  set DEFAULT(-injectuninstall) ""
  set DEFAULT(-prep)            ""
  set DEFAULT(-body)            ""
  set DEFAULT(-test)            ""
  set DEFAULT(-install)         ""
  set DEFAULT(-uninstall)       ""
  set DEFAULT(-start)           1
  set DEFAULT(-end)             0

  array set O [array get DEFAULT]
  array set O $args
  foreach o [array names O] {
    if {[info exists DEFAULT($o)]==0} { error "unknown option: $o" }
  }

................................................................................
    uplevel faultsim_test_result_int \$args [list $O(-injecterrlist)]
  "

  eval $O(-injectinstall)
  eval $O(-install)

  set stop 0
  for {set iFail $O(-start)}                        \
      {!$stop && ($O(-end)==0 || $iFail<=$O(-end))} \
      {incr iFail}                                  \
  {

    # Evaluate the -prep script.
    #
    eval $O(-prep)

    # Start the fault-injection. Run the -body script. Stop the fault
    # injection. Local var $nfail is set to the total number of faults