/ Check-in [7aea8c6d]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Improve test coverage of fts5_index.c.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 7aea8c6d99737c6c72078e0b4b9c5f8186021aa0
User & Date: dan 2015-05-15 18:13:14
Context
2015-05-16
20:04
Further test coverage improvements for fts5. check-in: 927d9a64 user: dan tags: fts5
2015-05-15
18:13
Improve test coverage of fts5_index.c. check-in: 7aea8c6d user: dan tags: fts5
12:18
Add test cases. And some fixes. check-in: adee7885 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_index.c.

  3458   3458           fts5NodeIterInit(pPg->buf.p, pPg->buf.n, &ss);
  3459   3459           while( ss.aData ) fts5NodeIterNext(&p->rc, &ss);
  3460   3460           fts5BufferSet(&p->rc, &pPg->term, ss.term.n, ss.term.p);
  3461   3461           pgno = ss.iChild;
  3462   3462           fts5NodeIterFree(&ss);
  3463   3463         }
  3464   3464       }
  3465         -    if( pSeg->nHeight==1 ){
  3466         -      pWriter->nEmpty = pSeg->pgnoLast-1;
  3467         -    }
  3468   3465       assert( p->rc!=SQLITE_OK || (pgno+pWriter->nEmpty)==pSeg->pgnoLast );
  3469   3466       pWriter->bFirstTermInPage = 1;
  3470   3467       assert( pWriter->aWriter[0].term.n==0 );
  3471   3468     }
  3472   3469   }
  3473   3470   
  3474   3471   /*
................................................................................
  4047   4044     return fts5IndexReturn(p); 
  4048   4045   }
  4049   4046   
  4050   4047   int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
  4051   4048     Fts5Structure *pStruct;
  4052   4049   
  4053   4050     pStruct = fts5StructureRead(p);
  4054         -  fts5IndexMerge(p, &pStruct, nMerge);
  4055         -  fts5StructureWrite(p, pStruct);
         4051  +  if( pStruct && pStruct->nLevel ){
         4052  +    fts5IndexMerge(p, &pStruct, nMerge);
         4053  +    fts5StructureWrite(p, pStruct);
         4054  +  }
  4056   4055     fts5StructureRelease(pStruct);
  4057   4056   
  4058   4057     return fts5IndexReturn(p);
  4059   4058   }
  4060   4059   
  4061   4060   
  4062   4061   /*
................................................................................
  4529   4528          || (flags & FTS5INDEX_QUERY_SCAN)==FTS5INDEX_QUERY_SCAN
  4530   4529     );
  4531   4530   
  4532   4531     if( sqlite3Fts5BufferGrow(&p->rc, &buf, nToken+1)==0 ){
  4533   4532       memcpy(&buf.p[1], pToken, nToken);
  4534   4533     }
  4535   4534   
         4535  +#ifdef SQLITE_DEBUG
         4536  +  if( flags & FTS5INDEX_QUERY_TEST_NOIDX ){
         4537  +    assert( flags & FTS5INDEX_QUERY_PREFIX );
         4538  +    iIdx = 1+pConfig->nPrefix;
         4539  +  }else
         4540  +#endif
  4536   4541     if( flags & FTS5INDEX_QUERY_PREFIX ){
  4537         -    if( flags & FTS5INDEX_QUERY_TEST_NOIDX ){
  4538         -      iIdx = 1+pConfig->nPrefix;
  4539         -    }else{
  4540         -      int nChar = fts5IndexCharlen(pToken, nToken);
  4541         -      for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){
  4542         -        if( pConfig->aPrefix[iIdx-1]==nChar ) break;
  4543         -      }
         4542  +    int nChar = fts5IndexCharlen(pToken, nToken);
         4543  +    for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){
         4544  +      if( pConfig->aPrefix[iIdx-1]==nChar ) break;
  4544   4545       }
  4545   4546     }
  4546   4547   
  4547   4548     pRet = (Fts5IndexIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5IndexIter));
  4548   4549     if( pRet ){
  4549         -    memset(pRet, 0, sizeof(Fts5IndexIter));
  4550         -
  4551   4550       pRet->pIndex = p;
  4552   4551       if( iIdx<=pConfig->nPrefix ){
  4553   4552         buf.p[0] = FTS5_MAIN_PREFIX + iIdx;
  4554   4553         pRet->pStruct = fts5StructureRead(p);
  4555   4554         if( pRet->pStruct ){
  4556   4555           fts5MultiIterNew(
  4557   4556               p, pRet->pStruct, 1, flags, buf.p, nToken+1, -1, 0, &pRet->pMulti
................................................................................
  4886   4885         pLvl->pData = 0;
  4887   4886       }
  4888   4887     }
  4889   4888     sqlite3_free(pIter->aLvl);
  4890   4889     fts5BufferFree(&pIter->term);
  4891   4890   }
  4892   4891   
         4892  +#ifdef SQLITE_DEBUG
  4893   4893   /*
  4894   4894   ** This function is purely an internal test. It does not contribute to 
  4895   4895   ** FTS functionality, or even the integrity-check, in any way.
  4896   4896   **
  4897   4897   ** Instead, it tests that the same set of pgno/rowid combinations are 
  4898   4898   ** visited regardless of whether the doclist-index identified by parameters
  4899   4899   ** iSegid/iLeaf is iterated in forwards or reverse order.
  4900   4900   */
  4901         -#ifdef SQLITE_DEBUG
  4902         -static void fts5DlidxIterTestReverse(
         4901  +static void fts5TestDlidxReverse(
  4903   4902     Fts5Index *p, 
  4904   4903     int iSegid,                     /* Segment id to load from */
  4905   4904     int iLeaf                       /* Load doclist-index for this leaf */
  4906   4905   ){
  4907   4906     Fts5DlidxIter *pDlidx = 0;
  4908   4907     u64 cksum1 = 13;
  4909   4908     u64 cksum2 = 13;
................................................................................
  4930   4929       cksum2 += iRowid + ((i64)pgno<<32);
  4931   4930     }
  4932   4931     fts5DlidxIterFree(pDlidx);
  4933   4932     pDlidx = 0;
  4934   4933   
  4935   4934     if( p->rc==SQLITE_OK && cksum1!=cksum2 ) p->rc = FTS5_CORRUPT;
  4936   4935   }
         4936  +
         4937  +static int fts5QueryCksum(
         4938  +  Fts5Index *p,                   /* Fts5 index object */
         4939  +  int iIdx,
         4940  +  const char *z,                  /* Index key to query for */
         4941  +  int n,                          /* Size of index key in bytes */
         4942  +  int flags,                      /* Flags for Fts5IndexQuery */
         4943  +  u64 *pCksum                     /* IN/OUT: Checksum value */
         4944  +){
         4945  +  u64 cksum = *pCksum;
         4946  +  Fts5IndexIter *pIdxIter = 0;
         4947  +  int rc = sqlite3Fts5IndexQuery(p, z, n, flags, &pIdxIter);
         4948  +
         4949  +  while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
         4950  +    const u8 *pPos;
         4951  +    int nPos;
         4952  +    i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
         4953  +    rc = sqlite3Fts5IterPoslist(pIdxIter, &pPos, &nPos);
         4954  +    if( rc==SQLITE_OK ){
         4955  +      Fts5PoslistReader sReader;
         4956  +      for(sqlite3Fts5PoslistReaderInit(-1, pPos, nPos, &sReader);
         4957  +          sReader.bEof==0;
         4958  +          sqlite3Fts5PoslistReaderNext(&sReader)
         4959  +      ){
         4960  +        int iCol = FTS5_POS2COLUMN(sReader.iPos);
         4961  +        int iOff = FTS5_POS2OFFSET(sReader.iPos);
         4962  +        cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
         4963  +      }
         4964  +      rc = sqlite3Fts5IterNext(pIdxIter);
         4965  +    }
         4966  +  }
         4967  +  sqlite3Fts5IterClose(pIdxIter);
         4968  +
         4969  +  *pCksum = cksum;
         4970  +  return rc;
         4971  +}
         4972  +
         4973  +
         4974  +/*
         4975  +** This function is also purely an internal test. It does not contribute to 
         4976  +** FTS functionality, or even the integrity-check, in any way.
         4977  +*/
         4978  +static void fts5TestTerm(
         4979  +  Fts5Index *p, 
         4980  +  Fts5Buffer *pPrev,              /* Previous term */
         4981  +  const char *z, int n,           /* Possibly new term to test */
         4982  +  u64 expected,
         4983  +  u64 *pCksum
         4984  +){
         4985  +  int rc = p->rc;
         4986  +  if( pPrev->n==0 ){
         4987  +    fts5BufferSet(&rc, pPrev, n, (const u8*)z);
         4988  +  }else
         4989  +  if( rc==SQLITE_OK && (pPrev->n!=n || memcmp(pPrev->p, z, n)) ){
         4990  +    u32 cksum3 = *pCksum;
         4991  +    const char *zTerm = &pPrev->p[1];  /* The term without the prefix-byte */
         4992  +    int nTerm = pPrev->n-1;            /* Size of zTerm in bytes */
         4993  +    int iIdx = (pPrev->p[0] - FTS5_MAIN_PREFIX);
         4994  +    int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX);
         4995  +    int rc;
         4996  +    u64 ck1 = 0;
         4997  +    u64 ck2 = 0;
         4998  +
         4999  +    /* Check that the results returned for ASC and DESC queries are
         5000  +    ** the same. If not, call this corruption.  */
         5001  +    rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, flags, &ck1);
         5002  +    if( rc==SQLITE_OK ){
         5003  +      int f = flags|FTS5INDEX_QUERY_DESC;
         5004  +      rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
         5005  +    }
         5006  +    if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
         5007  +
         5008  +    /* If this is a prefix query, check that the results returned if the
         5009  +    ** the index is disabled are the same. In both ASC and DESC order. */
         5010  +    if( iIdx>0 && rc==SQLITE_OK ){
         5011  +      int f = flags|FTS5INDEX_QUERY_TEST_NOIDX;
         5012  +      ck2 = 0;
         5013  +      rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
         5014  +      if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
         5015  +    }
         5016  +    if( iIdx>0 && rc==SQLITE_OK ){
         5017  +      int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC;
         5018  +      ck2 = 0;
         5019  +      rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
         5020  +      if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
         5021  +    }
         5022  +
         5023  +    cksum3 ^= ck1;
         5024  +    fts5BufferSet(&rc, pPrev, n, (const u8*)z);
         5025  +
         5026  +    if( rc==SQLITE_OK && cksum3!=expected ){
         5027  +      rc = FTS5_CORRUPT;
         5028  +    }
         5029  +    *pCksum = cksum3;
         5030  +  }
         5031  +  p->rc = rc;
         5032  +}
         5033  + 
  4937   5034   #else
  4938         -# define fts5DlidxIterTestReverse(x,y,z)
         5035  +# define fts5TestDlidxReverse(x,y,z)
         5036  +# define fts5TestTerm(u,v,w,x,y,z)
  4939   5037   #endif
  4940   5038   
  4941   5039   static void fts5IndexIntegrityCheckSegment(
  4942   5040     Fts5Index *p,                   /* FTS5 backend object */
  4943   5041     Fts5StructureSegment *pSeg      /* Segment to check internal consistency */
  4944   5042   ){
  4945   5043     Fts5BtreeIter iter;             /* Used to iterate through b-tree hierarchy */
................................................................................
  5042   5140           if( pLeaf ){
  5043   5141             if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT;
  5044   5142             fts5DataRelease(pLeaf);
  5045   5143           }
  5046   5144         }
  5047   5145   
  5048   5146         fts5DlidxIterFree(pDlidx);
  5049         -      fts5DlidxIterTestReverse(p, iSegid, iter.iLeaf);
         5147  +      fts5TestDlidxReverse(p, iSegid, iter.iLeaf);
  5050   5148       }
  5051   5149     }
  5052   5150   
  5053         -  /* Either iter.iLeaf must be the rightmost leaf-page in the segment, or 
  5054         -  ** else the segment has been completely emptied by an ongoing merge
  5055         -  ** operation. */
  5056         -  if( p->rc==SQLITE_OK 
  5057         -   && iter.iLeaf!=pSeg->pgnoLast 
  5058         -   && (pSeg->pgnoFirst || pSeg->pgnoLast) 
  5059         -  ){
         5151  +  /* Page iter.iLeaf must now be the rightmost leaf-page in the segment */
         5152  +  if( p->rc==SQLITE_OK && iter.iLeaf!=pSeg->pgnoLast ){
  5060   5153       p->rc = FTS5_CORRUPT;
  5061   5154     }
  5062   5155   
  5063   5156     fts5BtreeIterFree(&iter);
  5064   5157   }
  5065   5158   
  5066   5159   
  5067         -static int fts5QueryCksum(
  5068         -  Fts5Index *p,                   /* Fts5 index object */
  5069         -  int iIdx,
  5070         -  const char *z,                  /* Index key to query for */
  5071         -  int n,                          /* Size of index key in bytes */
  5072         -  int flags,                      /* Flags for Fts5IndexQuery */
  5073         -  u64 *pCksum                     /* IN/OUT: Checksum value */
  5074         -){
  5075         -  u64 cksum = *pCksum;
  5076         -  Fts5IndexIter *pIdxIter = 0;
  5077         -  int rc = sqlite3Fts5IndexQuery(p, z, n, flags, &pIdxIter);
  5078         -
  5079         -  while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
  5080         -    const u8 *pPos;
  5081         -    int nPos;
  5082         -    i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
  5083         -    rc = sqlite3Fts5IterPoslist(pIdxIter, &pPos, &nPos);
  5084         -    if( rc==SQLITE_OK ){
  5085         -      Fts5PoslistReader sReader;
  5086         -      for(sqlite3Fts5PoslistReaderInit(-1, pPos, nPos, &sReader);
  5087         -          sReader.bEof==0;
  5088         -          sqlite3Fts5PoslistReaderNext(&sReader)
  5089         -      ){
  5090         -        int iCol = FTS5_POS2COLUMN(sReader.iPos);
  5091         -        int iOff = FTS5_POS2OFFSET(sReader.iPos);
  5092         -        cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
  5093         -      }
  5094         -      rc = sqlite3Fts5IterNext(pIdxIter);
  5095         -    }
  5096         -  }
  5097         -  sqlite3Fts5IterClose(pIdxIter);
  5098         -
  5099         -  *pCksum = cksum;
  5100         -  return rc;
  5101         -}
  5102         -
  5103   5160   /*
  5104   5161   ** Run internal checks to ensure that the FTS index (a) is internally 
  5105   5162   ** consistent and (b) contains entries for which the XOR of the checksums
  5106   5163   ** as calculated by fts5IndexEntryCksum() is cksum.
  5107   5164   **
  5108   5165   ** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
  5109   5166   ** checksum does not match. Return SQLITE_OK if all checks pass without
  5110   5167   ** error, or some other SQLite error code if another error (e.g. OOM)
  5111   5168   ** occurs.
  5112   5169   */
  5113   5170   int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
  5114   5171     u64 cksum2 = 0;                 /* Checksum based on contents of indexes */
  5115         -  u64 cksum3 = 0;                 /* Checksum based on contents of indexes */
  5116         -  Fts5Buffer term = {0,0,0};      /* Buffer used to hold most recent term */
  5117   5172     Fts5Buffer poslist = {0,0,0};   /* Buffer used to hold a poslist */
  5118   5173     Fts5MultiSegIter *pIter;        /* Used to iterate through entire index */
  5119   5174     Fts5Structure *pStruct;         /* Index structure */
         5175  +
         5176  +  /* Used by extra internal tests only run if NDEBUG is not defined */
         5177  +  u64 cksum3 = 0;                 /* Checksum based on contents of indexes */
         5178  +  Fts5Buffer term = {0,0,0};      /* Buffer used to hold most recent term */
  5120   5179     
  5121   5180     /* Load the FTS index structure */
  5122   5181     pStruct = fts5StructureRead(p);
  5123   5182   
  5124   5183     /* Check that the internal nodes of each segment match the leaves */
  5125   5184     if( pStruct ){
  5126   5185       int iLvl, iSeg;
................................................................................
  5160   5219       while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
  5161   5220         int iCol = FTS5_POS2COLUMN(iPos);
  5162   5221         int iTokOff = FTS5_POS2OFFSET(iPos);
  5163   5222         cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
  5164   5223       }
  5165   5224   
  5166   5225       /* If this is a new term, query for it. Update cksum3 with the results. */
  5167         -    if( p->rc==SQLITE_OK && (term.n!=n || memcmp(term.p, z, n)) ){
  5168         -      const char *zTerm = &z[1];     /* The term without the prefix-byte */
  5169         -      int nTerm = n-1;               /* Size of zTerm in bytes */
  5170         -      int iIdx = (z[0] - FTS5_MAIN_PREFIX);
  5171         -      int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX);
  5172         -      int rc;
  5173         -      u64 ck1 = 0;
  5174         -      u64 ck2 = 0;
  5175         -
  5176         -      /* Check that the results returned for ASC and DESC queries are
  5177         -      ** the same. If not, call this corruption.  */
  5178         -      rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, flags, &ck1);
  5179         -      if( rc==SQLITE_OK ){
  5180         -        int f = flags|FTS5INDEX_QUERY_DESC;
  5181         -        rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
  5182         -      }
  5183         -      if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
  5184         -
  5185         -      /* If this is a prefix query, check that the results returned if the
  5186         -      ** the index is disabled are the same. In both ASC and DESC order. */
  5187         -      if( iIdx>0 && rc==SQLITE_OK ){
  5188         -        int f = flags|FTS5INDEX_QUERY_TEST_NOIDX;
  5189         -        ck2 = 0;
  5190         -        rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
  5191         -        if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
  5192         -      }
  5193         -      if( iIdx>0 && rc==SQLITE_OK ){
  5194         -        int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC;
  5195         -        ck2 = 0;
  5196         -        rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
  5197         -        if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
  5198         -      }
  5199         -
  5200         -      cksum3 ^= ck1;
  5201         -      fts5BufferSet(&rc, &term, n, (const u8*)z);
  5202         -      p->rc = rc;
  5203         -    }
         5226  +    fts5TestTerm(p, &term, z, n, cksum2, &cksum3);
  5204   5227     }
         5228  +  fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3);
         5229  +
  5205   5230     fts5MultiIterFree(p, pIter);
  5206         -
  5207   5231     if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT;
  5208         -  if( p->rc==SQLITE_OK && cksum!=cksum3 ) p->rc = FTS5_CORRUPT;
  5209   5232   
  5210   5233     fts5StructureRelease(pStruct);
  5211   5234     fts5BufferFree(&term);
  5212   5235     fts5BufferFree(&poslist);
  5213   5236     return fts5IndexReturn(p);
  5214   5237   }
  5215   5238   

Changes to ext/fts5/test/fts5corrupt2.test.

   111    111   
   112    112     do_execsql_test 2.$i.4 {
   113    113       ROLLBACK;
   114    114       INSERT INTO t1(t1) VALUES('integrity-check');
   115    115     } {}
   116    116   }
   117    117   
   118         -}
   119         -
   120    118   #-------------------------------------------------------------------------
   121    119   # Test that corruption in leaf page headers is detected by queries that use
   122    120   # doclist-indexes.
   123    121   #
   124    122   set doc "A B C D E F G H I J "
   125    123   do_execsql_test 3.0 {
   126    124     CREATE VIRTUAL TABLE x3 USING fts5(tt);
................................................................................
   203    201       } {1}
   204    202   
   205    203       execsql ROLLBACK
   206    204     }
   207    205   
   208    206     do_test 4.$tn.x { expr $nCorrupt>0 } 1
   209    207   }
          208  +
          209  +}
          210  +
          211  +set doc [string repeat "A B C " 1000]
          212  +do_execsql_test 4.0 {
          213  +  CREATE VIRTUAL TABLE x5 USING fts5(tt);
          214  +  INSERT INTO x5(x5, rank) VALUES('pgsz', 32);
          215  +  WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10) 
          216  +  INSERT INTO x5 SELECT $doc FROM ii;
          217  +}
          218  +
          219  +foreach {tn hdr} {
          220  +  1 "\x00\x01"
          221  +} {
          222  +  set tn2 0
          223  +  set nCorrupt 0
          224  +  foreach rowid [db eval {SELECT rowid FROM x5_data WHERE rowid>10}] {
          225  +    if {$rowid & $mask} continue
          226  +    incr tn2
          227  +    do_test 4.$tn.$tn2 {
          228  +      execsql BEGIN
          229  +
          230  +      set fd [db incrblob main x5_data block $rowid]
          231  +      fconfigure $fd -encoding binary -translation binary
          232  +      puts -nonewline $fd $hdr
          233  +      close $fd
          234  +
          235  +      catchsql { INSERT INTO x5(x5) VALUES('integrity-check') }
          236  +      set {} {}
          237  +    } {}
          238  +
          239  +    execsql ROLLBACK
          240  +  }
          241  +}
   210    242   
   211    243   
   212    244   sqlite3_fts5_may_be_corrupt 0
   213    245   finish_test
   214    246   

Added ext/fts5/test/fts5corrupt3.test.

            1  +# 2015 Apr 24
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# This file tests that FTS5 handles corrupt databases (i.e. internal
           13  +# inconsistencies in the backing tables) correctly. In this case 
           14  +# "correctly" means without crashing.
           15  +#
           16  +
           17  +source [file join [file dirname [info script]] fts5_common.tcl]
           18  +set testprefix fts5corrupt3
           19  +sqlite3_fts5_may_be_corrupt 1
           20  +
           21  +# Create a simple FTS5 table containing 100 documents. Each document 
           22  +# contains 10 terms, each of which start with the character "x".
           23  +#
           24  +expr srand(0)
           25  +db func rnddoc fts5_rnddoc
           26  +do_execsql_test 1.0 {
           27  +  CREATE VIRTUAL TABLE t1 USING fts5(x);
           28  +  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
           29  +  WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
           30  +  INSERT INTO t1 SELECT rnddoc(10) FROM ii;
           31  +}
           32  +set mask [expr 31 << 31]
           33  +
           34  +do_test 1.1 {
           35  +  # Pick out the rowid of the right-most b-tree leaf in the new segment.
           36  +  set rowid [db one {
           37  +    SELECT max(rowid) FROM t1_data WHERE ((rowid>>31) & 0x0F)==1
           38  +  }]
           39  +  set L [db one {SELECT length(block) FROM t1_data WHERE rowid = $rowid}]
           40  +  set {} {}
           41  +} {} 
           42  +
           43  +for {set i 0} {$i < $L} {incr i} {
           44  +  do_test 1.2.$i {
           45  +    catchsql {
           46  +      BEGIN;
           47  +      UPDATE t1_data SET block = substr(block, 1, $i) WHERE id = $rowid;
           48  +      INSERT INTO t1(t1) VALUES('integrity-check');
           49  +    }
           50  +  } {1 {database disk image is malformed}}
           51  +  catchsql ROLLBACK
           52  +}
           53  + 
           54  +
           55  +sqlite3_fts5_may_be_corrupt 0
           56  +finish_test
           57  +

Changes to ext/fts5/test/fts5merge.test.

   130    130   do_test 3.4 {
   131    131     execsql { INSERT INTO x8(x8, rank) VALUES('automerge', 4) }
   132    132     while {[not_merged x8]} {
   133    133       execsql { INSERT INTO x8(x8, rank) VALUES('merge', 1) }
   134    134     }
   135    135     fts5_level_segs x8
   136    136   } {0 1}
          137  +
          138  +#-------------------------------------------------------------------------
          139  +#
          140  +proc mydoc {} {
          141  +  set x [lindex {a b c d e f g h i j} [expr int(rand()*10)]]
          142  +  return [string repeat "$x " 30]
          143  +}
          144  +db func mydoc mydoc
          145  +
          146  +proc mycount {} {
          147  +  set res [list]
          148  +  foreach x {a b c d e f g h i j} {
          149  +    lappend res [db one {SELECT count(*) FROM x8 WHERE x8 MATCH $x}]
          150  +  }
          151  +  set res
          152  +}
          153  +
          154  +  #1 32
          155  +foreach {tn pgsz} {
          156  +  2 1000
          157  +} {
          158  +  do_execsql_test 4.$tn.1 {
          159  +    DROP TABLE IF EXISTS x8;
          160  +    CREATE VIRTUAL TABLE x8 USING fts5(i);
          161  +    INSERT INTO x8(x8, rank) VALUES('pgsz', $pgsz);
          162  +  }
          163  +
          164  +  do_execsql_test 4.$tn.2 {
          165  +    INSERT INTO x8(x8, rank) VALUES('merge', 1);
          166  +  }
          167  +
          168  +  do_execsql_test 4.$tn.3 {
          169  +    WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
          170  +      INSERT INTO x8 SELECT mydoc() FROM ii;
          171  +    WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
          172  +      INSERT INTO x8 SELECT mydoc() FROM ii;
          173  +    INSERT INTO x8(x8, rank) VALUES('automerge', 2);
          174  +  }
          175  +
          176  +  set expect [mycount]
          177  +    for {set i 0} {$i < 20} {incr i} {
          178  +      do_test 4.$tn.4.$i {
          179  +        execsql { INSERT INTO x8(x8, rank) VALUES('merge', 1); }
          180  +        mycount
          181  +      } $expect
          182  +      break
          183  +    }
          184  +  db eval {SELECT fts5_decode(rowid, block) AS r FROM x8_data} { puts $r }
          185  +}
   137    186   
   138    187   finish_test
   139    188