/ Check-in [45c73deb]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix an fts5 problem with interleaving reads and writes in a single transaction.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 45c73deb440496e848cb24d4c1326d4105dacfee8bbafb115e567051855e6518
User & Date: dan 2019-03-18 15:23:20
Context
2019-03-18
15:49
Fix a buffer overread that could occur when running fts5 prefix queries inside a transaction. check-in: b3fa58dd user: dan tags: trunk
15:23
Fix an fts5 problem with interleaving reads and writes in a single transaction. check-in: 45c73deb user: dan tags: trunk
10:30
Fix a typo in a comment. No changes to code. check-in: c2f50aa4 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5Int.h.

   579    579   /*
   580    580   ** Empty (but do not delete) a hash table.
   581    581   */
   582    582   void sqlite3Fts5HashClear(Fts5Hash*);
   583    583   
   584    584   int sqlite3Fts5HashQuery(
   585    585     Fts5Hash*,                      /* Hash table to query */
          586  +  int nPre,
   586    587     const char *pTerm, int nTerm,   /* Query term */
   587         -  const u8 **ppDoclist,           /* OUT: Pointer to doclist for pTerm */
          588  +  void **ppObj,                   /* OUT: Pointer to doclist for pTerm */
   588    589     int *pnDoclist                  /* OUT: Size of doclist in bytes */
   589    590   );
   590    591   
   591    592   int sqlite3Fts5HashScanInit(
   592    593     Fts5Hash*,                      /* Hash table to query */
   593    594     const char *pTerm, int nTerm    /* Query prefix */
   594    595   );

Changes to ext/fts5/fts5_hash.c.

   183    183   
   184    184     sqlite3_free(apOld);
   185    185     pHash->nSlot = nNew;
   186    186     pHash->aSlot = apNew;
   187    187     return SQLITE_OK;
   188    188   }
   189    189   
   190         -static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){
          190  +static int fts5HashAddPoslistSize(
          191  +  Fts5Hash *pHash, 
          192  +  Fts5HashEntry *p,
          193  +  Fts5HashEntry *p2
          194  +){
          195  +  int nRet = 0;
   191    196     if( p->iSzPoslist ){
   192         -    u8 *pPtr = (u8*)p;
          197  +    u8 *pPtr = p2 ? (u8*)p2 : (u8*)p;
          198  +    int nData = p->nData;
   193    199       if( pHash->eDetail==FTS5_DETAIL_NONE ){
   194         -      assert( p->nData==p->iSzPoslist );
          200  +      assert( nData==p->iSzPoslist );
   195    201         if( p->bDel ){
   196         -        pPtr[p->nData++] = 0x00;
          202  +        pPtr[nData++] = 0x00;
   197    203           if( p->bContent ){
   198         -          pPtr[p->nData++] = 0x00;
          204  +          pPtr[nData++] = 0x00;
   199    205           }
   200    206         }
   201    207       }else{
   202         -      int nSz = (p->nData - p->iSzPoslist - 1);       /* Size in bytes */
          208  +      int nSz = (nData - p->iSzPoslist - 1);       /* Size in bytes */
   203    209         int nPos = nSz*2 + p->bDel;                     /* Value of nPos field */
   204    210   
   205    211         assert( p->bDel==0 || p->bDel==1 );
   206    212         if( nPos<=127 ){
   207    213           pPtr[p->iSzPoslist] = (u8)nPos;
   208    214         }else{
   209    215           int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
   210    216           memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
   211    217           sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
   212         -        p->nData += (nByte-1);
          218  +        nData += (nByte-1);
   213    219         }
   214    220       }
   215    221   
   216         -    p->iSzPoslist = 0;
   217         -    p->bDel = 0;
   218         -    p->bContent = 0;
          222  +    nRet = nData - p->nData;
          223  +    if( p2==0 ){
          224  +      p->iSzPoslist = 0;
          225  +      p->bDel = 0;
          226  +      p->bContent = 0;
          227  +      p->nData = nData;
          228  +    }
   219    229     }
          230  +  return nRet;
   220    231   }
   221    232   
   222    233   /*
   223    234   ** Add an entry to the in-memory hash table. The key is the concatenation
   224    235   ** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos).
   225    236   **
   226    237   **     (bByte || pToken) -> (iRowid,iCol,iPos)
................................................................................
   324    335     assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) );
   325    336   
   326    337     pPtr = (u8*)p;
   327    338   
   328    339     /* If this is a new rowid, append the 4-byte size field for the previous
   329    340     ** entry, and the new rowid for this entry.  */
   330    341     if( iRowid!=p->iRowid ){
   331         -    fts5HashAddPoslistSize(pHash, p);
          342  +    fts5HashAddPoslistSize(pHash, p, 0);
   332    343       p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid);
   333    344       p->iRowid = iRowid;
   334    345       bNew = 1;
   335    346       p->iSzPoslist = p->nData;
   336    347       if( pHash->eDetail!=FTS5_DETAIL_NONE ){
   337    348         p->nData += 1;
   338    349         p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
................................................................................
   469    480   }
   470    481   
   471    482   /*
   472    483   ** Query the hash table for a doclist associated with term pTerm/nTerm.
   473    484   */
   474    485   int sqlite3Fts5HashQuery(
   475    486     Fts5Hash *pHash,                /* Hash table to query */
          487  +  int nPre,
   476    488     const char *pTerm, int nTerm,   /* Query term */
   477         -  const u8 **ppDoclist,           /* OUT: Pointer to doclist for pTerm */
          489  +  void **ppOut,                   /* OUT: Pointer to new object */
   478    490     int *pnDoclist                  /* OUT: Size of doclist in bytes */
   479    491   ){
   480    492     unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm);
   481    493     char *zKey = 0;
   482    494     Fts5HashEntry *p;
   483    495   
   484    496     for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
   485    497       zKey = fts5EntryKey(p);
   486    498       assert( p->nKey+1==(int)strlen(zKey) );
   487    499       if( nTerm==p->nKey+1 && memcmp(zKey, pTerm, nTerm)==0 ) break;
   488    500     }
   489    501   
   490    502     if( p ){
   491         -    fts5HashAddPoslistSize(pHash, p);
   492         -    *ppDoclist = (const u8*)&zKey[nTerm+1];
   493         -    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
          503  +    int nHashPre = sizeof(Fts5HashEntry) + nTerm + 1;
          504  +    int nList = p->nData - nHashPre;
          505  +    u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64(nPre + nList + 10));
          506  +    if( pRet ){
          507  +      Fts5HashEntry *pFaux = (Fts5HashEntry*)&pRet[nPre-nHashPre];
          508  +      memcpy(&pRet[nPre], &((u8*)p)[nHashPre], nList);
          509  +      nList += fts5HashAddPoslistSize(pHash, p, pFaux);
          510  +      *pnDoclist = nList;
          511  +    }else{
          512  +      *pnDoclist = 0;
          513  +      return SQLITE_NOMEM;
          514  +    }
   494    515     }else{
   495         -    *ppDoclist = 0;
          516  +    *ppOut = 0;
   496    517       *pnDoclist = 0;
   497    518     }
   498    519   
   499    520     return SQLITE_OK;
   500    521   }
   501    522   
   502    523   int sqlite3Fts5HashScanInit(
................................................................................
   521    542     const u8 **ppDoclist,           /* OUT: pointer to doclist */
   522    543     int *pnDoclist                  /* OUT: size of doclist in bytes */
   523    544   ){
   524    545     Fts5HashEntry *p;
   525    546     if( (p = pHash->pScan) ){
   526    547       char *zKey = fts5EntryKey(p);
   527    548       int nTerm = (int)strlen(zKey);
   528         -    fts5HashAddPoslistSize(pHash, p);
          549  +    fts5HashAddPoslistSize(pHash, p, 0);
   529    550       *pzTerm = zKey;
   530    551       *ppDoclist = (const u8*)&zKey[nTerm+1];
   531    552       *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
   532    553     }else{
   533    554       *pzTerm = 0;
   534    555       *ppDoclist = 0;
   535    556       *pnDoclist = 0;
   536    557     }
   537    558   }

Changes to ext/fts5/fts5_index.c.

  2453   2453   */
  2454   2454   static void fts5SegIterHashInit(
  2455   2455     Fts5Index *p,                   /* FTS5 backend */
  2456   2456     const u8 *pTerm, int nTerm,     /* Term to seek to */
  2457   2457     int flags,                      /* Mask of FTS5INDEX_XXX flags */
  2458   2458     Fts5SegIter *pIter              /* Object to populate */
  2459   2459   ){
  2460         -  const u8 *pList = 0;
  2461   2460     int nList = 0;
  2462   2461     const u8 *z = 0;
  2463   2462     int n = 0;
         2463  +  Fts5Data *pLeaf = 0;
  2464   2464   
  2465   2465     assert( p->pHash );
  2466   2466     assert( p->rc==SQLITE_OK );
  2467   2467   
  2468   2468     if( pTerm==0 || (flags & FTS5INDEX_QUERY_SCAN) ){
         2469  +    const u8 *pList = 0;
         2470  +
  2469   2471       p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm);
  2470   2472       sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &pList, &nList);
  2471   2473       n = (z ? (int)strlen((const char*)z) : 0);
         2474  +    if( pList ){
         2475  +      pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data));
         2476  +      if( pLeaf ){
         2477  +        pLeaf->p = pList;
         2478  +      }
         2479  +    }
  2472   2480     }else{
  2473         -    pIter->flags |= FTS5_SEGITER_ONETERM;
  2474         -    sqlite3Fts5HashQuery(p->pHash, (const char*)pTerm, nTerm, &pList, &nList);
         2481  +    p->rc = sqlite3Fts5HashQuery(p->pHash, sizeof(Fts5Data), 
         2482  +        (const char*)pTerm, nTerm, (void**)&pLeaf, &nList
         2483  +    );
         2484  +    if( pLeaf ){
         2485  +      pLeaf->p = (u8*)&pLeaf[1];
         2486  +    }
  2475   2487       z = pTerm;
  2476   2488       n = nTerm;
         2489  +    pIter->flags |= FTS5_SEGITER_ONETERM;
  2477   2490     }
  2478   2491   
  2479         -  if( pList ){
  2480         -    Fts5Data *pLeaf;
         2492  +  if( pLeaf ){
  2481   2493       sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z);
  2482         -    pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data));
  2483         -    if( pLeaf==0 ) return;
  2484         -    pLeaf->p = (u8*)pList;
  2485   2494       pLeaf->nn = pLeaf->szLeaf = nList;
  2486   2495       pIter->pLeaf = pLeaf;
  2487   2496       pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid);
  2488   2497       pIter->iEndofDoclist = pLeaf->nn;
  2489   2498   
  2490   2499       if( flags & FTS5INDEX_QUERY_DESC ){
  2491   2500         pIter->flags |= FTS5_SEGITER_REVERSE;

Changes to ext/fts5/test/fts5aa.test.

   423    423     INSERT INTO n1 VALUES('a b c d');
   424    424   }
   425    425   
   426    426   proc funk {} {
   427    427     db eval { UPDATE n1_config SET v=50 WHERE k='version' }
   428    428     set fd [db incrblob main n1_data block 10]
   429    429     fconfigure $fd -encoding binary -translation binary
   430         -  puts -nonewline $fd "\x44\x45"
          430  +#  puts -nonewline $fd "\x44\x45"
   431    431     close $fd
   432    432   }
   433    433   db func funk funk
   434    434   
   435    435   # This test case corrupts the structure record within the first invocation
   436    436   # of function funk(). Which used to cause the bm25() function to throw an
   437    437   # exception. But since bm25() can now used the cached structure record,
................................................................................
   598    598   }
   599    599   do_execsql_test 23.1 {
   600    600     SELECT * FROM t11, t10 WHERE t11.x = t10.x AND t10.rowid IS NULL;
   601    601   }
   602    602   do_execsql_test 23.2 {
   603    603     SELECT * FROM t11, t10 WHERE t10.rowid IS NULL;
   604    604   }
          605  +
          606  +#-------------------------------------------------------------------------
          607  +do_execsql_test 24.0 {
          608  +  CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL%);
          609  +  INSERT INTO t12 VALUES('aaaa');
          610  +}
          611  +do_execsql_test 24.1 {
          612  +  BEGIN;
          613  +    DELETE FROM t12 WHERE rowid=1;
          614  +    SELECT * FROM t12('aaaa');
          615  +    INSERT INTO t12 VALUES('aaaa');
          616  +  END;
          617  +}
          618  +do_execsql_test 24.2 {
          619  +  INSERT INTO t12(t12) VALUES('integrity-check');
          620  +}
          621  +do_execsql_test 24.3 {
          622  +    SELECT * FROM t12('aaaa');
          623  +} {aaaa}
   605    624   
   606    625   }
   607    626   
   608    627   expand_all_sql db
   609    628   finish_test