Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes to ext/fts5/fts5Int.h.

   476    476   ** this connection since it was created.
   477    477   */
   478    478   int sqlite3Fts5IndexReads(Fts5Index *p);
   479    479   
   480    480   int sqlite3Fts5IndexReinit(Fts5Index *p);
   481    481   int sqlite3Fts5IndexOptimize(Fts5Index *p);
   482    482   int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
          483  +int sqlite3Fts5IndexReset(Fts5Index *p);
   483    484   
   484    485   int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
   485    486   
   486    487   /*
   487    488   ** End of interface to code in fts5_index.c.
   488    489   **************************************************************************/
   489    490   
................................................................................
   618    619       Fts5Storage *p, const char*, sqlite3_value*, int
   619    620   );
   620    621   
   621    622   int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
   622    623   int sqlite3Fts5StorageRebuild(Fts5Storage *p);
   623    624   int sqlite3Fts5StorageOptimize(Fts5Storage *p);
   624    625   int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
          626  +int sqlite3Fts5StorageReset(Fts5Storage *p);
   625    627   
   626    628   /*
   627    629   ** End of interface to code in fts5_storage.c.
   628    630   **************************************************************************/
   629    631   
   630    632   
   631    633   /**************************************************************************

Changes to ext/fts5/fts5_index.c.

   300    300     sqlite3_blob *pReader;          /* RO incr-blob open on %_data table */
   301    301     sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
   302    302     sqlite3_stmt *pDeleter;         /* "DELETE FROM %_data ... id>=? AND id<=?" */
   303    303     sqlite3_stmt *pIdxWriter;       /* "INSERT ... %_idx VALUES(?,?,?,?)" */
   304    304     sqlite3_stmt *pIdxDeleter;      /* "DELETE FROM %_idx WHERE segid=? */
   305    305     sqlite3_stmt *pIdxSelect;
   306    306     int nRead;                      /* Total number of blocks read */
          307  +
          308  +  sqlite3_stmt *pDataVersion;
          309  +  i64 iStructVersion;             /* data_version when pStruct read */
          310  +  Fts5Structure *pStruct;         /* Current db structure (or NULL) */
   307    311   };
   308    312   
   309    313   struct Fts5DoclistIter {
   310    314     u8 *aEof;                       /* Pointer to 1 byte past end of doclist */
   311    315   
   312    316     /* Output variables. aPoslist==0 at EOF */
   313    317     i64 iRowid;
................................................................................
   954    958         }
   955    959         pLvl->aSeg = aNew;
   956    960       }else{
   957    961         *pRc = SQLITE_NOMEM;
   958    962       }
   959    963     }
   960    964   }
          965  +
          966  +static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){
          967  +  Fts5Structure *pRet = 0;
          968  +  Fts5Config *pConfig = p->pConfig;
          969  +  int iCookie;                    /* Configuration cookie */
          970  +  Fts5Data *pData;
          971  +
          972  +  pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
          973  +  if( p->rc==SQLITE_OK ){
          974  +    /* TODO: Do we need this if the leaf-index is appended? Probably... */
          975  +    memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING);
          976  +    p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
          977  +    if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
          978  +      p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
          979  +    }
          980  +    fts5DataRelease(pData);
          981  +    if( p->rc!=SQLITE_OK ){
          982  +      fts5StructureRelease(pRet);
          983  +      pRet = 0;
          984  +    }
          985  +  }
          986  +
          987  +  return pRet;
          988  +}
          989  +
          990  +static i64 fts5IndexDataVersion(Fts5Index *p){
          991  +  i64 iVersion = 0;
          992  +
          993  +  if( p->rc==SQLITE_OK ){
          994  +    if( p->pDataVersion==0 ){
          995  +      p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion, 
          996  +          sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb)
          997  +          );
          998  +      if( p->rc ) return 0;
          999  +    }
         1000  +
         1001  +    if( SQLITE_ROW==sqlite3_step(p->pDataVersion) ){
         1002  +      iVersion = sqlite3_column_int64(p->pDataVersion, 0);
         1003  +    }
         1004  +    p->rc = sqlite3_reset(p->pDataVersion);
         1005  +  }
         1006  +
         1007  +  return iVersion;
         1008  +}
   961   1009   
   962   1010   /*
   963   1011   ** Read, deserialize and return the structure record.
   964   1012   **
   965   1013   ** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array
   966   1014   ** are over-allocated as described for function fts5StructureDecode() 
   967   1015   ** above.
   968   1016   **
   969   1017   ** If an error occurs, NULL is returned and an error code left in the
   970   1018   ** Fts5Index handle. If an error has already occurred when this function
   971   1019   ** is called, it is a no-op.
   972   1020   */
   973   1021   static Fts5Structure *fts5StructureRead(Fts5Index *p){
   974         -  Fts5Config *pConfig = p->pConfig;
   975         -  Fts5Structure *pRet = 0;        /* Object to return */
   976         -  int iCookie;                    /* Configuration cookie */
   977         -  Fts5Data *pData;
   978         -
   979         -  pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
   980         -  if( p->rc ) return 0;
   981         -  /* TODO: Do we need this if the leaf-index is appended? Probably... */
   982         -  memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING);
   983         -  p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
   984         -  if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
   985         -    p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
   986         -  }
   987         -
   988         -  fts5DataRelease(pData);
   989         -  if( p->rc!=SQLITE_OK ){
   990         -    fts5StructureRelease(pRet);
   991         -    pRet = 0;
   992         -  }
   993         -  return pRet;
         1022  +
         1023  +  if( p->pStruct==0 ){
         1024  +    p->iStructVersion = fts5IndexDataVersion(p);
         1025  +    if( p->rc==SQLITE_OK ){
         1026  +      p->pStruct = fts5StructureReadUncached(p);
         1027  +    }
         1028  +  }
         1029  +
         1030  +#ifdef SQLITE_DEBUG
         1031  +  else{
         1032  +    Fts5Structure *pTest = fts5StructureReadUncached(p);
         1033  +    if( pTest ){
         1034  +      int i, j;
         1035  +      assert_nc( p->pStruct->nSegment==pTest->nSegment );
         1036  +      assert_nc( p->pStruct->nLevel==pTest->nLevel );
         1037  +      for(i=0; i<pTest->nLevel; i++){
         1038  +        assert_nc( p->pStruct->aLevel[i].nMerge==pTest->aLevel[i].nMerge );
         1039  +        assert_nc( p->pStruct->aLevel[i].nSeg==pTest->aLevel[i].nSeg );
         1040  +        for(j=0; j<pTest->aLevel[i].nSeg; j++){
         1041  +          Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j];
         1042  +          Fts5StructureSegment *p2 = &p->pStruct->aLevel[i].aSeg[j];
         1043  +          assert_nc( p1->iSegid==p2->iSegid );
         1044  +          assert_nc( p1->pgnoFirst==p2->pgnoFirst );
         1045  +          assert_nc( p1->pgnoLast==p2->pgnoLast );
         1046  +        }
         1047  +      }
         1048  +      fts5StructureRelease(pTest);
         1049  +    }
         1050  +  }
         1051  +#endif
         1052  +
         1053  +  if( p->rc!=SQLITE_OK ) return 0;
         1054  +  assert( p->iStructVersion!=0 );
         1055  +  assert( p->pStruct!=0 );
         1056  +  fts5StructureRef(p->pStruct);
         1057  +  return p->pStruct;
         1058  +}
         1059  +
         1060  +static void fts5StructureInvalidate(Fts5Index *p){
         1061  +  if( p->pStruct ){
         1062  +    fts5StructureRelease(p->pStruct);
         1063  +    p->pStruct = 0;
         1064  +  }
   994   1065   }
   995   1066   
   996   1067   /*
   997   1068   ** Return the total number of segments in index structure pStruct. This
   998   1069   ** function is only ever used as part of assert() conditions.
   999   1070   */
  1000   1071   #ifdef SQLITE_DEBUG
................................................................................
  4343   4414     int iSegid;
  4344   4415     int pgnoLast = 0;                 /* Last leaf page number in segment */
  4345   4416   
  4346   4417     /* Obtain a reference to the index structure and allocate a new segment-id
  4347   4418     ** for the new level-0 segment.  */
  4348   4419     pStruct = fts5StructureRead(p);
  4349   4420     iSegid = fts5AllocateSegid(p, pStruct);
         4421  +  fts5StructureInvalidate(p);
  4350   4422   
  4351   4423     if( iSegid ){
  4352   4424       const int pgsz = p->pConfig->pgsz;
  4353   4425       int eDetail = p->pConfig->eDetail;
  4354   4426       Fts5StructureSegment *pSeg;   /* New segment within pStruct */
  4355   4427       Fts5Buffer *pBuf;             /* Buffer in which to assemble leaf page */
  4356   4428       Fts5Buffer *pPgidx;           /* Buffer in which to assemble pgidx */
................................................................................
  4562   4634   int sqlite3Fts5IndexOptimize(Fts5Index *p){
  4563   4635     Fts5Structure *pStruct;
  4564   4636     Fts5Structure *pNew = 0;
  4565   4637   
  4566   4638     assert( p->rc==SQLITE_OK );
  4567   4639     fts5IndexFlush(p);
  4568   4640     pStruct = fts5StructureRead(p);
         4641  +  fts5StructureInvalidate(p);
  4569   4642   
  4570   4643     if( pStruct ){
  4571   4644       pNew = fts5IndexOptimizeStruct(p, pStruct);
  4572   4645     }
  4573   4646     fts5StructureRelease(pStruct);
  4574   4647   
  4575   4648     assert( pNew==0 || pNew->nSegment>0 );
................................................................................
  4592   4665   ** This is called to implement the special "VALUES('merge', $nMerge)"
  4593   4666   ** INSERT command.
  4594   4667   */
  4595   4668   int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
  4596   4669     Fts5Structure *pStruct = fts5StructureRead(p);
  4597   4670     if( pStruct ){
  4598   4671       int nMin = p->pConfig->nUsermerge;
         4672  +    fts5StructureInvalidate(p);
  4599   4673       if( nMerge<0 ){
  4600   4674         Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct);
  4601   4675         fts5StructureRelease(pStruct);
  4602   4676         pStruct = pNew;
  4603   4677         nMin = 2;
  4604   4678         nMerge = nMerge*-1;
  4605   4679       }
................................................................................
  5019   5093   ** to the database. Additionally, assume that the contents of the %_data
  5020   5094   ** table may have changed on disk. So any in-memory caches of %_data 
  5021   5095   ** records must be invalidated.
  5022   5096   */
  5023   5097   int sqlite3Fts5IndexRollback(Fts5Index *p){
  5024   5098     fts5CloseReader(p);
  5025   5099     fts5IndexDiscardData(p);
         5100  +  fts5StructureInvalidate(p);
  5026   5101     /* assert( p->rc==SQLITE_OK ); */
  5027   5102     return SQLITE_OK;
  5028   5103   }
  5029   5104   
  5030   5105   /*
  5031   5106   ** The %_data table is completely empty when this function is called. This
  5032   5107   ** function populates it with the initial structure objects for each index,
  5033   5108   ** and the initial version of the "averages" record (a zero-byte blob).
  5034   5109   */
  5035   5110   int sqlite3Fts5IndexReinit(Fts5Index *p){
  5036   5111     Fts5Structure s;
         5112  +  fts5StructureInvalidate(p);
  5037   5113     memset(&s, 0, sizeof(Fts5Structure));
  5038   5114     fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0);
  5039   5115     fts5StructureWrite(p, &s);
  5040   5116     return fts5IndexReturn(p);
  5041   5117   }
  5042   5118   
  5043   5119   /*
................................................................................
  5088   5164   /*
  5089   5165   ** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen().
  5090   5166   */
  5091   5167   int sqlite3Fts5IndexClose(Fts5Index *p){
  5092   5168     int rc = SQLITE_OK;
  5093   5169     if( p ){
  5094   5170       assert( p->pReader==0 );
         5171  +    fts5StructureInvalidate(p);
  5095   5172       sqlite3_finalize(p->pWriter);
  5096   5173       sqlite3_finalize(p->pDeleter);
  5097   5174       sqlite3_finalize(p->pIdxWriter);
  5098   5175       sqlite3_finalize(p->pIdxDeleter);
  5099   5176       sqlite3_finalize(p->pIdxSelect);
         5177  +    sqlite3_finalize(p->pDataVersion);
  5100   5178       sqlite3Fts5HashFree(p->pHash);
  5101   5179       sqlite3_free(p->zDataTbl);
  5102   5180       sqlite3_free(p);
  5103   5181     }
  5104   5182     return rc;
  5105   5183   }
  5106   5184   
................................................................................
  6348   6426     if( rc==SQLITE_OK ){
  6349   6427       rc = sqlite3_create_function(
  6350   6428           db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
  6351   6429       );
  6352   6430     }
  6353   6431     return rc;
  6354   6432   }
         6433  +
         6434  +
         6435  +int sqlite3Fts5IndexReset(Fts5Index *p){
         6436  +  assert( p->pStruct==0 || p->iStructVersion!=0 );
         6437  +  if( fts5IndexDataVersion(p)!=p->iStructVersion ){
         6438  +    fts5StructureInvalidate(p);
         6439  +  }
         6440  +  return fts5IndexReturn(p);
         6441  +}

Changes to ext/fts5/fts5_main.c.

   592    592         pInfo->aConstraintUsage[pC->iConsIndex].omit = (unsigned char)pC->omit;
   593    593       }
   594    594     }
   595    595   
   596    596     pInfo->idxNum = idxFlags;
   597    597     return SQLITE_OK;
   598    598   }
          599  +
          600  +static int fts5NewTransaction(Fts5Table *pTab){
          601  +  Fts5Cursor *pCsr;
          602  +  for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
          603  +    if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK;
          604  +  }
          605  +  return sqlite3Fts5StorageReset(pTab->pStorage);
          606  +}
   599    607   
   600    608   /*
   601    609   ** Implementation of xOpen method.
   602    610   */
   603    611   static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
   604    612     Fts5Table *pTab = (Fts5Table*)pVTab;
   605    613     Fts5Config *pConfig = pTab->pConfig;
   606         -  Fts5Cursor *pCsr;               /* New cursor object */
          614  +  Fts5Cursor *pCsr = 0;           /* New cursor object */
   607    615     int nByte;                      /* Bytes of space to allocate */
   608         -  int rc = SQLITE_OK;             /* Return code */
          616  +  int rc;                         /* Return code */
   609    617   
   610         -  nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
   611         -  pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
   612         -  if( pCsr ){
   613         -    Fts5Global *pGlobal = pTab->pGlobal;
   614         -    memset(pCsr, 0, nByte);
   615         -    pCsr->aColumnSize = (int*)&pCsr[1];
   616         -    pCsr->pNext = pGlobal->pCsr;
   617         -    pGlobal->pCsr = pCsr;
   618         -    pCsr->iCsrId = ++pGlobal->iNextId;
   619         -  }else{
   620         -    rc = SQLITE_NOMEM;
          618  +  rc = fts5NewTransaction(pTab);
          619  +  if( rc==SQLITE_OK ){
          620  +    nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
          621  +    pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
          622  +    if( pCsr ){
          623  +      Fts5Global *pGlobal = pTab->pGlobal;
          624  +      memset(pCsr, 0, nByte);
          625  +      pCsr->aColumnSize = (int*)&pCsr[1];
          626  +      pCsr->pNext = pGlobal->pCsr;
          627  +      pGlobal->pCsr = pCsr;
          628  +      pCsr->iCsrId = ++pGlobal->iNextId;
          629  +    }else{
          630  +      rc = SQLITE_NOMEM;
          631  +    }
   621    632     }
   622    633     *ppCsr = (sqlite3_vtab_cursor*)pCsr;
   623    634     return rc;
   624    635   }
   625    636   
   626    637   static int fts5StmtType(Fts5Cursor *pCsr){
   627    638     if( pCsr->ePlan==FTS5_PLAN_SCAN ){
................................................................................
  1574   1585     return rc;
  1575   1586   }
  1576   1587   
  1577   1588   /*
  1578   1589   ** Implementation of xBegin() method. 
  1579   1590   */
  1580   1591   static int fts5BeginMethod(sqlite3_vtab *pVtab){
  1581         -  UNUSED_PARAM(pVtab);  /* Call below is a no-op for NDEBUG builds */
  1582   1592     fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0);
         1593  +  fts5NewTransaction((Fts5Table*)pVtab);
  1583   1594     return SQLITE_OK;
  1584   1595   }
  1585   1596   
  1586   1597   /*
  1587   1598   ** Implementation of xCommit() method. This is a no-op. The contents of
  1588   1599   ** the pending-terms hash-table have already been flushed into the database
  1589   1600   ** by fts5SyncMethod().

Changes to ext/fts5/fts5_storage.c.

   635    635   int sqlite3Fts5StorageOptimize(Fts5Storage *p){
   636    636     return sqlite3Fts5IndexOptimize(p->pIndex);
   637    637   }
   638    638   
   639    639   int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){
   640    640     return sqlite3Fts5IndexMerge(p->pIndex, nMerge);
   641    641   }
          642  +
          643  +int sqlite3Fts5StorageReset(Fts5Storage *p){
          644  +  return sqlite3Fts5IndexReset(p->pIndex);
          645  +}
   642    646   
   643    647   /*
   644    648   ** Allocate a new rowid. This is used for "external content" tables when
   645    649   ** a NULL value is inserted into the rowid column. The new rowid is allocated
   646    650   ** by inserting a dummy row into the %_docsize table. The dummy will be
   647    651   ** overwritten later.
   648    652   **

Changes to ext/fts5/test/fts5corrupt3.test.

   175    175   for {set i 1} {1} {incr i} {
   176    176     set struct [db one {SELECT block FROM t1_data WHERE id=10}]
   177    177     binary scan $struct c* var
   178    178     set end [lindex $var end]
   179    179     if {$end<=$i} break
   180    180     lset var end [expr $end - $i]
   181    181     set struct [binary format c* $var]
          182  +
          183  +  db close
          184  +  sqlite3 db test.db
          185  +
   182    186     db eval {
   183    187       BEGIN;
   184    188       UPDATE t1_data SET block = $struct WHERE id=10;
   185    189     }
   186    190     do_test 4.1.$i {
   187    191       incr nErr [catch { db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x*' } }]
   188    192       set {} {}

Changes to ext/fts5/test/fts5dlidx.test.

   174    174   
   175    175   do_execsql_test 3.2 {
   176    176     SELECT rowid FROM abc WHERE abc 
   177    177     MATCH 'IteratorpItercurrentlypointstothefirstrowidofadoclist' 
   178    178     ORDER BY rowid DESC;
   179    179   } {16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1}
   180    180   
   181         -do_execsql_test 3.2 {
          181  +do_execsql_test 3.3 {
   182    182     INSERT INTO abc(abc) VALUES('integrity-check');
   183    183     INSERT INTO abc(abc) VALUES('optimize');
   184    184     INSERT INTO abc(abc) VALUES('integrity-check');
   185    185   }
   186    186   
   187    187   set v [lindex $vocab 0]
   188    188   set i 0
   189    189   foreach v $vocab {
   190         -  do_execsql_test 3.3.[incr i] {
          190  +  do_execsql_test 3.4.[incr i] {
   191    191       SELECT rowid FROM abc WHERE abc MATCH $v
   192    192     } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
   193    193   }
   194    194   
   195    195   } ;# foreach_detail_mode
   196    196   
   197    197   
   198    198   
   199    199   finish_test
   200    200   

Added ext/fts5/test/fts5multiclient.test.

            1  +# 2016 March 17
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +#
           12  +
           13  +source [file join [file dirname [info script]] fts5_common.tcl]
           14  +source $testdir/lock_common.tcl
           15  +
           16  +set testprefix fts5multiclient
           17  +return_if_no_fts5
           18  +
           19  +foreach_detail_mode $testprefix {
           20  +
           21  +do_multiclient_test tn {
           22  +
           23  +  do_test 1.$tn.1 {
           24  +    sql1 { CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%) }
           25  +    sql1 { INSERT INTO t1 VALUES('a b c') }
           26  +    sql2 { SELECT rowid FROM t1('b') }
           27  +  } {1}
           28  +
           29  +  do_test 1.$tn.2 {
           30  +    sql2 { INSERT INTO t1 VALUES('a b c') }
           31  +    sql1 { SELECT rowid FROM t1('b') }
           32  +  } {1 2}
           33  +
           34  +  do_test 1.$tn.3 {
           35  +    sql2 { INSERT INTO t1 VALUES('a b c') }
           36  +    sql1 { SELECT rowid FROM t1('b') }
           37  +  } {1 2 3}
           38  +
           39  +  do_test 1.$tn.4 {
           40  +    sql2 { INSERT INTO t1 VALUES('a b c') }
           41  +    sql1 { INSERT INTO t1 VALUES('a b c') }
           42  +    sql3 { INSERT INTO t1(t1) VALUES('integrity-check') }
           43  +  } {}
           44  +
           45  +};# do_multiclient_test
           46  +};# foreach_detail_mode
           47  +finish_test
           48  +