SQLite

Check-in [d8c6b246]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Different fix for the fts5 COMMIT-following-OOM problem first fixed by [fba3129d]. This one does not cause problems if an fts5 table is renamed and then dropped within the same transaction.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: d8c6b246944934a7a6e027b3f5b986fd64a19dd5c5c5175f4ea8586da59a6764
User & Date: dan 2023-12-07 14:41:58
Context
2023-12-11
13:48
Different fix for the fts5 COMMIT-following-OOM problem first fixed by [fba3129d]. This one does not cause problems if an fts5 table is renamed and then dropped within the same transaction. (check-in: 67da596d user: dan tags: branch-3.44)
2023-12-07
18:41
Fix a problem with handling OOM and other errors in fts5 when querying tokendata=1 tables. (check-in: bc911ab5 user: dan tags: trunk)
14:41
Different fix for the fts5 COMMIT-following-OOM problem first fixed by [fba3129d]. This one does not cause problems if an fts5 table is renamed and then dropped within the same transaction. (check-in: d8c6b246 user: dan tags: trunk)
14:09
Rework the jsonEachPathLength() routine in json_tree() so that it is less susceptible to problems due to goofy object labels. (check-in: 858b76a0 user: drh tags: trunk)
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_index.c.

356
357
358
359
360
361
362

363
364
365
366
367
368
369
  i64 iWriteRowid;                /* Rowid for current doc being written */
  int bDelete;                    /* Current write is a delete */
  int nContentlessDelete;         /* Number of contentless delete ops */
  int nPendingRow;                /* Number of INSERT in hash table */

  /* Error state. */
  int rc;                         /* Current error code */


  /* State used by the fts5DataXXX() functions. */
  sqlite3_blob *pReader;          /* RO incr-blob open on %_data table */
  sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
  sqlite3_stmt *pDeleter;         /* "DELETE FROM %_data ... id>=? AND id<=?" */
  sqlite3_stmt *pIdxWriter;       /* "INSERT ... %_idx VALUES(?,?,?,?)" */
  sqlite3_stmt *pIdxDeleter;      /* "DELETE FROM %_idx WHERE segid=?" */







>







356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
  i64 iWriteRowid;                /* Rowid for current doc being written */
  int bDelete;                    /* Current write is a delete */
  int nContentlessDelete;         /* Number of contentless delete ops */
  int nPendingRow;                /* Number of INSERT in hash table */

  /* Error state. */
  int rc;                         /* Current error code */
  int flushRc;

  /* State used by the fts5DataXXX() functions. */
  sqlite3_blob *pReader;          /* RO incr-blob open on %_data table */
  sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
  sqlite3_stmt *pDeleter;         /* "DELETE FROM %_data ... id>=? AND id<=?" */
  sqlite3_stmt *pIdxWriter;       /* "INSERT ... %_idx VALUES(?,?,?,?)" */
  sqlite3_stmt *pIdxDeleter;      /* "DELETE FROM %_idx WHERE segid=?" */
4116
4117
4118
4119
4120
4121
4122

4123
4124
4125
4126
4127
4128
4129
*/
static void fts5IndexDiscardData(Fts5Index *p){
  assert( p->pHash || p->nPendingData==0 );
  if( p->pHash ){
    sqlite3Fts5HashClear(p->pHash);
    p->nPendingData = 0;
    p->nPendingRow = 0;

  }
  p->nContentlessDelete = 0;
}

/*
** Return the size of the prefix, in bytes, that buffer 
** (pNew/<length-unknown>) shares with buffer (pOld/nOld).







>







4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
*/
static void fts5IndexDiscardData(Fts5Index *p){
  assert( p->pHash || p->nPendingData==0 );
  if( p->pHash ){
    sqlite3Fts5HashClear(p->pHash);
    p->nPendingData = 0;
    p->nPendingRow = 0;
    p->flushRc = SQLITE_OK;
  }
  p->nContentlessDelete = 0;
}

/*
** Return the size of the prefix, in bytes, that buffer 
** (pNew/<length-unknown>) shares with buffer (pOld/nOld).
5697
5698
5699
5700
5701
5702
5703




5704
5705
5706
5707
5708
5709
5710
5711


5712
5713
5714
5715
5716
5717
5718
}

/*
** Flush any data stored in the in-memory hash tables to the database.
*/
static void fts5IndexFlush(Fts5Index *p){
  /* Unless it is empty, flush the hash table to disk */




  if( p->nPendingData || p->nContentlessDelete ){
    assert( p->pHash );
    fts5FlushOneHash(p);
    if( p->rc==SQLITE_OK ){
      sqlite3Fts5HashClear(p->pHash);
      p->nPendingData = 0;
      p->nPendingRow = 0;
      p->nContentlessDelete = 0;


    }
  }
}

static Fts5Structure *fts5IndexOptimizeStruct(
  Fts5Index *p, 
  Fts5Structure *pStruct







>
>
>
>








>
>







5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
}

/*
** Flush any data stored in the in-memory hash tables to the database.
*/
static void fts5IndexFlush(Fts5Index *p){
  /* Unless it is empty, flush the hash table to disk */
  if( p->flushRc ){
    p->rc = p->flushRc;
    return;
  }
  if( p->nPendingData || p->nContentlessDelete ){
    assert( p->pHash );
    fts5FlushOneHash(p);
    if( p->rc==SQLITE_OK ){
      sqlite3Fts5HashClear(p->pHash);
      p->nPendingData = 0;
      p->nPendingRow = 0;
      p->nContentlessDelete = 0;
    }else if( p->nPendingData || p->nContentlessDelete ){
      p->flushRc = p->rc;
    }
  }
}

static Fts5Structure *fts5IndexOptimizeStruct(
  Fts5Index *p, 
  Fts5Structure *pStruct
6884
6885
6886
6887
6888
6889
6890
6891
6892
6893
6894
6895
6896
6897




6898
6899
6900
6901
6902
6903
6904
    Fts5Iter *pNew = 0;
    Fts5SegIter *pNewIter = 0;
    Fts5SegIter *pPrevIter = 0;

    int iLvl, iSeg, ii;

    pNew = fts5MultiIterAlloc(p, pStruct->nSegment);
    if( pNew==0 ) break;

    if( pSmall ){
      fts5BufferSet(&p->rc, &bSeek, pSmall->n, pSmall->p);
      fts5BufferAppendBlob(&p->rc, &bSeek, 1, (const u8*)"\0");
    }else{
      fts5BufferSet(&p->rc, &bSeek, nToken, pToken);




    }

    pNewIter = &pNew->aSeg[0];
    pPrevIter = (pPrev ? &pPrev->aSeg[0] : 0);
    for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
      for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){
        Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];







<
<





>
>
>
>







6892
6893
6894
6895
6896
6897
6898


6899
6900
6901
6902
6903
6904
6905
6906
6907
6908
6909
6910
6911
6912
6913
6914
    Fts5Iter *pNew = 0;
    Fts5SegIter *pNewIter = 0;
    Fts5SegIter *pPrevIter = 0;

    int iLvl, iSeg, ii;

    pNew = fts5MultiIterAlloc(p, pStruct->nSegment);


    if( pSmall ){
      fts5BufferSet(&p->rc, &bSeek, pSmall->n, pSmall->p);
      fts5BufferAppendBlob(&p->rc, &bSeek, 1, (const u8*)"\0");
    }else{
      fts5BufferSet(&p->rc, &bSeek, nToken, pToken);
    }
    if( p->rc ){
      sqlite3Fts5IterClose((Fts5IndexIter*)pNew);
      break;
    }

    pNewIter = &pNew->aSeg[0];
    pPrevIter = (pPrev ? &pPrev->aSeg[0] : 0);
    for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
      for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){
        Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];

Changes to ext/fts5/fts5_main.c.

114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

struct Fts5FullTable {
  Fts5Table p;                    /* Public class members from fts5Int.h */
  Fts5Storage *pStorage;          /* Document store */
  Fts5Global *pGlobal;            /* Global (connection wide) data */
  Fts5Cursor *pSortCsr;           /* Sort data from this cursor */
  int iSavepoint;                 /* Successful xSavepoint()+1 */
  int bInSavepoint;
#ifdef SQLITE_DEBUG
  struct Fts5TransactionState ts;
#endif
};

struct Fts5MatchPhrase {
  Fts5Buffer *pPoslist;           /* Pointer to current poslist */







|







114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

struct Fts5FullTable {
  Fts5Table p;                    /* Public class members from fts5Int.h */
  Fts5Storage *pStorage;          /* Document store */
  Fts5Global *pGlobal;            /* Global (connection wide) data */
  Fts5Cursor *pSortCsr;           /* Sort data from this cursor */
  int iSavepoint;                 /* Successful xSavepoint()+1 */
  
#ifdef SQLITE_DEBUG
  struct Fts5TransactionState ts;
#endif
};

struct Fts5MatchPhrase {
  Fts5Buffer *pPoslist;           /* Pointer to current poslist */
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
*/
static int fts5RenameMethod(
  sqlite3_vtab *pVtab,            /* Virtual table handle */
  const char *zName               /* New name of table */
){
  int rc;
  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
  pTab->bInSavepoint = 1;
  rc = sqlite3Fts5StorageRename(pTab->pStorage, zName);
  pTab->bInSavepoint = 0;
  return rc;
}

int sqlite3Fts5FlushToDisk(Fts5Table *pTab){
  fts5TripCursors((Fts5FullTable*)pTab);
  return sqlite3Fts5StorageSync(((Fts5FullTable*)pTab)->pStorage);
}

/*
** The xSavepoint() method.
**
** Flush the contents of the pending-terms table to disk.
*/
static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
  int rc = SQLITE_OK;
  char *zSql = 0;
  fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);

  if( pTab->bInSavepoint==0 ){
    zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')",
        pTab->p.pConfig->zDb, pTab->p.pConfig->zName, pTab->p.pConfig->zName
    );
    if( zSql ){
      pTab->bInSavepoint = 1;
      rc = sqlite3_exec(pTab->p.pConfig->db, zSql, 0, 0, 0);
      pTab->bInSavepoint = 0;
      sqlite3_free(zSql);
    }else{
      rc = SQLITE_NOMEM;
    }
    if( rc==SQLITE_OK ){
      pTab->iSavepoint = iSavepoint+1;
    }
  }

  return rc;
}

/*
** The xRelease() method.
**
** This is a no-op.







<

<
















|

|
<
<
<
<
<
<
<
<
<
<
<
<
|
|
|
<
<







2663
2664
2665
2666
2667
2668
2669

2670

2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689












2690
2691
2692


2693
2694
2695
2696
2697
2698
2699
*/
static int fts5RenameMethod(
  sqlite3_vtab *pVtab,            /* Virtual table handle */
  const char *zName               /* New name of table */
){
  int rc;
  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;

  rc = sqlite3Fts5StorageRename(pTab->pStorage, zName);

  return rc;
}

int sqlite3Fts5FlushToDisk(Fts5Table *pTab){
  fts5TripCursors((Fts5FullTable*)pTab);
  return sqlite3Fts5StorageSync(((Fts5FullTable*)pTab)->pStorage);
}

/*
** The xSavepoint() method.
**
** Flush the contents of the pending-terms table to disk.
*/
static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
  int rc = SQLITE_OK;

  fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
  rc = sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);












  if( rc==SQLITE_OK ){
    pTab->iSavepoint = iSavepoint+1;
  }


  return rc;
}

/*
** The xRelease() method.
**
** This is a no-op.
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
}

/*
** Run an integrity check on the FTS5 data structures.  Return a string
** if anything is found amiss.  Return a NULL pointer if everything is
** OK.
*/
static int fts5Integrity(
  sqlite3_vtab *pVtab,    /* the FTS5 virtual table to check */
  const char *zSchema,    /* Name of schema in which this table lives */
  const char *zTabname,   /* Name of the table itself */
  int isQuick,            /* True if this is a quick-check */
  char **pzErr            /* Write error message here */
){
  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;







|







2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
}

/*
** Run an integrity check on the FTS5 data structures.  Return a string
** if anything is found amiss.  Return a NULL pointer if everything is
** OK.
*/
static int fts5IntegrityMethod(
  sqlite3_vtab *pVtab,    /* the FTS5 virtual table to check */
  const char *zSchema,    /* Name of schema in which this table lives */
  const char *zTabname,   /* Name of the table itself */
  int isQuick,            /* True if this is a quick-check */
  char **pzErr            /* Write error message here */
){
  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
    /* xRollback     */ fts5RollbackMethod,
    /* xFindFunction */ fts5FindFunctionMethod,
    /* xRename       */ fts5RenameMethod,
    /* xSavepoint    */ fts5SavepointMethod,
    /* xRelease      */ fts5ReleaseMethod,
    /* xRollbackTo   */ fts5RollbackToMethod,
    /* xShadowName   */ fts5ShadowName,
    /* xIntegrity    */ fts5Integrity
  };

  int rc;
  Fts5Global *pGlobal = 0;

  pGlobal = (Fts5Global*)sqlite3_malloc(sizeof(Fts5Global));
  if( pGlobal==0 ){







|







3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
    /* xRollback     */ fts5RollbackMethod,
    /* xFindFunction */ fts5FindFunctionMethod,
    /* xRename       */ fts5RenameMethod,
    /* xSavepoint    */ fts5SavepointMethod,
    /* xRelease      */ fts5ReleaseMethod,
    /* xRollbackTo   */ fts5RollbackToMethod,
    /* xShadowName   */ fts5ShadowName,
    /* xIntegrity    */ fts5IntegrityMethod
  };

  int rc;
  Fts5Global *pGlobal = 0;

  pGlobal = (Fts5Global*)sqlite3_malloc(sizeof(Fts5Global));
  if( pGlobal==0 ){

Changes to ext/fts5/test/fts5misc.test.

87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
do_execsql_test 2.2.1 {
  CREATE TABLE t0(c0);
  CREATE VIRTUAL TABLE vt0 USING fts5(c0);
  BEGIN TRANSACTION;
  INSERT INTO vt0(c0) VALUES ('xyz');
}

breakpoint
do_execsql_test 2.2.2 {
  ALTER TABLE t0 RENAME TO t1;
}
do_execsql_test 2.2.3 {
  INSERT INTO vt0(vt0) VALUES('integrity-check');
}
do_execsql_test 2.2.4 {







<







87
88
89
90
91
92
93

94
95
96
97
98
99
100
do_execsql_test 2.2.1 {
  CREATE TABLE t0(c0);
  CREATE VIRTUAL TABLE vt0 USING fts5(c0);
  BEGIN TRANSACTION;
  INSERT INTO vt0(c0) VALUES ('xyz');
}


do_execsql_test 2.2.2 {
  ALTER TABLE t0 RENAME TO t1;
}
do_execsql_test 2.2.3 {
  INSERT INTO vt0(vt0) VALUES('integrity-check');
}
do_execsql_test 2.2.4 {
496
497
498
499
500
501
502















503
504
505
do_execsql_test 17.5 {
  SELECT c0 FROM t0 WHERE c0 GLOB '*f*';
} {assertionfaultproblem}
do_execsql_test 17.5 {
  SELECT c0 FROM t0 WHERE c0 GLOB '*faul*';
} {assertionfaultproblem}

















finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
do_execsql_test 17.5 {
  SELECT c0 FROM t0 WHERE c0 GLOB '*f*';
} {assertionfaultproblem}
do_execsql_test 17.5 {
  SELECT c0 FROM t0 WHERE c0 GLOB '*faul*';
} {assertionfaultproblem}

#-------------------------------------------------------------------------
reset_db
do_execsql_test 18.0 {
  BEGIN;
    CREATE VIRTUAL TABLE t1 USING fts5(text);
    ALTER TABLE t1 RENAME TO t2;
}

do_execsql_test 18.1 {
    DROP TABLE t2;
}

do_execsql_test 18.2 {
  COMMIT;
}

finish_test