SQLite

Changes On Branch rtree-performance
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch rtree-performance Excluding Merge-Ins

This is equivalent to a diff from 2170312c8d to f158b7d491

2023-09-15
20:28
Simplifications and performance optimizations for the RTree extension. (check-in: 04a333f5fa user: drh tags: trunk)
2023-09-14
01:46
Reduce the number of calls to sqlite3_mprintf() made by RTree. (Closed-Leaf check-in: f158b7d491 user: drh tags: rtree-performance)
2023-09-13
17:30
Omit the Reinsert algorithm from RTree. This causes most benchmarks to run faster, at the expense of having a slightly less dense and hence larger index (example: 33 entries/node versus 34 entries/node). (check-in: b3049a1d3d user: drh tags: rtree-performance)
17:11
Add JNI sqlite3_prepare_multi(), based on feedback. (check-in: fa1c153472 user: stephan tags: trunk)
11:36
Remove unused variable from the ChooseLeaf algorithm in r-tree. (check-in: fb2e4a8067 user: drh tags: rtree-performance)
11:24
Replace an if() condition in fts5 that is always true with an assert(). (check-in: 2170312c8d user: dan tags: trunk)
2023-09-12
23:21
CLI .import to accept EOF in lieu of record terminator on last field of CSV (with multiple field records), per RFC 4180. forum post 5b21c25bdfa (check-in: 6f7842f577 user: larrybr tags: trunk)

Changes to ext/rtree/geopoly.c.
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267

1268
1269
1270
1271
1272


1273
1274
1275
1276
1277
1278
1279
  (void)pAux;

  sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1);

  /* Allocate the sqlite3_vtab structure */
  nDb = strlen(argv[1]);
  nName = strlen(argv[2]);
  pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName+2);
  if( !pRtree ){
    return SQLITE_NOMEM;
  }
  memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2);
  pRtree->nBusy = 1;
  pRtree->base.pModule = &rtreeModule;
  pRtree->zDb = (char *)&pRtree[1];
  pRtree->zName = &pRtree->zDb[nDb+1];

  pRtree->eCoordType = RTREE_COORD_REAL32;
  pRtree->nDim = 2;
  pRtree->nDim2 = 4;
  memcpy(pRtree->zDb, argv[1], nDb);
  memcpy(pRtree->zName, argv[2], nName);




  /* Create/Connect to the underlying relational database schema. If
  ** that is successful, call sqlite3_declare_vtab() to configure
  ** the r-tree table schema.
  */
  pSql = sqlite3_str_new(db);







|



|




>





>
>







1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
  (void)pAux;

  sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1);

  /* Allocate the sqlite3_vtab structure */
  nDb = strlen(argv[1]);
  nName = strlen(argv[2]);
  pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName*2+8);
  if( !pRtree ){
    return SQLITE_NOMEM;
  }
  memset(pRtree, 0, sizeof(Rtree)+nDb+nName*2+8);
  pRtree->nBusy = 1;
  pRtree->base.pModule = &rtreeModule;
  pRtree->zDb = (char *)&pRtree[1];
  pRtree->zName = &pRtree->zDb[nDb+1];
  pRtree->zNodeName = &pRtree->zName[nName+1];
  pRtree->eCoordType = RTREE_COORD_REAL32;
  pRtree->nDim = 2;
  pRtree->nDim2 = 4;
  memcpy(pRtree->zDb, argv[1], nDb);
  memcpy(pRtree->zName, argv[2], nName);
  memcpy(pRtree->zNodeName, argv[2], nName);
  memcpy(&pRtree->zNodeName[nName], "_node", 6);


  /* Create/Connect to the underlying relational database schema. If
  ** that is successful, call sqlite3_declare_vtab() to configure
  ** the r-tree table schema.
  */
  pSql = sqlite3_str_new(db);
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
    }
    *pRowid = cell.iRowid;
    if( rc==SQLITE_OK ){
      rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf);
    }
    if( rc==SQLITE_OK ){
      int rc2;
      pRtree->iReinsertHeight = -1;
      rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0);
      rc2 = nodeRelease(pRtree, pLeaf);
      if( rc==SQLITE_OK ){
        rc = rc2;
      }
    }
  }







<







1682
1683
1684
1685
1686
1687
1688

1689
1690
1691
1692
1693
1694
1695
    }
    *pRowid = cell.iRowid;
    if( rc==SQLITE_OK ){
      rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf);
    }
    if( rc==SQLITE_OK ){
      int rc2;

      rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0);
      rc2 = nodeRelease(pRtree, pLeaf);
      if( rc==SQLITE_OK ){
        rc = rc2;
      }
    }
  }
Changes to ext/rtree/rtree.c.
162
163
164
165
166
167
168

169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
#endif
#ifdef SQLITE_DEBUG
  u8 bCorrupt;                /* Shadow table corruption detected */
#endif
  int iDepth;                 /* Current depth of the r-tree structure */
  char *zDb;                  /* Name of database containing r-tree table */
  char *zName;                /* Name of r-tree table */ 

  u32 nBusy;                  /* Current number of users of this structure */
  i64 nRowEst;                /* Estimated number of rows in this table */
  u32 nCursor;                /* Number of open cursors */
  u32 nNodeRef;               /* Number RtreeNodes with positive nRef */
  char *zReadAuxSql;          /* SQL for statement to read aux data */

  /* List of nodes removed during a CondenseTree operation. List is
  ** linked together via the pointer normally used for hash chains -
  ** RtreeNode.pNext. RtreeNode.iNode stores the depth of the sub-tree 
  ** headed by the node (leaf nodes have RtreeNode.iNode==0).
  */
  RtreeNode *pDeleted;
  int iReinsertHeight;        /* Height of sub-trees Reinsert() has run on */

  /* Blob I/O on xxx_node */
  sqlite3_blob *pNodeBlob;

  /* Statements to read/write/delete a record from xxx_node */
  sqlite3_stmt *pWriteNode;
  sqlite3_stmt *pDeleteNode;







>












<







162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181

182
183
184
185
186
187
188
#endif
#ifdef SQLITE_DEBUG
  u8 bCorrupt;                /* Shadow table corruption detected */
#endif
  int iDepth;                 /* Current depth of the r-tree structure */
  char *zDb;                  /* Name of database containing r-tree table */
  char *zName;                /* Name of r-tree table */ 
  char *zNodeName;            /* Name of the %_node table */
  u32 nBusy;                  /* Current number of users of this structure */
  i64 nRowEst;                /* Estimated number of rows in this table */
  u32 nCursor;                /* Number of open cursors */
  u32 nNodeRef;               /* Number RtreeNodes with positive nRef */
  char *zReadAuxSql;          /* SQL for statement to read aux data */

  /* List of nodes removed during a CondenseTree operation. List is
  ** linked together via the pointer normally used for hash chains -
  ** RtreeNode.pNext. RtreeNode.iNode stores the depth of the sub-tree 
  ** headed by the node (leaf nodes have RtreeNode.iNode==0).
  */
  RtreeNode *pDeleted;


  /* Blob I/O on xxx_node */
  sqlite3_blob *pNodeBlob;

  /* Statements to read/write/delete a record from xxx_node */
  sqlite3_stmt *pWriteNode;
  sqlite3_stmt *pDeleteNode;
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
  sqlite3_stmt *pReadParent;
  sqlite3_stmt *pWriteParent;
  sqlite3_stmt *pDeleteParent;

  /* Statement for writing to the "aux:" fields, if there are any */
  sqlite3_stmt *pWriteAux;

  RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ 
};

/* Possible values for Rtree.eCoordType: */
#define RTREE_COORD_REAL32 0
#define RTREE_COORD_INT32  1

/*







|







196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
  sqlite3_stmt *pReadParent;
  sqlite3_stmt *pWriteParent;
  sqlite3_stmt *pDeleteParent;

  /* Statement for writing to the "aux:" fields, if there are any */
  sqlite3_stmt *pWriteAux;

  RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */
};

/* Possible values for Rtree.eCoordType: */
#define RTREE_COORD_REAL32 0
#define RTREE_COORD_INT32  1

/*
733
734
735
736
737
738
739
740
741
742

743
744
745
746
747
748
749
750
751
    pRtree->pNodeBlob = pBlob;
    if( rc ){
      nodeBlobReset(pRtree);
      if( rc==SQLITE_NOMEM ) return SQLITE_NOMEM;
    }
  }
  if( pRtree->pNodeBlob==0 ){
    char *zTab = sqlite3_mprintf("%s_node", pRtree->zName);
    if( zTab==0 ) return SQLITE_NOMEM;
    rc = sqlite3_blob_open(pRtree->db, pRtree->zDb, zTab, "data", iNode, 0,

                           &pRtree->pNodeBlob);
    sqlite3_free(zTab);
  }
  if( rc ){
    nodeBlobReset(pRtree);
    *ppNode = 0;
    /* If unable to open an sqlite3_blob on the desired row, that can only
    ** be because the shadow tables hold erroneous data. */
    if( rc==SQLITE_ERROR ){







<
<
|
>

<







733
734
735
736
737
738
739


740
741
742

743
744
745
746
747
748
749
    pRtree->pNodeBlob = pBlob;
    if( rc ){
      nodeBlobReset(pRtree);
      if( rc==SQLITE_NOMEM ) return SQLITE_NOMEM;
    }
  }
  if( pRtree->pNodeBlob==0 ){


    rc = sqlite3_blob_open(pRtree->db, pRtree->zDb, pRtree->zNodeName,
                           "data", iNode, 0,
                           &pRtree->pNodeBlob);

  }
  if( rc ){
    nodeBlobReset(pRtree);
    *ppNode = 0;
    /* If unable to open an sqlite3_blob on the desired row, that can only
    ** be because the shadow tables hold erroneous data. */
    if( rc==SQLITE_ERROR ){
2078
2079
2080
2081
2082
2083
2084

2085

2086


2087
2088
2089
2090
2091
2092
2093
        pIdxInfo->aConstraintUsage[ii].omit = doOmit;
      }
    }
  }

  pIdxInfo->idxNum = 2;
  pIdxInfo->needToFreeIdxStr = 1;

  if( iIdx>0 && 0==(pIdxInfo->idxStr = sqlite3_mprintf("%s", zIdxStr)) ){

    return SQLITE_NOMEM;


  }

  nRow = pRtree->nRowEst >> (iIdx/2);
  pIdxInfo->estimatedCost = (double)6.0 * (double)nRow;
  pIdxInfo->estimatedRows = nRow;

  return rc;







>
|
>
|
>
>







2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
        pIdxInfo->aConstraintUsage[ii].omit = doOmit;
      }
    }
  }

  pIdxInfo->idxNum = 2;
  pIdxInfo->needToFreeIdxStr = 1;
  if( iIdx>0 ){
    pIdxInfo->idxStr = sqlite3_malloc( iIdx+1 );
    if( pIdxInfo->idxStr==0 ){
      return SQLITE_NOMEM;
    }
    memcpy(pIdxInfo->idxStr, zIdxStr, iIdx+1);
  }

  nRow = pRtree->nRowEst >> (iIdx/2);
  pIdxInfo->estimatedCost = (double)6.0 * (double)nRow;
  pIdxInfo->estimatedRows = nRow;

  return rc;
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170


2171


2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196

/*
** Return true if the area covered by p2 is a subset of the area covered
** by p1. False otherwise.
*/
static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){
  int ii;
  int isInt = (pRtree->eCoordType==RTREE_COORD_INT32);
  for(ii=0; ii<pRtree->nDim2; ii+=2){
    RtreeCoord *a1 = &p1->aCoord[ii];
    RtreeCoord *a2 = &p2->aCoord[ii];
    if( (!isInt && (a2[0].f<a1[0].f || a2[1].f>a1[1].f)) 
     || ( isInt && (a2[0].i<a1[0].i || a2[1].i>a1[1].i)) 


    ){


      return 0;
    }
  }
  return 1;
}

/*
** Return the amount cell p would grow by if it were unioned with pCell.
*/
static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){
  RtreeDValue area;
  RtreeCell cell;
  memcpy(&cell, p, sizeof(RtreeCell));
  area = cellArea(pRtree, &cell);
  cellUnion(pRtree, &cell, pCell);
  return (cellArea(pRtree, &cell)-area);
}

static RtreeDValue cellOverlap(
  Rtree *pRtree, 
  RtreeCell *p, 
  RtreeCell *aCell, 
  int nCell
){
  int ii;







|
|
|
|
<
|
>
>
|
>
>
|





<
<
<
<
<
<
<
<
<
<
<
<







2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170

2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182












2183
2184
2185
2186
2187
2188
2189

/*
** Return true if the area covered by p2 is a subset of the area covered
** by p1. False otherwise.
*/
static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){
  int ii;
  if( pRtree->eCoordType==RTREE_COORD_INT32 ){
    for(ii=0; ii<pRtree->nDim2; ii+=2){
      RtreeCoord *a1 = &p1->aCoord[ii];
      RtreeCoord *a2 = &p2->aCoord[ii];

      if( a2[0].i<a1[0].i || a2[1].i>a1[1].i ) return 0;
    }
  }else{
    for(ii=0; ii<pRtree->nDim2; ii+=2){
      RtreeCoord *a1 = &p1->aCoord[ii];
      RtreeCoord *a2 = &p2->aCoord[ii];
      if( a2[0].f<a1[0].f || a2[1].f>a1[1].f ) return 0;
    }
  }
  return 1;
}













static RtreeDValue cellOverlap(
  Rtree *pRtree, 
  RtreeCell *p, 
  RtreeCell *aCell, 
  int nCell
){
  int ii;
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243





2244







2245
2246
2247





2248
2249
2250
2251

2252
2253
2254

2255
2256


2257
2258
2259
2260
2261
2262
2263

2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
  int ii;
  RtreeNode *pNode = 0;
  rc = nodeAcquire(pRtree, 1, 0, &pNode);

  for(ii=0; rc==SQLITE_OK && ii<(pRtree->iDepth-iHeight); ii++){
    int iCell;
    sqlite3_int64 iBest = 0;

    RtreeDValue fMinGrowth = RTREE_ZERO;
    RtreeDValue fMinArea = RTREE_ZERO;

    int nCell = NCELL(pNode);
    RtreeCell cell;
    RtreeNode *pChild = 0;






    RtreeCell *aCell = 0;








    /* Select the child node which will be enlarged the least if pCell
    ** is inserted into it. Resolve ties by choosing the entry with





    ** the smallest area.
    */
    for(iCell=0; iCell<nCell; iCell++){
      int bBest = 0;

      RtreeDValue growth;
      RtreeDValue area;
      nodeGetCell(pRtree, pNode, iCell, &cell);

      growth = cellGrowth(pRtree, &cell, pCell);
      area = cellArea(pRtree, &cell);


      if( iCell==0||growth<fMinGrowth||(growth==fMinGrowth && area<fMinArea) ){
        bBest = 1;
      }
      if( bBest ){
        fMinGrowth = growth;
        fMinArea = area;
        iBest = cell.iRowid;

      }
    }

    sqlite3_free(aCell);
    rc = nodeAcquire(pRtree, iBest, pNode, &pChild);
    nodeRelease(pRtree, pNode);
    pNode = pChild;
  }

  *ppLeaf = pNode;
  return rc;







|


<

<


>
>
>
>
>
|
>
>
>
>
>
>
>
|
<
<
>
>
>
>
>
|
|
|
<
>
|
|
|
>
|
|
>
>
|
<
<
|
|
|
|
>



<







2222
2223
2224
2225
2226
2227
2228
2229
2230
2231

2232

2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248


2249
2250
2251
2252
2253
2254
2255
2256

2257
2258
2259
2260
2261
2262
2263
2264
2265
2266


2267
2268
2269
2270
2271
2272
2273
2274

2275
2276
2277
2278
2279
2280
2281
  int ii;
  RtreeNode *pNode = 0;
  rc = nodeAcquire(pRtree, 1, 0, &pNode);

  for(ii=0; rc==SQLITE_OK && ii<(pRtree->iDepth-iHeight); ii++){
    int iCell;
    sqlite3_int64 iBest = 0;
    int bFound = 0;
    RtreeDValue fMinGrowth = RTREE_ZERO;
    RtreeDValue fMinArea = RTREE_ZERO;

    int nCell = NCELL(pNode);

    RtreeNode *pChild = 0;

    /* First check to see if there is are any cells in pNode that completely
    ** contains pCell.  If two or more cells in pNode completely contain pCell
    ** then pick the smallest.
    */
    for(iCell=0; iCell<nCell; iCell++){
      RtreeCell cell;
      nodeGetCell(pRtree, pNode, iCell, &cell);
      if( cellContains(pRtree, &cell, pCell) ){
        RtreeDValue area = cellArea(pRtree, &cell);
        if( bFound==0 || area<fMinArea ){
          iBest = cell.iRowid;
          fMinArea = area;
          bFound = 1;
        }


      }
    }
    if( !bFound ){
      /* No cells of pNode will completely contain pCell.  So pick the
      ** cell of pNode that grows by the least amount when pCell is added.
      ** Break ties by selecting the smaller cell.
      */
      for(iCell=0; iCell<nCell; iCell++){

        RtreeCell cell;
        RtreeDValue growth;
        RtreeDValue area;
        nodeGetCell(pRtree, pNode, iCell, &cell);
        area = cellArea(pRtree, &cell);
        cellUnion(pRtree, &cell, pCell);
        growth = cellArea(pRtree, &cell)-area;
        if( iCell==0
         || growth<fMinGrowth
         || (growth==fMinGrowth && area<fMinArea)


        ){
          fMinGrowth = growth;
          fMinArea = area;
          iBest = cell.iRowid;
        }
      }
    }


    rc = nodeAcquire(pRtree, iBest, pNode, &pChild);
    nodeRelease(pRtree, pNode);
    pNode = pChild;
  }

  *ppLeaf = pNode;
  return rc;
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
  sqlite3_step(pRtree->pWriteParent);
  return sqlite3_reset(pRtree->pWriteParent);
}

static int rtreeInsertCell(Rtree *, RtreeNode *, RtreeCell *, int);


/*
** Arguments aIdx, aDistance and aSpare all point to arrays of size
** nIdx. The aIdx array contains the set of integers from 0 to 
** (nIdx-1) in no particular order. This function sorts the values
** in aIdx according to the indexed values in aDistance. For
** example, assuming the inputs:
**
**   aIdx      = { 0,   1,   2,   3 }
**   aDistance = { 5.0, 2.0, 7.0, 6.0 }
**
** this function sets the aIdx array to contain:
**
**   aIdx      = { 0,   1,   2,   3 }
**
** The aSpare array is used as temporary working space by the
** sorting algorithm.
*/
static void SortByDistance(
  int *aIdx, 
  int nIdx, 
  RtreeDValue *aDistance, 
  int *aSpare
){
  if( nIdx>1 ){
    int iLeft = 0;
    int iRight = 0;

    int nLeft = nIdx/2;
    int nRight = nIdx-nLeft;
    int *aLeft = aIdx;
    int *aRight = &aIdx[nLeft];

    SortByDistance(aLeft, nLeft, aDistance, aSpare);
    SortByDistance(aRight, nRight, aDistance, aSpare);

    memcpy(aSpare, aLeft, sizeof(int)*nLeft);
    aLeft = aSpare;

    while( iLeft<nLeft || iRight<nRight ){
      if( iLeft==nLeft ){
        aIdx[iLeft+iRight] = aRight[iRight];
        iRight++;
      }else if( iRight==nRight ){
        aIdx[iLeft+iRight] = aLeft[iLeft];
        iLeft++;
      }else{
        RtreeDValue fLeft = aDistance[aLeft[iLeft]];
        RtreeDValue fRight = aDistance[aRight[iRight]];
        if( fLeft<fRight ){
          aIdx[iLeft+iRight] = aLeft[iLeft];
          iLeft++;
        }else{
          aIdx[iLeft+iRight] = aRight[iRight];
          iRight++;
        }
      }
    }

#if 0
    /* Check that the sort worked */
    {
      int jj;
      for(jj=1; jj<nIdx; jj++){
        RtreeDValue left = aDistance[aIdx[jj-1]];
        RtreeDValue right = aDistance[aIdx[jj]];
        assert( left<=right );
      }
    }
#endif
  }
}

/*
** Arguments aIdx, aCell and aSpare all point to arrays of size
** nIdx. The aIdx array contains the set of integers from 0 to 
** (nIdx-1) in no particular order. This function sorts the values
** in aIdx according to dimension iDim of the cells in aCell. The
** minimum value of dimension iDim is considered first, the







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







2340
2341
2342
2343
2344
2345
2346







































































2347
2348
2349
2350
2351
2352
2353
  sqlite3_step(pRtree->pWriteParent);
  return sqlite3_reset(pRtree->pWriteParent);
}

static int rtreeInsertCell(Rtree *, RtreeNode *, RtreeCell *, int);










































































/*
** Arguments aIdx, aCell and aSpare all point to arrays of size
** nIdx. The aIdx array contains the set of integers from 0 to 
** (nIdx-1) in no particular order. This function sorts the values
** in aIdx according to dimension iDim of the cells in aCell. The
** minimum value of dimension iDim is considered first, the
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
    }else{
      rc = fixBoundingBox(pRtree, pNode);
    }
  }

  return rc;
}

static int Reinsert(
  Rtree *pRtree, 
  RtreeNode *pNode, 
  RtreeCell *pCell, 
  int iHeight
){
  int *aOrder;
  int *aSpare;
  RtreeCell *aCell;
  RtreeDValue *aDistance;
  int nCell;
  RtreeDValue aCenterCoord[RTREE_MAX_DIMENSIONS];
  int iDim;
  int ii;
  int rc = SQLITE_OK;
  int n;

  memset(aCenterCoord, 0, sizeof(RtreeDValue)*RTREE_MAX_DIMENSIONS);

  nCell = NCELL(pNode)+1;
  n = (nCell+1)&(~1);

  /* Allocate the buffers used by this operation. The allocation is
  ** relinquished before this function returns.
  */
  aCell = (RtreeCell *)sqlite3_malloc64(n * (
    sizeof(RtreeCell)     +         /* aCell array */
    sizeof(int)           +         /* aOrder array */
    sizeof(int)           +         /* aSpare array */
    sizeof(RtreeDValue)             /* aDistance array */
  ));
  if( !aCell ){
    return SQLITE_NOMEM;
  }
  aOrder    = (int *)&aCell[n];
  aSpare    = (int *)&aOrder[n];
  aDistance = (RtreeDValue *)&aSpare[n];

  for(ii=0; ii<nCell; ii++){
    if( ii==(nCell-1) ){
      memcpy(&aCell[ii], pCell, sizeof(RtreeCell));
    }else{
      nodeGetCell(pRtree, pNode, ii, &aCell[ii]);
    }
    aOrder[ii] = ii;
    for(iDim=0; iDim<pRtree->nDim; iDim++){
      aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2]);
      aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2+1]);
    }
  }
  for(iDim=0; iDim<pRtree->nDim; iDim++){
    aCenterCoord[iDim] = (aCenterCoord[iDim]/(nCell*(RtreeDValue)2));
  }

  for(ii=0; ii<nCell; ii++){
    aDistance[ii] = RTREE_ZERO;
    for(iDim=0; iDim<pRtree->nDim; iDim++){
      RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) - 
                               DCOORD(aCell[ii].aCoord[iDim*2]));
      aDistance[ii] += (coord-aCenterCoord[iDim])*(coord-aCenterCoord[iDim]);
    }
  }

  SortByDistance(aOrder, nCell, aDistance, aSpare);
  nodeZero(pRtree, pNode);

  for(ii=0; rc==SQLITE_OK && ii<(nCell-(RTREE_MINCELLS(pRtree)+1)); ii++){
    RtreeCell *p = &aCell[aOrder[ii]];
    nodeInsertCell(pRtree, pNode, p);
    if( p->iRowid==pCell->iRowid ){
      if( iHeight==0 ){
        rc = rowidWrite(pRtree, p->iRowid, pNode->iNode);
      }else{
        rc = parentWrite(pRtree, p->iRowid, pNode->iNode);
      }
    }
  }
  if( rc==SQLITE_OK ){
    rc = fixBoundingBox(pRtree, pNode);
  }
  for(; rc==SQLITE_OK && ii<nCell; ii++){
    /* Find a node to store this cell in. pNode->iNode currently contains
    ** the height of the sub-tree headed by the cell.
    */
    RtreeNode *pInsert;
    RtreeCell *p = &aCell[aOrder[ii]];
    rc = ChooseLeaf(pRtree, p, iHeight, &pInsert);
    if( rc==SQLITE_OK ){
      int rc2;
      rc = rtreeInsertCell(pRtree, pInsert, p, iHeight);
      rc2 = nodeRelease(pRtree, pInsert);
      if( rc==SQLITE_OK ){
        rc = rc2;
      }
    }
  }

  sqlite3_free(aCell);
  return rc;
}

/*
** Insert cell pCell into node pNode. Node pNode is the head of a 
** subtree iHeight high (leaf nodes have iHeight==0).
*/
static int rtreeInsertCell(
  Rtree *pRtree,
  RtreeNode *pNode,
  RtreeCell *pCell,
  int iHeight
){
  int rc = SQLITE_OK;
  if( iHeight>0 ){
    RtreeNode *pChild = nodeHashLookup(pRtree, pCell->iRowid);
    if( pChild ){
      nodeRelease(pRtree, pChild->pParent);
      nodeReference(pNode);
      pChild->pParent = pNode;
    }
  }
  if( nodeInsertCell(pRtree, pNode, pCell) ){
    if( iHeight<=pRtree->iReinsertHeight || pNode->iNode==1){
      rc = SplitNode(pRtree, pNode, pCell, iHeight);
    }else{
      pRtree->iReinsertHeight = iHeight;
      rc = Reinsert(pRtree, pNode, pCell, iHeight);
    }
  }else{
    rc = AdjustTree(pRtree, pNode, pCell);
    if( ALWAYS(rc==SQLITE_OK) ){
      if( iHeight==0 ){
        rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode);
      }else{
        rc = parentWrite(pRtree, pCell->iRowid, pNode->iNode);







|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<




















<
|
<
<
<
<







2823
2824
2825
2826
2827
2828
2829
2830





































































































2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850

2851




2852
2853
2854
2855
2856
2857
2858
    }else{
      rc = fixBoundingBox(pRtree, pNode);
    }
  }

  return rc;
}
	





































































































/*
** Insert cell pCell into node pNode. Node pNode is the head of a 
** subtree iHeight high (leaf nodes have iHeight==0).
*/
static int rtreeInsertCell(
  Rtree *pRtree,
  RtreeNode *pNode,
  RtreeCell *pCell,
  int iHeight
){
  int rc = SQLITE_OK;
  if( iHeight>0 ){
    RtreeNode *pChild = nodeHashLookup(pRtree, pCell->iRowid);
    if( pChild ){
      nodeRelease(pRtree, pChild->pParent);
      nodeReference(pNode);
      pChild->pParent = pNode;
    }
  }
  if( nodeInsertCell(pRtree, pNode, pCell) ){

    rc = SplitNode(pRtree, pNode, pCell, iHeight);




  }else{
    rc = AdjustTree(pRtree, pNode, pCell);
    if( ALWAYS(rc==SQLITE_OK) ){
      if( iHeight==0 ){
        rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode);
      }else{
        rc = parentWrite(pRtree, pCell->iRowid, pNode->iNode);
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
    *pRowid = cell.iRowid;

    if( rc==SQLITE_OK ){
      rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf);
    }
    if( rc==SQLITE_OK ){
      int rc2;
      pRtree->iReinsertHeight = -1;
      rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0);
      rc2 = nodeRelease(pRtree, pLeaf);
      if( rc==SQLITE_OK ){
        rc = rc2;
      }
    }
    if( rc==SQLITE_OK && pRtree->nAux ){







<







3187
3188
3189
3190
3191
3192
3193

3194
3195
3196
3197
3198
3199
3200
    *pRowid = cell.iRowid;

    if( rc==SQLITE_OK ){
      rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf);
    }
    if( rc==SQLITE_OK ){
      int rc2;

      rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0);
      rc2 = nodeRelease(pRtree, pLeaf);
      if( rc==SQLITE_OK ){
        rc = rc2;
      }
    }
    if( rc==SQLITE_OK && pRtree->nAux ){
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801

3802
3803
3804


3805
3806
3807
3808
3809
3810
3811
  }

  sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1);

  /* Allocate the sqlite3_vtab structure */
  nDb = (int)strlen(argv[1]);
  nName = (int)strlen(argv[2]);
  pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName+2);
  if( !pRtree ){
    return SQLITE_NOMEM;
  }
  memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2);
  pRtree->nBusy = 1;
  pRtree->base.pModule = &rtreeModule;
  pRtree->zDb = (char *)&pRtree[1];
  pRtree->zName = &pRtree->zDb[nDb+1];

  pRtree->eCoordType = (u8)eCoordType;
  memcpy(pRtree->zDb, argv[1], nDb);
  memcpy(pRtree->zName, argv[2], nName);




  /* Create/Connect to the underlying relational database schema. If
  ** that is successful, call sqlite3_declare_vtab() to configure
  ** the r-tree table schema.
  */
  pSql = sqlite3_str_new(db);







|



|




>



>
>







3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
  }

  sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1);

  /* Allocate the sqlite3_vtab structure */
  nDb = (int)strlen(argv[1]);
  nName = (int)strlen(argv[2]);
  pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName*2+8);
  if( !pRtree ){
    return SQLITE_NOMEM;
  }
  memset(pRtree, 0, sizeof(Rtree)+nDb+nName*2+8);
  pRtree->nBusy = 1;
  pRtree->base.pModule = &rtreeModule;
  pRtree->zDb = (char *)&pRtree[1];
  pRtree->zName = &pRtree->zDb[nDb+1];
  pRtree->zNodeName = &pRtree->zName[nName+1];
  pRtree->eCoordType = (u8)eCoordType;
  memcpy(pRtree->zDb, argv[1], nDb);
  memcpy(pRtree->zName, argv[2], nName);
  memcpy(pRtree->zNodeName, argv[2], nName);
  memcpy(&pRtree->zNodeName[nName], "_node", 6);


  /* Create/Connect to the underlying relational database schema. If
  ** that is successful, call sqlite3_declare_vtab() to configure
  ** the r-tree table schema.
  */
  pSql = sqlite3_str_new(db);
Changes to ext/rtree/rtree8.test.
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

# This test runs many SELECT queries simultaneously against a large 
# table, causing a collision in the hash-table used to store r-tree 
# nodes internally.
#
populate_t1 1500
do_rtree_integrity_test rtree8-1.3.0 t1
do_execsql_test rtree8-1.3.1 { SELECT max(nodeno) FROM t1_node } {164}
do_test rtree8-1.3.2 {
  set rowids [execsql {SELECT min(rowid) FROM t1_rowid GROUP BY nodeno}]
  set stmt_list [list]
  foreach row $rowids {
    set stmt [sqlite3_prepare db "SELECT * FROM t1 WHERE id = $row" -1 tail]
    sqlite3_step $stmt
    lappend res_list [sqlite3_column_int $stmt 0]







|







71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

# This test runs many SELECT queries simultaneously against a large 
# table, causing a collision in the hash-table used to store r-tree 
# nodes internally.
#
populate_t1 1500
do_rtree_integrity_test rtree8-1.3.0 t1
do_execsql_test rtree8-1.3.1 { SELECT max(nodeno) FROM t1_node } {183}
do_test rtree8-1.3.2 {
  set rowids [execsql {SELECT min(rowid) FROM t1_rowid GROUP BY nodeno}]
  set stmt_list [list]
  foreach row $rowids {
    set stmt [sqlite3_prepare db "SELECT * FROM t1 WHERE id = $row" -1 tail]
    sqlite3_step $stmt
    lappend res_list [sqlite3_column_int $stmt 0]
Changes to ext/rtree/rtreeA.test.
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
  4   "SELECT * FROM t1 WHERE x1<10 AND x2>12"
}

do_execsql_test rtreeA-1.1.1 {
  SELECT rtreecheck('main', 't1')
} {{Node 1 missing from database
Wrong number of entries in %_rowid table - expected 0, actual 500
Wrong number of entries in %_parent table - expected 0, actual 23}}

do_execsql_test  rtreeA-1.2.0 { DROP TABLE t1_node } {}
do_corruption_tests rtreeA-1.2 -error "database disk image is malformed" {
  1   "SELECT * FROM t1"
  2   "SELECT * FROM t1 WHERE rowid=5"
  3   "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
  4   "SELECT * FROM t1 WHERE x1<10 AND x2>12"







|







109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
  4   "SELECT * FROM t1 WHERE x1<10 AND x2>12"
}

do_execsql_test rtreeA-1.1.1 {
  SELECT rtreecheck('main', 't1')
} {{Node 1 missing from database
Wrong number of entries in %_rowid table - expected 0, actual 500
Wrong number of entries in %_parent table - expected 0, actual 25}}

do_execsql_test  rtreeA-1.2.0 { DROP TABLE t1_node } {}
do_corruption_tests rtreeA-1.2 -error "database disk image is malformed" {
  1   "SELECT * FROM t1"
  2   "SELECT * FROM t1 WHERE rowid=5"
  3   "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
  4   "SELECT * FROM t1 WHERE x1<10 AND x2>12"
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
  3   "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
}

do_execsql_test rtreeA-3.3.3.4 {
  SELECT rtreecheck('main', 't1')
} {{Rtree depth out of range (65535)
Wrong number of entries in %_rowid table - expected 0, actual 499
Wrong number of entries in %_parent table - expected 0, actual 23}}

#-------------------------------------------------------------------------
# Set the "number of entries" field on some nodes incorrectly.
#
create_t1
populate_t1
do_test rtreeA-4.1.0 { 







|







187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
  3   "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
}

do_execsql_test rtreeA-3.3.3.4 {
  SELECT rtreecheck('main', 't1')
} {{Rtree depth out of range (65535)
Wrong number of entries in %_rowid table - expected 0, actual 499
Wrong number of entries in %_parent table - expected 0, actual 25}}

#-------------------------------------------------------------------------
# Set the "number of entries" field on some nodes incorrectly.
#
create_t1
populate_t1
do_test rtreeA-4.1.0 { 
Changes to ext/rtree/rtreedoc.test.
597
598
599
600
601
602
603
604
605

606
607


608







609
610
611
612
613
614
615
    AND A.maxY>=B.minY AND A.minY<=B.maxY
    AND B.id=28269
} {$step < 100}
do_execsql_test 2.5.2 {
  SELECT A.id FROM demo_index AS A, demo_index AS B
    WHERE A.maxX>=B.minX AND A.minX<=B.maxX
    AND A.maxY>=B.minY AND A.minY<=B.maxY
    AND B.id=28269;
} {

  28293 28216 28322 28286 28269 
  28215 28336 28262 28291 28320 


  28313 28298 28287







}

# EVIDENCE-OF: R-02723-34107 Note that it is not necessary for all
# coordinates in an R*Tree index to be constrained in order for the
# index search to be efficient.
#
# EVIDENCE-OF: R-22490-27246 One might, for example, want to query all







|

>
|
|
>
>
|
>
>
>
>
>
>
>







597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
    AND A.maxY>=B.minY AND A.minY<=B.maxY
    AND B.id=28269
} {$step < 100}
do_execsql_test 2.5.2 {
  SELECT A.id FROM demo_index AS A, demo_index AS B
    WHERE A.maxX>=B.minX AND A.minX<=B.maxX
    AND A.maxY>=B.minY AND A.minY<=B.maxY
    AND B.id=28269 ORDER BY +A.id;
} {
  28215
  28216
  28262
  28269
  28286
  28287
  28291
  28293
  28298
  28313
  28320
  28322
  28336
}

# EVIDENCE-OF: R-02723-34107 Note that it is not necessary for all
# coordinates in an R*Tree index to be constrained in order for the
# index search to be efficient.
#
# EVIDENCE-OF: R-22490-27246 One might, for example, want to query all
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
# in the %_parent table as there are non-leaf cells in the r-tree
# structure, and that there is a non-leaf cell that corresponds to each
# entry in the %_parent table.
execsql BEGIN
do_test 3.6 {
  execsql { INSERT INTO rt2_parent VALUES(1000, 1000) }
  execsql { SELECT rtreecheck('rt2') }
} {{Wrong number of entries in %_parent table - expected 9, actual 10}}
execsql ROLLBACK



finish_test







|





1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
# in the %_parent table as there are non-leaf cells in the r-tree
# structure, and that there is a non-leaf cell that corresponds to each
# entry in the %_parent table.
execsql BEGIN
do_test 3.6 {
  execsql { INSERT INTO rt2_parent VALUES(1000, 1000) }
  execsql { SELECT rtreecheck('rt2') }
} {{Wrong number of entries in %_parent table - expected 10, actual 11}}
execsql ROLLBACK



finish_test
Changes to ext/rtree/rtreefuzz001.test.
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
|   2848: 00 00 00 00 00 00 00 00 00 00 00 00 00 89 50 01   ..............P.
|   2864: 04 00 93 24 00 01 00 02 00 00 00 00 00 00 00 02   ...$............
|   2880: ff ff ff 06 00 00 00 0c 00 00 00 01 00 00 00 0b   ................
|   2896: 00 00 00 00 00 00 00 02 40 00 00 00 00 00 00 00   ........@.......
| end crash-2e81f5dce5cbd4.db}]
  execsql { PRAGMA writable_schema = 1;}
  catchsql {UPDATE t1 SET ex= ex ISNULL}
} {1 {database disk image is malformed}}

do_test rtreefuzz001-600 {
  sqlite3 db {}
  db deserialize [decode_hexdb {
| size 20480 pagesize 4096 filename crash-7b37d80f000235.db
| page 1 offset 0
|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.







|







1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
|   2848: 00 00 00 00 00 00 00 00 00 00 00 00 00 89 50 01   ..............P.
|   2864: 04 00 93 24 00 01 00 02 00 00 00 00 00 00 00 02   ...$............
|   2880: ff ff ff 06 00 00 00 0c 00 00 00 01 00 00 00 0b   ................
|   2896: 00 00 00 00 00 00 00 02 40 00 00 00 00 00 00 00   ........@.......
| end crash-2e81f5dce5cbd4.db}]
  execsql { PRAGMA writable_schema = 1;}
  catchsql {UPDATE t1 SET ex= ex ISNULL}
} {0 {}}

do_test rtreefuzz001-600 {
  sqlite3 db {}
  db deserialize [decode_hexdb {
| size 20480 pagesize 4096 filename crash-7b37d80f000235.db
| page 1 offset 0
|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.