SQLITE_NOTICE(283): recovered 1 frames from WAL file /fossil/sqlite.fossil-wal

SQLite: Check-in [bacfe8cb]
/ Check-in [bacfe8cb]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Avoid constantly freeing and reallocing small internal buffers associated with LSM cursors. Instead, allow them to persist for the lifetime of the cursor if they are LSM_SEGMENTPTR_FREE_THRESHOLD (default 1024) bytes or smaller. This change is based on research by Martijn Blaauw.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: bacfe8cb3e4d3be736c6789b2ce55ef5603e5c7a289b05b37cae2203cd7f2290
User & Date: dan 2017-06-30 11:44:11
Original Comment: Avoid constantly freeing and reallocing small internal buffers associated with LSM cursors. Instead, allow them to persist for the lifetime of the cursor if they are LSM_SEGMENTPTR_FREE_THRESHOLD (default 1024) bytes or smaller.
Context
2017-06-30
18:12
Fix some minor typos in lsm1. check-in: 0ef777d7 user: mistachkin tags: trunk
11:44
Avoid constantly freeing and reallocing small internal buffers associated with LSM cursors. Instead, allow them to persist for the lifetime of the cursor if they are LSM_SEGMENTPTR_FREE_THRESHOLD (default 1024) bytes or smaller. This change is based on research by Martijn Blaauw. check-in: bacfe8cb user: dan tags: trunk
2017-06-29
21:33
In the command-line shell, add the -quote option to start up in quote mode. Enhance the ".mode" command so that it reports the current output mode if given no arguments. check-in: 5e3f9ea5 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Show Whitespace Changes Patch

Changes to ext/lsm1/lsm_sorted.c.

    94     94   
    95     95   #define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry)
    96     96   
    97     97   #define SEGMENT_BTREE_FLAG     0x0001
    98     98   #define PGFTR_SKIP_NEXT_FLAG   0x0002
    99     99   #define PGFTR_SKIP_THIS_FLAG   0x0004
   100    100   
          101  +
          102  +#ifndef LSM_SEGMENTPTR_FREE_THRESHOLD
          103  +# define LSM_SEGMENTPTR_FREE_THRESHOLD 1024
          104  +#endif
          105  +
   101    106   typedef struct SegmentPtr SegmentPtr;
   102    107   typedef struct Blob Blob;
   103    108   
   104    109   struct Blob {
   105    110     lsm_env *pEnv;
   106    111     void *pData;
   107    112     int nData;
................................................................................
  1144   1149       pLevel->nSplitKey = blob.nData;
  1145   1150       lsmFsPageRelease(pPg);
  1146   1151     }
  1147   1152   
  1148   1153     *pRc = rc;
  1149   1154   }
  1150   1155   
  1151         -static void segmentPtrReset(SegmentPtr *pPtr){
         1156  +/*
         1157  +** Reset a segment cursor. Also free its buffers if they are nThreshold
         1158  +** bytes or larger in size.
         1159  +*/
         1160  +static void segmentPtrReset(SegmentPtr *pPtr, int nThreshold){
  1152   1161     lsmFsPageRelease(pPtr->pPg);
  1153   1162     pPtr->pPg = 0;
  1154   1163     pPtr->nCell = 0;
  1155   1164     pPtr->pKey = 0;
  1156   1165     pPtr->nKey = 0;
  1157   1166     pPtr->pVal = 0;
  1158   1167     pPtr->nVal = 0;
  1159   1168     pPtr->eType = 0;
  1160   1169     pPtr->iCell = 0;
  1161         -  sortedBlobFree(&pPtr->blob1);
  1162         -  sortedBlobFree(&pPtr->blob2);
         1170  +  if( pPtr->blob1.nAlloc>=nThreshold ) sortedBlobFree(&pPtr->blob1);
         1171  +  if( pPtr->blob2.nAlloc>=nThreshold ) sortedBlobFree(&pPtr->blob2);
  1163   1172   }
  1164   1173   
  1165   1174   static int segmentPtrIgnoreSeparators(MultiCursor *pCsr, SegmentPtr *pPtr){
  1166   1175     return (pCsr->flags & CURSOR_READ_SEPARATORS)==0
  1167   1176         || (pPtr!=&pCsr->aPtr[pCsr->nPtr-1]);
  1168   1177   }
  1169   1178   
................................................................................
  1202   1211       if( rc!=LSM_OK ) return rc;
  1203   1212   
  1204   1213       if( svFlags && pPtr->pPg ){
  1205   1214         int res = sortedKeyCompare(pCsr->pDb->xCmp,
  1206   1215             rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey,
  1207   1216             pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey
  1208   1217         );
  1209         -      if( res<0 ) segmentPtrReset(pPtr);
         1218  +      if( res<0 ) segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD);
  1210   1219       }
  1211   1220   
  1212   1221       if( pPtr->pPg==0 && (svFlags & LSM_END_DELETE) ){
  1213   1222         Segment *pSeg = pPtr->pSeg;
  1214   1223         rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, pSeg->iFirst, &pPtr->pPg);
  1215   1224         if( rc!=LSM_OK ) return rc;
  1216   1225         pPtr->eType = LSM_START_DELETE | LSM_POINT_DELETE;
................................................................................
  1271   1280     if( rc==LSM_OK && pPtr->pPg ){
  1272   1281       rc = segmentPtrLoadCell(pPtr, bLast ? (pPtr->nCell-1) : 0);
  1273   1282       if( rc==LSM_OK && bLast && pPtr->pSeg!=&pLvl->lhs ){
  1274   1283         int res = sortedKeyCompare(pCsr->pDb->xCmp,
  1275   1284             rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey,
  1276   1285             pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey
  1277   1286         );
  1278         -      if( res<0 ) segmentPtrReset(pPtr);
         1287  +      if( res<0 ) segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD);
  1279   1288       }
  1280   1289     }
  1281   1290     
  1282   1291     bIgnore = segmentPtrIgnoreSeparators(pCsr, pPtr);
  1283   1292     if( rc==LSM_OK && pPtr->pPg && bIgnore && rtIsSeparator(pPtr->eType) ){
  1284   1293       rc = segmentPtrAdvance(pCsr, pPtr, bLast);
  1285   1294     }
................................................................................
  1607   1616       ptr.pLevel = pPtr->pLevel;
  1608   1617       ptr.pSeg = &ptr.pLevel->aRhs[ptr.pLevel->nRight-1];
  1609   1618       rc = sortedRhsFirst(pCsr, ptr.pLevel, &ptr);
  1610   1619       if( rc==LSM_OK ){
  1611   1620         rc = ptrFwdPointer(ptr.pPg, ptr.iCell, ptr.pSeg, &iOut, &bFound);
  1612   1621         ptr.pPg = 0;
  1613   1622       }
  1614         -    segmentPtrReset(&ptr);
         1623  +    segmentPtrReset(&ptr, 0);
  1615   1624     }
  1616   1625   
  1617   1626     *piPtr = iOut;
  1618   1627     return rc;
  1619   1628   }
  1620   1629   
  1621   1630   static int segmentPtrSeek(
................................................................................
  1654   1663   #endif
  1655   1664   
  1656   1665     assert( pPtr->nCell>0 
  1657   1666          || pPtr->pSeg->nSize==1 
  1658   1667          || lsmFsDbPageIsLast(pPtr->pSeg, pPtr->pPg)
  1659   1668     );
  1660   1669     if( pPtr->nCell==0 ){
  1661         -    segmentPtrReset(pPtr);
         1670  +    segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD);
  1662   1671     }else{
  1663   1672       iMin = 0;
  1664   1673       iMax = pPtr->nCell-1;
  1665   1674   
  1666   1675       while( 1 ){
  1667   1676         int iTry = (iMin+iMax)/2;
  1668   1677         void *pKeyT; int nKeyT;       /* Key for cell iTry */
................................................................................
  1712   1721                 pCsr->eType = pPtr->eType;
  1713   1722                 rc = sortedBlobSet(pEnv, &pCsr->key, pPtr->pKey, pPtr->nKey);
  1714   1723                 if( rc==LSM_OK ){
  1715   1724                   rc = sortedBlobSet(pEnv, &pCsr->val, pPtr->pVal, pPtr->nVal);
  1716   1725                 }
  1717   1726                 pCsr->flags |= CURSOR_SEEK_EQ;
  1718   1727               }
  1719         -            segmentPtrReset(pPtr);
         1728  +            segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD);
  1720   1729               break;
  1721   1730             }
  1722   1731             case LSM_SEEK_LE:
  1723   1732               if( res>0 ) rc = segmentPtrAdvance(pCsr, pPtr, 1);
  1724   1733               break;
  1725   1734             case LSM_SEEK_GE: {
  1726   1735               /* Figure out if we need to 'skip' the pointer forward or not */
................................................................................
  1939   1948         /* If the segment-pointer has settled on a key that is smaller than
  1940   1949         ** the splitkey, invalidate the segment-pointer.  */
  1941   1950         if( pPtr->pPg ){
  1942   1951           res = sortedKeyCompare(pCsr->pDb->xCmp, 
  1943   1952               rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, 
  1944   1953               pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey
  1945   1954           );
  1946         -        if( res<0 ) segmentPtrReset(pPtr);
         1955  +        if( res<0 ) segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD);
  1947   1956         }
  1948   1957   
  1949   1958         if( aPtr[i].pKey ) bHit = 1;
  1950   1959       }
  1951   1960   
  1952   1961       if( rc==LSM_OK && eSeek==LSM_SEEK_LE && bHit==0 ){
  1953   1962         rc = segmentPtrEnd(pCsr, &aPtr[0], 1);
................................................................................
  2215   2224   
  2216   2225     /* Close the tree cursor, if any. */
  2217   2226     lsmTreeCursorDestroy(pCsr->apTreeCsr[0]);
  2218   2227     lsmTreeCursorDestroy(pCsr->apTreeCsr[1]);
  2219   2228   
  2220   2229     /* Reset the segment pointers */
  2221   2230     for(i=0; i<pCsr->nPtr; i++){
  2222         -    segmentPtrReset(&pCsr->aPtr[i]);
         2231  +    segmentPtrReset(&pCsr->aPtr[i], 0);
  2223   2232     }
  2224   2233   
  2225   2234     /* And the b-tree cursor, if any */
  2226   2235     btreeCursorFree(pCsr->pBtCsr);
  2227   2236   
  2228   2237     /* Free allocations */
  2229   2238     lsmFree(pEnv, pCsr->aPtr);
................................................................................
  2870   2879         for(iRhs=0; iRhs<pLvl->nRight && rc==LSM_OK; iRhs++){
  2871   2880           rc = segmentPtrEnd(pCsr, &pPtr[iRhs+1], 1);
  2872   2881           if( pPtr[iRhs+1].pPg ) bHit = 1;
  2873   2882         }
  2874   2883         if( bHit==0 && rc==LSM_OK ){
  2875   2884           rc = segmentPtrEnd(pCsr, pPtr, 1);
  2876   2885         }else{
  2877         -        segmentPtrReset(pPtr);
         2886  +        segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD);
  2878   2887         }
  2879   2888       }else{
  2880   2889         int bLhs = (pPtr->pSeg==&pLvl->lhs);
  2881   2890         assert( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[0] );
  2882   2891   
  2883   2892         if( bLhs ){
  2884   2893           rc = segmentPtrEnd(pCsr, pPtr, 0);
  2885   2894           if( pPtr->pKey ) bHit = 1;
  2886   2895         }
  2887   2896         for(iRhs=0; iRhs<pLvl->nRight && rc==LSM_OK; iRhs++){
  2888   2897           if( bHit ){
  2889         -          segmentPtrReset(&pPtr[iRhs+1]);
         2898  +          segmentPtrReset(&pPtr[iRhs+1], LSM_SEGMENTPTR_FREE_THRESHOLD);
  2890   2899           }else{
  2891   2900             rc = sortedRhsFirst(pCsr, pLvl, &pPtr[iRhs+bLhs]);
  2892   2901           }
  2893   2902         }
  2894   2903       }
  2895   2904       i += pLvl->nRight;
  2896   2905     }
................................................................................
  2965   2974   }
  2966   2975   
  2967   2976   void lsmMCursorReset(MultiCursor *pCsr){
  2968   2977     int i;
  2969   2978     lsmTreeCursorReset(pCsr->apTreeCsr[0]);
  2970   2979     lsmTreeCursorReset(pCsr->apTreeCsr[1]);
  2971   2980     for(i=0; i<pCsr->nPtr; i++){
  2972         -    segmentPtrReset(&pCsr->aPtr[i]);
         2981  +    segmentPtrReset(&pCsr->aPtr[i], LSM_SEGMENTPTR_FREE_THRESHOLD);
  2973   2982     }
  2974   2983     pCsr->key.nData = 0;
  2975   2984   }
  2976   2985   
  2977   2986   static int treeCursorSeek(
  2978   2987     MultiCursor *pCsr,
  2979   2988     TreeCursor *pTreeCsr, 
................................................................................
  6075   6084           if( ptr1.pPg==0 ){
  6076   6085             assert( 0 );
  6077   6086           }
  6078   6087         }
  6079   6088       }
  6080   6089     }
  6081   6090   
  6082         -  segmentPtrReset(&ptr1);
  6083         -  segmentPtrReset(&ptr2);
         6091  +  segmentPtrReset(&ptr1, 0);
         6092  +  segmentPtrReset(&ptr2, 0);
  6084   6093     return LSM_OK;
  6085   6094   }
  6086   6095   
  6087   6096   /*
  6088   6097   ** This function is only included in the build if LSM_DEBUG_EXPENSIVE is 
  6089   6098   ** defined. Its only purpose is to evaluate various assert() statements to 
  6090   6099   ** verify that the database is well formed in certain respects.