SQLite4  Check-in [1f0129ac39]

/*
** Unlink database zDb and its supporting files (wal, shm, journal, and log).
** This function works with both lsm and sqlite3 databases.
*/
static int unlink_db(const char *zDb){
  int i;
  char *zBase = (char *)zDb;
  const char *azExt[] = { "", "-shm", "-wal", "-journal", "-log", 0 };

  if( 0==sqlite4_strnicmp("file:", zDb, 5) ){
    for(i=5; zDb[i] && zDb[i]!='?'; i++);
    zBase = sqlite4_mprintf(0, "%.*s", i-5, &zDb[5]);
  }

  for(i=0; azExt[i]; i++){
    char *zFile = sqlite4_mprintf(0, "%s%s", zBase, azExt[i]);
    unlink(zFile);
    sqlite4_free(0, zFile);
  }

  if( zBase!=zDb ){
    sqlite4_free(0, zBase);
  }
  return 0;
}

static char *create_schema_sql(int nIdx){
  char *zSchema;
  int i;

  return iRet;
}
/*
** End of integer query implementations.
*************************************************************************/

static int do_insert1_test4(
  const char *zFile,
  int nRow,                       /* Number of rows to insert in total */
  int nRowPerTrans,               /* Number of rows per transaction */
  int nIdx,                       /* Number of aux indexes (aside from PK) */
  int iSync                       /* PRAGMA synchronous value (0, 1 or 2) */
){
  char *zCreateTbl;               /* Create table statement */
  char *zInsert;                  /* INSERT statement */
  sqlite4_stmt *pInsert;          /* Compiled INSERT statement */
  sqlite4 *db = 0;                /* Database handle */
  int i;                          /* Counter to count nRow rows */
  int nMs;                        /* Test time in ms */

  lsm_db *pLsm;

  if( zFile==0 ) zFile = SQLITE4_DB_FILE;
  unlink_db(zFile);
  EXPLODE(  sqlite4_open(0, zFile, &db)  );
  sqlite4_kvstore_control(db, "main", SQLITE4_KVCTRL_LSM_HANDLE, &pLsm);
  i = iSync;
  lsm_config(pLsm, LSM_CONFIG_SAFETY, &i);
  assert( i==iSync );

  install_rblob_function4(db);

  nMs = testTimeGet();

  /* Print out the time taken by the test */
  printf("%.3f seconds\n", (double)nMs / 1000.0);
  return 0;
}
static int do_insert1_test3(
  const char *zFile,
  int nRow,                       /* Number of rows to insert in total */
  int nRowPerTrans,               /* Number of rows per transaction */
  int nIdx,                       /* Number of aux indexes (aside from PK) */
  int iSync                       /* PRAGMA synchronous value (0, 1 or 2) */
){
  char *zCreateTbl;               /* Create table statement */
  char *zInsert;                  /* INSERT statement */
  char *zSync;                    /* "PRAGMA synchronous=" statement */
  sqlite3_stmt *pInsert;          /* Compiled INSERT statement */
  sqlite3 *db = 0;                /* Database handle */
  int i;                          /* Counter to count nRow rows */
  int nMs;                        /* Test time in ms */

  if( zFile==0 ) zFile = SQLITE3_DB_FILE;
  unlink_db(zFile);
  EXPLODE( sqlite3_open(zFile, &db) );
  EXPLODE( sqlite3_exec(db, "PRAGMA journal_mode=WAL", 0, 0, 0) );
  zSync = sqlite4_mprintf(0, "PRAGMA synchronous=%d", iSync);
  EXPLODE( sqlite3_exec(db, zSync, 0, 0, 0) );
  sqlite4_free(0, zSync);

  install_rblob_function3(db);

    int nMax;
  } aArg[] = { 
    {"-db",           3,    4}, 
    {"-rows",         1,    10000000}, 
    {"-rowspertrans", 1,    10000000}, 
    {"-indexes",      0,    20}, 
    {"-sync",         0,    2}, 
    {"-file",         -1,    -1}, 
    {0,0,0}
  };
  int i;

  int iDb = 4;                    /* SQLite 3 or 4 */
  int nRow = 50000;               /* Total rows: 50000 */
  int nRowPerTrans = 10;          /* Total rows each transaction: 50000 */
  int nIdx = 3;                   /* Number of auxilliary indexes */
  int iSync = 1;                  /* PRAGMA synchronous setting */
  const char *zFile = 0;

  for(i=0; i<argc; i++){
    int iSel;
    int iVal;
    int rc;

    rc = testArgSelectX(aArg, "argument", sizeof(aArg[0]), argv[i], &iSel);
    if( rc!=0 ) return -1;
    if( i==argc-1 ){
      fprintf(stderr, "option %s requires an argument\n", aArg[iSel].zArg);
      return -1;
    }
    if( 0==strcmp("-file", aArg[iSel].zArg) ){
      zFile = argv[++i];
    }else{
      iVal = atoi(argv[++i]);
      if( iVal<aArg[iSel].nMin || iVal>aArg[iSel].nMax ){
        fprintf(stderr, "option %s out of range (%d..%d)\n", 
            aArg[iSel].zArg, aArg[iSel].nMin, aArg[iSel].nMax 
            );
        return -1;
      }

      switch( iSel ){
        case 0: iDb = iVal;          break;
        case 1: nRow = iVal;         break;
        case 2: nRowPerTrans = iVal; break;
        case 3: nIdx = iVal;         break;
        case 4: iSync = iVal;        break;
      }
    }
  }

  printf("insert1: db=%d rows=%d rowspertrans=%d indexes=%d sync=%d ... ", 
      iDb, nRow, nRowPerTrans, nIdx, iSync
  );
  fflush(stdout);
  if( iDb==3 ){
    do_insert1_test3(zFile, nRow, nRowPerTrans, nIdx, iSync);
  }else{
    do_insert1_test4(zFile, nRow, nRowPerTrans, nIdx, iSync);
  }

  return 0;
}

static int do_select1_test4(
  const char *zFile,
  int nRow,                       /* Number of rows to read in total */
  int nRowPerTrans,               /* Number of rows per transaction */
  int iIdx
){
  int nMs = 0;
  sqlite4_stmt *pSelect = 0;
  char *zSelect;
  sqlite4 *db;
  int i;
  int nTblRow;

  if( zFile==0 ) zFile = SQLITE4_DB_FILE;
  EXPLODE( sqlite4_open(0, zFile, &db) );
  install_rblob_function4(db);

  nTblRow = integer_query4(db, "SELECT count(*) FROM t1");

  /* Create the db schema and prepare the INSERT statement */
  zSelect = create_select_sql(iIdx);
  EXPLODE(  sqlite4_prepare(db, zSelect, -1, &pSelect, 0)  );

  sqlite4_close(db);
  sqlite4_free(0, zSelect);

  printf("%.3f seconds\n", (double)nMs / 1000.0);
  return 0;
}
static int do_select1_test3(
  const char *zFile,
  int nRow,                       /* Number of rows to read in total */
  int nRowPerTrans,               /* Number of rows per transaction */
  int iIdx
){
  int nMs = 0;
  sqlite3_stmt *pSelect = 0;
  char *zSelect;
  sqlite3 *db;
  int i;
  int nTblRow;
  

  if( zFile==0 ) zFile = SQLITE3_DB_FILE;
  EXPLODE( sqlite3_open(zFile, &db) );
  install_rblob_function3(db);

  nTblRow = integer_query3(db, "SELECT count(*) FROM t1");

  /* Create the db schema and prepare the INSERT statement */
  zSelect = create_select_sql(iIdx);
  EXPLODE(  sqlite3_prepare(db, zSelect, -1, &pSelect, 0)  );

    int nMin;
    int nMax;
  } aArg[] = {
    {"-db",           3,    4}, 
    {"-rows",         1,    10000000}, 
    {"-rowspertrans", 1,    10000000}, 
    {"-index",        0,    21}, 
    {"-file",        -1,    -1}, 
    {0,0,0}
  };
  int i;

  int iDb = 4;                    /* SQLite 3 or 4 */
  int nRow = 50000;               /* Total rows: 50000 */
  int nRowPerTrans = 10;          /* Total rows each transaction: 50000 */
  int iIdx = 0;
  const char *zFile = 0;

  for(i=0; i<argc; i++){
    int iSel;

    int rc;

    rc = testArgSelectX(aArg, "argument", sizeof(aArg[0]), argv[i], &iSel);
    if( rc!=0 ) return -1;
    if( i==argc-1 ){
      fprintf(stderr, "option %s requires an argument\n", aArg[iSel].zArg);
      return -1;
    }
    if( 0==strcmp("-file", aArg[iSel].zArg) ){
      zFile = argv[++i];
    }else{
      int iVal = atoi(argv[++i]);
      if( iVal<aArg[iSel].nMin || iVal>aArg[iSel].nMax ){
        fprintf(stderr, "option %s out of range (%d..%d)\n", 
            aArg[iSel].zArg, aArg[iSel].nMin, aArg[iSel].nMax 
            );
        return -1;
      }

      switch( iSel ){
        case 0: iDb = iVal;          break;
        case 1: nRow = iVal;         break;
        case 2: nRowPerTrans = iVal; break;
        case 3: iIdx = iVal;         break;
      }
    }
  }

  printf("select1: db=%d rows=%d rowspertrans=%d index=%d ... ", 
      iDb, nRow, nRowPerTrans, iIdx
  );
  fflush(stdout);
  if( iDb==3 ){
    do_select1_test3(zFile, nRow, nRowPerTrans, iIdx);
  }else{
    do_select1_test4(zFile, nRow, nRowPerTrans, iIdx);
  }

  return 0;
}

int main(int argc, char **argv){
  struct SqltestArg {

  if( pNew==0 ){
    rc = SQLITE4_NOMEM;
  }else{
    struct Config {
      const char *zParam;
      int eParam;
    } aConfig[] = {
      { "lsm_mmap", LSM_CONFIG_MMAP },
      { "lsm_page_size", LSM_CONFIG_PAGE_SIZE },
      { "lsm_block_size", LSM_CONFIG_BLOCK_SIZE },
      { "lsm_multiple_processes", LSM_CONFIG_MULTIPLE_PROCESSES }
    };

    memset(pNew, 0, sizeof(KVLsm));
    pNew->base.pStoreVfunc = &kvlsmMethods;
    pNew->base.pEnv = pEnv;
    rc = lsm_new(0, &pNew->pDb);
    if( rc==SQLITE4_OK ){
      int i;
      int bMmap = 0;
      lsm_config(pNew->pDb, LSM_CONFIG_MMAP, &bMmap);

      for(i=0; i<ArraySize(aConfig); i++){
        const char *zVal = sqlite4_uri_parameter(zName, aConfig[i].zParam);
        if( zVal ){
          int nVal = sqlite4Atoi(zVal);
          lsm_config(pNew->pDb, aConfig[i].eParam, &nVal);
        }
      }

void lsmEnvSleep(lsm_env *, int);

int lsmFsReadSyncedId(lsm_db *db, int, i64 *piVal);

int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, Pgno, int *);
Pgno lsmFsRedirectPage(FileSystem *, Redirect *, Pgno);

void lsmFsPurgeCache(FileSystem *);

/*
** End of functions from "lsm_file.c".
**************************************************************************/

/* 
** Functions from file "lsm_sorted.c".

    /* Make a copy of the database and log file names. */
    memcpy(pFS->zDb, zDb, nDb+1);
    memcpy(pFS->zLog, zDb, nDb);
    memcpy(&pFS->zLog[nDb], "-log", 5);

    /* Allocate the hash-table here. At some point, it should be changed
    ** so that it can grow dynamicly. */
    pFS->nCacheMax = 2048*1024 / pFS->nPagesize;
    pFS->nHash = 4096;
    pFS->apHash = lsmMallocZeroRc(pDb->pEnv, sizeof(Page *) * pFS->nHash, &rc);

    /* Open the database file */
    pLsmFile = lsmDbRecycleFd(pDb);
    if( pLsmFile ){
      pFS->pLsmFile = pLsmFile;

/*
** Configure the nominal page-size used by this file-system. Actual 
** pages may be smaller or larger than this value.
*/
void lsmFsSetPageSize(FileSystem *pFS, int nPgsz){
  pFS->nPagesize = nPgsz;
  pFS->nCacheMax = 2048*1024 / pFS->nPagesize;
}

/*
** Configure the block-size used by this file-system. Actual pages may be 
** smaller or larger than this value.
*/
void lsmFsSetBlockSize(FileSystem *pFS, int nBlocksize){

  Page **pp;

  iHash = fsHashKey(pFS->nHash, pPg->iPg);
  for(pp=&pFS->apHash[iHash]; *pp!=pPg; pp=&(*pp)->pHashNext);
  *pp = pPg->pHashNext;
}


/*
** Purge the page cache of all entries with nRef==0.
*/
void lsmFsPurgeCache(FileSystem *pFS){
  Page *pPg;

  pPg = pFS->pLruFirst;
  while( pPg ){
    Page *pNext = pPg->pLruNext;
    fsPageRemoveFromHash(pFS, pPg);
    if( pPg->flags & PAGE_FREE ){
      lsmFree(pFS->pEnv, pPg->aData);
    }
    lsmFree(pFS->pEnv, pPg);
    pPg = pNext;
    pFS->nCacheAlloc--;
  }
  pFS->pLruFirst = 0;
  pFS->pLruLast = 0;

  assert( pFS->nCacheAlloc<=pFS->nOut && pFS->nCacheAlloc>=0 );
}

/*
** Search the hash-table for page iPg. If an entry is round, return a pointer
** to it. Otherwise, return NULL.
**
** Either way, if argument piHash is not NULL set *piHash to the hash slot
** number that page iPg would be stored in before returning.
*/
static Page *fsPageFindInHash(FileSystem *pFS, Pgno iPg, int *piHash){
  Page *p;                        /* Return value */
  int iHash = fsHashKey(pFS->nHash, iPg);

  if( piHash ) *piHash = iHash;
  for(p=pFS->apHash[iHash]; p; p=p->pHashNext){
    if( p->iPg==iPg) break;
  }
  return p;
}

static int fsPageBuffer(
  FileSystem *pFS, 

  Page **ppOut
){
  int rc = LSM_OK;
  Page *pPage = 0;
  if( pFS->bUseMmap || pFS->pLruFirst==0 || pFS->nCacheAlloc<pFS->nCacheMax ){
    pPage = lsmMallocZero(pFS->pEnv, sizeof(Page));
    if( !pPage ){
      rc = LSM_NOMEM_BKPT;
    }else{
      pPage->aData = (u8 *)lsmMalloc(pFS->pEnv, pFS->nPagesize);
      pPage->flags = PAGE_FREE;
      if( !pPage->aData ){
        lsmFree(pFS->pEnv, pPage);
        rc = LSM_NOMEM_BKPT;
        pPage = 0;
      }
      pFS->nCacheAlloc++;


    }
  }else{
    u8 *aData;
    pPage = pFS->pLruFirst;
    aData = pPage->aData;
    fsPageRemoveFromLru(pFS, pPage);
    fsPageRemoveFromHash(pFS, pPage);

    memset(pPage, 0, sizeof(Page));
    pPage->flags = PAGE_FREE;
    pPage->aData = aData;
  }

  *ppOut = pPage;
  return rc;
}

static void fsPageBufferFree(Page *pPg){
  if( pPg->flags & PAGE_FREE ){
    lsmFree(pPg->pFS->pEnv, pPg->aData);

    /* Search the hash-table for the page */
    iHash = fsHashKey(pFS->nHash, iReal);
    for(p=pFS->apHash[iHash]; p; p=p->pHashNext){
      if( p->iPg==iReal) break;
    }

    if( p==0 ){
      rc = fsPageBuffer(pFS, &p);
      if( rc==LSM_OK ){
        int nSpace = 0;
        p->iPg = iReal;
        p->nRef = 0;
        p->pFS = pFS;
        assert( p->flags==0 || p->flags==PAGE_FREE );

  assert( p->pRedirect==0 );

  if( pFS->pCompress || bDefer ){
    /* In compressed database mode the page is not assigned a page number
    ** or location in the database file at this point. This will be done
    ** by the lsmFsPagePersist() call.  */
    rc = fsPageBuffer(pFS, &pPg);
    if( rc==LSM_OK ){
      pPg->pFS = pFS;
      pPg->pSeg = p;
      pPg->iPg = 0;
      pPg->flags |= PAGE_DIRTY;
      pPg->nData = pFS->nPagesize;
      assert( pPg->aData );

        i64 iOff = iFromOff + i*nSz;
        rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aData, nSz);
        if( rc==LSM_OK ){
          iOff = iToOff + i*nSz;
          rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, aData, nSz);
        }
      }
      lsmFsPurgeCache(pFS);
    }
  }

  /* Update append-point list if necessary */
  if( rc==LSM_OK ){
    int i;
    for(i=0; i<LSM_APPLIST_SZ; i++){

    if( rc==LSM_OK ) rc = lsmFsPagePersist(pPg);
    assert( pPg->nRef==1 );
    lsmFsPageRelease(pPg);
    pPg = pNext;
  }
  *pRc = rc;
}

static void fsRemoveHashEntry(FileSystem *pFS, Pgno iPg){
  Page *p;
  int iHash = fsHashKey(pFS->nHash, iPg);

  for(p=pFS->apHash[iHash]; p && p->iPg!=iPg; p=p->pHashNext);

  if( p ){
    assert( p->nRef==0 );
    fsPageRemoveFromHash(pFS, p);
    p->iPg = 0;
    iHash = fsHashKey(pFS->nHash, 0);
    p->pHashNext = pFS->apHash[iHash];
    pFS->apHash[iHash] = p;
  }
}

/*
** If the page passed as an argument is dirty, update the database file
** (or mapping of the database file) with its current contents and mark
** the page as clean.
**
** Return LSM_OK if the operation is a success, or an LSM error code

        assert( pPg->nData==pFS->nPagesize-4 );

        rc = fsAppendPage(pFS, pPg->pSeg, &pPg->iPg, &iPrev, &iNext);
        if( rc!=LSM_OK ) return rc;

        if( pFS->bUseMmap==0 ){
          int iHash = fsHashKey(pFS->nHash, pPg->iPg);
          fsRemoveHashEntry(pFS, pPg->iPg);
          pPg->pHashNext = pFS->apHash[iHash];
          pFS->apHash[iHash] = pPg;
          assert( pPg->pHashNext==0 || pPg->pHashNext->iPg!=pPg->iPg );
        }

        if( iPrev ){
          assert( iNext==0 );

      pPg->aData -= (pPg->flags & PAGE_HASPREV);
      pPg->flags &= ~PAGE_HASPREV;

      if( pFS->bUseMmap ){
        pPg->pHashNext = pFS->pFree;
        pFS->pFree = pPg;
      }else{
#if 0
        assert( pPg->pLruNext==0 );
        assert( pPg->pLruPrev==0 );
        fsPageRemoveFromHash(pFS, pPg);
        fsPageBufferFree(pPg);
#else
        fsPageAddToLru(pFS, pPg);
#endif
      }
    }
  }

  return rc;
}

    /* Load the database snapshot */
    if( rc==LSM_OK ){
      if( lsmCheckpointClientCacheOk(pDb)==0 ){
        lsmFreeSnapshot(pDb->pEnv, pDb->pClient);
        pDb->pClient = 0;
        lsmMCursorFreeCache(pDb);
        lsmFsPurgeCache(pDb->pFS);
        rc = lsmCheckpointLoad(pDb, &iSnap);
      }else{
        iSnap = 1;
      }
    }

    /* Take a read-lock on the tree and snapshot just loaded. Then check

  int nPage;                      /* Equivalent of nKB in pages */
  int nWrite = 0;                 /* Number of pages written */

  /* This function may not be called if pDb has an open read or write
  ** transaction. Return LSM_MISUSE if an application attempts this.  */
  if( pDb->nTransOpen || pDb->pCsr ) return LSM_MISUSE_BKPT;
  if( nMerge<=0 ) nMerge = pDb->nMerge;

  lsmFsPurgeCache(pDb->pFS);

  /* Convert from KB to pages */
  nPgsz = lsmFsPageSize(pDb->pFS);
  if( nKB>=0 ){
    nPage = ((i64)nKB * 1024 + nPgsz - 1) / nPgsz;
  }else{
    nPage = -1;

  int bKeys,                      /* Output the keys from each segment */
  int bVals,                      /* Output the values from each segment */
  const char *zWhy                /* Caption to print near top of dump */
){
  Snapshot *pDump = pSnap;
  Level *pTopLevel;
  char *zFree = 0;


  assert( pSnap );
  pTopLevel = lsmDbSnapshotLevel(pDump);
  if( pDb->xLog && pTopLevel ){
    static int nCall = 0;
    Level *pLevel;
    int iLevel = 0;