SQLite

AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE

lib_LTLIBRARIES = libsqlite3.la
libsqlite3_la_SOURCES = sqlite3.c
libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8

bin_PROGRAMS = sqlite3
sqlite3_SOURCES = shell.c sqlite3.c sqlite3.h
sqlite3_LDADD = @READLINE_LIBS@
sqlite3_SOURCES = shell.c sqlite3.h
EXTRA_sqlite3_SOURCES = sqlite3.c
sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS

include_HEADERS = sqlite3.h sqlite3ext.h

EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs
pkgconfigdir = ${libdir}/pkgconfig

#   --enable-static-shell
#
AC_ARG_ENABLE(static-shell, [AS_HELP_STRING(
  [--enable-static-shell], 
  [statically link libsqlite3 into shell tool [default=yes]])], 
  [], [enable_static_shell=yes])
if test x"$enable_static_shell" == "xyes"; then
  EXTRA_SHELL_OBJ=sqlite3.$OBJEXT
  EXTRA_SHELL_OBJ=sqlite3-sqlite3.$OBJEXT
else
  EXTRA_SHELL_OBJ=libsqlite3.la
fi
AC_SUBST(EXTRA_SHELL_OBJ)
#-----------------------------------------------------------------------

AC_CHECK_FUNCS(posix_fallocate)

  }

  if( iLevel==FTS3_SEGCURSOR_ALL ){
    /* This call is to merge all segments in the database to a single
    ** segment. The level of the new segment is equal to the numerically
    ** greatest segment level currently present in the database for this
    ** index. The idx of the new segment is always 0.  */
    if( csr.nSegment==1 ){
    if( csr.nSegment==1 && 0==fts3SegReaderIsPending(csr.apSegment[0]) ){
      rc = SQLITE_DONE;
      goto finished;
    }
    iNewLevel = iMaxLevel;
    bIgnoreEmpty = 1;

  }else{

  fts5_tokenizer *pTokApi;

  /* Values loaded from the %_config table */
  int iCookie;                    /* Incremented when %_config is modified */
  int pgsz;                       /* Approximate page size used in %_data */
  int nAutomerge;                 /* 'automerge' setting */
  int nCrisisMerge;               /* Maximum allowed segments per level */
  int nUsermerge;                 /* 'usermerge' setting */
  int nHashSize;                  /* Bytes of memory for in-memory hash */
  char *zRank;                    /* Name of rank function */
  char *zRankArgs;                /* Arguments to rank function */

  /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
  char **pzErrmsg;

Fts5ExprNode *sqlite3Fts5ParseNode(
  Fts5Parse *pParse,
  int eType,
  Fts5ExprNode *pLeft,
  Fts5ExprNode *pRight,
  Fts5ExprNearset *pNear
);

Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
  Fts5Parse *pParse,
  Fts5ExprNode *pLeft,
  Fts5ExprNode *pRight
);

Fts5ExprPhrase *sqlite3Fts5ParseTerm(
  Fts5Parse *pParse, 
  Fts5ExprPhrase *pPhrase, 
  Fts5Token *pToken,
  int bPrefix
);

*/


#include "fts5Int.h"

#define FTS5_DEFAULT_PAGE_SIZE   4050
#define FTS5_DEFAULT_AUTOMERGE      4
#define FTS5_DEFAULT_USERMERGE      4
#define FTS5_DEFAULT_CRISISMERGE   16
#define FTS5_DEFAULT_HASHSIZE    (1024*1024)

/* Maximum allowed page size */
#define FTS5_MAX_PAGE_SIZE (128*1024)

static int fts5_iswhitespace(char x){

    memcpy(zOut, zIn, nIn+1);
    if( fts5_isopenquote(zOut[0]) ){
      int ii = fts5Dequote(zOut);
      zRet = &zIn[ii];
      *pbQuoted = 1;
    }else{
      zRet = fts5ConfigSkipBareword(zIn);
      if( zRet ){
      zOut[zRet-zIn] = '\0';
        zOut[zRet-zIn] = '\0';
      }
    }
  }

  if( zRet==0 ){
    sqlite3_free(zOut);
  }else{
    *pzOut = zOut;

    if( nAutomerge<0 || nAutomerge>64 ){
      *pbBadkey = 1;
    }else{
      if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE;
      pConfig->nAutomerge = nAutomerge;
    }
  }

  else if( 0==sqlite3_stricmp(zKey, "usermerge") ){
    int nUsermerge = -1;
    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
      nUsermerge = sqlite3_value_int(pVal);
    }
    if( nUsermerge<2 || nUsermerge>16 ){
      *pbBadkey = 1;
    }else{
      pConfig->nUsermerge = nUsermerge;
    }
  }

  else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){
    int nCrisisMerge = -1;
    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
      nCrisisMerge = sqlite3_value_int(pVal);
    }
    if( nCrisisMerge<0 ){

  sqlite3_stmt *p = 0;
  int rc = SQLITE_OK;
  int iVersion = 0;

  /* Set default values */
  pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
  pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE;
  pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE;
  pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
  pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;

  zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
  if( zSql ){
    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0);
    sqlite3_free(zSql);

      }
      pNew->pIndex = 0;
      pNew->pConfig = pConfig;
      pNew->apExprPhrase = sParse.apPhrase;
      pNew->nPhrase = sParse.nPhrase;
      sParse.apPhrase = 0;
    }
  }else{
    sqlite3Fts5ParseNodeFree(sParse.pExpr);
  }

  sqlite3_free(sParse.apPhrase);
  *pzErr = sParse.zErr;
  return sParse.rc;
}

  int rc = SQLITE_OK;
  pNode->bEof = 0;
  pNode->bNomatch = 0;

  if( Fts5NodeIsString(pNode) ){
    /* Initialize all term iterators in the NEAR object. */
    rc = fts5ExprNearInitAll(pExpr, pNode);
  }else if( pNode->xNext==0 ){
    pNode->bEof = 1;
  }else{
    int i;
    int nEof = 0;
    for(i=0; i<pNode->nChild && rc==SQLITE_OK; i++){
      Fts5ExprNode *pChild = pNode->apChild[i];
      rc = fts5ExprNodeFirst(pExpr, pNode->apChild[i]);
      assert( pChild->bEof==0 || pChild->bEof==1 );

** equal to iFirst.
**
** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
** is not considered an error if the query does not match any documents.
*/
int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){
  Fts5ExprNode *pRoot = p->pRoot;
  int rc = SQLITE_OK;
  if( pRoot->xNext ){
    p->pIndex = pIdx;
    p->bDesc = bDesc;
    rc = fts5ExprNodeFirst(p, pRoot);
  int rc;                         /* Return code */

  p->pIndex = pIdx;
  p->bDesc = bDesc;
  rc = fts5ExprNodeFirst(p, pRoot);

    /* If not at EOF but the current rowid occurs earlier than iFirst in
    ** the iteration order, move to document iFirst or later. */
    if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){
      rc = fts5ExprNodeNext(p, pRoot, 1, iFirst);
    }
  /* If not at EOF but the current rowid occurs earlier than iFirst in
  ** the iteration order, move to document iFirst or later. */
  if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){
    rc = fts5ExprNodeNext(p, pRoot, 1, iFirst);
  }

    /* If the iterator is not at a real match, skip forward until it is. */
    while( pRoot->bNomatch ){
      assert( pRoot->bEof==0 && rc==SQLITE_OK );
      rc = fts5ExprNodeNext(p, pRoot, 0, 0);
  /* If the iterator is not at a real match, skip forward until it is. */
  while( pRoot->bNomatch ){
    assert( pRoot->bEof==0 && rc==SQLITE_OK );
    rc = fts5ExprNodeNext(p, pRoot, 0, 0);
    }
  }
  return rc;
}

/*
** Move to the next document 
**

  }

  if( pRet==0 ){
    assert( pParse->rc!=SQLITE_OK );
    sqlite3Fts5ParseNearsetFree(pNear);
    sqlite3Fts5ParsePhraseFree(pPhrase);
  }else{
    if( pRet->nPhrase>0 ){
      Fts5ExprPhrase *pLast = pRet->apPhrase[pRet->nPhrase-1];
      assert( pLast==pParse->apPhrase[pParse->nPhrase-2] );
      if( pPhrase->nTerm==0 ){
        fts5ExprPhraseFree(pPhrase);
        pRet->nPhrase--;
        pParse->nPhrase--;
        pPhrase = pLast;
      }else if( pLast->nTerm==0 ){
        fts5ExprPhraseFree(pLast);
        pParse->apPhrase[pParse->nPhrase-2] = pPhrase;
        pParse->nPhrase--;
        pRet->nPhrase--;
      }
    }
    pRet->apPhrase[pRet->nPhrase++] = pPhrase;
  }
  return pRet;
}

typedef struct TokenCtx TokenCtx;
struct TokenCtx {

  Fts5ExprPhrase *pPhrase = pCtx->pPhrase;

  UNUSED_PARAM2(iUnused1, iUnused2);

  /* If an error has already occurred, this is a no-op */
  if( pCtx->rc!=SQLITE_OK ) return pCtx->rc;

  assert( pPhrase==0 || pPhrase->nTerm>0 );
  if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){
  if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED) ){
    Fts5ExprTerm *pSyn;
    int nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1;
    pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte);
    if( pSyn==0 ){
      rc = SQLITE_NOMEM;
    }else{
      memset(pSyn, 0, nByte);

    rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize);
  }
  sqlite3_free(z);
  if( rc || (rc = sCtx.rc) ){
    pParse->rc = rc;
    fts5ExprPhraseFree(sCtx.pPhrase);
    sCtx.pPhrase = 0;
  }else if( sCtx.pPhrase ){
  }else{

    if( pAppend==0 ){
      if( (pParse->nPhrase % 8)==0 ){
        int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8);
        Fts5ExprPhrase **apNew;
        apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte);
        if( apNew==0 ){
          pParse->rc = SQLITE_NOMEM;
          fts5ExprPhraseFree(sCtx.pPhrase);
          return 0;
        }
        pParse->apPhrase = apNew;
      }
      pParse->nPhrase++;
    }

    pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
    assert( sCtx.pPhrase->nTerm>0 );
    sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
    if( sCtx.pPhrase==0 ){
      /* This happens when parsing a token or quoted phrase that contains
      ** no token characters at all. (e.g ... MATCH '""'). */
      sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase));
    }else if( sCtx.pPhrase->nTerm ){
      sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
    }
    pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
  }

  return sCtx.pPhrase;
}

/*
** Create a new FTS5 expression by cloning phrase iPhrase of the

}

void sqlite3Fts5ParseSetDistance(
  Fts5Parse *pParse, 
  Fts5ExprNearset *pNear,
  Fts5Token *p
){
  if( pNear ){
  int nNear = 0;
  int i;
  if( p->n ){
    for(i=0; i<p->n; i++){
      char c = (char)p->p[i];
      if( c<'0' || c>'9' ){
        sqlite3Fts5ParseError(
            pParse, "expected integer, got \"%.*s\"", p->n, p->p
        );
        return;
      }
      nNear = nNear * 10 + (p->p[i] - '0');
    }
  }else{
    nNear = FTS5_DEFAULT_NEARDIST;
  }
  pNear->nNear = nNear;
    int nNear = 0;
    int i;
    if( p->n ){
      for(i=0; i<p->n; i++){
        char c = (char)p->p[i];
        if( c<'0' || c>'9' ){
          sqlite3Fts5ParseError(
              pParse, "expected integer, got \"%.*s\"", p->n, p->p
              );
          return;
        }
        nNear = nNear * 10 + (p->p[i] - '0');
      }
    }else{
      nNear = FTS5_DEFAULT_NEARDIST;
    }
    pNear->nNear = nNear;
  }
}

/*
** The second argument passed to this function may be NULL, or it may be
** an existing Fts5Colset object. This function returns a pointer to
** a new colset object containing the contents of (p) with new value column
** number iCol appended. 

      pRet->eType = eType;
      pRet->pNear = pNear;
      fts5ExprAssignXNext(pRet);
      if( eType==FTS5_STRING ){
        int iPhrase;
        for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
          pNear->apPhrase[iPhrase]->pNode = pRet;
          if( pNear->apPhrase[iPhrase]->nTerm==0 ){
            pRet->xNext = 0;
            pRet->eType = FTS5_EOF;
          }
        }

        if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL 
         && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm!=1)
         && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1)
        ){
          assert( pParse->rc==SQLITE_OK );
          pParse->rc = SQLITE_ERROR;
          assert( pParse->zErr==0 );
          pParse->zErr = sqlite3_mprintf(
              "fts5: %s queries are not supported (detail!=full)", 
              pNear->nPhrase==1 ? "phrase": "NEAR"

  if( pRet==0 ){
    assert( pParse->rc!=SQLITE_OK );
    sqlite3Fts5ParseNodeFree(pLeft);
    sqlite3Fts5ParseNodeFree(pRight);
    sqlite3Fts5ParseNearsetFree(pNear);
  }
  return pRet;
}

Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
  Fts5Parse *pParse,              /* Parse context */
  Fts5ExprNode *pLeft,            /* Left hand child expression */
  Fts5ExprNode *pRight            /* Right hand child expression */
){
  Fts5ExprNode *pRet = 0;
  Fts5ExprNode *pPrev;

  if( pParse->rc ){
    sqlite3Fts5ParseNodeFree(pLeft);
    sqlite3Fts5ParseNodeFree(pRight);
  }else{

    assert( pLeft->eType==FTS5_STRING 
        || pLeft->eType==FTS5_TERM
        || pLeft->eType==FTS5_EOF
        || pLeft->eType==FTS5_AND
    );
    assert( pRight->eType==FTS5_STRING 
        || pRight->eType==FTS5_TERM 
        || pRight->eType==FTS5_EOF 
    );

    if( pLeft->eType==FTS5_AND ){
      pPrev = pLeft->apChild[pLeft->nChild-1];
    }else{
      pPrev = pLeft;
    }
    assert( pPrev->eType==FTS5_STRING 
        || pPrev->eType==FTS5_TERM 
        || pPrev->eType==FTS5_EOF 
        );

    if( pRight->eType==FTS5_EOF ){
      assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] );
      sqlite3Fts5ParseNodeFree(pRight);
      pRet = pLeft;
      pParse->nPhrase--;
    }
    else if( pPrev->eType==FTS5_EOF ){
      Fts5ExprPhrase **ap;

      if( pPrev==pLeft ){
        pRet = pRight;
      }else{
        pLeft->apChild[pLeft->nChild-1] = pRight;
        pRet = pLeft;
      }

      ap = &pParse->apPhrase[pParse->nPhrase-1-pRight->pNear->nPhrase];
      assert( ap[0]==pPrev->pNear->apPhrase[0] );
      memmove(ap, &ap[1], sizeof(Fts5ExprPhrase*)*pRight->pNear->nPhrase);
      pParse->nPhrase--;

      sqlite3Fts5ParseNodeFree(pPrev);
    }
    else{
      pRet = sqlite3Fts5ParseNode(pParse, FTS5_AND, pLeft, pRight, 0);
    }
  }

  return pRet;
}

static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
  int nByte = 0;
  Fts5ExprTerm *p;
  char *zQuoted;

  }

  return zRet;
}

static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){
  char *zRet = 0;
  if( pExpr->eType==0 ){
    return sqlite3_mprintf("\"\"");
  }else
  if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){
    Fts5ExprNearset *pNear = pExpr->pNear;
    int i; 
    int iTerm;

    if( pNear->pColset ){
      int iCol = pNear->pColset->aiCol[0];

    for(i=0; i<pExpr->nChild; i++){
      char *z = fts5ExprPrint(pConfig, pExpr->apChild[i]);
      if( z==0 ){
        sqlite3_free(zRet);
        zRet = 0;
      }else{
        int e = pExpr->apChild[i]->eType;
        int b = (e!=FTS5_STRING && e!=FTS5_TERM);
        int b = (e!=FTS5_STRING && e!=FTS5_TERM && e!=FTS5_EOF);
        zRet = fts5PrintfAppend(zRet, "%s%s%z%s", 
            (i==0 ? "" : zOp),
            (b?"(":""), z, (b?")":"")
        );
      }
      if( zRet==0 ) break;
    }

  fts5MultiIterFree(pIter);
  fts5BufferFree(&term);
  if( pnRem ) *pnRem -= writer.nLeafWritten;
}

/*
** Do up to nPg pages of automerge work on the index.
**
** Return true if any changes were actually made, or false otherwise.
*/
static void fts5IndexMerge(
static int fts5IndexMerge(
  Fts5Index *p,                   /* FTS5 backend object */
  Fts5Structure **ppStruct,       /* IN/OUT: Current structure of index */
  int nPg                         /* Pages of work to do */
  int nPg,                        /* Pages of work to do */
  int nMin                        /* Minimum number of segments to merge */
){
  int nRem = nPg;
  int bRet = 0;
  Fts5Structure *pStruct = *ppStruct;
  while( nRem>0 && p->rc==SQLITE_OK ){
    int iLvl;                   /* To iterate through levels */
    int iBestLvl = 0;           /* Level offering the most input segments */
    int nBest = 0;              /* Number of input segments on best level */

    /* Set iBestLvl to the level to read input segments from. */

    /* If nBest is still 0, then the index must be empty. */
#ifdef SQLITE_DEBUG
    for(iLvl=0; nBest==0 && iLvl<pStruct->nLevel; iLvl++){
      assert( pStruct->aLevel[iLvl].nSeg==0 );
    }
#endif

    if( nBest<p->pConfig->nAutomerge 
        && pStruct->aLevel[iBestLvl].nMerge==0 
    if( nBest<nMin && pStruct->aLevel[iBestLvl].nMerge==0 ){
      ){
      break;
    }
    bRet = 1;
    fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem);
    if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){
      fts5StructurePromote(p, iBestLvl+1, pStruct);
    }
  }
  *ppStruct = pStruct;
  return bRet;
}

/*
** A total of nLeaf leaf pages of data has just been flushed to a level-0
** segment. This function updates the write-counter accordingly and, if
** necessary, performs incremental merge work.
**

    /* Update the write-counter. While doing so, set nWork. */
    nWrite = pStruct->nWriteCounter;
    nWork = (int)(((nWrite + nLeaf) / p->nWorkUnit) - (nWrite / p->nWorkUnit));
    pStruct->nWriteCounter += nLeaf;
    nRem = (int)(p->nWorkUnit * nWork * pStruct->nLevel);

    fts5IndexMerge(p, ppStruct, nRem);
    fts5IndexMerge(p, ppStruct, nRem, p->pConfig->nAutomerge);
  }
}

static void fts5IndexCrisismerge(
  Fts5Index *p,                   /* FTS5 backend object */
  Fts5Structure **ppStruct        /* IN/OUT: Current structure of index */
){

  if( p->nPendingData ){
    assert( p->pHash );
    p->nPendingData = 0;
    fts5FlushOneHash(p);
  }
}


int sqlite3Fts5IndexOptimize(Fts5Index *p){
  Fts5Structure *pStruct;
static Fts5Structure *fts5IndexOptimizeStruct(
  Fts5Index *p, 
  Fts5Structure *pStruct
){
  Fts5Structure *pNew = 0;
  int nByte = sizeof(Fts5Structure);
  int nSeg = 0;
  int nSeg = pStruct->nSegment;
  int i;

  assert( p->rc==SQLITE_OK );
  fts5IndexFlush(p);
  pStruct = fts5StructureRead(p);

  if( pStruct ){
    assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
  /* Figure out if this structure requires optimization. A structure does
  ** not require optimization if either:
  **
  **  + it consists of fewer than two segments, or 
  **  + all segments are on the same level, or
  **  + all segments except one are currently inputs to a merge operation.
  **
  ** In the first case, return NULL. In the second, increment the ref-count
  ** on *pStruct and return a copy of the pointer to it.
  */
  if( nSeg<2 ) return 0;
  for(i=0; i<pStruct->nLevel; i++){
    int nThis = pStruct->aLevel[i].nSeg;
    if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){
      fts5StructureRef(pStruct);
      return pStruct;
    }
    assert( pStruct->aLevel[i].nMerge<=nThis );
  }

    nSeg = pStruct->nSegment;
    if( nSeg>1 ){
      int nByte = sizeof(Fts5Structure);
      nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
      pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
    }
  nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
  pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);

  }
  if( pNew ){
    Fts5StructureLevel *pLvl;
    int nByte = nSeg * sizeof(Fts5StructureSegment);
    pNew->nLevel = pStruct->nLevel+1;
    pNew->nRef = 1;
    pNew->nWriteCounter = pStruct->nWriteCounter;
    pLvl = &pNew->aLevel[pStruct->nLevel];

      pNew->nSegment = pLvl->nSeg = nSeg;
    }else{
      sqlite3_free(pNew);
      pNew = 0;
    }
  }

  return pNew;
}

int sqlite3Fts5IndexOptimize(Fts5Index *p){
  Fts5Structure *pStruct;
  Fts5Structure *pNew = 0;

  assert( p->rc==SQLITE_OK );
  fts5IndexFlush(p);
  pStruct = fts5StructureRead(p);

  if( pStruct ){
    pNew = fts5IndexOptimizeStruct(p, pStruct);
  }
  fts5StructureRelease(pStruct);

  assert( pNew==0 || pNew->nSegment>0 );
  if( pNew ){
    int iLvl = pNew->nLevel-1;
    int iLvl;
    for(iLvl=0; pNew->aLevel[iLvl].nSeg==0; iLvl++){}
    while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){
      int nRem = FTS5_OPT_WORK_UNIT;
      fts5IndexMergeLevel(p, &pNew, iLvl, &nRem);
    }

    fts5StructureWrite(p, pNew);
    fts5StructureRelease(pNew);
  }

  fts5StructureRelease(pStruct);
  return fts5IndexReturn(p); 
}

/*
** This is called to implement the special "VALUES('merge', $nMerge)"
** INSERT command.
*/
int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
  Fts5Structure *pStruct;

  Fts5Structure *pStruct = fts5StructureRead(p);
  if( pStruct ){
    int nMin = p->pConfig->nUsermerge;
    if( nMerge<0 ){
      Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct);
      fts5StructureRelease(pStruct);
      pStruct = pNew;
      nMin = 2;
      nMerge = nMerge*-1;
    }
  pStruct = fts5StructureRead(p);
  if( pStruct && pStruct->nLevel ){
    fts5IndexMerge(p, &pStruct, nMerge);
    fts5StructureWrite(p, pStruct);
  }
  fts5StructureRelease(pStruct);

    if( pStruct && pStruct->nLevel ){
      if( fts5IndexMerge(p, &pStruct, nMerge, nMin) ){
        fts5StructureWrite(p, pStruct);
      }
    }
    fts5StructureRelease(pStruct);
  }
  return fts5IndexReturn(p);
}

static void fts5AppendRowid(
  Fts5Index *p,
  i64 iDelta,
  Fts5Iter *pUnused,

      pTab->base.zErrMsg = sqlite3_mprintf(
          "cannot %s contentless fts5 table: %s", 
          (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName
      );
      rc = SQLITE_ERROR;
    }

    /* Case 1: DELETE */
    /* DELETE */
    else if( nArg==1 ){
      i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0);
    }

    /* Case 2: INSERT */
    /* INSERT */
    else if( eType0!=SQLITE_INTEGER ){     
      /* If this is a REPLACE, first remove the current entry (if any) */
      if( eConflict==SQLITE_REPLACE 
       && sqlite3_value_type(apVal[1])==SQLITE_INTEGER 
      ){
        i64 iNew = sqlite3_value_int64(apVal[1]);  /* Rowid to delete */
        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
      }
      fts5StorageInsert(&rc, pTab, apVal, pRowid);
    }

    /* Case 2: UPDATE */
    /* UPDATE */
    else{
      i64 iOld = sqlite3_value_int64(apVal[0]);  /* Old rowid */
      i64 iNew = sqlite3_value_int64(apVal[1]);  /* New rowid */
      if( iOld!=iNew ){
        if( eConflict==SQLITE_REPLACE ){
          rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
          if( rc==SQLITE_OK ){

/*
** Return a pointer to the fts5_api pointer for database connection db.
** If an error occurs, return NULL and leave an error in the database 
** handle (accessible using sqlite3_errcode()/errmsg()).
*/
static fts5_api *fts5_api_from_db(sqlite3 *db){
static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){
  fts5_api *pRet = 0;
  sqlite3_stmt *pStmt = 0;
  int rc;

  *ppApi = 0;
  if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0)
   && SQLITE_ROW==sqlite3_step(pStmt) 
   && sizeof(pRet)==sqlite3_column_bytes(pStmt, 0)
  ){
    memcpy(&pRet, sqlite3_column_blob(pStmt, 0), sizeof(pRet));
  }
  sqlite3_finalize(pStmt);
  return pRet;
  rc = sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0);
  if( rc==SQLITE_OK ){
    if( SQLITE_ROW==sqlite3_step(pStmt) 
        && sizeof(fts5_api*)==sqlite3_column_bytes(pStmt, 0)
      ){
      memcpy(ppApi, sqlite3_column_blob(pStmt, 0), sizeof(fts5_api*));
    }
    rc = sqlite3_finalize(pStmt);
  }

  return rc;
}


/*
** Argument f should be a flag accepted by matchinfo() (a valid character
** in the string passed as the second argument). If it is not, -1 is 
** returned. Otherwise, if f is a valid matchinfo flag, the value returned

int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){
  int rc;                         /* Return code */
  fts5_api *pApi;                 /* FTS5 API functions */

  /* Extract the FTS5 API pointer from the database handle. The 
  ** fts5_api_from_db() function above is copied verbatim from the 
  ** FTS5 documentation. Refer there for details. */
  pApi = fts5_api_from_db(db);
  rc = fts5_api_from_db(db, &pApi);
  if( rc!=SQLITE_OK ) return rc;

  /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered
  ** with this database handle, or an error (OOM perhaps?) has occurred.
  **
  ** Also check that the fts5_api object is version 2 or newer.  
  */ 
  if( pApi==0 || pApi->iVersion<2 ){

}

expr(A) ::= LP expr(X) RP. {A = X;}
expr(A) ::= exprlist(X).   {A = X;}

exprlist(A) ::= cnearset(X). {A = X;}
exprlist(A) ::= exprlist(X) cnearset(Y). {
  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
  A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
}

cnearset(A) ::= nearset(X). { 
  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
}
cnearset(A) ::= colset(X) COLON nearset(Y). { 
  sqlite3Fts5ParseSetColset(pParse, Y, X);

    fts5_test_queryphrase
    fts5_test_phrasecount
  } {
    sqlite3_fts5_create_function $db $f $f
  }
}

proc fts5_segcount {tbl} {
  set N 0
  foreach n [fts5_level_segs $tbl] { incr N $n }
  set N
}

proc fts5_level_segs {tbl} {
  set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
  set ret [list]
  foreach L [lrange [db one $sql] 1 end] {
    lappend ret [expr [llength $L] - 3]
  }

  set res [list 1 {malformed detail=... directive}]
  do_catchsql_test 11.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
}

do_catchsql_test 12.1 {
  INSERT INTO t1(t1, rank) VALUES('rank', NULL);;
} {1 {SQL logic error or missing database}}

#-------------------------------------------------------------------------
# errors in the 'usermerge' option
#
do_execsql_test 13.0 {
  CREATE VIRTUAL TABLE tt USING fts5(ttt);
}
foreach {tn val} {
  1     -1
  2     4.2
  3     17
  4     1
} {
  set sql "INSERT INTO tt(tt, rank) VALUES('usermerge', $val)"
  do_catchsql_test 13.$tn $sql {1 {SQL logic error or missing database}}
}

finish_test

  do_execsql_test $tn {SELECT fts5_expr($se_expr)} [list $res]
}

foreach {tn expr res} {
  1  {abc}                            {"abc"}
  2  {abc ""}                         {"abc"}
  3  {""}                             {}
  4  {abc OR ""}                      {"abc"}
  5  {abc NOT ""}                     {"abc"}
  6  {abc AND ""}                     {"abc"}
  7  {"" OR abc}                      {"abc"}
  8  {"" NOT abc}                     {"abc"}
  9  {"" AND abc}                     {"abc"}
  4  {abc OR ""}                      {"abc" OR ""}
  5  {abc NOT ""}                     {"abc" NOT ""}
  6  {abc AND ""}                     {"abc" AND ""}
  7  {"" OR abc}                      {"" OR "abc"}
  8  {"" NOT abc}                     {"" NOT "abc"}
  9  {"" AND abc}                     {"" AND "abc"}
  10 {abc + "" + def}                 {"abc" + "def"}
  11 {abc "" def}                     {"abc" AND "def"}
  12 {r+e OR w}                       {"r" + "e" OR "w"}

  13 {a AND b NOT c}                  {"a" AND ("b" NOT "c")}
  14 {a OR b NOT c}                   {"a" OR ("b" NOT "c")}
  15 {a NOT b AND c}                  {("a" NOT "b") AND "c"}

  if {[detail_is_none]==0} {
    do_faultsim_test 3 -faults oom-* -body {
      execsql { SELECT rowid FROM t1('b:2') }
    } -test {
      faultsim_test_result {0 {1 3}} {1 SQLITE_NOMEM}
    }
  }

} ;# foreach_detail_mode...


do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE x2 USING fts5(a);
  INSERT INTO x2(x2, rank) VALUES('crisismerge', 2);
  INSERT INTO x2(x2, rank) VALUES('pgsz', 32);
  INSERT INTO x2 VALUES('a b c d');
  INSERT INTO x2 VALUES('e f g h');
  INSERT INTO x2 VALUES('i j k l');
  INSERT INTO x2 VALUES('m n o p');
  INSERT INTO x2 VALUES('q r s t');
  INSERT INTO x2 VALUES('u v w x');
  INSERT INTO x2 VALUES('y z a b');
}
faultsim_save_and_close

do_faultsim_test 4 -faults oom-* -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { INSERT INTO x2(x2) VALUES('optimize') }
} -test {
  faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
}


finish_test

# 2014 June 17
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the FTS5 module.
#

source [file join [file dirname [info script]] fts5_common.tcl]
return_if_no_fts5
set testprefix fts5fuzz1


#-------------------------------------------------------------------------
reset_db
do_catchsql_test 1.1 {
  CREATE VIRTUAL TABLE f1 USING fts5(a b);
} {/1 {parse error in.*}/}


#-------------------------------------------------------------------------
reset_db
do_execsql_test 2.1 {
  CREATE VIRTUAL TABLE f1 USING fts5(a, b);
  INSERT INTO f1 VALUES('a b', 'c d');
  INSERT INTO f1 VALUES('e f', 'a b');
}

do_execsql_test 2.2.1 {
  SELECT rowid FROM f1('""');
} {}

do_execsql_test 2.2.2 {
  SELECT rowid FROM f1('"" AND a');
} {}


do_execsql_test 2.2.3 {
  SELECT rowid FROM f1('"" a');
} {1 2}

do_execsql_test 2.2.4 {
  SELECT rowid FROM f1('"" OR a');
} {1 2}

do_execsql_test 2.3 {
  SELECT a, b FROM f1('NEAR("")');
} {}

do_execsql_test 2.4 {
  SELECT a, b FROM f1('NEAR("", 5)');
} {}

do_execsql_test 2.5 {
  SELECT a, b FROM f1('NEAR("" c, 5)');
} {{a b} {c d}}

do_execsql_test 2.6 {
  SELECT a, b FROM f1('NEAR("" c d, 5)');
} {{a b} {c d}}

do_execsql_test 2.7 {
  SELECT a, b FROM f1('NEAR(c d, 5)');
} {{a b} {c d}}

do_execsql_test 2.8 {
  SELECT rowid FROM f1('NEAR("a" "b", 5)');
} {1 2}

#-------------------------------------------------------------------------
reset_db
do_execsql_test 3.2 {
  CREATE VIRTUAL TABLE f2 USING fts5(o, t, tokenize="ascii separators abc");
  SELECT * FROM f2('a+4');
} {}



#-------------------------------------------------------------------------
reset_db
do_catchsql_test 4.1 {
  CREATE VIRTUAL TABLE f2 USING fts5(o, t);
  SELECT * FROM f2('(8 AND 9)`AND 10');
} {1 {fts5: syntax error near "`"}}

finish_test

    WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<$::nRowPerSeg)
      INSERT INTO x8 SELECT repeat('x y ', i % 16) FROM ii;

    WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<$::nRowPerSeg)
      INSERT INTO x8 SELECT repeat('x y ', i % 16) FROM ii;

    INSERT INTO x8(x8, rank) VALUES('automerge', 2);
    INSERT INTO x8(x8, rank) VALUES('usermerge', 2);
  }

  for {set tn 1} {[lindex [fts5_level_segs x8] 0]>0} {incr tn} {
    do_execsql_test $testname.$tn {
      INSERT INTO x8(x8, rank) VALUES('merge', 1);
      INSERT INTO x8(x8) VALUES('integrity-check');
    }

  set ::nRow $nRow
  do_test $testname.1 {
    for {set i 0} {$i < $::nRow} {incr i} {
      execsql { INSERT INTO x8 VALUES( rnddoc(($i%16) + 5) ) }
      while {[not_merged x8]} {
        execsql {
          INSERT INTO x8(x8, rank) VALUES('automerge', 2);
          INSERT INTO x8(x8, rank) VALUES('usermerge', 2);
          INSERT INTO x8(x8, rank) VALUES('merge', 1);
          INSERT INTO x8(x8, rank) VALUES('automerge', 16);
          INSERT INTO x8(x8, rank) VALUES('usermerge', 16);
          INSERT INTO x8(x8) VALUES('integrity-check');
        }
      }
    }
  } {}
}
proc not_merged {tbl} {
  set segs [fts5_level_segs $tbl]
  foreach s $segs { if {$s>1} { return 1 } }
  return 0
}

do_merge2_test 2.1    5
do_merge2_test 2.2   10
do_merge2_test 2.3   20

#-------------------------------------------------------------------------
# Test that an auto-merge will complete any merge that has already been
# Test that a merge will complete any merge that has already been
# started, even if the number of input segments is less than the current
# value of the 'automerge' configuration parameter.
# value of the 'usermerge' configuration parameter.
#
db func rnddoc fts5_rnddoc

do_execsql_test 3.1 {
  DROP TABLE IF EXISTS x8;
  CREATE VIRTUAL TABLE x8 USING fts5(i);
  INSERT INTO x8(x8, rank) VALUES('pgsz', 32);
  INSERT INTO x8 VALUES(rnddoc(100));
  INSERT INTO x8 VALUES(rnddoc(100));
}
do_test 3.2 {
  execsql {
    INSERT INTO x8(x8, rank) VALUES('automerge', 4);
    INSERT INTO x8(x8, rank) VALUES('usermerge', 4);
    INSERT INTO x8(x8, rank) VALUES('merge', 1);
  }
  fts5_level_segs x8
} {2}

do_test 3.3 {
  execsql {
    INSERT INTO x8(x8, rank) VALUES('automerge', 2);
    INSERT INTO x8(x8, rank) VALUES('usermerge', 2);
    INSERT INTO x8(x8, rank) VALUES('merge', 1);
  }
  fts5_level_segs x8
} {2 1}

do_test 3.4 {
  execsql { INSERT INTO x8(x8, rank) VALUES('automerge', 4) }
  execsql { INSERT INTO x8(x8, rank) VALUES('usermerge', 4) }
  while {[not_merged x8]} {
    execsql { INSERT INTO x8(x8, rank) VALUES('merge', 1) }
  }
  fts5_level_segs x8
} {0 1}

#-------------------------------------------------------------------------

  }

  do_execsql_test 4.$tn.3 {
    WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
      INSERT INTO x8 SELECT mydoc() FROM ii;
    WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
      INSERT INTO x8 SELECT mydoc() FROM ii;
    INSERT INTO x8(x8, rank) VALUES('automerge', 2);
    INSERT INTO x8(x8, rank) VALUES('usermerge', 2);
  }

  set expect [mycount]
    for {set i 0} {$i < 20} {incr i} {
      do_test 4.$tn.4.$i {
        execsql { INSERT INTO x8(x8, rank) VALUES('merge', 1); }
        mycount
      } $expect
      break
    }
#  db eval {SELECT fts5_decode(rowid, block) AS r FROM x8_data} { puts $r }
}

#-------------------------------------------------------------------------
# Test that the 'merge' command does not modify the database if there is
# no work to do. 

do_execsql_test 5.1 {
  CREATE VIRTUAL TABLE x9 USING fts5(one, two);
  INSERT INTO x9(x9, rank) VALUES('pgsz', 32);
  INSERT INTO x9(x9, rank) VALUES('automerge', 2);
  INSERT INTO x9(x9, rank) VALUES('usermerge', 2);
  INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
  INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
  INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
  INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
  INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
  INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
  INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
  INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
}

do_test 5.2 {
  while 1 {
    set nChange [db total_changes]
    execsql { INSERT INTO x9(x9, rank) VALUES('merge', 1); }
    set nChange [expr [db total_changes] - $nChange]
    #puts $nChange
    if {$nChange<2} break
  }
} {}


#--------------------------------------------------------------------------
# Test that running 'merge' on an empty database does not cause a 
# problem.
#
reset_db
do_execsql_test 6.0 {
  CREATE VIRTUAL TABLE g1 USING fts5(a, b);
}
do_execsql_test 6.1 {
  INSERT INTO g1(g1, rank) VALUES('merge', 10);
}
do_execsql_test 6.2 {
  INSERT INTO g1(g1, rank) VALUES('merge', -10);
}
do_execsql_test 6.3 {
  INSERT INTO g1(g1) VALUES('integrity-check');
}



finish_test

set testprefix fts5optimize

# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}

#
# 1.* - Warm body tests for index optimization using ('optimize')
#
# 2.* - Warm body tests for index optimization using ('merge', -1)
#

proc rnddoc {nWord} {
  set vocab {a b c d e f g h i j k l m n o p q r s t u v w x y z}
  set nVocab [llength $vocab]
  set ret [list]
  for {set i 0} {$i < $nWord} {incr i} {
    lappend ret [lindex $vocab [expr {int(rand() * $nVocab)}]]
  }
  return $ret
}


foreach {tn nStep} {
  1 2
  2 10
  3 50
  4 500
} {
if {$tn!=4} continue
  reset_db
  db func rnddoc rnddoc
  do_execsql_test 1.$tn.1 {
    CREATE VIRTUAL TABLE t1 USING fts5(x, y);
  }
  do_test 1.$tn.2 {
    for {set i 0} {$i < $nStep} {incr i} {

  do_execsql_test 1.$tn.4 {
    INSERT INTO t1(t1) VALUES('optimize');
  }

  do_execsql_test 1.$tn.5 {
    INSERT INTO t1(t1) VALUES('integrity-check');
  }
}


  do_test 1.$tn.6 { fts5_segcount t1 } 1
}

foreach {tn nStep} {
  1 2
  2 10
  3 50
  4 500
} {
  reset_db
  db func rnddoc rnddoc
  do_execsql_test 1.$tn.1 {
    CREATE VIRTUAL TABLE t1 USING fts5(x, y);
  }
  do_test 2.$tn.2 {
    for {set i 0} {$i < $nStep} {incr i} {
      execsql { INSERT INTO t1 VALUES( rnddoc(5), rnddoc(5) ) }
    }
  } {}

  do_execsql_test 2.$tn.3 {
    INSERT INTO t1(t1) VALUES('integrity-check');
  }

  do_test 2.$tn.4 {
    execsql { INSERT INTO t1(t1, rank) VALUES('merge', -1) }
    while 1 {
      set c [db total_changes]
      execsql { INSERT INTO t1(t1, rank) VALUES('merge', 1) }
      set c [expr [db total_changes]-$c]
      if {$c<2} break
    }
  } {}

  do_execsql_test 2.$tn.5 {
    INSERT INTO t1(t1) VALUES('integrity-check');
  }

  do_test 2.$tn.6 { fts5_segcount t1 } 1
}
finish_test

  sqlite3_value **argv
){
  const unsigned char *zIn = sqlite3_value_text(argv[0]);
  int nIn = sqlite3_value_bytes(argv[0]);
  int c, sz;
  int scriptMask = 0;
  int res;
  int seenDigit = 0;
# define SCRIPT_LATIN       0x0001
# define SCRIPT_CYRILLIC    0x0002
# define SCRIPT_GREEK       0x0004
# define SCRIPT_HEBREW      0x0008
# define SCRIPT_ARABIC      0x0010

  while( nIn>0 ){
    c = utf8Read(zIn, nIn, &sz);
    zIn += sz;
    nIn -= sz;
    if( c<0x02af ){
    if( c<0x02af && (c>=0x80 || midClass[c&0x7f]<CCLASS_DIGIT) ){
      scriptMask |= SCRIPT_LATIN;
      if( c>=0x80 || midClass[c&0x7f]<CCLASS_DIGIT ){
        scriptMask |= SCRIPT_LATIN;
      }else if( c>='0' && c<='9' ){
        seenDigit = 1;
      }
    }else if( c>=0x0400 && c<=0x04ff ){
      scriptMask |= SCRIPT_CYRILLIC;
    }else if( c>=0x0386 && c<=0x03ce ){
      scriptMask |= SCRIPT_GREEK;
    }else if( c>=0x0590 && c<=0x05ff ){
      scriptMask |= SCRIPT_HEBREW;
    }else if( c>=0x0600 && c<=0x06ff ){
      scriptMask |= SCRIPT_ARABIC;
    }
  }
  if( scriptMask==0 && seenDigit ) scriptMask = SCRIPT_LATIN;
  switch( scriptMask ){
    case 0:                res = 999; break;
    case SCRIPT_LATIN:     res = 215; break;
    case SCRIPT_CYRILLIC:  res = 220; break;
    case SCRIPT_GREEK:     res = 200; break;
    case SCRIPT_HEBREW:    res = 125; break;
    case SCRIPT_ARABIC:    res = 160; break;

  if( sqlite3StrICmp(z, "_ROWID_")==0 ) return 1;
  if( sqlite3StrICmp(z, "ROWID")==0 ) return 1;
  if( sqlite3StrICmp(z, "OID")==0 ) return 1;
  return 0;
}

/*
** Return true if we are able to the IN operator optimization on a
** query of the form
** pX is the RHS of an IN operator.  If pX is a SELECT statement 
** that can be simplified to a direct table access, then return
**
**       x IN (SELECT ...)
** a pointer to the SELECT statement.  If pX is not a SELECT statement,
**
** Where the SELECT... clause is as specified by the parameter to this
** routine.
** or if the SELECT statement needs to be manifested into a transient
** table, then return NULL.
**
** The Select object passed in has already been preprocessed and no
** errors have been found.
*/
#ifndef SQLITE_OMIT_SUBQUERY
static int isCandidateForInOpt(Select *p){
static Select *isCandidateForInOpt(Expr *pX){
  Select *p;
  SrcList *pSrc;
  ExprList *pEList;
  Expr *pRes;
  Table *pTab;
  if( p==0 ) return 0;                   /* right-hand side of IN is SELECT */
  if( !ExprHasProperty(pX, EP_xIsSelect) ) return 0;  /* Not a subquery */
  if( ExprHasProperty(pX, EP_VarSelect)  ) return 0;  /* Correlated subq */
  p = pX->x.pSelect;
  if( p->pPrior ) return 0;              /* Not a compound SELECT */
  if( p->selFlags & (SF_Distinct|SF_Aggregate) ){
    testcase( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct );
    testcase( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate );
    return 0; /* No DISTINCT keyword and no aggregate functions */
  }
  assert( p->pGroupBy==0 );              /* Has no GROUP BY clause */
  if( p->pLimit ) return 0;              /* Has no LIMIT clause */
  assert( p->pOffset==0 );               /* No LIMIT means no OFFSET */
  if( p->pWhere ) return 0;              /* Has no WHERE clause */
  pSrc = p->pSrc;
  assert( pSrc!=0 );
  if( pSrc->nSrc!=1 ) return 0;          /* Single term in FROM clause */
  if( pSrc->a[0].pSelect ) return 0;     /* FROM is not a subquery or view */
  pTab = pSrc->a[0].pTab;
  if( NEVER(pTab==0) ) return 0;
  assert( pTab!=0 );
  assert( pTab->pSelect==0 );            /* FROM clause is not a view */
  if( IsVirtual(pTab) ) return 0;        /* FROM clause not a virtual table */
  pEList = p->pEList;
  if( pEList->nExpr!=1 ) return 0;       /* One column in the result set */
  pRes = pEList->a[0].pExpr;
  if( pEList->a[0].pExpr->op!=TK_COLUMN ) return 0; /* Result is a column */
  return 1;
  if( pRes->op!=TK_COLUMN ) return 0;    /* Result is a column */
  assert( pRes->iTable==pSrc->a[0].iCursor );  /* Not a correlated subquery */
  return p;
}
#endif /* SQLITE_OMIT_SUBQUERY */

/*
** Code an OP_Once instruction and allocate space for its flag. Return the 
** address of the new instruction.
*/

  assert( pX->op==TK_IN );
  mustBeUnique = (inFlags & IN_INDEX_LOOP)!=0;

  /* Check to see if an existing table or index can be used to
  ** satisfy the query.  This is preferable to generating a new 
  ** ephemeral table.
  */
  p = (ExprHasProperty(pX, EP_xIsSelect) ? pX->x.pSelect : 0);
  if( pParse->nErr==0 && isCandidateForInOpt(p) ){
  if( pParse->nErr==0 && (p = isCandidateForInOpt(pX))!=0 ){
    sqlite3 *db = pParse->db;              /* Database connection */
    Table *pTab;                           /* Table <table>. */
    Expr *pExpr;                           /* Expression <column> */
    i16 iCol;                              /* Index of column <column> */
    i16 iDb;                               /* Database idx for pTab */

    assert( p );                        /* Because of isCandidateForInOpt(p) */
    assert( p->pEList!=0 );             /* Because of isCandidateForInOpt(p) */
    assert( p->pEList->a[0].pExpr!=0 ); /* Because of isCandidateForInOpt(p) */
    assert( p->pSrc!=0 );               /* Because of isCandidateForInOpt(p) */
    pTab = p->pSrc->a[0].pTab;
    pExpr = p->pEList->a[0].pExpr;
    iCol = (i16)pExpr->iColumn;
   

/*
** Return true if the underlying VFS for the given pager supports the
** primitives necessary for write-ahead logging.
*/
int sqlite3PagerWalSupported(Pager *pPager){
  const sqlite3_io_methods *pMethods = pPager->fd->pMethods;
  if( pPager->noLock ) return 0;
  return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap);
}

/*
** Attempt to take an exclusive lock on the database file. If a PENDING lock
** is obtained instead, immediately release it.
*/

#if defined(SQLITE_HAVE_OS_TRACE) || defined(SQLITE_TEST) || \
    (defined(SQLITE_DEBUG) && SQLITE_OS_WIN)
# define SQLITE_NEED_ERR_NAME
#else
# undef  SQLITE_NEED_ERR_NAME
#endif

/*
** SQLITE_ENABLE_EXPLAIN_COMMENTS is incompatible with SQLITE_OMIT_EXPLAIN
*/
#ifdef SQLITE_OMIT_EXPLAIN
# undef SQLITE_ENABLE_EXPLAIN_COMMENTS
#endif

/*
** Return true (non-zero) if the input is an integer that is too large
** to fit in 32-bits.  This macro is used inside of various testcase()
** macros to verify that we have tested SQLite for large-file support.
*/
#define IS_BIG_INT(X)  (((X)&~(i64)0xffffffff)!=0)

u64 sqlite3LogEstToInt(LogEst x){
  u64 n;
  if( x<10 ) return 1;
  n = x%10;
  x /= 10;
  if( n>=5 ) n -= 2;
  else if( n>=1 ) n -= 1;
  if( x>=3 ){
    return x>60 ? (u64)LARGEST_INT64 : (n+8)<<(x-3);
#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \
    defined(SQLITE_EXPLAIN_ESTIMATED_ROWS)
  if( x>60 ) return (u64)LARGEST_INT64;
  }
  return (n+8)>>(3-x);
#else
  /* If only SQLITE_ENABLE_STAT3_OR_STAT4 is on, then the largest input
  ** possible to this routine is 310, resulting in a maximum x of 31 */
  assert( x<=60 );
#endif
  return x>=3 ? (n+8)<<(x-3) : (n+8)>>(3-x);
}
#endif /* defined SCANSTAT or STAT4 or ESTIMATED_ROWS */

**    (3)  The template has same or fewer dependencies than the current loop
**    (4)  The template has the same or lower cost than the current loop
*/
static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
  WhereLoop **ppPrev, *p;
  WhereInfo *pWInfo = pBuilder->pWInfo;
  sqlite3 *db = pWInfo->pParse->db;
  int rc;

  /* If pBuilder->pOrSet is defined, then only keep track of the costs
  ** and prereqs.
  */
  if( pBuilder->pOrSet!=0 ){
    if( pTemplate->nLTerm ){
#if WHERETRACE_ENABLED

        sqlite3DebugPrintf(" delete: ");
        whereLoopPrint(pToDel, pBuilder->pWC);
      }
#endif
      whereLoopDelete(db, pToDel);
    }
  }
  whereLoopXfer(db, p, pTemplate);
  rc = whereLoopXfer(db, p, pTemplate);
  if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
    Index *pIndex = p->u.btree.pIndex;
    if( pIndex && pIndex->tnum==0 ){
      p->u.btree.pIndex = 0;
    }
  }
  return SQLITE_OK;
  return rc;
}

/*
** Adjust the WhereLoop.nOut value downward to account for terms of the
** WHERE clause that reference the loop but which are not used by an
** index.
*

  /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated
  ** that the scan will visit at most one row. Clear it otherwise. */
  if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){
    pNew->wsFlags |= WHERE_ONEROW;
  }else{
    pNew->wsFlags &= ~WHERE_ONEROW;
  }
  whereLoopInsert(pBuilder, pNew);
  rc = whereLoopInsert(pBuilder, pNew);
  if( pNew->u.vtab.needFree ){
    sqlite3_free(pNew->u.vtab.idxStr);
    pNew->u.vtab.needFree = 0;
  }
  WHERETRACE(0xffff, ("  bIn=%d prereqIn=%04llx prereqOut=%04llx\n",
                      *pbIn, (sqlite3_uint64)mPrereq,
                      (sqlite3_uint64)(pNew->prereq & ~mPrereq)));

  return SQLITE_OK;
  return rc;
}


/*
** Add all WhereLoop objects for a table of the join identified by
** pBuilder->pNew->iTab.  That table is guaranteed to be a virtual table.
**

if {$tcl_platform(platform)=="windows"} {
  set PROG "sqlite3_analyzer.exe"
} else {
  set PROG "./sqlite3_analyzer"
}
if {![file exe $PROG]} {
  set PROG [file normalize [file join $::cmdlinearg(TESTFIXTURE_HOME) $PROG]]
  if {![file exe $PROG]} {
  puts "analyzer1 cannot run because $PROG is not available"
  finish_test
  return
    puts "analyzer1 cannot run because $PROG is not available"
    finish_test
    return
  }
}
db close
forcedelete test.db test.db-journal test.db-wal
sqlite3 db test.db

do_test analyzer1-1.0 {
  db eval {

do_test autovacuum-2.4.3 {
  execsql {
    SELECT rootpage FROM sqlite_master ORDER by rootpage
  }
} {3 4 5 6 7 8 9 10}

# Right now there are 5 free pages in the database. Consume and then free
# a 520 pages. Then create 520 tables. This ensures that at least some of the
# all 520 pages. Then create 520 tables. This ensures that at least some of the
# desired root-pages reside on the second free-list trunk page, and that the
# trunk itself is required at some point.
do_test autovacuum-2.4.4 {
  execsql "
    INSERT INTO av3 VALUES ('[make_str abcde [expr 1020*520 + 500]]');
    DELETE FROM av3;
  "
} {}
set root_page_list [list]
set pending_byte_page [expr ($::sqlite_pending_byte / 1024) + 1]

# unusable_pages
# These are either the pending_byte page or the pointer map pages
#
unset -nocomplain unusable_page
if {[sqlite3 -has-codec]} {
  array set unusable_page {205 1 408 1}
} else {
  array set unusable_page {207 1 412 1}
}
set unusable_page($pending_byte_page) 1

for {set i 3} {$i<=532} {incr i} {
  # 207 and 412 are pointer-map pages.
  if {![info exists unusable_page($i)]} {
  if { $i!=207 && $i!=412 && $i != $pending_byte_page} {
    lappend root_page_list $i
  }
}
if {$i >= $pending_byte_page} {
  lappend root_page_list $i
}
do_test autovacuum-2.4.5 {

array set ::incompatible [list]
proc do_allbackcompat_test {script} {

  foreach bin $::BC(binaries) {
    set nErr [set_test_counter errors]
    foreach dir {0 1} {

      set bintag [string map {testfixture {}} $bin]
      set bintag $bin
      regsub {.*testfixture\.} $bintag {} bintag
      set bintag [string map {\.exe {}} $bintag]
      if {$bintag == ""} {set bintag self}
      set ::bcname ".$bintag.$dir."

      rename do_test _do_test
      proc do_test {nm sql res} {
        set nm [regsub {\.} $nm $::bcname]

# schema cookie and change counter. Doing that could cause other clients
# to become confused and continue using out-of-date cache data.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix backup4

# The codec logic does not work for zero-length database files.  A database
# file must contain at least one page in order to be recognized as an
# encrypted database.
do_not_use_codec

#-------------------------------------------------------------------------
# At one point this test was failing because [db] was using an out of
# date schema in test case 1.2.
#
do_execsql_test 1.0 {
  CREATE TABLE t1(x, y, UNIQUE(x, y));

proc bc_find_binaries {zCaption} {
  # Search for binaries to test against. Any executable files that match
  # our naming convention are assumed to be testfixture binaries to test
  # against.
  #
  set binaries [list]
  set self [file tail [info nameofexec]]
  set self [info nameofexec]
  set pattern "$self?*"
  if {$::tcl_platform(platform)=="windows"} {
    set pattern [string map {\.exe {}} $pattern]
  }
  foreach file [glob -nocomplain $pattern] {
    if {$file==$self} continue
    if {[file executable $file] && [file isfile $file]} {lappend binaries $file}

  proc code2 {tcl} { testfixture $::bc_chan $tcl }
  proc sql1 sql { code1 [list db eval $sql] }
  proc sql2 sql { code2 [list db eval $sql] }

  code1 { sqlite3 db test.db }
  code2 { sqlite3 db test.db }

  set bintag [string map {testfixture {}} $bin]
  set bintag $bin
  regsub {.*testfixture\.} $bintag {} bintag
  set bintag [string map {\.exe {}} $bintag]
  if {$bintag == ""} {set bintag self}
  set saved_prefix $::testprefix
  append ::testprefix ".$bintag"

  uplevel $script

#***********************************************************************
# 
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix bestindex1

ifcapable !vtab {
  finish_test
  return
}

register_tcl_module db

proc vtab_command {method args} {
  switch -- $method {
    xConnect {
      return "CREATE TABLE t1(a, b, c)"

  do_eqp_test 2.2.$mode.6 { 
    SELECT rowid FROM t1 WHERE a IN ('one', 'four') ORDER BY +rowid
  } $plan($mode)
}

finish_test

#
#***********************************************************************

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix bestindex2

ifcapable !vtab {
  finish_test
  return
}

#-------------------------------------------------------------------------
# Virtual table callback for table named $tbl, with the columns specified
# by list argument $cols. e.g. if the function is invoked as:
#
#   vtab_cmd t1 {a b c} ...
#

  0 0 0 {SCAN TABLE x1} 
  0 1 1 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:}
  0 2 2 {SCAN TABLE t2 VIRTUAL TABLE INDEX 0:indexed(c=?)} 
  0 3 3 {SCAN TABLE t3 VIRTUAL TABLE INDEX 0:indexed(e=?)}
}

finish_test

# Test some specific circumstances to do with shared cache mode.
#


set testdir [file dirname $argv0]
source $testdir/tester.tcl
set ::testprefix close

# This module bypasses the "-key" logic in tester.tcl, so it cannot run
# with the codec enabled.
do_not_use_codec

do_execsql_test 1.0 {
  CREATE TABLE t1(x);
  INSERT INTO t1 VALUES('one');
  INSERT INTO t1 VALUES('two');
  INSERT INTO t1 VALUES('three');
}

# segfault if it sees a corrupt database file.
#
# $Id: corrupt3.test,v 1.2 2007/04/06 21:42:22 drh Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Do not use a codec for tests in this file, as the database file is
# This module uses hard-coded offsets which do not work if the reserved_bytes
# manipulated directly using tcl scripts (using the [hexio_write] command).
#
do_not_use_codec
# value is nonzero.
if {[nonzero_reserved_bytes]} {finish_test; return;}

# These tests deal with corrupt database files
#
database_may_be_corrupt

# We must have the page_size pragma for these tests to work.
#

# segfault if it sees a corrupt database file.
#
# $Id: corrupt4.test,v 1.1 2007/09/07 14:32:07 drh Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Do not use a codec for tests in this file, as the database file is
# This module uses hard-coded offsets which do not work if the reserved_bytes
# manipulated directly using tcl scripts (using the [hexio_write] command).
#
do_not_use_codec
# value is nonzero.
if {[nonzero_reserved_bytes]} {finish_test; return;}

# These tests deal with corrupt database files
#
database_may_be_corrupt

# We must have the page_size pragma for these tests to work.
#

# on corrupt SerialTypeLen values.
#
# $Id: corrupt6.test,v 1.2 2008/05/19 15:37:10 shane Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Do not use a codec for tests in this file, as the database file is
# This module uses hard-coded offsets which do not work if the reserved_bytes
# manipulated directly using tcl scripts (using the [hexio_write] command).
#
do_not_use_codec
# value is nonzero.
if {[nonzero_reserved_bytes]} {finish_test; return;}

# These tests deal with corrupt database files
#
database_may_be_corrupt

# We must have the page_size pragma for these tests to work.
#

# on corrupt cell offsets in a btree page.
#
# $Id: corrupt7.test,v 1.8 2009/08/10 10:18:08 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Do not use a codec for tests in this file, as the database file is
# This module uses hard-coded offsets which do not work if the reserved_bytes
# manipulated directly using tcl scripts (using the [hexio_write] command).
#
do_not_use_codec
# value is nonzero.
if {[nonzero_reserved_bytes]} {finish_test; return;}

# These tests deal with corrupt database files
#
database_may_be_corrupt

# We must have the page_size pragma for these tests to work.
#

# segfault if it sees a corrupt database file.  It specifcally
# focuses on rowid order corruption.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Do not use a codec for tests in this file, as the database file is
# This module uses hard-coded offsets which do not work if the reserved_bytes
# manipulated directly using tcl scripts (using the [hexio_write] command).
#
do_not_use_codec
# value is nonzero.
if {[nonzero_reserved_bytes]} {finish_test; return;}

# These tests deal with corrupt database files
#
database_may_be_corrupt

# Do not run the tests in this file if ENABLE_OVERSIZE_CELL_CHECK is on.
#

#***********************************************************************
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix corruptG

# Do not use a codec for tests in this file, as the database file is
# This module uses hard-coded offsets which do not work if the reserved_bytes
# manipulated directly using tcl scripts (using the [hexio_write] command).
#
do_not_use_codec
# value is nonzero.
if {[nonzero_reserved_bytes]} {finish_test; return;}

# These tests deal with corrupt database files
#
database_may_be_corrupt

# Create a simple database with a single entry.  Then corrupt the
# header-size varint on the index payload so that it maps into a

#***********************************************************************
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix corruptH

# Do not use a codec for tests in this file, as the database file is
# manipulated directly using tcl scripts (using the [hexio_write] command).
#
# This module uses hard-coded offsets which do not work if the reserved_bytes
# value is nonzero.
if {[nonzero_reserved_bytes]} {finish_test; return;}

do_not_use_codec
database_may_be_corrupt

# The corruption migrations tested by the code in this file are not detected
# mmap mode.
#
# The reason is that in mmap mode, the different queries may use different
# PgHdr objects for the same page (same data, but different PgHdr container 

set testprefix corruptI

if {[permutation]=="mmap"} {
  finish_test
  return
}

# Do not use a codec for tests in this file, as the database file is
# manipulated directly using tcl scripts (using the [hexio_write] command).
#
# This module uses hard-coded offsets which do not work if the reserved_bytes
# value is nonzero.
if {[nonzero_reserved_bytes]} {finish_test; return;}

do_not_use_codec
database_may_be_corrupt

# Initialize the database.
#
do_execsql_test 1.1 {
  PRAGMA page_size=1024;
  PRAGMA auto_vacuum=0;

set testprefix corruptJ

if {[permutation]=="mmap"} {
  finish_test
  return
}

# Do not use a codec for tests in this file, as the database file is
# manipulated directly using tcl scripts (using the [hexio_write] command).
#
# This module uses hard-coded offsets which do not work if the reserved_bytes
# value is nonzero.
if {[nonzero_reserved_bytes]} {finish_test; return;}

do_not_use_codec
database_may_be_corrupt

# Initialize the database.
#
do_execsql_test 1.1 {
  PRAGMA page_size=1024;
  PRAGMA auto_vacuum=0;

set testdir [file dirname $argv0]
source $testdir/tester.tcl

ifcapable !crashtest {
  finish_test
  return
}
do_not_use_codec

do_test crash8-1.1 {
  execsql {
    PRAGMA auto_vacuum=OFF;
    CREATE TABLE t1(a, b);
    CREATE INDEX i1 ON t1(a, b);
    INSERT INTO t1 VALUES(1, randstr(1000,1000));

#
#***********************************************************************
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix e_uri

do_not_use_codec
db close

proc parse_uri {uri} {
  testvfs tvfs2
  testvfs tvfs 
  tvfs filter xOpen
  tvfs script parse_uri_open_cb

} {1024 1}
do_test e_vacuum-1.3.1.2 {
  execsql { PRAGMA page_size = 2048 }
  execsql { PRAGMA auto_vacuum = NONE }
  execsql { PRAGMA page_size ; PRAGMA auto_vacuum }
} {1024 1}

if {![nonzero_reserved_bytes]} {
# EVIDENCE-OF: R-08570-19916 However, when not in write-ahead log mode,
# the page_size and/or auto_vacuum properties of an existing database
# may be changed by using the page_size and/or pragma auto_vacuum
# pragmas and then immediately VACUUMing the database.
#
do_test e_vacuum-1.3.2.1 {
  execsql { PRAGMA journal_mode = delete }
  execsql { PRAGMA page_size = 2048 }
  execsql { PRAGMA auto_vacuum = NONE }
  execsql VACUUM
  execsql { PRAGMA page_size ; PRAGMA auto_vacuum }
} {2048 0}

# EVIDENCE-OF: R-48521-51450 When in write-ahead log mode, only the
# auto_vacuum support property can be changed using VACUUM.
#
ifcapable wal {
do_test e_vacuum-1.3.3.1 {
  execsql { PRAGMA journal_mode = wal }
  execsql { PRAGMA page_size ; PRAGMA auto_vacuum }
} {2048 0}
do_test e_vacuum-1.3.3.2 {
  execsql { PRAGMA page_size = 1024 }
  execsql { PRAGMA auto_vacuum = FULL }
  execsql VACUUM
  execsql { PRAGMA page_size ; PRAGMA auto_vacuum }
} {2048 1}
}

  # EVIDENCE-OF: R-08570-19916 However, when not in write-ahead log mode,
  # the page_size and/or auto_vacuum properties of an existing database
  # may be changed by using the page_size and/or pragma auto_vacuum
  # pragmas and then immediately VACUUMing the database.
  #
  do_test e_vacuum-1.3.2.1 {
    execsql { PRAGMA journal_mode = delete }
    execsql { PRAGMA page_size = 2048 }
    execsql { PRAGMA auto_vacuum = NONE }
    execsql VACUUM
    execsql { PRAGMA page_size ; PRAGMA auto_vacuum }
  } {2048 0}
  
  # EVIDENCE-OF: R-48521-51450 When in write-ahead log mode, only the
  # auto_vacuum support property can be changed using VACUUM.
  #
  ifcapable wal {
    do_test e_vacuum-1.3.3.1 {
      execsql { PRAGMA journal_mode = wal }
      execsql { PRAGMA page_size ; PRAGMA auto_vacuum }
    } {2048 0}
    do_test e_vacuum-1.3.3.2 {
      execsql { PRAGMA page_size = 1024 }
      execsql { PRAGMA auto_vacuum = FULL }
      execsql VACUUM
      execsql { PRAGMA page_size ; PRAGMA auto_vacuum }
    } {2048 1}
  }
}
  
# EVIDENCE-OF: R-38001-03952 VACUUM only works on the main database. It
# is not possible to VACUUM an attached database file.
forcedelete test.db2
create_db { PRAGMA auto_vacuum = NONE }
do_execsql_test e_vacuum-2.1.1 {
  ATTACH 'test.db2' AS aux;
  PRAGMA aux.page_size = 1024;

# accessing the same coherent view of the "test.db-shm" file. This doesn't
# work on OpenBSD.
#
if {$tcl_platform(os) == "OpenBSD"} {
  finish_test
  return
}

# This module uses hard-coded offsets which do not work if the reserved_bytes
# value is nonzero.
if {[nonzero_reserved_bytes]} {finish_test; return;}


proc read_nbackfill {} {
  seek $::shmfd 96
  binary scan [read $::shmfd 4] n nBackfill
  set nBackfill
}
proc read_mxframe {} {

  1 0 0 {SCAN TABLE t1 USING COVERING INDEX i1}
  2 0 0 {SCAN TABLE t2}
  2 0 0 {USE TEMP B-TREE FOR ORDER BY}
  0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (EXCEPT)}
}


if {![nonzero_reserved_bytes]} {
#-------------------------------------------------------------------------
# The following tests - eqp-6.* - test that the example C code on 
# documentation page eqp.html works. The C code is duplicated in test1.c
# and wrapped in Tcl command [print_explain_query_plan] 
#
set boilerplate {
  proc explain_query_plan {db sql} {
    set stmt [sqlite3_prepare_v2 db $sql -1 DUMMY]
    print_explain_query_plan $stmt
    sqlite3_finalize $stmt
  }
  sqlite3 db test.db
  explain_query_plan db {%SQL%}
  db close
  exit
}

# Do a "Print Explain Query Plan" test.
proc do_peqp_test {tn sql res} {
  set fd [open script.tcl w]
  puts $fd [string map [list %SQL% $sql] $::boilerplate]
  close $fd

  uplevel do_test $tn [list {
    set fd [open "|[info nameofexec] script.tcl"]
    set data [read $fd]
    close $fd
    set data
  }] [list $res]
}

do_peqp_test 6.1 {
  SELECT a, b FROM t1 EXCEPT SELECT d, 99 FROM t2 ORDER BY 1
} [string trimleft {
  #-------------------------------------------------------------------------
  # The following tests - eqp-6.* - test that the example C code on 
  # documentation page eqp.html works. The C code is duplicated in test1.c
  # and wrapped in Tcl command [print_explain_query_plan] 
  #
  set boilerplate {
    proc explain_query_plan {db sql} {
      set stmt [sqlite3_prepare_v2 db $sql -1 DUMMY]
      print_explain_query_plan $stmt
      sqlite3_finalize $stmt
    }
    sqlite3 db test.db
    explain_query_plan db {%SQL%}
    db close
    exit
  }
  
  # Do a "Print Explain Query Plan" test.
  proc do_peqp_test {tn sql res} {
    set fd [open script.tcl w]
    puts $fd [string map [list %SQL% $sql] $::boilerplate]
    close $fd
  
    uplevel do_test $tn [list {
      set fd [open "|[info nameofexec] script.tcl"]
      set data [read $fd]
      close $fd
      set data
    }] [list $res]
  }
  
  do_peqp_test 6.1 {
    SELECT a, b FROM t1 EXCEPT SELECT d, 99 FROM t2 ORDER BY 1
  } [string trimleft {
1 0 0 SCAN TABLE t1 USING COVERING INDEX i2
2 0 0 SCAN TABLE t2
2 0 0 USE TEMP B-TREE FOR ORDER BY
0 0 0 COMPOUND SUBQUERIES 1 AND 2 (EXCEPT)
}]
}

#-------------------------------------------------------------------------
# The following tests - eqp-7.* - test that queries that use the OP_Count
# optimization return something sensible with EQP.
#
drop_all_tables

} {33 1 1057 1 2081 1 3105 1}
do_execsql_test 2.7 { INSERT INTO t2(t2) VALUES('integrity-check') }

do_execsql_test 2.8 {
  INSERT INTO t2(words) SELECT words FROM t1;
  SELECT level, count(*) FROM t2_segdir GROUP BY level;
} {0 2 1024 2 2048 2 3072 2}

#-------------------------------------------------------------------------
# Check that 'optimize' works when there is data in the in-memory hash
# table, but no segments at all on disk.
#
do_execsql_test 3.1 {
  CREATE VIRTUAL TABLE fts USING fts4 (t);
  INSERT INTO fts (fts) VALUES ('optimize');
}
do_execsql_test 3.2 {
  INSERT INTO fts(fts) VALUES('integrity-check');
  SELECT count(*) FROM fts_segdir;
} {0}
do_execsql_test 3.3 {
  BEGIN;
  INSERT INTO fts (rowid, t) VALUES (2, 'test');
  INSERT INTO fts (fts) VALUES ('optimize');
  COMMIT;
  SELECT level, idx FROM fts_segdir;
} {0 0}
do_execsql_test 3.4 {
  INSERT INTO fts(fts) VALUES('integrity-check');
  SELECT rowid FROM fts WHERE fts MATCH 'test';
} {2}
do_execsql_test 3.5 {
  INSERT INTO fts (fts) VALUES ('optimize');
  INSERT INTO fts(fts) VALUES('integrity-check');
}
do_test 3.6 {
  set c1 [db total_changes]
  execsql { INSERT INTO fts (fts) VALUES ('optimize') }
  expr {[db total_changes] - $c1}
} {1}
do_test 3.7 {
  execsql { INSERT INTO fts (rowid, t) VALUES (3, 'xyz') }
  set c1 [db total_changes]
  execsql { INSERT INTO fts (fts) VALUES ('optimize') }
  expr {([db total_changes] - $c1) > 1}
} {1}
do_test 3.8 {
  set c1 [db total_changes]
  execsql { INSERT INTO fts (fts) VALUES ('optimize') }
  expr {[db total_changes] - $c1}
} {1}

finish_test

do_execsql_test 6.3.1 {
  CREATE TABLE x1(a);
  CREATE TABLE x2(b);
  INSERT INTO x1 VALUES(1), (1), (2);
  INSERT INTO x2 VALUES(1), (2);
  SELECT count(*) FROM x2 WHERE b IN (SELECT DISTINCT a FROM x1 LIMIT 2);
} {2}

#-------------------------------------------------------------------------
# Test to confirm that bug [5e3c886796e5] is fixed.
#
do_execsql_test 7.1 {
  CREATE TABLE y1(a, b);
  CREATE TABLE y2(c);

  INSERT INTO y1 VALUES(1,     'one');
  INSERT INTO y1 VALUES('two', 'two');
  INSERT INTO y1 VALUES(3,     'three');

  INSERT INTO y2 VALUES('one');
  INSERT INTO y2 VALUES('two');
  INSERT INTO y2 VALUES('three');
} {}

do_execsql_test 7.2.1 {
  SELECT a FROM y1 WHERE b NOT IN (SELECT a FROM y2);
} {1 3}
do_execsql_test 7.2.2 {
  SELECT a FROM y1 WHERE b IN (SELECT a FROM y2);
} {two}

do_execsql_test 7.3.1 {
  CREATE INDEX y2c ON y2(c);
  SELECT a FROM y1 WHERE b NOT IN (SELECT a FROM y2);
} {1 3}
do_execsql_test 7.3.2 {
  SELECT a FROM y1 WHERE b IN (SELECT a FROM y2);
} {two}

finish_test



finish_test

  db close
  forcedelete test.db test.db-journal

  sqlite3 db test.db
  execsql "PRAGMA mmap_size = 0"
  execsql "PRAGMA auto_vacuum = $AutoVacuumMode"

  # Extra value added to size answers
  set ib2_extra 0
  if {$AutoVacuumMode} {incr ib2_extra}
  if {[nonzero_reserved_bytes]} {incr ib2_extra}

  do_test incrblob-2.$AutoVacuumMode.1 {
    set ::str [string repeat abcdefghij 2900]
    execsql {
      BEGIN;
      CREATE TABLE blobs(k PRIMARY KEY, v BLOB, i INTEGER);
      DELETE FROM blobs;
      INSERT INTO blobs VALUES('one', $::str || randstr(500,500), 45);
      COMMIT;
    }
    expr [file size test.db]/1024
  } [expr 31 + $AutoVacuumMode]
  } [expr 31 + $ib2_extra]

  ifcapable autovacuum {
    do_test incrblob-2.$AutoVacuumMode.2 {
      execsql {
        PRAGMA auto_vacuum;
      }
    } $AutoVacuumMode

    close $::blob
  
    # If the database is not in auto-vacuum mode, the whole of
    # the overflow-chain must be scanned. In auto-vacuum mode,
    # sqlite uses the ptrmap pages to avoid reading the other pages.
    #
    nRead db
  } [expr $AutoVacuumMode ? 4 : 30]
  } [expr $AutoVacuumMode ? 4 : 30+$ib2_extra]

  do_test incrblob-2.$AutoVacuumMode.4 {
    string range [db one {SELECT v FROM blobs}] end-19 end
  } $::fragment

  do_test incrblob-2.$AutoVacuumMode.5 {
    # Open and close the db to make sure the page cache is empty.

    flush $::blob
  
    # If the database is not in auto-vacuum mode, the whole of
    # the overflow-chain must be scanned. In auto-vacuum mode,
    # sqlite uses the ptrmap pages to avoid reading the other pages.
    #
    nRead db
  } [expr $AutoVacuumMode ? 4 : 30]
  } [expr $AutoVacuumMode ? 4 : 30 + $ib2_extra]

  # Pages 1 (the write-counter) and 32 (the blob data) were written.
  do_test incrblob-2.$AutoVacuumMode.6 {
    close $::blob
    nWrite db
  } 2

    execsql { PRAGMA mmap_size = 0 }

    execsql { SELECT i FROM blobs } 
  } {45}

  do_test incrblob-2.$AutoVacuumMode.9 {
    nRead db
  } [expr $AutoVacuumMode ? 4 : 30]
  } [expr $AutoVacuumMode ? 4 : 30 + $ib2_extra]
}
sqlite3_soft_heap_limit $cmdlinearg(soft-heap-limit)

#------------------------------------------------------------------------
# incrblob-3.*: 
#
# Test the outcome of trying to write to a read-only blob handle.

# incrblob-5.*: 
#
#     Test that opening a blob in an attached database works.
#
ifcapable attach {
  do_test incrblob-5.1 {
    forcedelete test2.db test2.db-journal
    set ::size [expr [file size [info script]]]
    set ::size [expr [file size $::cmdlinearg(INFO_SCRIPT)]]
    execsql {
      ATTACH 'test2.db' AS aux;
      CREATE TABLE aux.files(name, text);
      INSERT INTO aux.files VALUES('this one', zeroblob($::size));
    }
    set fd  [db incrblob aux files text 1]
    fconfigure $fd -translation binary
    set fd2 [open [info script]]
    set fd2 [open $::cmdlinearg(INFO_SCRIPT)]
    fconfigure $fd2 -translation binary
    puts -nonewline $fd [read $fd2]
    close $fd
    close $fd2
    set ::text [db one {select text from aux.files}]
    string length $::text
  } [file size [info script]]
  } [file size $::cmdlinearg(INFO_SCRIPT)]
  do_test incrblob-5.2 {
    set fd2 [open [info script]]
    set fd2 [open $::cmdlinearg(INFO_SCRIPT)]
    fconfigure $fd2 -translation binary
    set ::data [read $fd2]
    close $fd2
    set ::data
  } $::text
}

    execsql {
      SELECT d FROM t1;
    }
  } {15}

}

set fd [open [info script]]
set fd [open $::cmdlinearg(INFO_SCRIPT)]
fconfigure $fd -translation binary
set ::data [read $fd 14000]
close $fd

db close
forcedelete test.db test.db-journal
sqlite3 db test.db

  finish_test
  return
}

source $testdir/malloc_common.tcl

unset -nocomplain ::fd ::data
set ::fd [open [info script]]
set ::fd [open $::cmdlinearg(INFO_SCRIPT)]
set ::data [read $::fd]
close $::fd

do_malloc_test 1 -tclprep {
  set bytes [file size [info script]]
  set bytes [file size $::cmdlinearg(INFO_SCRIPT)]
  execsql {
    CREATE TABLE blobs(k, v BLOB);
    INSERT INTO blobs VALUES(1, zeroblob($::bytes));
  }
} -tclbody {
  set ::blob [db incrblob blobs v 1]
  fconfigure $::blob -translation binary

    # that the file is now greater than 20000 bytes in size.
    list [expr [file size test.db]>20000] [nSync]
  } {1 0}
  do_test io-3.3 {
    # The COMMIT requires a single fsync() - to the database file.
    execsql { COMMIT }
    list [file size test.db] [nSync]
  } {39936 1}
  } "[expr {[nonzero_reserved_bytes]?40960:39936}] 1"
}

#----------------------------------------------------------------------
# Test cases io-4.* test the IOCAP_SAFE_APPEND optimization.
#
sqlite3_simulate_device -char safe_append

  expr {$pg_used<24}
} 1
do_test memsubsys1-7.4 {
  set pg_ovfl [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2]
} 0
do_test memsubsys1-7.5 {
  set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2]
  expr {$maxreq<4100}
  expr {$maxreq<4100 + 4200*[nonzero_reserved_bytes]}
} 1
do_test memsubsys1-7.6 {
  set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2]
} 1
do_test memsubsys1-7.7 {
  set s_ovfl [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0] 2]
} 0

      sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count"
    } {32 ok 77}

    # Have connection 2 shrink the file. Check connection 1 can still read it.
    sql2 { DELETE FROM t1 WHERE rowid%2; }
    do_test $t.$tn.2 {
      sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count"
    } {16 ok 42}
    } "16 ok [expr {42+[nonzero_reserved_bytes]}]"

    # Have connection 2 grow the file. Check connection 1 can still read it.
    sql2 { INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1 }
    do_test $t.$tn.3 {
      sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count"
    } {32 ok 79}

    # Have connection 2 grow the file again. Check connection 1 is still ok.
    sql2 { INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1 }
    do_test $t.$tn.4 {
      sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count"
    } {64 ok 149}

    # Check that the number of pages read by connection 1 indicates that the
    # "PRAGMA mmap_size" command worked.
    if {[nonzero_reserved_bytes]==0} {
    do_test $t.$tn.5 { nRead db } $nRead
      do_test $t.$tn.5 { nRead db } $nRead
    }
  }
}

set ::rcnt 0
proc rblob {n} {
  set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF]
  set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]]

ifcapable !mmap||!vtab {
  finish_test
  return
}
source $testdir/lock_common.tcl
set testprefix mmap3

# A codec shuts down memory-mapped I/O
if {[nonzero_reserved_bytes]} {finish_test; return;}

do_test mmap3-1.0 {
  load_static_extension db wholenumber
  db eval {
    PRAGMA mmap_size=100000;
    CREATE TABLE t1(x, y);
    CREATE VIRTUAL TABLE nums USING wholenumber;
    INSERT INTO t1 SELECT value, randomblob(value) FROM nums

# SQLite always converts NaN into NULL so it is not possible to write
# a NaN value into the database file using SQLite.  The following series
# of tests writes a normal floating point value (0.5) into the database,
# then writes directly into the database file to change the 0.5 into NaN.
# Then it reads the value of the database to verify it is converted into
# NULL.
#
if {![nonzero_reserved_bytes]} {
do_test nan-3.1 {
  db eval {
    DELETE FROM t1;
    INSERT INTO t1 VALUES(0.5);
    PRAGMA auto_vacuum=OFF;
    PRAGMA page_size=1024;
    VACUUM;
  }
  hexio_read test.db 2040 8
} {3FE0000000000000}
do_test nan-3.2 {
  db eval {
    SELECT x, typeof(x) FROM t1
  }
} {0.5 real}
do_test nan-3.3 {
  db close
  hexio_write test.db 2040 FFF8000000000000
  sqlite3 db test.db
  db eval {SELECT x, typeof(x) FROM t1}
} {{} null}
do_test nan-3.4 {
  db close
  hexio_write test.db 2040 7FF8000000000000
  sqlite3 db test.db
  db eval {SELECT x, typeof(x) FROM t1}
} {{} null}
do_test nan-3.5 {
  db close
  hexio_write test.db 2040 FFFFFFFFFFFFFFFF
  sqlite3 db test.db
  db eval {SELECT x, typeof(x) FROM t1}
} {{} null}
do_test nan-3.6 {
  db close
  hexio_write test.db 2040 7FFFFFFFFFFFFFFF
  sqlite3 db test.db
  db eval {SELECT x, typeof(x) FROM t1}
} {{} null}
  do_test nan-3.1 {
    db eval {
      DELETE FROM t1;
      INSERT INTO t1 VALUES(0.5);
      PRAGMA auto_vacuum=OFF;
      PRAGMA page_size=1024;
      VACUUM;
    }
    hexio_read test.db 2040 8
  } {3FE0000000000000}
  do_test nan-3.2 {
    db eval {
      SELECT x, typeof(x) FROM t1
    }
  } {0.5 real}
  do_test nan-3.3 {
    db close
    hexio_write test.db 2040 FFF8000000000000
    sqlite3 db test.db
    db eval {SELECT x, typeof(x) FROM t1}
  } {{} null}
  do_test nan-3.4 {
    db close
    hexio_write test.db 2040 7FF8000000000000
    sqlite3 db test.db
    db eval {SELECT x, typeof(x) FROM t1}
  } {{} null}
  do_test nan-3.5 {
    db close
    hexio_write test.db 2040 FFFFFFFFFFFFFFFF
    sqlite3 db test.db
    db eval {SELECT x, typeof(x) FROM t1}
  } {{} null}
  do_test nan-3.6 {
    db close
    hexio_write test.db 2040 7FFFFFFFFFFFFFFF
    sqlite3 db test.db
    db eval {SELECT x, typeof(x) FROM t1}
  } {{} null}
}

# Verify that the sqlite3AtoF routine is able to handle extreme
# numbers.
#
do_test nan-4.1 {
  db eval {DELETE FROM t1}
  db eval "INSERT INTO t1 VALUES([string repeat 9 307].0)"

       xCheckReservedLock $::tvfs_calls(xCheckReservedLock) \
       xAccess $::tvfs_calls(xAccess)
} {xLock 0 xUnlock 0 xCheckReservedLock 0 xAccess 0}

db2 close
db close
tvfs delete

# 2016-03-11:  Make sure all works when transitioning to WAL mode under nolock.
#
do_test nolock-4.1 {
  forcedelete test.db
  sqlite3 db file:test.db?nolock=1 -uri 1
  db eval {
     PRAGMA journal_mode=WAL;
     CREATE TABLE t1(x);
     INSERT INTO t1 VALUES('youngling');
     SELECT * FROM t1;
  }
} {delete youngling}
db close

do_test nolock-4.2 {
  forcedelete test.db
  sqlite3 db test.db
  db eval {
    PRAGMA journal_mode=WAL;
    CREATE TABLE t1(x);
    INSERT INTO t1 VALUES('catbird');
    SELECT * FROM t1;
  }
} {wal catbird}
do_test nolock-4.3 {
  db close
  sqlite3 db file:test.db?nolock=1 -uri 1
  set rc [catch {db eval {SELECT * FROM t1}} msg]
  lappend rc $msg
} {1 {unable to open database file}}

finish_test

  testvfs tv -default 1
  tv sectorsize 4096
  faultsim_delete_and_reopen

  execsql { PRAGMA page_size = 1024 }
  for {set ii 0} {$ii < 4} {incr ii} { execsql "CREATE TABLE t${ii}(a, b)" }
} {}
if {[nonzero_reserved_bytes]} {
  # backup with a page size changes is not possible with the codec
  #
do_test pager1-9.3.2 {
  sqlite3 db2 test.db2
  do_test pager1-9.3.2codec {
    sqlite3 db2 test.db2

  execsql {
    PRAGMA page_size = 4096;
    PRAGMA synchronous = OFF;
    CREATE TABLE t1(a, b);
    CREATE TABLE t2(a, b);
  } db2

  sqlite3_backup B db2 main db main
  B step 30
  list [B step 10000] [B finish]
} {SQLITE_DONE SQLITE_OK}
do_test pager1-9.3.3 {
  db2 close
  db close
  tv delete
  file size test.db2
} [file size test.db]
    execsql {
      PRAGMA page_size = 4096;
      PRAGMA synchronous = OFF;
      CREATE TABLE t1(a, b);
      CREATE TABLE t2(a, b);
    } db2
    sqlite3_backup B db2 main db main
    B step 30
    list [B step 10000] [B finish]
  } {SQLITE_READONLY SQLITE_READONLY}
  do_test pager1-9.3.3codec {
    db2 close
    db close
    tv delete
    file size test.db2
  } [file size test.db2]
} else {
  do_test pager1-9.3.2 {
    sqlite3 db2 test.db2
    execsql {
      PRAGMA page_size = 4096;
      PRAGMA synchronous = OFF;
      CREATE TABLE t1(a, b);
      CREATE TABLE t2(a, b);
    } db2
    sqlite3_backup B db2 main db main
    B step 30
    list [B step 10000] [B finish]
  } {SQLITE_DONE SQLITE_OK}
  do_test pager1-9.3.3 {
    db2 close
    db close
    tv delete
    file size test.db2
  } [file size test.db]
}

do_test pager1-9.4.1 {
  faultsim_delete_and_reopen
  sqlite3 db2 test.db2
  execsql {
    PRAGMA page_size = 4096;
    CREATE TABLE t1(a, b);

    PRAGMA auto_vacuum = full;
    PRAGMA locking_mode=exclusive;
    CREATE TABLE t1(a, b);
    INSERT INTO t1 VALUES(1, 2);
  }
  file size test.db
} [expr 1024*3]
if {[nonzero_reserved_bytes]} {
  # VACUUM with size changes is not possible with the codec.
do_test pager1-29.2 {
  execsql {
    PRAGMA page_size = 4096;
    VACUUM;
  }
  file size test.db
} [expr 4096*3]
  do_test pager1-29.2 {
    catchsql {
      PRAGMA page_size = 4096;
      VACUUM;
    }
  } {1 {attempt to write a readonly database}}
} else {
  do_test pager1-29.2 {
    execsql {
      PRAGMA page_size = 4096;
      VACUUM;
    }
    file size test.db
  } [expr 4096*3]
}

#-------------------------------------------------------------------------
# Test that if an empty database file (size 0 bytes) is opened in 
# exclusive-locking mode, any journal file is deleted from the file-system
# without being rolled back. And that the RESERVED lock obtained while
# doing this is not released.
#

# The focus of the tests in this file are to verify that the
# pager optimizations implemented in version 3.3.14 work.
#
# $Id: pageropt.test,v 1.5 2008/08/20 14:49:25 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl
do_not_use_codec

ifcapable {!pager_pragmas||secure_delete||direct_read} {
  finish_test
  return
}

# Run the SQL statement supplied by the argument and return

      puts "  $d"
      puts ""
    }
  }
  exit -1
}

if {[info script] == $argv0} {
if {[file tail $argv0] == "permutations.test"} {
  proc main {argv} {
    if {[llength $argv]==0} {
      help
    } else {
      set suite [lindex $argv 0]
      set suite [file tail [lindex $argv 0]]
      if {[info exists ::testspec($suite)]==0} help
      set extra ""
      if {[llength $argv]>1} { set extra [list -files [lrange $argv 1 end]] }
      eval run_tests $suite $::testspec($suite) $extra
    }
  }
  main $argv

  catchsql {PRAGMA data_store_directory}
} {0 {}}

forcedelete data_dir
} ;# endif windows

database_may_be_corrupt
if {![nonzero_reserved_bytes]} {

do_test 21.1 {
  # Create a corrupt database in testerr.db. And a non-corrupt at test.db.
  #
  db close
  forcedelete test.db
  sqlite3 db test.db
  execsql { 
    PRAGMA page_size = 1024;
    PRAGMA auto_vacuum = 0;
    CREATE TABLE t1(a PRIMARY KEY, b);
    INSERT INTO t1 VALUES(1, 1);
  }
  for {set i 0} {$i < 10} {incr i} {
    execsql { INSERT INTO t1 SELECT a + (1 << $i), b + (1 << $i) FROM t1 }
  }
  db close
  forcecopy test.db testerr.db
  hexio_write testerr.db 15000 [string repeat 55 100]
} {100}

set mainerr {*** in database main ***
  do_test 21.1 {
    # Create a corrupt database in testerr.db. And a non-corrupt at test.db.
    #
    db close
    forcedelete test.db
    sqlite3 db test.db
    execsql { 
      PRAGMA page_size = 1024;
      PRAGMA auto_vacuum = 0;
      CREATE TABLE t1(a PRIMARY KEY, b);
      INSERT INTO t1 VALUES(1, 1);
    }
    for {set i 0} {$i < 10} {incr i} {
      execsql { INSERT INTO t1 SELECT a + (1 << $i), b + (1 << $i) FROM t1 }
    }
    db close
    forcecopy test.db testerr.db
    hexio_write testerr.db 15000 [string repeat 55 100]
  } {100}
  
  set mainerr {*** in database main ***
Multiple uses for byte 672 of page 15}
set auxerr {*** in database aux ***
  set auxerr {*** in database aux ***
Multiple uses for byte 672 of page 15}

set mainerr {/{\*\*\* in database main \*\*\*
  
  set mainerr {/{\*\*\* in database main \*\*\*
Multiple uses for byte 672 of page 15}.*/}
set auxerr {/{\*\*\* in database aux \*\*\*
  set auxerr {/{\*\*\* in database aux \*\*\*
Multiple uses for byte 672 of page 15}.*/}

do_test 22.2 {
  catch { db close }
  sqlite3 db testerr.db
  execsql { PRAGMA integrity_check }
} $mainerr

do_test 22.3.1 {
  catch { db close }
  sqlite3 db test.db
  execsql { 
    ATTACH 'testerr.db' AS 'aux';
    PRAGMA integrity_check;
  }
} $auxerr
do_test 22.3.2 {
  execsql { PRAGMA main.integrity_check; }
} {ok}
do_test 22.3.3 {
  execsql { PRAGMA aux.integrity_check; }
} $auxerr

do_test 22.4.1 {
  catch { db close }
  sqlite3 db testerr.db
  execsql { 
    ATTACH 'test.db' AS 'aux';
    PRAGMA integrity_check;
  }
} $mainerr
do_test 22.4.2 {
  execsql { PRAGMA main.integrity_check; }
} $mainerr
do_test 22.4.3 {
  execsql { PRAGMA aux.integrity_check; }
} {ok}

  
  do_test 22.2 {
    catch { db close }
    sqlite3 db testerr.db
    execsql { PRAGMA integrity_check }
  } $mainerr
  
  do_test 22.3.1 {
    catch { db close }
    sqlite3 db test.db
    execsql { 
      ATTACH 'testerr.db' AS 'aux';
      PRAGMA integrity_check;
    }
  } $auxerr
  do_test 22.3.2 {
    execsql { PRAGMA main.integrity_check; }
  } {ok}
  do_test 22.3.3 {
    execsql { PRAGMA aux.integrity_check; }
  } $auxerr
  
  do_test 22.4.1 {
    catch { db close }
    sqlite3 db testerr.db
    execsql { 
      ATTACH 'test.db' AS 'aux';
      PRAGMA integrity_check;
    }
  } $mainerr
  do_test 22.4.2 {
    execsql { PRAGMA main.integrity_check; }
  } $mainerr
  do_test 22.4.3 {
    execsql { PRAGMA aux.integrity_check; }
  } {ok}
}
  
db close
forcedelete test.db test.db-wal test.db-journal
sqlite3 db test.db
sqlite3 db2 test.db
do_test 23.1 {
  db eval {
    CREATE TABLE t1(a INTEGER PRIMARY KEY,b,c,d);

# This file implements regression tests for SQLite library.
#
# This file implements tests for PRAGMA data_version command.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
do_not_use_codec

do_execsql_test pragma3-100 {
  PRAGMA data_version;
} {1}
do_execsql_test pragma3-101 {
  PRAGMA temp.data_version;
} {1}

  SELECT spellfix1_scriptcode('וַיֹּ֥אמֶר אֱלֹהִ֖ים יְהִ֣י א֑וֹר וַֽיְהִי־אֽוֹר׃');
} {125}
do_execsql_test 140 {
  SELECT spellfix1_scriptcode('فِي ذَلِكَ الوَقتِ، قالَ اللهُ: لِيَكُنْ نُورٌ. فَصَارَ نُورٌ.');
} {160}
do_execsql_test 200 {
  SELECT spellfix1_scriptcode('+3.14159');
} {999}
} {215}
do_execsql_test 210 {
  SELECT spellfix1_scriptcode('And God said: "Да будет свет"');
} {998}
do_execsql_test 220 {
  SELECT spellfix1_scriptcode('+3.14159 light');
} {215}
do_execsql_test 230 {
  SELECT spellfix1_scriptcode('+3.14159 свет');
} {220}
do_execsql_test 240 {
  SELECT spellfix1_scriptcode('וַיֹּ֥אמֶר +3.14159');
} {125}

finish_test

set testprefix stat

ifcapable !vtab||!compound {
  finish_test
  return
}

# This module uses hard-coded results that depend on exact measurements of
# pages sizes at the byte level, and hence will not work if the reserved_bytes
# value is nonzero.
if {[nonzero_reserved_bytes]} {finish_test; return;}

set ::asc 1
proc a_string {n} { string range [string repeat [incr ::asc]. $n] 1 $n }
db func a_string a_string

register_dbstat_vtab db
do_execsql_test stat-0.0 {

#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl

set testprefix superlock
do_not_use_codec

# Test organization:
#
#   1.*: Test superlock on a rollback database. Test that once the db is
#        superlocked, it is not possible for a second client to read from
#        it.
#

do_catchsql_test 6.7 { SELECT * FROM t1 } {1 {no such table: t1}}
do_catchsql_test 6.8 { SELECT * FROM t2 } {0 {a b}}

db_swap test.db2 test.db
do_catchsql_test 6.9 { SELECT * FROM t1 } {0 {1 2 3 4}}
do_catchsql_test 6.10 { SELECT * FROM t2 } {1 {no such table: t2}}

if {[nonzero_reserved_bytes]} {
  # Vacuum with a size change is not allowed with the codec
  do_execsql_test  6.11codec { 
    PRAGMA journal_mode = delete;
    VACUUM;
    PRAGMA journal_mode = wal;
    INSERT INTO t1 VALUES(5, 6);
  } {delete wal}
} else {
do_execsql_test  6.11 { 
  PRAGMA journal_mode = delete;
  PRAGMA page_size = 512;
  VACUUM;
  PRAGMA journal_mode = wal;
  INSERT INTO t1 VALUES(5, 6);
} {delete wal}
  do_execsql_test  6.11 { 
    PRAGMA journal_mode = delete;
    PRAGMA page_size = 512;
    VACUUM;
    PRAGMA journal_mode = wal;
    INSERT INTO t1 VALUES(5, 6);
  } {delete wal}
}

db_swap test.db2 test.db
do_catchsql_test 6.12 { SELECT * FROM t1 } {1 {no such table: t1}}
do_catchsql_test 6.13 { SELECT * FROM t2 } {0 {a b}}

db_swap test.db2 test.db
do_catchsql_test 6.14 { SELECT * FROM t1 } {0 {1 2 3 4 5 6}}
do_catchsql_test 6.15 { SELECT * FROM t2 } {1 {no such table: t2}}

finish_test

# $Id: tclsqlite.test,v 1.73 2009/03/16 13:19:36 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Check the error messages generated by tclsqlite
#
set r "sqlite_orig HANDLE FILENAME ?-vfs VFSNAME? ?-readonly BOOLEAN? ?-create BOOLEAN? ?-nomutex BOOLEAN? ?-fullmutex BOOLEAN? ?-uri BOOLEAN?"
if {[sqlite3 -has-codec]} {
  set r "sqlite_orig HANDLE FILENAME ?-key CODEC-KEY?"
  append r " ?-key CODECKEY?"
} else {
  set r "sqlite_orig HANDLE FILENAME ?-vfs VFSNAME? ?-readonly BOOLEAN? ?-create BOOLEAN? ?-nomutex BOOLEAN? ?-fullmutex BOOLEAN? ?-uri BOOLEAN?"
}
do_test tcl-1.1 {
  set v [catch {sqlite3 bogus} msg]
  regsub {really_sqlite3} $msg {sqlite3} msg
  lappend v $msg
} [list 1 "wrong # args: should be \"$r\""]
do_test tcl-1.2 {

# This command should be called after loading tester.tcl from within
# all test scripts that are incompatible with encryption codecs.
#
proc do_not_use_codec {} {
  set ::do_not_use_codec 1
  reset_db
}

# Return true if the "reserved_bytes" integer on database files is non-zero.
#
proc nonzero_reserved_bytes {} {
  return [sqlite3 -has-codec]
}

# Print a HELP message and exit
#
proc print_help_and_quit {} {
  puts {Options:
  --pause                  Wait for user input before continuing
  --soft-heap-limit=N      Set the soft-heap-limit to N

  #   --soak=N
  #   --file-retries=N
  #   --file-retry-delay=N
  #   --start=[$permutation:]$testfile
  #   --match=$pattern
  #   --verbose=$val
  #   --output=$filename
  #   -q                                      Reduce output
  #   --testdir=$dir                          Run tests in subdirectory $dir
  #   --help
  #
  set cmdlinearg(soft-heap-limit)    0
  set cmdlinearg(maxerror)        1000
  set cmdlinearg(malloctrace)        0
  set cmdlinearg(backtrace)         10
  set cmdlinearg(binarylog)          0
  set cmdlinearg(soak)               0
  set cmdlinearg(file-retries)       0
  set cmdlinearg(file-retry-delay)   0
  set cmdlinearg(start)             ""
  set cmdlinearg(match)             ""
  set cmdlinearg(verbose)           ""
  set cmdlinearg(output)            ""
  set cmdlinearg(testdir)           "testdir"

  set leftover [list]
  foreach a $argv {
    switch -regexp -- $a {
      {^-+pause$} {
        # Wait for user input before continuing. This is to give the user an
        # opportunity to connect profiling tools to the process.

      }
      {^-+backtrace=.+$} {
        foreach {dummy cmdlinearg(backtrace)} [split $a =] break
        sqlite3_memdebug_backtrace $value
      }
      {^-+binarylog=.+$} {
        foreach {dummy cmdlinearg(binarylog)} [split $a =] break
        set cmdlinearg(binarylog) [file normalize $cmdlinearg(binarylog)]
      }
      {^-+soak=.+$} {
        foreach {dummy cmdlinearg(soak)} [split $a =] break
        set ::G(issoak) $cmdlinearg(soak)
      }
      {^-+file-retries=.+$} {
        foreach {dummy cmdlinearg(file-retries)} [split $a =] break

        set ::G(match) $cmdlinearg(match)
        if {$::G(match) == ""} {unset ::G(match)}
      }

      {^-+output=.+$} {
        foreach {dummy cmdlinearg(output)} [split $a =] break
        set cmdlinearg(output) [file normalize $cmdlinearg(output)]
        if {$cmdlinearg(verbose)==""} {
          set cmdlinearg(verbose) 2
        }
      }
      {^-+verbose=.+$} {
        foreach {dummy cmdlinearg(verbose)} [split $a =] break
        if {$cmdlinearg(verbose)=="file"} {
          set cmdlinearg(verbose) 2
        } elseif {[string is boolean -strict $cmdlinearg(verbose)]==0} {
          error "option --verbose= must be set to a boolean or to \"file\""
        }
      }
      {^-+testdir=.*$} {
        foreach {dummy cmdlinearg(testdir)} [split $a =] break
      }
      {.*help.*} {
         print_help_and_quit
      }
      {^-q$} {
        set cmdlinearg(output) test-out.txt
        set cmdlinearg(verbose) 2
      }

      default {
        lappend leftover $a
        lappend leftover [file normalize $a]
      }
    }
  }
  set testdir [file normalize $testdir]
  set cmdlinearg(TESTFIXTURE_HOME) [pwd]
  set cmdlinearg(INFO_SCRIPT) [file normalize [info script]]
  set argv0 [file normalize $argv0]
  if {$cmdlinearg(testdir)!=""} {
    file mkdir $cmdlinearg(testdir)
    cd $cmdlinearg(testdir)
  }
  set argv $leftover

  # Install the malloc layer used to inject OOM errors. And the 'automatic'
  # extensions. This only needs to be done once for the process.
  #
  sqlite3_shutdown

#
# This file implements tests to verify that ticket #4018 has been
# fixed.  
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
do_not_use_codec

proc testsql {sql} {
  set fd [open tf_main.tcl w]
  puts $fd [subst -nocommands {
    sqlite3_test_control_pending_byte 0x0010000
    sqlite3 db test.db
    set rc [catch { db eval {$sql} } msg]

do_test wal-21.3 {
  execsql { PRAGMA integrity_check }
} {ok}

#-------------------------------------------------------------------------
# Test reading and writing of databases with different page-sizes.
#
incr ::do_not_use_codec
foreach pgsz {512 1024 2048 4096 8192 16384 32768 65536} {
  do_multiclient_test tn [string map [list %PGSZ% $pgsz] {
    do_test wal-22.%PGSZ%.$tn.1 {
      sql1 {
        PRAGMA main.page_size = %PGSZ%;
        PRAGMA auto_vacuum = 0;
        PRAGMA journal_mode = WAL;
        CREATE TABLE t1(x UNIQUE);
        INSERT INTO t1 SELECT randomblob(800);
        INSERT INTO t1 SELECT randomblob(800);
        INSERT INTO t1 SELECT randomblob(800);
      }
    } {wal}
    do_test wal-22.%PGSZ%.$tn.2 { sql2 { PRAGMA integrity_check } } {ok}
    do_test wal-22.%PGSZ%.$tn.3 {
      sql1 {PRAGMA wal_checkpoint}
      expr {[file size test.db] % %PGSZ%}
    } {0}
  }]
}
incr ::do_not_use_codec -1

#-------------------------------------------------------------------------
# Test that when 1 or more pages are recovered from a WAL file, 
# sqlite3_log() is invoked to report this to the user.
#
ifcapable curdir {
  set walfile [file nativename [file join [get_pwd] test.db-wal]]

#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
source $testdir/wal_common.tcl
ifcapable !wal {finish_test ; return }
do_not_use_codec

set testprefix wal5

proc db_page_count  {{file test.db}} { expr [file size $file] / 1024 }
proc wal_page_count {{file test.db}} { wal_frame_count ${file}-wal 1024 }

# first read transaction is executed), and the "PRAGMA page_size = XXX"
# is a no-op.
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set ::testprefix wal8
ifcapable !wal {finish_test ; return }
do_not_use_codec

db close
forcedelete test.db test.db-wal

sqlite3 db test.db
sqlite3 db2 test.db

      INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 16 */
      INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 32 */
      INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 64 */
    COMMIT;
  }
} {}
do_test walbak-2.2 {
  forcedelete abc.db
  db backup abc.db
  sqlite3 db2 abc.db
  string compare [sig db] [sig db2]
} {0}

do_test walbak-2.3 {
  sqlite3_backup B db2 main db main

      PRAGMA page_size = 2048;
      PRAGMA journal_mode = PERSIST;
      CREATE TABLE xx(x);
    }
  }

} {
  if {$tn==4 && [sqlite3 -has-codec]} continue
  foreach f [glob -nocomplain test.db*] { forcedelete $f }

  eval $setup

  do_test walbak-3.$tn.1 {
    execsql {
      CREATE TABLE t1(a, b);

      INSERT INTO t2 SELECT x||y, y||x FROM t2;
      INSERT INTO t2 SELECT x||y, y||x FROM t2;
      INSERT INTO t2 SELECT x||y, y||x FROM t2;
      INSERT INTO t2 SELECT x||y, y||x FROM t2;
      INSERT INTO t2 SELECT x||y, y||x FROM t2;
    }
    file size test.db-wal
  } {147800}
  } [expr {[nonzero_reserved_bytes]?148848:147800}]
  do_test 1.4.4.2 {
    csql1 { SELECT * FROM t1 }
  } {0 {a b c d e f g h i j k l 1 2 3 4 5 6}}
  do_test 1.4.4.3 {
    csql2 COMMIT
    csql1 { SELECT count(*) FROM t2 }
  } {0 512}

#
do_execsql_test where2-13.1 {
  CREATE TABLE t13(a,b);
  CREATE INDEX t13a ON t13(a);
  INSERT INTO t13 VALUES(4,5);
  SELECT * FROM t13 WHERE (1=2 AND a=3) OR a=4;
} {4 5}

# https://www.sqlite.org/src/info/5e3c886796e5512e  (2016-03-09)
# Correlated subquery on the RHS of an IN operator 
#
do_execsql_test where2-14.1 {
  CREATE TABLE t14a(x INTEGER PRIMARY KEY);
  INSERT INTO t14a(x) VALUES(1),(2),(3),(4);
  CREATE TABLE t14b(y INTEGER PRIMARY KEY);
  INSERT INTO t14b(y) VALUES(1);
  SELECT x FROM t14a WHERE x NOT IN (SELECT x FROM t14b);
} {}

finish_test

        REM
        REM NOTE: Copy the "sqlite3.pdb" file to the appropriate directory for
        REM       the build and platform beneath the binary directory unless we
        REM       are prevented from doing so.
        REM
        IF NOT DEFINED NOSYMBOLS (
          IF EXIST "%DLL_PDB_FILE_NAME%" (
          %__ECHO% XCOPY "%DLL_PDB_FILE_NAME%" "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
            %__ECHO% XCOPY "%DLL_PDB_FILE_NAME%" "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%

          IF ERRORLEVEL 1 (
            ECHO Failed to copy "%DLL_PDB_FILE_NAME%" to "%BINARYDIRECTORY%\%%B\%%D\".
            GOTO errors
            IF ERRORLEVEL 1 (
              ECHO Failed to copy "%DLL_PDB_FILE_NAME%" to "%BINARYDIRECTORY%\%%B\%%D\".
              GOTO errors
            )
          )
        )

        REM
        REM NOTE: If requested, also build the shell executable.
        REM
        IF DEFINED BUILD_ALL_SHELL (

          REM
          REM NOTE: Copy the "sqlite3sh.pdb" file to the appropriate directory
          REM       for the build and platform beneath the binary directory
          REM       unless we are prevented from doing so.
          REM
          IF NOT DEFINED NOSYMBOLS (
            IF EXIST "%EXE_PDB_FILE_NAME%" (
            %__ECHO% XCOPY "%EXE_PDB_FILE_NAME%" "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
              %__ECHO% XCOPY "%EXE_PDB_FILE_NAME%" "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%

            IF ERRORLEVEL 1 (
              ECHO Failed to copy "%EXE_PDB_FILE_NAME%" to "%BINARYDIRECTORY%\%%B\%%D\".
              GOTO errors
              IF ERRORLEVEL 1 (
                ECHO Failed to copy "%EXE_PDB_FILE_NAME%" to "%BINARYDIRECTORY%\%%B\%%D\".
                GOTO errors
              )
            )
          )
        )
      )
    )
  )