/ Check-in [8d2f52bb]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Unless the "--freelist-corrupt" option is specified, do not have the .recover command attempt to recover data from pages that are on the database free-list.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | dbdata
Files: files | file ages | folders
SHA3-256: 8d2f52bb640d6d0f84b18d746043e56f45a73ace93239be1d036701f7f4018fd
User & Date: dan 2019-04-25 19:23:15
Context
2019-04-25
20:06
Merge latest trunk changes into this branch. check-in: 1da302d8 user: dan tags: dbdata
19:23
Unless the "--freelist-corrupt" option is specified, do not have the .recover command attempt to recover data from pages that are on the database free-list. check-in: 8d2f52bb user: dan tags: dbdata
16:20
Fix a bug preventing .recover from working on databases where the final page of the db is corrupt. check-in: 959bbd11 user: dan tags: dbdata
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/shell.c.in.

  3929   3929       p->lineno = nLine;
  3930   3930     }
  3931   3931     sqlite3_free(a);
  3932   3932     utf8_printf(stderr,"Error on line %d of --hexdb input\n", nLine);
  3933   3933     return 0;
  3934   3934   }
  3935   3935   #endif /* SQLITE_ENABLE_DESERIALIZE */
         3936  +
         3937  +/*
         3938  +** Scalar function "shell_int32". The first argument to this function
         3939  +** must be a blob. The second a non-negative integer. This function
         3940  +** reads and returns a 32-bit big-endian integer from byte
         3941  +** offset (4*<arg2>) of the blob.
         3942  +*/
         3943  +static void shellInt32(
         3944  +  sqlite3_context *context, 
         3945  +  int argc, 
         3946  +  sqlite3_value **argv
         3947  +){
         3948  +  const unsigned char *pBlob;
         3949  +  int nBlob;
         3950  +  int iInt;
         3951  +  
         3952  +  nBlob = sqlite3_value_bytes(argv[0]);
         3953  +  pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]);
         3954  +  iInt = sqlite3_value_int(argv[1]);
         3955  +
         3956  +  if( iInt>=0 && (iInt+1)*4<=nBlob ){
         3957  +    const unsigned char *a = &pBlob[iInt*4];
         3958  +    sqlite3_int64 iVal = ((sqlite3_int64)a[0]<<24)
         3959  +                       + ((sqlite3_int64)a[1]<<16)
         3960  +                       + ((sqlite3_int64)a[2]<< 8)
         3961  +                       + ((sqlite3_int64)a[3]<< 0);
         3962  +    sqlite3_result_int64(context, iVal);
         3963  +  }
         3964  +}
  3936   3965   
  3937   3966   /*
  3938   3967   ** Scalar function "shell_escape_crnl" used by the .recover command.
  3939   3968   ** The argument passed to this function is the output of built-in
  3940   3969   ** function quote(). If the first character of the input is "'", 
  3941   3970   ** indicating that the value passed to quote() was a text value,
  3942   3971   ** then this function searches the input for "\n" and "\r" characters
................................................................................
  4101   4130                               shellAddSchemaName, 0, 0);
  4102   4131       sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0,
  4103   4132                               shellModuleSchema, 0, 0);
  4104   4133       sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p,
  4105   4134                               shellPutsFunc, 0, 0);
  4106   4135       sqlite3_create_function(p->db, "shell_escape_crnl", 1, SQLITE_UTF8, 0,
  4107   4136                               shellEscapeCrnl, 0, 0);
         4137  +    sqlite3_create_function(p->db, "shell_int32", 2, SQLITE_UTF8, 0,
         4138  +                            shellInt32, 0, 0);
  4108   4139   #ifndef SQLITE_NOHAVE_SYSTEM
  4109   4140       sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0,
  4110   4141                               editFunc, 0, 0);
  4111   4142       sqlite3_create_function(p->db, "edit", 2, SQLITE_UTF8, 0,
  4112   4143                               editFunc, 0, 0);
  4113   4144   #endif
  4114   4145       if( p->openMode==SHELL_OPEN_ZIPFILE ){
................................................................................
  6370   6401   ** on stream pState->out.
  6371   6402   */
  6372   6403   static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){
  6373   6404     int rc = SQLITE_OK;
  6374   6405     sqlite3_stmt *pLoop = 0;        /* Loop through all root pages */
  6375   6406     sqlite3_stmt *pPages = 0;       /* Loop through all pages in a group */
  6376   6407     sqlite3_stmt *pCells = 0;       /* Loop through all cells in a page */
         6408  +  int i;
         6409  +
         6410  +  int bFreelist = 1;              /* 0 if --freelist-corrupt is specified */
         6411  +  for(i=1; i<nArg; i++){
         6412  +    char *z = azArg[i];
         6413  +    int n;
         6414  +    if( z[0]=='-' && z[1]=='-' ) z++;
         6415  +    n = strlen(z);
         6416  +    if( n<=17 && memcmp("-freelist-corrupt", z, n)==0 ){
         6417  +      bFreelist = 0;
         6418  +    }
         6419  +    else{
         6420  +      raw_printf(stderr, 
         6421  +        "unexpected option: %s - expected \"--freelist-corrupt\"\n", 
         6422  +        azArg[i]
         6423  +      );
         6424  +      return 1;
         6425  +    }
         6426  +  }
  6377   6427   
  6378   6428     shellExec(pState->db, &rc, 
  6379   6429       /* Attach an in-memory database named 'recovery'. Create an indexed 
  6380   6430       ** cache of the sqlite_dbptr virtual table. */
  6381   6431       "ATTACH '' AS recovery;"
  6382   6432       "CREATE TABLE recovery.dbptr("
  6383   6433       "      pgno, child, PRIMARY KEY(child, pgno)"
................................................................................
  6391   6441       /* Delete all pointers to any pages that have more than one pointer
  6392   6442       ** to them. Such pages will be treated as root pages when recovering
  6393   6443       ** data.  */
  6394   6444       "DELETE FROM recovery.dbptr WHERE child IN ("
  6395   6445       "  SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1"
  6396   6446       ");"
  6397   6447   
         6448  +    "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);"
         6449  +  );
         6450  +
         6451  +  if( bFreelist ){
         6452  +    shellExec(pState->db, &rc,
         6453  +      "WITH trunk(pgno) AS ("
         6454  +      "  SELECT shell_int32("
         6455  +      "      (SELECT data FROM sqlite_dbpage WHERE pgno=1), 8) AS x "
         6456  +      "      WHERE x>0"
         6457  +      "    UNION"
         6458  +      "  SELECT shell_int32("
         6459  +      "      (SELECT data FROM sqlite_dbpage WHERE pgno=trunk.pgno), 0) AS x "
         6460  +      "      FROM trunk WHERE x>0"
         6461  +      "),"
         6462  +      "freelist(data, n, freepgno) AS ("
         6463  +      "  SELECT data, shell_int32(data, 1)-1, t.pgno "
         6464  +      "      FROM trunk t, sqlite_dbpage s WHERE s.pgno=t.pgno"
         6465  +      "    UNION ALL"
         6466  +      "  SELECT data, n-1, shell_int32(data, 2+n) "
         6467  +      "      FROM freelist WHERE n>=0"
         6468  +      ")"
         6469  +      "REPLACE INTO recovery.freelist SELECT freepgno FROM freelist;"
         6470  +    );
         6471  +  }
         6472  +
         6473  +  shellExec(pState->db, &rc, 
  6398   6474       /* Create the "map" table that will (eventually) contain instructions
  6399   6475       ** for dealing with each page in the db that contains one or more 
  6400   6476       ** records. */
  6401   6477       "CREATE TABLE recovery.map("
  6402   6478         "pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT"
  6403   6479       ");"
  6404   6480   
................................................................................
  6420   6496       "      SELECT 0, i, (SELECT pgno FROM recovery.dbptr WHERE child=i)"
  6421   6497       "        UNION ALL"
  6422   6498       "      SELECT i, p.parent, "
  6423   6499       "        (SELECT pgno FROM recovery.dbptr WHERE child=p.parent) FROM p"
  6424   6500       "    )"
  6425   6501       "    SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)"
  6426   6502       ") "
  6427         -    "FROM pages WHERE maxlen > 0;"
         6503  +    "FROM pages WHERE maxlen > 0 AND i NOT IN freelist;"
  6428   6504       "UPDATE recovery.map AS o SET intkey = ("
  6429   6505       "  SELECT substr(data, 1, 1)==X'0D' FROM sqlite_dbpage WHERE pgno=o.pgno"
  6430   6506       ");"
  6431   6507   
  6432   6508       /* Extract data from page 1 and any linked pages into table
  6433   6509       ** recovery.schema. With the same schema as an sqlite_master table.  */
  6434   6510       "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);"