/ Check-in [7461d2e1]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the ".recovery" command to the shell tool. For recovering the maximum amount data from corrupt databases. Still needs work.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | dbdata
Files: files | file ages | folders
SHA3-256: 7461d2e120f2149315ddac2676d51d7445bcdb8e97543effd9c30603517ef9da
User & Date: dan 2019-04-20 20:57:28
Context
2019-04-22
20:52
Enhance the ".recover" command. Fix a problem with overflow pages in dbdata.c. check-in: f193ca58 user: dan tags: dbdata
2019-04-20
20:57
Add the ".recovery" command to the shell tool. For recovering the maximum amount data from corrupt databases. Still needs work. check-in: 7461d2e1 user: dan tags: dbdata
2019-04-18
21:14
Add the sqlite_dbptr virtual table to the dbdata extension. For querying the links between b-tree pages. check-in: 3213a15f user: dan tags: dbdata
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Changes to ext/misc/dbdata.c.

63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
..
90
91
92
93
94
95
96
97
98



99
100
101
102
103
104
105
...
301
302
303
304
305
306
307








































































308
309
310
311
312
313
314
...
431
432
433
434
435
436
437
438
439
440


441




442
443
444

445
446
447
448


449
450
451
452
453
454
455
456
457
458
...
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500


501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
...
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
**       schema TEXT HIDDEN
**     );
*/
#if !defined(SQLITEINT_H)
#include "sqlite3ext.h"

typedef unsigned char u8;
typedef unsigned int u32;

#endif
SQLITE_EXTENSION_INIT1
#include <string.h>
#include <assert.h>

typedef struct DbdataTable DbdataTable;
................................................................................
  int iCell;                      /* Current cell number */
  int bOnePage;                   /* True to stop after one page */
  sqlite3_int64 iRowid;

  /* Only for the sqlite_dbdata table */
  u8 *pRec;                       /* Buffer containing current record */
  int nRec;                       /* Size of pRec[] in bytes */
  int nField;                     /* Number of fields in pRec */
  int iField;                     /* Current field number */



  sqlite3_int64 iIntkey;          /* Integer key value */
};

/* The sqlite_dbdata table */
struct DbdataTable {
  sqlite3_vtab base;              /* Base class.  Must be first */
  sqlite3 *db;                    /* The database connection */
................................................................................
    v = (v<<7) + (z[i]&0x7f);
    if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; }
  }
  v = (v<<8) + (z[i]&0xff);
  *pVal = v;
  return 9;
}









































































/*
** Move a dbdata cursor to the next entry in the file.
*/
static int dbdataNext(sqlite3_vtab_cursor *pCursor){
  DbdataCursor *pCsr = (DbdataCursor*)pCursor;
  DbdataTable *pTab = (DbdataTable*)pCursor->pVtab;
................................................................................
            memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy);
            nRem -= nCopy;
  
            sqlite3_free(aOvfl);
          }
        }
  
        /* Figure out how many fields in the record */
        pCsr->nField = 0;
        iHdr = dbdataGetVarint(pCsr->pRec, &nHdr);


        while( iHdr<nHdr ){




          sqlite3_int64 iDummy;
          iHdr += dbdataGetVarint(&pCsr->pRec[iHdr], &iDummy);
          pCsr->nField++;

        }
  
        pCsr->iField = (bHasRowid ? -2 : -1);
      }


  
      pCsr->iField++;
      if( pCsr->iField<pCsr->nField ) return SQLITE_OK;
  
      /* Advance to the next cell. The next iteration of the loop will load
      ** the record and so on. */
      sqlite3_free(pCsr->pRec);
      pCsr->pRec = 0;
      pCsr->iCell++;
    }
................................................................................
  DbdataTable *pTab = (DbdataTable*)pCursor->pVtab;
  int rc;
  const char *zSchema = "main";

  dbdataResetCursor(pCsr);
  assert( pCsr->iPgno==1 );
  if( idxNum & 0x01 ){
    zSchema = sqlite3_value_text(argv[0]);
  }
  if( idxNum & 0x02 ){
    pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]);
    pCsr->bOnePage = 1;
  }

  rc = sqlite3_prepare_v2(pTab->db, 
      "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1,
      &pCsr->pStmt, 0
  );
  if( rc==SQLITE_OK ){
    rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT);


  }
  if( rc==SQLITE_OK ){
    rc = dbdataNext(pCursor);
  }
  return rc;
}

static int dbdataValueBytes(int eType){
  switch( eType ){
    case 0: case 8: case 9:
    case 10: case 11:
      return 0;
    case 1:
      return 1;
    case 2:
      return 2;
    case 3:
      return 3;
    case 4:
      return 4;
    case 5:
      return 6;
    case 6:
    case 7:
      return 8;
    default:
      return ((eType-12) / 2);
  }
}

static void dbdataValue(sqlite3_context *pCtx, int eType, u8 *pData){
  switch( eType ){
    case 0: 
    case 10: 
    case 11: 
      sqlite3_result_null(pCtx);
      break;
    
    case 8: 
      sqlite3_result_int(pCtx, 0);
      break;
    case 9:
      sqlite3_result_int(pCtx, 1);
      break;

    case 1: case 2: case 3: case 4: case 5: case 6: case 7: {
      sqlite3_uint64 v = (signed char)pData[0];
      pData++;
      switch( eType ){
        case 7:
        case 6:  v = (v<<16) + (pData[0]<<8) + pData[1];  pData += 2;
        case 5:  v = (v<<16) + (pData[0]<<8) + pData[1];  pData += 2;
        case 4:  v = (v<<8) + pData[0];  pData++;
        case 3:  v = (v<<8) + pData[0];  pData++;
        case 2:  v = (v<<8) + pData[0];  pData++;
      }

      if( eType==7 ){
        double r;
        memcpy(&r, &v, sizeof(r));
        sqlite3_result_double(pCtx, r);
      }else{
        sqlite3_result_int64(pCtx, (sqlite3_int64)v);
      }
      break;
    }

    default: {
      int n = ((eType-12) / 2);
      if( eType % 2 ){
        sqlite3_result_text(pCtx, pData, n, SQLITE_TRANSIENT);
      }else{
        sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT);
      }
    }
  }
}

/* Return a column for the sqlite_dbdata table */
static int dbdataColumn(
  sqlite3_vtab_cursor *pCursor, 
  sqlite3_context *ctx, 
  int i
){
  DbdataCursor *pCsr = (DbdataCursor*)pCursor;
................................................................................
      case DBDATA_COLUMN_FIELD:
        sqlite3_result_int(ctx, pCsr->iField);
        break;
      case DBDATA_COLUMN_VALUE: {
        if( pCsr->iField<0 ){
          sqlite3_result_int64(ctx, pCsr->iIntkey);
        }else{
          int iHdr;
          sqlite3_int64 iType;
          sqlite3_int64 iOff;
          int i;
          iHdr = dbdataGetVarint(pCsr->pRec, &iOff);
          for(i=0; i<pCsr->iField; i++){
            iHdr += dbdataGetVarint(&pCsr->pRec[iHdr], &iType);
            iOff += dbdataValueBytes(iType);
          }
          dbdataGetVarint(&pCsr->pRec[iHdr], &iType);

          dbdataValue(ctx, iType, &pCsr->pRec[iOff]);
        }
        break;
      }
    }
  }
  return SQLITE_OK;
}







|







 







|

>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







<
<

>
>
|
>
>
>
>
|
|
<
>

|
<
|
>
>
|
<
<







 







|












>
>







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







<

<
<
|
<
<
<
<
<
<
|







63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
..
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
...
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
...
506
507
508
509
510
511
512


513
514
515
516
517
518
519
520
521
522

523
524
525

526
527
528
529


530
531
532
533
534
535
536
...
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587







































































588
589
590
591
592
593
594
...
621
622
623
624
625
626
627

628


629






630
631
632
633
634
635
636
637
**       schema TEXT HIDDEN
**     );
*/
#if !defined(SQLITEINT_H) 
#include "sqlite3ext.h"

typedef unsigned char u8;
typedef unsigned long u32;

#endif
SQLITE_EXTENSION_INIT1
#include <string.h>
#include <assert.h>

typedef struct DbdataTable DbdataTable;
................................................................................
  int iCell;                      /* Current cell number */
  int bOnePage;                   /* True to stop after one page */
  sqlite3_int64 iRowid;

  /* Only for the sqlite_dbdata table */
  u8 *pRec;                       /* Buffer containing current record */
  int nRec;                       /* Size of pRec[] in bytes */
  int nHdr;                       /* Size of header in bytes */
  int iField;                     /* Current field number */
  u8 *pHdrPtr;
  u8 *pPtr;
  
  sqlite3_int64 iIntkey;          /* Integer key value */
};

/* The sqlite_dbdata table */
struct DbdataTable {
  sqlite3_vtab base;              /* Base class.  Must be first */
  sqlite3 *db;                    /* The database connection */
................................................................................
    v = (v<<7) + (z[i]&0x7f);
    if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; }
  }
  v = (v<<8) + (z[i]&0xff);
  *pVal = v;
  return 9;
}

static int dbdataValueBytes(int eType){
  switch( eType ){
    case 0: case 8: case 9:
    case 10: case 11:
      return 0;
    case 1:
      return 1;
    case 2:
      return 2;
    case 3:
      return 3;
    case 4:
      return 4;
    case 5:
      return 6;
    case 6:
    case 7:
      return 8;
    default:
      return ((eType-12) / 2);
  }
}

static void dbdataValue(sqlite3_context *pCtx, int eType, u8 *pData){
  switch( eType ){
    case 0: 
    case 10: 
    case 11: 
      sqlite3_result_null(pCtx);
      break;
    
    case 8: 
      sqlite3_result_int(pCtx, 0);
      break;
    case 9:
      sqlite3_result_int(pCtx, 1);
      break;

    case 1: case 2: case 3: case 4: case 5: case 6: case 7: {
      sqlite3_uint64 v = (signed char)pData[0];
      pData++;
      switch( eType ){
        case 7:
        case 6:  v = (v<<16) + (pData[0]<<8) + pData[1];  pData += 2;
        case 5:  v = (v<<16) + (pData[0]<<8) + pData[1];  pData += 2;
        case 4:  v = (v<<8) + pData[0];  pData++;
        case 3:  v = (v<<8) + pData[0];  pData++;
        case 2:  v = (v<<8) + pData[0];  pData++;
      }

      if( eType==7 ){
        double r;
        memcpy(&r, &v, sizeof(r));
        sqlite3_result_double(pCtx, r);
      }else{
        sqlite3_result_int64(pCtx, (sqlite3_int64)v);
      }
      break;
    }

    default: {
      int n = ((eType-12) / 2);
      if( eType % 2 ){
        sqlite3_result_text(pCtx, (const char*)pData, n, SQLITE_TRANSIENT);
      }else{
        sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT);
      }
    }
  }
}


/*
** Move a dbdata cursor to the next entry in the file.
*/
static int dbdataNext(sqlite3_vtab_cursor *pCursor){
  DbdataCursor *pCsr = (DbdataCursor*)pCursor;
  DbdataTable *pTab = (DbdataTable*)pCursor->pVtab;
................................................................................
            memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy);
            nRem -= nCopy;
  
            sqlite3_free(aOvfl);
          }
        }
  


        iHdr = dbdataGetVarint(pCsr->pRec, &nHdr);
        pCsr->nHdr = nHdr;
        pCsr->pHdrPtr = &pCsr->pRec[iHdr];
        pCsr->pPtr = &pCsr->pRec[pCsr->nHdr];
        pCsr->iField = (bHasRowid ? -1 : 0);
      }else{
        pCsr->iField++;
        if( pCsr->iField>0 ){
          sqlite3_int64 iType;
          pCsr->pHdrPtr += dbdataGetVarint(pCsr->pHdrPtr, &iType);

          pCsr->pPtr += dbdataValueBytes(iType);
        }
      }


      if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->pRec[pCsr->nHdr] ){
        return SQLITE_OK;
      }


  
      /* Advance to the next cell. The next iteration of the loop will load
      ** the record and so on. */
      sqlite3_free(pCsr->pRec);
      pCsr->pRec = 0;
      pCsr->iCell++;
    }
................................................................................
  DbdataTable *pTab = (DbdataTable*)pCursor->pVtab;
  int rc;
  const char *zSchema = "main";

  dbdataResetCursor(pCsr);
  assert( pCsr->iPgno==1 );
  if( idxNum & 0x01 ){
    zSchema = (const char*)sqlite3_value_text(argv[0]);
  }
  if( idxNum & 0x02 ){
    pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]);
    pCsr->bOnePage = 1;
  }

  rc = sqlite3_prepare_v2(pTab->db, 
      "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1,
      &pCsr->pStmt, 0
  );
  if( rc==SQLITE_OK ){
    rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT);
  }else{
    pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
  }
  if( rc==SQLITE_OK ){
    rc = dbdataNext(pCursor);
  }
  return rc;
}








































































/* Return a column for the sqlite_dbdata table */
static int dbdataColumn(
  sqlite3_vtab_cursor *pCursor, 
  sqlite3_context *ctx, 
  int i
){
  DbdataCursor *pCsr = (DbdataCursor*)pCursor;
................................................................................
      case DBDATA_COLUMN_FIELD:
        sqlite3_result_int(ctx, pCsr->iField);
        break;
      case DBDATA_COLUMN_VALUE: {
        if( pCsr->iField<0 ){
          sqlite3_result_int64(ctx, pCsr->iIntkey);
        }else{

          sqlite3_int64 iType;


          dbdataGetVarint(pCsr->pHdrPtr, &iType);






          dbdataValue(ctx, iType, pCsr->pPtr);
        }
        break;
      }
    }
  }
  return SQLITE_OK;
}

Changes to main.mk.

734
735
736
737
738
739
740

741
742
743
744
745
746
747
	$(TOP)/ext/misc/fileio.c \
	$(TOP)/ext/misc/completion.c \
	$(TOP)/ext/misc/sqlar.c \
	$(TOP)/ext/expert/sqlite3expert.c \
	$(TOP)/ext/expert/sqlite3expert.h \
	$(TOP)/ext/misc/zipfile.c \
	$(TOP)/ext/misc/memtrace.c \

        $(TOP)/src/test_windirent.c

shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
	tclsh $(TOP)/tool/mkshellc.tcl >shell.c










>







734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
	$(TOP)/ext/misc/fileio.c \
	$(TOP)/ext/misc/completion.c \
	$(TOP)/ext/misc/sqlar.c \
	$(TOP)/ext/expert/sqlite3expert.c \
	$(TOP)/ext/expert/sqlite3expert.h \
	$(TOP)/ext/misc/zipfile.c \
	$(TOP)/ext/misc/memtrace.c \
	$(TOP)/ext/misc/dbdata.c \
        $(TOP)/src/test_windirent.c

shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
	tclsh $(TOP)/tool/mkshellc.tcl >shell.c



Changes to src/shell.c.in.

943
944
945
946
947
948
949


950
951
952
953
954
955
956
....
1098
1099
1100
1101
1102
1103
1104

1105
1106
1107
1108
1109
1110
1111
....
3995
3996
3997
3998
3999
4000
4001

4002
4003
4004
4005
4006
4007
4008
....
6022
6023
6024
6025
6026
6027
6028








































































































































































































































6029
6030
6031
6032
6033
6034
6035
....
6308
6309
6310
6311
6312
6313
6314





6315
6316
6317
6318
6319
6320
6321
....
6346
6347
6348
6349
6350
6351
6352

6353

6354
6355
6356
6357
6358
6359
6360
INCLUDE ../ext/misc/memtrace.c
#ifdef SQLITE_HAVE_ZLIB
INCLUDE ../ext/misc/zipfile.c
INCLUDE ../ext/misc/sqlar.c
#endif
INCLUDE ../ext/expert/sqlite3expert.h
INCLUDE ../ext/expert/sqlite3expert.c



#if defined(SQLITE_ENABLE_SESSION)
/*
** State information for a single open session
*/
typedef struct OpenSession OpenSession;
struct OpenSession {
................................................................................
#define SHFLG_Pagecache      0x00000001 /* The --pagecache option is used */
#define SHFLG_Lookaside      0x00000002 /* Lookaside memory is used */
#define SHFLG_Backslash      0x00000004 /* The --backslash option is used */
#define SHFLG_PreserveRowid  0x00000008 /* .dump preserves rowid values */
#define SHFLG_Newlines       0x00000010 /* .dump --newline flag */
#define SHFLG_CountChanges   0x00000020 /* .changes setting */
#define SHFLG_Echo           0x00000040 /* .echo or --echo setting */


/*
** Macros for testing and setting shellFlgs
*/
#define ShellHasFlag(P,X)    (((P)->shellFlgs & (X))!=0)
#define ShellSetFlag(P,X)    ((P)->shellFlgs|=(X))
#define ShellClearFlag(P,X)  ((P)->shellFlgs&=(~(X)))
................................................................................
    }
#ifndef SQLITE_OMIT_LOAD_EXTENSION
    sqlite3_enable_load_extension(p->db, 1);
#endif
    sqlite3_fileio_init(p->db, 0, 0);
    sqlite3_shathree_init(p->db, 0, 0);
    sqlite3_completion_init(p->db, 0, 0);

#ifdef SQLITE_HAVE_ZLIB
    sqlite3_zipfile_init(p->db, 0, 0);
    sqlite3_sqlar_init(p->db, 0, 0);
#endif
    sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0,
                            shellAddSchemaName, 0, 0);
    sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0,
................................................................................

  return rc;
}
/* End of the ".archive" or ".ar" command logic
**********************************************************************************/
#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */










































































































































































































































/*
** If an input line begins with "." then invoke this routine to
** process that line.
**
** Return 1 on error, 2 to exit, and 0 otherwise.
*/
................................................................................
      utf8_printf(stderr, "Enter \".dbconfig\" with no arguments for a list\n");
    }   
  }else

  if( c=='d' && n>=3 && strncmp(azArg[0], "dbinfo", n)==0 ){
    rc = shell_dbinfo_command(p, nArg, azArg);
  }else






  if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){
    const char *zLike = 0;
    int i;
    int savedShowHeader = p->showHeader;
    int savedShellFlags = p->shellFlgs;
    ShellClearFlag(p, SHFLG_PreserveRowid|SHFLG_Newlines|SHFLG_Echo);
................................................................................
                           "?--newlines? ?LIKE-PATTERN?\n");
        rc = 1;
        goto meta_command_exit;
      }else{
        zLike = azArg[i];
      }
    }

    open_db(p, 0);

    /* When playing back a "dump", the content might appear in an order
    ** which causes immediate foreign key constraints to be violated.
    ** So disable foreign-key constraint enforcement to prevent problems. */
    raw_printf(p->out, "PRAGMA foreign_keys=OFF;\n");
    raw_printf(p->out, "BEGIN TRANSACTION;\n");
    p->writableSchema = 0;
    p->showHeader = 0;







>
>







 







>







 







>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>
>
>







 







>

>







943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
....
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
....
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
....
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
....
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
....
6587
6588
6589
6590
6591
6592
6593
6594
6595
6596
6597
6598
6599
6600
6601
6602
6603
INCLUDE ../ext/misc/memtrace.c
#ifdef SQLITE_HAVE_ZLIB
INCLUDE ../ext/misc/zipfile.c
INCLUDE ../ext/misc/sqlar.c
#endif
INCLUDE ../ext/expert/sqlite3expert.h
INCLUDE ../ext/expert/sqlite3expert.c

INCLUDE ../ext/misc/dbdata.c

#if defined(SQLITE_ENABLE_SESSION)
/*
** State information for a single open session
*/
typedef struct OpenSession OpenSession;
struct OpenSession {
................................................................................
#define SHFLG_Pagecache      0x00000001 /* The --pagecache option is used */
#define SHFLG_Lookaside      0x00000002 /* Lookaside memory is used */
#define SHFLG_Backslash      0x00000004 /* The --backslash option is used */
#define SHFLG_PreserveRowid  0x00000008 /* .dump preserves rowid values */
#define SHFLG_Newlines       0x00000010 /* .dump --newline flag */
#define SHFLG_CountChanges   0x00000020 /* .changes setting */
#define SHFLG_Echo           0x00000040 /* .echo or --echo setting */
#define SHFLG_Recover        0x00000080 /* .dump is --recover */

/*
** Macros for testing and setting shellFlgs
*/
#define ShellHasFlag(P,X)    (((P)->shellFlgs & (X))!=0)
#define ShellSetFlag(P,X)    ((P)->shellFlgs|=(X))
#define ShellClearFlag(P,X)  ((P)->shellFlgs&=(~(X)))
................................................................................
    }
#ifndef SQLITE_OMIT_LOAD_EXTENSION
    sqlite3_enable_load_extension(p->db, 1);
#endif
    sqlite3_fileio_init(p->db, 0, 0);
    sqlite3_shathree_init(p->db, 0, 0);
    sqlite3_completion_init(p->db, 0, 0);
    sqlite3_dbdata_init(p->db, 0, 0);
#ifdef SQLITE_HAVE_ZLIB
    sqlite3_zipfile_init(p->db, 0, 0);
    sqlite3_sqlar_init(p->db, 0, 0);
#endif
    sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0,
                            shellAddSchemaName, 0, 0);
    sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0,
................................................................................

  return rc;
}
/* End of the ".archive" or ".ar" command logic
**********************************************************************************/
#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */

static void shellExec(sqlite3 *db, int *pRc, const char *zSql){
  int rc = *pRc;
  if( rc==SQLITE_OK ){
    char *zErr = 0;
    rc = sqlite3_exec(db, zSql, 0, 0, &zErr);
    if( rc!=SQLITE_OK ){
      raw_printf(stderr, "SQL error: %s\n", zErr);
    }
    *pRc = rc;
  }
}

static void *shellMalloc(int *pRc, sqlite3_int64 nByte){
  void *pRet = 0;
  if( *pRc==SQLITE_OK ){
    pRet = sqlite3_malloc64(nByte);
    if( pRet==0 ){
      *pRc = SQLITE_NOMEM;
    }else{
      memset(pRet, 0, nByte);
    }
  }
  return pRet;
}

static char *shellMPrintf(int *pRc, const char *zFmt, ...){
  char *z = 0;
  if( *pRc==SQLITE_OK ){
    va_list ap;
    va_start(ap, zFmt);
    z = sqlite3_vmprintf(zFmt, ap);
    va_end(ap);
    if( z==0 ){
      *pRc = SQLITE_NOMEM;
    }
  }
  return z;
}

typedef struct RecoverTable RecoverTable;
struct RecoverTable {
  char *zName;                    /* Name of table */
  char *zQuoted;                  /* Quoted version of zName */
  char *zCreate;                  /* SQL to create table in default schema */
  int nCol;                       /* Number of columns in table */
  char **azlCol;                  /* Array of column lists */
};

/*
** Free a RecoverTable object allocated by recoverNewTable()
*/
static void recoverFreeTable(RecoverTable *pTab){
  if( pTab ){
    sqlite3_free(pTab->zName);
    sqlite3_free(pTab->zQuoted);
    sqlite3_free(pTab->zCreate);
    if( pTab->azlCol ){
      int i;
      for(i=0; i<pTab->nCol; i++){
        sqlite3_free(pTab->azlCol[i]);
      }
      sqlite3_free(pTab->azlCol);
    }
    sqlite3_free(pTab);
  }
}

static RecoverTable *recoverNewTable(
  ShellState *pState, 
  int *pRc,
  int iRoot,
  int nCol
){
  RecoverTable *pRet = 0;

  pRet = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable));
  if( pRet ){
    sqlite3_stmt *pStmt = 0;
    pRet->zName = shellMPrintf(pRc, "orphan_%d_%d", nCol, iRoot);
    pRet->zQuoted = shellMPrintf(pRc, "%Q", pRet->zName);
    pRet->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * nCol);
    pRet->nCol = nCol;

    shellPreparePrintf(pState->db, pRc, &pStmt, 
      "WITH s(i) AS ("
      "  SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<%d"
      ")"
      "SELECT i-1, group_concat('c' || i, ', ') OVER (ORDER BY i) FROM s",
      nCol
    );
    while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
      int idx = sqlite3_column_int(pStmt, 0);
      const char *zText = (const char*)sqlite3_column_text(pStmt, 1);
      pRet->azlCol[idx] = shellMPrintf(pRc, "%s", zText);
    }
    shellFinalize(pRc, pStmt);

    pRet->zCreate = shellMPrintf(pRc, "CREATE TABLE %Q (id, %s)", 
        pRet->zName, pRet->azlCol[nCol-1]
    );
  }

  if( *pRc!=SQLITE_OK ){
    recoverFreeTable(pRet);
    pRet = 0;
  }

  return pRet;
}

/*
** This function is called to recover data from the database. A script
** to construct a new database containing all recovered data is output
** on stream pState->out.
*/
static int recoverDatabaseCmd(ShellState *pState){
  const char *zSql;
  int rc = SQLITE_OK;
  sqlite3_stmt *pLoop = 0;        /* Loop through all root pages */

  shellExec(pState->db, &rc, 
    /* Attach an in-memory database named 'recovery'. Create an indexed 
    ** cache of the sqlite_dbptr virtual table. */
    "ATTACH '' AS recovery;"
    "CREATE TABLE recovery.dbptr("
    "      pgno, child, PRIMARY KEY(child, pgno)"
    ") WITHOUT ROWID;"
    "INSERT OR IGNORE INTO dbptr(pgno, child) SELECT * FROM sqlite_dbptr;"

    /* Delete any pointer to page 1. This ensures that page 1 is considered
    ** a root page, regardless of how corrupt the db is. */
    "DELETE FROM recovery.dbptr WHERE child = 1;"

    /* Delete all pointers to any pages that have more than one pointer
    ** to them. Such pages will be treated as root pages when recovering
    ** data.  */
    "DELETE FROM recovery.dbptr WHERE child IN ("
    "  SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1"
    ");"

    /* Create the "map" table that will (eventually) contain instructions
    ** for dealing with each page in the db that contains one or more 
    ** records. */
    "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, maxlen INT, root INT);"

    /* Populate table [map]. If there are circular loops of pages in the
    ** database, the following adds all pages in such a loop to the map
    ** as individual root pages. This could be handled better.  */
    "WITH pages(i, maxlen) AS ("
    "  SELECT page_count, max(field+1) "
    "      FROM pragma_page_count, sqlite_dbdata WHERE pgno=page_count"
    "    UNION ALL"
    "  SELECT * FROM (SELECT i-1, max(field+1)"
    "      FROM pages, sqlite_dbdata WHERE pgno=i-1 AND i>=2)"
    ")"
    "INSERT INTO recovery.map(pgno, maxlen, root) SELECT i, maxlen, ("
    "    WITH p(orig, pgno, parent) AS ("
    "      SELECT 0, i, (SELECT pgno FROM recovery.dbptr WHERE child=i)"
    "        UNION ALL"
    "      SELECT i, p.parent, "
    "        (SELECT pgno FROM recovery.dbptr WHERE child=p.parent) FROM p"
    "    )"
    "    SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)"
    ") "
    "FROM pages WHERE maxlen > 0;"

    /* Extract data from page 1 and any linked pages into table
    ** recovery.schema. With the same schema as an sqlite_master table.  */
    "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);"
    "INSERT INTO recovery.schema SELECT "
    "  max(CASE WHEN field=0 THEN value ELSE NULL END),"
    "  max(CASE WHEN field=1 THEN value ELSE NULL END),"
    "  max(CASE WHEN field=2 THEN value ELSE NULL END),"
    "  max(CASE WHEN field=3 THEN value ELSE NULL END),"
    "  max(CASE WHEN field=4 THEN value ELSE NULL END)"
    "FROM sqlite_dbdata WHERE pgno IN ("
    "  SELECT pgno FROM recovery.map WHERE root=1"
    ")"
    "GROUP BY pgno, cell;"
  );

#if 0
  zSql = "SELECT type ||','|| name ||','|| tbl_name ||','|| rootpage ||','|| sql FROM recovery.schema;";
  shellPrepare(pState->db, &rc, zSql, &pLoop);
  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){
    raw_printf(pState->out, "%s\n", (const char*)sqlite3_column_text(pLoop, 0));
  }
  shellFinalize(&rc, pLoop);
  return rc;
#endif

  /* Loop through each root page. */
  zSql = "SELECT root,max(maxlen) FROM recovery.map WHERE root>1 GROUP BY root";
  shellPrepare(pState->db, &rc, zSql, &pLoop);
  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){
    int iRoot = sqlite3_column_int(pLoop, 0);
    int nCol = sqlite3_column_int(pLoop, 1);
    RecoverTable *pTab;

    pTab = recoverNewTable(pState, &rc, iRoot, nCol);
    if( pTab ){
      sqlite3_stmt *pData = 0;
      raw_printf(pState->out, "%s;\n", pTab->zCreate);
      shellPreparePrintf(pState->db, &rc, &pData, 
        "SELECT max(field), group_concat(quote(value), ', ') "
        "FROM sqlite_dbdata WHERE pgno IN ("
        "  SELECT pgno FROM recovery.map WHERE root=%d"
        ")"
        "GROUP BY pgno, cell;", iRoot
      );
      while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pData) ){
        int iMax = sqlite3_column_int(pData, 0);
        const char *zVal = (const char*)sqlite3_column_text(pData, 1);
        if( iMax+1==pTab->nCol ){
          raw_printf(pState->out, "INSERT INTO %s VALUES( %s );\n", 
              pTab->zQuoted, zVal);
        }else{
          raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", 
              pTab->zQuoted, pTab->azlCol[iMax], zVal
          );
        }
      }
      shellFinalize(&rc, pData);
    }
    recoverFreeTable(pTab);
  }
  shellFinalize(&rc, pLoop);

  sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0);
  return rc;
}


/*
** If an input line begins with "." then invoke this routine to
** process that line.
**
** Return 1 on error, 2 to exit, and 0 otherwise.
*/
................................................................................
      utf8_printf(stderr, "Enter \".dbconfig\" with no arguments for a list\n");
    }   
  }else

  if( c=='d' && n>=3 && strncmp(azArg[0], "dbinfo", n)==0 ){
    rc = shell_dbinfo_command(p, nArg, azArg);
  }else

  if( c=='r' && strncmp(azArg[0], "recover", n)==0 ){
    open_db(p, 0);
    rc = recoverDatabaseCmd(p);
  }else

  if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){
    const char *zLike = 0;
    int i;
    int savedShowHeader = p->showHeader;
    int savedShellFlags = p->shellFlgs;
    ShellClearFlag(p, SHFLG_PreserveRowid|SHFLG_Newlines|SHFLG_Echo);
................................................................................
                           "?--newlines? ?LIKE-PATTERN?\n");
        rc = 1;
        goto meta_command_exit;
      }else{
        zLike = azArg[i];
      }
    }

    open_db(p, 0);

    /* When playing back a "dump", the content might appear in an order
    ** which causes immediate foreign key constraints to be violated.
    ** So disable foreign-key constraint enforcement to prevent problems. */
    raw_printf(p->out, "PRAGMA foreign_keys=OFF;\n");
    raw_printf(p->out, "BEGIN TRANSACTION;\n");
    p->writableSchema = 0;
    p->showHeader = 0;

Changes to tool/mkshellc.tcl.

36
37
38
39
40
41
42

43
44
45

46
47
48

49
50
51
52


53
54
55
56
57
58
59
60

61
62
63
64
65
66
    if {[info exists typedef_seen($line)]} {
      return "/* $line */"
    }
    set typedef_seen($line) 1
  }
  return $line
}

while {1} {
  set lx [omit_redundant_typedefs [gets $in]]
  if {[eof $in]} break;

  if {[regexp {^INCLUDE } $lx]} {
    set cfile [lindex $lx 1]
    puts $out "/************************* Begin $cfile ******************/"

    set in2 [open $topdir/src/$cfile rb]
    while {![eof $in2]} {
      set lx [omit_redundant_typedefs [gets $in2]]
      if {[regexp {^#include "sqlite} $lx]} continue


      if {[regexp {^# *include "test_windirent.h"} $lx]} {
        set lx "/* $lx */"
      }
      set lx [string map [list __declspec(dllexport) {}] $lx]
      puts $out $lx
    }
    close $in2
    puts $out "/************************* End $cfile ********************/"

    continue
  }
  puts $out $lx
}
close $in
close $out







>



>



>



|
>
>








>






36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
    if {[info exists typedef_seen($line)]} {
      return "/* $line */"
    }
    set typedef_seen($line) 1
  }
  return $line
}
set iLine 0
while {1} {
  set lx [omit_redundant_typedefs [gets $in]]
  if {[eof $in]} break;
  incr iLine
  if {[regexp {^INCLUDE } $lx]} {
    set cfile [lindex $lx 1]
    puts $out "/************************* Begin $cfile ******************/"
    puts $out "#line 1 \"$cfile\""
    set in2 [open $topdir/src/$cfile rb]
    while {![eof $in2]} {
      set lx [omit_redundant_typedefs [gets $in2]]
      if {[regexp {^#include "sqlite} $lx]} {
        set lx "/* $lx */"
      }
      if {[regexp {^# *include "test_windirent.h"} $lx]} {
        set lx "/* $lx */"
      }
      set lx [string map [list __declspec(dllexport) {}] $lx]
      puts $out $lx
    }
    close $in2
    puts $out "/************************* End $cfile ********************/"
    puts $out "#line [expr $iLine+1] \"shell.c.in\""
    continue
  }
  puts $out $lx
}
close $in
close $out