/ Check-in [f8560c60]
Login
Home Timeline Branches Tags Tickets Wiki More...
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix an error in the SQLITE_MAX_MEMORY implementation resulting from a bad merge. Update the OSSFuzz interface so that it times out after running the byte-code engine for 10 seconds.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: f8560c60d10c0365b33342ab05b5a953987b0471
User & Date: drh 2017-03-10 17:03:11
Context
2017-03-10
18:36
Remove the rbu_round_trip.tcl script. It is now part of project "test-dbs". check-in: b5bf2957 user: dan tags: trunk
17:03
Fix an error in the SQLITE_MAX_MEMORY implementation resulting from a bad merge. Update the OSSFuzz interface so that it times out after running the byte-code engine for 10 seconds. check-in: f8560c60 user: drh tags: trunk
16:22
Add the -DSQLITE_MAX_MEMORY=N compile-time option. The default is no limit. check-in: eabd4ef4 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to Makefile.in. 

562
563
564
565
566
567
568

569
570
571
572
573
574
575

#
SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
# SHELL_OPT += -DSQLITE_ENABLE_FTS5
SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ

FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
DBFUZZ_OPT = 

# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#
all:	sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la)








>








562
563
564
565
566
567
568
569
570
571
572
573
574
575
576

#
SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
# SHELL_OPT += -DSQLITE_ENABLE_FTS5
SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
DBFUZZ_OPT = 

# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#
all:	sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la)

Changes to Makefile.msc. 

1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520

!ENDIF

# <<mark>>
# Extra compiler options for various test tools.
#
MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5
FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c
OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
DBSELFTEST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
ST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0









|








1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520

!ENDIF

# <<mark>>
# Extra compiler options for various test tools.
#
MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5
FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000
FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c
OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
DBSELFTEST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
ST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0

Changes to main.mk. 

473
474
475
476
477
478
479

480
481
482
483
484
485
486

# Extra compiler options for various shell tools
#
SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5

DBFUZZ_OPT =
KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
ST_OPT = -DSQLITE_THREADSAFE=0

# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#








>








473
474
475
476
477
478
479
480
481
482
483
484
485
486
487

# Extra compiler options for various shell tools
#
SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
DBFUZZ_OPT =
KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
ST_OPT = -DSQLITE_THREADSAFE=0

# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#

Changes to src/malloc.c. 

228
229
230
231
232
233
234

235
236
237
238
239
240
241
242

  ** mode and specifically when the DMD "Dark Matter Detector" is enabled
  ** or else a crash results.  Hence, do not attempt to optimize out the
  ** following xRoundup() call. */
  nFull = sqlite3GlobalConfig.m.xRoundup(n);

#ifdef SQLITE_MAX_MEMORY
  if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)+nFull>SQLITE_MAX_MEMORY ){

    return 0;
  }
#endif

  sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n);
  if( mem0.alarmThreshold>0 ){
    sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED);
    if( nUsed >= mem0.alarmThreshold - nFull ){








>
|








228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243

  ** mode and specifically when the DMD "Dark Matter Detector" is enabled
  ** or else a crash results.  Hence, do not attempt to optimize out the
  ** following xRoundup() call. */
  nFull = sqlite3GlobalConfig.m.xRoundup(n);

#ifdef SQLITE_MAX_MEMORY
  if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)+nFull>SQLITE_MAX_MEMORY ){
    *pp = 0;
    return;
  }
#endif

  sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n);
  if( mem0.alarmThreshold>0 ){
    sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED);
    if( nUsed >= mem0.alarmThreshold - nFull ){

Changes to test/ossfuzz.c. 

1
2
3
4
5
6
7

















8
9
10
11



12
13
14


15
16
17
18
19
20
21
..
27
28
29
30
31
32
33
34
35
36
37
38
39
40

41
42
43
44
45
46
47
..
52
53
54
55
56
57
58
59
60
61


62
63
64
65
66
67
68
69
70
71
72
73
74
75

/*
** This module interfaces SQLite to the Google OSS-Fuzz, fuzzer as a service.
** (https://github.com/google/oss-fuzz)
*/
#include <stddef.h>
#include <stdint.h>
#include "sqlite3.h"


















#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
/*
** Progress handler callback



*/
static int progress_handler(void *pReturn) {
  return *(int*)pReturn;


}
#endif

/*
** Callback for sqlite3_exec().
*/
static int exec_handler(void *pCnt, int argc, char **argv, char **namev){
................................................................................
}

/*
** Main entry point.  The fuzzer invokes this function with each
** fuzzed input.
*/
int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
  int progressArg = 0;     /* 1 causes progress handler abort */
  int execCnt = 0;         /* Abort row callback when count reaches zero */
  char *zErrMsg = 0;       /* Error message returned by sqlite_exec() */
  sqlite3 *db;             /* The database connection */
  uint8_t uSelector;       /* First byte of input data[] */
  int rc;                  /* Return code from various interfaces */
  char *zSql;              /* Zero-terminated copy of data[] */


  if( size<3 ) return 0;   /* Early out if unsufficient data */

  /* Extract the selector byte from the beginning of the input.  But only
  ** do this if the second byte is a \n.  If the second byte is not \n,
  ** then use a default selector */
  if( data[1]=='\n' ){
................................................................................

  /* Open the database connection.  Only use an in-memory database. */
  rc = sqlite3_open_v2("fuzz.db", &db,
           SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY, 0);
  if( rc ) return 0;

#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
  /* Bit 0 of the selector enables progress callbacks.  Bit 1 is the
  ** return code from progress callbacks */
  if( uSelector & 1 ){


    sqlite3_progress_handler(db, 4, progress_handler, (void*)&progressArg);
  }
#endif
  uSelector >>= 1;
  progressArg = uSelector & 1;  uSelector >>= 1;

  /* Bit 2 of the selector enables foreign key constraints */
  sqlite3_db_config(db, SQLITE_DBCONFIG_ENABLE_FKEY, uSelector&1, &rc);
  uSelector >>= 1;

  /* Remaining bits of the selector determine a limit on the number of
  ** output rows */
  execCnt = uSelector + 1;









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



|
>
>
>


<
>
>







 







<






>







 







|
|
<
>
>
|
<

<
<

|








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

34
35
36
37
38
39
40
41
42
..
48
49
50
51
52
53
54

55
56
57
58
59
60
61
62
63
64
65
66
67
68
..
73
74
75
76
77
78
79
80
81

82
83
84

85


86
87
88
89
90
91
92
93
94

/*
** This module interfaces SQLite to the Google OSS-Fuzz, fuzzer as a service.
** (https://github.com/google/oss-fuzz)
*/
#include <stddef.h>
#include <stdint.h>
#include "sqlite3.h"

/* Return the current real-world time in milliseconds since the
** Julian epoch (-4714-11-24).
*/
static sqlite3_int64 timeOfDay(void){
  static sqlite3_vfs *clockVfs = 0;
  sqlite3_int64 t;
  if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0);
  if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){
    clockVfs->xCurrentTimeInt64(clockVfs, &t);
  }else{
    double r;
    clockVfs->xCurrentTime(clockVfs, &r);
    t = (sqlite3_int64)(r*86400000.0);
  }
  return t;
}

#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
/*
** Progress handler callback.
**
** The argument is the cutoff-time after which all processing should
** stop.  So return non-zero if the cut-off time is exceeded.
*/
static int progress_handler(void *pReturn) {

  sqlite3_int64 iCutoffTime = *(sqlite3_int64*)pReturn;
  return timeOfDay()>=iCutoffTime;
}
#endif

/*
** Callback for sqlite3_exec().
*/
static int exec_handler(void *pCnt, int argc, char **argv, char **namev){
................................................................................
}

/*
** Main entry point.  The fuzzer invokes this function with each
** fuzzed input.
*/
int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {

  int execCnt = 0;         /* Abort row callback when count reaches zero */
  char *zErrMsg = 0;       /* Error message returned by sqlite_exec() */
  sqlite3 *db;             /* The database connection */
  uint8_t uSelector;       /* First byte of input data[] */
  int rc;                  /* Return code from various interfaces */
  char *zSql;              /* Zero-terminated copy of data[] */
  sqlite3_int64 iCutoff;   /* Cutoff timer */

  if( size<3 ) return 0;   /* Early out if unsufficient data */

  /* Extract the selector byte from the beginning of the input.  But only
  ** do this if the second byte is a \n.  If the second byte is not \n,
  ** then use a default selector */
  if( data[1]=='\n' ){
................................................................................

  /* Open the database connection.  Only use an in-memory database. */
  rc = sqlite3_open_v2("fuzz.db", &db,
           SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY, 0);
  if( rc ) return 0;

#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
  /* Invoke the progress handler every 500 thousand instructions (approximately
  ** 20 to 40 times per second) to check to see if we are taking too long.

  */
  iCutoff = timeOfDay() + 10000;  /* Now + 10 seconds */
  sqlite3_progress_handler(db, 500000, progress_handler, (void*)&iCutoff);

#endif



  /* Bit 1 of the selector enables foreign key constraints */
  sqlite3_db_config(db, SQLITE_DBCONFIG_ENABLE_FKEY, uSelector&1, &rc);
  uSelector >>= 1;

  /* Remaining bits of the selector determine a limit on the number of
  ** output rows */
  execCnt = uSelector + 1;

This page was generated in about 0.013s by Fossil 2.10 [9d9ef82234] 2019-10-04 21:41:13