SQLite

Check-in [2dfc986391]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Reduce the number of distinct token symbols in the parser so that the parser can store tokens in a single byte and thus be smaller in size. (CVS 2776)
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 2dfc9863919c9eb1fd1064ab2817d752dd7f9293
User & Date: drh 2005-11-24 22:22:30.000
Context
2005-11-24
22:33
Make sure sqliteInt.h is included before any system includes. This is required for QNX. Ticket #1478. (CVS 2777) (check-in: ab76453553 user: drh tags: trunk)
22:22
Reduce the number of distinct token symbols in the parser so that the parser can store tokens in a single byte and thus be smaller in size. (CVS 2776) (check-in: 2dfc986391 user: drh tags: trunk)
14:34
Comment changes. No changes to code. (CVS 2775) (check-in: 786e23b295 user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to Makefile.in.
335
336
337
338
339
340
341
342
343
344


345
346
347
348
349
350
351
	$(LTCOMPILE) -c $(TOP)/src/os_win.c

parse.lo:	parse.c $(HDR)
	$(LTCOMPILE) -c parse.c

parse.h:	parse.c

parse.c:	$(TOP)/src/parse.y lemon$(BEXE)
	cp $(TOP)/src/parse.y .
	./lemon $(OPTS) parse.y



pragma.lo:	$(TOP)/src/pragma.c $(HDR)
	$(LTCOMPILE) -c $(TOP)/src/pragma.c

prepare.lo:	$(TOP)/src/prepare.c $(HDR)
	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/prepare.c








|


>
>







335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
	$(LTCOMPILE) -c $(TOP)/src/os_win.c

parse.lo:	parse.c $(HDR)
	$(LTCOMPILE) -c parse.c

parse.h:	parse.c

parse.c:	$(TOP)/src/parse.y lemon$(BEXE) $(TOP)/addopcodes.awk
	cp $(TOP)/src/parse.y .
	./lemon $(OPTS) parse.y
	mv parse.h parse.h.temp
	awk -f $(TOP)/addopcodes.awk parse.h.temp >parse.h

pragma.lo:	$(TOP)/src/pragma.c $(HDR)
	$(LTCOMPILE) -c $(TOP)/src/pragma.c

prepare.lo:	$(TOP)/src/prepare.c $(HDR)
	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/prepare.c

Added addopcodes.awk.
































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
#!/usr/bin/awk
#
# This script appends additional token codes to the end of the
# parse.h file that lemon generates.  These extra token codes are
# not used by the parser.  But they are used by the tokenizer and/or
# the code generator.
#
#
BEGIN {
  max = 0
}
/^#define TK_/ {
  print $0
  if( max<$3 ) max = $3
}
END {
  printf "#define TK_%-29s %4d\n", "TO_TEXT",         max+1
  printf "#define TK_%-29s %4d\n", "TO_BLOB",         max+2
  printf "#define TK_%-29s %4d\n", "TO_NUMERIC",      max+3
  printf "#define TK_%-29s %4d\n", "TO_INT",          max+4
  printf "#define TK_%-29s %4d\n", "TO_REAL",         max+5
  printf "#define TK_%-29s %4d\n", "END_OF_FILE",     max+6
  printf "#define TK_%-29s %4d\n", "ILLEGAL",         max+7
  printf "#define TK_%-29s %4d\n", "SPACE",           max+8
  printf "#define TK_%-29s %4d\n", "UNCLOSED_STRING", max+9
  printf "#define TK_%-29s %4d\n", "COMMENT",         max+10
  printf "#define TK_%-29s %4d\n", "FUNCTION",        max+11
  printf "#define TK_%-29s %4d\n", "COLUMN",          max+12
  printf "#define TK_%-29s %4d\n", "AGG_FUNCTION",    max+13
  printf "#define TK_%-29s %4d\n", "AGG_COLUMN",      max+14
  printf "#define TK_%-29s %4d\n", "CONST_FUNC",      max+15
}
Changes to main.mk.
269
270
271
272
273
274
275
276
277
278


279
280
281
282
283
284
285
	$(TCCX) -c $(TOP)/src/os_win.c

parse.o:	parse.c $(HDR)
	$(TCCX) -c parse.c

parse.h:	parse.c

parse.c:	$(TOP)/src/parse.y lemon
	cp $(TOP)/src/parse.y .
	./lemon $(OPTS) parse.y



pragma.o:	$(TOP)/src/pragma.c $(HDR)
	$(TCCX) $(TCL_FLAGS) -c $(TOP)/src/pragma.c

prepare.o:	$(TOP)/src/prepare.c $(HDR)
	$(TCCX) $(TCL_FLAGS) -c $(TOP)/src/prepare.c








|


>
>







269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
	$(TCCX) -c $(TOP)/src/os_win.c

parse.o:	parse.c $(HDR)
	$(TCCX) -c parse.c

parse.h:	parse.c

parse.c:	$(TOP)/src/parse.y lemon $(TOP)/addopcodes.awk
	cp $(TOP)/src/parse.y .
	./lemon $(OPTS) parse.y
	mv parse.h parse.h.temp
	awk -f $(TOP)/addopcodes.awk parse.h.temp >parse.h

pragma.o:	$(TOP)/src/pragma.c $(HDR)
	$(TCCX) $(TCL_FLAGS) -c $(TOP)/src/pragma.c

prepare.o:	$(TOP)/src/prepare.c $(HDR)
	$(TCCX) $(TCL_FLAGS) -c $(TOP)/src/prepare.c

Changes to mkopcodeh.awk.
18
19
20
21
22
23
24














25
26
27
28
29
30
31
# as an optimization.  During parsing, things like expression operators
# are coded with TK_ values such as TK_ADD, TK_DIVIDE, and so forth.  Later
# during code generation, we need to generate corresponding opcodes like
# OP_Add and OP_Divide.  By making TK_ADD==OP_Add and TK_DIVIDE==OP_Divide,
# code to translation from one to the other is avoided.  This makes the
# code generator run (infinitesimally) faster and more importantly it makes
# the total library smaller.














#


# Remember the TK_ values from the parse.h file
/^#define TK_/ {
  tk[$2] = $3
}







>
>
>
>
>
>
>
>
>
>
>
>
>
>







18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
# as an optimization.  During parsing, things like expression operators
# are coded with TK_ values such as TK_ADD, TK_DIVIDE, and so forth.  Later
# during code generation, we need to generate corresponding opcodes like
# OP_Add and OP_Divide.  By making TK_ADD==OP_Add and TK_DIVIDE==OP_Divide,
# code to translation from one to the other is avoided.  This makes the
# code generator run (infinitesimally) faster and more importantly it makes
# the total library smaller.
#
# This script also scans for lines of the form:
#
#       case OP_aaaa:       /* no-push */
#
# When the no-push comment is found on an opcode, it means that that
# opcode does not leave a result on the stack.  But identifying which
# opcodes leave results on the stack it is possible to determine a
# much smaller upper bound on the size of the stack.  This allows
# a smaller stack to be allocated, which is important to embedded
# systems with limited memory space.  This script generates a series
# of "NOPUSH_MASK" defines that contain bitmaps of opcodes that leave
# results on the stack.  The NOPUSH_MASK defines are used in vdbeaux.c
# to help determine the maximum stack size.
#


# Remember the TK_ values from the parse.h file
/^#define TK_/ {
  tk[$2] = $3
}
Changes to src/parse.y.
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
**
*************************************************************************
** This file contains SQLite's grammar for SQL.  Process this file
** using the lemon parser generator to generate C code that runs
** the parser.  Lemon will also generate a header file containing
** numeric codes for all of the tokens.
**
** @(#) $Id: parse.y,v 1.184 2005/11/14 22:29:05 drh Exp $
*/

// All token codes are small integers with #defines that begin with "TK_"
%token_prefix TK_

// The type of the data attached to each token is Token.  This is also the
// default type for non-terminals.







|







10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
**
*************************************************************************
** This file contains SQLite's grammar for SQL.  Process this file
** using the lemon parser generator to generate C code that runs
** the parser.  Lemon will also generate a header file containing
** numeric codes for all of the tokens.
**
** @(#) $Id: parse.y,v 1.185 2005/11/24 22:22:30 drh Exp $
*/

// All token codes are small integers with #defines that begin with "TK_"
%token_prefix TK_

// The type of the data attached to each token is Token.  This is also the
// default type for non-terminals.
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
/*
** An instance of this structure holds the ATTACH key and the key type.
*/
struct AttachKey { int type;  Token key; };

} // end %include

// These are extra tokens used by the lexer but never seen by the
// parser.  We put them in a rule so that the parser generator will
// add them to the parse.h output file.
//
%nonassoc END_OF_FILE ILLEGAL SPACE UNCLOSED_STRING COMMENT FUNCTION
          COLUMN AGG_FUNCTION AGG_COLUMN CONST_FUNC.

// Extra tokens used by the code generator by never seen by the parser.
%nonassoc TO_TEXT TO_BLOB TO_NUMERIC TO_INT TO_REAL.

// Input is a single SQL command
input ::= cmdlist.
cmdlist ::= cmdlist ecmd.
cmdlist ::= ecmd.
cmdx ::= cmd.           { sqlite3FinishCoding(pParse); }
ecmd ::= SEMI.
ecmd ::= explain cmdx SEMI.







<
<
<
<
<
<
<
<
<
<







84
85
86
87
88
89
90










91
92
93
94
95
96
97
/*
** An instance of this structure holds the ATTACH key and the key type.
*/
struct AttachKey { int type;  Token key; };

} // end %include











// Input is a single SQL command
input ::= cmdlist.
cmdlist ::= cmdlist ecmd.
cmdlist ::= ecmd.
cmdx ::= cmd.           { sqlite3FinishCoding(pParse); }
ecmd ::= SEMI.
ecmd ::= explain cmdx SEMI.
Changes to src/vdbeaux.c.
198
199
200
201
202
203
204

205
206
207
208
209
210
211
  static const u32 masks[5] = {
    NOPUSH_MASK_0 + (NOPUSH_MASK_1<<16),
    NOPUSH_MASK_2 + (NOPUSH_MASK_3<<16),
    NOPUSH_MASK_4 + (NOPUSH_MASK_5<<16),
    NOPUSH_MASK_6 + (NOPUSH_MASK_7<<16),
    NOPUSH_MASK_8 + (NOPUSH_MASK_9<<16)
  };

  return (masks[op>>5] & (1<<(op&0x1F)));
}

#ifndef NDEBUG
int sqlite3VdbeOpcodeNoPush(u8 op){
  return opcodeNoPush(op);
}







>







198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
  static const u32 masks[5] = {
    NOPUSH_MASK_0 + (NOPUSH_MASK_1<<16),
    NOPUSH_MASK_2 + (NOPUSH_MASK_3<<16),
    NOPUSH_MASK_4 + (NOPUSH_MASK_5<<16),
    NOPUSH_MASK_6 + (NOPUSH_MASK_7<<16),
    NOPUSH_MASK_8 + (NOPUSH_MASK_9<<16)
  };
  assert( op>=0 && op<32*5 );
  return (masks[op>>5] & (1<<(op&0x1F)));
}

#ifndef NDEBUG
int sqlite3VdbeOpcodeNoPush(u8 op){
  return opcodeNoPush(op);
}