/ Check-in [de9f8ef6]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a memory leak that could follow an OOM condition in fts5.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: de9f8ef6ebf036df5a558cd78fb4927da2d83ce8
User & Date: dan 2015-05-19 11:32:01
Context
2015-05-19
11:38
Update Makefile.in so that the amalgamation files built by "make sqlite3.c" include fts5. check-in: 2870a805 user: dan tags: fts5
11:32
Fix a memory leak that could follow an OOM condition in fts5. check-in: de9f8ef6 user: dan tags: fts5
2015-05-18
20:34
Improve test coverage of fts5_storage.c. check-in: 4dfe2e58 user: dan tags: fts5
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Changes to ext/fts5/fts5_tcl.c.

150
151
152
153
154
155
156



157
158
159
160
161
162
163
  Tcl_IncrRefCount(pEval);
  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zToken, nToken));
  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iStart));
  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iEnd));

  rc = Tcl_EvalObjEx(p->interp, pEval, 0);
  Tcl_DecrRefCount(pEval);




  return rc;
}

static int xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []);

static int xQueryPhraseCb(







>
>
>







150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
  Tcl_IncrRefCount(pEval);
  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zToken, nToken));
  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iStart));
  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iEnd));

  rc = Tcl_EvalObjEx(p->interp, pEval, 0);
  Tcl_DecrRefCount(pEval);
  if( rc==TCL_OK ){
    rc = f5tResultToErrorCode(Tcl_GetStringResult(p->interp));
  }

  return rc;
}

static int xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []);

static int xQueryPhraseCb(

Changes to ext/fts5/fts5_tokenize.c.

51
52
53
54
55
56
57
58







59
60
61
62
63
64
65
..
81
82
83
84
85
86
87




88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
    if( (zArg[i] & 0x80)==0 ){
      p->aTokenChar[(int)zArg[i]] = (unsigned char)bTokenChars;
    }
  }
}

/*
** Create a "ascii" tokenizer.







*/
static int fts5AsciiCreate(
  void *pCtx, 
  const char **azArg, int nArg,
  Fts5Tokenizer **ppOut
){
  int rc = SQLITE_OK;
................................................................................
        }else
        if( 0==sqlite3_stricmp(azArg[i], "separators") ){
          fts5AsciiAddExceptions(p, zArg, 0);
        }else{
          rc = SQLITE_ERROR;
        }
      }




    }
  }

  *ppOut = (Fts5Tokenizer*)p;
  return rc;
}

/*
** Delete a "ascii" tokenizer.
*/
static void fts5AsciiDelete(Fts5Tokenizer *p){
  sqlite3_free(p);
}


static void asciiFold(char *aOut, const char *aIn, int nByte){
  int i;
  for(i=0; i<nByte; i++){
    char c = aIn[i];
    if( c>='A' && c<='Z' ) c += 32;
    aOut[i] = c;







|
>
>
>
>
>
>
>







 







>
>
>
>







<
<
<
<
<
<
<







51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
..
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105







106
107
108
109
110
111
112
    if( (zArg[i] & 0x80)==0 ){
      p->aTokenChar[(int)zArg[i]] = (unsigned char)bTokenChars;
    }
  }
}

/*
** Delete a "ascii" tokenizer.
*/
static void fts5AsciiDelete(Fts5Tokenizer *p){
  sqlite3_free(p);
}

/*
** Create an "ascii" tokenizer.
*/
static int fts5AsciiCreate(
  void *pCtx, 
  const char **azArg, int nArg,
  Fts5Tokenizer **ppOut
){
  int rc = SQLITE_OK;
................................................................................
        }else
        if( 0==sqlite3_stricmp(azArg[i], "separators") ){
          fts5AsciiAddExceptions(p, zArg, 0);
        }else{
          rc = SQLITE_ERROR;
        }
      }
      if( rc!=SQLITE_OK ){
        fts5AsciiDelete((Fts5Tokenizer*)p);
        p = 0;
      }
    }
  }

  *ppOut = (Fts5Tokenizer*)p;
  return rc;
}









static void asciiFold(char *aOut, const char *aIn, int nByte){
  int i;
  for(i=0; i<nByte; i++){
    char c = aIn[i];
    if( c>='A' && c<='Z' ) c += 32;
    aOut[i] = c;

Changes to ext/fts5/test/fts5_common.tcl.

47
48
49
50
51
52
53

54
55
56
57
58
59
60
  }
  set res
}

proc test_append_token {varname token iStart iEnd} {
  upvar $varname var
  lappend var $token

}
proc fts5_test_tokenize {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
    set tokens [list]
    $cmd xTokenize [$cmd xColumnText $i] [list test_append_token tokens]
    lappend res $tokens







>







47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
  }
  set res
}

proc test_append_token {varname token iStart iEnd} {
  upvar $varname var
  lappend var $token
  return "SQLITE_OK"
}
proc fts5_test_tokenize {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
    set tokens [list]
    $cmd xTokenize [$cmd xColumnText $i] [list test_append_token tokens]
    lappend res $tokens

Changes to ext/fts5/test/fts5tokenizer.test.

5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
..
96
97
98
99
100
101
102
103

































104
105
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on the fts5 tokenizers
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5tokenizer


do_execsql_test 1.0 {
................................................................................
    9 vw  1
  } {
    do_execsql_test 5.$tn.2.$tn2 "
      SELECT rowid FROM t1 WHERE t1 MATCH '\"$token\"'
    " $res
  }
}


































finish_test








|







 








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
..
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on the built-in fts5 tokenizers. 
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5tokenizer


do_execsql_test 1.0 {
................................................................................
    9 vw  1
  } {
    do_execsql_test 5.$tn.2.$tn2 "
      SELECT rowid FROM t1 WHERE t1 MATCH '\"$token\"'
    " $res
  }
}

#-------------------------------------------------------------------------
# Miscellaneous tests for the ascii tokenizer.
#
# 5.1.*: Test that the ascii tokenizer ignores non-ASCII characters in the
#        'separators' option. But unicode61 does not.
#
# 5.2.*: An option without an argument is an error.
#

do_test 5.1.1 {
  execsql "
    CREATE VIRTUAL TABLE a1 USING fts5(x, tokenize=`ascii separators '\u1234'`);
    INSERT INTO a1 VALUES('abc\u1234def');
  "
  execsql { SELECT rowid FROM a1 WHERE a1 MATCH 'def' } 
} {}

do_test 5.1.2 {
  execsql "
    CREATE VIRTUAL TABLE a2 USING fts5(
        x, tokenize=`unicode61 separators '\u1234'`);
    INSERT INTO a2 VALUES('abc\u1234def');
  "
  execsql { SELECT rowid FROM a2 WHERE a2 MATCH 'def' } 
} {1}

do_catchsql_test 5.2 {
  CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'ascii tokenchars');
} {1 {error in tokenizer constructor}}
do_catchsql_test 5.3 {
  CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'ascii opt arg');
} {1 {error in tokenizer constructor}}

finish_test