Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Update fts5 to support "<colset> : ( <expr> )" for column filtering, as well as "<colset> : NEAR(...)" and "<colset> : <phrase>". |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA3-256: |
c847543f8bb1376fef52bca72b419116 |
| User & Date: | dan 2017-04-12 17:50:12 |
Context
|
2017-04-13
| ||
| 00:12 | Fix a regression caused by the fix for ticket [6c9b5514077fed34551f98e64c09a1] - control characters allowed in JSON. (check-in: 8e7b6118 user: drh tags: trunk) | |
|
2017-04-12
| ||
| 17:50 | Update fts5 to support "<colset> : ( <expr> )" for column filtering, as well as "<colset> : NEAR(...)" and "<colset> : <phrase>". (check-in: c847543f user: dan tags: trunk) | |
| 17:38 | Improved \n and \r escapes in the ext/misc/dbdump.c utility function. The implementation of dbdump.c now matches the implementation in the CLI. (check-in: f2643315 user: drh tags: trunk) | |
Changes
Changes to ext/fts5/fts5Int.h.
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 |
);
void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
/*
** End of interface to code in fts5_expr.c.
**************************************************************************/
|
| |
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 |
);
void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
/*
** End of interface to code in fts5_expr.c.
**************************************************************************/
|
Changes to ext/fts5/fts5_expr.c.
1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 |
assert( pParse->rc!=SQLITE_OK );
sqlite3_free(pColset);
}
return pRet;
}
void sqlite3Fts5ParseSetColset(
Fts5Parse *pParse,
Fts5ExprNearset *pNear,
Fts5Colset *pColset
){
if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
pParse->rc = SQLITE_ERROR;
pParse->zErr = sqlite3_mprintf(
"fts5: column queries are not supported (detail=none)"
);
sqlite3_free(pColset);
return;
}
if( pNear ){
pNear->pColset = pColset;
}else{
sqlite3_free(pColset);
}
}
static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
switch( pNode->eType ){
case FTS5_STRING: {
Fts5ExprNearset *pNear = pNode->pNear;
if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1
|
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > < < < < < < < > > |
1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 |
assert( pParse->rc!=SQLITE_OK );
sqlite3_free(pColset);
}
return pRet;
}
/*
** If argument pOrig is NULL, or if (*pRc) is set to anything other than
** SQLITE_OK when this function is called, NULL is returned.
**
** Otherwise, a copy of (*pOrig) is made into memory obtained from
** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation
** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned.
*/
static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){
Fts5Colset *pRet;
if( pOrig ){
int nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int);
pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte);
if( pRet ){
memcpy(pRet, pOrig, nByte);
}
}else{
pRet = 0;
}
return pRet;
}
/*
** Remove from colset pColset any columns that are not also in colset pMerge.
*/
static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){
int iIn = 0; /* Next input in pColset */
int iMerge = 0; /* Next input in pMerge */
int iOut = 0; /* Next output slot in pColset */
while( iIn<pColset->nCol && iMerge<pMerge->nCol ){
int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge];
if( iDiff==0 ){
pColset->aiCol[iOut++] = pMerge->aiCol[iMerge];
iMerge++;
iIn++;
}else if( iDiff>0 ){
iMerge++;
}else{
iIn++;
}
}
pColset->nCol = iOut;
}
/*
** Recursively apply colset pColset to expression node pNode and all of
** its decendents. If (*ppFree) is not NULL, it contains a spare copy
** of pColset. This function may use the spare copy and set (*ppFree) to
** zero, or it may create copies of pColset using fts5CloneColset().
*/
static void fts5ParseSetColset(
Fts5Parse *pParse,
Fts5ExprNode *pNode,
Fts5Colset *pColset,
Fts5Colset **ppFree
){
if( pParse->rc==SQLITE_OK ){
assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING
|| pNode->eType==FTS5_AND || pNode->eType==FTS5_OR
|| pNode->eType==FTS5_NOT || pNode->eType==FTS5_EOF
);
if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){
Fts5ExprNearset *pNear = pNode->pNear;
if( pNear->pColset ){
fts5MergeColset(pNear->pColset, pColset);
if( pNear->pColset->nCol==0 ){
pNode->eType = FTS5_EOF;
pNode->xNext = 0;
}
}else if( *ppFree ){
pNear->pColset = pColset;
*ppFree = 0;
}else{
pNear->pColset = fts5CloneColset(&pParse->rc, pColset);
}
}else{
int i;
assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 );
for(i=0; i<pNode->nChild; i++){
fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree);
}
}
}
}
/*
** Apply colset pColset to expression node pExpr and all of its descendents.
*/
void sqlite3Fts5ParseSetColset(
Fts5Parse *pParse,
Fts5ExprNode *pExpr,
Fts5Colset *pColset
){
Fts5Colset *pFree = pColset;
if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
pParse->rc = SQLITE_ERROR;
pParse->zErr = sqlite3_mprintf(
"fts5: column queries are not supported (detail=none)"
);
}else{
fts5ParseSetColset(pParse, pExpr, pColset, &pFree);
}
sqlite3_free(pFree);
}
static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
switch( pNode->eType ){
case FTS5_STRING: {
Fts5ExprNearset *pNear = pNode->pNear;
if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1
|
Changes to ext/fts5/fts5_index.c.
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
....
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
|
while( p<pEnd && *p!=0x01 ){
while( *p++ & 0x80 );
}
return p - (*pa);
}
static int fts5IndexExtractColset (
Fts5Colset *pColset, /* Colset to filter on */
const u8 *pPos, int nPos, /* Position list */
Fts5Buffer *pBuf /* Output buffer */
){
int rc = SQLITE_OK;
int i;
fts5BufferZero(pBuf);
for(i=0; i<pColset->nCol; i++){
const u8 *pSub = pPos;
int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
if( nSub ){
fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
}
}
return rc;
}
/*
** xSetOutputs callback used by detail=none tables.
*/
static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){
assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE );
................................................................................
/* All data is stored on the current page. Populate the output
** variables to point into the body of the page object. */
const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
if( pColset->nCol==1 ){
pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]);
pIter->base.pData = a;
}else{
fts5BufferZero(&pIter->poslist);
fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist);
pIter->base.pData = pIter->poslist.p;
pIter->base.nData = pIter->poslist.n;
}
}else{
/* The data is distributed over two or more pages. Copy it into the
** Fts5Iter.poslist buffer and then set the output pointer to point
** to this buffer. */
|
|
>
|
|
<
|
|
|
|
|
|
|
|
<
>
>
|
|
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
....
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
|
while( p<pEnd && *p!=0x01 ){
while( *p++ & 0x80 );
}
return p - (*pa);
}
static void fts5IndexExtractColset(
int *pRc,
Fts5Colset *pColset, /* Colset to filter on */
const u8 *pPos, int nPos, /* Position list */
Fts5Buffer *pBuf /* Output buffer */
){
if( *pRc==SQLITE_OK ){
int i;
fts5BufferZero(pBuf);
for(i=0; i<pColset->nCol; i++){
const u8 *pSub = pPos;
int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
if( nSub ){
fts5BufferAppendBlob(pRc, pBuf, nSub, pSub);
}
}
}
}
/*
** xSetOutputs callback used by detail=none tables.
*/
static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){
assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE );
................................................................................
/* All data is stored on the current page. Populate the output
** variables to point into the body of the page object. */
const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
if( pColset->nCol==1 ){
pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]);
pIter->base.pData = a;
}else{
int *pRc = &pIter->pIndex->rc;
fts5BufferZero(&pIter->poslist);
fts5IndexExtractColset(pRc, pColset, a, pSeg->nPos, &pIter->poslist);
pIter->base.pData = pIter->poslist.p;
pIter->base.nData = pIter->poslist.n;
}
}else{
/* The data is distributed over two or more pages. Copy it into the
** Fts5Iter.poslist buffer and then set the output pointer to point
** to this buffer. */
|
Changes to ext/fts5/fts5parse.y.
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
...
133
134
135
136
137
138
139
140
141
142
143
144
145
146
|
%type cnearset {Fts5ExprNode*}
%type expr {Fts5ExprNode*}
%type exprlist {Fts5ExprNode*}
%destructor cnearset { sqlite3Fts5ParseNodeFree($$); }
%destructor expr { sqlite3Fts5ParseNodeFree($$); }
%destructor exprlist { sqlite3Fts5ParseNodeFree($$); }
expr(A) ::= expr(X) AND expr(Y). {
A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
}
expr(A) ::= expr(X) OR expr(Y). {
A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
}
expr(A) ::= expr(X) NOT expr(Y). {
A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
}
expr(A) ::= LP expr(X) RP. {A = X;}
expr(A) ::= exprlist(X). {A = X;}
exprlist(A) ::= cnearset(X). {A = X;}
exprlist(A) ::= exprlist(X) cnearset(Y). {
A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
}
cnearset(A) ::= nearset(X). {
A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X);
}
cnearset(A) ::= colset(X) COLON nearset(Y). {
sqlite3Fts5ParseSetColset(pParse, Y, X);
A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y);
}
%type colset {Fts5Colset*}
%destructor colset { sqlite3_free($$); }
%type colsetlist {Fts5Colset*}
%destructor colsetlist { sqlite3_free($$); }
colset(A) ::= MINUS LCP colsetlist(X) RCP. {
A = sqlite3Fts5ParseColsetInvert(pParse, X);
................................................................................
}
colsetlist(A) ::= colsetlist(Y) STRING(X). {
A = sqlite3Fts5ParseColset(pParse, Y, &X); }
colsetlist(A) ::= STRING(X). {
A = sqlite3Fts5ParseColset(pParse, 0, &X);
}
%type nearset {Fts5ExprNearset*}
%type nearphrases {Fts5ExprNearset*}
%destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
%destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }
nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
85
86
87
88
89
90
91
92
93
94
95
96
97
98
...
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
|
%type cnearset {Fts5ExprNode*}
%type expr {Fts5ExprNode*}
%type exprlist {Fts5ExprNode*}
%destructor cnearset { sqlite3Fts5ParseNodeFree($$); }
%destructor expr { sqlite3Fts5ParseNodeFree($$); }
%destructor exprlist { sqlite3Fts5ParseNodeFree($$); }
%type colset {Fts5Colset*}
%destructor colset { sqlite3_free($$); }
%type colsetlist {Fts5Colset*}
%destructor colsetlist { sqlite3_free($$); }
colset(A) ::= MINUS LCP colsetlist(X) RCP. {
A = sqlite3Fts5ParseColsetInvert(pParse, X);
................................................................................
}
colsetlist(A) ::= colsetlist(Y) STRING(X). {
A = sqlite3Fts5ParseColset(pParse, Y, &X); }
colsetlist(A) ::= STRING(X). {
A = sqlite3Fts5ParseColset(pParse, 0, &X);
}
expr(A) ::= expr(X) AND expr(Y). {
A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
}
expr(A) ::= expr(X) OR expr(Y). {
A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
}
expr(A) ::= expr(X) NOT expr(Y). {
A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
}
expr(A) ::= colset(X) COLON LP expr(Y) RP. {
sqlite3Fts5ParseSetColset(pParse, Y, X);
A = Y;
}
expr(A) ::= LP expr(X) RP. {A = X;}
expr(A) ::= exprlist(X). {A = X;}
exprlist(A) ::= cnearset(X). {A = X;}
exprlist(A) ::= exprlist(X) cnearset(Y). {
A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
}
cnearset(A) ::= nearset(X). {
A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X);
}
cnearset(A) ::= colset(X) COLON nearset(Y). {
A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y);
sqlite3Fts5ParseSetColset(pParse, A, X);
}
%type nearset {Fts5ExprNearset*}
%type nearphrases {Fts5ExprNearset*}
%destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
%destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }
nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }
|
Changes to ext/fts5/test/fts5colset.test.
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 |
5 " - {d d c} : a" {1 2}
6 "- {d c b a} : a" {}
7 "-{\"a\"} : b" {1 2 3}
8 "- c : a" {1 2 4}
9 "-c : a" {1 2 4}
10 "-\"c\" : a" {1 2 4}
} {
breakpoint
do_execsql_test 1.$tn {
SELECT rowid FROM t1($q)
} $res
}
}
finish_test
|
< > > > > > > > > > > > > > | |
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 |
5 " - {d d c} : a" {1 2}
6 "- {d c b a} : a" {}
7 "-{\"a\"} : b" {1 2 3}
8 "- c : a" {1 2 4}
9 "-c : a" {1 2 4}
10 "-\"c\" : a" {1 2 4}
} {
do_execsql_test 1.$tn {
SELECT rowid FROM t1($q)
} $res
}
foreach {tn q res} {
0 {{a} : (a AND ":")} {}
1 "{a b c} : (a AND d)" {2 3}
2 "{a b c} : (a AND b:d)" {3}
3 "{a b c} : (a AND d:d)" {}
4 "{b} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {3 4}
5 "{a} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {2 3}
6 "{a} : ( {b a} : ( {c b} : ( {d b c a} : ( d OR c ) ) ) )" {}
7 "{a b c} : (b:a AND c:b)" {2}
} {
do_execsql_test 2.$tn {
SELECT rowid FROM t1($q)
} $res
}
}
finish_test
|
Changes to ext/fts5/test/fts5faultB.test.
102 103 104 105 106 107 108 109 110 111 |
do_faultsim_test 3.3 -faults oom* -body {
execsql {
SELECT rowid FROM x1('c') WHERE rowid>1;
}
} -test {
faultsim_test_result {0 {2 3}}
}
finish_test
|
> > > > > > > > > > > > > > > > > > > > > > > > |
102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 |
do_faultsim_test 3.3 -faults oom* -body {
execsql {
SELECT rowid FROM x1('c') WHERE rowid>1;
}
} -test {
faultsim_test_result {0 {2 3}}
}
#-------------------------------------------------------------------------
# Test OOM injection with nested colsets.
#
reset_db
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d);
INSERT INTO t1 VALUES('a', 'b', 'c', 'd'); -- 1
INSERT INTO t1 VALUES('d', 'a', 'b', 'c'); -- 2
INSERT INTO t1 VALUES('c', 'd', 'a', 'b'); -- 3
INSERT INTO t1 VALUES('b', 'c', 'd', 'a'); -- 4
}
do_faultsim_test 4.1 -faults oom* -body {
execsql { SELECT rowid FROM t1('{a b c} : (b:a AND c:b)'); }
} -test {
faultsim_test_result {0 2}
}
do_faultsim_test 4.2 -faults oom* -body {
execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') }
} -test {
faultsim_test_result {0 {2 3}}
}
finish_test
|