/*
** 2014 May 31
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
*/
#include "fts5.h"
#include <string.h>
#include <assert.h>
/**************************************************************************
** Start of simple tokenizer implementation.
*/
/*
** For tokenizers with no "unicode" modifier, the set of token characters
** is the same as the set of ASCII range alphanumeric characters.
*/
static unsigned char aSimpleTokenChar[128] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00..0x0F */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x10..0x1F */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20..0x2F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30..0x3F */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40..0x4F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x50..0x5F */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60..0x6F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x70..0x7F */
};
typedef struct SimpleTokenizer SimpleTokenizer;
struct SimpleTokenizer {
unsigned char aTokenChar[128];
};
static void fts5SimpleAddExceptions(
SimpleTokenizer *p,
const char *zArg,
int bTokenChars
){
int i;
for(i=0; zArg[i]; i++){
if( (zArg[i] & 0x80)==0 ){
p->aTokenChar[(int)zArg[i]] = (unsigned char)bTokenChars;
}
}
}
/*
** Create a "simple" tokenizer.
*/
static int fts5SimpleCreate(
void *pCtx,
const char **azArg, int nArg,
Fts5Tokenizer **ppOut
){
int rc = SQLITE_OK;
SimpleTokenizer *p = 0;
if( nArg%2 ){
rc = SQLITE_ERROR;
}else{
p = sqlite3_malloc(sizeof(SimpleTokenizer));
if( p==0 ){
rc = SQLITE_NOMEM;
}else{
int i;
memset(p, 0, sizeof(SimpleTokenizer));
memcpy(p->aTokenChar, aSimpleTokenChar, sizeof(aSimpleTokenChar));
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
fts5SimpleAddExceptions(p, zArg, 1);
}else
if( 0==sqlite3_stricmp(azArg[i], "separators") ){
fts5SimpleAddExceptions(p, zArg, 0);
}else{
rc = SQLITE_ERROR;
}
}
}
}
*ppOut = (Fts5Tokenizer*)p;
return rc;
}
/*
** Delete a "simple" tokenizer.
*/
static void fts5SimpleDelete(Fts5Tokenizer *p){
sqlite3_free(p);
}
static void simpleFold(char *aOut, const char *aIn, int nByte){
int i;
for(i=0; i<nByte; i++){
char c = aIn[i];
if( c>='A' && c<='Z' ) c += 32;
aOut[i] = c;
}
}
/*
** Tokenize some text using the simple tokenizer.
*/
static int fts5SimpleTokenize(
Fts5Tokenizer *pTokenizer,
void *pCtx,
const char *pText, int nText,
int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd)
){
SimpleTokenizer *p = (SimpleTokenizer*)pTokenizer;
int rc = SQLITE_OK;
int ie;
int is = 0;
char aFold[64];
int nFold = sizeof(aFold);
char *pFold = aFold;
unsigned char *a = p->aTokenChar;
while( is<nText && rc==SQLITE_OK ){
int nByte;
/* Skip any leading divider characters. */
while( is<nText && ((pText[is]&0x80) || a[(int)pText[is]]==0) ){
is++;
}
if( is==nText ) break;
/* Count the token characters */
ie = is+1;
while( ie<nText && ((pText[ie]&0x80)==0 && a[(int)pText[ie]] ) ){
ie++;
}
/* Fold to lower case */
nByte = ie-is;
if( nByte>nFold ){
if( pFold!=aFold ) sqlite3_free(pFold);
pFold = sqlite3_malloc(nByte*2);
if( pFold==0 ){
rc = SQLITE_NOMEM;
break;
}
nFold = nByte*2;
}
simpleFold(pFold, &pText[is], nByte);
/* Invoke the token callback */
rc = xToken(pCtx, pFold, nByte, is, ie);
is = ie+1;
}
if( pFold!=aFold ) sqlite3_free(pFold);
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
return rc;
}
/**************************************************************************
** Start of unicode61 tokenizer implementation.
*/
/*
** Functions in fts5_unicode2.c.
*/
int sqlite3Fts5UnicodeIsalnum(int c);
int sqlite3Fts5UnicodeIsdiacritic(int c);
int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
/*
** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied
** from the sqlite3 source file utf.c. If this file is compiled as part
** of the amalgamation, they are not required.
*/
#ifndef SQLITE_AMALGAMATION
static const unsigned char sqlite3Utf8Trans1[] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00,
};
#define READ_UTF8(zIn, zTerm, c) \
c = *(zIn++); \
if( c>=0xc0 ){ \
c = sqlite3Utf8Trans1[c-0xc0]; \
while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \
c = (c<<6) + (0x3f & *(zIn++)); \
} \
if( c<0x80 \
|| (c&0xFFFFF800)==0xD800 \
|| (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \
}
#define WRITE_UTF8(zOut, c) { \
if( c<0x00080 ){ \
*zOut++ = (unsigned char)(c&0xFF); \
} \
else if( c<0x00800 ){ \
*zOut++ = 0xC0 + (unsigned char)((c>>6)&0x1F); \
*zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
} \
else if( c<0x10000 ){ \
*zOut++ = 0xE0 + (unsigned char)((c>>12)&0x0F); \
*zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \
*zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
}else{ \
*zOut++ = 0xF0 + (unsigned char)((c>>18) & 0x07); \
*zOut++ = 0x80 + (unsigned char)((c>>12) & 0x3F); \
*zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \
*zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
} \
}
#endif /* ifndef SQLITE_AMALGAMATION */
typedef struct Unicode61Tokenizer Unicode61Tokenizer;
struct Unicode61Tokenizer {
int bRemoveDiacritic; /* True if remove_diacritics=1 is set */
int nException;
int *aiException;
};
static int fts5UnicodeAddExceptions(
Unicode61Tokenizer *p, /* Tokenizer object */
const char *z, /* Characters to treat as exceptions */
int bTokenChars /* 1 for 'tokenchars', 0 for 'separators' */
){
int rc = SQLITE_OK;
int n = strlen(z);
int *aNew;
if( n>0 ){
aNew = (int*)sqlite3_realloc(p->aiException, (n+p->nException)*sizeof(int));
if( aNew ){
int nNew = p->nException;
const unsigned char *zCsr = (const unsigned char*)z;
const unsigned char *zTerm = (const unsigned char*)&z[n];
while( zCsr<zTerm ){
int iCode;
int bToken;
READ_UTF8(zCsr, zTerm, iCode);
bToken = sqlite3Fts5UnicodeIsalnum(iCode);
assert( (bToken==0 || bToken==1) );
assert( (bTokenChars==0 || bTokenChars==1) );
if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){
int i;
for(i=0; i<nNew; i++){
if( aNew[i]>iCode ) break;
}
memmove(&aNew[i+1], &aNew[i], (nNew-i)*sizeof(int));
aNew[i] = iCode;
nNew++;
}
}
p->aiException = aNew;
p->nException = nNew;
}else{
rc = SQLITE_NOMEM;
}
}
return rc;
}
/*
** Return true if the p->aiException[] array contains the value iCode.
*/
static int fts5UnicodeIsException(Unicode61Tokenizer *p, int iCode){
if( p->nException>0 ){
int *a = p->aiException;
int iLo = 0;
int iHi = p->nException-1;
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
if( iCode==a[iTest] ){
return 1;
}else if( iCode>a[iTest] ){
iLo = iTest+1;
}else{
iHi = iTest-1;
}
}
}
return 0;
}
/*
** Create a "unicode61" tokenizer.
*/
static int fts5UnicodeCreate(
void *pCtx,
const char **azArg, int nArg,
Fts5Tokenizer **ppOut
){
int rc = SQLITE_OK; /* Return code */
Unicode61Tokenizer *p = 0; /* New tokenizer object */
if( nArg%2 ){
rc = SQLITE_ERROR;
}else{
p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer));
if( p ){
int i;
memset(p, 0, sizeof(Unicode61Tokenizer));
p->bRemoveDiacritic = 1;
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
rc = SQLITE_ERROR;
}
p->bRemoveDiacritic = (zArg[0]=='1');
}else
if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
rc = fts5UnicodeAddExceptions(p, zArg, 1);
}else
if( 0==sqlite3_stricmp(azArg[i], "separators") ){
rc = fts5UnicodeAddExceptions(p, zArg, 0);
}else{
rc = SQLITE_ERROR;
}
}
}else{
rc = SQLITE_NOMEM;
}
*ppOut = (Fts5Tokenizer*)p;
}
return rc;
}
/*
** Delete a "unicode61" tokenizer.
*/
static void fts5UnicodeDelete(Fts5Tokenizer *pTok){
Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTok;
sqlite3_free(p->aiException);
sqlite3_free(p);
return;
}
/*
** Return true if, for the purposes of tokenizing with the tokenizer
** passed as the first argument, codepoint iCode is considered a token
** character (not a separator).
*/
static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){
assert( (sqlite3Fts5UnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 );
return sqlite3Fts5UnicodeIsalnum(iCode) ^ fts5UnicodeIsException(p, iCode);
}
/*
** Tokenize some text using a unicode61 tokenizer.
*/
static int fts5UnicodeTokenize(
Fts5Tokenizer *pTokenizer,
void *pCtx,
const char *pText, int nText,
int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd)
){
Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer;
const unsigned char *zInput = (const unsigned char*)pText;
const unsigned char *zTerm = &zInput[nText];
const unsigned char *z = zInput;
int rc = SQLITE_OK;
int nBuf = 0;
unsigned char *zBuf = 0;
unsigned char *zOut = 0;
while( rc==SQLITE_OK && z<zTerm ){
int iCode;
int bAlnum;
const unsigned char *zStart;
const unsigned char *zCode;
if( zOut==zBuf ) zStart = z;
zCode = z;
READ_UTF8(z, zTerm, iCode);
bAlnum = fts5UnicodeIsAlnum(p, iCode);
if( bAlnum==0 && zOut>zBuf ){
bAlnum = sqlite3Fts5UnicodeIsdiacritic(iCode);
}
if( bAlnum ){
int iOut;
/* Grow the output buffer if required */
while( (zOut-zBuf)+4>=nBuf ){
unsigned char *zNew;
nBuf = (nBuf ? nBuf*2 : 128);
zNew = sqlite3_realloc(zBuf, nBuf);
if( zNew==0 ){
rc = SQLITE_NOMEM;
goto tokenize_finished;
}else{
zOut = &zNew[zOut-zBuf];
zBuf = zNew;
}
}
/* Write the new character to it */
iOut = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic);
if( iOut ) WRITE_UTF8(zOut, iOut);
}
if( zOut>zBuf && (bAlnum==0 || z>=zTerm) ){
int ie = (bAlnum ? z : zCode) - zInput;
rc = xToken(pCtx, (const char*)zBuf, zOut-zBuf, zStart-zInput, ie);
zOut = zBuf;
}
}
tokenize_finished:
sqlite3_free(zBuf);
return rc;
}
/**************************************************************************
** Start of porter stemmer implementation.
*/
/* Any tokens larger than this (in bytes) are passed through without
** stemming. */
#define FTS5_PORTER_MAX_TOKEN 64
typedef struct PorterTokenizer PorterTokenizer;
struct PorterTokenizer {
fts5_tokenizer tokenizer; /* Parent tokenizer module */
Fts5Tokenizer *pTokenizer; /* Parent tokenizer instance */
char aBuf[FTS5_PORTER_MAX_TOKEN + 64];
};
/*
** Delete a "porter" tokenizer.
*/
static void fts5PorterDelete(Fts5Tokenizer *pTok){
if( pTok ){
PorterTokenizer *p = (PorterTokenizer*)pTok;
if( p->pTokenizer ){
p->tokenizer.xDelete(p->pTokenizer);
}
sqlite3_free(p);
}
}
/*
** Create a "porter" tokenizer.
*/
static int fts5PorterCreate(
void *pCtx,
const char **azArg, int nArg,
Fts5Tokenizer **ppOut
){
fts5_api *pApi = (fts5_api*)pCtx;
int rc = SQLITE_OK;
PorterTokenizer *pRet;
void *pUserdata = 0;
pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer));
if( pRet ){
memset(pRet, 0, sizeof(PorterTokenizer));
rc = pApi->xFindTokenizer(pApi, "simple", &pUserdata, &pRet->tokenizer);
}else{
rc = SQLITE_NOMEM;
}
if( rc==SQLITE_OK ){
rc = pRet->tokenizer.xCreate(pUserdata, 0, 0, &pRet->pTokenizer);
}
if( rc!=SQLITE_OK ){
fts5PorterDelete((Fts5Tokenizer*)pRet);
pRet = 0;
}
*ppOut = (Fts5Tokenizer*)pRet;
return rc;
}
typedef struct PorterContext PorterContext;
struct PorterContext {
void *pCtx;
int (*xToken)(void*, const char*, int, int, int);
char *aBuf;
};
typedef struct PorterRule PorterRule;
struct PorterRule {
const char *zSuffix;
int nSuffix;
int (*xCond)(char *zStem, int nStem);
const char *zOutput;
int nOutput;
};
static int fts5PorterApply(char *aBuf, int *pnBuf, PorterRule *aRule){
int ret = -1;
int nBuf = *pnBuf;
PorterRule *p;
for(p=aRule; p->zSuffix; p++){
assert( strlen(p->zSuffix)==p->nSuffix );
assert( strlen(p->zOutput)==p->nOutput );
if( nBuf<p->nSuffix ) continue;
if( 0==memcmp(&aBuf[nBuf - p->nSuffix], p->zSuffix, p->nSuffix) ) break;
}
if( p->zSuffix ){
int nStem = nBuf - p->nSuffix;
if( p->xCond==0 || p->xCond(aBuf, nStem) ){
memcpy(&aBuf[nStem], p->zOutput, p->nOutput);
*pnBuf = nStem + p->nOutput;
ret = p - aRule;
}
}
return ret;
}
static int fts5PorterIsVowel(char c, int bYIsVowel){
return (
c=='a' || c=='e' || c=='i' || c=='o' || c=='u' || (bYIsVowel && c=='y')
);
}
static int fts5PorterGobbleVC(char *zStem, int nStem, int bPrevCons){
int i;
int bCons = bPrevCons;
/* Scan for a vowel */
for(i=0; i<nStem; i++){
if( 0==(bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) break;
}
/* Scan for a consonent */
for(i++; i<nStem; i++){
if( (bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) return i+1;
}
return 0;
}
/* porter rule condition: (m > 0) */
static int fts5Porter_MGt0(char *zStem, int nStem){
return !!fts5PorterGobbleVC(zStem, nStem, 0);
}
/* porter rule condition: (m > 1) */
static int fts5Porter_MGt1(char *zStem, int nStem){
int n;
n = fts5PorterGobbleVC(zStem, nStem, 0);
if( n && fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){
return 1;
}
return 0;
}
/* porter rule condition: (m = 1) */
static int fts5Porter_MEq1(char *zStem, int nStem){
int n;
n = fts5PorterGobbleVC(zStem, nStem, 0);
if( n && 0==fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){
return 1;
}
return 0;
}
/* porter rule condition: (*o) */
static int fts5Porter_Ostar(char *zStem, int nStem){
if( zStem[nStem-1]=='w' || zStem[nStem-1]=='x' || zStem[nStem-1]=='y' ){
return 0;
}else{
int i;
int mask = 0;
int bCons = 0;
for(i=0; i<nStem; i++){
bCons = !fts5PorterIsVowel(zStem[i], bCons);
assert( bCons==0 || bCons==1 );
mask = (mask << 1) + bCons;
}
return ((mask & 0x0007)==0x0005);
}
}
/* porter rule condition: (m > 1 and (*S or *T)) */
static int fts5Porter_MGt1_and_S_or_T(char *zStem, int nStem){
return nStem>0
&& (zStem[nStem-1]=='s' || zStem[nStem-1]=='t')
&& fts5Porter_MGt1(zStem, nStem);
}
/* porter rule condition: (*v*) */
static int fts5Porter_Vowel(char *zStem, int nStem){
int i;
for(i=0; i<nStem; i++){
if( fts5PorterIsVowel(zStem[i], i>0) ){
return 1;
}
}
return 0;
}
static int fts5PorterCb(
void *pCtx,
const char *pToken,
int nToken,
int iStart,
int iEnd
){
PorterContext *p = (PorterContext*)pCtx;
PorterRule aStep1A[] = {
{ "sses", 4, 0, "ss", 2 },
{ "ies", 3, 0, "i", 1 },
{ "ss", 2, 0, "ss", 2 },
{ "s", 1, 0, "", 0 },
{ 0, 0, 0, 0 }
};
PorterRule aStep1B[] = {
{ "eed", 3, fts5Porter_MGt0, "ee", 2 },
{ "ed", 2, fts5Porter_Vowel, "", 0 },
{ "ing", 3, fts5Porter_Vowel, "", 0 },
{ 0, 0, 0, 0 }
};
PorterRule aStep1B2[] = {
{ "at", 2, 0, "ate", 3 },
{ "bl", 2, 0, "ble", 3 },
{ "iz", 2, 0, "ize", 3 },
{ 0, 0, 0, 0 }
};
PorterRule aStep1C[] = {
{ "y", 1, fts5Porter_Vowel, "i", 1 },
{ 0, 0, 0, 0 }
};
PorterRule aStep2[] = {
{ "ational", 7, fts5Porter_MGt0, "ate", 3},
{ "tional", 6, fts5Porter_MGt0, "tion", 4},
{ "enci", 4, fts5Porter_MGt0, "ence", 4},
{ "anci", 4, fts5Porter_MGt0, "ance", 4},
{ "izer", 4, fts5Porter_MGt0, "ize", 3},
{ "logi", 4, fts5Porter_MGt0, "log", 3}, /* added post 1979 */
{ "bli", 3, fts5Porter_MGt0, "ble", 3}, /* modified post 1979 */
{ "alli", 4, fts5Porter_MGt0, "al", 2},
{ "entli", 5, fts5Porter_MGt0, "ent", 3},
{ "eli", 3, fts5Porter_MGt0, "e", 1},
{ "ousli", 5, fts5Porter_MGt0, "ous", 3},
{ "ization", 7, fts5Porter_MGt0, "ize", 3},
{ "ation", 5, fts5Porter_MGt0, "ate", 3},
{ "ator", 4, fts5Porter_MGt0, "ate", 3},
{ "alism", 5, fts5Porter_MGt0, "al", 2},
{ "iveness", 7, fts5Porter_MGt0, "ive", 3},
{ "fulness", 7, fts5Porter_MGt0, "ful", 3},
{ "ousness", 7, fts5Porter_MGt0, "ous", 3},
{ "aliti", 5, fts5Porter_MGt0, "al", 2},
{ "iviti", 5, fts5Porter_MGt0, "ive", 3},
{ "biliti", 6, fts5Porter_MGt0, "ble", 3},
{ 0, 0, 0, 0 }
};
PorterRule aStep3[] = {
{ "icate", 5, fts5Porter_MGt0, "ic", 2},
{ "ative", 5, fts5Porter_MGt0, "", 0},
{ "alize", 5, fts5Porter_MGt0, "al", 2},
{ "iciti", 5, fts5Porter_MGt0, "ic", 2},
{ "ical", 4, fts5Porter_MGt0, "ic", 2},
{ "ful", 3, fts5Porter_MGt0, "", 0},
{ "ness", 4, fts5Porter_MGt0, "", 0},
{ 0, 0, 0, 0 }
};
PorterRule aStep4[] = {
{ "al", 2, fts5Porter_MGt1, "", 0},
{ "ance", 4, fts5Porter_MGt1, "", 0},
{ "ence", 4, fts5Porter_MGt1, "", 0},
{ "er", 2, fts5Porter_MGt1, "", 0},
{ "ic", 2, fts5Porter_MGt1, "", 0},
{ "able", 4, fts5Porter_MGt1, "", 0},
{ "ible", 4, fts5Porter_MGt1, "", 0},
{ "ant", 3, fts5Porter_MGt1, "", 0},
{ "ement", 5, fts5Porter_MGt1, "", 0},
{ "ment", 4, fts5Porter_MGt1, "", 0},
{ "ent", 3, fts5Porter_MGt1, "", 0},
{ "ion", 3, fts5Porter_MGt1_and_S_or_T, "", 0},
{ "ou", 2, fts5Porter_MGt1, "", 0},
{ "ism", 3, fts5Porter_MGt1, "", 0},
{ "ate", 3, fts5Porter_MGt1, "", 0},
{ "iti", 3, fts5Porter_MGt1, "", 0},
{ "ous", 3, fts5Porter_MGt1, "", 0},
{ "ive", 3, fts5Porter_MGt1, "", 0},
{ "ize", 3, fts5Porter_MGt1, "", 0},
{ 0, 0, 0, 0 }
};
char *aBuf;
int nBuf;
int n;
if( nToken>FTS5_PORTER_MAX_TOKEN || nToken<3 ) goto pass_through;
aBuf = p->aBuf;
nBuf = nToken;
memcpy(aBuf, pToken, nBuf);
/* Step 1. */
fts5PorterApply(aBuf, &nBuf, aStep1A);
n = fts5PorterApply(aBuf, &nBuf, aStep1B);
if( n==1 || n==2 ){
if( fts5PorterApply(aBuf, &nBuf, aStep1B2)<0 ){
char c = aBuf[nBuf-1];
if( fts5PorterIsVowel(c, 0)==0
&& c!='l' && c!='s' && c!='z' && c==aBuf[nBuf-2]
){
nBuf--;
}else if( fts5Porter_MEq1(aBuf, nBuf) && fts5Porter_Ostar(aBuf, nBuf) ){
aBuf[nBuf++] = 'e';
}
}
}
fts5PorterApply(aBuf, &nBuf, aStep1C);
/* Steps 2 through 4. */
fts5PorterApply(aBuf, &nBuf, aStep2);
fts5PorterApply(aBuf, &nBuf, aStep3);
fts5PorterApply(aBuf, &nBuf, aStep4);
/* Step 5a. */
if( nBuf>0 && aBuf[nBuf-1]=='e' ){
if( fts5Porter_MGt1(aBuf, nBuf-1)
|| (fts5Porter_MEq1(aBuf, nBuf-1) && !fts5Porter_Ostar(aBuf, nBuf-1))
){
nBuf--;
}
}
/* Step 5b. */
if( nBuf>1 && aBuf[nBuf-1]=='l'
&& aBuf[nBuf-2]=='l' && fts5Porter_MGt1(aBuf, nBuf-1)
){
nBuf--;
}
return p->xToken(p->pCtx, aBuf, nBuf, iStart, iEnd);
pass_through:
return p->xToken(p->pCtx, pToken, nToken, iStart, iEnd);
}
/*
** Tokenize using the porter tokenizer.
*/
static int fts5PorterTokenize(
Fts5Tokenizer *pTokenizer,
void *pCtx,
const char *pText, int nText,
int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd)
){
PorterTokenizer *p = (PorterTokenizer*)pTokenizer;
PorterContext sCtx;
sCtx.xToken = xToken;
sCtx.pCtx = pCtx;
sCtx.aBuf = p->aBuf;
return p->tokenizer.xTokenize(
p->pTokenizer, (void*)&sCtx, pText, nText, fts5PorterCb
);
}
/*
** Register all built-in tokenizers with FTS5.
*/
int sqlite3Fts5TokenizerInit(fts5_api *pApi){
struct BuiltinTokenizer {
const char *zName;
fts5_tokenizer x;
} aBuiltin[] = {
{ "porter", {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }},
{ "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}},
{ "simple", {fts5SimpleCreate, fts5SimpleDelete, fts5SimpleTokenize }}
};
int rc = SQLITE_OK; /* Return code */
int i; /* To iterate through builtin functions */
for(i=0; rc==SQLITE_OK && i<sizeof(aBuiltin)/sizeof(aBuiltin[0]); i++){
rc = pApi->xCreateTokenizer(pApi,
aBuiltin[i].zName,
(void*)pApi,
&aBuiltin[i].x,
0
);
}
return SQLITE_OK;
}