/ Check-in [c3c672af]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a bug in the fts5 porter tokenizer preventing it from passing xCreate() arguments through to its parent tokenizer.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: c3c672af97edf2ae5d793f6fa47364370aa4f4ec
User & Date: dan 2015-07-31 14:43:02
Context
2015-07-31
15:13
Fix a problem causing [sqldiff --rbu] to fail on tables for which all columns are part of the PRIMARY KEY. check-in: 93449e70 user: dan tags: trunk
14:43
Fix a bug in the fts5 porter tokenizer preventing it from passing xCreate() arguments through to its parent tokenizer. check-in: c3c672af user: dan tags: trunk
2015-07-30
20:26
Add the --rbu switch to the "sqldiff" utility. check-in: 098bea26 user: dan tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_tokenize.c.

   533    533     if( pRet ){
   534    534       memset(pRet, 0, sizeof(PorterTokenizer));
   535    535       rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer);
   536    536     }else{
   537    537       rc = SQLITE_NOMEM;
   538    538     }
   539    539     if( rc==SQLITE_OK ){
   540         -    rc = pRet->tokenizer.xCreate(pUserdata, 0, 0, &pRet->pTokenizer);
          540  +    int nArg2 = (nArg>0 ? nArg-1 : 0);
          541  +    const char **azArg2 = (nArg2 ? &azArg[1] : 0);
          542  +    rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer);
   541    543     }
   542    544   
   543    545     if( rc!=SQLITE_OK ){
   544    546       fts5PorterDelete((Fts5Tokenizer*)pRet);
   545    547       pRet = 0;
   546    548     }
   547    549     *ppOut = (Fts5Tokenizer*)pRet;

Changes to ext/fts5/test/fts5tokenizer.test.

   242    242     INSERT INTO e6 VALUES('\u0E08\u0E07\u0E09');
   243    243     CREATE VIRTUAL TABLE e7 USING fts5vocab(e6, 'row');
   244    244     SELECT term FROM e7;
   245    245     ROLLBACK;
   246    246   }] [subst {
   247    247     brown dog fox jumped lazy over quick the \u0E08 \u0E09
   248    248   }]
          249  +
          250  +# Test that the porter tokenizer correctly passes arguments through to
          251  +# its parent tokenizer.
          252  +do_execsql_test 8.3 {
          253  +  BEGIN;
          254  +  CREATE VIRTUAL TABLE e6 USING fts5(x,
          255  +    tokenize="porter unicode61 separators ABCDEFGHIJKLMNOPQRSTUVWXYZ"
          256  +  );
          257  +  INSERT INTO e6 VALUES('theAquickBbrownCfoxDjumpedWoverXtheYlazyZdog');
          258  +  CREATE VIRTUAL TABLE e7 USING fts5vocab(e6, 'row');
          259  +  SELECT term FROM e7;
          260  +  ROLLBACK;
          261  +} {
          262  +  brown dog fox jump lazi over quick the
          263  +}
   249    264   
   250    265   finish_test
   251    266