/ Check-in [047aaf83]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Tests and fixes for fts5 external content tables.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 047aaf830d1e72f0fdad3832a0b617e769d66468
User & Date: dan 2015-01-05 20:41:39
Context
2015-01-06
14:38
Further fixes and test cases related to external content tables. check-in: ce6a899b user: dan tags: fts5
2015-01-05
20:41
Tests and fixes for fts5 external content tables. check-in: 047aaf83 user: dan tags: fts5
2015-01-03
20:44
Add support for external content tables to fts5. check-in: 17ef5b59 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.c.

   252    252         break;
   253    253     }
   254    254   }
   255    255   #else
   256    256   # define fts5CheckTransactionState(x,y,z)
   257    257   #endif
   258    258   
          259  +/*
          260  +** Return true if pTab is a contentless table.
          261  +*/
          262  +static int fts5IsContentless(Fts5Table *pTab){
          263  +  return pTab->pConfig->eContent==FTS5_CONTENT_NONE;
          264  +}
   259    265   
   260    266   /*
   261    267   ** Close a virtual table handle opened by fts5InitVtab(). If the bDestroy
   262    268   ** argument is non-zero, attempt delete the shadow tables from teh database
   263    269   */
   264    270   static int fts5FreeVtab(Fts5Table *pTab, int bDestroy){
   265    271     int rc = SQLITE_OK;
................................................................................
   913    919             }
   914    920           }
   915    921         }
   916    922       }else{
   917    923         /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup
   918    924         ** by rowid (ePlan==FTS5_PLAN_ROWID).  */
   919    925         int eStmt = fts5StmtType(idxNum);
   920         -      rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt);
          926  +      rc = sqlite3Fts5StorageStmt(
          927  +          pTab->pStorage, eStmt, &pCsr->pStmt, &pTab->base.zErrMsg
          928  +      );
   921    929         if( rc==SQLITE_OK ){
   922    930           if( ePlan==FTS5_PLAN_ROWID ){
   923    931             sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
   924    932           }
   925    933           rc = fts5NextMethod(pCursor);
   926    934         }
   927    935       }
................................................................................
   991    999   static int fts5SeekCursor(Fts5Cursor *pCsr){
   992   1000     int rc = SQLITE_OK;
   993   1001   
   994   1002     /* If the cursor does not yet have a statement handle, obtain one now. */ 
   995   1003     if( pCsr->pStmt==0 ){
   996   1004       Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
   997   1005       int eStmt = fts5StmtType(pCsr->idxNum);
   998         -    rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt);
         1006  +    rc = sqlite3Fts5StorageStmt(
         1007  +        pTab->pStorage, eStmt, &pCsr->pStmt, &pTab->base.zErrMsg
         1008  +    );
   999   1009       assert( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) );
  1000   1010     }
  1001   1011   
  1002   1012     if( rc==SQLITE_OK && CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){
  1003   1013       assert( pCsr->pExpr );
  1004   1014       sqlite3_reset(pCsr->pStmt);
  1005   1015       sqlite3_bind_int64(pCsr->pStmt, 1, fts5CursorRowid(pCsr));
................................................................................
  1096   1106     **   1. The "old" rowid, or NULL.
  1097   1107     **   2. The "new" rowid.
  1098   1108     **   3. Values for each of the nCol matchable columns.
  1099   1109     **   4. Values for the two hidden columns (<tablename> and "rank").
  1100   1110     */
  1101   1111     assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) );
  1102   1112   
  1103         -  if( nArg>1 ){
  1104         -    sqlite3_value *pCmd = sqlite3_value_type(apVal[2 + pConfig->nCol]);
  1105         -    if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){
  1106         -      const char *z = sqlite3_value_text(pCmd);
  1107         -      if( pConfig->bExternalContent && sqlite3_stricmp("delete", z) ){
  1108         -        return fts5SpecialDelete(pTab, apVal, pRowid);
  1109         -      }else{
  1110         -        return fts5SpecialInsert(pTab, pCmd, apVal[2 + pConfig->nCol + 1]);
  1111         -      }
  1112         -    }
  1113         -  }
  1114         -
  1115   1113     eType0 = sqlite3_value_type(apVal[0]);
  1116   1114     eConflict = sqlite3_vtab_on_conflict(pConfig->db);
  1117   1115   
  1118   1116     assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
  1119   1117     assert( pVtab->zErrMsg==0 );
  1120   1118   
  1121   1119     if( rc==SQLITE_OK && eType0==SQLITE_INTEGER ){
  1122         -    i64 iDel = sqlite3_value_int64(apVal[0]);    /* Rowid to delete */
  1123         -    rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
         1120  +    if( fts5IsContentless(pTab) ){
         1121  +      pTab->base.zErrMsg = sqlite3_mprintf(
         1122  +          "cannot %s contentless fts5 table: %s", 
         1123  +          (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName
         1124  +      );
         1125  +      rc = SQLITE_ERROR;
         1126  +    }else{
         1127  +      i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
         1128  +      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
         1129  +    }
         1130  +  }else if( nArg>1 ){
         1131  +    sqlite3_value *pCmd = apVal[2 + pConfig->nCol];
         1132  +    if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){
         1133  +      const char *z = sqlite3_value_text(pCmd);
         1134  +      if( pConfig->eContent!=FTS5_CONTENT_NORMAL 
         1135  +       && 0==sqlite3_stricmp("delete", z) 
         1136  +      ){
         1137  +        return fts5SpecialDelete(pTab, apVal, pRowid);
         1138  +      }else{
         1139  +        return fts5SpecialInsert(pTab, pCmd, apVal[2 + pConfig->nCol + 1]);
         1140  +      }
         1141  +    }
  1124   1142     }
         1143  +
  1125   1144   
  1126   1145     if( rc==SQLITE_OK && nArg>1 ){
  1127   1146       rc = sqlite3Fts5StorageInsert(pTab->pStorage, apVal, eConflict, pRowid);
  1128   1147     }
  1129   1148   
  1130   1149     return rc;
  1131   1150   }
................................................................................
  1324   1343   
  1325   1344   static int fts5ApiColumnText(
  1326   1345     Fts5Context *pCtx, 
  1327   1346     int iCol, 
  1328   1347     const char **pz, 
  1329   1348     int *pn
  1330   1349   ){
         1350  +  int rc = SQLITE_OK;
  1331   1351     Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  1332         -  int rc = fts5SeekCursor(pCsr);
  1333         -  if( rc==SQLITE_OK ){
  1334         -    *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1);
  1335         -    *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
         1352  +  if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){
         1353  +    *pz = 0;
         1354  +    *pn = 0;
         1355  +  }else{
         1356  +    rc = fts5SeekCursor(pCsr);
         1357  +    if( rc==SQLITE_OK ){
         1358  +      *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1);
         1359  +      *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
         1360  +    }
  1336   1361     }
  1337   1362     return rc;
  1338   1363   }
  1339   1364   
  1340   1365   static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
  1341   1366     Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  1342   1367     Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
................................................................................
  1562   1587   ** the row that the supplied cursor currently points to.
  1563   1588   */
  1564   1589   static int fts5ColumnMethod(
  1565   1590     sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
  1566   1591     sqlite3_context *pCtx,          /* Context for sqlite3_result_xxx() calls */
  1567   1592     int iCol                        /* Index of column to read value from */
  1568   1593   ){
  1569         -  Fts5Config *pConfig = ((Fts5Table*)(pCursor->pVtab))->pConfig;
         1594  +  Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
         1595  +  Fts5Config *pConfig = pTab->pConfig;
  1570   1596     Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  1571   1597     int rc = SQLITE_OK;
  1572   1598     
  1573   1599     assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
  1574   1600   
  1575   1601     if( pCsr->idxNum==FTS5_PLAN_SPECIAL ){
  1576   1602       if( iCol==pConfig->nCol ){
................................................................................
  1593   1619           FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_MATCH
  1594   1620        || FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SORTED_MATCH
  1595   1621       ){
  1596   1622         if( pCsr->pRank || SQLITE_OK==(rc = fts5FindRankFunction(pCsr)) ){
  1597   1623           fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, pCsr->nRankArg, pCsr->apRankArg);
  1598   1624         }
  1599   1625       }
  1600         -  }else{
         1626  +  }else if( !fts5IsContentless(pTab) ){
  1601   1627       rc = fts5SeekCursor(pCsr);
  1602   1628       if( rc==SQLITE_OK ){
  1603   1629         sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1));
  1604   1630       }
  1605   1631     }
  1606   1632     return rc;
  1607   1633   }

Changes to ext/fts5/fts5Int.h.

    72     72     sqlite3 *db;                    /* Database handle */
    73     73     char *zDb;                      /* Database holding FTS index (e.g. "main") */
    74     74     char *zName;                    /* Name of FTS index */
    75     75     int nCol;                       /* Number of columns */
    76     76     char **azCol;                   /* Column names */
    77     77     int nPrefix;                    /* Number of prefix indexes */
    78     78     int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
    79         -  int bExternalContent;           /* Content is external */
    80         -  char *zContent;                 /* "content=" option value (or NULL) */ 
    81         -  char *zContentRowid;            /* "content_rowid=" option value (or NULL) */ 
           79  +  int eContent;                   /* An FTS5_CONTENT value */
           80  +  char *zContent;                 /* content table */ 
           81  +  char *zContentRowid;            /* "content_rowid=" option value */ 
    82     82     Fts5Tokenizer *pTok;
    83     83     fts5_tokenizer *pTokApi;
    84     84   
    85     85     /* Values loaded from the %_config table */
    86     86     int iCookie;                    /* Incremented when %_config is modified */
    87     87     int pgsz;                       /* Approximate page size used in %_data */
    88     88     int nAutomerge;                 /* 'automerge' setting */
    89     89     char *zRank;                    /* Name of rank function */
    90     90     char *zRankArgs;                /* Arguments to rank function */
    91     91   };
           92  +
           93  +#define FTS5_CONTENT_NORMAL   0
           94  +#define FTS5_CONTENT_NONE     1
           95  +#define FTS5_CONTENT_EXTERNAL 2
           96  +
           97  +
    92     98   
    93     99   int sqlite3Fts5ConfigParse(
    94    100       Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
    95    101   );
    96    102   void sqlite3Fts5ConfigFree(Fts5Config*);
    97    103   
    98    104   int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
................................................................................
   397    403   int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
   398    404   
   399    405   int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
   400    406   int sqlite3Fts5StorageInsert(Fts5Storage *p, sqlite3_value **apVal, int, i64*);
   401    407   
   402    408   int sqlite3Fts5StorageIntegrity(Fts5Storage *p);
   403    409   
   404         -int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **);
          410  +int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
   405    411   void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
   406    412   
   407    413   int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
   408    414   int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
   409    415   int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
   410    416   
   411    417   int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);

Changes to ext/fts5/fts5_aux.c.

   218    218   
   219    219     iCol = sqlite3_value_int(apVal[0]);
   220    220     memset(&ctx, 0, sizeof(HighlightContext));
   221    221     ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
   222    222     ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
   223    223     rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn);
   224    224   
   225         -  if( rc==SQLITE_OK ){
   226         -    rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter);
          225  +  if( ctx.zIn ){
          226  +    if( rc==SQLITE_OK ){
          227  +      rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter);
          228  +    }
          229  +
          230  +    if( rc==SQLITE_OK ){
          231  +      rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
          232  +    }
          233  +    fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
          234  +
          235  +    if( rc==SQLITE_OK ){
          236  +      sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
          237  +    }else{
          238  +      sqlite3_result_error_code(pCtx, rc);
          239  +    }
          240  +    sqlite3_free(ctx.zOut);
   227    241     }
   228         -
   229         -  if( rc==SQLITE_OK ){
   230         -    rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx, fts5HighlightCb);
   231         -  }
   232         -  fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
   233         -
   234         -  if( rc==SQLITE_OK ){
   235         -    sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
   236         -  }else{
   237         -    sqlite3_result_error_code(pCtx, rc);
   238         -  }
   239         -  sqlite3_free(ctx.zOut);
   240    242   }
   241    243   /*
   242    244   ** End of highlight() implementation.
   243    245   **************************************************************************/
   244    246   
   245    247   /*
   246    248   ** Implementation of snippet() function.
................................................................................
   271    273       const char *zErr = "wrong number of arguments to function snippet()";
   272    274       sqlite3_result_error(pCtx, zErr, -1);
   273    275       return;
   274    276     }
   275    277   
   276    278     memset(&ctx, 0, sizeof(HighlightContext));
   277    279     iCol = sqlite3_value_int(apVal[0]);
   278         -  rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn);
   279    280     ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
   280    281     ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
   281    282     zEllips = (const char*)sqlite3_value_text(apVal[3]);
   282    283     nToken = sqlite3_value_int(apVal[4]);
   283    284     iBestLast = nToken-1;
   284    285   
   285    286     iBestCol = (iCol>=0 ? iCol : 0);
................................................................................
   324    325   
   325    326     if( rc==SQLITE_OK ){
   326    327       rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
   327    328     }
   328    329     if( rc==SQLITE_OK ){
   329    330       rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
   330    331     }
   331         -  if( rc==SQLITE_OK ){
   332         -    rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
   333         -  }
   334         -
   335         -  if( (iBestStart+nToken-1)>iBestLast ){
   336         -    iBestStart -= (iBestStart+nToken-1-iBestLast) / 2;
   337         -  }
   338         -  if( iBestStart+nToken>nColSize ){
   339         -    iBestStart = nColSize - nToken;
   340         -  }
   341         -  if( iBestStart<0 ) iBestStart = 0;
   342         -
   343         -  ctx.iRangeStart = iBestStart;
   344         -  ctx.iRangeEnd = iBestStart + nToken - 1;
   345         -
   346         -  if( iBestStart>0 ){
   347         -    fts5HighlightAppend(&rc, &ctx, zEllips, -1);
   348         -  }
   349         -  if( rc==SQLITE_OK ){
   350         -    rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx, fts5HighlightCb);
   351         -  }
   352         -  if( ctx.iRangeEnd>=(nColSize-1) ){
   353         -    fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
   354         -  }else{
   355         -    fts5HighlightAppend(&rc, &ctx, zEllips, -1);
   356         -  }
   357         -
   358         -  if( rc==SQLITE_OK ){
   359         -    sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
   360         -  }else{
   361         -    sqlite3_result_error_code(pCtx, rc);
   362         -  }
   363         -  sqlite3_free(ctx.zOut);
          332  +  if( ctx.zIn ){
          333  +    if( rc==SQLITE_OK ){
          334  +      rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
          335  +    }
          336  +
          337  +    if( (iBestStart+nToken-1)>iBestLast ){
          338  +      iBestStart -= (iBestStart+nToken-1-iBestLast) / 2;
          339  +    }
          340  +    if( iBestStart+nToken>nColSize ){
          341  +      iBestStart = nColSize - nToken;
          342  +    }
          343  +    if( iBestStart<0 ) iBestStart = 0;
          344  +
          345  +    ctx.iRangeStart = iBestStart;
          346  +    ctx.iRangeEnd = iBestStart + nToken - 1;
          347  +
          348  +    if( iBestStart>0 ){
          349  +      fts5HighlightAppend(&rc, &ctx, zEllips, -1);
          350  +    }
          351  +    if( rc==SQLITE_OK ){
          352  +      rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
          353  +    }
          354  +    if( ctx.iRangeEnd>=(nColSize-1) ){
          355  +      fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
          356  +    }else{
          357  +      fts5HighlightAppend(&rc, &ctx, zEllips, -1);
          358  +    }
          359  +
          360  +    if( rc==SQLITE_OK ){
          361  +      sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
          362  +    }else{
          363  +      sqlite3_result_error_code(pCtx, rc);
          364  +    }
          365  +    sqlite3_free(ctx.zOut);
          366  +  }
   364    367     sqlite3_free(aSeen);
   365    368   }
   366    369   
   367    370   /************************************************************************/
   368    371   
   369    372   /*
   370    373   ** The first time the bm25() function is called for a query, an instance

Changes to ext/fts5/fts5_config.c.

   330    330       sqlite3_free(azArg);
   331    331       sqlite3_free(pDel);
   332    332       return rc;
   333    333     }
   334    334   
   335    335     if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){
   336    336       int rc = SQLITE_OK;
   337         -    if( pConfig->zContent ){
          337  +    if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
   338    338         *pzErr = sqlite3_mprintf("multiple content=... directives");
   339    339         rc = SQLITE_ERROR;
   340    340       }else{
   341         -      pConfig->zContent = sqlite3_mprintf("%Q.%Q", pConfig->zDb, zArg);
   342         -      pConfig->bExternalContent = 1;
          341  +      if( zArg[0] ){
          342  +        pConfig->eContent = FTS5_CONTENT_EXTERNAL;
          343  +        pConfig->zContent = sqlite3_mprintf("%Q.%Q", pConfig->zDb, zArg);
          344  +      }else{
          345  +        pConfig->eContent = FTS5_CONTENT_NONE;
          346  +        pConfig->zContent = sqlite3_mprintf(
          347  +            "%Q.'%q_docsize'", pConfig->zDb, pConfig->zName
          348  +        );
          349  +      }
   343    350         if( pConfig->zContent==0 ) rc = SQLITE_NOMEM;
   344    351       }
   345    352       return rc;
   346    353     }
   347    354   
   348    355     if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){
   349    356       int rc = SQLITE_OK;
................................................................................
   469    476     ** already been allocated. Otherwise, allocate an instance of the default
   470    477     ** tokenizer (simple) now.  */
   471    478     if( rc==SQLITE_OK && pRet->pTok==0 ){
   472    479       rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
   473    480     }
   474    481   
   475    482     /* If no zContent option was specified, fill in the default values. */
   476         -  if( rc==SQLITE_OK && pRet->zContent==0 ){
          483  +  if( rc==SQLITE_OK && pRet->eContent==FTS5_CONTENT_NORMAL ){
   477    484       pRet->zContent = sqlite3_mprintf("%Q.'%q_content'", pRet->zDb, pRet->zName);
   478    485       if( pRet->zContent==0 ){
   479    486         rc = SQLITE_NOMEM;
   480    487       }else{
   481    488         sqlite3_free(pRet->zContentRowid);
   482    489         pRet->zContentRowid = 0;
   483    490       }

Changes to ext/fts5/fts5_storage.c.

    50     50   ** Fts5Storage.pInsertDocsize - if they have not already been prepared.
    51     51   ** Return SQLITE_OK if successful, or an SQLite error code if an error
    52     52   ** occurs.
    53     53   */
    54     54   static int fts5StorageGetStmt(
    55     55     Fts5Storage *p,                 /* Storage handle */
    56     56     int eStmt,                      /* FTS5_STMT_XXX constant */
    57         -  sqlite3_stmt **ppStmt           /* OUT: Prepared statement handle */
           57  +  sqlite3_stmt **ppStmt,          /* OUT: Prepared statement handle */
           58  +  char **pzErrMsg                 /* OUT: Error message (if any) */
    58     59   ){
    59     60     int rc = SQLITE_OK;
    60     61   
    61     62     assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
    62     63     if( p->aStmt[eStmt]==0 ){
    63     64       const char *azStmt[] = {
    64     65         "SELECT * FROM %s ORDER BY id ASC",               /* SCAN_ASC */
................................................................................
   113    114       }
   114    115   
   115    116       if( zSql==0 ){
   116    117         rc = SQLITE_NOMEM;
   117    118       }else{
   118    119         rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0);
   119    120         sqlite3_free(zSql);
          121  +      if( rc!=SQLITE_OK && pzErrMsg ){
          122  +        *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db));
          123  +      }
   120    124       }
   121    125     }
   122    126   
   123    127     *ppStmt = p->aStmt[eStmt];
   124    128     return rc;
   125    129   }
   126    130   
................................................................................
   201    205   
   202    206     memset(p, 0, nByte);
   203    207     p->aTotalSize = (i64*)&p[1];
   204    208     p->pConfig = pConfig;
   205    209     p->pIndex = pIndex;
   206    210   
   207    211     if( bCreate ){
   208         -    if( pConfig->bExternalContent==0 ){
          212  +    if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
   209    213         char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10);
   210    214         if( zDefn==0 ){
   211    215           rc = SQLITE_NOMEM;
   212    216         }else{
   213    217           int i;
   214    218           int iOff = sprintf(zDefn, "id INTEGER PRIMARY KEY");
   215    219           for(i=0; i<pConfig->nCol; i++){
................................................................................
   250    254       /* Finalize all SQL statements */
   251    255       for(i=0; i<ArraySize(p->aStmt); i++){
   252    256         sqlite3_finalize(p->aStmt[i]);
   253    257       }
   254    258   
   255    259       /* If required, remove the shadow tables from the database */
   256    260       if( bDestroy ){
   257         -      rc = sqlite3Fts5DropTable(p->pConfig, "content");
          261  +      if( p->pConfig->eContent==FTS5_CONTENT_NORMAL ){
          262  +        rc = sqlite3Fts5DropTable(p->pConfig, "content");
          263  +      }
   258    264         if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "docsize");
   259    265         if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "config");
   260    266       }
   261    267   
   262    268       sqlite3_free(p);
   263    269     }
   264    270     return rc;
................................................................................
   294    300   ** remove the %_content row at this time though.
   295    301   */
   296    302   static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){
   297    303     Fts5Config *pConfig = p->pConfig;
   298    304     sqlite3_stmt *pSeek;            /* SELECT to read row iDel from %_data */
   299    305     int rc;                         /* Return code */
   300    306   
   301         -  rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek);
          307  +  rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
   302    308     if( rc==SQLITE_OK ){
   303    309       int rc2;
   304    310       sqlite3_bind_int64(pSeek, 1, iDel);
   305    311       if( sqlite3_step(pSeek)==SQLITE_ROW ){
   306    312         int iCol;
   307    313         Fts5InsertCtx ctx;
   308    314         ctx.pStorage = p;
................................................................................
   334    340   */
   335    341   static int fts5StorageInsertDocsize(
   336    342     Fts5Storage *p,                 /* Storage module to write to */
   337    343     i64 iRowid,                     /* id value */
   338    344     Fts5Buffer *pBuf                /* sz value */
   339    345   ){
   340    346     sqlite3_stmt *pReplace = 0;
   341         -  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace);
          347  +  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
   342    348     if( rc==SQLITE_OK ){
   343    349       sqlite3_bind_int64(pReplace, 1, iRowid);
   344    350       sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
   345    351       sqlite3_step(pReplace);
   346    352       rc = sqlite3_reset(pReplace);
   347    353     }
   348    354     return rc;
................................................................................
   420    426     /* Delete the index records */
   421    427     if( rc==SQLITE_OK ){
   422    428       rc = fts5StorageDeleteFromIndex(p, iDel);
   423    429     }
   424    430   
   425    431     /* Delete the %_docsize record */
   426    432     if( rc==SQLITE_OK ){
   427         -    rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel);
          433  +    rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
   428    434     }
   429    435     if( rc==SQLITE_OK ){
   430    436       sqlite3_bind_int64(pDel, 1, iDel);
   431    437       sqlite3_step(pDel);
   432    438       rc = sqlite3_reset(pDel);
   433    439     }
   434    440   
   435    441     /* Delete the %_content record */
   436    442     if( rc==SQLITE_OK ){
   437         -    rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel);
          443  +    rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0);
   438    444     }
   439    445     if( rc==SQLITE_OK ){
   440    446       sqlite3_bind_int64(pDel, 1, iDel);
   441    447       sqlite3_step(pDel);
   442    448       rc = sqlite3_reset(pDel);
   443    449     }
   444    450   
................................................................................
   455    461     i64 iDel, 
   456    462     sqlite3_value **apVal
   457    463   ){
   458    464     Fts5Config *pConfig = p->pConfig;
   459    465     int rc;
   460    466     sqlite3_stmt *pDel;
   461    467   
   462         -  assert( p->pConfig->bExternalContent );
          468  +  assert( p->pConfig->eContent!=FTS5_CONTENT_NORMAL );
   463    469     rc = fts5StorageLoadTotals(p, 1);
   464    470   
   465    471     /* Delete the index records */
   466    472     if( rc==SQLITE_OK ){
   467    473       int iCol;
   468    474       Fts5InsertCtx ctx;
   469    475       ctx.pStorage = p;
................................................................................
   473    479       for(iCol=0; rc==SQLITE_OK && iCol<pConfig->nCol; iCol++){
   474    480         rc = sqlite3Fts5Tokenize(pConfig, 
   475    481           (const char*)sqlite3_value_text(apVal[iCol]),
   476    482           sqlite3_value_bytes(apVal[iCol]),
   477    483           (void*)&ctx,
   478    484           fts5StorageInsertCallback
   479    485         );
   480         -      p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
          486  +      p->aTotalSize[iCol] -= (i64)ctx.szCol;
   481    487       }
   482    488       p->nTotalRow--;
   483    489     }
   484    490   
   485    491     /* Delete the %_docsize record */
   486    492     if( rc==SQLITE_OK ){
   487         -    rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel);
          493  +    rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
   488    494     }
   489    495     if( rc==SQLITE_OK ){
   490    496       sqlite3_bind_int64(pDel, 1, iDel);
   491    497       sqlite3_step(pDel);
   492    498       rc = sqlite3_reset(pDel);
   493    499     }
   494    500   
................................................................................
   505    511   ** Allocate a new rowid. This is used for "external content" tables when
   506    512   ** a NULL value is inserted into the rowid column. The new rowid is allocated
   507    513   ** by inserting a dummy row into the %_docsize table. The dummy will be
   508    514   ** overwritten later.
   509    515   */
   510    516   static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){
   511    517     sqlite3_stmt *pReplace = 0;
   512         -  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace);
          518  +  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
   513    519     if( rc==SQLITE_OK ){
   514    520       sqlite3_bind_null(pReplace, 1);
   515    521       sqlite3_bind_null(pReplace, 2);
   516    522       sqlite3_step(pReplace);
   517    523       rc = sqlite3_reset(pReplace);
   518    524     }
   519    525     if( rc==SQLITE_OK ){
................................................................................
   539    545     Fts5InsertCtx ctx;              /* Tokenization callback context object */
   540    546     Fts5Buffer buf;                 /* Buffer used to build up %_docsize blob */
   541    547   
   542    548     memset(&buf, 0, sizeof(Fts5Buffer));
   543    549     rc = fts5StorageLoadTotals(p, 1);
   544    550   
   545    551     /* Insert the new row into the %_content table. */
   546         -  if( rc==SQLITE_OK && pConfig->bExternalContent==0 ){
   547         -    if( pConfig->bExternalContent ){
          552  +  if( rc==SQLITE_OK ){
          553  +    if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
   548    554         if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
   549    555           *piRowid = sqlite3_value_int64(apVal[1]);
   550    556         }else{
   551    557           rc = fts5StorageNewRowid(p, piRowid);
   552    558         }
   553    559       }else{
   554    560         if( eConflict==SQLITE_REPLACE ){
................................................................................
   556    562           if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
   557    563             rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
   558    564           }
   559    565         }else{
   560    566           eStmt = FTS5_STMT_INSERT_CONTENT;
   561    567         }
   562    568         if( rc==SQLITE_OK ){
   563         -        rc = fts5StorageGetStmt(p, eStmt, &pInsert);
          569  +        rc = fts5StorageGetStmt(p, eStmt, &pInsert, 0);
   564    570         }
   565    571         for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
   566    572           rc = sqlite3_bind_value(pInsert, i, apVal[i]);
   567    573         }
   568    574         if( rc==SQLITE_OK ){
   569    575           sqlite3_step(pInsert);
   570    576           rc = sqlite3_reset(pInsert);
................................................................................
   678    684     aTotalSize = (i64*)sqlite3_malloc(pConfig->nCol * (sizeof(int)+sizeof(i64)));
   679    685     if( !aTotalSize ) return SQLITE_NOMEM;
   680    686     aColSize = (int*)&aTotalSize[pConfig->nCol];
   681    687     memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol);
   682    688   
   683    689     /* Generate the expected index checksum based on the contents of the
   684    690     ** %_content table. This block stores the checksum in ctx.cksum. */
   685         -  rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN_ASC, &pScan);
          691  +  rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN_ASC, &pScan, 0);
   686    692     if( rc==SQLITE_OK ){
   687    693       int rc2;
   688    694       while( SQLITE_ROW==sqlite3_step(pScan) ){
   689    695         int i;
   690    696         ctx.iRowid = sqlite3_column_int64(pScan, 0);
   691    697         ctx.szCol = 0;
   692    698         rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
................................................................................
   741    747     return rc;
   742    748   }
   743    749   
   744    750   /*
   745    751   ** Obtain an SQLite statement handle that may be used to read data from the
   746    752   ** %_content table.
   747    753   */
   748         -int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **pp){
          754  +int sqlite3Fts5StorageStmt(
          755  +  Fts5Storage *p, 
          756  +  int eStmt, 
          757  +  sqlite3_stmt **pp, 
          758  +  char **pzErrMsg
          759  +){
   749    760     int rc;
   750    761     assert( eStmt==FTS5_STMT_SCAN_ASC 
   751    762          || eStmt==FTS5_STMT_SCAN_DESC
   752    763          || eStmt==FTS5_STMT_LOOKUP
   753    764     );
   754         -  rc = fts5StorageGetStmt(p, eStmt, pp);
          765  +  rc = fts5StorageGetStmt(p, eStmt, pp, pzErrMsg);
   755    766     if( rc==SQLITE_OK ){
   756    767       assert( p->aStmt[eStmt]==*pp );
   757    768       p->aStmt[eStmt] = 0;
   758    769     }
   759    770     return rc;
   760    771   }
   761    772   
................................................................................
   801    812   **
   802    813   ** An SQLite error code is returned if an error occurs, or SQLITE_OK
   803    814   ** otherwise.
   804    815   */
   805    816   int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
   806    817     int nCol = p->pConfig->nCol;
   807    818     sqlite3_stmt *pLookup = 0;
   808         -  int rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup);
          819  +  int rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
   809    820     if( rc==SQLITE_OK ){
   810    821       int bCorrupt = 1;
   811    822       sqlite3_bind_int64(pLookup, 1, iRowid);
   812    823       if( SQLITE_ROW==sqlite3_step(pLookup) ){
   813    824         const u8 *aBlob = sqlite3_column_blob(pLookup, 0);
   814    825         int nBlob = sqlite3_column_bytes(pLookup, 0);
   815    826         if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){
................................................................................
   869    880   
   870    881   int sqlite3Fts5StorageConfigValue(
   871    882     Fts5Storage *p, 
   872    883     const char *z, 
   873    884     sqlite3_value *pVal
   874    885   ){
   875    886     sqlite3_stmt *pReplace = 0;
   876         -  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace);
          887  +  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace, 0);
   877    888     if( rc==SQLITE_OK ){
   878    889       sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_TRANSIENT);
   879    890       sqlite3_bind_value(pReplace, 2, pVal);
   880    891       sqlite3_step(pReplace);
   881    892       rc = sqlite3_reset(pReplace);
   882    893     }
   883    894     if( rc==SQLITE_OK ){

Added ext/fts5/test/fts5content.test.

            1  +# 2014 Dec 20
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +#
           13  +
           14  +if {![info exists testdir]} {
           15  +  set testdir [file join [file dirname [info script]] .. .. .. test]
           16  +}
           17  +source $testdir/tester.tcl
           18  +set testprefix fts5content
           19  +
           20  +do_execsql_test 1.1 {
           21  +  CREATE VIRTUAL TABLE f1 USING fts5(a, b, content='');
           22  +  INSERT INTO f1(rowid, a, b) VALUES(1, 'one',   'o n e');
           23  +  INSERT INTO f1(rowid, a, b) VALUES(2, 'two',   't w o');
           24  +  INSERT INTO f1(rowid, a, b) VALUES(3, 'three', 't h r e e');
           25  +}
           26  +
           27  +do_execsql_test 1.2 {
           28  +  SELECT rowid FROM f1 WHERE f1 MATCH 'o';
           29  +} {2 1}
           30  +
           31  +do_execsql_test 1.3 {
           32  +  INSERT INTO f1(a, b) VALUES('four',   'f o u r');
           33  +  SELECT rowid FROM f1 WHERE f1 MATCH 'o';
           34  +} {4 2 1}
           35  +
           36  +do_execsql_test 1.4 {
           37  +  SELECT rowid, a, b FROM f1 WHERE f1 MATCH 'o';
           38  +} {4 {} {} 2 {} {} 1 {} {}}
           39  +
           40  +do_execsql_test 1.5 {
           41  +  SELECT rowid, highlight(f1, 0, '[', ']') FROM f1 WHERE f1 MATCH 'o';
           42  +} {4 {} 2 {} 1 {}}
           43  +
           44  +do_execsql_test 1.6 {
           45  +  SELECT rowid, highlight(f1, 0, '[', ']') IS NULL FROM f1 WHERE f1 MATCH 'o';
           46  +} {4 1 2 1 1 1}
           47  +
           48  +do_execsql_test 1.7 {
           49  +  SELECT rowid, snippet(f1, -1, '[', ']', '...', 5) IS NULL 
           50  +  FROM f1 WHERE f1 MATCH 'o';
           51  +} {4 1 2 1 1 1}
           52  +
           53  +do_execsql_test 1.8 {
           54  +  SELECT rowid, snippet(f1, 1, '[', ']', '...', 5) IS NULL 
           55  +  FROM f1 WHERE f1 MATCH 'o';
           56  +} {4 1 2 1 1 1}
           57  +
           58  +do_execsql_test 1.9 {
           59  +  SELECT rowid FROM f1;
           60  +} {4 3 2 1}
           61  +
           62  +do_execsql_test 1.10 {
           63  +  SELECT * FROM f1;
           64  +} {{} {}  {} {}  {} {}  {} {}}
           65  +
           66  +do_execsql_test 1.11 {
           67  +  SELECT rowid, a, b FROM f1 ORDER BY rowid ASC;
           68  +} {1 {} {}  2 {} {}  3 {} {}  4 {} {}}
           69  +
           70  +do_execsql_test 1.12 {
           71  +  SELECT a IS NULL FROM f1;
           72  +} {1 1 1 1}
           73  +
           74  +do_catchsql_test 1.13 {
           75  +  DELETE FROM f1 WHERE rowid = 2;
           76  +} {1 {cannot DELETE from contentless fts5 table: f1}}
           77  +
           78  +do_catchsql_test 1.14 {
           79  +  UPDATE f1 SET a = 'a b c' WHERE rowid = 2;
           80  +} {1 {cannot UPDATE contentless fts5 table: f1}}
           81  +
           82  +do_execsql_test 1.15 {
           83  +  INSERT INTO f1(f1, rowid, a, b) VALUES('delete', 2, 'two', 't w o');
           84  +} {}
           85  +
           86  +db eval { SELECT fts5_decode(id, block) AS d FROM f1_data } { puts $d }
           87  +
           88  +breakpoint
           89  +do_execsql_test 1.16 {
           90  +  SELECT rowid FROM f1 WHERE f1 MATCH 'o';
           91  +} {4 1}
           92  +do_execsql_test 1.17 {
           93  +  SELECT rowid FROM f1;
           94  +} {4 3 1}
           95  +
           96  +
           97  +
           98  +
           99  +finish_test