ADDED ext/misc/shardvtab.c Index: ext/misc/shardvtab.c ================================================================== --- /dev/null +++ ext/misc/shardvtab.c @@ -0,0 +1,375 @@ +/* +** 2019-04-26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a virtual-table that can be used to access a +** sharded table implemented as the UNION ALL of various separate tables. +*/ +#if !defined(SQLITEINT_H) +#include "sqlite3ext.h" +#endif +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +/* shardvtab_vtab is a subclass of sqlite3_vtab which is +** underlying representation of the virtual table +*/ +typedef struct shardvtab_vtab shardvtab_vtab; +struct shardvtab_vtab { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* The database connection */ + char *zView; /* Name of view that implements the shard */ + int nCol; /* Number of columns in the view */ + char **azCol; /* Names of the columns, individually malloced */ +}; + +/* shardvtab_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct shardvtab_cursor shardvtab_cursor; +struct shardvtab_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_stmt *pStmt; /* Prepared statement to access the shard */ + int rcLastStep; /* Last return from sqlite3_step() */ +}; + +/* +** The shardvtabConnect() method is invoked to create a new +** shard virtual table. +** +** Think of this routine as the constructor for shardvtab_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the shardvtab_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against the virtual table will look like. +*/ +static int shardvtabConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + shardvtab_vtab *pNew; + int rc; + char *zSql; + sqlite3_str *pSchema; + sqlite3_stmt *pStmt = 0; + const char *zView = 0; + char **azCol = 0; + int nCol = 0; + char cSep; + int i; + + if( argc!=4 || argv[0]==0 ){ + *pzErr = sqlite3_mprintf("one argument requires: the name of a view"); + return SQLITE_ERROR; + } + zView = argv[3]; + zSql = sqlite3_mprintf("SELECT * FROM \"%w\"", zView); + if( zSql==0 ){ + return SQLITE_NOMEM; + } + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + *pzErr = sqlite3_mprintf("not a valid view: \"%w\"", zView); + return SQLITE_NOMEM; + } + pSchema = sqlite3_str_new(db); + if( pSchema==0 ){ + sqlite3_finalize(pStmt); + return SQLITE_NOMEM; + } + sqlite3_str_appendall(pSchema, "CREATE TABLE x"); + cSep = '('; + for(i=0; idb = db; + pNew->zView = (char*)&pNew[1]; + memcpy(pNew->zView, zView, n); + pNew->nCol = nCol; + pNew->azCol = azCol; + } + return SQLITE_OK; + +shardvtab_connect_error: + sqlite3_finalize(pStmt); + for(i=0; inCol; i++) sqlite3_free(p->azCol[i]); + sqlite3_free(p->azCol); + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Constructor for a new shardvtab_cursor object. +*/ +static int shardvtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + shardvtab_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a shardvtab_cursor. +*/ +static int shardvtabClose(sqlite3_vtab_cursor *cur){ + shardvtab_cursor *pCur = (shardvtab_cursor*)cur; + sqlite3_finalize(pCur->pStmt); + sqlite3_free(pCur); + return SQLITE_OK; +} + + +/* +** Advance a shardvtab_cursor to its next row of output. +*/ +static int shardvtabNext(sqlite3_vtab_cursor *cur){ + shardvtab_cursor *pCur = (shardvtab_cursor*)cur; + int rc; + rc = pCur->rcLastStep = sqlite3_step(pCur->pStmt); + if( rc==SQLITE_ROW || rc==SQLITE_DONE ) return SQLITE_OK; + return rc; +} + +/* +** Return values of columns for the row at which the shardvtab_cursor +** is currently pointing. +*/ +static int shardvtabColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + shardvtab_cursor *pCur = (shardvtab_cursor*)cur; + sqlite3_result_value(ctx, sqlite3_column_value(pCur->pStmt, i)); + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int shardvtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + *pRowid = 0; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int shardvtabEof(sqlite3_vtab_cursor *cur){ + shardvtab_cursor *pCur = (shardvtab_cursor*)cur; + return pCur->rcLastStep!=SQLITE_ROW; +} + +/* +** This method is called to "rewind" the shardvtab_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to shardvtabColumn() or shardvtabRowid() or +** shardvtabEof(). +*/ +static int shardvtabFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + shardvtab_cursor *pCur = (shardvtab_cursor *)pVtabCursor; + shardvtab_vtab *pTab = (shardvtab_vtab *)pVtabCursor->pVtab; + int rc; + sqlite3_finalize(pCur->pStmt); + pCur->pStmt = 0; + rc = sqlite3_prepare_v2(pTab->db, idxStr, -1, &pCur->pStmt, 0); + if( rc==SQLITE_OK ){ + int i; + for(i=0; ipStmt, i+1, argv[i]); + } + }else{ + sqlite3_finalize(pCur->pStmt); + pCur->pStmt = 0; + } + pCur->rcLastStep = rc; + return rc; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +*/ +static int shardvtabBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *p +){ + shardvtab_vtab *pTab = (shardvtab_vtab*)tab; + int i; + int n; + sqlite3_stmt *pStmt; + int rc; + sqlite3_str *pSql; + char *zSep = "WHERE"; + char *zSql; + pSql = sqlite3_str_new(pTab->db); + if( pSql==0 ) return SQLITE_NOMEM; + sqlite3_str_appendf(pSql, "SELECT * FROM \"%w\"", pTab->zView); + for(i=n=0; inConstraint; i++){ + const char *zOp; + int iCol; + if( p->aConstraint[i].usable==0 ) continue; + iCol = p->aConstraint[i].iColumn; + if( iCol<0 ) continue; + zOp = 0; + switch( p->aConstraint[i].op ){ + case SQLITE_INDEX_CONSTRAINT_EQ: zOp = "=="; break; + case SQLITE_INDEX_CONSTRAINT_GT: zOp = ">"; break; + case SQLITE_INDEX_CONSTRAINT_LE: zOp = "<="; break; + case SQLITE_INDEX_CONSTRAINT_LT: zOp = "<"; break; + case SQLITE_INDEX_CONSTRAINT_GE: zOp = ">="; break; + case SQLITE_INDEX_CONSTRAINT_MATCH: zOp = "MATCH"; break; + case SQLITE_INDEX_CONSTRAINT_LIKE: zOp = "LIKE"; break; + case SQLITE_INDEX_CONSTRAINT_GLOB: zOp = "GLOB"; break; + case SQLITE_INDEX_CONSTRAINT_REGEXP: zOp = "REGEXP"; break; + case SQLITE_INDEX_CONSTRAINT_NE: zOp = "<>"; break; + case SQLITE_INDEX_CONSTRAINT_IS: zOp = "IS"; break; + } + if( zOp ){ + n++; + p->aConstraintUsage[i].argvIndex = n; + sqlite3_str_appendf(pSql, " %s (\"%w\" %s ?%d)", + zSep, pTab->azCol[iCol], zOp, n); + zSep = "AND"; + } + } + zSql = sqlite3_str_finish(pSql); + if( zSql==0 ){ + return SQLITE_NOMEM; + } + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + int x = sqlite3_stmt_status(pStmt, SQLITE_STMTSTATUS_EST_COST, 0); + p->estimatedCost = pow(2.0, 0.1*x); + p->estimatedRows = + sqlite3_stmt_status(pStmt, SQLITE_STMTSTATUS_EST_ROWS, 0); + p->idxStr = zSql; + p->needToFreeIdxStr = 1; + }else{ + sqlite3_free(zSql); + } + sqlite3_finalize(pStmt); + return rc; +} + +/* +** This following structure defines all the methods for the +** virtual table. +*/ +static sqlite3_module shardvtabModule = { + /* iVersion */ 0, + /* xCreate */ shardvtabConnect, + /* xConnect */ shardvtabConnect, + /* xBestIndex */ shardvtabBestIndex, + /* xDisconnect */ shardvtabDisconnect, + /* xDestroy */ shardvtabDisconnect, + /* xOpen */ shardvtabOpen, + /* xClose */ shardvtabClose, + /* xFilter */ shardvtabFilter, + /* xNext */ shardvtabNext, + /* xEof */ shardvtabEof, + /* xColumn */ shardvtabColumn, + /* xRowid */ shardvtabRowid, + /* xUpdate */ 0, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindMethod */ 0, + /* xRename */ 0, + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0, + /* xShadowName */ 0 +}; + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_shardvtab_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + rc = sqlite3_create_module(db, "shardvtab", &shardvtabModule, 0); + return rc; +} Index: src/shell.c.in ================================================================== --- src/shell.c.in +++ src/shell.c.in @@ -1770,11 +1770,11 @@ } /* ** Display and reset the EXPLAIN QUERY PLAN data */ -static void eqp_render(ShellState *p){ +static void eqp_render(ShellState *p, sqlite3_stmt *pStmt){ EQPGraphRow *pRow = p->sGraph.pRow; if( pRow ){ if( pRow->zText[0]=='-' ){ if( pRow->pNext==0 ){ eqp_reset(p); @@ -1782,11 +1782,14 @@ } utf8_printf(p->out, "%s\n", pRow->zText+3); p->sGraph.pRow = pRow->pNext; sqlite3_free(pRow); }else{ - utf8_printf(p->out, "QUERY PLAN\n"); + int iCost, nRow; + iCost = sqlite3_stmt_status(pStmt, SQLITE_STMTSTATUS_EST_COST, 0); + nRow = sqlite3_stmt_status(pStmt, SQLITE_STMTSTATUS_EST_ROWS, 0); + utf8_printf(p->out, "QUERY PLAN (log est cost=%d rows=%d)\n", iCost, nRow); } p->sGraph.zPrefix[0] = 0; eqp_render_level(p, 0); eqp_reset(p); } @@ -3073,14 +3076,14 @@ if( rc==SQLITE_OK ){ while( sqlite3_step(pExplain)==SQLITE_ROW ){ const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3); int iEqpId = sqlite3_column_int(pExplain, 0); int iParentId = sqlite3_column_int(pExplain, 1); - if( zEQPLine[0]=='-' ) eqp_render(pArg); + if( zEQPLine[0]=='-' ) eqp_render(pArg, pExplain); eqp_append(pArg, iEqpId, iParentId, zEQPLine); } - eqp_render(pArg); + eqp_render(pArg, pExplain); } sqlite3_finalize(pExplain); sqlite3_free(zEQP); if( pArg->autoEQP>=AUTOEQP_full ){ /* Also do an EXPLAIN for ".eqp full" mode */ @@ -3124,11 +3127,11 @@ } bind_prepared_stmt(pArg, pStmt); exec_prepared_stmt(pArg, pStmt); explain_data_delete(pArg); - eqp_render(pArg); + eqp_render(pArg, pStmt); /* print usage stats if stats on */ if( pArg && pArg->statsOn ){ display_stats(db, pArg, 0); } Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -7845,10 +7845,19 @@ ** [[SQLITE_STMTSTATUS_MEMUSED]]
SQLITE_STMTSTATUS_MEMUSED
**
^This is the approximate number of bytes of heap memory ** used to store the prepared statement. ^This value is not actually ** a counter, and so the resetFlg parameter to sqlite3_stmt_status() ** is ignored when the opcode is SQLITE_STMTSTATUS_MEMUSED. +** +** [[SQLITE_STMTSTATUS_EST_ROWS]]
SQLITE_STMTSTATUS_EST_ROWS
+**
^A return value of X indicates that the query planner estimated +** that the query will return pow(2,X/10.0) rows. +** +** [[SQLITE_STMTSTATUS_EST_COST]]
SQLITE_STMTSTATUS_EST_COST
+**
^A return value of X indicates that the query planner estimated +** the relative cost of running this statement to completion is +** pow(2,X/10.0). **
** */ #define SQLITE_STMTSTATUS_FULLSCAN_STEP 1 #define SQLITE_STMTSTATUS_SORT 2 @@ -7855,10 +7864,12 @@ #define SQLITE_STMTSTATUS_AUTOINDEX 3 #define SQLITE_STMTSTATUS_VM_STEP 4 #define SQLITE_STMTSTATUS_REPREPARE 5 #define SQLITE_STMTSTATUS_RUN 6 #define SQLITE_STMTSTATUS_MEMUSED 99 +#define SQLITE_STMTSTATUS_EST_ROWS 100 +#define SQLITE_STMTSTATUS_EST_COST 101 /* ** CAPI3REF: Custom Page Cache Object ** ** The sqlite3_pcache type is opaque. It is implemented by Index: src/vdbe.h ================================================================== --- src/vdbe.h +++ src/vdbe.h @@ -262,10 +262,11 @@ #endif void sqlite3VdbeSwap(Vdbe*,Vdbe*); VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*); sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe*, int, u8); void sqlite3VdbeSetVarmask(Vdbe*, int); +void sqlite3VdbeUpdateCostEstimates(Parse*, LogEst, LogEst); #ifndef SQLITE_OMIT_TRACE char *sqlite3VdbeExpandSql(Vdbe*, const char*); #endif int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); int sqlite3BlobCompare(const Mem*, const Mem*); Index: src/vdbeInt.h ================================================================== --- src/vdbeInt.h +++ src/vdbeInt.h @@ -423,10 +423,12 @@ bft changeCntOn:1; /* True to update the change-counter */ bft runOnlyOnce:1; /* Automatically expire on reset */ bft usesStmtJournal:1; /* True if uses a statement journal */ bft readOnly:1; /* True for statements that do not write */ bft bIsReader:1; /* True for statements that read */ + LogEst nRowEst; /* Query planner of estimated number of output rows */ + LogEst iCostEst; /* Query planner cost estimate */ yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */ yDbMask lockMask; /* Subset of btreeMask that requires a lock */ u32 aCounter[7]; /* Counters used by sqlite3_stmt_status() */ char *zSql; /* Text of the SQL statement that generated this */ #ifdef SQLITE_ENABLE_NORMALIZE Index: src/vdbeapi.c ================================================================== --- src/vdbeapi.c +++ src/vdbeapi.c @@ -1656,31 +1656,43 @@ /* ** Return the value of a status counter for a prepared statement */ int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ Vdbe *pVdbe = (Vdbe*)pStmt; - u32 v; -#ifdef SQLITE_ENABLE_API_ARMOR - if( !pStmt - || (op!=SQLITE_STMTSTATUS_MEMUSED && (op<0||op>=ArraySize(pVdbe->aCounter))) - ){ + u32 v = 0; + if( !pStmt ){ (void)SQLITE_MISUSE_BKPT; return 0; } -#endif - if( op==SQLITE_STMTSTATUS_MEMUSED ){ - sqlite3 *db = pVdbe->db; - sqlite3_mutex_enter(db->mutex); - v = 0; - db->pnBytesFreed = (int*)&v; - sqlite3VdbeClearObject(db, pVdbe); - sqlite3DbFree(db, pVdbe); - db->pnBytesFreed = 0; - sqlite3_mutex_leave(db->mutex); - }else{ - v = pVdbe->aCounter[op]; - if( resetFlag ) pVdbe->aCounter[op] = 0; + switch( op ){ + case SQLITE_STMTSTATUS_MEMUSED: { + sqlite3 *db = pVdbe->db; + sqlite3_mutex_enter(db->mutex); + v = 0; + db->pnBytesFreed = (int*)&v; + sqlite3VdbeClearObject(db, pVdbe); + sqlite3DbFree(db, pVdbe); + db->pnBytesFreed = 0; + sqlite3_mutex_leave(db->mutex); + break; + } + case SQLITE_STMTSTATUS_EST_ROWS: { + v = pVdbe->nRowEst; + break; + } + case SQLITE_STMTSTATUS_EST_COST: { + v = pVdbe->iCostEst; + break; + } + default: { + if( op>=0 && opaCounter) ){ + v = pVdbe->aCounter[op]; + if( resetFlag ) pVdbe->aCounter[op] = 0; + }else{ + (void)SQLITE_MISUSE_BKPT; + } + } } return (int)v; } /* Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -4845,10 +4845,24 @@ v->expmask |= 0x80000000; }else{ v->expmask |= ((u32)1 << (iVar-1)); } } + +/* +** Update the estimated cost fields +*/ +void sqlite3VdbeUpdateCostEstimates(Parse *pParse, LogEst iCost, LogEst nRow){ + Vdbe *v = pParse->pVdbe; + if( v->iCostEst ){ + v->iCostEst = sqlite3LogEstAdd(v->iCostEst, iCost+pParse->nQueryLoop) + 1; + if( nRow > v->nRowEst ) v->nRowEst = nRow; + }else{ + v->nRowEst = nRow; + v->iCostEst = iCost + 1; + } +} /* ** Cause a function to throw an error if it was call from OP_PureFunc ** rather than OP_Function. ** Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -4361,10 +4361,11 @@ } } pWInfo->nRowOut = pFrom->nRow; + pWInfo->iTotalCost = pFrom->rCost; /* Free temporary memory and return success */ sqlite3DbFreeNN(db, pSpace); return SQLITE_OK; } @@ -5143,10 +5144,11 @@ SrcList *pTabList = pWInfo->pTabList; sqlite3 *db = pParse->db; /* Generate loop termination code. */ + sqlite3VdbeUpdateCostEstimates(pParse, pWInfo->iTotalCost, pWInfo->nRowOut); VdbeModuleComment((v, "End WHERE-core")); for(i=pWInfo->nLevel-1; i>=0; i--){ int addr; pLevel = &pWInfo->a[i]; pLoop = pLevel->pWLoop; Index: src/whereInt.h ================================================================== --- src/whereInt.h +++ src/whereInt.h @@ -460,10 +460,11 @@ u8 bOrderedInnerLoop; /* True if only the inner-most loop is ordered */ int iTop; /* The very beginning of the WHERE loop */ WhereLoop *pLoops; /* List of all WhereLoop objects */ Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ LogEst nRowOut; /* Estimated number of output rows */ + LogEst iTotalCost; /* Cost estimate for the whole plan */ WhereClause sWC; /* Decomposition of the WHERE clause */ WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ WhereLevel a[1]; /* Information about each nest loop in WHERE */ };