Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Have zonefile store encryption keys in a hash-table instead of a linked list. Add extra tests for key management. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | zonefile |
Files: | files | file ages | folders |
SHA3-256: |
3a63ea652546a4c63eccd72665becff3 |
User & Date: | dan 2018-02-21 16:36:08.835 |
Context
2018-02-21
| ||
21:15 | Modifications to the zonefile module to make it easier to add a cache of uncompressed frame content. (check-in: d9d5cc62f1 user: dan tags: zonefile) | |
16:36 | Have zonefile store encryption keys in a hash-table instead of a linked list. Add extra tests for key management. (check-in: 3a63ea6525 user: dan tags: zonefile) | |
10:43 | In zonefile, change the "file TEXT" column back to "fileid INTEGER". The fileid can be used as a key with the associated zonefile_files table, which contains more information than just the filename. (check-in: 38d23888cf user: dan tags: zonefile) | |
Changes
Changes to ext/zonefile/zonefile.c.
︙ | ︙ | |||
11 12 13 14 15 16 17 18 19 20 21 22 23 24 | ****************************************************************************** */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 #ifndef SQLITE_OMIT_VIRTUALTABLE #ifndef SQLITE_AMALGAMATION typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; typedef unsigned char u8; typedef unsigned short u16; typedef unsigned long u32; #define MIN(a,b) ((a)<(b) ? (a) : (b)) | > > > > | 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | ****************************************************************************** */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 #ifndef SQLITE_OMIT_VIRTUALTABLE #include <stdio.h> #include <string.h> #include <assert.h> #ifndef SQLITE_AMALGAMATION typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; typedef unsigned char u8; typedef unsigned short u16; typedef unsigned long u32; #define MIN(a,b) ((a)<(b) ? (a) : (b)) |
︙ | ︙ | |||
126 127 128 129 130 131 132 133 134 135 | ** It is harmless to pass in a NULL pointer. */ static void zonefileCodecDestroy(ZonefileCodec *pCodec){ sqlite3_free(pCodec); } #endif /* SQLITE_HAVE_ZONEFILE_CODEC */ typedef struct ZonefileGlobal ZonefileGlobal; typedef struct ZonefileKey ZonefileKey; struct ZonefileGlobal { | > > > > > > > > | | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 | ** It is harmless to pass in a NULL pointer. */ static void zonefileCodecDestroy(ZonefileCodec *pCodec){ sqlite3_free(pCodec); } #endif /* SQLITE_HAVE_ZONEFILE_CODEC */ /* ** All zonefile and zonefile_files virtual table instances that belong ** to the same database handle (sqlite3*) share a single instance of the ** ZonefileGlobal object. This global object contains a table of ** configured encryption keys for the various zonefiles in the system. */ typedef struct ZonefileGlobal ZonefileGlobal; typedef struct ZonefileKey ZonefileKey; struct ZonefileGlobal { int nEntry; /* Number of entries in the hash table */ int nHash; /* Size of aHash[] array */ ZonefileKey **aHash; /* Hash buckets */ }; struct ZonefileKey { const char *zName; /* Zonefile table name */ const char *zDb; /* Database name ("main", "temp" etc.) */ i64 iFileid; /* File id */ const char *zKey; /* Key buffer */ int nKey; /* Size of zKey in bytes */ u32 iHash; /* zonefileKeyHash() value */ ZonefileKey *pHashNext; /* Next colliding key in hash table */ }; static u32 zonefileKeyHash( const char *zDb, const char *zTab, i64 iFileid ){ u32 iHash = 0; int i; for(i=0; zDb[i]; i++) iHash += (iHash<<3) + (u8)zDb[i]; for(i=0; zTab[i]; i++) iHash += (iHash<<3) + (u8)zTab[i]; return (iHash ^ (iFileid & 0xFFFFFFFF)); } /* ** Store encryption key zKey in the key-store passed as the first argument. ** Return SQLITE_OK if successful, or an SQLite error code (SQLITE_NOMEM) ** otherwise. */ static int zonefileKeyStore( ZonefileGlobal *pGlobal, const char *zDb, /* Database containing zonefile table */ const char *zTab, /* Name of zonefile table */ i64 iFileid, /* File-id to configure key for */ const char *zKey /* Key to store */ ){ ZonefileKey **pp; u32 iHash = zonefileKeyHash(zDb, zTab, iFileid); /* Remove any old entry */ if( pGlobal->nHash ){ for(pp=&pGlobal->aHash[iHash%pGlobal->nHash]; *pp; pp=&((*pp)->pHashNext)){ ZonefileKey *pThis = *pp; if( pThis->iFileid==iFileid && 0==sqlite3_stricmp(zTab, pThis->zName) && 0==sqlite3_stricmp(zDb, pThis->zDb) ){ pGlobal->nEntry--; *pp = pThis->pHashNext; sqlite3_free(pThis); break; } } } if( zKey ){ int nKey = strlen(zKey); int nDb = strlen(zDb); int nTab = strlen(zTab); ZonefileKey *pNew; /* Resize the hash-table, if necessary */ if( pGlobal->nEntry>=pGlobal->nHash ){ int i; int n = pGlobal->nHash ? pGlobal->nHash*2 : 16; ZonefileKey **a = (ZonefileKey**)sqlite3_malloc(n*sizeof(ZonefileKey*)); if( a==0 ) return SQLITE_NOMEM; memset(a, 0, n*sizeof(ZonefileKey*)); for(i=0; i<pGlobal->nHash; i++){ ZonefileKey *p; ZonefileKey *pNext; for(p=pGlobal->aHash[i]; p; p=pNext){ pNext = p->pHashNext; p->pHashNext = a[p->iHash % n]; a[p->iHash % n] = p; } } sqlite3_free(pGlobal->aHash); pGlobal->aHash = a; pGlobal->nHash = n; } pNew = (ZonefileKey*)sqlite3_malloc( sizeof(ZonefileKey) + nKey+1 + nDb+1 + nTab+1 ); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(ZonefileKey)); pNew->iFileid = iFileid; pNew->iHash = iHash; pNew->zKey = (const char*)&pNew[1]; pNew->nKey = nKey; pNew->zDb = &pNew->zKey[nKey+1]; pNew->zName = &pNew->zDb[nDb+1]; memcpy((char*)pNew->zKey, zKey, nKey+1); memcpy((char*)pNew->zDb, zDb, nDb+1); memcpy((char*)pNew->zName, zTab, nTab+1); pNew->pHashNext = pGlobal->aHash[iHash % pGlobal->nHash]; pGlobal->aHash[iHash % pGlobal->nHash] = pNew; } return SQLITE_OK; } /* ** Search the key-store passed as the first argument for an encryption ** key to use with the file with file-id iFileid in zonefile table zTab ** in database zDb. If successful, set (*pzKey) to point to the key ** buffer and return the size of the key in bytes. ** ** If no key is found, return 0. The final value of (*pzKey) is undefined ** in this case. */ static int zonefileKeyFind( ZonefileGlobal *pGlobal, const char *zDb, /* Database containing zonefile table */ const char *zTab, /* Name of zonefile table */ i64 iFileid, /* File-id to configure key for */ const char **pzKey /* OUT: Pointer to key buffer */ ){ if( pGlobal->nHash ){ ZonefileKey *pKey; u32 iHash = zonefileKeyHash(zDb, zTab, iFileid); for(pKey=pGlobal->aHash[iHash%pGlobal->nHash]; pKey; pKey=pKey->pHashNext){ if( pKey->iFileid==iFileid && 0==sqlite3_stricmp(zDb, pKey->zDb) && 0==sqlite3_stricmp(zTab, pKey->zName) ){ *pzKey = pKey->zKey; return pKey->nKey; } } } return 0; } /* ** The pointer passed as the only argument must actually point to a ** ZonefileGlobal structure. This function frees the structure and all ** of its components. */ static void zonefileKeyDestroy(void *p){ ZonefileGlobal *pGlobal = (ZonefileGlobal*)p; int i; for(i=0; i<pGlobal->nHash; i++){ ZonefileKey *pKey; ZonefileKey *pNext; for(pKey=pGlobal->aHash[i]; pKey; pKey=pNext){ pNext = pKey->pHashNext; sqlite3_free(pKey); } } sqlite3_free(pGlobal->aHash); sqlite3_free(pGlobal); } #define ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE (64*1024) #define ZONEFILE_DEFAULT_ENCRYPTION 1 #define ZONEFILE_DEFAULT_COMPRESSION 0 #define ZONEFILE_DEFAULT_DICTSIZE (64*1024) |
︙ | ︙ | |||
174 175 176 177 178 179 180 | static u32 zonefileGet32(const u8 *aBuf){ return (((u32)aBuf[0]) << 24) + (((u32)aBuf[1]) << 16) + (((u32)aBuf[2]) << 8) + (((u32)aBuf[3]) << 0); } | < < < < | 332 333 334 335 336 337 338 339 340 341 342 343 344 345 | static u32 zonefileGet32(const u8 *aBuf){ return (((u32)aBuf[0]) << 24) + (((u32)aBuf[1]) << 16) + (((u32)aBuf[2]) << 8) + (((u32)aBuf[3]) << 0); } static int zfGenericOpen(void **pp, u8 *aDict, int nDict){ *pp = 0; return SQLITE_OK; } static void zfGenericClose(void *p){ } static int zfGenericUncompressSize( |
︙ | ︙ | |||
1640 1641 1642 1643 1644 1645 1646 | sqlite3_free(aKey); } zonefileFileClose(pFd); return rc; } | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 | sqlite3_free(aKey); } zonefileFileClose(pFd); return rc; } /* ** zonefile_files virtual table module xUpdate method. ** ** A delete specifies a single argument - the rowid of the row to remove. ** ** Update and insert operations pass: ** |
︙ | ︙ | |||
1708 1709 1710 1711 1712 1713 1714 | int rc = SQLITE_OK; ZonefileFilesTab *pTab = (ZonefileFilesTab*)pVtab; if( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ){ if( nVal>1 && sqlite3_value_nochange(apVal[2]) ){ const char *zKey = (const char*)sqlite3_value_text(apVal[3]); i64 iFileid = sqlite3_value_int64(apVal[0]); | | > > | 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 | int rc = SQLITE_OK; ZonefileFilesTab *pTab = (ZonefileFilesTab*)pVtab; if( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ){ if( nVal>1 && sqlite3_value_nochange(apVal[2]) ){ const char *zKey = (const char*)sqlite3_value_text(apVal[3]); i64 iFileid = sqlite3_value_int64(apVal[0]); return zonefileKeyStore( pTab->pGlobal, pTab->zDb, pTab->zBase, iFileid, zKey ); }else{ if( pTab->pDelete==0 ){ rc = zonefilePrepare(pTab->db, &pTab->pDelete, &pVtab->zErrMsg, "DELETE FROM %Q.'%q_shadow_file' WHERE fileid=?", pTab->zDb, pTab->zBase ); } |
︙ | ︙ | |||
1762 1763 1764 1765 1766 1767 1768 | if( rc==SQLITE_OK ){ iFileid = sqlite3_last_insert_rowid(pTab->db); rc = zonefilePopulateIndex(pTab, zFile, iFileid); } if( rc==SQLITE_OK ){ const char *zKey = (const char*)sqlite3_value_text(apVal[3]); | | | 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 | if( rc==SQLITE_OK ){ iFileid = sqlite3_last_insert_rowid(pTab->db); rc = zonefilePopulateIndex(pTab, zFile, iFileid); } if( rc==SQLITE_OK ){ const char *zKey = (const char*)sqlite3_value_text(apVal[3]); rc = zonefileKeyStore(pTab->pGlobal, pTab->zDb, pTab->zBase,iFileid,zKey); } } return rc; } typedef struct ZonefileTab ZonefileTab; |
︙ | ︙ | |||
2119 2120 2121 2122 2123 2124 2125 | zonefileCtxError(pCtx, "failed to uncompress frame"); } sqlite3_free(aUn); return rc; } | < < < < < < < < < < < < < < < < | 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 | zonefileCtxError(pCtx, "failed to uncompress frame"); } sqlite3_free(aUn); return rc; } static int zonefileGetFile( sqlite3_context *pCtx, /* Leave error message here */ ZonefileCsr *pCsr, /* Cursor object */ const char **pzFile /* OUT: Pointer to current file */ ){ ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab; int rc = SQLITE_OK; |
︙ | ︙ | |||
2224 2225 2226 2227 2228 2229 2230 | rc = pCmpMethod->xOpen(&pCmp, aDict, nDict); } sqlite3_free(aDict); } /* Find the encryption method and key. */ if( hdr.encryptionType ){ | > | | | | | 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 | rc = pCmpMethod->xOpen(&pCmp, aDict, nDict); } sqlite3_free(aDict); } /* Find the encryption method and key. */ if( hdr.encryptionType ){ i64 iFileid = sqlite3_column_int64(pCsr->pSelect, 1); const char *z = 0; int n = zonefileKeyFind(pTab->pGlobal, pTab->zDb, pTab->zName, iFileid, &z); if( n==0 ){ zErr = sqlite3_mprintf("missing encryption key for file \"%s\"", zFile); rc = SQLITE_ERROR; }else{ rc = zonefileCodecCreate(hdr.encryptionType, (u8*)z, n, &pCodec, &zErr); } } /* Read some data into memory. If the data is uncompressed, then just ** the required record is read. Otherwise, the entire frame is read ** into memory. */ if( rc==SQLITE_OK ){ |
︙ | ︙ | |||
2334 2335 2336 2337 2338 2339 2340 | */ static int zonefileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ ZonefileCsr *pCsr = (ZonefileCsr*)cur; *pRowid = sqlite3_column_int64(pCsr->pSelect, 0); return SQLITE_OK; } | < < < < < < < < < < < | 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 | */ static int zonefileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ ZonefileCsr *pCsr = (ZonefileCsr*)cur; *pRowid = sqlite3_column_int64(pCsr->pSelect, 0); return SQLITE_OK; } /* ** Register the "zonefile" extensions. */ static int zonefileRegister(sqlite3 *db){ static sqlite3_module filesModule = { 0, /* iVersion */ zffCreate, /* xCreate - create a table */ |
︙ | ︙ | |||
2434 2435 2436 2437 2438 2439 2440 | if( rc==SQLITE_OK ){ rc = sqlite3_create_module( db, "zonefile_files", &filesModule, (void*)pGlobal ); } if( rc==SQLITE_OK ){ rc = sqlite3_create_module_v2(db, "zonefile", &zonefileModule, | | | 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 | if( rc==SQLITE_OK ){ rc = sqlite3_create_module( db, "zonefile_files", &filesModule, (void*)pGlobal ); } if( rc==SQLITE_OK ){ rc = sqlite3_create_module_v2(db, "zonefile", &zonefileModule, (void*)pGlobal, zonefileKeyDestroy ); pGlobal = 0; } sqlite3_free(pGlobal); return rc; } |
︙ | ︙ |
Added ext/zonefile/zonefileenc.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | # 2018 Feb 11 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # The focus of this file is testing the zonefile extension. # if {![info exists testdir]} { set testdir [file join [file dirname [info script]] .. .. test] } source [file join $testdir tester.tcl] set testprefix zonefileenc load_static_extension db zonefile set K { braking bramble brambles brambly bran branch branched branches branching branchings brand branded } set nFile 100 do_execsql_test 1.0 { CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB); CREATE TABLE rr(k INTEGER PRIMARY KEY, v); } do_test 1.1 { for {set i 0} {$i < $nFile} {incr i} { set k [lindex $K [expr $i % [llength $K]]] execsql { DELETE FROM zz; INSERT INTO zz VALUES($i*10+1, 1, -1, randomblob(100)); INSERT INTO zz VALUES($i*10+2, 2, -1, randomblob(100)); INSERT INTO zz VALUES($i*10+3, 1, -1, randomblob(100)); INSERT INTO rr SELECT k,v FROM zz; WITH p(n,v) AS ( VALUES('encryptionType', 'xor') UNION ALL VALUES('encryptionKey', $k) ) SELECT zonefile_write('test' || $i || '.zonefile', 'zz', json_group_object(n, v) ) FROM p; } } } {} do_test 1.2 { execsql { CREATE VIRTUAL TABLE gg USING zonefile; } for {set i 0} {$i < $nFile} {incr i} { execsql { INSERT INTO gg_files(filename) VALUES('test' || $i || '.zonefile') } } } {} do_catchsql_test 1.3 { SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v; } {1 {missing encryption key for file "test0.zonefile"}} do_execsql_test 1.4 { UPDATE gg_files SET ekey = 'braking' WHERE filename='test0.zonefile'; } do_catchsql_test 1.5 { SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v; } {1 {missing encryption key for file "test1.zonefile"}} proc k {i} { lindex $::K [expr $i % [llength $::K]] } db func k k do_execsql_test 1.6 { UPDATE gg_files SET ekey = k(rowid-1); } do_execsql_test 1.7 { SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v; } {0} do_execsql_test 1.8 { SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v==gg.v; } {300} finish_test |