Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Have the ota extension perform an incremental checkpoint after generating the wal file. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | ota-update |
| Files: | files | file ages | folders |
| SHA1: |
0bf1301aacb3b717b4cc020fbda9fab0 |
| User & Date: | dan 2014-10-20 16:24:23 |
Context
|
2014-10-20
| ||
| 16:34 | Merge version-3.8.7 changes with this branch. (check-in: d380a648 user: dan tags: ota-update) | |
| 16:24 | Have the ota extension perform an incremental checkpoint after generating the wal file. (check-in: 0bf1301a user: dan tags: ota-update) | |
|
2014-09-19
| ||
| 18:08 | Add further tests to ota5.test. Add "ota.test", for running all ota tests. (check-in: 95ffdaa5 user: dan tags: ota-update) | |
Changes
Changes to ext/ota/ota1.test.
120 120 CREATE INDEX i1 ON t1(b, c); 121 121 CREATE INDEX i2 ON t1(c, b); 122 122 CREATE INDEX i3 ON t1(a, b, c, a, b, c); 123 123 } 124 124 } { 125 125 reset_db 126 126 execsql $schema 127 - 127 + 128 128 do_test 1.$tn2.$tn.1 { 129 129 create_ota1 ota.db 130 130 $cmd test.db ota.db 131 131 } {SQLITE_DONE} 132 - 132 + 133 133 do_execsql_test 1.$tn2.$tn.2 { 134 134 SELECT * FROM t1 ORDER BY a ASC; 135 135 } { 136 136 1 2 3 137 137 2 two three 138 138 3 {} 8.2 139 139 } 140 - 140 + 141 141 do_execsql_test 1.$tn2.$tn.3 { PRAGMA integrity_check } ok 142 142 } 143 143 } 144 144 145 145 #------------------------------------------------------------------------- 146 146 # Check that an OTA cannot be applied to a table that has no PK. 147 147 #
Changes to ext/ota/sqlite3ota.c.
18 18 19 19 #include "sqlite3.h" 20 20 #include "sqlite3ota.h" 21 21 22 22 23 23 /* 24 24 ** The ota_state table is used to save the state of a partially applied 25 -** update so that it can be resumed later. The table contains at most a 26 -** single row: 27 -** 28 -** "tbl" -> Table currently being written (target database names). 29 -** 30 -** "idx" -> Index currently being written (target database names). 31 -** Or, if the main table is being written, a NULL value. 32 -** 33 -** "row" -> Number of rows for this object already processed 34 -** 35 -** "progress" -> total number of key/value b-tree operations performed 36 -** so far as part of this ota update. 37 -*/ 25 +** update so that it can be resumed later. The table consists of integer 26 +** keys mapped to values as follows: 27 +** 28 +** OTA_STATE_STAGE: 29 +** May be set to integer values 1, 2 or 3. As follows: 30 +** 0: Nothing at all has been done. 31 +** 1: the *-ota file is currently under construction. 32 +** 2: the *-ota file has been constructed, but not yet moved 33 +** to the *-wal path. 34 +** 3: the checkpoint is underway. 35 +** 36 +** OTA_STATE_TBL: 37 +** Only valid if STAGE==1. The target database name of the table 38 +** currently being written. 39 +** 40 +** OTA_STATE_IDX: 41 +** Only valid if STAGE==1. The target database name of the index 42 +** currently being written, or NULL if the main table is currently being 43 +** updated. 44 +** 45 +** OTA_STATE_ROW: 46 +** Only valid if STAGE==1. Number of rows already processed for the current 47 +** table/index. 48 +** 49 +** OTA_STATE_PROGRESS: 50 +** Total number of sqlite3ota_step() calls made so far as part of this 51 +** ota update. 52 +** 53 +** OTA_STATE_CKPT: 54 +** Valid if STAGE==3. The blob to pass to sqlite3ckpt_start() to resume 55 +** the incremental checkpoint. 56 +** 57 +*/ 58 +#define OTA_STATE_STAGE 1 59 +#define OTA_STATE_TBL 2 60 +#define OTA_STATE_IDX 3 61 +#define OTA_STATE_ROW 4 62 +#define OTA_STATE_PROGRESS 5 63 +#define OTA_STATE_CKPT 6 64 + 65 +#define OTA_STAGE_OAL 1 66 +#define OTA_STAGE_COPY 2 67 +#define OTA_STAGE_CKPT 3 68 +#define OTA_STAGE_DONE 4 69 + 70 + 38 71 #define OTA_CREATE_STATE "CREATE TABLE IF NOT EXISTS ota.ota_state" \ 39 - "(tbl, idx, row, progress)" 72 + "(k INTEGER PRIMARY KEY, v)" 40 73 41 74 typedef struct OtaState OtaState; 42 75 typedef struct OtaObjIter OtaObjIter; 43 76 44 77 /* 45 78 ** A structure to store values read from the ota_state table in memory. 46 79 */ 47 80 struct OtaState { 81 + int eStage; 48 82 char *zTbl; 49 83 char *zIdx; 84 + unsigned char *pCkptState; 85 + int nCkptState; 50 86 int nRow; 51 87 sqlite3_int64 nProgress; 52 88 }; 53 89 54 90 /* 55 91 ** An iterator of this type is used to iterate through all objects in 56 92 ** the target database that require updating. For each such table, the ................................................................................ 84 120 sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */ 85 121 }; 86 122 87 123 /* 88 124 ** OTA handle. 89 125 */ 90 126 struct sqlite3ota { 127 + int eStage; /* Value of OTA_STATE_STAGE field */ 91 128 sqlite3 *db; /* "main" -> target db, "ota" -> ota db */ 92 129 char *zTarget; /* Path to target db */ 130 + char *zOta; /* Path to ota db */ 93 131 int rc; /* Value returned by last ota_step() call */ 94 132 char *zErrmsg; /* Error message if rc!=SQLITE_OK */ 95 133 int nStep; /* Rows processed for current object */ 96 134 int nProgress; /* Rows processed for all objects */ 97 - OtaObjIter objiter; 135 + OtaObjIter objiter; /* Iterator for skipping through tbl/idx */ 136 + sqlite3_ckpt *pCkpt; /* Incr-checkpoint handle */ 98 137 }; 99 138 100 139 /* 101 140 ** Prepare the SQL statement in buffer zSql against database handle db. 102 141 ** If successful, set *ppStmt to point to the new statement and return 103 142 ** SQLITE_OK. 104 143 ** ................................................................................ 737 776 memcpy(pIter->zMask, zMask, pIter->nTblCol); 738 777 } 739 778 sqlite3_free(zWhere); 740 779 sqlite3_free(zSet); 741 780 } 742 781 return p->rc; 743 782 } 783 + 784 +static void otaOpenDatabase(sqlite3ota *p){ 785 + assert( p->rc==SQLITE_OK ); 786 + sqlite3_close(p->db); 787 + p->db = 0; 788 + 789 + p->rc = sqlite3_open(p->zTarget, &p->db); 790 + if( p->rc ){ 791 + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); 792 + } 793 + otaMPrintfExec(p, "ATTACH %Q AS ota", p->zOta); 794 +} 795 + 796 +/* 797 +** This routine is a copy of the sqlite3FileSuffix3() routine from the core. 798 +** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined. 799 +** 800 +** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database 801 +** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and 802 +** if filename in z[] has a suffix (a.k.a. "extension") that is longer than 803 +** three characters, then shorten the suffix on z[] to be the last three 804 +** characters of the original suffix. 805 +** 806 +** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always 807 +** do the suffix shortening regardless of URI parameter. 808 +** 809 +** Examples: 810 +** 811 +** test.db-journal => test.nal 812 +** test.db-wal => test.wal 813 +** test.db-shm => test.shm 814 +** test.db-mj7f3319fa => test.9fa 815 +*/ 816 +static void otaFileSuffix3(const char *zBase, char *z){ 817 +#ifdef SQLITE_ENABLE_8_3_NAMES 818 +#if SQLITE_ENABLE_8_3_NAMES<2 819 + if( sqlite3_uri_boolean(zBase, "8_3_names", 0) ) 820 +#endif 821 + { 822 + int i, sz; 823 + sz = sqlite3Strlen30(z); 824 + for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} 825 + if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4); 826 + } 827 +#endif 828 +} 829 + 830 +/* 831 +** Move the "*-oal" file corresponding to the target database to the 832 +** "*-wal" location. If an error occurs, leave an error code and error 833 +** message in the ota handle. 834 +*/ 835 +static void otaMoveOalFile(sqlite3ota *p){ 836 + const char *zBase = sqlite3_db_filename(p->db, "main"); 837 + 838 + char *zWal = sqlite3_mprintf("%s-wal", zBase); 839 + char *zOal = sqlite3_mprintf("%s-oal", zBase); 840 + 841 + assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); 842 + if( zWal==0 || zOal==0 ){ 843 + p->rc = SQLITE_NOMEM; 844 + }else{ 845 + /* Move the *-oal file to *-wal. At this point connection p->db is 846 + ** holding a SHARED lock on the target database file (because it is 847 + ** in WAL mode). So no other connection may be writing the db. */ 848 + otaFileSuffix3(zBase, zWal); 849 + otaFileSuffix3(zBase, zOal); 850 + rename(zOal, zWal); 851 + 852 + /* Re-open the databases. */ 853 + otaObjIterFinalize(&p->objiter); 854 + otaOpenDatabase(p); 855 + p->eStage = OTA_STAGE_CKPT; 856 + } 857 + 858 + sqlite3_free(zWal); 859 + sqlite3_free(zOal); 860 +} 744 861 745 862 /* 746 863 ** The SELECT statement iterating through the keys for the current object 747 864 ** (p->objiter.pSelect) currently points to a valid row. This function 748 865 ** determines the type of operation requested by this row and returns 749 866 ** one of the following values to indicate the result: 750 867 ** ................................................................................ 858 975 /* no-op */ 859 976 assert( eType==OTA_DELETE && pIter->zIdx ); 860 977 } 861 978 } 862 979 863 980 return p->rc; 864 981 } 982 + 983 +/* 984 +** Increment the schema cookie of the main database opened by p->db. 985 +*/ 986 +static void otaIncrSchemaCookie(sqlite3ota *p){ 987 + int iCookie = 1000000; 988 + sqlite3_stmt *pStmt; 989 + 990 + assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); 991 + p->rc = prepareAndCollectError(p->db, &pStmt, &p->zErrmsg, 992 + "PRAGMA schema_version" 993 + ); 994 + if( p->rc==SQLITE_OK ){ 995 + if( SQLITE_ROW==sqlite3_step(pStmt) ){ 996 + iCookie = sqlite3_column_int(pStmt, 0); 997 + } 998 + p->rc = sqlite3_finalize(pStmt); 999 + } 1000 + if( p->rc==SQLITE_OK ){ 1001 + otaMPrintfExec(p, "PRAGMA schema_version = %d", iCookie+1); 1002 + } 1003 +} 865 1004 866 1005 /* 867 1006 ** Step the OTA object. 868 1007 */ 869 1008 int sqlite3ota_step(sqlite3ota *p){ 870 1009 if( p ){ 871 - OtaObjIter *pIter = &p->objiter; 872 - while( p && p->rc==SQLITE_OK && pIter->zTbl ){ 873 - 874 - if( pIter->bCleanup ){ 875 - /* Clean up the ota_tmp_xxx table for the previous table. It 876 - ** cannot be dropped as there are currently active SQL statements. 877 - ** But the contents can be deleted. */ 878 - otaMPrintfExec(p, "DELETE FROM ota.'ota_tmp_%q'", pIter->zTbl); 879 - }else{ 880 - otaObjIterPrepareAll(p, pIter, 0); 881 - 882 - /* Advance to the next row to process. */ 1010 + switch( p->eStage ){ 1011 + case OTA_STAGE_OAL: { 1012 + OtaObjIter *pIter = &p->objiter; 1013 + while( p && p->rc==SQLITE_OK && pIter->zTbl ){ 1014 + 1015 + if( pIter->bCleanup ){ 1016 + /* Clean up the ota_tmp_xxx table for the previous table. It 1017 + ** cannot be dropped as there are currently active SQL statements. 1018 + ** But the contents can be deleted. */ 1019 + otaMPrintfExec(p, "DELETE FROM ota.'ota_tmp_%q'", pIter->zTbl); 1020 + }else{ 1021 + otaObjIterPrepareAll(p, pIter, 0); 1022 + 1023 + /* Advance to the next row to process. */ 1024 + if( p->rc==SQLITE_OK ){ 1025 + int rc = sqlite3_step(pIter->pSelect); 1026 + if( rc==SQLITE_ROW ){ 1027 + p->nProgress++; 1028 + p->nStep++; 1029 + return otaStep(p); 1030 + } 1031 + p->rc = sqlite3_reset(pIter->pSelect); 1032 + p->nStep = 0; 1033 + } 1034 + } 1035 + 1036 + otaObjIterNext(p, pIter); 1037 + } 1038 + 1039 + if( p->rc==SQLITE_OK && pIter->zTbl==0 ){ 1040 + p->nProgress++; 1041 + otaIncrSchemaCookie(p); 1042 + if( p->rc==SQLITE_OK ){ 1043 + p->rc = sqlite3_exec(p->db, "COMMIT", 0, 0, &p->zErrmsg); 1044 + } 1045 + if( p->rc==SQLITE_OK ){ 1046 + otaMoveOalFile(p); 1047 + } 1048 + } 1049 + break; 1050 + } 1051 + 1052 + case OTA_STAGE_CKPT: { 1053 + 1054 + if( p->rc==SQLITE_OK && p->pCkpt==0 ){ 1055 + p->rc = sqlite3_ckpt_open(p->db, 0, 0, &p->pCkpt); 1056 + } 883 1057 if( p->rc==SQLITE_OK ){ 884 - int rc = sqlite3_step(pIter->pSelect); 885 - if( rc==SQLITE_ROW ){ 886 - p->nStep++; 887 - p->nProgress++; 888 - return otaStep(p); 1058 + if( SQLITE_OK!=sqlite3_ckpt_step(p->pCkpt) ){ 1059 + p->rc = sqlite3_ckpt_close(p->pCkpt, 0, 0); 1060 + p->pCkpt = 0; 1061 + if( p->rc==SQLITE_OK ){ 1062 + p->eStage = OTA_STAGE_DONE; 1063 + p->rc = SQLITE_DONE; 1064 + } 889 1065 } 890 - p->rc = sqlite3_reset(pIter->pSelect); 891 - p->nStep = 0; 1066 + p->nProgress++; 892 1067 } 1068 + 1069 + break; 893 1070 } 894 1071 895 - otaObjIterNext(p, pIter); 896 - } 897 - 898 - if( p->rc==SQLITE_OK && pIter->zTbl==0 ){ 899 - p->rc = SQLITE_DONE; 1072 + default: 1073 + break; 900 1074 } 901 1075 } 902 1076 return p->rc; 903 1077 } 904 1078 905 1079 static void otaSaveTransactionState(sqlite3ota *p){ 906 - otaMPrintfExec(p, 907 - "INSERT OR REPLACE INTO ota.ota_state(rowid, tbl, idx, row, progress)" 908 - "VALUES(1, %Q, %Q, %d, %lld)", 909 - p->objiter.zTbl, p->objiter.zIdx, p->nStep, p->nProgress 1080 + sqlite3_stmt *pInsert; 1081 + int rc; 1082 + 1083 + assert( (p->rc==SQLITE_OK || p->rc==SQLITE_DONE) && p->zErrmsg==0 ); 1084 + rc = prepareFreeAndCollectError(p->db, &pInsert, &p->zErrmsg, 1085 + sqlite3_mprintf( 1086 + "INSERT OR REPLACE INTO ota.ota_state(k, v) VALUES " 1087 + "(%d, %d), " 1088 + "(%d, %Q), " 1089 + "(%d, %Q), " 1090 + "(%d, %d), " 1091 + "(%d, %lld), " 1092 + "(%d, ?) ", 1093 + OTA_STATE_STAGE, p->eStage, 1094 + OTA_STATE_TBL, p->objiter.zTbl, 1095 + OTA_STATE_IDX, p->objiter.zIdx, 1096 + OTA_STATE_ROW, p->nStep, 1097 + OTA_STATE_PROGRESS, p->nProgress, 1098 + OTA_STATE_CKPT 1099 + ) 910 1100 ); 1101 + assert( pInsert==0 || rc==SQLITE_OK ); 1102 + if( rc==SQLITE_OK ){ 1103 + if( p->pCkpt ){ 1104 + unsigned char *pCkptState = 0; 1105 + int nCkptState = 0; 1106 + rc = sqlite3_ckpt_close(p->pCkpt, &pCkptState, &nCkptState); 1107 + p->pCkpt = 0; 1108 + sqlite3_bind_blob(pInsert, 1, pCkptState, nCkptState, SQLITE_TRANSIENT); 1109 + sqlite3_free(pCkptState); 1110 + } 1111 + } 1112 + if( rc==SQLITE_OK ){ 1113 + sqlite3_step(pInsert); 1114 + rc = sqlite3_finalize(pInsert); 1115 + }else{ 1116 + sqlite3_finalize(pInsert); 1117 + } 1118 + 1119 + if( rc!=SQLITE_OK ){ 1120 + p->rc = rc; 1121 + } 1122 +} 1123 + 1124 +static char *otaStrndup(char *zStr, int nStr, int *pRc){ 1125 + char *zRet = 0; 1126 + assert( *pRc==SQLITE_OK ); 1127 + 1128 + if( zStr ){ 1129 + int nCopy = nStr; 1130 + if( nCopy<0 ) nCopy = strlen(zStr) + 1; 1131 + zRet = (char*)sqlite3_malloc(nCopy); 1132 + if( zRet ){ 1133 + memcpy(zRet, zStr, nCopy); 1134 + }else{ 1135 + *pRc = SQLITE_NOMEM; 1136 + } 1137 + } 1138 + 1139 + return zRet; 1140 +} 1141 + 1142 +static void otaFreeState(OtaState *p){ 1143 + sqlite3_free(p->zTbl); 1144 + sqlite3_free(p->zIdx); 1145 + sqlite3_free(p->pCkptState); 1146 + sqlite3_free(p); 911 1147 } 912 1148 913 1149 /* 914 1150 ** Allocate an OtaState object and load the contents of the ota_state 915 1151 ** table into it. Return a pointer to the new object. It is the 916 1152 ** responsibility of the caller to eventually free the object using 917 1153 ** sqlite3_free(). 918 1154 ** 919 1155 ** If an error occurs, leave an error code and message in the ota handle 920 1156 ** and return NULL. 921 1157 */ 922 1158 static OtaState *otaLoadState(sqlite3ota *p){ 923 - const char *zSelect = "SELECT tbl, idx, row, progress FROM ota.ota_state"; 1159 + const char *zSelect = "SELECT k, v FROM ota.ota_state"; 924 1160 OtaState *pRet = 0; 925 1161 sqlite3_stmt *pStmt; 926 1162 int rc; 1163 + int rc2; 927 1164 928 1165 assert( p->rc==SQLITE_OK ); 929 - rc = prepareAndCollectError(p->db, &pStmt, &p->zErrmsg, zSelect); 930 - if( rc==SQLITE_OK ){ 931 - if( sqlite3_step(pStmt)==SQLITE_ROW ){ 932 - const char *zIdx = (const char*)sqlite3_column_text(pStmt, 1); 933 - const char *zTbl = (const char*)sqlite3_column_text(pStmt, 0); 934 - int nIdx = zIdx ? (strlen(zIdx) + 1) : 0; 935 - int nTbl = strlen(zTbl) + 1; 936 - int nByte = sizeof(OtaState) + nTbl + nIdx; 1166 + pRet = (OtaState*)sqlite3_malloc(sizeof(OtaState)); 1167 + if( pRet==0 ){ 1168 + rc = SQLITE_NOMEM; 1169 + }else{ 1170 + memset(pRet, 0, sizeof(OtaState)); 1171 + rc = prepareAndCollectError(p->db, &pStmt, &p->zErrmsg, zSelect); 1172 + } 937 1173 938 - pRet = (OtaState*)sqlite3_malloc(nByte); 939 - if( pRet ){ 940 - pRet->zTbl = (char*)&pRet[1]; 941 - memcpy(pRet->zTbl, sqlite3_column_text(pStmt, 0), nTbl); 942 - if( zIdx ){ 943 - pRet->zIdx = &pRet->zTbl[nTbl]; 944 - memcpy(pRet->zIdx, zIdx, nIdx); 945 - }else{ 946 - pRet->zIdx = 0; 1174 + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ 1175 + switch( sqlite3_column_int(pStmt, 0) ){ 1176 + case OTA_STATE_STAGE: 1177 + pRet->eStage = sqlite3_column_int(pStmt, 1); 1178 + if( pRet->eStage!=OTA_STAGE_OAL 1179 + && pRet->eStage!=OTA_STAGE_COPY 1180 + && pRet->eStage!=OTA_STAGE_CKPT 1181 + ){ 1182 + p->rc = SQLITE_CORRUPT; 947 1183 } 948 - pRet->nRow = sqlite3_column_int(pStmt, 2); 949 - pRet->nProgress = sqlite3_column_int64(pStmt, 3); 950 - } 951 - }else{ 952 - pRet = (OtaState*)sqlite3_malloc(sizeof(OtaState)); 953 - if( pRet ){ 954 - memset(pRet, 0, sizeof(*pRet)); 955 - } 956 - } 957 - rc = sqlite3_finalize(pStmt); 958 - if( rc==SQLITE_OK && pRet==0 ) rc = SQLITE_NOMEM; 959 - if( rc!=SQLITE_OK ){ 960 - sqlite3_free(pRet); 961 - pRet = 0; 1184 + break; 1185 + 1186 + case OTA_STATE_TBL: 1187 + pRet->zTbl = otaStrndup((char*)sqlite3_column_text(pStmt, 1), -1, &rc); 1188 + break; 1189 + 1190 + case OTA_STATE_IDX: 1191 + pRet->zIdx = otaStrndup((char*)sqlite3_column_text(pStmt, 1), -1, &rc); 1192 + break; 1193 + 1194 + case OTA_STATE_ROW: 1195 + pRet->nRow = sqlite3_column_int(pStmt, 1); 1196 + break; 1197 + 1198 + case OTA_STATE_PROGRESS: 1199 + pRet->nProgress = sqlite3_column_int64(pStmt, 1); 1200 + break; 1201 + 1202 + case OTA_STATE_CKPT: 1203 + pRet->nCkptState = sqlite3_column_bytes(pStmt, 1); 1204 + pRet->pCkptState = (unsigned char*)otaStrndup( 1205 + (char*)sqlite3_column_blob(pStmt, 1), pRet->nCkptState, &rc 1206 + ); 1207 + break; 1208 + 1209 + default: 1210 + rc = SQLITE_CORRUPT; 1211 + break; 962 1212 } 963 1213 } 1214 + rc2 = sqlite3_finalize(pStmt); 1215 + if( rc==SQLITE_OK ) rc = rc2; 964 1216 965 1217 p->rc = rc; 966 1218 return pRet; 967 1219 } 968 1220 969 1221 static int otaStrCompare(const char *z1, const char *z2){ 970 1222 if( z1==0 && z2==0 ) return 0; ................................................................................ 995 1247 rc = otaObjIterPrepareAll(p, &p->objiter, p->nStep); 996 1248 } 997 1249 998 1250 p->rc = rc; 999 1251 } 1000 1252 } 1001 1253 1002 -/* 1003 -** This routine is a copy of the sqlite3FileSuffix3() routine from the core. 1004 -** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined. 1005 -** 1006 -** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database 1007 -** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and 1008 -** if filename in z[] has a suffix (a.k.a. "extension") that is longer than 1009 -** three characters, then shorten the suffix on z[] to be the last three 1010 -** characters of the original suffix. 1011 -** 1012 -** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always 1013 -** do the suffix shortening regardless of URI parameter. 1014 -** 1015 -** Examples: 1016 -** 1017 -** test.db-journal => test.nal 1018 -** test.db-wal => test.wal 1019 -** test.db-shm => test.shm 1020 -** test.db-mj7f3319fa => test.9fa 1021 -*/ 1022 -static void otaFileSuffix3(const char *zBase, char *z){ 1023 -#ifdef SQLITE_ENABLE_8_3_NAMES 1024 -#if SQLITE_ENABLE_8_3_NAMES<2 1025 - if( sqlite3_uri_boolean(zBase, "8_3_names", 0) ) 1026 -#endif 1027 - { 1028 - int i, sz; 1029 - sz = sqlite3Strlen30(z); 1030 - for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} 1031 - if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4); 1032 - } 1033 -#endif 1034 -} 1035 - 1036 -/* 1037 -** Move the "*-oal" file corresponding to the target database to the 1038 -** "*-wal" location. If an error occurs, leave an error code and error 1039 -** message in the ota handle. 1040 -*/ 1041 -static void otaMoveOalFile(const char *zBase, sqlite3ota *p){ 1042 - char *zWal = sqlite3_mprintf("%s-wal", p->zTarget); 1043 - char *zOal = sqlite3_mprintf("%s-oal", p->zTarget); 1044 - 1045 - assert( p->rc==SQLITE_DONE && p->zErrmsg==0 ); 1046 - if( zWal==0 || zOal==0 ){ 1047 - p->rc = SQLITE_NOMEM; 1048 - }else{ 1049 - otaFileSuffix3(zBase, zWal); 1050 - otaFileSuffix3(zBase, zOal); 1051 - rename(zOal, zWal); 1052 - } 1053 - 1054 - sqlite3_free(zWal); 1055 - sqlite3_free(zOal); 1056 -} 1057 - 1058 1254 /* 1059 1255 ** If there is a "*-oal" file in the file-system corresponding to the 1060 1256 ** target database in the file-system, delete it. If an error occurs, 1061 1257 ** leave an error code and error message in the ota handle. 1062 1258 */ 1063 1259 static void otaDeleteOalFile(sqlite3ota *p){ 1064 1260 char *zOal = sqlite3_mprintf("%s-oal", p->zTarget); ................................................................................ 1069 1265 1070 1266 /* 1071 1267 ** Open and return a new OTA handle. 1072 1268 */ 1073 1269 sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ 1074 1270 sqlite3ota *p; 1075 1271 int nTarget = strlen(zTarget); 1272 + int nOta = strlen(zOta); 1076 1273 1077 - p = (sqlite3ota*)sqlite3_malloc(sizeof(sqlite3ota)+nTarget+1); 1274 + p = (sqlite3ota*)sqlite3_malloc(sizeof(sqlite3ota)+nTarget+1+nOta+1); 1078 1275 if( p ){ 1079 1276 OtaState *pState = 0; 1080 1277 1081 1278 /* Open the target database */ 1082 1279 memset(p, 0, sizeof(sqlite3ota)); 1083 1280 p->zTarget = (char*)&p[1]; 1084 1281 memcpy(p->zTarget, zTarget, nTarget+1); 1085 - p->rc = sqlite3_open(zTarget, &p->db); 1086 - if( p->rc ){ 1087 - p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); 1088 - } 1089 - otaMPrintfExec(p, "ATTACH %Q AS ota", zOta); 1282 + p->zOta = &p->zTarget[nTarget+1]; 1283 + memcpy(p->zOta, zOta, nOta+1); 1284 + otaOpenDatabase(p); 1090 1285 1091 1286 /* If it has not already been created, create the ota_state table */ 1092 1287 if( p->rc==SQLITE_OK ){ 1093 1288 p->rc = sqlite3_exec(p->db, OTA_CREATE_STATE, 0, 0, &p->zErrmsg); 1094 1289 } 1095 1290 1096 1291 if( p->rc==SQLITE_OK ){ 1097 1292 pState = otaLoadState(p); 1098 - if( pState && pState->zTbl==0 ){ 1099 - otaDeleteOalFile(p); 1293 + assert( pState || p->rc!=SQLITE_OK ); 1294 + if( pState ){ 1295 + if( pState->eStage==0 ){ 1296 + otaDeleteOalFile(p); 1297 + p->eStage = 1; 1298 + }else{ 1299 + p->eStage = pState->eStage; 1300 + } 1301 + p->nProgress = pState->nProgress; 1100 1302 } 1101 1303 } 1304 + assert( p->rc!=SQLITE_OK || p->eStage!=0 ); 1102 1305 1103 - if( p->rc==SQLITE_OK ){ 1104 - const char *zScript = 1105 - "PRAGMA journal_mode=off;" 1106 - "PRAGMA pager_ota_mode=1;" 1107 - "PRAGMA ota_mode=1;" 1108 - "BEGIN IMMEDIATE;" 1109 - ; 1110 - p->rc = sqlite3_exec(p->db, zScript, 0, 0, &p->zErrmsg); 1306 + if( p->eStage==OTA_STAGE_OAL ){ 1307 + if( p->rc==SQLITE_OK ){ 1308 + const char *zScript = 1309 + "PRAGMA journal_mode=off;" 1310 + "PRAGMA pager_ota_mode=1;" 1311 + "PRAGMA ota_mode=1;" 1312 + "BEGIN IMMEDIATE;" 1313 + ; 1314 + p->rc = sqlite3_exec(p->db, zScript, 0, 0, &p->zErrmsg); 1315 + } 1316 + 1317 + /* Point the object iterator at the first object */ 1318 + if( p->rc==SQLITE_OK ){ 1319 + p->rc = otaObjIterFirst(p, &p->objiter); 1320 + } 1321 + 1322 + if( p->rc==SQLITE_OK ){ 1323 + otaLoadTransactionState(p, pState); 1324 + } 1325 + }else if( p->rc==SQLITE_OK && p->eStage==OTA_STAGE_CKPT ){ 1326 + p->rc = sqlite3_ckpt_open( 1327 + p->db, pState->pCkptState, pState->nCkptState, &p->pCkpt 1328 + ); 1111 1329 } 1112 1330 1113 - /* Point the object iterator at the first object */ 1114 - if( p->rc==SQLITE_OK ){ 1115 - p->rc = otaObjIterFirst(p, &p->objiter); 1116 - } 1117 - 1118 - if( p->rc==SQLITE_OK ){ 1119 - p->nProgress = pState->nProgress; 1120 - otaLoadTransactionState(p, pState); 1121 - } 1122 - 1123 - sqlite3_free(pState); 1331 + otaFreeState(pState); 1124 1332 } 1125 1333 1126 1334 return p; 1127 1335 } 1128 1336 1129 1337 /* 1130 1338 ** Close the OTA handle. 1131 1339 */ 1132 1340 int sqlite3ota_close(sqlite3ota *p, char **pzErrmsg){ 1133 1341 int rc; 1134 1342 if( p ){ 1135 - const char *zBase = sqlite3_db_filename(p->db, "main"); 1136 1343 1137 1344 /* If the update has not been fully applied, save the state in 1138 1345 ** the ota db. If successful, this call also commits the open 1139 1346 ** transaction on the ota db. */ 1140 1347 assert( p->rc!=SQLITE_ROW ); 1141 - if( p->rc==SQLITE_OK ){ 1348 + if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){ 1142 1349 assert( p->zErrmsg==0 ); 1143 1350 otaSaveTransactionState(p); 1144 1351 } 1145 1352 1146 - /* Close all open statement handles. */ 1353 + /* Close any open statement handles. */ 1147 1354 otaObjIterFinalize(&p->objiter); 1148 1355 1149 1356 /* Commit the transaction to the *-oal file. */ 1150 - if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){ 1151 - rc = sqlite3_exec(p->db, "COMMIT", 0, 0, &p->zErrmsg); 1152 - if( rc!=SQLITE_OK ) p->rc = rc; 1357 + if( p->rc==SQLITE_OK && p->eStage==OTA_STAGE_OAL ){ 1358 + p->rc = sqlite3_exec(p->db, "COMMIT", 0, 0, &p->zErrmsg); 1359 + } 1360 + 1361 + if( p->rc==SQLITE_OK && p->eStage==OTA_STAGE_CKPT ){ 1362 + p->rc = sqlite3_exec(p->db, "PRAGMA pager_ota_mode=2", 0, 0, &p->zErrmsg); 1153 1363 } 1154 1364 1155 - /* Close the open database handles */ 1365 + /* Close the open database handle */ 1366 + if( p->pCkpt ) sqlite3_ckpt_close(p->pCkpt, 0, 0); 1156 1367 sqlite3_close(p->db); 1157 1368 1158 - /* If the OTA has been completely applied and no error occurred, move 1159 - ** the *-oal file to *-wal. */ 1160 - if( p->rc==SQLITE_DONE ){ 1161 - otaMoveOalFile(zBase, p); 1162 - } 1163 - 1164 1369 rc = p->rc; 1165 1370 *pzErrmsg = p->zErrmsg; 1166 1371 sqlite3_free(p); 1167 1372 }else{ 1168 1373 rc = SQLITE_NOMEM; 1169 1374 *pzErrmsg = 0; 1170 1375 }
Changes to src/main.c.
1763 1763 sqlite3Error(db, rc); 1764 1764 } 1765 1765 rc = sqlite3ApiExit(db, rc); 1766 1766 sqlite3_mutex_leave(db->mutex); 1767 1767 return rc; 1768 1768 #endif 1769 1769 } 1770 + 1771 +int sqlite3_ckpt_open( 1772 + sqlite3 *db, 1773 + unsigned char *a, int n, 1774 + sqlite3_ckpt **ppCkpt 1775 +){ 1776 + Pager *pPager; 1777 + Btree *pBt; 1778 + int rc; 1779 + 1780 + *ppCkpt = 0; 1781 + sqlite3_mutex_enter(db->mutex); 1782 + pBt = db->aDb[0].pBt; 1783 + pPager = sqlite3BtreePager(pBt); 1784 + rc = sqlite3PagerWalCheckpointStart(db, pPager, a, n, ppCkpt); 1785 + sqlite3_mutex_leave(db->mutex); 1786 + return rc; 1787 +} 1770 1788 1771 1789 1772 1790 /* 1773 1791 ** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points 1774 1792 ** to contains a zero-length string, all attached databases are 1775 1793 ** checkpointed. 1776 1794 */
Changes to src/pager.c.
619 619 ** otaMode 620 620 ** This variable is normally 0. It is set to 1 by the PagerSetOtaMode() 621 621 ** function - as a result of a "PRAGMA pager_ota_mode=1" command. Once 622 622 ** the *-oal file has been opened and it has been determined that the 623 623 ** database file has not been modified since it was created, this variable 624 624 ** is set to 2. 625 625 ** 626 +** noCkptOnClose 627 +** 626 628 ** 627 629 */ 628 630 struct Pager { 629 631 sqlite3_vfs *pVfs; /* OS functions to use for IO */ 630 632 u8 exclusiveMode; /* Boolean. True if locking_mode==EXCLUSIVE */ 631 633 u8 journalMode; /* One of the PAGER_JOURNALMODE_* values */ 632 634 u8 useJournal; /* Use a rollback journal on this file */ ................................................................................ 7284 7286 return sqlite3WalFramesize(pPager->pWal); 7285 7287 } 7286 7288 #endif 7287 7289 7288 7290 /* 7289 7291 ** Set or clear the "OTA mode" flag. 7290 7292 */ 7291 -int sqlite3PagerSetOtaMode(Pager *pPager, int bOta){ 7292 - if( pPager->pWal || pPager->eState!=PAGER_OPEN ){ 7293 +int sqlite3PagerSetOtaMode(Pager *pPager, int iOta){ 7294 + if( iOta==1 && (pPager->pWal || pPager->eState!=PAGER_OPEN) ){ 7293 7295 return SQLITE_ERROR; 7294 7296 } 7295 - pPager->otaMode = 1; 7297 + pPager->otaMode = iOta; 7296 7298 return SQLITE_OK; 7297 7299 } 7300 + 7301 +int sqlite3PagerWalCheckpointStart( 7302 + sqlite3 *db, 7303 + Pager *pPager, 7304 + u8 *a, int n, 7305 + sqlite3_ckpt **ppCkpt 7306 +){ 7307 + return sqlite3WalCheckpointStart(db, pPager->pWal, a, n, 7308 + pPager->xBusyHandler, pPager->pBusyHandlerArg, 7309 + pPager->ckptSyncFlags, ppCkpt 7310 + ); 7311 +} 7298 7312 7299 7313 #endif /* SQLITE_OMIT_DISKIO */
Changes to src/pager.h.
205 205 #else 206 206 # define disable_simulated_io_errors() 207 207 # define enable_simulated_io_errors() 208 208 #endif 209 209 210 210 int sqlite3PagerSetOtaMode(Pager *pPager, int bOta); 211 211 void sqlite3PagerWalSalt(Pager *pPager, u32 *aSalt); 212 +int sqlite3PagerWalCheckpointStart(sqlite3*, Pager*, u8*, int, sqlite3_ckpt**); 212 213 213 214 #endif /* _PAGER_H_ */
Changes to src/pragma.c.
896 896 ** Other clients see a rollback-mode database on which the pager_ota_mode 897 897 ** client is holding a SHARED lock. 898 898 */ 899 899 case PragTyp_PAGER_OTA_MODE: { 900 900 Btree *pBt = pDb->pBt; 901 901 assert( pBt!=0 ); 902 902 if( zRight ){ 903 - int iArg = !!sqlite3Atoi(zRight); 903 + int iArg = sqlite3Atoi(zRight); 904 904 if( sqlite3BtreeIsInReadTrans(pBt) ){ 905 905 sqlite3ErrorMsg(pParse, 906 906 "cannot set pager_ota_mode with open transaction" 907 907 ); 908 908 }else if( sqlite3PagerSetOtaMode(sqlite3BtreePager(pBt), iArg) ){ 909 909 sqlite3ErrorMsg(pParse, "cannot set pager_ota_mode in wal mode"); 910 910 }
Changes to src/sqlite.h.in.
7460 7460 int bDelete, /* Zero for insert, non-zero for delete */ 7461 7461 const char *zIndex, /* Index to write to */ 7462 7462 sqlite3_stmt**, /* OUT: New statement handle */ 7463 7463 const char ***pazColl, /* OUT: Collation sequences for each column */ 7464 7464 int **paiCol, int *pnCol /* OUT: See above */ 7465 7465 ); 7466 7466 7467 +/* 7468 +** Incremental checkpoint API. 7469 +** 7470 +** An incremental checkpoint handle is opened using the sqlite3_ckpt_open() 7471 +** API. To begin a new checkpoint, the second and third arguments should both 7472 +** be passed zero. To resume an earlier checkpoint, the second and third 7473 +** arguments should specify a buffer returned by an earlier call to 7474 +** sqlite3_ckpt_close(). When resuming a checkpoint, if the database or WAL 7475 +** file has been modified since the checkpoint was suspended, the 7476 +** sqlite3_ckpt_open() call fails with SQLITE_MISMATCH. 7477 +** 7478 +** Each time sqlite3_ckpt_step() is called on an open checkpoint handle, a 7479 +** single page is copied from the WAL file to the database. If no error 7480 +** occurs, but the checkpoint is not finished, SQLITE_OK is returned. If the 7481 +** checkpoint has been finished (and so sqlite3_ckpt_step() should not be 7482 +** called again), SQLITE_DONE is returned. Otherwise, if an error occurs, 7483 +** some other SQLite error code is returned. 7484 +** 7485 +** Calling sqlite3_ckpt_close() closes an open checkpoint handle. If the 7486 +** checkpoint has finished and no error has occurred, SQLITE_OK is returned 7487 +** and the two output parameters zeroed. Or, if an error has occurred, an 7488 +** error code is returned and the two output parameters are zeroed. Finally, 7489 +** if the checkpoint is not finished but no error has occurred, SQLITE_OK is 7490 +** returned and the first output variable set to point to a buffer allocated 7491 +** using sqlite3_malloc() containing the serialized state of the checkpoint. 7492 +** The contents of this buffer may be passed to a later call to 7493 +** sqlite3_ckpt_open() to restart the checkpoint. The second output variable 7494 +** is set to the size of the buffer in bytes. 7495 +*/ 7496 +typedef struct sqlite3_ckpt sqlite3_ckpt; 7497 +int sqlite3_ckpt_open(sqlite3*, unsigned char *a, int n, sqlite3_ckpt **ppCkpt); 7498 +int sqlite3_ckpt_step(sqlite3_ckpt*); 7499 +int sqlite3_ckpt_close(sqlite3_ckpt*, unsigned char **pa, int *pn); 7500 + 7467 7501 /* 7468 7502 ** Undo the hack that converts floating point types to integer for 7469 7503 ** builds on processors without floating point support. 7470 7504 */ 7471 7505 #ifdef SQLITE_OMIT_FLOATING_POINT 7472 7506 # undef double 7473 7507 #endif 7474 7508 7475 7509 #ifdef __cplusplus 7476 7510 } /* End of the 'extern "C"' block */ 7477 7511 #endif 7478 7512 #endif /* _SQLITE3_H_ */
Changes to src/wal.c.
478 478 ht_slot *aIndex; /* i0, i1, i2... such that aPgno[iN] ascend */ 479 479 u32 *aPgno; /* Array of page numbers. */ 480 480 int nEntry; /* Nr. of entries in aPgno[] and aIndex[] */ 481 481 int iZero; /* Frame number associated with aPgno[0] */ 482 482 } aSegment[1]; /* One for every 32KB page in the wal-index */ 483 483 }; 484 484 485 +/* 486 +** walCheckpoint 487 +*/ 488 +typedef struct WalCkpt WalCkpt; 489 +struct WalCkpt { 490 + sqlite3 *db; /* Database pointer (incremental only) */ 491 + int szPage; /* Database page-size */ 492 + int sync_flags; /* Flags for OsSync() (or 0) */ 493 + u32 mxSafeFrame; /* Max frame that can be backfilled */ 494 + u32 mxPage; /* Max database page to write */ 495 + volatile WalCkptInfo *pInfo; /* The checkpoint status information */ 496 + WalIterator *pIter; /* Wal iterator context */ 497 + Wal *pWal; /* Pointer to owner object */ 498 + u8 *aBuf; /* Temporary page-sized buffer to use */ 499 + int rc; /* Error code. SQLITE_DONE -> finished */ 500 + int nStep; /* Number of times pIter has been stepped */ 501 +}; 502 + 485 503 /* 486 504 ** Define the parameters of the hash tables in the wal-index file. There 487 505 ** is a hash-table following every HASHTABLE_NPAGE page numbers in the 488 506 ** wal-index. 489 507 ** 490 508 ** Changing any of these constants will alter the wal-index format and 491 509 ** create incompatibilities. ................................................................................ 1618 1636 /* 1619 1637 ** The cache of the wal-index header must be valid to call this function. 1620 1638 ** Return the page-size in bytes used by the database. 1621 1639 */ 1622 1640 static int walPagesize(Wal *pWal){ 1623 1641 return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); 1624 1642 } 1643 + 1644 +static int walCheckpointStart( 1645 + Wal *pWal, 1646 + u8 *aBuf, /* Page-sized temporary buffer */ 1647 + int nBuf, /* Size of aBuf[] in bytes */ 1648 + int (*xBusy)(void*), /* Function to call when busy (or NULL) */ 1649 + void *pBusyArg, /* Context argument for xBusyHandler */ 1650 + int sync_flags, /* Flags for OsSync() (or 0) */ 1651 + WalCkpt *p /* Allocated object to populate */ 1652 +){ 1653 + int rc; /* Return code */ 1654 + int i; /* Iterator variable */ 1655 + 1656 + memset(p, 0, sizeof(WalCkpt)); 1657 + 1658 + if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ 1659 + return SQLITE_CORRUPT_BKPT; 1660 + } 1661 + 1662 + p->szPage = walPagesize(pWal); 1663 + p->pWal = pWal; 1664 + p->aBuf = aBuf; 1665 + p->sync_flags = sync_flags; 1666 + testcase( p->szPage<=32768 ); 1667 + testcase( p->szPage>=65536 ); 1668 + p->pInfo = walCkptInfo(pWal); 1669 + if( p->pInfo->nBackfill>=pWal->hdr.mxFrame ) return SQLITE_OK; 1670 + 1671 + /* Allocate the iterator */ 1672 + rc = walIteratorInit(pWal, &p->pIter); 1673 + if( rc!=SQLITE_OK ) return rc; 1674 + assert( p->pIter ); 1675 + 1676 + /* Compute in mxSafeFrame the index of the last frame of the WAL that is 1677 + ** safe to write into the database. Frames beyond mxSafeFrame might 1678 + ** overwrite database pages that are in use by active readers and thus 1679 + ** cannot be backfilled from the WAL. 1680 + */ 1681 + p->mxSafeFrame = pWal->hdr.mxFrame; 1682 + p->mxPage = pWal->hdr.nPage; 1683 + for(i=1; i<WAL_NREADER; i++){ 1684 + u32 y = p->pInfo->aReadMark[i]; 1685 + if( p->mxSafeFrame>y ){ 1686 + assert( y<=pWal->hdr.mxFrame ); 1687 + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); 1688 + if( rc==SQLITE_OK ){ 1689 + p->pInfo->aReadMark[i] = (i==1 ? p->mxSafeFrame : READMARK_NOT_USED); 1690 + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); 1691 + }else if( rc==SQLITE_BUSY ){ 1692 + p->mxSafeFrame = y; 1693 + xBusy = 0; 1694 + }else{ 1695 + walIteratorFree(p->pIter); 1696 + p->pIter = 0; 1697 + return rc; 1698 + } 1699 + } 1700 + } 1701 + 1702 + if( p->pInfo->nBackfill>=p->mxSafeFrame 1703 + || (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0), 1))!=SQLITE_OK 1704 + ){ 1705 + walIteratorFree(p->pIter); 1706 + p->pIter = 0; 1707 + } 1708 + if( rc==SQLITE_BUSY ) rc = SQLITE_OK; 1709 + 1710 + if( rc==SQLITE_OK && p->pIter ){ 1711 + /* Sync the WAL to disk */ 1712 + if( sync_flags ){ 1713 + rc = sqlite3OsSync(pWal->pWalFd, sync_flags); 1714 + } 1715 + 1716 + /* If the database may grow as a result of this checkpoint, hint 1717 + ** about the eventual size of the db file to the VFS layer. */ 1718 + if( rc==SQLITE_OK ){ 1719 + i64 nSize; /* Current size of database file */ 1720 + i64 nReq = ((i64)p->mxPage * p->szPage); 1721 + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); 1722 + if( rc==SQLITE_OK && nSize<nReq ){ 1723 + sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); 1724 + } 1725 + } 1726 + } 1727 + 1728 + return rc; 1729 +} 1730 + 1731 +static int walCheckpointStep(WalCkpt *p){ 1732 + u32 iDbpage = 0; /* Next database page to write */ 1733 + u32 iFrame = 0; /* Wal frame containing data for iDbpage */ 1734 + int rc = SQLITE_DONE; 1735 + 1736 + assert( p->rc==SQLITE_OK ); 1737 + while( p->pIter && 0==walIteratorNext(p->pIter, &iDbpage, &iFrame) ){ 1738 + i64 iOffset; 1739 + assert( walFramePgno(p->pWal, iFrame)==iDbpage ); 1740 + p->nStep++; 1741 + if( iFrame<=p->pInfo->nBackfill 1742 + || iFrame>p->mxSafeFrame 1743 + || iDbpage>p->mxPage 1744 + ){ 1745 + continue; 1746 + } 1747 + 1748 + iOffset = walFrameOffset(iFrame, p->szPage) + WAL_FRAME_HDRSIZE; 1749 + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ 1750 + rc = sqlite3OsRead(p->pWal->pWalFd, p->aBuf, p->szPage, iOffset); 1751 + if( rc!=SQLITE_OK ) break; 1752 + iOffset = (iDbpage-1)*(i64)p->szPage; 1753 + testcase( IS_BIG_INT(iOffset) ); 1754 + rc = sqlite3OsWrite(p->pWal->pDbFd, p->aBuf, p->szPage, iOffset); 1755 + break; 1756 + } 1757 + 1758 + p->rc = rc; 1759 + return rc; 1760 +} 1761 + 1762 +static int walCheckpointFinalize(WalCkpt *p){ 1763 + if( p->pIter ){ 1764 + int rc = p->rc; 1765 + Wal *pWal = p->pWal; 1766 + 1767 + /* If work was completed */ 1768 + if( p->pIter && rc==SQLITE_DONE ){ 1769 + rc = SQLITE_OK; 1770 + if( p->mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ 1771 + i64 szDb = pWal->hdr.nPage*(i64)p->szPage; 1772 + testcase( IS_BIG_INT(szDb) ); 1773 + rc = sqlite3OsTruncate(pWal->pDbFd, szDb); 1774 + if( rc==SQLITE_OK && p->sync_flags ){ 1775 + rc = sqlite3OsSync(pWal->pDbFd, p->sync_flags); 1776 + } 1777 + } 1778 + if( rc==SQLITE_OK ){ 1779 + p->pInfo->nBackfill = p->mxSafeFrame; 1780 + } 1781 + p->rc = rc; 1782 + } 1783 + 1784 + /* Release the reader lock held while backfilling */ 1785 + walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); 1786 + walIteratorFree(p->pIter); 1787 + p->pIter = 0; 1788 + } 1789 + 1790 + return p->rc; 1791 +} 1625 1792 1626 1793 /* 1627 1794 ** Copy as much content as we can from the WAL back into the database file 1628 1795 ** in response to an sqlite3_wal_checkpoint() request or the equivalent. 1629 1796 ** 1630 1797 ** The amount of information copies from WAL to database might be limited 1631 1798 ** by active readers. This routine will never overwrite a database page ................................................................................ 1656 1823 */ 1657 1824 static int walCheckpoint( 1658 1825 Wal *pWal, /* Wal connection */ 1659 1826 int eMode, /* One of PASSIVE, FULL or RESTART */ 1660 1827 int (*xBusyCall)(void*), /* Function to call when busy */ 1661 1828 void *pBusyArg, /* Context argument for xBusyHandler */ 1662 1829 int sync_flags, /* Flags for OsSync() (or 0) */ 1663 - u8 *zBuf /* Temporary buffer to use */ 1830 + u8 *zBuf, /* Temporary buffer to use */ 1831 + int nBuf /* Size of zBuf in bytes */ 1664 1832 ){ 1665 1833 int rc; /* Return code */ 1666 - int szPage; /* Database page-size */ 1667 - WalIterator *pIter = 0; /* Wal iterator context */ 1668 - u32 iDbpage = 0; /* Next database page to write */ 1669 - u32 iFrame = 0; /* Wal frame containing data for iDbpage */ 1670 - u32 mxSafeFrame; /* Max frame that can be backfilled */ 1671 - u32 mxPage; /* Max database page to write */ 1672 - int i; /* Loop counter */ 1673 - volatile WalCkptInfo *pInfo; /* The checkpoint status information */ 1674 1834 int (*xBusy)(void*) = 0; /* Function to call when waiting for locks */ 1675 - 1676 - szPage = walPagesize(pWal); 1677 - testcase( szPage<=32768 ); 1678 - testcase( szPage>=65536 ); 1679 - pInfo = walCkptInfo(pWal); 1680 - if( pInfo->nBackfill>=pWal->hdr.mxFrame ) return SQLITE_OK; 1681 - 1682 - /* Allocate the iterator */ 1683 - rc = walIteratorInit(pWal, &pIter); 1684 - if( rc!=SQLITE_OK ){ 1685 - return rc; 1686 - } 1687 - assert( pIter ); 1835 + WalCkpt sC; 1688 1836 1689 1837 if( eMode!=SQLITE_CHECKPOINT_PASSIVE ) xBusy = xBusyCall; 1690 - 1691 - /* Compute in mxSafeFrame the index of the last frame of the WAL that is 1692 - ** safe to write into the database. Frames beyond mxSafeFrame might 1693 - ** overwrite database pages that are in use by active readers and thus 1694 - ** cannot be backfilled from the WAL. 1695 - */ 1696 - mxSafeFrame = pWal->hdr.mxFrame; 1697 - mxPage = pWal->hdr.nPage; 1698 - for(i=1; i<WAL_NREADER; i++){ 1699 - u32 y = pInfo->aReadMark[i]; 1700 - if( mxSafeFrame>y ){ 1701 - assert( y<=pWal->hdr.mxFrame ); 1702 - rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); 1703 - if( rc==SQLITE_OK ){ 1704 - pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED); 1705 - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); 1706 - }else if( rc==SQLITE_BUSY ){ 1707 - mxSafeFrame = y; 1708 - xBusy = 0; 1709 - }else{ 1710 - goto walcheckpoint_out; 1711 - } 1712 - } 1713 - } 1714 - 1715 - if( pInfo->nBackfill<mxSafeFrame 1716 - && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0), 1))==SQLITE_OK 1717 - ){ 1718 - i64 nSize; /* Current size of database file */ 1719 - u32 nBackfill = pInfo->nBackfill; 1720 - 1721 - /* Sync the WAL to disk */ 1722 - if( sync_flags ){ 1723 - rc = sqlite3OsSync(pWal->pWalFd, sync_flags); 1724 - } 1725 - 1726 - /* If the database may grow as a result of this checkpoint, hint 1727 - ** about the eventual size of the db file to the VFS layer. 1728 - */ 1729 - if( rc==SQLITE_OK ){ 1730 - i64 nReq = ((i64)mxPage * szPage); 1731 - rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); 1732 - if( rc==SQLITE_OK && nSize<nReq ){ 1733 - sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); 1734 - } 1735 - } 1736 - 1737 - 1738 - /* Iterate through the contents of the WAL, copying data to the db file. */ 1739 - while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ 1740 - i64 iOffset; 1741 - assert( walFramePgno(pWal, iFrame)==iDbpage ); 1742 - if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ) continue; 1743 - iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; 1744 - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ 1745 - rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); 1746 - if( rc!=SQLITE_OK ) break; 1747 - iOffset = (iDbpage-1)*(i64)szPage; 1748 - testcase( IS_BIG_INT(iOffset) ); 1749 - rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); 1750 - if( rc!=SQLITE_OK ) break; 1751 - } 1752 - 1753 - /* If work was actually accomplished... */ 1754 - if( rc==SQLITE_OK ){ 1755 - if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ 1756 - i64 szDb = pWal->hdr.nPage*(i64)szPage; 1757 - testcase( IS_BIG_INT(szDb) ); 1758 - rc = sqlite3OsTruncate(pWal->pDbFd, szDb); 1759 - if( rc==SQLITE_OK && sync_flags ){ 1760 - rc = sqlite3OsSync(pWal->pDbFd, sync_flags); 1761 - } 1762 - } 1763 - if( rc==SQLITE_OK ){ 1764 - pInfo->nBackfill = mxSafeFrame; 1765 - } 1766 - } 1767 - 1768 - /* Release the reader lock held while backfilling */ 1769 - walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); 1770 - } 1771 - 1772 - if( rc==SQLITE_BUSY ){ 1773 - /* Reset the return code so as not to report a checkpoint failure 1774 - ** just because there are active readers. */ 1775 - rc = SQLITE_OK; 1776 - } 1838 + rc = walCheckpointStart(pWal, zBuf, nBuf, xBusy, pBusyArg, sync_flags, &sC); 1839 + if( sC.pIter==0 ) goto walcheckpoint_out; 1840 + assert( rc==SQLITE_OK ); 1841 + 1842 + /* Step the checkpoint object until it reports something other than 1843 + ** SQLITE_OK. */ 1844 + while( SQLITE_OK==(rc = walCheckpointStep(&sC)) ); 1845 + if( rc==SQLITE_DONE ) rc = SQLITE_OK; 1846 + rc = walCheckpointFinalize(&sC); 1777 1847 1778 1848 /* If this is an SQLITE_CHECKPOINT_RESTART operation, and the entire wal 1779 1849 ** file has been copied into the database file, then block until all 1780 1850 ** readers have finished using the wal file. This ensures that the next 1781 1851 ** process to write to the database restarts the wal file. 1782 1852 */ 1783 1853 if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ 1784 1854 assert( pWal->writeLock ); 1785 - if( pInfo->nBackfill<pWal->hdr.mxFrame ){ 1855 + if( sC.pInfo->nBackfill<pWal->hdr.mxFrame ){ 1786 1856 rc = SQLITE_BUSY; 1787 1857 }else if( eMode==SQLITE_CHECKPOINT_RESTART ){ 1788 - assert( mxSafeFrame==pWal->hdr.mxFrame ); 1858 + assert( sC.mxSafeFrame==pWal->hdr.mxFrame ); 1789 1859 rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); 1790 1860 if( rc==SQLITE_OK ){ 1791 1861 walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); 1792 1862 } 1793 1863 } 1794 1864 } 1795 1865 1796 1866 walcheckpoint_out: 1797 - walIteratorFree(pIter); 1867 + walIteratorFree(sC.pIter); 1798 1868 return rc; 1799 1869 } 1800 1870 1801 1871 /* 1802 1872 ** If the WAL file is currently larger than nMax bytes in size, truncate 1803 1873 ** it to exactly nMax bytes. If an error occurs while doing so, ignore it. 1804 1874 */ ................................................................................ 2967 3037 if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ 2968 3038 sqlite3OsUnfetch(pWal->pDbFd, 0, 0); 2969 3039 } 2970 3040 } 2971 3041 2972 3042 /* Copy data from the log to the database file. */ 2973 3043 if( rc==SQLITE_OK ){ 2974 - if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ 2975 - rc = SQLITE_CORRUPT_BKPT; 2976 - }else{ 2977 - rc = walCheckpoint(pWal, eMode2, xBusy, pBusyArg, sync_flags, zBuf); 2978 - } 3044 + rc = walCheckpoint(pWal, eMode2, xBusy, pBusyArg, sync_flags, zBuf, nBuf); 2979 3045 2980 3046 /* If no error occurred, set the output variables. */ 2981 3047 if( rc==SQLITE_OK || rc==SQLITE_BUSY ){ 2982 3048 if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; 2983 3049 if( pnCkpt ) *pnCkpt = (int)(walCkptInfo(pWal)->nBackfill); 2984 3050 } 2985 3051 } ................................................................................ 2997 3063 /* Release the locks. */ 2998 3064 sqlite3WalEndWriteTransaction(pWal); 2999 3065 walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); 3000 3066 pWal->ckptLock = 0; 3001 3067 WALTRACE(("WAL%p: checkpoint %s\n", pWal, rc ? "failed" : "ok")); 3002 3068 return (rc==SQLITE_OK && eMode!=eMode2 ? SQLITE_BUSY : rc); 3003 3069 } 3070 + 3071 +int sqlite3_ckpt_step(sqlite3_ckpt *pCkpt){ 3072 + int rc; 3073 + WalCkpt *p = (WalCkpt*)pCkpt; 3074 + sqlite3_mutex_enter(p->db->mutex); 3075 + rc = walCheckpointStep(p); 3076 + sqlite3_mutex_leave(p->db->mutex); 3077 + return rc; 3078 +} 3079 + 3080 +int sqlite3_ckpt_close(sqlite3_ckpt *pCkpt, u8 **paState, int *pnState){ 3081 + int rc; 3082 + WalCkpt *p = (WalCkpt*)pCkpt; 3083 + sqlite3 *db = p->db; 3084 + Wal *pWal = p->pWal; 3085 + sqlite3_mutex_enter(db->mutex); 3086 + if( paState ){ 3087 + *paState = 0; 3088 + *pnState = 0; 3089 + if( p->rc==SQLITE_OK ){ 3090 + u8 *aState = sqlite3_malloc(sizeof(u32) * 3); 3091 + if( aState==0 ){ 3092 + p->rc = SQLITE_NOMEM; 3093 + }else{ 3094 + *pnState = sizeof(u32)*3; 3095 + sqlite3Put4byte(&aState[0], p->nStep); 3096 + sqlite3Put4byte(&aState[4], p->pWal->hdr.aCksum[0]); 3097 + sqlite3Put4byte(&aState[8], p->pWal->hdr.aCksum[1]); 3098 + *paState = aState; 3099 + } 3100 + } 3101 + } 3102 + rc = walCheckpointFinalize(p); 3103 + walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); 3104 + pWal->ckptLock = 0; 3105 + sqlite3_free(p); 3106 + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); 3107 + sqlite3_mutex_leave(db->mutex); 3108 + return rc; 3109 +} 3110 + 3111 +int sqlite3WalCheckpointStart( 3112 + sqlite3 *db, /* Database connection */ 3113 + Wal *pWal, /* Wal connection */ 3114 + u8 *aState, int nState, /* Checkpoint state to restore */ 3115 + int (*xBusy)(void*), /* Function to call when busy */ 3116 + void *pBusyArg, /* Context argument for xBusyHandler */ 3117 + int sync_flags, /* Flags to sync db file with (or 0) */ 3118 + sqlite3_ckpt **ppCkpt /* OUT: Incremental checkpoint object */ 3119 +){ 3120 + WalCkpt *p = 0; 3121 + int isChanged = 0; 3122 + int rc; 3123 + int pgsz; 3124 + 3125 + *ppCkpt = 0; 3126 + if( pWal->readOnly ) return SQLITE_READONLY; 3127 + WALTRACE(("WAL%p: checkpoint begins\n", pWal)); 3128 + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); 3129 + if( rc ){ 3130 + /* Usually this is SQLITE_BUSY meaning that another thread or process 3131 + ** is already running a checkpoint, or maybe a recovery. But it might 3132 + ** also be SQLITE_IOERR. */ 3133 + return rc; 3134 + } 3135 + pWal->ckptLock = 1; 3136 + 3137 + /* Read the wal-index header. */ 3138 + rc = walIndexReadHdr(pWal, &isChanged); 3139 + if( rc!=SQLITE_OK ) goto ckptstart_out; 3140 + if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ 3141 + sqlite3OsUnfetch(pWal->pDbFd, 0, 0); 3142 + } 3143 + 3144 + pgsz = walPagesize(pWal); 3145 + p = sqlite3_malloc(sizeof(WalCkpt) + pgsz); 3146 + if( p==0 ){ 3147 + rc = SQLITE_NOMEM; 3148 + goto ckptstart_out; 3149 + } 3150 + 3151 + rc = walCheckpointStart( 3152 + pWal, (u8*)&p[1], pgsz, xBusy, pBusyArg, sync_flags, p 3153 + ); 3154 + p->db = db; 3155 + 3156 + if( rc==SQLITE_OK && aState ){ 3157 + if( nState!=sizeof(u32)*3 ){ 3158 + rc = SQLITE_CORRUPT_BKPT; 3159 + }else{ 3160 + int i; 3161 + if( pWal->hdr.aCksum[0]!=sqlite3Get4byte(&aState[4]) 3162 + || pWal->hdr.aCksum[1]!=sqlite3Get4byte(&aState[8]) 3163 + ){ 3164 + rc = SQLITE_MISMATCH; 3165 + }else{ 3166 + p->nStep = (int)sqlite3Get4byte(aState); 3167 + sqlite3Put4byte(&aState[4], pWal->hdr.aCksum[0]); 3168 + sqlite3Put4byte(&aState[8], pWal->hdr.aCksum[1]); 3169 + for(i=0; rc==SQLITE_OK && i<p->nStep; i++){ 3170 + u32 dummy1, dummy2; 3171 + rc = walIteratorNext(p->pIter, &dummy1, &dummy2); 3172 + } 3173 + } 3174 + } 3175 + } 3176 + 3177 + ckptstart_out: 3178 + if( rc!=SQLITE_OK ){ 3179 + if( p ) walIteratorFree(p->pIter); 3180 + walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); 3181 + pWal->ckptLock = 0; 3182 + sqlite3_free(p); 3183 + p = 0; 3184 + } 3185 + *ppCkpt = (sqlite3_ckpt*)p; 3186 + return rc; 3187 +} 3004 3188 3005 3189 /* Return the value to pass to a sqlite3_wal_hook callback, the 3006 3190 ** number of frames in the WAL at the point of the last commit since 3007 3191 ** sqlite3WalCallback() was called. If no commits have occurred since 3008 3192 ** the last call, then return 0. 3009 3193 */ 3010 3194 int sqlite3WalCallback(Wal *pWal){
Changes to src/wal.h.
123 123 /* Return true if the argument is non-NULL and the WAL module is using 124 124 ** heap-memory for the wal-index. Otherwise, if the argument is NULL or the 125 125 ** WAL module is using shared-memory, return false. 126 126 */ 127 127 int sqlite3WalHeapMemory(Wal *pWal); 128 128 129 129 int sqlite3WalCheckSalt(Wal *pWal, sqlite3_file*); 130 + 131 +int sqlite3WalCheckpointStart(sqlite3 *, 132 + Wal *pWal, /* Wal connection */ 133 + u8 *aState, int nState, /* Checkpoint state to restore */ 134 + int (*xBusy)(void*), /* Function to call when busy */ 135 + void *pBusyArg, /* Context argument for xBusyHandler */ 136 + int sync_flags, /* Flags to sync db file with (or 0) */ 137 + sqlite3_ckpt **ppCkpt /* OUT: Incremental checkpoint object */ 138 +); 130 139 131 140 #ifdef SQLITE_ENABLE_ZIPVFS 132 141 /* If the WAL file is not empty, return the number of bytes of content 133 142 ** stored in each frame (i.e. the db page-size when the WAL was created). 134 143 */ 135 144 int sqlite3WalFramesize(Wal *pWal); 136 145 #endif 137 146 138 147 #endif /* ifndef SQLITE_OMIT_WAL */ 139 148 #endif /* _WAL_H_ */
Changes to test/wal.test.
371 371 list [file size test.db] [file size test.db-wal] 372 372 } [list 2048 [wal_file_size 3 1024]] 373 373 374 374 # Execute some transactions in auto-vacuum mode to test database file 375 375 # truncation. 376 376 # 377 377 do_test wal-8.1 { 378 +breakpoint 378 379 reopen_db 379 380 catch { db close } 380 381 forcedelete test.db test.db-wal 381 382 382 383 sqlite3 db test.db 383 384 db function blob blob 384 385 execsql {