/ Check-in [e26ef165]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add support for "fossil deltas" to RBU and "sqldiff --rbu".
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: e26ef165fe2f7524684af0d269d38475ea8b9489
User & Date: dan 2015-07-31 19:52:03
Context
2015-08-01
18:18
Add extra tests for RBU and FTS3/4. check-in: 34190449 user: dan tags: trunk
2015-07-31
19:52
Add support for "fossil deltas" to RBU and "sqldiff --rbu". check-in: e26ef165 user: dan tags: trunk
18:59
Fix the sqlite3_stmt_busy() interface so that it always returns FALSE after the statement has returned SQLITE_DONE, even for ROLLBACK statements. Clarify the documentation. check-in: 047d3475 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/rbu/rbudiff.test.

    53     53   }
    54     54   
    55     55   proc rbudiff_cksum {db1} {
    56     56     set txt ""
    57     57   
    58     58     sqlite3 dbtmp $db1
    59     59     foreach tbl [dbtmp eval {SELECT name FROM sqlite_master WHERE type='table'}] {
           60  +    set cols [list]
           61  +    dbtmp eval "PRAGMA table_info = $tbl" { lappend cols "quote( $name )" }
    60     62       append txt [dbtmp eval \
    61         -      "SELECT a || '.' || b || '.' || c FROM $tbl ORDER BY 1"
           63  +      "SELECT [join $cols {||'.'||}] FROM $tbl ORDER BY 1"
    62     64       ]
    63     65     }
    64     66     dbtmp close
    65     67   
    66     68     md5 $txt
    67     69   }
    68     70   
................................................................................
    90     92       INSERT INTO t1 VALUES('u', 'v', 'w');
    91     93       INSERT INTO t1 VALUES('x', 'y', 'z');
    92     94     } {
    93     95       DELETE FROM t1 WHERE a='u';
    94     96       INSERT INTO t1 VALUES('a', 'b', 'c');
    95     97     }
    96     98   
           99  +  3 {
          100  +    CREATE TABLE t1(i INTEGER PRIMARY KEY, x);
          101  +    INSERT INTO t1 VALUES(1,
          102  +      X'0000000000000000111111111111111122222222222222223333333333333333'
          103  +    );
          104  +    CREATE TABLE t2(y INTEGER PRIMARY KEY, x);
          105  +    INSERT INTO t2 VALUES(1,
          106  +        X'0000000000000000111111111111111122222222222222223333333333333333'
          107  +    );
          108  +  } {
          109  +    DELETE FROM t1;
          110  +    INSERT INTO t1 VALUES(1,
          111  +      X'0000000000000000111111111111111122222555555552223333333333333333'
          112  +    );
          113  +    DELETE FROM t2;
          114  +    INSERT INTO t2 VALUES(1,
          115  +        X'0000000000000000111111111111111122222222222222223333333FFF333333'
          116  +    );
          117  +  }
          118  +
    97    119   } {
    98         -  
    99    120     catch { db close }
   100    121   
   101    122     forcedelete test.db test.db2
   102    123     sqlite3 db test.db
   103    124     db eval "$init"
   104    125     sqlite3 db test.db2
   105    126     db eval "$init ; $mod"

Changes to ext/rbu/sqlite3rbu.c.

   356    356     char *zDel;                     /* Delete this when closing file */
   357    357   
   358    358     const char *zWal;               /* Wal filename for this main db file */
   359    359     rbu_file *pWalFd;               /* Wal file descriptor for this main db */
   360    360     rbu_file *pMainNext;            /* Next MAIN_DB file */
   361    361   };
   362    362   
          363  +
          364  +/*************************************************************************
          365  +** The following three functions, found below:
          366  +**
          367  +**   rbuDeltaGetInt()
          368  +**   rbuDeltaChecksum()
          369  +**   rbuDeltaApply()
          370  +**
          371  +** are lifted from the fossil source code (http://fossil-scm.org). They
          372  +** are used to implement the scalar SQL function rbu_fossil_delta().
          373  +*/
          374  +
          375  +/*
          376  +** Read bytes from *pz and convert them into a positive integer.  When
          377  +** finished, leave *pz pointing to the first character past the end of
          378  +** the integer.  The *pLen parameter holds the length of the string
          379  +** in *pz and is decremented once for each character in the integer.
          380  +*/
          381  +static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){
          382  +  static const signed char zValue[] = {
          383  +    -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
          384  +    -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
          385  +    -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
          386  +     0,  1,  2,  3,  4,  5,  6,  7,    8,  9, -1, -1, -1, -1, -1, -1,
          387  +    -1, 10, 11, 12, 13, 14, 15, 16,   17, 18, 19, 20, 21, 22, 23, 24,
          388  +    25, 26, 27, 28, 29, 30, 31, 32,   33, 34, 35, -1, -1, -1, -1, 36,
          389  +    -1, 37, 38, 39, 40, 41, 42, 43,   44, 45, 46, 47, 48, 49, 50, 51,
          390  +    52, 53, 54, 55, 56, 57, 58, 59,   60, 61, 62, -1, -1, -1, 63, -1,
          391  +  };
          392  +  unsigned int v = 0;
          393  +  int c;
          394  +  unsigned char *z = (unsigned char*)*pz;
          395  +  unsigned char *zStart = z;
          396  +  while( (c = zValue[0x7f&*(z++)])>=0 ){
          397  +     v = (v<<6) + c;
          398  +  }
          399  +  z--;
          400  +  *pLen -= z - zStart;
          401  +  *pz = (char*)z;
          402  +  return v;
          403  +}
          404  +
          405  +/*
          406  +** Compute a 32-bit checksum on the N-byte buffer.  Return the result.
          407  +*/
          408  +static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){
          409  +  const unsigned char *z = (const unsigned char *)zIn;
          410  +  unsigned sum0 = 0;
          411  +  unsigned sum1 = 0;
          412  +  unsigned sum2 = 0;
          413  +  unsigned sum3 = 0;
          414  +  while(N >= 16){
          415  +    sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]);
          416  +    sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]);
          417  +    sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]);
          418  +    sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]);
          419  +    z += 16;
          420  +    N -= 16;
          421  +  }
          422  +  while(N >= 4){
          423  +    sum0 += z[0];
          424  +    sum1 += z[1];
          425  +    sum2 += z[2];
          426  +    sum3 += z[3];
          427  +    z += 4;
          428  +    N -= 4;
          429  +  }
          430  +  sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24);
          431  +  switch(N){
          432  +    case 3:   sum3 += (z[2] << 8);
          433  +    case 2:   sum3 += (z[1] << 16);
          434  +    case 1:   sum3 += (z[0] << 24);
          435  +    default:  ;
          436  +  }
          437  +  return sum3;
          438  +}
          439  +
          440  +/*
          441  +** Apply a delta.
          442  +**
          443  +** The output buffer should be big enough to hold the whole output
          444  +** file and a NUL terminator at the end.  The delta_output_size()
          445  +** routine will determine this size for you.
          446  +**
          447  +** The delta string should be null-terminated.  But the delta string
          448  +** may contain embedded NUL characters (if the input and output are
          449  +** binary files) so we also have to pass in the length of the delta in
          450  +** the lenDelta parameter.
          451  +**
          452  +** This function returns the size of the output file in bytes (excluding
          453  +** the final NUL terminator character).  Except, if the delta string is
          454  +** malformed or intended for use with a source file other than zSrc,
          455  +** then this routine returns -1.
          456  +**
          457  +** Refer to the delta_create() documentation above for a description
          458  +** of the delta file format.
          459  +*/
          460  +static int rbuDeltaApply(
          461  +  const char *zSrc,      /* The source or pattern file */
          462  +  int lenSrc,            /* Length of the source file */
          463  +  const char *zDelta,    /* Delta to apply to the pattern */
          464  +  int lenDelta,          /* Length of the delta */
          465  +  char *zOut             /* Write the output into this preallocated buffer */
          466  +){
          467  +  unsigned int limit;
          468  +  unsigned int total = 0;
          469  +#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST
          470  +  char *zOrigOut = zOut;
          471  +#endif
          472  +
          473  +  limit = rbuDeltaGetInt(&zDelta, &lenDelta);
          474  +  if( *zDelta!='\n' ){
          475  +    /* ERROR: size integer not terminated by "\n" */
          476  +    return -1;
          477  +  }
          478  +  zDelta++; lenDelta--;
          479  +  while( *zDelta && lenDelta>0 ){
          480  +    unsigned int cnt, ofst;
          481  +    cnt = rbuDeltaGetInt(&zDelta, &lenDelta);
          482  +    switch( zDelta[0] ){
          483  +      case '@': {
          484  +        zDelta++; lenDelta--;
          485  +        ofst = rbuDeltaGetInt(&zDelta, &lenDelta);
          486  +        if( lenDelta>0 && zDelta[0]!=',' ){
          487  +          /* ERROR: copy command not terminated by ',' */
          488  +          return -1;
          489  +        }
          490  +        zDelta++; lenDelta--;
          491  +        total += cnt;
          492  +        if( total>limit ){
          493  +          /* ERROR: copy exceeds output file size */
          494  +          return -1;
          495  +        }
          496  +        if( ofst+cnt > lenSrc ){
          497  +          /* ERROR: copy extends past end of input */
          498  +          return -1;
          499  +        }
          500  +        memcpy(zOut, &zSrc[ofst], cnt);
          501  +        zOut += cnt;
          502  +        break;
          503  +      }
          504  +      case ':': {
          505  +        zDelta++; lenDelta--;
          506  +        total += cnt;
          507  +        if( total>limit ){
          508  +          /* ERROR:  insert command gives an output larger than predicted */
          509  +          return -1;
          510  +        }
          511  +        if( cnt>lenDelta ){
          512  +          /* ERROR: insert count exceeds size of delta */
          513  +          return -1;
          514  +        }
          515  +        memcpy(zOut, zDelta, cnt);
          516  +        zOut += cnt;
          517  +        zDelta += cnt;
          518  +        lenDelta -= cnt;
          519  +        break;
          520  +      }
          521  +      case ';': {
          522  +        zDelta++; lenDelta--;
          523  +        zOut[0] = 0;
          524  +#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST
          525  +        if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){
          526  +          /* ERROR:  bad checksum */
          527  +          return -1;
          528  +        }
          529  +#endif
          530  +        if( total!=limit ){
          531  +          /* ERROR: generated size does not match predicted size */
          532  +          return -1;
          533  +        }
          534  +        return total;
          535  +      }
          536  +      default: {
          537  +        /* ERROR: unknown delta operator */
          538  +        return -1;
          539  +      }
          540  +    }
          541  +  }
          542  +  /* ERROR: unterminated delta */
          543  +  return -1;
          544  +}
          545  +
          546  +static int rbuDeltaOutputSize(const char *zDelta, int lenDelta){
          547  +  int size;
          548  +  size = rbuDeltaGetInt(&zDelta, &lenDelta);
          549  +  if( *zDelta!='\n' ){
          550  +    /* ERROR: size integer not terminated by "\n" */
          551  +    return -1;
          552  +  }
          553  +  return size;
          554  +}
          555  +
          556  +/*
          557  +** End of code taken from fossil.
          558  +*************************************************************************/
          559  +
          560  +/*
          561  +** Implementation of SQL scalar function rbu_fossil_delta().
          562  +**
          563  +** This function applies a fossil delta patch to a blob. Exactly two
          564  +** arguments must be passed to this function. The first is the blob to
          565  +** patch and the second the patch to apply. If no error occurs, this
          566  +** function returns the patched blob.
          567  +*/
          568  +static void rbuFossilDeltaFunc(
          569  +  sqlite3_context *context,
          570  +  int argc,
          571  +  sqlite3_value **argv
          572  +){
          573  +  const char *aDelta;
          574  +  int nDelta;
          575  +  const char *aOrig;
          576  +  int nOrig;
          577  +
          578  +  int nOut;
          579  +  int nOut2;
          580  +  char *aOut;
          581  +
          582  +  assert( argc==2 );
          583  +
          584  +  nOrig = sqlite3_value_bytes(argv[0]);
          585  +  aOrig = (const char*)sqlite3_value_blob(argv[0]);
          586  +  nDelta = sqlite3_value_bytes(argv[1]);
          587  +  aDelta = (const char*)sqlite3_value_blob(argv[1]);
          588  +
          589  +  /* Figure out the size of the output */
          590  +  nOut = rbuDeltaOutputSize(aDelta, nDelta);
          591  +  if( nOut<0 ){
          592  +    sqlite3_result_error(context, "corrupt fossil delta", -1);
          593  +    return;
          594  +  }
          595  +
          596  +  aOut = sqlite3_malloc(nOut+1);
          597  +  if( aOut==0 ){
          598  +    sqlite3_result_error_nomem(context);
          599  +  }else{
          600  +    int nOut2 = rbuDeltaApply(aOrig, nOrig, aDelta, nDelta, aOut);
          601  +    if( nOut2!=nOut ){
          602  +      sqlite3_result_error(context, "corrupt fossil delta", -1);
          603  +    }else{
          604  +      sqlite3_result_blob(context, aOut, nOut, sqlite3_free);
          605  +    }
          606  +  }
          607  +}
          608  +
   363    609   
   364    610   /*
   365    611   ** Prepare the SQL statement in buffer zSql against database handle db.
   366    612   ** If successful, set *ppStmt to point to the new statement and return
   367    613   ** SQLITE_OK. 
   368    614   **
   369    615   ** Otherwise, if an error does occur, set *ppStmt to NULL and return
................................................................................
  1296   1542           char c = zMask[pIter->aiSrcOrder[i]];
  1297   1543           if( c=='x' ){
  1298   1544             zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", 
  1299   1545                 zList, zSep, pIter->azTblCol[i], i+1
  1300   1546             );
  1301   1547             zSep = ", ";
  1302   1548           }
  1303         -        if( c=='d' ){
         1549  +        else if( c=='d' ){
  1304   1550             zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_delta(\"%w\", ?%d)", 
  1305   1551                 zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1
  1306   1552             );
  1307   1553             zSep = ", ";
         1554  +        }
         1555  +        else if( c=='f' ){
         1556  +          zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_fossil_delta(\"%w\", ?%d)", 
         1557  +              zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1
         1558  +          );
         1559  +          zSep = ", ";
  1308   1560           }
  1309   1561         }
  1310   1562       }
  1311   1563     }
  1312   1564     return zList;
  1313   1565   }
  1314   1566   
................................................................................
  1895   2147     }
  1896   2148   
  1897   2149     if( p->rc==SQLITE_OK ){
  1898   2150       p->rc = sqlite3_create_function(p->dbMain, 
  1899   2151           "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0
  1900   2152       );
  1901   2153     }
         2154  +
         2155  +  if( p->rc==SQLITE_OK ){
         2156  +    p->rc = sqlite3_create_function(p->dbMain, 
         2157  +        "rbu_fossil_delta", 2, SQLITE_UTF8, 0, rbuFossilDeltaFunc, 0, 0
         2158  +    );
         2159  +  }
  1902   2160   
  1903   2161     if( p->rc==SQLITE_OK ){
  1904   2162       p->rc = sqlite3_create_function(p->dbRbu, 
  1905   2163           "rbu_target_name", 1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0
  1906   2164       );
  1907   2165     }
  1908   2166   
................................................................................
  2331   2589         sqlite3_stmt *pUpdate = 0;
  2332   2590         assert( eType==RBU_UPDATE );
  2333   2591         rbuGetUpdateStmt(p, pIter, zMask, &pUpdate);
  2334   2592         if( pUpdate ){
  2335   2593           for(i=0; p->rc==SQLITE_OK && i<pIter->nCol; i++){
  2336   2594             char c = zMask[pIter->aiSrcOrder[i]];
  2337   2595             pVal = sqlite3_column_value(pIter->pSelect, i);
  2338         -          if( pIter->abTblPk[i] || c=='x' || c=='d' ){
         2596  +          if( pIter->abTblPk[i] || c!='.' ){
  2339   2597               p->rc = sqlite3_bind_value(pUpdate, i+1, pVal);
  2340   2598             }
  2341   2599           }
  2342   2600           if( p->rc==SQLITE_OK 
  2343   2601            && (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) 
  2344   2602           ){
  2345   2603             /* Bind the rbu_rowid value to column _rowid_ */

Changes to ext/rbu/sqlite3rbu.h.

   184    184   ** For example, this row:
   185    185   **
   186    186   **   INSERT INTO data_t1(a, b, c, rbu_control) VALUES(4, NULL, 'usa', '..d');
   187    187   **
   188    188   ** is similar to an UPDATE statement such as: 
   189    189   **
   190    190   **   UPDATE t1 SET c = rbu_delta(c, 'usa') WHERE a = 4;
          191  +**
          192  +** Finally, if an 'f' character appears in place of a 'd' or 's' in an 
          193  +** ota_control string, the contents of the data_xxx table column is assumed
          194  +** to be a "fossil delta" - a patch to be applied to a blob value in the
          195  +** format used by the fossil source-code management system. In this case
          196  +** the existing value within the target database table must be of type BLOB. 
          197  +** It is replaced by the result of applying the specified fossil delta to
          198  +** itself.
   191    199   **
   192    200   ** If the target database table is a virtual table or a table with no PRIMARY
   193    201   ** KEY, the rbu_control value should not include a character corresponding 
   194    202   ** to the rbu_rowid value. For example, this:
   195    203   **
   196    204   **   INSERT INTO data_ft1(a, b, rbu_rowid, rbu_control) 
   197    205   **       VALUES(NULL, 'usa', 12, '.x');

Changes to tool/sqldiff.c.

    19     19   ** run the utility.
    20     20   */
    21     21   #include <stdio.h>
    22     22   #include <stdlib.h>
    23     23   #include <stdarg.h>
    24     24   #include <ctype.h>
    25     25   #include <string.h>
           26  +#include <assert.h>
    26     27   #include "sqlite3.h"
    27     28   
    28     29   /*
    29     30   ** All global variables are gathered into the "g" singleton.
    30     31   */
    31     32   struct GlobalVars {
    32     33     const char *zArgv0;       /* Name of program */
................................................................................
   750    751       }
   751    752     }else{
   752    753       runtimeError("table %s missing from one or both databases", safeId(zTab));
   753    754     }
   754    755     sqlite3_finalize(pStmt);
   755    756   }
   756    757   
          758  +/**************************************************************************
          759  +** The following code is copied from fossil. It is used to generate the
          760  +** fossil delta blobs sometimes used in RBU update records.
          761  +*/
          762  +
          763  +typedef unsigned short u16;
          764  +typedef unsigned int u32;
          765  +typedef unsigned char u8;
          766  +
          767  +/*
          768  +** The width of a hash window in bytes.  The algorithm only works if this
          769  +** is a power of 2.
          770  +*/
          771  +#define NHASH 16
          772  +
          773  +/*
          774  +** The current state of the rolling hash.
          775  +**
          776  +** z[] holds the values that have been hashed.  z[] is a circular buffer.
          777  +** z[i] is the first entry and z[(i+NHASH-1)%NHASH] is the last entry of
          778  +** the window.
          779  +**
          780  +** Hash.a is the sum of all elements of hash.z[].  Hash.b is a weighted
          781  +** sum.  Hash.b is z[i]*NHASH + z[i+1]*(NHASH-1) + ... + z[i+NHASH-1]*1.
          782  +** (Each index for z[] should be module NHASH, of course.  The %NHASH operator
          783  +** is omitted in the prior expression for brevity.)
          784  +*/
          785  +typedef struct hash hash;
          786  +struct hash {
          787  +  u16 a, b;         /* Hash values */
          788  +  u16 i;            /* Start of the hash window */
          789  +  char z[NHASH];    /* The values that have been hashed */
          790  +};
          791  +
          792  +/*
          793  +** Initialize the rolling hash using the first NHASH characters of z[]
          794  +*/
          795  +static void hash_init(hash *pHash, const char *z){
          796  +  u16 a, b, i;
          797  +  a = b = 0;
          798  +  for(i=0; i<NHASH; i++){
          799  +    a += z[i];
          800  +    b += (NHASH-i)*z[i];
          801  +    pHash->z[i] = z[i];
          802  +  }
          803  +  pHash->a = a & 0xffff;
          804  +  pHash->b = b & 0xffff;
          805  +  pHash->i = 0;
          806  +}
          807  +
          808  +/*
          809  +** Advance the rolling hash by a single character "c"
          810  +*/
          811  +static void hash_next(hash *pHash, int c){
          812  +  u16 old = pHash->z[pHash->i];
          813  +  pHash->z[pHash->i] = c;
          814  +  pHash->i = (pHash->i+1)&(NHASH-1);
          815  +  pHash->a = pHash->a - old + c;
          816  +  pHash->b = pHash->b - NHASH*old + pHash->a;
          817  +}
          818  +
          819  +/*
          820  +** Return a 32-bit hash value
          821  +*/
          822  +static u32 hash_32bit(hash *pHash){
          823  +  return (pHash->a & 0xffff) | (((u32)(pHash->b & 0xffff))<<16);
          824  +}
          825  +
          826  +/*
          827  +** Write an base-64 integer into the given buffer.
          828  +*/
          829  +static void putInt(unsigned int v, char **pz){
          830  +  static const char zDigits[] =
          831  +    "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~";
          832  +  /*  123456789 123456789 123456789 123456789 123456789 123456789 123 */
          833  +  int i, j;
          834  +  char zBuf[20];
          835  +  if( v==0 ){
          836  +    *(*pz)++ = '0';
          837  +    return;
          838  +  }
          839  +  for(i=0; v>0; i++, v>>=6){
          840  +    zBuf[i] = zDigits[v&0x3f];
          841  +  }
          842  +  for(j=i-1; j>=0; j--){
          843  +    *(*pz)++ = zBuf[j];
          844  +  }
          845  +}
          846  +
          847  +/*
          848  +** Read bytes from *pz and convert them into a positive integer.  When
          849  +** finished, leave *pz pointing to the first character past the end of
          850  +** the integer.  The *pLen parameter holds the length of the string
          851  +** in *pz and is decremented once for each character in the integer.
          852  +*/
          853  +static unsigned int getInt(const char **pz, int *pLen){
          854  +  static const signed char zValue[] = {
          855  +    -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
          856  +    -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
          857  +    -1, -1, -1, -1, -1, -1, -1, -1,   -1, -1, -1, -1, -1, -1, -1, -1,
          858  +     0,  1,  2,  3,  4,  5,  6,  7,    8,  9, -1, -1, -1, -1, -1, -1,
          859  +    -1, 10, 11, 12, 13, 14, 15, 16,   17, 18, 19, 20, 21, 22, 23, 24,
          860  +    25, 26, 27, 28, 29, 30, 31, 32,   33, 34, 35, -1, -1, -1, -1, 36,
          861  +    -1, 37, 38, 39, 40, 41, 42, 43,   44, 45, 46, 47, 48, 49, 50, 51,
          862  +    52, 53, 54, 55, 56, 57, 58, 59,   60, 61, 62, -1, -1, -1, 63, -1,
          863  +  };
          864  +  unsigned int v = 0;
          865  +  int c;
          866  +  unsigned char *z = (unsigned char*)*pz;
          867  +  unsigned char *zStart = z;
          868  +  while( (c = zValue[0x7f&*(z++)])>=0 ){
          869  +     v = (v<<6) + c;
          870  +  }
          871  +  z--;
          872  +  *pLen -= z - zStart;
          873  +  *pz = (char*)z;
          874  +  return v;
          875  +}
          876  +
          877  +/*
          878  +** Return the number digits in the base-64 representation of a positive integer
          879  +*/
          880  +static int digit_count(int v){
          881  +  unsigned int i, x;
          882  +  for(i=1, x=64; v>=x; i++, x <<= 6){}
          883  +  return i;
          884  +}
          885  +
          886  +/*
          887  +** Compute a 32-bit checksum on the N-byte buffer.  Return the result.
          888  +*/
          889  +static unsigned int checksum(const char *zIn, size_t N){
          890  +  const unsigned char *z = (const unsigned char *)zIn;
          891  +  unsigned sum0 = 0;
          892  +  unsigned sum1 = 0;
          893  +  unsigned sum2 = 0;
          894  +  unsigned sum3 = 0;
          895  +  while(N >= 16){
          896  +    sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]);
          897  +    sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]);
          898  +    sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]);
          899  +    sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]);
          900  +    z += 16;
          901  +    N -= 16;
          902  +  }
          903  +  while(N >= 4){
          904  +    sum0 += z[0];
          905  +    sum1 += z[1];
          906  +    sum2 += z[2];
          907  +    sum3 += z[3];
          908  +    z += 4;
          909  +    N -= 4;
          910  +  }
          911  +  sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24);
          912  +  switch(N){
          913  +    case 3:   sum3 += (z[2] << 8);
          914  +    case 2:   sum3 += (z[1] << 16);
          915  +    case 1:   sum3 += (z[0] << 24);
          916  +    default:  ;
          917  +  }
          918  +  return sum3;
          919  +}
          920  +
          921  +/*
          922  +** Create a new delta.
          923  +**
          924  +** The delta is written into a preallocated buffer, zDelta, which
          925  +** should be at least 60 bytes longer than the target file, zOut.
          926  +** The delta string will be NUL-terminated, but it might also contain
          927  +** embedded NUL characters if either the zSrc or zOut files are
          928  +** binary.  This function returns the length of the delta string
          929  +** in bytes, excluding the final NUL terminator character.
          930  +**
          931  +** Output Format:
          932  +**
          933  +** The delta begins with a base64 number followed by a newline.  This
          934  +** number is the number of bytes in the TARGET file.  Thus, given a
          935  +** delta file z, a program can compute the size of the output file
          936  +** simply by reading the first line and decoding the base-64 number
          937  +** found there.  The delta_output_size() routine does exactly this.
          938  +**
          939  +** After the initial size number, the delta consists of a series of
          940  +** literal text segments and commands to copy from the SOURCE file.
          941  +** A copy command looks like this:
          942  +**
          943  +**     NNN@MMM,
          944  +**
          945  +** where NNN is the number of bytes to be copied and MMM is the offset
          946  +** into the source file of the first byte (both base-64).   If NNN is 0
          947  +** it means copy the rest of the input file.  Literal text is like this:
          948  +**
          949  +**     NNN:TTTTT
          950  +**
          951  +** where NNN is the number of bytes of text (base-64) and TTTTT is the text.
          952  +**
          953  +** The last term is of the form
          954  +**
          955  +**     NNN;
          956  +**
          957  +** In this case, NNN is a 32-bit bigendian checksum of the output file
          958  +** that can be used to verify that the delta applied correctly.  All
          959  +** numbers are in base-64.
          960  +**
          961  +** Pure text files generate a pure text delta.  Binary files generate a
          962  +** delta that may contain some binary data.
          963  +**
          964  +** Algorithm:
          965  +**
          966  +** The encoder first builds a hash table to help it find matching
          967  +** patterns in the source file.  16-byte chunks of the source file
          968  +** sampled at evenly spaced intervals are used to populate the hash
          969  +** table.
          970  +**
          971  +** Next we begin scanning the target file using a sliding 16-byte
          972  +** window.  The hash of the 16-byte window in the target is used to
          973  +** search for a matching section in the source file.  When a match
          974  +** is found, a copy command is added to the delta.  An effort is
          975  +** made to extend the matching section to regions that come before
          976  +** and after the 16-byte hash window.  A copy command is only issued
          977  +** if the result would use less space that just quoting the text
          978  +** literally. Literal text is added to the delta for sections that
          979  +** do not match or which can not be encoded efficiently using copy
          980  +** commands.
          981  +*/
          982  +static int rbuDeltaCreate(
          983  +  const char *zSrc,      /* The source or pattern file */
          984  +  unsigned int lenSrc,   /* Length of the source file */
          985  +  const char *zOut,      /* The target file */
          986  +  unsigned int lenOut,   /* Length of the target file */
          987  +  char *zDelta           /* Write the delta into this buffer */
          988  +){
          989  +  int i, base;
          990  +  char *zOrigDelta = zDelta;
          991  +  hash h;
          992  +  int nHash;                 /* Number of hash table entries */
          993  +  int *landmark;             /* Primary hash table */
          994  +  int *collide;              /* Collision chain */
          995  +  int lastRead = -1;         /* Last byte of zSrc read by a COPY command */
          996  +
          997  +  /* Add the target file size to the beginning of the delta
          998  +  */
          999  +  putInt(lenOut, &zDelta);
         1000  +  *(zDelta++) = '\n';
         1001  +
         1002  +  /* If the source file is very small, it means that we have no
         1003  +  ** chance of ever doing a copy command.  Just output a single
         1004  +  ** literal segment for the entire target and exit.
         1005  +  */
         1006  +  if( lenSrc<=NHASH ){
         1007  +    putInt(lenOut, &zDelta);
         1008  +    *(zDelta++) = ':';
         1009  +    memcpy(zDelta, zOut, lenOut);
         1010  +    zDelta += lenOut;
         1011  +    putInt(checksum(zOut, lenOut), &zDelta);
         1012  +    *(zDelta++) = ';';
         1013  +    return zDelta - zOrigDelta;
         1014  +  }
         1015  +
         1016  +  /* Compute the hash table used to locate matching sections in the
         1017  +  ** source file.
         1018  +  */
         1019  +  nHash = lenSrc/NHASH;
         1020  +  collide = sqlite3_malloc( nHash*2*sizeof(int) );
         1021  +  landmark = &collide[nHash];
         1022  +  memset(landmark, -1, nHash*sizeof(int));
         1023  +  memset(collide, -1, nHash*sizeof(int));
         1024  +  for(i=0; i<lenSrc-NHASH; i+=NHASH){
         1025  +    int hv;
         1026  +    hash_init(&h, &zSrc[i]);
         1027  +    hv = hash_32bit(&h) % nHash;
         1028  +    collide[i/NHASH] = landmark[hv];
         1029  +    landmark[hv] = i/NHASH;
         1030  +  }
         1031  +
         1032  +  /* Begin scanning the target file and generating copy commands and
         1033  +  ** literal sections of the delta.
         1034  +  */
         1035  +  base = 0;    /* We have already generated everything before zOut[base] */
         1036  +  while( base+NHASH<lenOut ){
         1037  +    int iSrc, iBlock;
         1038  +    unsigned int bestCnt, bestOfst=0, bestLitsz=0;
         1039  +    hash_init(&h, &zOut[base]);
         1040  +    i = 0;     /* Trying to match a landmark against zOut[base+i] */
         1041  +    bestCnt = 0;
         1042  +    while( 1 ){
         1043  +      int hv;
         1044  +      int limit = 250;
         1045  +
         1046  +      hv = hash_32bit(&h) % nHash;
         1047  +      iBlock = landmark[hv];
         1048  +      while( iBlock>=0 && (limit--)>0 ){
         1049  +        /*
         1050  +        ** The hash window has identified a potential match against
         1051  +        ** landmark block iBlock.  But we need to investigate further.
         1052  +        **
         1053  +        ** Look for a region in zOut that matches zSrc. Anchor the search
         1054  +        ** at zSrc[iSrc] and zOut[base+i].  Do not include anything prior to
         1055  +        ** zOut[base] or after zOut[outLen] nor anything after zSrc[srcLen].
         1056  +        **
         1057  +        ** Set cnt equal to the length of the match and set ofst so that
         1058  +        ** zSrc[ofst] is the first element of the match.  litsz is the number
         1059  +        ** of characters between zOut[base] and the beginning of the match.
         1060  +        ** sz will be the overhead (in bytes) needed to encode the copy
         1061  +        ** command.  Only generate copy command if the overhead of the
         1062  +        ** copy command is less than the amount of literal text to be copied.
         1063  +        */
         1064  +        int cnt, ofst, litsz;
         1065  +        int j, k, x, y;
         1066  +        int sz;
         1067  +
         1068  +        /* Beginning at iSrc, match forwards as far as we can.  j counts
         1069  +        ** the number of characters that match */
         1070  +        iSrc = iBlock*NHASH;
         1071  +        for(j=0, x=iSrc, y=base+i; x<lenSrc && y<lenOut; j++, x++, y++){
         1072  +          if( zSrc[x]!=zOut[y] ) break;
         1073  +        }
         1074  +        j--;
         1075  +
         1076  +        /* Beginning at iSrc-1, match backwards as far as we can.  k counts
         1077  +        ** the number of characters that match */
         1078  +        for(k=1; k<iSrc && k<=i; k++){
         1079  +          if( zSrc[iSrc-k]!=zOut[base+i-k] ) break;
         1080  +        }
         1081  +        k--;
         1082  +
         1083  +        /* Compute the offset and size of the matching region */
         1084  +        ofst = iSrc-k;
         1085  +        cnt = j+k+1;
         1086  +        litsz = i-k;  /* Number of bytes of literal text before the copy */
         1087  +        /* sz will hold the number of bytes needed to encode the "insert"
         1088  +        ** command and the copy command, not counting the "insert" text */
         1089  +        sz = digit_count(i-k)+digit_count(cnt)+digit_count(ofst)+3;
         1090  +        if( cnt>=sz && cnt>bestCnt ){
         1091  +          /* Remember this match only if it is the best so far and it
         1092  +          ** does not increase the file size */
         1093  +          bestCnt = cnt;
         1094  +          bestOfst = iSrc-k;
         1095  +          bestLitsz = litsz;
         1096  +        }
         1097  +
         1098  +        /* Check the next matching block */
         1099  +        iBlock = collide[iBlock];
         1100  +      }
         1101  +
         1102  +      /* We have a copy command that does not cause the delta to be larger
         1103  +      ** than a literal insert.  So add the copy command to the delta.
         1104  +      */
         1105  +      if( bestCnt>0 ){
         1106  +        if( bestLitsz>0 ){
         1107  +          /* Add an insert command before the copy */
         1108  +          putInt(bestLitsz,&zDelta);
         1109  +          *(zDelta++) = ':';
         1110  +          memcpy(zDelta, &zOut[base], bestLitsz);
         1111  +          zDelta += bestLitsz;
         1112  +          base += bestLitsz;
         1113  +        }
         1114  +        base += bestCnt;
         1115  +        putInt(bestCnt, &zDelta);
         1116  +        *(zDelta++) = '@';
         1117  +        putInt(bestOfst, &zDelta);
         1118  +        *(zDelta++) = ',';
         1119  +        if( bestOfst + bestCnt -1 > lastRead ){
         1120  +          lastRead = bestOfst + bestCnt - 1;
         1121  +        }
         1122  +        bestCnt = 0;
         1123  +        break;
         1124  +      }
         1125  +
         1126  +      /* If we reach this point, it means no match is found so far */
         1127  +      if( base+i+NHASH>=lenOut ){
         1128  +        /* We have reached the end of the file and have not found any
         1129  +        ** matches.  Do an "insert" for everything that does not match */
         1130  +        putInt(lenOut-base, &zDelta);
         1131  +        *(zDelta++) = ':';
         1132  +        memcpy(zDelta, &zOut[base], lenOut-base);
         1133  +        zDelta += lenOut-base;
         1134  +        base = lenOut;
         1135  +        break;
         1136  +      }
         1137  +
         1138  +      /* Advance the hash by one character.  Keep looking for a match */
         1139  +      hash_next(&h, zOut[base+i+NHASH]);
         1140  +      i++;
         1141  +    }
         1142  +  }
         1143  +  /* Output a final "insert" record to get all the text at the end of
         1144  +  ** the file that does not match anything in the source file.
         1145  +  */
         1146  +  if( base<lenOut ){
         1147  +    putInt(lenOut-base, &zDelta);
         1148  +    *(zDelta++) = ':';
         1149  +    memcpy(zDelta, &zOut[base], lenOut-base);
         1150  +    zDelta += lenOut-base;
         1151  +  }
         1152  +  /* Output the final checksum record. */
         1153  +  putInt(checksum(zOut, lenOut), &zDelta);
         1154  +  *(zDelta++) = ';';
         1155  +  sqlite3_free(collide);
         1156  +  return zDelta - zOrigDelta;
         1157  +}
         1158  +
         1159  +/*
         1160  +** End of code copied from fossil.
         1161  +**************************************************************************/
         1162  +
   757   1163   static void strPrintfArray(
   758   1164     Str *pStr,                      /* String object to append to */
   759   1165     const char *zSep,               /* Separator string */
   760   1166     const char *zFmt,               /* Format for each entry */
   761   1167     char **az, int n                /* Array of strings & its size (or -1) */
   762   1168   ){
   763   1169     int i;
................................................................................
   775   1181     Str *pSql
   776   1182   ){
   777   1183     int i;
   778   1184   
   779   1185     /* First the newly inserted rows: **/ 
   780   1186     strPrintf(pSql, "SELECT ");
   781   1187     strPrintfArray(pSql, ", ", "%s", azCol, -1);
   782         -  strPrintf(pSql, ", 0");         /* Set ota_control to 0 for an insert */
         1188  +  strPrintf(pSql, ", 0, ");       /* Set ota_control to 0 for an insert */
         1189  +  strPrintfArray(pSql, ", ", "NULL", azCol, -1);
   783   1190     strPrintf(pSql, " FROM aux.%Q AS n WHERE NOT EXISTS (\n", zTab);
   784   1191     strPrintf(pSql, "    SELECT 1 FROM ", zTab);
   785   1192     strPrintf(pSql, " main.%Q AS o WHERE ", zTab);
   786   1193     strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
   787   1194     strPrintf(pSql, "\n)");
   788   1195   
   789   1196     /* Deleted rows: */
   790   1197     strPrintf(pSql, "\nUNION ALL\nSELECT ");
   791   1198     strPrintfArray(pSql, ", ", "%s", azCol, nPK);
   792   1199     if( azCol[nPK] ){
   793   1200       strPrintf(pSql, ", ");
   794   1201       strPrintfArray(pSql, ", ", "NULL", &azCol[nPK], -1);
   795   1202     }
   796         -  strPrintf(pSql, ", 1");         /* Set ota_control to 1 for a delete */
         1203  +  strPrintf(pSql, ", 1, ");       /* Set ota_control to 1 for a delete */
         1204  +  strPrintfArray(pSql, ", ", "NULL", azCol, -1);
   797   1205     strPrintf(pSql, " FROM main.%Q AS n WHERE NOT EXISTS (\n", zTab);
   798   1206     strPrintf(pSql, "    SELECT 1 FROM ", zTab);
   799   1207     strPrintf(pSql, " aux.%Q AS o WHERE ", zTab);
   800   1208     strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
   801   1209     strPrintf(pSql, "\n) ");
   802   1210   
   803   1211     /* Updated rows. If all table columns are part of the primary key, there 
................................................................................
   817   1225         strPrintf(pSql, "' ||\n");
   818   1226       }else{
   819   1227         strPrintf(pSql, ",\n");
   820   1228       }
   821   1229       strPrintfArray(pSql, " ||\n", 
   822   1230           "    CASE WHEN n.%s IS o.%s THEN '.' ELSE 'x' END", &azCol[nPK], -1
   823   1231       );
   824         -    strPrintf(pSql, "\nAS ota_control");
         1232  +    strPrintf(pSql, "\nAS ota_control, ");
         1233  +    strPrintfArray(pSql, ", ", "NULL", azCol, nPK);
         1234  +    strPrintf(pSql, ",\n");
         1235  +    strPrintfArray(pSql, " ,\n", 
         1236  +        "    CASE WHEN n.%s IS o.%s THEN NULL ELSE o.%s END", &azCol[nPK], -1
         1237  +    );
   825   1238   
   826   1239       strPrintf(pSql, "\nFROM main.%Q AS o, aux.%Q AS n\nWHERE ", zTab, zTab);
   827   1240       strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
   828   1241       strPrintf(pSql, " AND ota_control LIKE '%%x%%'");
   829   1242     }
   830   1243   
   831   1244     /* Now add an ORDER BY clause to sort everything by PK. */
................................................................................
   852   1265   
   853   1266     /* Grab the column names and PK details for the table(s). If no usable PK
   854   1267     ** columns are found, bail out early.  */
   855   1268     azCol = columnNames("main", zTab, &nPK, &bOtaRowid);
   856   1269     if( azCol==0 ){
   857   1270       runtimeError("table %s has no usable PK columns", zTab);
   858   1271     }
         1272  +  for(nCol=0; azCol[nCol]; nCol++);
   859   1273   
   860   1274     /* Build and output the CREATE TABLE statement for the data_xxx table */
   861   1275     strPrintf(&ct, "CREATE TABLE IF NOT EXISTS 'data_%q'(", zTab);
   862   1276     if( bOtaRowid ) strPrintf(&ct, "rbu_rowid, ");
   863   1277     strPrintfArray(&ct, ", ", "%s", &azCol[bOtaRowid], -1);
   864   1278     strPrintf(&ct, ", rbu_control);");
   865         -
   866   1279   
   867   1280     /* Get the SQL for the query to retrieve data from the two databases */
   868   1281     getRbudiffQuery(zTab, azCol, nPK, bOtaRowid, &sql);
   869   1282   
   870   1283     /* Build the first part of the INSERT statement output for each row
   871   1284     ** in the data_xxx table. */
   872   1285     strPrintf(&insert, "INSERT INTO 'data_%q' (", zTab);
   873   1286     if( bOtaRowid ) strPrintf(&insert, "rbu_rowid, ");
   874   1287     strPrintfArray(&insert, ", ", "%s", &azCol[bOtaRowid], -1);
   875   1288     strPrintf(&insert, ", rbu_control) VALUES(");
   876   1289   
   877   1290     pStmt = db_prepare("%s", sql.z);
   878         -  nCol = sqlite3_column_count(pStmt);
         1291  +
   879   1292     while( sqlite3_step(pStmt)==SQLITE_ROW ){
         1293  +    
         1294  +    /* If this is the first row output, print out the CREATE TABLE 
         1295  +    ** statement first. And then set ct.z to NULL so that it is not 
         1296  +    ** printed again.  */
   880   1297       if( ct.z ){
   881   1298         fprintf(out, "%s\n", ct.z);
   882   1299         strFree(&ct);
   883   1300       }
   884   1301   
         1302  +    /* Output the first part of the INSERT statement */
   885   1303       fprintf(out, "%s", insert.z);
   886         -    for(i=0; i<nCol; i++){
   887         -      if( i>0 ) fprintf(out, ", ");
   888         -      printQuoted(out, sqlite3_column_value(pStmt, i));
         1304  +
         1305  +    if( sqlite3_column_type(pStmt, nCol)==SQLITE_INTEGER ){
         1306  +      for(i=0; i<=nCol; i++){
         1307  +        if( i>0 ) fprintf(out, ", ");
         1308  +        printQuoted(out, sqlite3_column_value(pStmt, i));
         1309  +      }
         1310  +    }else{
         1311  +      char *zOtaControl;
         1312  +      int nOtaControl = sqlite3_column_bytes(pStmt, nCol);
         1313  +
         1314  +      zOtaControl = (char*)sqlite3_malloc(nOtaControl);
         1315  +      memcpy(zOtaControl, sqlite3_column_text(pStmt, nCol), nOtaControl+1);
         1316  +
         1317  +      for(i=0; i<nCol; i++){
         1318  +        int bDone = 0;
         1319  +        if( i>=nPK 
         1320  +            && sqlite3_column_type(pStmt, i)==SQLITE_BLOB
         1321  +            && sqlite3_column_type(pStmt, nCol+1+i)==SQLITE_BLOB
         1322  +        ){
         1323  +          const char *aSrc = sqlite3_column_blob(pStmt, nCol+1+i);
         1324  +          int nSrc = sqlite3_column_bytes(pStmt, nCol+1+i);
         1325  +          const char *aFinal = sqlite3_column_blob(pStmt, i);
         1326  +          int nFinal = sqlite3_column_bytes(pStmt, i);
         1327  +          char *aDelta;
         1328  +          int nDelta;
         1329  +
         1330  +          aDelta = sqlite3_malloc(nFinal + 60);
         1331  +          nDelta = rbuDeltaCreate(aSrc, nSrc, aFinal, nFinal, aDelta);
         1332  +          if( nDelta<nFinal ){
         1333  +            int j;
         1334  +            fprintf(out, "x'");
         1335  +            for(j=0; j<nDelta; j++) fprintf(out, "%02x", (u8)aDelta[j]);
         1336  +            fprintf(out, "'");
         1337  +            zOtaControl[i-bOtaRowid] = 'f';
         1338  +            bDone = 1;
         1339  +          }
         1340  +          sqlite3_free(aDelta);
         1341  +        }
         1342  +
         1343  +        if( bDone==0 ){
         1344  +          printQuoted(out, sqlite3_column_value(pStmt, i));
         1345  +        }
         1346  +        fprintf(out, ", ");
         1347  +      }
         1348  +      fprintf(out, "'%s'", zOtaControl);
         1349  +      sqlite3_free(zOtaControl);
   889   1350       }
         1351  +
         1352  +    /* And the closing bracket of the insert statement */
   890   1353       fprintf(out, ");\n");
   891   1354     }
   892   1355   
   893   1356     sqlite3_finalize(pStmt);
   894   1357   
   895   1358     strFree(&ct);
   896   1359     strFree(&sql);