/ Check-in [e91012b6]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Backport the stack-size reduction patch in (3673) to version 3.3.13. (CVS 3979)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | branch_3_3_13
Files: files | file ages | folders
SHA1:e91012b603014765523cda77a4dda93f938c7358
User & Date: drh 2007-05-10 21:31:40
Context
2007-05-10
21:31
Backport the stack-size reduction patch in (3673) to version 3.3.13. (CVS 3979) Leaf check-in: e91012b6 user: drh tags: branch_3_3_13
2007-02-13
12:49
Expand the cursor index in SrcList_item to 32-bits to accomodate really huge VDBE programs resulting from deeply nested triggers. This is an interim fix until we rework triggers to use subroutines. (CVS 3640) check-in: 22769e2d user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Deleted ext/fts1/ft_hash.c.

     1         -/*
     2         -** 2001 September 22
     3         -**
     4         -** The author disclaims copyright to this source code.  In place of
     5         -** a legal notice, here is a blessing:
     6         -**
     7         -**    May you do good and not evil.
     8         -**    May you find forgiveness for yourself and forgive others.
     9         -**    May you share freely, never taking more than you give.
    10         -**
    11         -*************************************************************************
    12         -** This is the implementation of generic hash-tables used in SQLite.
    13         -** We've modified it slightly to serve as a standalone hash table
    14         -** implementation for the full-text indexing module.
    15         -*/
    16         -#include <assert.h>
    17         -#include <stdlib.h>
    18         -#include <string.h>
    19         -
    20         -#include "ft_hash.h"
    21         -
    22         -void *malloc_and_zero(int n){
    23         -  void *p = malloc(n);
    24         -  if( p ){
    25         -    memset(p, 0, n);
    26         -  }
    27         -  return p;
    28         -}
    29         -
    30         -/* Turn bulk memory into a hash table object by initializing the
    31         -** fields of the Hash structure.
    32         -**
    33         -** "pNew" is a pointer to the hash table that is to be initialized.
    34         -** keyClass is one of the constants HASH_INT, HASH_POINTER,
    35         -** HASH_BINARY, or HASH_STRING.  The value of keyClass 
    36         -** determines what kind of key the hash table will use.  "copyKey" is
    37         -** true if the hash table should make its own private copy of keys and
    38         -** false if it should just use the supplied pointer.  CopyKey only makes
    39         -** sense for HASH_STRING and HASH_BINARY and is ignored
    40         -** for other key classes.
    41         -*/
    42         -void HashInit(Hash *pNew, int keyClass, int copyKey){
    43         -  assert( pNew!=0 );
    44         -  assert( keyClass>=HASH_STRING && keyClass<=HASH_BINARY );
    45         -  pNew->keyClass = keyClass;
    46         -#if 0
    47         -  if( keyClass==HASH_POINTER || keyClass==HASH_INT ) copyKey = 0;
    48         -#endif
    49         -  pNew->copyKey = copyKey;
    50         -  pNew->first = 0;
    51         -  pNew->count = 0;
    52         -  pNew->htsize = 0;
    53         -  pNew->ht = 0;
    54         -  pNew->xMalloc = malloc_and_zero;
    55         -  pNew->xFree = free;
    56         -}
    57         -
    58         -/* Remove all entries from a hash table.  Reclaim all memory.
    59         -** Call this routine to delete a hash table or to reset a hash table
    60         -** to the empty state.
    61         -*/
    62         -void HashClear(Hash *pH){
    63         -  HashElem *elem;         /* For looping over all elements of the table */
    64         -
    65         -  assert( pH!=0 );
    66         -  elem = pH->first;
    67         -  pH->first = 0;
    68         -  if( pH->ht ) pH->xFree(pH->ht);
    69         -  pH->ht = 0;
    70         -  pH->htsize = 0;
    71         -  while( elem ){
    72         -    HashElem *next_elem = elem->next;
    73         -    if( pH->copyKey && elem->pKey ){
    74         -      pH->xFree(elem->pKey);
    75         -    }
    76         -    pH->xFree(elem);
    77         -    elem = next_elem;
    78         -  }
    79         -  pH->count = 0;
    80         -}
    81         -
    82         -#if 0 /* NOT USED */
    83         -/*
    84         -** Hash and comparison functions when the mode is HASH_INT
    85         -*/
    86         -static int intHash(const void *pKey, int nKey){
    87         -  return nKey ^ (nKey<<8) ^ (nKey>>8);
    88         -}
    89         -static int intCompare(const void *pKey1, int n1, const void *pKey2, int n2){
    90         -  return n2 - n1;
    91         -}
    92         -#endif
    93         -
    94         -#if 0 /* NOT USED */
    95         -/*
    96         -** Hash and comparison functions when the mode is HASH_POINTER
    97         -*/
    98         -static int ptrHash(const void *pKey, int nKey){
    99         -  uptr x = Addr(pKey);
   100         -  return x ^ (x<<8) ^ (x>>8);
   101         -}
   102         -static int ptrCompare(const void *pKey1, int n1, const void *pKey2, int n2){
   103         -  if( pKey1==pKey2 ) return 0;
   104         -  if( pKey1<pKey2 ) return -1;
   105         -  return 1;
   106         -}
   107         -#endif
   108         -
   109         -/*
   110         -** Hash and comparison functions when the mode is HASH_STRING
   111         -*/
   112         -static int strHash(const void *pKey, int nKey){
   113         -  const char *z = (const char *)pKey;
   114         -  int h = 0;
   115         -  if( nKey<=0 ) nKey = (int) strlen(z);
   116         -  while( nKey > 0  ){
   117         -    h = (h<<3) ^ h ^ *z++;
   118         -    nKey--;
   119         -  }
   120         -  return h & 0x7fffffff;
   121         -}
   122         -static int strCompare(const void *pKey1, int n1, const void *pKey2, int n2){
   123         -  if( n1!=n2 ) return 1;
   124         -  return strncmp((const char*)pKey1,(const char*)pKey2,n1);
   125         -}
   126         -
   127         -/*
   128         -** Hash and comparison functions when the mode is HASH_BINARY
   129         -*/
   130         -static int binHash(const void *pKey, int nKey){
   131         -  int h = 0;
   132         -  const char *z = (const char *)pKey;
   133         -  while( nKey-- > 0 ){
   134         -    h = (h<<3) ^ h ^ *(z++);
   135         -  }
   136         -  return h & 0x7fffffff;
   137         -}
   138         -static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){
   139         -  if( n1!=n2 ) return 1;
   140         -  return memcmp(pKey1,pKey2,n1);
   141         -}
   142         -
   143         -/*
   144         -** Return a pointer to the appropriate hash function given the key class.
   145         -**
   146         -** The C syntax in this function definition may be unfamilar to some 
   147         -** programmers, so we provide the following additional explanation:
   148         -**
   149         -** The name of the function is "hashFunction".  The function takes a
   150         -** single parameter "keyClass".  The return value of hashFunction()
   151         -** is a pointer to another function.  Specifically, the return value
   152         -** of hashFunction() is a pointer to a function that takes two parameters
   153         -** with types "const void*" and "int" and returns an "int".
   154         -*/
   155         -static int (*hashFunction(int keyClass))(const void*,int){
   156         -#if 0  /* HASH_INT and HASH_POINTER are never used */
   157         -  switch( keyClass ){
   158         -    case HASH_INT:     return &intHash;
   159         -    case HASH_POINTER: return &ptrHash;
   160         -    case HASH_STRING:  return &strHash;
   161         -    case HASH_BINARY:  return &binHash;;
   162         -    default: break;
   163         -  }
   164         -  return 0;
   165         -#else
   166         -  if( keyClass==HASH_STRING ){
   167         -    return &strHash;
   168         -  }else{
   169         -    assert( keyClass==HASH_BINARY );
   170         -    return &binHash;
   171         -  }
   172         -#endif
   173         -}
   174         -
   175         -/*
   176         -** Return a pointer to the appropriate hash function given the key class.
   177         -**
   178         -** For help in interpreted the obscure C code in the function definition,
   179         -** see the header comment on the previous function.
   180         -*/
   181         -static int (*compareFunction(int keyClass))(const void*,int,const void*,int){
   182         -#if 0 /* HASH_INT and HASH_POINTER are never used */
   183         -  switch( keyClass ){
   184         -    case HASH_INT:     return &intCompare;
   185         -    case HASH_POINTER: return &ptrCompare;
   186         -    case HASH_STRING:  return &strCompare;
   187         -    case HASH_BINARY:  return &binCompare;
   188         -    default: break;
   189         -  }
   190         -  return 0;
   191         -#else
   192         -  if( keyClass==HASH_STRING ){
   193         -    return &strCompare;
   194         -  }else{
   195         -    assert( keyClass==HASH_BINARY );
   196         -    return &binCompare;
   197         -  }
   198         -#endif
   199         -}
   200         -
   201         -/* Link an element into the hash table
   202         -*/
   203         -static void insertElement(
   204         -  Hash *pH,              /* The complete hash table */
   205         -  struct _ht *pEntry,    /* The entry into which pNew is inserted */
   206         -  HashElem *pNew         /* The element to be inserted */
   207         -){
   208         -  HashElem *pHead;       /* First element already in pEntry */
   209         -  pHead = pEntry->chain;
   210         -  if( pHead ){
   211         -    pNew->next = pHead;
   212         -    pNew->prev = pHead->prev;
   213         -    if( pHead->prev ){ pHead->prev->next = pNew; }
   214         -    else             { pH->first = pNew; }
   215         -    pHead->prev = pNew;
   216         -  }else{
   217         -    pNew->next = pH->first;
   218         -    if( pH->first ){ pH->first->prev = pNew; }
   219         -    pNew->prev = 0;
   220         -    pH->first = pNew;
   221         -  }
   222         -  pEntry->count++;
   223         -  pEntry->chain = pNew;
   224         -}
   225         -
   226         -
   227         -/* Resize the hash table so that it cantains "new_size" buckets.
   228         -** "new_size" must be a power of 2.  The hash table might fail 
   229         -** to resize if sqliteMalloc() fails.
   230         -*/
   231         -static void rehash(Hash *pH, int new_size){
   232         -  struct _ht *new_ht;            /* The new hash table */
   233         -  HashElem *elem, *next_elem;    /* For looping over existing elements */
   234         -  int (*xHash)(const void*,int); /* The hash function */
   235         -
   236         -  assert( (new_size & (new_size-1))==0 );
   237         -  new_ht = (struct _ht *)pH->xMalloc( new_size*sizeof(struct _ht) );
   238         -  if( new_ht==0 ) return;
   239         -  if( pH->ht ) pH->xFree(pH->ht);
   240         -  pH->ht = new_ht;
   241         -  pH->htsize = new_size;
   242         -  xHash = hashFunction(pH->keyClass);
   243         -  for(elem=pH->first, pH->first=0; elem; elem = next_elem){
   244         -    int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
   245         -    next_elem = elem->next;
   246         -    insertElement(pH, &new_ht[h], elem);
   247         -  }
   248         -}
   249         -
   250         -/* This function (for internal use only) locates an element in an
   251         -** hash table that matches the given key.  The hash for this key has
   252         -** already been computed and is passed as the 4th parameter.
   253         -*/
   254         -static HashElem *findElementGivenHash(
   255         -  const Hash *pH,     /* The pH to be searched */
   256         -  const void *pKey,   /* The key we are searching for */
   257         -  int nKey,
   258         -  int h               /* The hash for this key. */
   259         -){
   260         -  HashElem *elem;                /* Used to loop thru the element list */
   261         -  int count;                     /* Number of elements left to test */
   262         -  int (*xCompare)(const void*,int,const void*,int);  /* comparison function */
   263         -
   264         -  if( pH->ht ){
   265         -    struct _ht *pEntry = &pH->ht[h];
   266         -    elem = pEntry->chain;
   267         -    count = pEntry->count;
   268         -    xCompare = compareFunction(pH->keyClass);
   269         -    while( count-- && elem ){
   270         -      if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ 
   271         -        return elem;
   272         -      }
   273         -      elem = elem->next;
   274         -    }
   275         -  }
   276         -  return 0;
   277         -}
   278         -
   279         -/* Remove a single entry from the hash table given a pointer to that
   280         -** element and a hash on the element's key.
   281         -*/
   282         -static void removeElementGivenHash(
   283         -  Hash *pH,         /* The pH containing "elem" */
   284         -  HashElem* elem,   /* The element to be removed from the pH */
   285         -  int h             /* Hash value for the element */
   286         -){
   287         -  struct _ht *pEntry;
   288         -  if( elem->prev ){
   289         -    elem->prev->next = elem->next; 
   290         -  }else{
   291         -    pH->first = elem->next;
   292         -  }
   293         -  if( elem->next ){
   294         -    elem->next->prev = elem->prev;
   295         -  }
   296         -  pEntry = &pH->ht[h];
   297         -  if( pEntry->chain==elem ){
   298         -    pEntry->chain = elem->next;
   299         -  }
   300         -  pEntry->count--;
   301         -  if( pEntry->count<=0 ){
   302         -    pEntry->chain = 0;
   303         -  }
   304         -  if( pH->copyKey && elem->pKey ){
   305         -    pH->xFree(elem->pKey);
   306         -  }
   307         -  pH->xFree( elem );
   308         -  pH->count--;
   309         -  if( pH->count<=0 ){
   310         -    assert( pH->first==0 );
   311         -    assert( pH->count==0 );
   312         -    HashClear(pH);
   313         -  }
   314         -}
   315         -
   316         -/* Attempt to locate an element of the hash table pH with a key
   317         -** that matches pKey,nKey.  Return the data for this element if it is
   318         -** found, or NULL if there is no match.
   319         -*/
   320         -void *HashFind(const Hash *pH, const void *pKey, int nKey){
   321         -  int h;             /* A hash on key */
   322         -  HashElem *elem;    /* The element that matches key */
   323         -  int (*xHash)(const void*,int);  /* The hash function */
   324         -
   325         -  if( pH==0 || pH->ht==0 ) return 0;
   326         -  xHash = hashFunction(pH->keyClass);
   327         -  assert( xHash!=0 );
   328         -  h = (*xHash)(pKey,nKey);
   329         -  assert( (pH->htsize & (pH->htsize-1))==0 );
   330         -  elem = findElementGivenHash(pH,pKey,nKey, h & (pH->htsize-1));
   331         -  return elem ? elem->data : 0;
   332         -}
   333         -
   334         -/* Insert an element into the hash table pH.  The key is pKey,nKey
   335         -** and the data is "data".
   336         -**
   337         -** If no element exists with a matching key, then a new
   338         -** element is created.  A copy of the key is made if the copyKey
   339         -** flag is set.  NULL is returned.
   340         -**
   341         -** If another element already exists with the same key, then the
   342         -** new data replaces the old data and the old data is returned.
   343         -** The key is not copied in this instance.  If a malloc fails, then
   344         -** the new data is returned and the hash table is unchanged.
   345         -**
   346         -** If the "data" parameter to this function is NULL, then the
   347         -** element corresponding to "key" is removed from the hash table.
   348         -*/
   349         -void *HashInsert(Hash *pH, const void *pKey, int nKey, void *data){
   350         -  int hraw;             /* Raw hash value of the key */
   351         -  int h;                /* the hash of the key modulo hash table size */
   352         -  HashElem *elem;       /* Used to loop thru the element list */
   353         -  HashElem *new_elem;   /* New element added to the pH */
   354         -  int (*xHash)(const void*,int);  /* The hash function */
   355         -
   356         -  assert( pH!=0 );
   357         -  xHash = hashFunction(pH->keyClass);
   358         -  assert( xHash!=0 );
   359         -  hraw = (*xHash)(pKey, nKey);
   360         -  assert( (pH->htsize & (pH->htsize-1))==0 );
   361         -  h = hraw & (pH->htsize-1);
   362         -  elem = findElementGivenHash(pH,pKey,nKey,h);
   363         -  if( elem ){
   364         -    void *old_data = elem->data;
   365         -    if( data==0 ){
   366         -      removeElementGivenHash(pH,elem,h);
   367         -    }else{
   368         -      elem->data = data;
   369         -    }
   370         -    return old_data;
   371         -  }
   372         -  if( data==0 ) return 0;
   373         -  new_elem = (HashElem*)pH->xMalloc( sizeof(HashElem) );
   374         -  if( new_elem==0 ) return data;
   375         -  if( pH->copyKey && pKey!=0 ){
   376         -    new_elem->pKey = pH->xMalloc( nKey );
   377         -    if( new_elem->pKey==0 ){
   378         -      pH->xFree(new_elem);
   379         -      return data;
   380         -    }
   381         -    memcpy((void*)new_elem->pKey, pKey, nKey);
   382         -  }else{
   383         -    new_elem->pKey = (void*)pKey;
   384         -  }
   385         -  new_elem->nKey = nKey;
   386         -  pH->count++;
   387         -  if( pH->htsize==0 ){
   388         -    rehash(pH,8);
   389         -    if( pH->htsize==0 ){
   390         -      pH->count = 0;
   391         -      pH->xFree(new_elem);
   392         -      return data;
   393         -    }
   394         -  }
   395         -  if( pH->count > pH->htsize ){
   396         -    rehash(pH,pH->htsize*2);
   397         -  }
   398         -  assert( pH->htsize>0 );
   399         -  assert( (pH->htsize & (pH->htsize-1))==0 );
   400         -  h = hraw & (pH->htsize-1);
   401         -  insertElement(pH, &pH->ht[h], new_elem);
   402         -  new_elem->data = data;
   403         -  return 0;
   404         -}

Deleted ext/fts1/ft_hash.h.

     1         -/*
     2         -** 2001 September 22
     3         -**
     4         -** The author disclaims copyright to this source code.  In place of
     5         -** a legal notice, here is a blessing:
     6         -**
     7         -**    May you do good and not evil.
     8         -**    May you find forgiveness for yourself and forgive others.
     9         -**    May you share freely, never taking more than you give.
    10         -**
    11         -*************************************************************************
    12         -** This is the header file for the generic hash-table implemenation
    13         -** used in SQLite.  We've modified it slightly to serve as a standalone
    14         -** hash table implementation for the full-text indexing module.
    15         -**
    16         -*/
    17         -#ifndef _HASH_H_
    18         -#define _HASH_H_
    19         -
    20         -/* Forward declarations of structures. */
    21         -typedef struct Hash Hash;
    22         -typedef struct HashElem HashElem;
    23         -
    24         -/* A complete hash table is an instance of the following structure.
    25         -** The internals of this structure are intended to be opaque -- client
    26         -** code should not attempt to access or modify the fields of this structure
    27         -** directly.  Change this structure only by using the routines below.
    28         -** However, many of the "procedures" and "functions" for modifying and
    29         -** accessing this structure are really macros, so we can't really make
    30         -** this structure opaque.
    31         -*/
    32         -struct Hash {
    33         -  char keyClass;          /* HASH_INT, _POINTER, _STRING, _BINARY */
    34         -  char copyKey;           /* True if copy of key made on insert */
    35         -  int count;              /* Number of entries in this table */
    36         -  HashElem *first;        /* The first element of the array */
    37         -  void *(*xMalloc)(int);  /* malloc() function to use */
    38         -  void (*xFree)(void *);  /* free() function to use */
    39         -  int htsize;             /* Number of buckets in the hash table */
    40         -  struct _ht {            /* the hash table */
    41         -    int count;               /* Number of entries with this hash */
    42         -    HashElem *chain;         /* Pointer to first entry with this hash */
    43         -  } *ht;
    44         -};
    45         -
    46         -/* Each element in the hash table is an instance of the following 
    47         -** structure.  All elements are stored on a single doubly-linked list.
    48         -**
    49         -** Again, this structure is intended to be opaque, but it can't really
    50         -** be opaque because it is used by macros.
    51         -*/
    52         -struct HashElem {
    53         -  HashElem *next, *prev;   /* Next and previous elements in the table */
    54         -  void *data;              /* Data associated with this element */
    55         -  void *pKey; int nKey;    /* Key associated with this element */
    56         -};
    57         -
    58         -/*
    59         -** There are 4 different modes of operation for a hash table:
    60         -**
    61         -**   HASH_INT         nKey is used as the key and pKey is ignored.
    62         -**
    63         -**   HASH_POINTER     pKey is used as the key and nKey is ignored.
    64         -**
    65         -**   HASH_STRING      pKey points to a string that is nKey bytes long
    66         -**                           (including the null-terminator, if any).  Case
    67         -**                           is respected in comparisons.
    68         -**
    69         -**   HASH_BINARY      pKey points to binary data nKey bytes long. 
    70         -**                           memcmp() is used to compare keys.
    71         -**
    72         -** A copy of the key is made for HASH_STRING and HASH_BINARY
    73         -** if the copyKey parameter to HashInit is 1.  
    74         -*/
    75         -/* #define HASH_INT       1 // NOT USED */
    76         -/* #define HASH_POINTER   2 // NOT USED */
    77         -#define HASH_STRING    3
    78         -#define HASH_BINARY    4
    79         -
    80         -/*
    81         -** Access routines.  To delete, insert a NULL pointer.
    82         -*/
    83         -void HashInit(Hash*, int keytype, int copyKey);
    84         -void *HashInsert(Hash*, const void *pKey, int nKey, void *pData);
    85         -void *HashFind(const Hash*, const void *pKey, int nKey);
    86         -void HashClear(Hash*);
    87         -
    88         -/*
    89         -** Macros for looping over all elements of a hash table.  The idiom is
    90         -** like this:
    91         -**
    92         -**   Hash h;
    93         -**   HashElem *p;
    94         -**   ...
    95         -**   for(p=HashFirst(&h); p; p=HashNext(p)){
    96         -**     SomeStructure *pData = HashData(p);
    97         -**     // do something with pData
    98         -**   }
    99         -*/
   100         -#define HashFirst(H)  ((H)->first)
   101         -#define HashNext(E)   ((E)->next)
   102         -#define HashData(E)   ((E)->data)
   103         -#define HashKey(E)    ((E)->pKey)
   104         -#define HashKeysize(E) ((E)->nKey)
   105         -
   106         -/*
   107         -** Number of entries in a hash table
   108         -*/
   109         -#define HashCount(H)  ((H)->count)
   110         -
   111         -#endif /* _HASH_H_ */

Deleted ext/fts1/fulltext.c.

     1         -/* The author disclaims copyright to this source code.
     2         - *
     3         - * This is an SQLite module implementing full-text search.
     4         - */
     5         -
     6         -#include <assert.h>
     7         -#if !defined(__APPLE__)
     8         -#include <malloc.h>
     9         -#else
    10         -#include <stdlib.h>
    11         -#endif
    12         -#include <stdio.h>
    13         -#include <string.h>
    14         -#include <ctype.h>
    15         -
    16         -#include "fulltext.h"
    17         -#include "ft_hash.h"
    18         -#include "tokenizer.h"
    19         -#include "sqlite3.h"
    20         -#include "sqlite3ext.h"
    21         -SQLITE_EXTENSION_INIT1
    22         -
    23         -/* utility functions */
    24         -
    25         -/* We encode variable-length integers in little-endian order using seven bits
    26         - * per byte as follows:
    27         -**
    28         -** KEY:
    29         -**         A = 0xxxxxxx    7 bits of data and one flag bit
    30         -**         B = 1xxxxxxx    7 bits of data and one flag bit
    31         -**
    32         -**  7 bits - A
    33         -** 14 bits - BA
    34         -** 21 bits - BBA
    35         -** and so on.
    36         -*/
    37         -
    38         -/* We may need up to VARINT_MAX bytes to store an encoded 64-bit integer. */
    39         -#define VARINT_MAX 10
    40         -
    41         -/* Write a 64-bit variable-length integer to memory starting at p[0].
    42         - * The length of data written will be between 1 and VARINT_MAX bytes.
    43         - * The number of bytes written is returned. */
    44         -static int putVarint(char *p, sqlite_int64 v){
    45         -  unsigned char *q = (unsigned char *) p;
    46         -  sqlite_uint64 vu = v;
    47         -  do{
    48         -    *q++ = (unsigned char) ((vu & 0x7f) | 0x80);
    49         -    vu >>= 7;
    50         -  }while( vu!=0 );
    51         -  q[-1] &= 0x7f;  /* turn off high bit in final byte */
    52         -  assert( q - (unsigned char *)p <= VARINT_MAX );
    53         -  return (int) (q - (unsigned char *)p);
    54         -}
    55         -
    56         -/* Read a 64-bit variable-length integer from memory starting at p[0].
    57         - * Return the number of bytes read, or 0 on error.
    58         - * The value is stored in *v. */
    59         -static int getVarint(const char *p, sqlite_int64 *v){
    60         -  const unsigned char *q = (const unsigned char *) p;
    61         -  sqlite_uint64 x = 0, y = 1;
    62         -  while( (*q & 0x80) == 0x80 ){
    63         -    x += y * (*q++ & 0x7f);
    64         -    y <<= 7;
    65         -    if( q - (unsigned char *)p >= VARINT_MAX ){  /* bad data */
    66         -      assert( 0 );
    67         -      return 0;
    68         -    }
    69         -  }
    70         -  x += y * (*q++);
    71         -  *v = (sqlite_int64) x;
    72         -  return (int) (q - (unsigned char *)p);
    73         -}
    74         -
    75         -static int getVarint32(const char *p, int *pi){
    76         - sqlite_int64 i;
    77         - int ret = getVarint(p, &i);
    78         - *pi = (int) i;
    79         - assert( *pi==i );
    80         - return ret;
    81         -}
    82         -
    83         -/*** Document lists ***
    84         - *
    85         - * A document list holds a sorted list of varint-encoded document IDs.
    86         - *
    87         - * A doclist with type DL_POSITIONS_OFFSETS is stored like this:
    88         - *
    89         - * array {
    90         - *   varint docid;
    91         - *   array {
    92         - *     varint position;     (delta from previous position plus 1, or 0 for end)
    93         - *     varint startOffset;  (delta from previous startOffset)
    94         - *     varint endOffset;    (delta from startOffset)
    95         - *   }
    96         - * }
    97         - *
    98         - * Here, array { X } means zero or more occurrences of X, adjacent in memory.
    99         - *
   100         - * A doclist with type DL_POSITIONS is like the above, but holds only docids
   101         - * and positions without offset information.
   102         - *
   103         - * A doclist with type DL_DOCIDS is like the above, but holds only docids
   104         - * without positions or offset information.
   105         - *
   106         - * On disk, every document list has positions and offsets, so we don't bother
   107         - * to serialize a doclist's type.
   108         - * 
   109         - * We don't yet delta-encode document IDs; doing so will probably be a
   110         - * modest win.
   111         - *
   112         - * NOTE(shess) I've thought of a slightly (1%) better offset encoding.
   113         - * After the first offset, estimate the next offset by using the
   114         - * current token position and the previous token position and offset,
   115         - * offset to handle some variance.  So the estimate would be
   116         - * (iPosition*w->iStartOffset/w->iPosition-64), which is delta-encoded
   117         - * as normal.  Offsets more than 64 chars from the estimate are
   118         - * encoded as the delta to the previous start offset + 128.  An
   119         - * additional tiny increment can be gained by using the end offset of
   120         - * the previous token to make the estimate a tiny bit more precise.
   121         -*/
   122         -
   123         -typedef enum DocListType {
   124         -  DL_DOCIDS,              /* docids only */
   125         -  DL_POSITIONS,           /* docids + positions */
   126         -  DL_POSITIONS_OFFSETS    /* docids + positions + offsets */
   127         -} DocListType;
   128         -
   129         -typedef struct DocList {
   130         -  char *pData;
   131         -  int nData;
   132         -  DocListType iType;
   133         -  int iLastPos;       /* the last position written */
   134         -  int iLastOffset;    /* the last start offset written */
   135         -} DocList;
   136         -
   137         -/* Initialize a new DocList to hold the given data. */
   138         -static void docListInit(DocList *d, DocListType iType,
   139         -                        const char *pData, int nData){
   140         -  d->nData = nData;
   141         -  if( nData>0 ){
   142         -    d->pData = malloc(nData);
   143         -    memcpy(d->pData, pData, nData);
   144         -  } else {
   145         -    d->pData = NULL;
   146         -  }
   147         -  d->iType = iType;
   148         -  d->iLastPos = 0;
   149         -  d->iLastOffset = 0;
   150         -}
   151         -
   152         -/* Create a new dynamically-allocated DocList. */
   153         -static DocList *docListNew(DocListType iType){
   154         -  DocList *d = (DocList *) malloc(sizeof(DocList));
   155         -  docListInit(d, iType, 0, 0);
   156         -  return d;
   157         -}
   158         -
   159         -static void docListDestroy(DocList *d){
   160         -  free(d->pData);
   161         -#ifndef NDEBUG
   162         -  memset(d, 0x55, sizeof(*d));
   163         -#endif
   164         -}
   165         -
   166         -static void docListDelete(DocList *d){
   167         -  docListDestroy(d);
   168         -  free(d);
   169         -}
   170         -
   171         -static char *docListEnd(DocList *d){
   172         -  return d->pData + d->nData;
   173         -}
   174         -
   175         -/* Append a varint to a DocList's data. */
   176         -static void appendVarint(DocList *d, sqlite_int64 i){
   177         -  char c[VARINT_MAX];
   178         -  int n = putVarint(c, i);
   179         -  d->pData = realloc(d->pData, d->nData + n);
   180         -  memcpy(d->pData + d->nData, c, n);
   181         -  d->nData += n;
   182         -}
   183         -
   184         -static void docListAddDocid(DocList *d, sqlite_int64 iDocid){
   185         -  appendVarint(d, iDocid);
   186         -  d->iLastPos = 0;
   187         -}
   188         -
   189         -/* Add a position to the last position list in a doclist. */
   190         -static void docListAddPos(DocList *d, int iPos){
   191         -  assert( d->iType>=DL_POSITIONS );
   192         -  appendVarint(d, iPos-d->iLastPos+1);
   193         -  d->iLastPos = iPos;
   194         -}
   195         -
   196         -static void docListAddPosOffset(DocList *d, int iPos,
   197         -                                int iStartOffset, int iEndOffset){
   198         -  assert( d->iType==DL_POSITIONS_OFFSETS );
   199         -  docListAddPos(d, iPos);
   200         -  appendVarint(d, iStartOffset-d->iLastOffset);
   201         -  d->iLastOffset = iStartOffset;
   202         -  appendVarint(d, iEndOffset-iStartOffset);
   203         -}
   204         -
   205         -/* Terminate the last position list in the given doclist. */
   206         -static void docListAddEndPos(DocList *d){
   207         -  appendVarint(d, 0);
   208         -}
   209         -
   210         -typedef struct DocListReader {
   211         -  DocList *pDoclist;
   212         -  char *p;
   213         -  int iLastPos;    /* the last position read */
   214         -} DocListReader;
   215         -
   216         -static void readerInit(DocListReader *r, DocList *pDoclist){
   217         -  r->pDoclist = pDoclist;
   218         -  if( pDoclist!=NULL ){
   219         -    r->p = pDoclist->pData;
   220         -  }
   221         -  r->iLastPos = 0;
   222         -}
   223         -
   224         -static int readerAtEnd(DocListReader *pReader){
   225         -  return pReader->p >= docListEnd(pReader->pDoclist);
   226         -}
   227         -
   228         -/* Peek at the next docid without advancing the read pointer. */
   229         -static sqlite_int64 peekDocid(DocListReader *pReader){
   230         -  sqlite_int64 ret;
   231         -  assert( !readerAtEnd(pReader) );
   232         -  getVarint(pReader->p, &ret);
   233         -  return ret;
   234         -}
   235         -
   236         -/* Read the next docid. */
   237         -static sqlite_int64 readDocid(DocListReader *pReader){
   238         -  sqlite_int64 ret;
   239         -  assert( !readerAtEnd(pReader) );
   240         -  pReader->p += getVarint(pReader->p, &ret);
   241         -  pReader->iLastPos = 0;
   242         -  return ret;
   243         -}
   244         -
   245         -/* Read the next position from a position list.
   246         - * Returns the position, or -1 at the end of the list. */
   247         -static int readPosition(DocListReader *pReader){
   248         -  int i;
   249         -  int iType = pReader->pDoclist->iType;
   250         -  assert( iType>=DL_POSITIONS );
   251         -  assert( !readerAtEnd(pReader) );
   252         -
   253         -  pReader->p += getVarint32(pReader->p, &i);
   254         -  if( i==0 ){
   255         -    pReader->iLastPos = -1;
   256         -    return -1;
   257         -  }
   258         -  pReader->iLastPos += ((int) i)-1;
   259         -  if( iType>=DL_POSITIONS_OFFSETS ){
   260         -    /* Skip over offsets, ignoring them for now. */
   261         -    int iStart, iEnd;
   262         -    pReader->p += getVarint32(pReader->p, &iStart);
   263         -    pReader->p += getVarint32(pReader->p, &iEnd);
   264         -  }
   265         -  return pReader->iLastPos;
   266         -}
   267         -
   268         -/* Skip past the end of a position list. */
   269         -static void skipPositionList(DocListReader *pReader){
   270         -  while( readPosition(pReader)!=-1 )
   271         -    ;
   272         -}
   273         -
   274         -/* Skip over a docid, including its position list if the doclist has
   275         - * positions. */
   276         -static void skipDocument(DocListReader *pReader){
   277         -  readDocid(pReader);
   278         -  if( pReader->pDoclist->iType >= DL_POSITIONS ){
   279         -    skipPositionList(pReader);
   280         -  }
   281         -}
   282         -
   283         -static sqlite_int64 firstDocid(DocList *d){
   284         -  DocListReader r;
   285         -  readerInit(&r, d);
   286         -  return readDocid(&r);
   287         -}
   288         -
   289         -/* Doclist multi-tool.  Pass pUpdate==NULL to delete the indicated docid;
   290         - * otherwise pUpdate, which must contain only the single docid [iDocid], is
   291         - * inserted (if not present) or updated (if already present). */
   292         -static int docListUpdate(DocList *d, sqlite_int64 iDocid, DocList *pUpdate){
   293         -  int modified = 0;
   294         -  DocListReader reader;
   295         -  char *p;
   296         -
   297         -  if( pUpdate!=NULL ){
   298         -    assert( d->iType==pUpdate->iType);
   299         -    assert( iDocid==firstDocid(pUpdate) );
   300         -  }
   301         -
   302         -  readerInit(&reader, d);
   303         -  while( !readerAtEnd(&reader) && peekDocid(&reader)<iDocid ){
   304         -    skipDocument(&reader);
   305         -  }
   306         -
   307         -  p = reader.p;
   308         -  /* Delete if there is a matching element. */
   309         -  if( !readerAtEnd(&reader) && iDocid==peekDocid(&reader) ){
   310         -    skipDocument(&reader);
   311         -    memmove(p, reader.p, docListEnd(d) - reader.p);
   312         -    d->nData -= (reader.p - p);
   313         -    modified = 1;
   314         -  }
   315         -
   316         -  /* Insert if indicated. */
   317         -  if( pUpdate!=NULL ){
   318         -    int iDoclist = p-d->pData;
   319         -    docListAddEndPos(pUpdate);
   320         -
   321         -    d->pData = realloc(d->pData, d->nData+pUpdate->nData);
   322         -    p = d->pData + iDoclist;
   323         -
   324         -    memmove(p+pUpdate->nData, p, docListEnd(d) - p);
   325         -    memcpy(p, pUpdate->pData, pUpdate->nData);
   326         -    d->nData += pUpdate->nData;
   327         -    modified = 1;
   328         -  }
   329         -
   330         -  return modified;
   331         -}
   332         -
   333         -/* Split the second half of doclist d into a separate doclist d2.  Returns 1
   334         - * if successful, or 0 if d contains a single document and hence can't be
   335         - * split. */
   336         -static int docListSplit(DocList *d, DocList *d2){
   337         -  const char *pSplitPoint = d->pData + d->nData / 2;
   338         -  DocListReader reader;
   339         -
   340         -  readerInit(&reader, d);
   341         -  while( reader.p<pSplitPoint ){
   342         -    skipDocument(&reader);
   343         -  }
   344         -  if( readerAtEnd(&reader) ) return 0;
   345         -  docListInit(d2, d->iType, reader.p, docListEnd(d) - reader.p);
   346         -  d->nData = reader.p - d->pData;
   347         -  d->pData = realloc(d->pData, d->nData);
   348         -  return 1;
   349         -}
   350         -
   351         -/* A DocListMerge computes the AND of an in-memory DocList [in] and a chunked
   352         - * on-disk doclist, resulting in another in-memory DocList [out].  [in]
   353         - * and [out] may or may not store position information according to the
   354         - * caller's wishes.  The on-disk doclist always comes with positions.
   355         - *
   356         - * The caller must read each chunk of the on-disk doclist in succession and
   357         - * pass it to mergeBlock().
   358         - *
   359         - * If [in] has positions, then the merge output contains only documents with
   360         - * matching positions in the two input doclists.  If [in] does not have
   361         - * positions, then the merge output contains all documents common to the two
   362         - * input doclists.
   363         - *
   364         - * If [in] is NULL, then the on-disk doclist is copied to [out] directly.
   365         - *
   366         - * A merge is performed using an integer [iOffset] provided by the caller.
   367         - * [iOffset] is subtracted from each position in the on-disk doclist for the
   368         - * purpose of position comparison; this is helpful in implementing phrase
   369         - * searches.
   370         - *
   371         - * A DocListMerge is not yet able to propagate offsets through query
   372         - * processing; we should add that capability soon.
   373         -*/
   374         -typedef struct DocListMerge {
   375         -  DocListReader in;
   376         -  DocList *pOut;
   377         -  int iOffset;
   378         -} DocListMerge;
   379         -
   380         -static void mergeInit(DocListMerge *m,
   381         -                      DocList *pIn, int iOffset, DocList *pOut){
   382         -  readerInit(&m->in, pIn);
   383         -  m->pOut = pOut;
   384         -  m->iOffset = iOffset;
   385         -
   386         -  /* can't handle offsets yet */
   387         -  assert( pIn==NULL || pIn->iType <= DL_POSITIONS );
   388         -  assert( pOut->iType <= DL_POSITIONS );
   389         -}
   390         -
   391         -/* A helper function for mergeBlock(), below.  Merge the position lists
   392         - * pointed to by m->in and pBlockReader.
   393         - * If the merge matches, write [iDocid] to m->pOut; if m->pOut
   394         - * has positions then write all matching positions as well. */
   395         -static void mergePosList(DocListMerge *m, sqlite_int64 iDocid,
   396         -                  DocListReader *pBlockReader){
   397         -  int block_pos = readPosition(pBlockReader);
   398         -  int in_pos = readPosition(&m->in);
   399         -  int match = 0;
   400         -  while( block_pos!=-1 || in_pos!=-1 ){
   401         -    if( block_pos-m->iOffset==in_pos ){
   402         -      if( !match ){
   403         -        docListAddDocid(m->pOut, iDocid);
   404         -        match = 1;
   405         -      }
   406         -      if( m->pOut->iType >= DL_POSITIONS ){
   407         -        docListAddPos(m->pOut, in_pos);
   408         -      }
   409         -      block_pos = readPosition(pBlockReader);
   410         -      in_pos = readPosition(&m->in);
   411         -    } else if( in_pos==-1 || (block_pos!=-1 && block_pos-m->iOffset<in_pos) ){
   412         -      block_pos = readPosition(pBlockReader);
   413         -    } else {
   414         -      in_pos = readPosition(&m->in);
   415         -    }
   416         -  }
   417         -  if( m->pOut->iType >= DL_POSITIONS && match ){
   418         -    docListAddEndPos(m->pOut);
   419         -  }
   420         -}
   421         -
   422         -/* Merge one block of an on-disk doclist into a DocListMerge. */
   423         -static void mergeBlock(DocListMerge *m, DocList *pBlock){
   424         -  DocListReader blockReader;
   425         -  assert( pBlock->iType >= DL_POSITIONS );
   426         -  readerInit(&blockReader, pBlock);
   427         -  while( !readerAtEnd(&blockReader) ){
   428         -    sqlite_int64 iDocid = readDocid(&blockReader);
   429         -    if( m->in.pDoclist!=NULL ){
   430         -      while( 1 ){
   431         -        if( readerAtEnd(&m->in) ) return;  /* nothing more to merge */
   432         -        if( peekDocid(&m->in)>=iDocid ) break;
   433         -        skipDocument(&m->in);
   434         -      }
   435         -      if( peekDocid(&m->in)>iDocid ){  /* [pIn] has no match with iDocid */
   436         -        skipPositionList(&blockReader);  /* skip this docid in the block */
   437         -        continue;
   438         -      }
   439         -      readDocid(&m->in);
   440         -    }
   441         -    /* We have a document match. */
   442         -    if( m->in.pDoclist==NULL || m->in.pDoclist->iType < DL_POSITIONS ){
   443         -      /* We don't need to do a poslist merge. */
   444         -      docListAddDocid(m->pOut, iDocid);
   445         -      if( m->pOut->iType >= DL_POSITIONS ){
   446         -        /* Copy all positions to the output doclist. */
   447         -        while( 1 ){
   448         -          int pos = readPosition(&blockReader);
   449         -          if( pos==-1 ) break;
   450         -          docListAddPos(m->pOut, pos);
   451         -        }
   452         -        docListAddEndPos(m->pOut);
   453         -      } else skipPositionList(&blockReader);
   454         -      continue;
   455         -    }
   456         -    mergePosList(m, iDocid, &blockReader);
   457         -  }
   458         -}
   459         -
   460         -static char *string_dup_n(const char *s, int n){
   461         -  char *str = malloc(n + 1);
   462         -  memcpy(str, s, n);
   463         -  str[n] = '\0';
   464         -  return str;
   465         -}
   466         -
   467         -/* Duplicate a string; the caller must free() the returned string.
   468         - * (We don't use strdup() since it's not part of the standard C library and
   469         - * may not be available everywhere.) */
   470         -static char *string_dup(const char *s){
   471         -  return string_dup_n(s, strlen(s));
   472         -}
   473         -
   474         -/* Format a string, replacing each occurrence of the % character with
   475         - * zName.  This may be more convenient than sqlite_mprintf()
   476         - * when one string is used repeatedly in a format string.
   477         - * The caller must free() the returned string. */
   478         -static char *string_format(const char *zFormat, const char *zName){
   479         -  const char *p;
   480         -  size_t len = 0;
   481         -  size_t nName = strlen(zName);
   482         -  char *result;
   483         -  char *r;
   484         -
   485         -  /* first compute length needed */
   486         -  for(p = zFormat ; *p ; ++p){
   487         -    len += (*p=='%' ? nName : 1);
   488         -  }
   489         -  len += 1;  /* for null terminator */
   490         -
   491         -  r = result = malloc(len);
   492         -  for(p = zFormat; *p; ++p){
   493         -    if( *p=='%' ){
   494         -      memcpy(r, zName, nName);
   495         -      r += nName;
   496         -    } else {
   497         -      *r++ = *p;
   498         -    }
   499         -  }
   500         -  *r++ = '\0';
   501         -  assert( r == result + len );
   502         -  return result;
   503         -}
   504         -
   505         -static int sql_exec(sqlite3 *db, const char *zName, const char *zFormat){
   506         -  char *zCommand = string_format(zFormat, zName);
   507         -  int rc = sqlite3_exec(db, zCommand, NULL, 0, NULL);
   508         -  free(zCommand);
   509         -  return rc;
   510         -}
   511         -
   512         -static int sql_prepare(sqlite3 *db, const char *zName, sqlite3_stmt **ppStmt,
   513         -                const char *zFormat){
   514         -  char *zCommand = string_format(zFormat, zName);
   515         -  int rc = sqlite3_prepare(db, zCommand, -1, ppStmt, NULL);
   516         -  free(zCommand);
   517         -  return rc;
   518         -}
   519         -
   520         -/* end utility functions */
   521         -
   522         -#define QUERY_GENERIC 0
   523         -#define QUERY_FULLTEXT 1
   524         -
   525         -#define CHUNK_MAX 1024
   526         -
   527         -typedef enum fulltext_statement {
   528         -  CONTENT_INSERT_STMT,
   529         -  CONTENT_SELECT_STMT,
   530         -  CONTENT_DELETE_STMT,
   531         -
   532         -  TERM_SELECT_STMT,
   533         -  TERM_CHUNK_SELECT_STMT,
   534         -  TERM_INSERT_STMT,
   535         -  TERM_UPDATE_STMT,
   536         -  TERM_DELETE_STMT,
   537         -
   538         -  MAX_STMT                     /* Always at end! */
   539         -} fulltext_statement;
   540         -
   541         -/* These must exactly match the enum above. */
   542         -/* TODO(adam): Is there some risk that a statement (in particular,
   543         -** pTermSelectStmt) will be used in two cursors at once, e.g.  if a
   544         -** query joins a virtual table to itself?  If so perhaps we should
   545         -** move some of these to the cursor object.
   546         -*/
   547         -static const char *fulltext_zStatement[MAX_STMT] = {
   548         -  /* CONTENT_INSERT */ "insert into %_content (rowid, content) values (?, ?)",
   549         -  /* CONTENT_SELECT */ "select content from %_content where rowid = ?",
   550         -  /* CONTENT_DELETE */ "delete from %_content where rowid = ?",
   551         -
   552         -  /* TERM_SELECT */
   553         -  "select rowid, doclist from %_term where term = ? and first = ?",
   554         -  /* TERM_CHUNK_SELECT */
   555         -  "select max(first) from %_term where term = ? and first <= ?",
   556         -  /* TERM_INSERT */
   557         -  "insert into %_term (term, first, doclist) values (?, ?, ?)",
   558         -  /* TERM_UPDATE */ "update %_term set doclist = ? where rowid = ?",
   559         -  /* TERM_DELETE */ "delete from %_term where rowid = ?",
   560         -};
   561         -
   562         -typedef struct fulltext_vtab {
   563         -  sqlite3_vtab base;
   564         -  sqlite3 *db;
   565         -  const char *zName;               /* virtual table name */
   566         -  sqlite3_tokenizer *pTokenizer;   /* tokenizer for inserts and queries */
   567         -
   568         -  /* Precompiled statements which we keep as long as the table is
   569         -  ** open.
   570         -  */
   571         -  sqlite3_stmt *pFulltextStatements[MAX_STMT];
   572         -} fulltext_vtab;
   573         -
   574         -typedef struct fulltext_cursor {
   575         -  sqlite3_vtab_cursor base;
   576         -  int iCursorType;  /* QUERY_GENERIC or QUERY_FULLTEXT */
   577         -
   578         -  sqlite3_stmt *pStmt;
   579         -
   580         -  int eof;
   581         -
   582         -  /* The following is used only when iCursorType == QUERY_FULLTEXT. */
   583         -  DocListReader result;
   584         -} fulltext_cursor;
   585         -
   586         -static struct fulltext_vtab *cursor_vtab(fulltext_cursor *c){
   587         -  return (fulltext_vtab *) c->base.pVtab;
   588         -}
   589         -
   590         -static sqlite3_module fulltextModule;   /* forward declaration */
   591         -
   592         -/* Puts a freshly-prepared statement determined by iStmt in *ppStmt.
   593         -** If the indicated statement has never been prepared, it is prepared
   594         -** and cached, otherwise the cached version is reset.
   595         -*/
   596         -static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt,
   597         -                             sqlite3_stmt **ppStmt){
   598         -  assert( iStmt<MAX_STMT );
   599         -  if( v->pFulltextStatements[iStmt]==NULL ){
   600         -    int rc = sql_prepare(v->db, v->zName, &v->pFulltextStatements[iStmt],
   601         -                         fulltext_zStatement[iStmt]);
   602         -    if( rc!=SQLITE_OK ) return rc;
   603         -  } else {
   604         -    int rc = sqlite3_reset(v->pFulltextStatements[iStmt]);
   605         -    if( rc!=SQLITE_OK ) return rc;
   606         -  }
   607         -
   608         -  *ppStmt = v->pFulltextStatements[iStmt];
   609         -  return SQLITE_OK;
   610         -}
   611         -
   612         -/* Step the indicated statement, handling errors SQLITE_BUSY (by
   613         -** retrying) and SQLITE_SCHEMA (by re-preparing and transferring
   614         -** bindings to the new statement).
   615         -** TODO(adam): We should extend this function so that it can work with
   616         -** statements declared locally, not only globally cached statements.
   617         -*/
   618         -static int sql_step_statement(fulltext_vtab *v, fulltext_statement iStmt,
   619         -                              sqlite3_stmt **ppStmt){
   620         -  int rc;
   621         -  sqlite3_stmt *s = *ppStmt;
   622         -  assert( iStmt<MAX_STMT );
   623         -  assert( s==v->pFulltextStatements[iStmt] );
   624         -
   625         -  while( (rc=sqlite3_step(s))!=SQLITE_DONE && rc!=SQLITE_ROW ){
   626         -    sqlite3_stmt *pNewStmt;
   627         -
   628         -    if( rc==SQLITE_BUSY ) continue;
   629         -    if( rc!=SQLITE_ERROR ) return rc;
   630         -
   631         -    rc = sqlite3_reset(s);
   632         -    if( rc!=SQLITE_SCHEMA ) return SQLITE_ERROR;
   633         -
   634         -    v->pFulltextStatements[iStmt] = NULL;   /* Still in s */
   635         -    rc = sql_get_statement(v, iStmt, &pNewStmt);
   636         -    if( rc!=SQLITE_OK ) goto err;
   637         -    *ppStmt = pNewStmt;
   638         -
   639         -    rc = sqlite3_transfer_bindings(s, pNewStmt);
   640         -    if( rc!=SQLITE_OK ) goto err;
   641         -
   642         -    rc = sqlite3_finalize(s);
   643         -    if( rc!=SQLITE_OK ) return rc;
   644         -    s = pNewStmt;
   645         -  }
   646         -  return rc;
   647         -
   648         - err:
   649         -  sqlite3_finalize(s);
   650         -  return rc;
   651         -}
   652         -
   653         -/* Like sql_step_statement(), but convert SQLITE_DONE to SQLITE_OK.
   654         -** Useful for statements like UPDATE, where we expect no results.
   655         -*/
   656         -static int sql_single_step_statement(fulltext_vtab *v,
   657         -                                     fulltext_statement iStmt,
   658         -                                     sqlite3_stmt **ppStmt){
   659         -  int rc = sql_step_statement(v, iStmt, ppStmt);
   660         -  return (rc==SQLITE_DONE) ? SQLITE_OK : rc;
   661         -}
   662         -
   663         -/* insert into %_content (rowid, content) values ([rowid], [zContent]) */
   664         -static int content_insert(fulltext_vtab *v, sqlite3_value *rowid,
   665         -                          const char *zContent, int nContent){
   666         -  sqlite3_stmt *s;
   667         -  int rc = sql_get_statement(v, CONTENT_INSERT_STMT, &s);
   668         -  if( rc!=SQLITE_OK ) return rc;
   669         -
   670         -  rc = sqlite3_bind_value(s, 1, rowid);
   671         -  if( rc!=SQLITE_OK ) return rc;
   672         -
   673         -  rc = sqlite3_bind_text(s, 2, zContent, nContent, SQLITE_STATIC);
   674         -  if( rc!=SQLITE_OK ) return rc;
   675         -
   676         -  return sql_single_step_statement(v, CONTENT_INSERT_STMT, &s);
   677         -}
   678         -
   679         -/* select content from %_content where rowid = [iRow]
   680         - * The caller must delete the returned string. */
   681         -static int content_select(fulltext_vtab *v, sqlite_int64 iRow,
   682         -                          char **pzContent){
   683         -  sqlite3_stmt *s;
   684         -  int rc = sql_get_statement(v, CONTENT_SELECT_STMT, &s);
   685         -  if( rc!=SQLITE_OK ) return rc;
   686         -
   687         -  rc = sqlite3_bind_int64(s, 1, iRow);
   688         -  if( rc!=SQLITE_OK ) return rc;
   689         -
   690         -  rc = sql_step_statement(v, CONTENT_SELECT_STMT, &s);
   691         -  if( rc!=SQLITE_ROW ) return rc;
   692         -
   693         -  *pzContent = string_dup((const char *)sqlite3_column_text(s, 0));
   694         -
   695         -  /* We expect only one row.  We must execute another sqlite3_step()
   696         -   * to complete the iteration; otherwise the table will remain locked. */
   697         -  rc = sqlite3_step(s);
   698         -  if( rc==SQLITE_DONE ) return SQLITE_OK;
   699         -
   700         -  free(*pzContent);
   701         -  return rc;
   702         -}
   703         -
   704         -/* delete from %_content where rowid = [iRow ] */
   705         -static int content_delete(fulltext_vtab *v, sqlite_int64 iRow){
   706         -  sqlite3_stmt *s;
   707         -  int rc = sql_get_statement(v, CONTENT_DELETE_STMT, &s);
   708         -  if( rc!=SQLITE_OK ) return rc;
   709         -
   710         -  rc = sqlite3_bind_int64(s, 1, iRow);
   711         -  if( rc!=SQLITE_OK ) return rc;
   712         -
   713         -  return sql_single_step_statement(v, CONTENT_DELETE_STMT, &s);
   714         -}
   715         -
   716         -/* select rowid, doclist from %_term where term = [zTerm] and first = [iFirst]
   717         - * If found, returns SQLITE_OK; the caller must free the returned doclist.
   718         - * If no rows found, returns SQLITE_ERROR. */
   719         -static int term_select(fulltext_vtab *v, const char *zTerm, int nTerm,
   720         -                       sqlite_int64 iFirst,
   721         -                       sqlite_int64 *rowid,
   722         -                       DocList *out){
   723         -  sqlite3_stmt *s;
   724         -  int rc = sql_get_statement(v, TERM_SELECT_STMT, &s);
   725         -  if( rc!=SQLITE_OK ) return rc;
   726         -
   727         -  rc = sqlite3_bind_text(s, 1, zTerm, nTerm, SQLITE_TRANSIENT);
   728         -  if( rc!=SQLITE_OK ) return rc;
   729         -
   730         -  rc = sqlite3_bind_int64(s, 2, iFirst);
   731         -  if( rc!=SQLITE_OK ) return rc;
   732         -
   733         -  rc = sql_step_statement(v, TERM_SELECT_STMT, &s);
   734         -  if( rc!=SQLITE_ROW ) return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
   735         -
   736         -  *rowid = sqlite3_column_int64(s, 0);
   737         -  docListInit(out, DL_POSITIONS_OFFSETS,
   738         -              sqlite3_column_blob(s, 1), sqlite3_column_bytes(s, 1));
   739         -
   740         -  /* We expect only one row.  We must execute another sqlite3_step()
   741         -   * to complete the iteration; otherwise the table will remain locked. */
   742         -  rc = sqlite3_step(s);
   743         -  return rc==SQLITE_DONE ? SQLITE_OK : rc;
   744         -}
   745         -
   746         -/* select max(first) from %_term where term = [zTerm] and first <= [iFirst]
   747         - * If found, returns SQLITE_ROW and result in *piResult; if the query returns
   748         - * NULL (meaning no row found) returns SQLITE_DONE.
   749         - */
   750         -static int term_chunk_select(fulltext_vtab *v, const char *zTerm, int nTerm,
   751         -                           sqlite_int64 iFirst, sqlite_int64 *piResult){
   752         -  sqlite3_stmt *s;
   753         -  int rc = sql_get_statement(v, TERM_CHUNK_SELECT_STMT, &s);
   754         -  if( rc!=SQLITE_OK ) return rc;
   755         -
   756         -  rc = sqlite3_bind_text(s, 1, zTerm, nTerm, SQLITE_STATIC);
   757         -  if( rc!=SQLITE_OK ) return rc;
   758         -
   759         -  rc = sqlite3_bind_int64(s, 2, iFirst);
   760         -  if( rc!=SQLITE_OK ) return rc;
   761         -
   762         -  rc = sql_step_statement(v, TERM_CHUNK_SELECT_STMT, &s);
   763         -  if( rc!=SQLITE_ROW ) return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
   764         -
   765         -  switch( sqlite3_column_type(s, 0) ){
   766         -    case SQLITE_NULL:
   767         -      rc = SQLITE_DONE;
   768         -      break;
   769         -    case SQLITE_INTEGER:
   770         -     *piResult = sqlite3_column_int64(s, 0);
   771         -     break;
   772         -    default:
   773         -      return SQLITE_ERROR;
   774         -  }
   775         -  /* We expect only one row.  We must execute another sqlite3_step()
   776         -   * to complete the iteration; otherwise the table will remain locked. */
   777         -  if( sqlite3_step(s) != SQLITE_DONE ) return SQLITE_ERROR;
   778         -  return rc;
   779         -}
   780         -
   781         -/* insert into %_term (term, first, doclist)
   782         -               values ([zTerm], [iFirst], [doclist]) */
   783         -static int term_insert(fulltext_vtab *v, const char *zTerm, int nTerm,
   784         -                       sqlite_int64 iFirst, DocList *doclist){
   785         -  sqlite3_stmt *s;
   786         -  int rc = sql_get_statement(v, TERM_INSERT_STMT, &s);
   787         -  if( rc!=SQLITE_OK ) return rc;
   788         -
   789         -  rc = sqlite3_bind_text(s, 1, zTerm, nTerm, SQLITE_STATIC);
   790         -  if( rc!=SQLITE_OK ) return rc;
   791         -
   792         -  rc = sqlite3_bind_int64(s, 2, iFirst);
   793         -  if( rc!=SQLITE_OK ) return rc;
   794         -
   795         -  rc = sqlite3_bind_blob(s, 3, doclist->pData, doclist->nData, SQLITE_STATIC);
   796         -  if( rc!=SQLITE_OK ) return rc;
   797         -
   798         -  return sql_single_step_statement(v, TERM_INSERT_STMT, &s);
   799         -}
   800         -
   801         -/* update %_term set doclist = [doclist] where rowid = [rowid] */
   802         -static int term_update(fulltext_vtab *v, sqlite_int64 rowid,
   803         -                       DocList *doclist){
   804         -  sqlite3_stmt *s;
   805         -  int rc = sql_get_statement(v, TERM_UPDATE_STMT, &s);
   806         -  if( rc!=SQLITE_OK ) return rc;
   807         -
   808         -  rc = sqlite3_bind_blob(s, 1, doclist->pData, doclist->nData,
   809         -                         SQLITE_STATIC);
   810         -  if( rc!=SQLITE_OK ) return rc;
   811         -
   812         -  rc = sqlite3_bind_int64(s, 2, rowid);
   813         -  if( rc!=SQLITE_OK ) return rc;
   814         -
   815         -  return sql_single_step_statement(v, TERM_UPDATE_STMT, &s);
   816         -}
   817         -
   818         -static int term_delete(fulltext_vtab *v, sqlite_int64 rowid){
   819         -  sqlite3_stmt *s;
   820         -  int rc = sql_get_statement(v, TERM_DELETE_STMT, &s);
   821         -  if( rc!=SQLITE_OK ) return rc;
   822         -
   823         -  rc = sqlite3_bind_int64(s, 1, rowid);
   824         -  if( rc!=SQLITE_OK ) return rc;
   825         -
   826         -  return sql_single_step_statement(v, TERM_DELETE_STMT, &s);
   827         -}
   828         -
   829         -static void fulltext_vtab_destroy(fulltext_vtab *v){
   830         -  int iStmt;
   831         -
   832         -  for( iStmt=0; iStmt<MAX_STMT; iStmt++ ){
   833         -    if( v->pFulltextStatements[iStmt]!=NULL ){
   834         -      sqlite3_finalize(v->pFulltextStatements[iStmt]);
   835         -      v->pFulltextStatements[iStmt] = NULL;
   836         -    }
   837         -  }
   838         -
   839         -  if( v->pTokenizer!=NULL ){
   840         -    v->pTokenizer->pModule->xDestroy(v->pTokenizer);
   841         -    v->pTokenizer = NULL;
   842         -  }
   843         -
   844         -  free((void *) v->zName);
   845         -  free(v);
   846         -}
   847         -
   848         -/* Current interface:
   849         -** argv[0] - module name
   850         -** argv[1] - database name
   851         -** argv[2] - table name
   852         -** argv[3] - tokenizer name (optional, a sensible default is provided)
   853         -** argv[4..] - passed to tokenizer (optional based on tokenizer)
   854         -**/
   855         -static int fulltextConnect(sqlite3 *db, void *pAux, int argc, char **argv,
   856         -                           sqlite3_vtab **ppVTab){
   857         -  int rc;
   858         -  fulltext_vtab *v;
   859         -  sqlite3_tokenizer_module *m = NULL;
   860         -
   861         -  assert( argc>=3 );
   862         -  v = (fulltext_vtab *) malloc(sizeof(fulltext_vtab));
   863         -  /* sqlite will initialize v->base */
   864         -  v->db = db;
   865         -  v->zName = string_dup(argv[2]);
   866         -  v->pTokenizer = NULL;
   867         -
   868         -  if( argc==3 ){
   869         -    get_simple_tokenizer_module(&m);
   870         -  } else {
   871         -    /* TODO(shess) For now, add new tokenizers as else if clauses. */
   872         -    if( !strcmp(argv[3], "simple") ){
   873         -      get_simple_tokenizer_module(&m);
   874         -    } else {
   875         -      assert( "unrecognized tokenizer"==NULL );
   876         -    }
   877         -  }
   878         -
   879         -  /* TODO(shess) Since tokenization impacts the index, the parameters
   880         -  ** to the tokenizer need to be identical when a persistent virtual
   881         -  ** table is re-created.  One solution would be a meta-table to track
   882         -  ** such information in the database.  Then we could verify that the
   883         -  ** information is identical on subsequent creates.
   884         -  */
   885         -  /* TODO(shess) Why isn't argv already (const char **)? */
   886         -  rc = m->xCreate(argc-3, (const char **) (argv+3), &v->pTokenizer);
   887         -  if( rc!=SQLITE_OK ) return rc;
   888         -  v->pTokenizer->pModule = m;
   889         -
   890         -  /* TODO: verify the existence of backing tables foo_content, foo_term */
   891         -
   892         -  rc = sqlite3_declare_vtab(db, "create table x(content text)");
   893         -  if( rc!=SQLITE_OK ) return rc;
   894         -
   895         -  memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements));
   896         -
   897         -  *ppVTab = &v->base;
   898         -  return SQLITE_OK;
   899         -}
   900         -
   901         -static int fulltextCreate(sqlite3 *db, void *pAux, int argc, char **argv,
   902         -                          sqlite3_vtab **ppVTab){
   903         -  int rc;
   904         -  assert( argc>=3 );
   905         -
   906         -  /* The %_content table holds the text of each full-text item, with
   907         -  ** the rowid used as the docid.
   908         -  **
   909         -  ** The %_term table maps each term to a document list blob
   910         -  ** containing elements sorted by ascending docid, each element
   911         -  ** encoded as:
   912         -  **
   913         -  **   docid varint-encoded
   914         -  **   token count varint-encoded
   915         -  **   "count" token elements (poslist):
   916         -  **     position varint-encoded as delta from previous position
   917         -  **     start offset varint-encoded as delta from previous start offset
   918         -  **     end offset varint-encoded as delta from start offset
   919         -  **
   920         -  ** Additionally, doclist blobs can be chunked into multiple rows,
   921         -  ** using "first" to order the blobs.  "first" is simply the first
   922         -  ** docid in the blob.
   923         -  */
   924         -  /*
   925         -  ** NOTE(shess) That last sentence is incorrect in the face of
   926         -  ** deletion, which can leave a doclist that doesn't contain the
   927         -  ** first from that row.  I _believe_ this does not matter to the
   928         -  ** operation of the system, but it might be reasonable to update
   929         -  ** appropriately in case this assumption becomes more important.
   930         -  */
   931         -  rc = sql_exec(db, argv[2],
   932         -    "create table %_content(content text);"
   933         -    "create table %_term(term text, first integer, doclist blob);"
   934         -    "create index %_index on %_term(term, first)");
   935         -  if( rc!=SQLITE_OK ) return rc;
   936         -
   937         -  return fulltextConnect(db, pAux, argc, argv, ppVTab);
   938         -}
   939         -
   940         -/* Decide how to handle an SQL query.
   941         - * At the moment, MATCH queries can include implicit boolean ANDs; we
   942         - * haven't implemented phrase searches or OR yet. */
   943         -static int fulltextBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
   944         -  int i;
   945         -
   946         -  for(i=0; i<pInfo->nConstraint; ++i){
   947         -    const struct sqlite3_index_constraint *pConstraint;
   948         -    pConstraint = &pInfo->aConstraint[i];
   949         -    if( pConstraint->iColumn==0 &&
   950         -        pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH &&
   951         -        pConstraint->usable ){   /* a full-text search */
   952         -      pInfo->aConstraintUsage[i].argvIndex = 1;
   953         -      pInfo->aConstraintUsage[i].omit = 1;
   954         -      pInfo->idxNum = QUERY_FULLTEXT;
   955         -      pInfo->estimatedCost = 1.0;   /* an arbitrary value for now */
   956         -      return SQLITE_OK;
   957         -    }
   958         -  }
   959         -  pInfo->idxNum = QUERY_GENERIC;
   960         -  return SQLITE_OK;
   961         -}
   962         -
   963         -static int fulltextDisconnect(sqlite3_vtab *pVTab){
   964         -  fulltext_vtab_destroy((fulltext_vtab *)pVTab);
   965         -  return SQLITE_OK;
   966         -}
   967         -
   968         -static int fulltextDestroy(sqlite3_vtab *pVTab){
   969         -  fulltext_vtab *v = (fulltext_vtab *)pVTab;
   970         -
   971         -  int rc = sql_exec(v->db, v->zName,
   972         -                    "drop table %_content; drop table %_term");
   973         -  if( rc!=SQLITE_OK ) return rc;
   974         -
   975         -  fulltext_vtab_destroy((fulltext_vtab *)pVTab);
   976         -  return SQLITE_OK;
   977         -}
   978         -
   979         -static int fulltextOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
   980         -  fulltext_cursor *c;
   981         -
   982         -  c = (fulltext_cursor *) calloc(sizeof(fulltext_cursor), 1);
   983         -  /* sqlite will initialize c->base */
   984         -  *ppCursor = &c->base;
   985         -
   986         -  return SQLITE_OK;
   987         -}
   988         -
   989         -static int fulltextClose(sqlite3_vtab_cursor *pCursor){
   990         -  fulltext_cursor *c = (fulltext_cursor *) pCursor;
   991         -  sqlite3_finalize(c->pStmt);
   992         -  if( c->result.pDoclist!=NULL ){
   993         -    docListDelete(c->result.pDoclist);
   994         -  }
   995         -  free(c);
   996         -  return SQLITE_OK;
   997         -}
   998         -
   999         -static int fulltextNext(sqlite3_vtab_cursor *pCursor){
  1000         -  fulltext_cursor *c = (fulltext_cursor *) pCursor;
  1001         -  sqlite_int64 iDocid;
  1002         -  int rc;
  1003         -
  1004         -  switch( c->iCursorType ){
  1005         -    case QUERY_GENERIC:
  1006         -      /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
  1007         -      rc = sqlite3_step(c->pStmt);
  1008         -      switch( rc ){
  1009         -        case SQLITE_ROW:
  1010         -          c->eof = 0;
  1011         -          return SQLITE_OK;
  1012         -        case SQLITE_DONE:
  1013         -          c->eof = 1;
  1014         -          return SQLITE_OK;
  1015         -        default:
  1016         -          c->eof = 1;
  1017         -          return rc;
  1018         -      }
  1019         -    case QUERY_FULLTEXT:
  1020         -      rc = sqlite3_reset(c->pStmt);
  1021         -      if( rc!=SQLITE_OK ) return rc;
  1022         -
  1023         -      if( readerAtEnd(&c->result)){
  1024         -        c->eof = 1;
  1025         -        return SQLITE_OK;
  1026         -      }
  1027         -      iDocid = readDocid(&c->result);
  1028         -      rc = sqlite3_bind_int64(c->pStmt, 1, iDocid);
  1029         -      if( rc!=SQLITE_OK ) return rc;
  1030         -      /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
  1031         -      rc = sqlite3_step(c->pStmt);
  1032         -      if( rc==SQLITE_ROW ){   /* the case we expect */
  1033         -        c->eof = 0;
  1034         -        return SQLITE_OK;
  1035         -      }
  1036         -      /* an error occurred; abort */
  1037         -      return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
  1038         -    default:
  1039         -      assert( 0 );
  1040         -      return SQLITE_ERROR;  /* not reached */
  1041         -  }
  1042         -}
  1043         -
  1044         -static int term_select_doclist(fulltext_vtab *v, const char *pTerm, int nTerm,
  1045         -                               sqlite3_stmt **ppStmt){
  1046         -  int rc;
  1047         -  if( *ppStmt ){
  1048         -    rc = sqlite3_reset(*ppStmt);
  1049         -  } else {
  1050         -    rc = sql_prepare(v->db, v->zName, ppStmt,
  1051         -      "select doclist from %_term where term = ? order by first");
  1052         -  }
  1053         -  if( rc!=SQLITE_OK ) return rc;
  1054         -
  1055         -  rc = sqlite3_bind_text(*ppStmt, 1, pTerm, nTerm, SQLITE_TRANSIENT);
  1056         -  if( rc!=SQLITE_OK ) return rc;
  1057         -
  1058         -  return sqlite3_step(*ppStmt);   /* TODO(adamd): handle schema error */
  1059         -}
  1060         -
  1061         -/* Read the posting list for [zTerm]; AND it with the doclist [in] to
  1062         - * produce the doclist [out], using the given offset [iOffset] for phrase
  1063         - * matching.
  1064         - * (*pSelect) is used to hold an SQLite statement used inside this function;
  1065         - * the caller should initialize *pSelect to NULL before the first call.
  1066         - */
  1067         -static int query_merge(fulltext_vtab *v, sqlite3_stmt **pSelect,
  1068         -                       const char *zTerm,
  1069         -                       DocList *pIn, int iOffset, DocList *out){
  1070         -  int rc;
  1071         -  DocListMerge merge;
  1072         -
  1073         -  if( pIn!=NULL && !pIn->nData ){
  1074         -    /* If [pIn] is already empty, there's no point in reading the
  1075         -     * posting list to AND it in; return immediately. */
  1076         -      return SQLITE_OK;
  1077         -  }
  1078         -
  1079         -  rc = term_select_doclist(v, zTerm, -1, pSelect);
  1080         -  if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ) return rc;
  1081         -
  1082         -  mergeInit(&merge, pIn, iOffset, out);
  1083         -  while( rc==SQLITE_ROW ){
  1084         -    DocList block;
  1085         -    docListInit(&block, DL_POSITIONS_OFFSETS,
  1086         -                sqlite3_column_blob(*pSelect, 0),
  1087         -                sqlite3_column_bytes(*pSelect, 0));
  1088         -    mergeBlock(&merge, &block);
  1089         -    docListDestroy(&block);
  1090         -
  1091         -    rc = sqlite3_step(*pSelect);
  1092         -    if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ){
  1093         -      return rc;
  1094         -    }
  1095         -  }
  1096         -  
  1097         -  return SQLITE_OK;
  1098         -}
  1099         -
  1100         -typedef struct QueryTerm {
  1101         -  int is_phrase;    /* true if this term begins a new phrase */
  1102         -  const char *zTerm;
  1103         -} QueryTerm;
  1104         -
  1105         -/* A parsed query.
  1106         - *
  1107         - * As an example, parsing the query ["four score" years "new nation"] will
  1108         - * yield a Query with 5 terms:
  1109         - *   "four",   is_phrase = 1
  1110         - *   "score",  is_phrase = 0
  1111         - *   "years",  is_phrase = 1
  1112         - *   "new",    is_phrase = 1
  1113         - *   "nation", is_phrase = 0
  1114         - */
  1115         -typedef struct Query {
  1116         -  int nTerms;
  1117         -  QueryTerm *pTerm;
  1118         -} Query;
  1119         -
  1120         -static void query_add(Query *q, int is_phrase, const char *zTerm){
  1121         -  QueryTerm *t;
  1122         -  ++q->nTerms;
  1123         -  q->pTerm = realloc(q->pTerm, q->nTerms * sizeof(q->pTerm[0]));
  1124         -  t = &q->pTerm[q->nTerms - 1];
  1125         -  t->is_phrase = is_phrase;
  1126         -  t->zTerm = zTerm;
  1127         -}
  1128         -    
  1129         -static void query_free(Query *q){
  1130         -  int i;
  1131         -  for(i = 0; i < q->nTerms; ++i){
  1132         -    free((void *) q->pTerm[i].zTerm);
  1133         -  }
  1134         -  free(q->pTerm);
  1135         -}
  1136         -
  1137         -static int tokenize_segment(sqlite3_tokenizer *pTokenizer,
  1138         -                            const char *zQuery, int in_phrase,
  1139         -                            Query *pQuery){
  1140         -  sqlite3_tokenizer_module *pModule = pTokenizer->pModule;
  1141         -  sqlite3_tokenizer_cursor *pCursor;
  1142         -  int is_first = 1;
  1143         -  
  1144         -  int rc = pModule->xOpen(pTokenizer, zQuery, -1, &pCursor);
  1145         -  if( rc!=SQLITE_OK ) return rc;
  1146         -  pCursor->pTokenizer = pTokenizer;
  1147         -
  1148         -  while( 1 ){
  1149         -    const char *zToken;
  1150         -    int nToken, iStartOffset, iEndOffset, dummy_pos;
  1151         -
  1152         -    rc = pModule->xNext(pCursor,
  1153         -                        &zToken, &nToken,
  1154         -                        &iStartOffset, &iEndOffset,
  1155         -                        &dummy_pos);
  1156         -    if( rc!=SQLITE_OK ) break;
  1157         -    query_add(pQuery, !in_phrase || is_first, string_dup_n(zToken, nToken));
  1158         -    is_first = 0;
  1159         -  }
  1160         -
  1161         -  return pModule->xClose(pCursor);
  1162         -}
  1163         -
  1164         -/* Parse a query string, yielding a Query object. */
  1165         -static int parse_query(fulltext_vtab *v, const char *zQuery, Query *pQuery){
  1166         -  char *zQuery1 = string_dup(zQuery);
  1167         -  int in_phrase = 0;
  1168         -  char *s = zQuery1;
  1169         -  pQuery->nTerms = 0;
  1170         -  pQuery->pTerm = NULL;
  1171         -
  1172         -  while( *s ){
  1173         -    char *t = s;
  1174         -    while( *t ){
  1175         -      if( *t=='"' ){
  1176         -        *t++ = '\0';
  1177         -        break;
  1178         -      }
  1179         -      ++t;
  1180         -    }
  1181         -    if( *s ){
  1182         -      tokenize_segment(v->pTokenizer, s, in_phrase, pQuery);
  1183         -    }
  1184         -    s = t;
  1185         -    in_phrase = !in_phrase;
  1186         -  }
  1187         -  
  1188         -  free(zQuery1);
  1189         -  return SQLITE_OK;
  1190         -}
  1191         -
  1192         -/* Perform a full-text query; return a list of documents in [pResult]. */
  1193         -static int fulltext_query(fulltext_vtab *v, const char *zQuery,
  1194         -                          DocList **pResult){
  1195         -  Query q;
  1196         -  int phrase_start = -1;
  1197         -  int i;
  1198         -  sqlite3_stmt *pSelect = NULL;
  1199         -  DocList *d = NULL;
  1200         -
  1201         -  int rc = parse_query(v, zQuery, &q);
  1202         -  if( rc!=SQLITE_OK ) return rc;
  1203         -
  1204         -  /* Merge terms. */
  1205         -  for(i = 0 ; i < q.nTerms ; ++i){
  1206         -    /* In each merge step, we need to generate positions whenever we're
  1207         -     * processing a phrase which hasn't ended yet. */
  1208         -    int need_positions = i<q.nTerms-1 && !q.pTerm[i+1].is_phrase;
  1209         -    DocList *next = docListNew(need_positions ? DL_POSITIONS : DL_DOCIDS);
  1210         -    if( q.pTerm[i].is_phrase ){
  1211         -      phrase_start = i;
  1212         -    }
  1213         -    rc = query_merge(v, &pSelect, q.pTerm[i].zTerm, d, i - phrase_start, next);
  1214         -    if( rc!=SQLITE_OK ) break;
  1215         -    if( d!=NULL ){
  1216         -      docListDelete(d);
  1217         -    }
  1218         -    d = next;
  1219         -  }
  1220         -
  1221         -  sqlite3_finalize(pSelect);
  1222         -  query_free(&q);
  1223         -  *pResult = d;
  1224         -  return rc;
  1225         -}
  1226         -
  1227         -static int fulltextFilter(sqlite3_vtab_cursor *pCursor,
  1228         -                          int idxNum, const char *idxStr,
  1229         -                          int argc, sqlite3_value **argv){
  1230         -  fulltext_cursor *c = (fulltext_cursor *) pCursor;
  1231         -  fulltext_vtab *v = cursor_vtab(c);
  1232         -  int rc;
  1233         -  const char *zStatement;
  1234         -
  1235         -  c->iCursorType = idxNum;
  1236         -  switch( idxNum ){
  1237         -    case QUERY_GENERIC:
  1238         -      zStatement = "select rowid, content from %_content";
  1239         -      break;
  1240         -
  1241         -    case QUERY_FULLTEXT:   /* full-text search */
  1242         -    {
  1243         -      const char *zQuery = (const char *)sqlite3_value_text(argv[0]);
  1244         -      DocList *pResult;
  1245         -      assert( argc==1 );
  1246         -      rc = fulltext_query(v, zQuery, &pResult);
  1247         -      if( rc!=SQLITE_OK ) return rc;
  1248         -      readerInit(&c->result, pResult);
  1249         -      zStatement = "select rowid, content from %_content where rowid = ?";
  1250         -      break;
  1251         -    }
  1252         -
  1253         -    default:
  1254         -      assert( 0 );
  1255         -  }
  1256         -
  1257         -  rc = sql_prepare(v->db, v->zName, &c->pStmt, zStatement);
  1258         -  if( rc!=SQLITE_OK ) return rc;
  1259         -
  1260         -  return fulltextNext(pCursor);
  1261         -}
  1262         -
  1263         -static int fulltextEof(sqlite3_vtab_cursor *pCursor){
  1264         -  fulltext_cursor *c = (fulltext_cursor *) pCursor;
  1265         -  return c->eof;
  1266         -}
  1267         -
  1268         -static int fulltextColumn(sqlite3_vtab_cursor *pCursor,
  1269         -                          sqlite3_context *pContext, int idxCol){
  1270         -  fulltext_cursor *c = (fulltext_cursor *) pCursor;
  1271         -  const char *s;
  1272         -
  1273         -  assert( idxCol==0 );
  1274         -  s = (const char *) sqlite3_column_text(c->pStmt, 1);
  1275         -  sqlite3_result_text(pContext, s, -1, SQLITE_TRANSIENT);
  1276         -
  1277         -  return SQLITE_OK;
  1278         -}
  1279         -
  1280         -static int fulltextRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
  1281         -  fulltext_cursor *c = (fulltext_cursor *) pCursor;
  1282         -
  1283         -  *pRowid = sqlite3_column_int64(c->pStmt, 0);
  1284         -  return SQLITE_OK;
  1285         -}
  1286         -
  1287         -/* Build a hash table containing all terms in zText. */
  1288         -static int build_terms(Hash *terms, sqlite3_tokenizer *pTokenizer,
  1289         -                       const char *zText, sqlite_int64 iDocid){
  1290         -  sqlite3_tokenizer_cursor *pCursor;
  1291         -  const char *pToken;
  1292         -  int nTokenBytes;
  1293         -  int iStartOffset, iEndOffset, iPosition;
  1294         -
  1295         -  int rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
  1296         -  if( rc!=SQLITE_OK ) return rc;
  1297         -
  1298         -  pCursor->pTokenizer = pTokenizer;
  1299         -  HashInit(terms, HASH_STRING, 1);
  1300         -  while( SQLITE_OK==pTokenizer->pModule->xNext(pCursor,
  1301         -                                               &pToken, &nTokenBytes,
  1302         -                                               &iStartOffset, &iEndOffset,
  1303         -                                               &iPosition) ){
  1304         -    DocList *p;
  1305         -
  1306         -    /* Positions can't be negative; we use -1 as a terminator internally. */
  1307         -    if( iPosition<0 ) {
  1308         -      rc = SQLITE_ERROR;  
  1309         -      goto err;
  1310         -    }
  1311         -
  1312         -    p = HashFind(terms, pToken, nTokenBytes);
  1313         -    if( p==NULL ){
  1314         -      p = docListNew(DL_POSITIONS_OFFSETS);
  1315         -      docListAddDocid(p, iDocid);
  1316         -      HashInsert(terms, pToken, nTokenBytes, p);
  1317         -    }
  1318         -    docListAddPosOffset(p, iPosition, iStartOffset, iEndOffset);
  1319         -  }
  1320         -
  1321         -err:
  1322         -  /* TODO(shess) Check return?  Should this be able to cause errors at
  1323         -  ** this point?  Actually, same question about sqlite3_finalize(),
  1324         -  ** though one could argue that failure there means that the data is
  1325         -  ** not durable.  *ponder*
  1326         -  */
  1327         -  pTokenizer->pModule->xClose(pCursor);
  1328         -  return rc;
  1329         -}
  1330         -/* Update the %_terms table to map the term [zTerm] to the given rowid. */
  1331         -static int index_insert_term(fulltext_vtab *v, const char *zTerm, int nTerm,
  1332         -                             sqlite_int64 iDocid, DocList *p){
  1333         -  sqlite_int64 iFirst;
  1334         -  sqlite_int64 iIndexRow;
  1335         -  DocList doclist;
  1336         -
  1337         -  int rc = term_chunk_select(v, zTerm, nTerm, iDocid, &iFirst);
  1338         -  if( rc==SQLITE_DONE ){
  1339         -    docListInit(&doclist, DL_POSITIONS_OFFSETS, 0, 0);
  1340         -    if( docListUpdate(&doclist, iDocid, p) ){
  1341         -      rc = term_insert(v, zTerm, nTerm, iDocid, &doclist);
  1342         -      docListDestroy(&doclist);
  1343         -      return rc;
  1344         -    }
  1345         -    return SQLITE_OK;
  1346         -  }
  1347         -  if( rc!=SQLITE_ROW ) return SQLITE_ERROR;
  1348         -
  1349         -  /* This word is in the index; add this document ID to its blob. */
  1350         -
  1351         -  rc = term_select(v, zTerm, nTerm, iFirst, &iIndexRow, &doclist);
  1352         -  if( rc!=SQLITE_OK ) return rc;
  1353         -
  1354         -  if( docListUpdate(&doclist, iDocid, p) ){
  1355         -    /* If the blob is too big, split it in half. */
  1356         -    if( doclist.nData>CHUNK_MAX ){
  1357         -      DocList half;
  1358         -      if( docListSplit(&doclist, &half) ){
  1359         -        rc = term_insert(v, zTerm, nTerm, firstDocid(&half), &half);
  1360         -        docListDestroy(&half);
  1361         -        if( rc!=SQLITE_OK ) goto err;
  1362         -      }
  1363         -    }
  1364         -    rc = term_update(v, iIndexRow, &doclist);
  1365         -  }
  1366         -
  1367         -err:
  1368         -  docListDestroy(&doclist);
  1369         -  return rc;
  1370         -}
  1371         -
  1372         -/* Insert a row into the full-text index; set *piRowid to be the ID of the
  1373         - * new row. */
  1374         -static int index_insert(fulltext_vtab *v,
  1375         -                        sqlite3_value *pRequestRowid, const char *zText,
  1376         -                        sqlite_int64 *piRowid){
  1377         -  Hash terms;  /* maps term string -> PosList */
  1378         -  HashElem *e;
  1379         -
  1380         -  int rc = content_insert(v, pRequestRowid, zText, -1);
  1381         -  if( rc!=SQLITE_OK ) return rc;
  1382         -  *piRowid = sqlite3_last_insert_rowid(v->db);
  1383         -
  1384         -  if( !zText ) return SQLITE_OK;   /* nothing to index */
  1385         -
  1386         -  rc = build_terms(&terms, v->pTokenizer, zText, *piRowid);
  1387         -  if( rc!=SQLITE_OK ) return rc;
  1388         -
  1389         -  for(e=HashFirst(&terms); e; e=HashNext(e)){
  1390         -    DocList *p = HashData(e);
  1391         -    rc = index_insert_term(v, HashKey(e), HashKeysize(e), *piRowid, p);
  1392         -    if( rc!=SQLITE_OK ) break;
  1393         -  }
  1394         -
  1395         -  for(e=HashFirst(&terms); e; e=HashNext(e)){
  1396         -    DocList *p = HashData(e);
  1397         -    docListDelete(p);
  1398         -  }
  1399         -  HashClear(&terms);
  1400         -  return rc;
  1401         -}
  1402         -
  1403         -static int index_delete_term(fulltext_vtab *v, const char *zTerm, int nTerm,
  1404         -                             sqlite_int64 iDocid){
  1405         -  sqlite_int64 iFirst;
  1406         -  sqlite_int64 iIndexRow;
  1407         -  DocList doclist;
  1408         -
  1409         -  int rc = term_chunk_select(v, zTerm, nTerm, iDocid, &iFirst);
  1410         -  if( rc!=SQLITE_ROW ) return SQLITE_ERROR;
  1411         -
  1412         -  rc = term_select(v, zTerm, nTerm, iFirst, &iIndexRow, &doclist);
  1413         -  if( rc!=SQLITE_OK ) return rc;
  1414         -
  1415         -  if( docListUpdate(&doclist, iDocid, NULL) ){
  1416         -    if( doclist.nData>0 ){
  1417         -      rc = term_update(v, iIndexRow, &doclist);
  1418         -    } else {  /* empty posting list */
  1419         -      rc = term_delete(v, iIndexRow);
  1420         -    }
  1421         -  }
  1422         -  docListDestroy(&doclist);
  1423         -  return rc;
  1424         -}
  1425         -
  1426         -/* Delete a row from the full-text index. */
  1427         -static int index_delete(fulltext_vtab *v, sqlite_int64 iRow){
  1428         -  char *zText;
  1429         -  Hash terms;
  1430         -  HashElem *e;
  1431         -
  1432         -  int rc = content_select(v, iRow, &zText);
  1433         -  if( rc!=SQLITE_OK ) return rc;
  1434         -
  1435         -  rc = build_terms(&terms, v->pTokenizer, zText, iRow);
  1436         -  free(zText);
  1437         -  if( rc!=SQLITE_OK ) return rc;
  1438         -
  1439         -  for(e=HashFirst(&terms); e; e=HashNext(e)){
  1440         -    rc = index_delete_term(v, HashKey(e), HashKeysize(e), iRow);
  1441         -    if( rc!=SQLITE_OK ) break;
  1442         -  }
  1443         -  for(e=HashFirst(&terms); e; e=HashNext(e)){
  1444         -    DocList *p = HashData(e);
  1445         -    docListDelete(p);
  1446         -  }
  1447         -  HashClear(&terms);
  1448         -
  1449         -  return content_delete(v, iRow);
  1450         -}
  1451         -
  1452         -static int fulltextUpdate(sqlite3_vtab *pVtab, int nArg, sqlite3_value **ppArg,
  1453         -                   sqlite_int64 *pRowid){
  1454         -  fulltext_vtab *v = (fulltext_vtab *) pVtab;
  1455         -
  1456         -  if( nArg<2 ){
  1457         -    return index_delete(v, sqlite3_value_int64(ppArg[0]));
  1458         -  }
  1459         -
  1460         -  if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){
  1461         -    return SQLITE_ERROR;   /* an update; not yet supported */
  1462         -  }
  1463         -
  1464         -  assert( nArg==3 );    /* ppArg[1] = rowid, ppArg[2] = content */
  1465         -  return index_insert(v, ppArg[1],
  1466         -                      (const char *)sqlite3_value_text(ppArg[2]), pRowid);
  1467         -}
  1468         -
  1469         -static sqlite3_module fulltextModule = {
  1470         -  0,
  1471         -  fulltextCreate,
  1472         -  fulltextConnect,
  1473         -  fulltextBestIndex,
  1474         -  fulltextDisconnect,
  1475         -  fulltextDestroy,
  1476         -  fulltextOpen,
  1477         -  fulltextClose,
  1478         -  fulltextFilter,
  1479         -  fulltextNext,
  1480         -  fulltextEof,
  1481         -  fulltextColumn,
  1482         -  fulltextRowid,
  1483         -  fulltextUpdate
  1484         -};
  1485         -
  1486         -int fulltext_init(sqlite3 *db){
  1487         - return sqlite3_create_module(db, "fulltext", &fulltextModule, 0);
  1488         -}
  1489         -
  1490         -#if !SQLITE_CORE
  1491         -int sqlite3_extension_init(sqlite3 *db, char **pzErrMsg,
  1492         -                           const sqlite3_api_routines *pApi){
  1493         - SQLITE_EXTENSION_INIT2(pApi)
  1494         - return fulltext_init(db);
  1495         -}
  1496         -#endif

Deleted ext/fts1/fulltext.h.

     1         -#include "sqlite3.h"
     2         -
     3         -#ifdef __cplusplus
     4         -extern "C" {
     5         -#endif  /* __cplusplus */
     6         -
     7         -int fulltext_init(sqlite3 *db);
     8         -
     9         -#ifdef __cplusplus
    10         -}  /* extern "C" */
    11         -#endif  /* __cplusplus */

Deleted ext/fts1/simple_tokenizer.c.

     1         -/*
     2         -** The author disclaims copyright to this source code.
     3         -**
     4         -*************************************************************************
     5         -** Implementation of the "simple" full-text-search tokenizer.
     6         -*/
     7         -
     8         -#include <assert.h>
     9         -#if !defined(__APPLE__)
    10         -#include <malloc.h>
    11         -#else
    12         -#include <stdlib.h>
    13         -#endif
    14         -#include <stdio.h>
    15         -#include <string.h>
    16         -#include <ctype.h>
    17         -
    18         -#include "tokenizer.h"
    19         -
    20         -/* Duplicate a string; the caller must free() the returned string.
    21         - * (We don't use strdup() since it's not part of the standard C library and
    22         - * may not be available everywhere.) */
    23         -/* TODO(shess) Copied from fulltext.c, consider util.c for such
    24         -** things. */
    25         -static char *string_dup(const char *s){
    26         -  char *str = malloc(strlen(s) + 1);
    27         -  strcpy(str, s);
    28         -  return str;
    29         -}
    30         -
    31         -typedef struct simple_tokenizer {
    32         -  sqlite3_tokenizer base;
    33         -  const char *zDelim;          /* token delimiters */
    34         -} simple_tokenizer;
    35         -
    36         -typedef struct simple_tokenizer_cursor {
    37         -  sqlite3_tokenizer_cursor base;
    38         -  const char *pInput;          /* input we are tokenizing */
    39         -  int nBytes;                  /* size of the input */
    40         -  const char *pCurrent;        /* current position in pInput */
    41         -  int iToken;                  /* index of next token to be returned */
    42         -  char *zToken;                /* storage for current token */
    43         -  int nTokenBytes;             /* actual size of current token */
    44         -  int nTokenAllocated;         /* space allocated to zToken buffer */
    45         -} simple_tokenizer_cursor;
    46         -
    47         -static sqlite3_tokenizer_module simpleTokenizerModule;/* forward declaration */
    48         -
    49         -static int simpleCreate(
    50         -  int argc, const char **argv,
    51         -  sqlite3_tokenizer **ppTokenizer
    52         -){
    53         -  simple_tokenizer *t;
    54         -
    55         -  t = (simple_tokenizer *) malloc(sizeof(simple_tokenizer));
    56         -  /* TODO(shess) Delimiters need to remain the same from run to run,
    57         -  ** else we need to reindex.  One solution would be a meta-table to
    58         -  ** track such information in the database, then we'd only want this
    59         -  ** information on the initial create.
    60         -  */
    61         -  if( argc>1 ){
    62         -    t->zDelim = string_dup(argv[1]);
    63         -  } else {
    64         -    /* Build a string excluding alphanumeric ASCII characters */
    65         -    char zDelim[0x80];               /* nul-terminated, so nul not a member */
    66         -    int i, j;
    67         -    for(i=1, j=0; i<0x80; i++){
    68         -      if( !isalnum(i) ){
    69         -        zDelim[j++] = i;
    70         -      }
    71         -    }
    72         -    zDelim[j++] = '\0';
    73         -    assert( j<=sizeof(zDelim) );
    74         -    t->zDelim = string_dup(zDelim);
    75         -  }
    76         -
    77         -  *ppTokenizer = &t->base;
    78         -  return SQLITE_OK;
    79         -}
    80         -
    81         -static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
    82         -  simple_tokenizer *t = (simple_tokenizer *) pTokenizer;
    83         -
    84         -  free((void *) t->zDelim);
    85         -  free(t);
    86         -
    87         -  return SQLITE_OK;
    88         -}
    89         -
    90         -static int simpleOpen(
    91         -  sqlite3_tokenizer *pTokenizer,
    92         -  const char *pInput, int nBytes,
    93         -  sqlite3_tokenizer_cursor **ppCursor
    94         -){
    95         -  simple_tokenizer_cursor *c;
    96         -
    97         -  c = (simple_tokenizer_cursor *) malloc(sizeof(simple_tokenizer_cursor));
    98         -  c->pInput = pInput;
    99         -  c->nBytes = nBytes<0 ? (int) strlen(pInput) : nBytes;
   100         -  c->pCurrent = c->pInput;        /* start tokenizing at the beginning */
   101         -  c->iToken = 0;
   102         -  c->zToken = NULL;               /* no space allocated, yet. */
   103         -  c->nTokenBytes = 0;
   104         -  c->nTokenAllocated = 0;
   105         -
   106         -  *ppCursor = &c->base;
   107         -  return SQLITE_OK;
   108         -}
   109         -
   110         -static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
   111         -  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
   112         -
   113         -  if( NULL!=c->zToken ){
   114         -    free(c->zToken);
   115         -  }
   116         -  free(c);
   117         -
   118         -  return SQLITE_OK;
   119         -}
   120         -
   121         -static int simpleNext(
   122         -  sqlite3_tokenizer_cursor *pCursor,
   123         -  const char **ppToken, int *pnBytes,
   124         -  int *piStartOffset, int *piEndOffset, int *piPosition
   125         -){
   126         -  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
   127         -  simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
   128         -  int ii;
   129         -
   130         -  while( c->pCurrent-c->pInput<c->nBytes ){
   131         -    int n = (int) strcspn(c->pCurrent, t->zDelim);
   132         -    if( n>0 ){
   133         -      if( n+1>c->nTokenAllocated ){
   134         -        c->zToken = realloc(c->zToken, n+1);
   135         -      }
   136         -      for(ii=0; ii<n; ii++){
   137         -        /* TODO(shess) This needs expansion to handle UTF-8
   138         -        ** case-insensitivity.
   139         -        */
   140         -        char ch = c->pCurrent[ii];
   141         -        c->zToken[ii] = (unsigned char)ch<0x80 ? tolower(ch) : ch;
   142         -      }
   143         -      c->zToken[n] = '\0';
   144         -      *ppToken = c->zToken;
   145         -      *pnBytes = n;
   146         -      *piStartOffset = (int) (c->pCurrent-c->pInput);
   147         -      *piEndOffset = *piStartOffset+n;
   148         -      *piPosition = c->iToken++;
   149         -      c->pCurrent += n + 1;
   150         -
   151         -      return SQLITE_OK;
   152         -    }
   153         -    c->pCurrent += n + 1;
   154         -    /* TODO(shess) could strspn() to skip delimiters en masse.  Needs
   155         -    ** to happen in two places, though, which is annoying.
   156         -    */
   157         -  }
   158         -  return SQLITE_DONE;
   159         -}
   160         -
   161         -static sqlite3_tokenizer_module simpleTokenizerModule = {
   162         -  0,
   163         -  simpleCreate,
   164         -  simpleDestroy,
   165         -  simpleOpen,
   166         -  simpleClose,
   167         -  simpleNext,
   168         -};
   169         -
   170         -void get_simple_tokenizer_module(
   171         -  sqlite3_tokenizer_module **ppModule
   172         -){
   173         -  *ppModule = &simpleTokenizerModule;
   174         -}

Deleted ext/fts1/tokenizer.h.

     1         -/*
     2         -** 2006 July 10
     3         -**
     4         -** The author disclaims copyright to this source code.
     5         -**
     6         -*************************************************************************
     7         -** Defines the interface to tokenizers used by fulltext-search.  There
     8         -** are three basic components:
     9         -**
    10         -** sqlite3_tokenizer_module is a singleton defining the tokenizer
    11         -** interface functions.  This is essentially the class structure for
    12         -** tokenizers.
    13         -**
    14         -** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
    15         -** including customization information defined at creation time.
    16         -**
    17         -** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
    18         -** tokens from a particular input.
    19         -*/
    20         -#ifndef _TOKENIZER_H_
    21         -#define _TOKENIZER_H_
    22         -
    23         -/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
    24         -** If tokenizers are to be allowed to call sqlite3_*() functions, then
    25         -** we will need a way to register the API consistently.
    26         -*/
    27         -#include "sqlite3.h"
    28         -
    29         -/*
    30         -** Structures used by the tokenizer interface.
    31         -*/
    32         -typedef struct sqlite3_tokenizer sqlite3_tokenizer;
    33         -typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
    34         -typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
    35         -
    36         -struct sqlite3_tokenizer_module {
    37         -  int iVersion;                  /* currently 0 */
    38         -
    39         -  /*
    40         -  ** Create and destroy a tokenizer.  argc/argv are passed down from
    41         -  ** the fulltext virtual table creation to allow customization.
    42         -  */
    43         -  int (*xCreate)(int argc, const char **argv,
    44         -                 sqlite3_tokenizer **ppTokenizer);
    45         -  int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
    46         -
    47         -  /*
    48         -  ** Tokenize a particular input.  Call xOpen() to prepare to
    49         -  ** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then
    50         -  ** xClose() to free any internal state.  The pInput passed to
    51         -  ** xOpen() must exist until the cursor is closed.  The ppToken
    52         -  ** result from xNext() is only valid until the next call to xNext()
    53         -  ** or until xClose() is called.
    54         -  */
    55         -  /* TODO(shess) current implementation requires pInput to be
    56         -  ** nul-terminated.  This should either be fixed, or pInput/nBytes
    57         -  ** should be converted to zInput.
    58         -  */
    59         -  int (*xOpen)(sqlite3_tokenizer *pTokenizer,
    60         -               const char *pInput, int nBytes,
    61         -               sqlite3_tokenizer_cursor **ppCursor);
    62         -  int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
    63         -  int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
    64         -               const char **ppToken, int *pnBytes,
    65         -               int *piStartOffset, int *piEndOffset, int *piPosition);
    66         -};
    67         -
    68         -struct sqlite3_tokenizer {
    69         -  sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */
    70         -  /* Tokenizer implementations will typically add additional fields */
    71         -};
    72         -
    73         -struct sqlite3_tokenizer_cursor {
    74         -  sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */
    75         -  /* Tokenizer implementations will typically add additional fields */
    76         -};
    77         -
    78         -/*
    79         -** Get the module for a tokenizer which generates tokens based on a
    80         -** set of non-token characters.  The default is to break tokens at any
    81         -** non-alnum character, though the set of delimiters can also be
    82         -** specified by the first argv argument to xCreate().
    83         -*/
    84         -/* TODO(shess) This doesn't belong here.  Need some sort of
    85         -** registration process.
    86         -*/
    87         -void get_simple_tokenizer_module(sqlite3_tokenizer_module **ppModule);
    88         -
    89         -#endif /* _TOKENIZER_H_ */

Deleted sqlite3.def.

     1         -EXPORTS
     2         -sqlite3_aggregate_context
     3         -sqlite3_aggregate_count
     4         -sqlite3_bind_blob
     5         -sqlite3_bind_double
     6         -sqlite3_bind_int
     7         -sqlite3_bind_int64
     8         -sqlite3_bind_null
     9         -sqlite3_bind_parameter_count
    10         -sqlite3_bind_parameter_index
    11         -sqlite3_bind_parameter_name
    12         -sqlite3_bind_text
    13         -sqlite3_bind_text16
    14         -sqlite3_busy_handler
    15         -sqlite3_busy_timeout
    16         -sqlite3_changes
    17         -sqlite3_close
    18         -sqlite3_collation_needed
    19         -sqlite3_collation_needed16
    20         -sqlite3_column_blob
    21         -sqlite3_column_bytes
    22         -sqlite3_column_bytes16
    23         -sqlite3_column_count
    24         -sqlite3_column_decltype
    25         -sqlite3_column_decltype16
    26         -sqlite3_column_double
    27         -sqlite3_column_int
    28         -sqlite3_column_int64
    29         -sqlite3_column_name
    30         -sqlite3_column_name16
    31         -sqlite3_column_text
    32         -sqlite3_column_text16
    33         -sqlite3_column_type
    34         -sqlite3_commit_hook
    35         -sqlite3_complete
    36         -sqlite3_complete16
    37         -sqlite3_create_collation
    38         -sqlite3_create_collation16
    39         -sqlite3_create_function
    40         -sqlite3_create_function16
    41         -sqlite3_data_count
    42         -sqlite3_db_handle
    43         -sqlite3_enable_load_extension
    44         -sqlite3_enable_shared_cache
    45         -sqlite3_errcode
    46         -sqlite3_errmsg
    47         -sqlite3_errmsg16
    48         -sqlite3_exec
    49         -sqlite3_expired
    50         -sqlite3_finalize
    51         -sqlite3_free
    52         -sqlite3_free_table
    53         -sqlite3_get_autocommit
    54         -sqlite3_get_auxdata
    55         -sqlite3_get_table
    56         -sqlite3_global_recover
    57         -sqlite3_interrupt
    58         -sqlite3_last_insert_rowid
    59         -sqlite3_libversion
    60         -sqlite3_libversion_number
    61         -sqlite3_load_extension
    62         -sqlite3_malloc
    63         -sqlite3_mprintf
    64         -sqlite3_open
    65         -sqlite3_open16
    66         -sqlite3_prepare
    67         -sqlite3_prepare16
    68         -sqlite3_progress_handler
    69         -sqlite3_realloc
    70         -sqlite3_reset
    71         -sqlite3_result_blob
    72         -sqlite3_result_double
    73         -sqlite3_result_error
    74         -sqlite3_result_error16
    75         -sqlite3_result_int
    76         -sqlite3_result_int64
    77         -sqlite3_result_null
    78         -sqlite3_result_text
    79         -sqlite3_result_text16
    80         -sqlite3_result_text16be
    81         -sqlite3_result_text16le
    82         -sqlite3_result_value
    83         -sqlite3_rollback_hook
    84         -sqlite3_set_authorizer
    85         -sqlite3_set_auxdata
    86         -sqlite3_snprintf
    87         -sqlite3_step
    88         -sqlite3_thread_cleanup
    89         -sqlite3_total_changes
    90         -sqlite3_trace
    91         -sqlite3_transfer_bindings
    92         -sqlite3_update_hook
    93         -sqlite3_user_data
    94         -sqlite3_value_blob
    95         -sqlite3_value_bytes
    96         -sqlite3_value_bytes16
    97         -sqlite3_value_double
    98         -sqlite3_value_int
    99         -sqlite3_value_int64
   100         -sqlite3_value_text
   101         -sqlite3_value_text16
   102         -sqlite3_value_text16be
   103         -sqlite3_value_text16le
   104         -sqlite3_value_type
   105         -sqlite3_vmprintf

Deleted src/experimental.c.

     1         -/*
     2         -** 2005 January 20
     3         -**
     4         -** The author disclaims copyright to this source code.  In place of
     5         -** a legal notice, here is a blessing:
     6         -**
     7         -**    May you do good and not evil.
     8         -**    May you find forgiveness for yourself and forgive others.
     9         -**    May you share freely, never taking more than you give.
    10         -**
    11         -*************************************************************************
    12         -** This file contains C code routines that are not a part of the official
    13         -** SQLite API.  These routines are unsupported.
    14         -**
    15         -** $Id: experimental.c,v 1.4 2006/01/31 20:49:13 drh Exp $
    16         -*/
    17         -#include "sqliteInt.h"
    18         -#include "os.h"
    19         -
    20         -/*
    21         -** Set all the parameters in the compiled SQL statement to NULL.
    22         -*/
    23         -int sqlite3_clear_bindings(sqlite3_stmt *pStmt){
    24         -  int i;
    25         -  int rc = SQLITE_OK;
    26         -  for(i=1; rc==SQLITE_OK && i<=sqlite3_bind_parameter_count(pStmt); i++){
    27         -    rc = sqlite3_bind_null(pStmt, i);
    28         -  }
    29         -  return rc;
    30         -}
    31         -
    32         -/*
    33         -** Sleep for a little while.  Return the amount of time slept.
    34         -*/
    35         -int sqlite3_sleep(int ms){
    36         -  return sqlite3OsSleep(ms);
    37         -}

Deleted src/md5.c.

     1         -/*
     2         -** SQLite uses this code for testing only.  It is not a part of
     3         -** the SQLite library.  This file implements two new TCL commands
     4         -** "md5" and "md5file" that compute md5 checksums on arbitrary text
     5         -** and on complete files.  These commands are used by the "testfixture"
     6         -** program to help verify the correct operation of the SQLite library.
     7         -**
     8         -** The original use of these TCL commands was to test the ROLLBACK
     9         -** feature of SQLite.  First compute the MD5-checksum of the database.
    10         -** Then make some changes but rollback the changes rather than commit
    11         -** them.  Compute a second MD5-checksum of the file and verify that the
    12         -** two checksums are the same.  Such is the original use of this code.
    13         -** New uses may have been added since this comment was written.
    14         -*/
    15         -/*
    16         - * This code implements the MD5 message-digest algorithm.
    17         - * The algorithm is due to Ron Rivest.  This code was
    18         - * written by Colin Plumb in 1993, no copyright is claimed.
    19         - * This code is in the public domain; do with it what you wish.
    20         - *
    21         - * Equivalent code is available from RSA Data Security, Inc.
    22         - * This code has been tested against that, and is equivalent,
    23         - * except that you don't need to include two pages of legalese
    24         - * with every copy.
    25         - *
    26         - * To compute the message digest of a chunk of bytes, declare an
    27         - * MD5Context structure, pass it to MD5Init, call MD5Update as
    28         - * needed on buffers full of bytes, and then call MD5Final, which
    29         - * will fill a supplied 16-byte array with the digest.
    30         - */
    31         -#include <tcl.h>
    32         -#include <string.h>
    33         -#include "sqlite3.h"
    34         -
    35         -/*
    36         - * If compiled on a machine that doesn't have a 32-bit integer,
    37         - * you just set "uint32" to the appropriate datatype for an
    38         - * unsigned 32-bit integer.  For example:
    39         - *
    40         - *       cc -Duint32='unsigned long' md5.c
    41         - *
    42         - */
    43         -#ifndef uint32
    44         -#  define uint32 unsigned int
    45         -#endif
    46         -
    47         -struct Context {
    48         -  uint32 buf[4];
    49         -  uint32 bits[2];
    50         -  unsigned char in[64];
    51         -};
    52         -typedef char MD5Context[88];
    53         -
    54         -/*
    55         - * Note: this code is harmless on little-endian machines.
    56         - */
    57         -static void byteReverse (unsigned char *buf, unsigned longs){
    58         -        uint32 t;
    59         -        do {
    60         -                t = (uint32)((unsigned)buf[3]<<8 | buf[2]) << 16 |
    61         -                            ((unsigned)buf[1]<<8 | buf[0]);
    62         -                *(uint32 *)buf = t;
    63         -                buf += 4;
    64         -        } while (--longs);
    65         -}
    66         -/* The four core functions - F1 is optimized somewhat */
    67         -
    68         -/* #define F1(x, y, z) (x & y | ~x & z) */
    69         -#define F1(x, y, z) (z ^ (x & (y ^ z)))
    70         -#define F2(x, y, z) F1(z, x, y)
    71         -#define F3(x, y, z) (x ^ y ^ z)
    72         -#define F4(x, y, z) (y ^ (x | ~z))
    73         -
    74         -/* This is the central step in the MD5 algorithm. */
    75         -#define MD5STEP(f, w, x, y, z, data, s) \
    76         -        ( w += f(x, y, z) + data,  w = w<<s | w>>(32-s),  w += x )
    77         -
    78         -/*
    79         - * The core of the MD5 algorithm, this alters an existing MD5 hash to
    80         - * reflect the addition of 16 longwords of new data.  MD5Update blocks
    81         - * the data and converts bytes into longwords for this routine.
    82         - */
    83         -static void MD5Transform(uint32 buf[4], const uint32 in[16]){
    84         -        register uint32 a, b, c, d;
    85         -
    86         -        a = buf[0];
    87         -        b = buf[1];
    88         -        c = buf[2];
    89         -        d = buf[3];
    90         -
    91         -        MD5STEP(F1, a, b, c, d, in[ 0]+0xd76aa478,  7);
    92         -        MD5STEP(F1, d, a, b, c, in[ 1]+0xe8c7b756, 12);
    93         -        MD5STEP(F1, c, d, a, b, in[ 2]+0x242070db, 17);
    94         -        MD5STEP(F1, b, c, d, a, in[ 3]+0xc1bdceee, 22);
    95         -        MD5STEP(F1, a, b, c, d, in[ 4]+0xf57c0faf,  7);
    96         -        MD5STEP(F1, d, a, b, c, in[ 5]+0x4787c62a, 12);
    97         -        MD5STEP(F1, c, d, a, b, in[ 6]+0xa8304613, 17);
    98         -        MD5STEP(F1, b, c, d, a, in[ 7]+0xfd469501, 22);
    99         -        MD5STEP(F1, a, b, c, d, in[ 8]+0x698098d8,  7);
   100         -        MD5STEP(F1, d, a, b, c, in[ 9]+0x8b44f7af, 12);
   101         -        MD5STEP(F1, c, d, a, b, in[10]+0xffff5bb1, 17);
   102         -        MD5STEP(F1, b, c, d, a, in[11]+0x895cd7be, 22);
   103         -        MD5STEP(F1, a, b, c, d, in[12]+0x6b901122,  7);
   104         -        MD5STEP(F1, d, a, b, c, in[13]+0xfd987193, 12);
   105         -        MD5STEP(F1, c, d, a, b, in[14]+0xa679438e, 17);
   106         -        MD5STEP(F1, b, c, d, a, in[15]+0x49b40821, 22);
   107         -
   108         -        MD5STEP(F2, a, b, c, d, in[ 1]+0xf61e2562,  5);
   109         -        MD5STEP(F2, d, a, b, c, in[ 6]+0xc040b340,  9);
   110         -        MD5STEP(F2, c, d, a, b, in[11]+0x265e5a51, 14);
   111         -        MD5STEP(F2, b, c, d, a, in[ 0]+0xe9b6c7aa, 20);
   112         -        MD5STEP(F2, a, b, c, d, in[ 5]+0xd62f105d,  5);
   113         -        MD5STEP(F2, d, a, b, c, in[10]+0x02441453,  9);
   114         -        MD5STEP(F2, c, d, a, b, in[15]+0xd8a1e681, 14);
   115         -        MD5STEP(F2, b, c, d, a, in[ 4]+0xe7d3fbc8, 20);
   116         -        MD5STEP(F2, a, b, c, d, in[ 9]+0x21e1cde6,  5);
   117         -        MD5STEP(F2, d, a, b, c, in[14]+0xc33707d6,  9);
   118         -        MD5STEP(F2, c, d, a, b, in[ 3]+0xf4d50d87, 14);
   119         -        MD5STEP(F2, b, c, d, a, in[ 8]+0x455a14ed, 20);
   120         -        MD5STEP(F2, a, b, c, d, in[13]+0xa9e3e905,  5);
   121         -        MD5STEP(F2, d, a, b, c, in[ 2]+0xfcefa3f8,  9);
   122         -        MD5STEP(F2, c, d, a, b, in[ 7]+0x676f02d9, 14);
   123         -        MD5STEP(F2, b, c, d, a, in[12]+0x8d2a4c8a, 20);
   124         -
   125         -        MD5STEP(F3, a, b, c, d, in[ 5]+0xfffa3942,  4);
   126         -        MD5STEP(F3, d, a, b, c, in[ 8]+0x8771f681, 11);
   127         -        MD5STEP(F3, c, d, a, b, in[11]+0x6d9d6122, 16);
   128         -        MD5STEP(F3, b, c, d, a, in[14]+0xfde5380c, 23);
   129         -        MD5STEP(F3, a, b, c, d, in[ 1]+0xa4beea44,  4);
   130         -        MD5STEP(F3, d, a, b, c, in[ 4]+0x4bdecfa9, 11);
   131         -        MD5STEP(F3, c, d, a, b, in[ 7]+0xf6bb4b60, 16);
   132         -        MD5STEP(F3, b, c, d, a, in[10]+0xbebfbc70, 23);
   133         -        MD5STEP(F3, a, b, c, d, in[13]+0x289b7ec6,  4);
   134         -        MD5STEP(F3, d, a, b, c, in[ 0]+0xeaa127fa, 11);
   135         -        MD5STEP(F3, c, d, a, b, in[ 3]+0xd4ef3085, 16);
   136         -        MD5STEP(F3, b, c, d, a, in[ 6]+0x04881d05, 23);
   137         -        MD5STEP(F3, a, b, c, d, in[ 9]+0xd9d4d039,  4);
   138         -        MD5STEP(F3, d, a, b, c, in[12]+0xe6db99e5, 11);
   139         -        MD5STEP(F3, c, d, a, b, in[15]+0x1fa27cf8, 16);
   140         -        MD5STEP(F3, b, c, d, a, in[ 2]+0xc4ac5665, 23);
   141         -
   142         -        MD5STEP(F4, a, b, c, d, in[ 0]+0xf4292244,  6);
   143         -        MD5STEP(F4, d, a, b, c, in[ 7]+0x432aff97, 10);
   144         -        MD5STEP(F4, c, d, a, b, in[14]+0xab9423a7, 15);
   145         -        MD5STEP(F4, b, c, d, a, in[ 5]+0xfc93a039, 21);
   146         -        MD5STEP(F4, a, b, c, d, in[12]+0x655b59c3,  6);
   147         -        MD5STEP(F4, d, a, b, c, in[ 3]+0x8f0ccc92, 10);
   148         -        MD5STEP(F4, c, d, a, b, in[10]+0xffeff47d, 15);
   149         -        MD5STEP(F4, b, c, d, a, in[ 1]+0x85845dd1, 21);
   150         -        MD5STEP(F4, a, b, c, d, in[ 8]+0x6fa87e4f,  6);
   151         -        MD5STEP(F4, d, a, b, c, in[15]+0xfe2ce6e0, 10);
   152         -        MD5STEP(F4, c, d, a, b, in[ 6]+0xa3014314, 15);
   153         -        MD5STEP(F4, b, c, d, a, in[13]+0x4e0811a1, 21);
   154         -        MD5STEP(F4, a, b, c, d, in[ 4]+0xf7537e82,  6);
   155         -        MD5STEP(F4, d, a, b, c, in[11]+0xbd3af235, 10);
   156         -        MD5STEP(F4, c, d, a, b, in[ 2]+0x2ad7d2bb, 15);
   157         -        MD5STEP(F4, b, c, d, a, in[ 9]+0xeb86d391, 21);
   158         -
   159         -        buf[0] += a;
   160         -        buf[1] += b;
   161         -        buf[2] += c;
   162         -        buf[3] += d;
   163         -}
   164         -
   165         -/*
   166         - * Start MD5 accumulation.  Set bit count to 0 and buffer to mysterious
   167         - * initialization constants.
   168         - */
   169         -static void MD5Init(MD5Context *pCtx){
   170         -        struct Context *ctx = (struct Context *)pCtx;
   171         -        ctx->buf[0] = 0x67452301;
   172         -        ctx->buf[1] = 0xefcdab89;
   173         -        ctx->buf[2] = 0x98badcfe;
   174         -        ctx->buf[3] = 0x10325476;
   175         -        ctx->bits[0] = 0;
   176         -        ctx->bits[1] = 0;
   177         -}
   178         -
   179         -/*
   180         - * Update context to reflect the concatenation of another buffer full
   181         - * of bytes.
   182         - */
   183         -static 
   184         -void MD5Update(MD5Context *pCtx, const unsigned char *buf, unsigned int len){
   185         -        struct Context *ctx = (struct Context *)pCtx;
   186         -        uint32 t;
   187         -
   188         -        /* Update bitcount */
   189         -
   190         -        t = ctx->bits[0];
   191         -        if ((ctx->bits[0] = t + ((uint32)len << 3)) < t)
   192         -                ctx->bits[1]++; /* Carry from low to high */
   193         -        ctx->bits[1] += len >> 29;
   194         -
   195         -        t = (t >> 3) & 0x3f;    /* Bytes already in shsInfo->data */
   196         -
   197         -        /* Handle any leading odd-sized chunks */
   198         -
   199         -        if ( t ) {
   200         -                unsigned char *p = (unsigned char *)ctx->in + t;
   201         -
   202         -                t = 64-t;
   203         -                if (len < t) {
   204         -                        memcpy(p, buf, len);
   205         -                        return;
   206         -                }
   207         -                memcpy(p, buf, t);
   208         -                byteReverse(ctx->in, 16);
   209         -                MD5Transform(ctx->buf, (uint32 *)ctx->in);
   210         -                buf += t;
   211         -                len -= t;
   212         -        }
   213         -
   214         -        /* Process data in 64-byte chunks */
   215         -
   216         -        while (len >= 64) {
   217         -                memcpy(ctx->in, buf, 64);
   218         -                byteReverse(ctx->in, 16);
   219         -                MD5Transform(ctx->buf, (uint32 *)ctx->in);
   220         -                buf += 64;
   221         -                len -= 64;
   222         -        }
   223         -
   224         -        /* Handle any remaining bytes of data. */
   225         -
   226         -        memcpy(ctx->in, buf, len);
   227         -}
   228         -
   229         -/*
   230         - * Final wrapup - pad to 64-byte boundary with the bit pattern 
   231         - * 1 0* (64-bit count of bits processed, MSB-first)
   232         - */
   233         -static void MD5Final(unsigned char digest[16], MD5Context *pCtx){
   234         -        struct Context *ctx = (struct Context *)pCtx;
   235         -        unsigned count;
   236         -        unsigned char *p;
   237         -
   238         -        /* Compute number of bytes mod 64 */
   239         -        count = (ctx->bits[0] >> 3) & 0x3F;
   240         -
   241         -        /* Set the first char of padding to 0x80.  This is safe since there is
   242         -           always at least one byte free */
   243         -        p = ctx->in + count;
   244         -        *p++ = 0x80;
   245         -
   246         -        /* Bytes of padding needed to make 64 bytes */
   247         -        count = 64 - 1 - count;
   248         -
   249         -        /* Pad out to 56 mod 64 */
   250         -        if (count < 8) {
   251         -                /* Two lots of padding:  Pad the first block to 64 bytes */
   252         -                memset(p, 0, count);
   253         -                byteReverse(ctx->in, 16);
   254         -                MD5Transform(ctx->buf, (uint32 *)ctx->in);
   255         -
   256         -                /* Now fill the next block with 56 bytes */
   257         -                memset(ctx->in, 0, 56);
   258         -        } else {
   259         -                /* Pad block to 56 bytes */
   260         -                memset(p, 0, count-8);
   261         -        }
   262         -        byteReverse(ctx->in, 14);
   263         -
   264         -        /* Append length in bits and transform */
   265         -        ((uint32 *)ctx->in)[ 14 ] = ctx->bits[0];
   266         -        ((uint32 *)ctx->in)[ 15 ] = ctx->bits[1];
   267         -
   268         -        MD5Transform(ctx->buf, (uint32 *)ctx->in);
   269         -        byteReverse((unsigned char *)ctx->buf, 4);
   270         -        memcpy(digest, ctx->buf, 16);
   271         -        memset(ctx, 0, sizeof(ctx));    /* In case it's sensitive */
   272         -}
   273         -
   274         -/*
   275         -** Convert a digest into base-16.  digest should be declared as
   276         -** "unsigned char digest[16]" in the calling function.  The MD5
   277         -** digest is stored in the first 16 bytes.  zBuf should
   278         -** be "char zBuf[33]".
   279         -*/
   280         -static void DigestToBase16(unsigned char *digest, char *zBuf){
   281         -  static char const zEncode[] = "0123456789abcdef";
   282         -  int i, j;
   283         -
   284         -  for(j=i=0; i<16; i++){
   285         -    int a = digest[i];
   286         -    zBuf[j++] = zEncode[(a>>4)&0xf];
   287         -    zBuf[j++] = zEncode[a & 0xf];
   288         -  }
   289         -  zBuf[j] = 0;
   290         -}
   291         -
   292         -/*
   293         -** A TCL command for md5.  The argument is the text to be hashed.  The
   294         -** Result is the hash in base64.  
   295         -*/
   296         -static int md5_cmd(void*cd, Tcl_Interp *interp, int argc, const char **argv){
   297         -  MD5Context ctx;
   298         -  unsigned char digest[16];
   299         -
   300         -  if( argc!=2 ){
   301         -    Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0], 
   302         -        " TEXT\"", 0);
   303         -    return TCL_ERROR;
   304         -  }
   305         -  MD5Init(&ctx);
   306         -  MD5Update(&ctx, (unsigned char*)argv[1], (unsigned)strlen(argv[1]));
   307         -  MD5Final(digest, &ctx);
   308         -  DigestToBase16(digest, interp->result);
   309         -  return TCL_OK;
   310         -}
   311         -
   312         -/*
   313         -** A TCL command to take the md5 hash of a file.  The argument is the
   314         -** name of the file.
   315         -*/
   316         -static int md5file_cmd(void*cd, Tcl_Interp*interp, int argc, const char **argv){
   317         -  FILE *in;
   318         -  MD5Context ctx;
   319         -  unsigned char digest[16];
   320         -  char zBuf[10240];
   321         -
   322         -  if( argc!=2 ){
   323         -    Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0], 
   324         -        " FILENAME\"", 0);
   325         -    return TCL_ERROR;
   326         -  }
   327         -  in = fopen(argv[1],"rb");
   328         -  if( in==0 ){
   329         -    Tcl_AppendResult(interp,"unable to open file \"", argv[1], 
   330         -         "\" for reading", 0);
   331         -    return TCL_ERROR;
   332         -  }
   333         -  MD5Init(&ctx);
   334         -  for(;;){
   335         -    int n;
   336         -    n = fread(zBuf, 1, sizeof(zBuf), in);
   337         -    if( n<=0 ) break;
   338         -    MD5Update(&ctx, (unsigned char*)zBuf, (unsigned)n);
   339         -  }
   340         -  fclose(in);
   341         -  MD5Final(digest, &ctx);
   342         -  DigestToBase16(digest, interp->result);
   343         -  return TCL_OK;
   344         -}
   345         -
   346         -/*
   347         -** Register the two TCL commands above with the TCL interpreter.
   348         -*/
   349         -int Md5_Init(Tcl_Interp *interp){
   350         -  Tcl_CreateCommand(interp, "md5", (Tcl_CmdProc*)md5_cmd, 0, 0);
   351         -  Tcl_CreateCommand(interp, "md5file", (Tcl_CmdProc*)md5file_cmd, 0, 0);
   352         -  return TCL_OK;
   353         -}
   354         -
   355         -/*
   356         -** During testing, the special md5sum() aggregate function is available.
   357         -** inside SQLite.  The following routines implement that function.
   358         -*/
   359         -static void md5step(sqlite3_context *context, int argc, sqlite3_value **argv){
   360         -  MD5Context *p;
   361         -  int i;
   362         -  if( argc<1 ) return;
   363         -  p = sqlite3_aggregate_context(context, sizeof(*p));
   364         -  if( p==0 ) return;
   365         -  if( sqlite3_aggregate_count(context)==1 ){
   366         -    MD5Init(p);
   367         -  }
   368         -  for(i=0; i<argc; i++){
   369         -    const char *zData = (char*)sqlite3_value_text(argv[i]);
   370         -    if( zData ){
   371         -      MD5Update(p, (unsigned char*)zData, strlen(zData));
   372         -    }
   373         -  }
   374         -}
   375         -static void md5finalize(sqlite3_context *context){
   376         -  MD5Context *p;
   377         -  unsigned char digest[16];
   378         -  char zBuf[33];
   379         -  p = sqlite3_aggregate_context(context, sizeof(*p));
   380         -  MD5Final(digest,p);
   381         -  DigestToBase16(digest, zBuf);
   382         -  sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
   383         -}
   384         -void Md5_Register(sqlite3 *db){
   385         -  sqlite3_create_function(db, "md5sum", -1, SQLITE_UTF8, 0, 0, 
   386         -      md5step, md5finalize);
   387         -}

Deleted src/os_test.c.

     1         -/*
     2         -** 2004 May 22
     3         -**
     4         -** The author disclaims copyright to this source code.  In place of
     5         -** a legal notice, here is a blessing:
     6         -**
     7         -**    May you do good and not evil.
     8         -**    May you find forgiveness for yourself and forgive others.
     9         -**    May you share freely, never taking more than you give.
    10         -**
    11         -******************************************************************************
    12         -**
    13         -** This file contains code that is specific to Unix systems. It is used
    14         -** for testing SQLite only.
    15         -*/
    16         -#if OS_TEST              /* This file is used for the test backend only */
    17         -#include "sqliteInt.h"
    18         -#include "os.h"          /* Must be first to enable large file support */
    19         -
    20         -#define sqlite3OsOpenReadWrite     sqlite3RealOpenReadWrite
    21         -#define sqlite3OsOpenExclusive     sqlite3RealOpenExclusive
    22         -#define sqlite3OsOpenReadOnly      sqlite3RealOpenReadOnly
    23         -#define sqlite3OsOpenDirectory     sqlite3RealOpenDirectory
    24         -#define sqlite3OsClose             sqlite3RealClose
    25         -#define sqlite3OsRead              sqlite3RealRead
    26         -#define sqlite3OsWrite             sqlite3RealWrite
    27         -#define sqlite3OsSeek              sqlite3RealSeek
    28         -#define sqlite3OsSync              sqlite3RealSync
    29         -#define sqlite3OsTruncate          sqlite3RealTruncate
    30         -#define sqlite3OsFileSize          sqlite3RealFileSize
    31         -#define sqlite3OsLock              sqlite3RealLock
    32         -#define sqlite3OsUnlock            sqlite3RealUnlock
    33         -#define sqlite3OsCheckReservedLock sqlite3RealCheckReservedLock
    34         -
    35         -#define OsFile OsRealFile
    36         -#define OS_UNIX 1
    37         -#include "os_unix.c"
    38         -#undef OS_UNIX
    39         -#undef OsFile
    40         -
    41         -#undef sqlite3OsOpenReadWrite     
    42         -#undef sqlite3OsOpenExclusive     
    43         -#undef sqlite3OsOpenReadOnly      
    44         -#undef sqlite3OsOpenDirectory     
    45         -#undef sqlite3OsClose             
    46         -#undef sqlite3OsRead              
    47         -#undef sqlite3OsWrite             
    48         -#undef sqlite3OsSeek              
    49         -#undef sqlite3OsSync              
    50         -#undef sqlite3OsTruncate          
    51         -#undef sqlite3OsFileSize          
    52         -#undef sqlite3OsLock              
    53         -#undef sqlite3OsUnlock            
    54         -#undef sqlite3OsCheckReservedLock 
    55         -
    56         -#define BLOCKSIZE 512
    57         -#define BLOCK_OFFSET(x) ((x) * BLOCKSIZE)
    58         -
    59         -
    60         -/*
    61         -** The following variables control when a simulated crash occurs.
    62         -**
    63         -** If iCrashDelay is non-zero, then zCrashFile contains (full path) name of
    64         -** a file that SQLite will call sqlite3OsSync() on. Each time this happens
    65         -** iCrashDelay is decremented. If iCrashDelay is zero after being
    66         -** decremented, a "crash" occurs during the sync() operation.
    67         -**
    68         -** In other words, a crash occurs the iCrashDelay'th time zCrashFile is
    69         -** synced.
    70         -*/
    71         -static int iCrashDelay = 0;
    72         -char zCrashFile[256];
    73         -
    74         -/*
    75         -** Set the value of the two crash parameters.
    76         -*/
    77         -void sqlite3SetCrashParams(int iDelay, char const *zFile){
    78         -  sqlite3OsEnterMutex();
    79         -  assert( strlen(zFile)<256 );
    80         -  strcpy(zCrashFile, zFile);
    81         -  iCrashDelay = iDelay;
    82         -  sqlite3OsLeaveMutex();
    83         -}
    84         -
    85         -/*
    86         -** File zPath is being sync()ed. Return non-zero if this should
    87         -** cause a crash.
    88         -*/
    89         -static int crashRequired(char const *zPath){
    90         -  int r;
    91         -  int n;
    92         -  sqlite3OsEnterMutex();
    93         -  n = strlen(zCrashFile);
    94         -  if( zCrashFile[n-1]=='*' ){
    95         -    n--;
    96         -  }else if( strlen(zPath)>n ){
    97         -    n = strlen(zPath);
    98         -  }
    99         -  r = 0;
   100         -  if( iCrashDelay>0 && strncmp(zPath, zCrashFile, n)==0 ){
   101         -    iCrashDelay--;
   102         -    if( iCrashDelay<=0 ){
   103         -      r = 1;
   104         -    }
   105         -  }
   106         -  sqlite3OsLeaveMutex();
   107         -  return r;
   108         -}
   109         -
   110         -
   111         -static OsTestFile *pAllFiles = 0;
   112         -
   113         -/*
   114         -** Initialise the os_test.c specific fields of pFile.
   115         -*/
   116         -static void initFile(OsFile *id, char const *zName){
   117         -  OsTestFile *pFile = (OsTestFile *)
   118         -      sqliteMalloc(sizeof(OsTestFile) + strlen(zName)+1);
   119         -  pFile->nMaxWrite = 0; 
   120         -  pFile->nBlk = 0; 
   121         -  pFile->apBlk = 0; 
   122         -  pFile->zName = (char *)(&pFile[1]);
   123         -  strcpy(pFile->zName, zName);
   124         -  *id = pFile;
   125         -  pFile->pNext = pAllFiles;
   126         -  pAllFiles = pFile;
   127         -}
   128         -
   129         -/*
   130         -** Undo the work done by initFile. Delete the OsTestFile structure
   131         -** and unlink the structure from the pAllFiles list.
   132         -*/
   133         -static void closeFile(OsFile *id){
   134         -  OsTestFile *pFile = *id;
   135         -  if( pFile==pAllFiles ){
   136         -    pAllFiles = pFile->pNext;
   137         -  }else{
   138         -    OsTestFile *p;
   139         -    for(p=pAllFiles; p->pNext!=pFile; p=p->pNext ){
   140         -      assert( p );
   141         -    }
   142         -    p->pNext = pFile->pNext;
   143         -  }
   144         -  sqliteFree(pFile);
   145         -  *id = 0;
   146         -}
   147         -
   148         -/*
   149         -** Return the current seek offset from the start of the file. This
   150         -** is unix-only code.
   151         -*/
   152         -static i64 osTell(OsTestFile *pFile){
   153         -  return lseek(pFile->fd.h, 0, SEEK_CUR);
   154         -}
   155         -
   156         -/*
   157         -** Load block 'blk' into the cache of pFile.
   158         -*/
   159         -static int cacheBlock(OsTestFile *pFile, int blk){
   160         -  if( blk>=pFile->nBlk ){
   161         -    int n = ((pFile->nBlk * 2) + 100 + blk);
   162         -    /* if( pFile->nBlk==0 ){ printf("DIRTY %s\n", pFile->zName); } */
   163         -    pFile->apBlk = (u8 **)sqliteRealloc(pFile->apBlk, n * sizeof(u8*));
   164         -    if( !pFile->apBlk ) return SQLITE_NOMEM;
   165         -    memset(&pFile->apBlk[pFile->nBlk], 0, (n - pFile->nBlk)*sizeof(u8*));
   166         -    pFile->nBlk = n;
   167         -  }
   168         -
   169         -  if( !pFile->apBlk[blk] ){
   170         -    i64 filesize;
   171         -    int rc;
   172         -
   173         -    u8 *p = sqliteMalloc(BLOCKSIZE);
   174         -    if( !p ) return SQLITE_NOMEM;
   175         -    pFile->apBlk[blk] = p;
   176         -
   177         -    rc = sqlite3RealFileSize(&pFile->fd, &filesize);
   178         -    if( rc!=SQLITE_OK ) return rc;
   179         -
   180         -    if( BLOCK_OFFSET(blk)<filesize ){
   181         -      int len = BLOCKSIZE;
   182         -      rc = sqlite3RealSeek(&pFile->fd, blk*BLOCKSIZE);
   183         -      if( BLOCK_OFFSET(blk+1)>filesize ){
   184         -        len = filesize - BLOCK_OFFSET(blk);
   185         -      }
   186         -      if( rc!=SQLITE_OK ) return rc;
   187         -      rc = sqlite3RealRead(&pFile->fd, p, len);
   188         -      if( rc!=SQLITE_OK ) return rc;
   189         -    }
   190         -  }
   191         -
   192         -  return SQLITE_OK;
   193         -}
   194         -
   195         -/* #define TRACE_WRITECACHE */
   196         -
   197         -/*
   198         -** Write the cache of pFile to disk. If crash is non-zero, randomly
   199         -** skip blocks when writing. The cache is deleted before returning.
   200         -*/
   201         -static int writeCache2(OsTestFile *pFile, int crash){
   202         -  int i;
   203         -  int nMax = pFile->nMaxWrite;
   204         -  i64 offset;
   205         -  int rc = SQLITE_OK;
   206         -
   207         -  offset = osTell(pFile);
   208         -  for(i=0; i<pFile->nBlk; i++){
   209         -    u8 *p = pFile->apBlk[i];
   210         -    if( p ){
   211         -      int skip = 0;
   212         -      int trash = 0;
   213         -      if( crash ){
   214         -        char random;
   215         -        sqlite3Randomness(1, &random);
   216         -        if( random & 0x01 ){
   217         -          if( random & 0x02 ){
   218         -            trash = 1;
   219         -#ifdef TRACE_WRITECACHE
   220         -printf("Trashing block %d of %s\n", i, pFile->zName); 
   221         -#endif
   222         -          }else{
   223         -            skip = 1;
   224         -#ifdef TRACE_WRITECACHE
   225         -printf("Skiping block %d of %s\n", i, pFile->zName); 
   226         -#endif
   227         -          }
   228         -        }else{
   229         -#ifdef TRACE_WRITECACHE
   230         -printf("Writing block %d of %s\n", i, pFile->zName); 
   231         -#endif
   232         -        }
   233         -      }
   234         -      if( rc==SQLITE_OK ){
   235         -        rc = sqlite3RealSeek(&pFile->fd, BLOCK_OFFSET(i));
   236         -      }
   237         -      if( rc==SQLITE_OK && !skip ){
   238         -        int len = BLOCKSIZE;
   239         -        if( BLOCK_OFFSET(i+1)>nMax ){
   240         -          len = nMax-BLOCK_OFFSET(i);
   241         -        }
   242         -        if( len>0 ){
   243         -          if( trash ){
   244         -            sqlite3Randomness(len, p);
   245         -          }
   246         -          rc = sqlite3RealWrite(&pFile->fd, p, len);
   247         -        }
   248         -      }
   249         -      sqliteFree(p);
   250         -    }
   251         -  }
   252         -  sqliteFree(pFile->apBlk);
   253         -  pFile->nBlk = 0;
   254         -  pFile->apBlk = 0;
   255         -  pFile->nMaxWrite = 0;
   256         -
   257         -  if( rc==SQLITE_OK ){
   258         -    rc = sqlite3RealSeek(&pFile->fd, offset);
   259         -  }
   260         -  return rc;
   261         -}
   262         -
   263         -/*
   264         -** Write the cache to disk.
   265         -*/
   266         -static int writeCache(OsTestFile *pFile){
   267         -  if( pFile->apBlk ){
   268         -    int c = crashRequired(pFile->zName);
   269         -    if( c ){
   270         -      OsTestFile *p;
   271         -#ifdef TRACE_WRITECACHE
   272         -      printf("\nCrash during sync of %s\n", pFile->zName);
   273         -#endif
   274         -      for(p=pAllFiles; p; p=p->pNext){
   275         -        writeCache2(p, 1);
   276         -      }
   277         -      exit(-1);
   278         -    }else{
   279         -      return writeCache2(pFile, 0);
   280         -    }
   281         -  }
   282         -  return SQLITE_OK;
   283         -}
   284         -
   285         -/*
   286         -** Close the file.
   287         -*/
   288         -int sqlite3OsClose(OsFile *id){
   289         -  if( !(*id) ) return SQLITE_OK;
   290         -  if( (*id)->fd.isOpen ){
   291         -    /* printf("CLOSE %s (%d blocks)\n", (*id)->zName, (*id)->nBlk); */
   292         -    writeCache(*id);
   293         -    sqlite3RealClose(&(*id)->fd);
   294         -  }
   295         -  closeFile(id);
   296         -  return SQLITE_OK;
   297         -}
   298         -
   299         -int sqlite3OsRead(OsFile *id, void *pBuf, int amt){
   300         -  i64 offset;       /* The current offset from the start of the file */
   301         -  i64 end;          /* The byte just past the last byte read */
   302         -  int blk;            /* Block number the read starts on */
   303         -  int i;
   304         -  u8 *zCsr;
   305         -  int rc = SQLITE_OK;
   306         -  OsTestFile *pFile = *id;
   307         -
   308         -  offset = osTell(pFile);
   309         -  end = offset+amt;
   310         -  blk = (offset/BLOCKSIZE);
   311         -
   312         -  zCsr = (u8 *)pBuf;
   313         -  for(i=blk; i*BLOCKSIZE<end; i++){
   314         -    int off = 0;
   315         -    int len = 0;
   316         -
   317         -
   318         -    if( BLOCK_OFFSET(i) < offset ){
   319         -      off = offset-BLOCK_OFFSET(i);
   320         -    }
   321         -    len = BLOCKSIZE - off;
   322         -    if( BLOCK_OFFSET(i+1) > end ){
   323         -      len = len - (BLOCK_OFFSET(i+1)-end);
   324         -    }
   325         -
   326         -    if( i<pFile->nBlk && pFile->apBlk[i]){
   327         -      u8 *pBlk = pFile->apBlk[i];
   328         -      memcpy(zCsr, &pBlk[off], len);
   329         -    }else{
   330         -      rc = sqlite3RealSeek(&pFile->fd, BLOCK_OFFSET(i) + off);
   331         -      if( rc!=SQLITE_OK ) return rc;
   332         -      rc = sqlite3RealRead(&pFile->fd, zCsr, len);
   333         -      if( rc!=SQLITE_OK ) return rc;
   334         -    }
   335         -
   336         -    zCsr += len;
   337         -  }
   338         -  assert( zCsr==&((u8 *)pBuf)[amt] );
   339         -
   340         -  rc = sqlite3RealSeek(&pFile->fd, end);
   341         -  return rc;
   342         -}
   343         -
   344         -int sqlite3OsWrite(OsFile *id, const void *pBuf, int amt){
   345         -  i64 offset;       /* The current offset from the start of the file */
   346         -  i64 end;          /* The byte just past the last byte written */
   347         -  int blk;            /* Block number the write starts on */
   348         -  int i;
   349         -  const u8 *zCsr;
   350         -  int rc = SQLITE_OK;
   351         -  OsTestFile *pFile = *id;
   352         -
   353         -  offset = osTell(pFile);
   354         -  end = offset+amt;
   355         -  blk = (offset/BLOCKSIZE);
   356         -
   357         -  zCsr = (u8 *)pBuf;
   358         -  for(i=blk; i*BLOCKSIZE<end; i++){
   359         -    u8 *pBlk;
   360         -    int off = 0;
   361         -    int len = 0;
   362         -
   363         -    /* Make sure the block is in the cache */
   364         -    rc = cacheBlock(pFile, i);
   365         -    if( rc!=SQLITE_OK ) return rc;
   366         -
   367         -    /* Write into the cache */
   368         -    pBlk = pFile->apBlk[i];
   369         -    assert( pBlk );
   370         -
   371         -    if( BLOCK_OFFSET(i) < offset ){
   372         -      off = offset-BLOCK_OFFSET(i);
   373         -    }
   374         -    len = BLOCKSIZE - off;
   375         -    if( BLOCK_OFFSET(i+1) > end ){
   376         -      len = len - (BLOCK_OFFSET(i+1)-end);
   377         -    }
   378         -    memcpy(&pBlk[off], zCsr, len);
   379         -    zCsr += len;
   380         -  }
   381         -  if( pFile->nMaxWrite<end ){
   382         -    pFile->nMaxWrite = end;
   383         -  }
   384         -  assert( zCsr==&((u8 *)pBuf)[amt] );
   385         -
   386         -  rc = sqlite3RealSeek(&pFile->fd, end);
   387         -  return rc;
   388         -}
   389         -
   390         -/*
   391         -** Sync the file. First flush the write-cache to disk, then call the
   392         -** real sync() function.
   393         -*/
   394         -int sqlite3OsSync(OsFile *id, int dataOnly){
   395         -  int rc;
   396         -  /* printf("SYNC %s (%d blocks)\n", (*id)->zName, (*id)->nBlk); */
   397         -  rc = writeCache(*id);
   398         -  if( rc!=SQLITE_OK ) return rc;
   399         -  rc = sqlite3RealSync(&(*id)->fd, dataOnly);
   400         -  return rc;
   401         -}
   402         -
   403         -/*
   404         -** Truncate the file. Set the internal OsFile.nMaxWrite variable to the new
   405         -** file size to ensure that nothing in the write-cache past this point
   406         -** is written to disk.
   407         -*/
   408         -int sqlite3OsTruncate(OsFile *id, i64 nByte){
   409         -  (*id)->nMaxWrite = nByte;
   410         -  return sqlite3RealTruncate(&(*id)->fd, nByte);
   411         -}
   412         -
   413         -/*
   414         -** Return the size of the file. If the cache contains a write that extended
   415         -** the file, then return this size instead of the on-disk size.
   416         -*/
   417         -int sqlite3OsFileSize(OsFile *id, i64 *pSize){
   418         -  int rc = sqlite3RealFileSize(&(*id)->fd, pSize);
   419         -  if( rc==SQLITE_OK && pSize && *pSize<(*id)->nMaxWrite ){
   420         -    *pSize = (*id)->nMaxWrite;
   421         -  }
   422         -  return rc;
   423         -}
   424         -
   425         -/*
   426         -** The three functions used to open files. All that is required is to
   427         -** initialise the os_test.c specific fields and then call the corresponding
   428         -** os_unix.c function to really open the file.
   429         -*/
   430         -int sqlite3OsOpenReadWrite(const char *zFilename, OsFile *id, int *pReadonly){
   431         -  initFile(id, zFilename);
   432         -  return sqlite3RealOpenReadWrite(zFilename, &(*id)->fd, pReadonly);
   433         -}
   434         -int sqlite3OsOpenExclusive(const char *zFilename, OsFile *id, int delFlag){
   435         -  initFile(id, zFilename);
   436         -  return sqlite3RealOpenExclusive(zFilename, &(*id)->fd, delFlag);
   437         -}
   438         -int sqlite3OsOpenReadOnly(const char *zFilename, OsFile *id){
   439         -  initFile(id, zFilename);
   440         -  return sqlite3RealOpenReadOnly(zFilename, &(*id)->fd);
   441         -}
   442         -
   443         -/*
   444         -** These six function calls are passed straight through to the os_unix.c
   445         -** backend.
   446         -*/
   447         -int sqlite3OsSeek(OsFile *id, i64 offset){
   448         -  return sqlite3RealSeek(&(*id)->fd, offset);
   449         -}
   450         -int sqlite3OsCheckReservedLock(OsFile *id){
   451         -  return sqlite3RealCheckReservedLock(&(*id)->fd);
   452         -}
   453         -int sqlite3OsLock(OsFile *id, int locktype){
   454         -  return sqlite3RealLock(&(*id)->fd, locktype);
   455         -}
   456         -int sqlite3OsUnlock(OsFile *id, int locktype){
   457         -  return sqlite3RealUnlock(&(*id)->fd, locktype);
   458         -}
   459         -int sqlite3OsOpenDirectory(const char *zDirname, OsFile *id){
   460         -  return sqlite3RealOpenDirectory(zDirname, &(*id)->fd);
   461         -}
   462         -
   463         -#endif /* OS_TEST */

Deleted src/os_test.h.

     1         -/*
     2         -** 2004 May 22
     3         -**
     4         -** The author disclaims copyright to this source code.  In place of
     5         -** a legal notice, here is a blessing:
     6         -**
     7         -**    May you do good and not evil.
     8         -**    May you find forgiveness for yourself and forgive others.
     9         -**    May you share freely, never taking more than you give.
    10         -**
    11         -******************************************************************************
    12         -**
    13         -*/
    14         -#ifndef _SQLITE_OS_TEST_H_
    15         -#define _SQLITE_OS_TEST_H_
    16         -
    17         -#define OsFile OsRealFile
    18         -#define OS_UNIX 1
    19         -#include "os_unix.h"
    20         -#undef OS_UNIX
    21         -#undef OsFile
    22         -#undef SET_FULLSYNC
    23         -
    24         -/* Include sqliteInt.h now to get the type u8. */
    25         -#include "sqliteInt.h"
    26         -
    27         -typedef struct OsTestFile* OsFile;
    28         -typedef struct OsTestFile OsTestFile;
    29         -struct OsTestFile {
    30         -  u8 **apBlk;       /* Array of blocks that have been written to. */
    31         -  int nBlk;         /* Size of apBlock. */
    32         -  int nMaxWrite;    /* Largest offset written to. */
    33         -  char *zName;      /* File name */
    34         -  OsRealFile fd;
    35         -  OsTestFile *pNext;
    36         -};
    37         -
    38         -void sqlite3SetCrashParams(int iDelay, char const *zFile);
    39         -
    40         -#endif /* _SQLITE_OS_UNIX_H_ */

Deleted src/os_unix.h.

     1         -/*
     2         -** 2004 May 22
     3         -**
     4         -** The author disclaims copyright to this source code.  In place of
     5         -** a legal notice, here is a blessing:
     6         -**
     7         -**    May you do good and not evil.
     8         -**    May you find forgiveness for yourself and forgive others.
     9         -**    May you share freely, never taking more than you give.
    10         -**
    11         -******************************************************************************
    12         -**
    13         -** This header file defined OS-specific features for Unix.
    14         -*/
    15         -#ifndef _SQLITE_OS_UNIX_H_
    16         -#define _SQLITE_OS_UNIX_H_
    17         -
    18         -/*
    19         -** Helpful hint:  To get this to compile on HP/UX, add -D_INCLUDE_POSIX_SOURCE
    20         -** to the compiler command line.
    21         -*/
    22         -
    23         -/*
    24         -** These #defines should enable >2GB file support on Posix if the
    25         -** underlying operating system supports it.  If the OS lacks
    26         -** large file support, or if the OS is windows, these should be no-ops.
    27         -**
    28         -** Large file support can be disabled using the -DSQLITE_DISABLE_LFS switch
    29         -** on the compiler command line.  This is necessary if you are compiling
    30         -** on a recent machine (ex: RedHat 7.2) but you want your code to work
    31         -** on an older machine (ex: RedHat 6.0).  If you compile on RedHat 7.2
    32         -** without this option, LFS is enable.  But LFS does not exist in the kernel
    33         -** in RedHat 6.0, so the code won't work.  Hence, for maximum binary
    34         -** portability you should omit LFS.
    35         -**
    36         -** Similar is true for MacOS.  LFS is only supported on MacOS 9 and later.
    37         -*/
    38         -#ifndef SQLITE_DISABLE_LFS
    39         -# define _LARGE_FILE       1
    40         -# ifndef _FILE_OFFSET_BITS
    41         -#   define _FILE_OFFSET_BITS 64
    42         -# endif
    43         -# define _LARGEFILE_SOURCE 1
    44         -#endif
    45         -
    46         -/*
    47         -** standard include files.
    48         -*/
    49         -#include <sys/types.h>
    50         -#include <sys/stat.h>
    51         -#include <fcntl.h>
    52         -#include <unistd.h>
    53         -
    54         -/*
    55         -** Macros used to determine whether or not to use threads.  The
    56         -** SQLITE_UNIX_THREADS macro is defined if we are synchronizing for
    57         -** Posix threads and SQLITE_W32_THREADS is defined if we are
    58         -** synchronizing using Win32 threads.
    59         -*/
    60         -#if defined(THREADSAFE) && THREADSAFE
    61         -# include <pthread.h>
    62         -# define SQLITE_UNIX_THREADS 1
    63         -#endif
    64         -
    65         -/*
    66         -** The OsFile structure is a operating-system independing representation
    67         -** of an open file handle.  It is defined differently for each architecture.
    68         -**
    69         -** This is the definition for Unix.
    70         -**
    71         -** OsFile.locktype takes one of the values SHARED_LOCK, RESERVED_LOCK,
    72         -** PENDING_LOCK or EXCLUSIVE_LOCK.
    73         -*/
    74         -typedef struct OsFile OsFile;
    75         -struct OsFile {
    76         -  struct Pager *pPager;     /* The pager that owns this OsFile.  Might be 0 */
    77         -  struct openCnt *pOpen;    /* Info about all open fd's on this inode */
    78         -  struct lockInfo *pLock;   /* Info about locks on this inode */
    79         -  int h;                    /* The file descriptor */
    80         -  unsigned char locktype;   /* The type of lock held on this fd */
    81         -  unsigned char isOpen;     /* True if needs to be closed */
    82         -  unsigned char fullSync;   /* Use F_FULLSYNC if available */
    83         -  int dirfd;                /* File descriptor for the directory */
    84         -#ifdef SQLITE_UNIX_THREADS
    85         -  pthread_t tid;            /* The thread authorized to use this OsFile */
    86         -#endif
    87         -};
    88         -
    89         -/*
    90         -** A macro to set the OsFile.fullSync flag, if it exists.
    91         -*/
    92         -#define SET_FULLSYNC(x,y)  ((x).fullSync = (y))
    93         -
    94         -/*
    95         -** Maximum number of characters in a temporary file name
    96         -*/
    97         -#define SQLITE_TEMPNAME_SIZE 200
    98         -
    99         -/*
   100         -** Minimum interval supported by sqlite3OsSleep().
   101         -*/
   102         -#if defined(HAVE_USLEEP) && HAVE_USLEEP
   103         -# define SQLITE_MIN_SLEEP_MS 1
   104         -#else
   105         -# define SQLITE_MIN_SLEEP_MS 1000
   106         -#endif
   107         -
   108         -/*
   109         -** Default permissions when creating a new file
   110         -*/
   111         -#ifndef SQLITE_DEFAULT_FILE_PERMISSIONS
   112         -# define SQLITE_DEFAULT_FILE_PERMISSIONS 0644
   113         -#endif
   114         -
   115         -
   116         -#endif /* _SQLITE_OS_UNIX_H_ */

Deleted src/os_win.h.

     1         -/*
     2         -** 2004 May 22
     3         -**
     4         -** The author disclaims copyright to this source code.  In place of
     5         -** a legal notice, here is a blessing:
     6         -**
     7         -**    May you do good and not evil.
     8         -**    May you find forgiveness for yourself and forgive others.
     9         -**    May you share freely, never taking more than you give.
    10         -**
    11         -******************************************************************************
    12         -**
    13         -** This header file defines OS-specific features for Win32
    14         -*/
    15         -#ifndef _SQLITE_OS_WIN_H_
    16         -#define _SQLITE_OS_WIN_H_
    17         -
    18         -#include <windows.h>
    19         -#include <winbase.h>
    20         -
    21         -/*
    22         -** The OsFile structure is a operating-system independing representation
    23         -** of an open file handle.  It is defined differently for each architecture.
    24         -**
    25         -** This is the definition for Win32.
    26         -*/
    27         -typedef struct OsFile OsFile;
    28         -struct OsFile {
    29         -  HANDLE h;               /* Handle for accessing the file */
    30         -  unsigned char locktype; /* Type of lock currently held on this file */
    31         -  unsigned char isOpen;   /* True if needs to be closed */
    32         -  short sharedLockByte;   /* Randomly chosen byte used as a shared lock */
    33         -};
    34         -
    35         -
    36         -#define SQLITE_TEMPNAME_SIZE (MAX_PATH+50)
    37         -#define SQLITE_MIN_SLEEP_MS 1
    38         -
    39         -
    40         -#endif /* _SQLITE_OS_WIN_H_ */

Changes to src/pager.c.

    14     14   ** The pager is used to access a database disk file.  It implements
    15     15   ** atomic commit and rollback through the use of a journal file that
    16     16   ** is separate from the database file.  The pager also implements file
    17     17   ** locking to prevent two processes from writing the same database
    18     18   ** file simultaneously, or one process from reading the database while
    19     19   ** another is writing.
    20     20   **
    21         -** @(#) $Id: pager.c,v 1.282 2007/01/05 02:00:47 drh Exp $
           21  +** @(#) $Id: pager.c,v 1.282.2.1 2007/05/10 21:31:40 drh Exp $
    22     22   */
    23     23   #ifndef SQLITE_OMIT_DISKIO
    24     24   #include "sqliteInt.h"
    25     25   #include "os.h"
    26     26   #include "pager.h"
    27     27   #include <assert.h>
    28     28   #include <string.h>
................................................................................
   280    280     void *(*xCodec)(void*,void*,Pgno,int); /* Routine for en/decoding data */
   281    281     void *pCodecArg;            /* First argument to xCodec() */
   282    282     int nHash;                  /* Size of the pager hash table */
   283    283     PgHdr **aHash;              /* Hash table to map page number to PgHdr */
   284    284   #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
   285    285     Pager *pNext;               /* Linked list of pagers in this thread */
   286    286   #endif
          287  +  char *pTmpSpace;            /* Pager.pageSize bytes of space for tmp use */
   287    288   };
   288    289   
   289    290   /*
   290    291   ** If SQLITE_TEST is defined then increment the variable given in
   291    292   ** the argument
   292    293   */
   293    294   #ifdef SQLITE_TEST
................................................................................
   991    992   ** need to survive power failures.
   992    993   */
   993    994   static int pager_playback_one_page(Pager *pPager, OsFile *jfd, int useCksum){
   994    995     int rc;
   995    996     PgHdr *pPg;                   /* An existing page in the cache */
   996    997     Pgno pgno;                    /* The page number of a page in journal */
   997    998     u32 cksum;                    /* Checksum used for sanity checking */
   998         -  u8 aData[SQLITE_MAX_PAGE_SIZE];  /* Temp storage for a page */
          999  +  u8 *aData = (u8 *)pPager->pTmpSpace;   /* Temp storage for a page */
   999   1000   
  1000   1001     /* useCksum should be true for the main journal and false for
  1001   1002     ** statement journals.  Verify that this is always the case
  1002   1003     */
  1003   1004     assert( jfd == (useCksum ? pPager->jfd : pPager->stfd) );
  1004         -
         1005  +  assert( aData );
  1005   1006   
  1006   1007     rc = read32bits(jfd, &pgno);
  1007   1008     if( rc!=SQLITE_OK ) return rc;
  1008         -  rc = sqlite3OsRead(jfd, &aData, pPager->pageSize);
         1009  +  rc = sqlite3OsRead(jfd, aData, pPager->pageSize);
  1009   1010     if( rc!=SQLITE_OK ) return rc;
  1010   1011     pPager->journalOff += pPager->pageSize + 4;
  1011   1012   
  1012   1013     /* Sanity checking on the page.  This is more important that I originally
  1013   1014     ** thought.  If a power failure occurs while the journal is being written,
  1014   1015     ** it could cause invalid data to be written into the journal.  We need to
  1015   1016     ** detect this invalid data (with high probability) and ignore it.
................................................................................
  1180   1181   ** cache content and the easiest way to do that is to reread the old content
  1181   1182   ** back from the disk.
  1182   1183   */
  1183   1184   static int pager_reload_cache(Pager *pPager){
  1184   1185     PgHdr *pPg;
  1185   1186     int rc = SQLITE_OK;
  1186   1187     for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
  1187         -    char zBuf[SQLITE_MAX_PAGE_SIZE];
         1188  +    char *zBuf = pPager->pTmpSpace;        /* Temp storage for one page */
  1188   1189       if( !pPg->dirty ) continue;
  1189   1190       if( (int)pPg->pgno <= pPager->origDbSize ){
  1190   1191         rc = sqlite3OsSeek(pPager->fd, pPager->pageSize*(i64)(pPg->pgno-1));
  1191   1192         if( rc==SQLITE_OK ){
  1192   1193           rc = sqlite3OsRead(pPager->fd, zBuf, pPager->pageSize);
  1193   1194         }
  1194   1195         TRACE3("REFETCH %d page %d\n", PAGERID(pPager), pPg->pgno);
................................................................................
  1669   1670     /* Allocate the Pager structure. As part of the same allocation, allocate
  1670   1671     ** space for the full paths of the file, directory and journal 
  1671   1672     ** (Pager.zFilename, Pager.zDirectory and Pager.zJournal).
  1672   1673     */
  1673   1674     if( zFullPathname ){
  1674   1675       nameLen = strlen(zFullPathname);
  1675   1676       pPager = sqliteMalloc( sizeof(*pPager) + nameLen*3 + 30 );
         1677  +    if( pPager && rc==SQLITE_OK ){
         1678  +      pPager->pTmpSpace = (char *)sqliteMallocRaw(SQLITE_DEFAULT_PAGE_SIZE);
         1679  +    }
  1676   1680     }
         1681  +
  1677   1682   
  1678   1683     /* If an error occured in either of the blocks above, free the memory 
  1679   1684     ** pointed to by zFullPathname, free the Pager structure and close the 
  1680   1685     ** file. Since the pager is not allocated there is no need to set 
  1681   1686     ** any Pager.errMask variables.
  1682   1687     */
  1683         -  if( !pPager || !zFullPathname || rc!=SQLITE_OK ){
         1688  +  if( !pPager || !zFullPathname || !pPager->pTmpSpace || rc!=SQLITE_OK ){
  1684   1689       sqlite3OsClose(&fd);
  1685   1690       sqliteFree(zFullPathname);
  1686   1691       sqliteFree(pPager);
  1687   1692       return ((rc==SQLITE_OK)?SQLITE_NOMEM:rc);
  1688   1693     }
  1689   1694   
  1690   1695     TRACE3("OPEN %d %s\n", FILEHANDLEID(fd), zFullPathname);
................................................................................
  1772   1777   ** size is inappropriate, then an alternative page size is selected
  1773   1778   ** and returned.
  1774   1779   */
  1775   1780   int sqlite3pager_set_pagesize(Pager *pPager, int pageSize){
  1776   1781     assert( pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE );
  1777   1782     if( !pPager->memDb ){
  1778   1783       pPager->pageSize = pageSize;
         1784  +    sqlite3ReallocOrFree((void **)&pPager->pTmpSpace, pageSize);
  1779   1785     }
  1780   1786     return pPager->pageSize;
  1781   1787   }
  1782   1788   
  1783   1789   /*
  1784   1790   ** The following set of routines are used to disable the simulated
  1785   1791   ** I/O error mechanism.  These routines are used to avoid simulated
................................................................................
  2104   2110     }else{
  2105   2111       Pager *pTmp;
  2106   2112       for(pTmp = pTsd->pPager; pTmp->pNext!=pPager; pTmp=pTmp->pNext){}
  2107   2113       pTmp->pNext = pPager->pNext;
  2108   2114     }
  2109   2115   #endif
  2110   2116     sqliteFree(pPager->aHash);
         2117  +  sqliteFree(pPager->pTmpSpace);
  2111   2118     sqliteFree(pPager);
  2112   2119     return SQLITE_OK;
  2113   2120   }
  2114   2121   
  2115   2122   /*
  2116   2123   ** Return the page number for the given page data.
  2117   2124   */

Deleted src/server.c.

     1         -/*
     2         -** 2006 January 07
     3         -**
     4         -** The author disclaims copyright to this source code.  In place of
     5         -** a legal notice, here is a blessing:
     6         -**
     7         -**    May you do good and not evil.
     8         -**    May you find forgiveness for yourself and forgive others.
     9         -**    May you share freely, never taking more than you give.
    10         -**
    11         -******************************************************************************
    12         -**
    13         -** This file contains demonstration code.  Nothing in this file gets compiled
    14         -** or linked into the SQLite library unless you use a non-standard option:
    15         -**
    16         -**      -DSQLITE_SERVER=1
    17         -**
    18         -** The configure script will never generate a Makefile with the option
    19         -** above.  You will need to manually modify the Makefile if you want to
    20         -** include any of the code from this file in your project.  Or, at your
    21         -** option, you may copy and paste the code from this file and
    22         -** thereby avoiding a recompile of SQLite.
    23         -**
    24         -**
    25         -** This source file demonstrates how to use SQLite to create an SQL database 
    26         -** server thread in a multiple-threaded program.  One or more client threads
    27         -** send messages to the server thread and the server thread processes those
    28         -** messages in the order received and returns the results to the client.
    29         -**
    30         -** One might ask:  "Why bother?  Why not just let each thread connect
    31         -** to the database directly?"  There are a several of reasons to
    32         -** prefer the client/server approach.
    33         -**
    34         -**    (1)  Some systems (ex: Redhat9) have broken threading implementations
    35         -**         that prevent SQLite database connections from being used in
    36         -**         a thread different from the one where they were created.  With
    37         -**         the client/server approach, all database connections are created
    38         -**         and used within the server thread.  Client calls to the database
    39         -**         can be made from multiple threads (though not at the same time!)
    40         -**
    41         -**    (2)  Beginning with SQLite version 3.3.0, when two or more 
    42         -**         connections to the same database occur within the same thread,
    43         -**         they can optionally share their database cache.  This reduces
    44         -**         I/O and memory requirements.  Cache shared is controlled using
    45         -**         the sqlite3_enable_shared_cache() API.
    46         -**
    47         -**    (3)  Database connections on a shared cache use table-level locking
    48         -**         instead of file-level locking for improved concurrency.
    49         -**
    50         -**    (4)  Database connections on a shared cache can by optionally
    51         -**         set to READ UNCOMMITTED isolation.  (The default isolation for
    52         -**         SQLite is SERIALIZABLE.)  When this occurs, readers will
    53         -**         never be blocked by a writer and writers will not be
    54         -**         blocked by readers.  There can still only be a single writer
    55         -**         at a time, but multiple readers can simultaneously exist with
    56         -**         that writer.  This is a huge increase in concurrency.
    57         -**
    58         -** To summarize the rational for using a client/server approach: prior
    59         -** to SQLite version 3.3.0 it probably was not worth the trouble.  But
    60         -** with SQLite version 3.3.0 and beyond you can get significant performance
    61         -** and concurrency improvements and memory usage reductions by going
    62         -** client/server.
    63         -**
    64         -** Note:  The extra features of version 3.3.0 described by points (2)
    65         -** through (4) above are only available if you compile without the
    66         -** option -DSQLITE_OMIT_SHARED_CACHE. 
    67         -**
    68         -** Here is how the client/server approach works:  The database server
    69         -** thread is started on this procedure:
    70         -**
    71         -**       void *sqlite3_server(void *NotUsed);
    72         -**
    73         -** The sqlite_server procedure runs as long as the g.serverHalt variable
    74         -** is false.  A mutex is used to make sure no more than one server runs
    75         -** at a time.  The server waits for messages to arrive on a message
    76         -** queue and processes the messages in order.
    77         -**
    78         -** Two convenience routines are provided for starting and stopping the
    79         -** server thread:
    80         -**
    81         -**       void sqlite3_server_start(void);
    82         -**       void sqlite3_server_stop(void);
    83         -**
    84         -** Both of the convenience routines return immediately.  Neither will
    85         -** ever give an error.  If a server is already started or already halted,
    86         -** then the routines are effectively no-ops.
    87         -**
    88         -** Clients use the following interfaces:
    89         -**
    90         -**       sqlite3_client_open
    91         -**       sqlite3_client_prepare
    92         -**       sqlite3_client_step
    93         -**       sqlite3_client_reset
    94         -**       sqlite3_client_finalize
    95         -**       sqlite3_client_close
    96         -**
    97         -** These interfaces work exactly like the standard core SQLite interfaces
    98         -** having the same names without the "_client_" infix.  Many other SQLite
    99         -** interfaces can be used directly without having to send messages to the
   100         -** server as long as SQLITE_ENABLE_MEMORY_MANAGEMENT is not defined.
   101         -** The following interfaces fall into this second category:
   102         -**
   103         -**       sqlite3_bind_*
   104         -**       sqlite3_changes
   105         -**       sqlite3_clear_bindings
   106         -**       sqlite3_column_*
   107         -**       sqlite3_complete
   108         -**       sqlite3_create_collation
   109         -**       sqlite3_create_function
   110         -**       sqlite3_data_count
   111         -**       sqlite3_db_handle
   112         -**       sqlite3_errcode
   113         -**       sqlite3_errmsg
   114         -**       sqlite3_last_insert_rowid
   115         -**       sqlite3_total_changes
   116         -**       sqlite3_transfer_bindings
   117         -**
   118         -** A single SQLite connection (an sqlite3* object) or an SQLite statement
   119         -** (an sqlite3_stmt* object) should only be passed to a single interface
   120         -** function at a time.  The connections and statements can be passed from
   121         -** any thread to any of the functions listed in the second group above as
   122         -** long as the same connection is not in use by two threads at once and
   123         -** as long as SQLITE_ENABLE_MEMORY_MANAGEMENT is not defined.  Additional
   124         -** information about the SQLITE_ENABLE_MEMORY_MANAGEMENT constraint is
   125         -** below.
   126         -**
   127         -** The busy handler for all database connections should remain turned
   128         -** off.  That means that any lock contention will cause the associated
   129         -** sqlite3_client_step() call to return immediately with an SQLITE_BUSY
   130         -** error code.  If a busy handler is enabled and lock contention occurs,
   131         -** then the entire server thread will block.  This will cause not only
   132         -** the requesting client to block but every other database client as
   133         -** well.  It is possible to enhance the code below so that lock
   134         -** contention will cause the message to be placed back on the top of
   135         -** the queue to be tried again later.  But such enhanced processing is
   136         -** not included here, in order to keep the example simple.
   137         -**
   138         -** This example code assumes the use of pthreads.  Pthreads
   139         -** implementations are available for windows.  (See, for example
   140         -** http://sourceware.org/pthreads-win32/announcement.html.)  Or, you
   141         -** can translate the locking and thread synchronization code to use
   142         -** windows primitives easily enough.  The details are left as an
   143         -** exercise to the reader.
   144         -**
   145         -**** Restrictions Associated With SQLITE_ENABLE_MEMORY_MANAGEMENT ****
   146         -**
   147         -** If you compile with SQLITE_ENABLE_MEMORY_MANAGEMENT defined, then
   148         -** SQLite includes code that tracks how much memory is being used by
   149         -** each thread.  These memory counts can become confused if memory
   150         -** is allocated by one thread and then freed by another.  For that
   151         -** reason, when SQLITE_ENABLE_MEMORY_MANAGEMENT is used, all operations
   152         -** that might allocate or free memory should be performanced in the same
   153         -** thread that originally created the database connection.  In that case,
   154         -** many of the operations that are listed above as safe to be performed
   155         -** in separate threads would need to be sent over to the server to be
   156         -** done there.  If SQLITE_ENABLE_MEMORY_MANAGEMENT is defined, then
   157         -** the following functions can be used safely from different threads
   158         -** without messing up the allocation counts:
   159         -**
   160         -**       sqlite3_bind_parameter_name
   161         -**       sqlite3_bind_parameter_index
   162         -**       sqlite3_changes
   163         -**       sqlite3_column_blob
   164         -**       sqlite3_column_count
   165         -**       sqlite3_complete
   166         -**       sqlite3_data_count
   167         -**       sqlite3_db_handle
   168         -**       sqlite3_errcode
   169         -**       sqlite3_errmsg
   170         -**       sqlite3_last_insert_rowid
   171         -**       sqlite3_total_changes
   172         -**
   173         -** The remaining functions are not thread-safe when memory management
   174         -** is enabled.  So one would have to define some new interface routines
   175         -** along the following lines:
   176         -**
   177         -**       sqlite3_client_bind_*
   178         -**       sqlite3_client_clear_bindings
   179         -**       sqlite3_client_column_*
   180         -**       sqlite3_client_create_collation
   181         -**       sqlite3_client_create_function
   182         -**       sqlite3_client_transfer_bindings
   183         -**
   184         -** The example code in this file is intended for use with memory
   185         -** management turned off.  So the implementation of these additional
   186         -** client interfaces is left as an exercise to the reader.
   187         -**
   188         -** It may seem surprising to the reader that the list of safe functions
   189         -** above does not include things like sqlite3_bind_int() or
   190         -** sqlite3_column_int().  But those routines might, in fact, allocate
   191         -** or deallocate memory.  In the case of sqlite3_bind_int(), if the
   192         -** parameter was previously bound to a string that string might need
   193         -** to be deallocated before the new integer value is inserted.  In
   194         -** the case of sqlite3_column_int(), the value of the column might be
   195         -** a UTF-16 string which will need to be converted to UTF-8 then into
   196         -** an integer.
   197         -*/
   198         -
   199         -/*
   200         -** Only compile the code in this file on UNIX with a THREADSAFE build
   201         -** and only if the SQLITE_SERVER macro is defined.
   202         -*/
   203         -#ifdef SQLITE_SERVER
   204         -#if defined(OS_UNIX) && OS_UNIX && defined(THREADSAFE) && THREADSAFE
   205         -
   206         -/*
   207         -** We require only pthreads and the public interface of SQLite.
   208         -*/
   209         -#include <pthread.h>
   210         -#include "sqlite3.h"
   211         -
   212         -/*
   213         -** Messages are passed from client to server and back again as 
   214         -** instances of the following structure.
   215         -*/
   216         -typedef struct SqlMessage SqlMessage;
   217         -struct SqlMessage {
   218         -  int op;                      /* Opcode for the message */
   219         -  sqlite3 *pDb;                /* The SQLite connection */
   220         -  sqlite3_stmt *pStmt;         /* A specific statement */
   221         -  int errCode;                 /* Error code returned */
   222         -  const char *zIn;             /* Input filename or SQL statement */
   223         -  int nByte;                   /* Size of the zIn parameter for prepare() */
   224         -  const char *zOut;            /* Tail of the SQL statement */
   225         -  SqlMessage *pNext;           /* Next message in the queue */
   226         -  SqlMessage *pPrev;           /* Previous message in the queue */
   227         -  pthread_mutex_t clientMutex; /* Hold this mutex to access the message */
   228         -  pthread_cond_t clientWakeup; /* Signal to wake up the client */
   229         -};
   230         -
   231         -/*
   232         -** Legal values for SqlMessage.op
   233         -*/
   234         -#define MSG_Open       1  /* sqlite3_open(zIn, &pDb) */
   235         -#define MSG_Prepare    2  /* sqlite3_prepare(pDb, zIn, nByte, &pStmt, &zOut) */
   236         -#define MSG_Step       3  /* sqlite3_step(pStmt) */
   237         -#define MSG_Reset      4  /* sqlite3_reset(pStmt) */
   238         -#define MSG_Finalize   5  /* sqlite3_finalize(pStmt) */
   239         -#define MSG_Close      6  /* sqlite3_close(pDb) */
   240         -#define MSG_Done       7  /* Server has finished with this message */
   241         -
   242         -
   243         -/*
   244         -** State information about the server is stored in a static variable
   245         -** named "g" as follows:
   246         -*/
   247         -static struct ServerState {
   248         -  pthread_mutex_t queueMutex;   /* Hold this mutex to access the msg queue */
   249         -  pthread_mutex_t serverMutex;  /* Held by the server while it is running */
   250         -  pthread_cond_t serverWakeup;  /* Signal this condvar to wake up the server */
   251         -  volatile int serverHalt;      /* Server halts itself when true */
   252         -  SqlMessage *pQueueHead;       /* Head of the message queue */
   253         -  SqlMessage *pQueueTail;       /* Tail of the message queue */
   254         -} g = {
   255         -  PTHREAD_MUTEX_INITIALIZER,
   256         -  PTHREAD_MUTEX_INITIALIZER,
   257         -  PTHREAD_COND_INITIALIZER,
   258         -};
   259         -
   260         -/*
   261         -** Send a message to the server.  Block until we get a reply.
   262         -**
   263         -** The mutex and condition variable in the message are uninitialized
   264         -** when this routine is called.  This routine takes care of 
   265         -** initializing them and destroying them when it has finished.
   266         -*/
   267         -static void sendToServer(SqlMessage *pMsg){
   268         -  /* Initialize the mutex and condition variable on the message
   269         -  */
   270         -  pthread_mutex_init(&pMsg->clientMutex, 0);
   271         -  pthread_cond_init(&pMsg->clientWakeup, 0);
   272         -
   273         -  /* Add the message to the head of the server's message queue.
   274         -  */
   275         -  pthread_mutex_lock(&g.queueMutex);
   276         -  pMsg->pNext = g.pQueueHead;
   277         -  if( g.pQueueHead==0 ){
   278         -    g.pQueueTail = pMsg;
   279         -  }else{
   280         -    g.pQueueHead->pPrev = pMsg;
   281         -  }
   282         -  pMsg->pPrev = 0;
   283         -  g.pQueueHead = pMsg;
   284         -  pthread_mutex_unlock(&g.queueMutex);
   285         -
   286         -  /* Signal the server that the new message has be queued, then
   287         -  ** block waiting for the server to process the message.
   288         -  */
   289         -  pthread_mutex_lock(&pMsg->clientMutex);
   290         -  pthread_cond_signal(&g.serverWakeup);
   291         -  while( pMsg->op!=MSG_Done ){
   292         -    pthread_cond_wait(&pMsg->clientWakeup, &pMsg->clientMutex);
   293         -  }
   294         -  pthread_mutex_unlock(&pMsg->clientMutex);
   295         -
   296         -  /* Destroy the mutex and condition variable of the message.
   297         -  */
   298         -  pthread_mutex_destroy(&pMsg->clientMutex);
   299         -  pthread_cond_destroy(&pMsg->clientWakeup);
   300         -}
   301         -
   302         -/*
   303         -** The following 6 routines are client-side implementations of the
   304         -** core SQLite interfaces:
   305         -**
   306         -**        sqlite3_open
   307         -**        sqlite3_prepare
   308         -**        sqlite3_step
   309         -**        sqlite3_reset
   310         -**        sqlite3_finalize
   311         -**        sqlite3_close
   312         -**
   313         -** Clients should use the following client-side routines instead of 
   314         -** the core routines above.
   315         -**
   316         -**        sqlite3_client_open
   317         -**        sqlite3_client_prepare
   318         -**        sqlite3_client_step
   319         -**        sqlite3_client_reset
   320         -**        sqlite3_client_finalize
   321         -**        sqlite3_client_close
   322         -**
   323         -** Each of these routines creates a message for the desired operation,
   324         -** sends that message to the server, waits for the server to process
   325         -** then message and return a response.
   326         -*/
   327         -int sqlite3_client_open(const char *zDatabaseName, sqlite3 **ppDb){
   328         -  SqlMessage msg;
   329         -  msg.op = MSG_Open;
   330         -  msg.zIn = zDatabaseName;
   331         -  sendToServer(&msg);
   332         -  *ppDb = msg.pDb;
   333         -  return msg.errCode;
   334         -}
   335         -int sqlite3_client_prepare(
   336         -  sqlite3 *pDb,
   337         -  const char *zSql,
   338         -  int nByte,
   339         -  sqlite3_stmt **ppStmt,
   340         -  const char **pzTail
   341         -){
   342         -  SqlMessage msg;
   343         -  msg.op = MSG_Prepare;
   344         -  msg.pDb = pDb;
   345         -  msg.zIn = zSql;
   346         -  msg.nByte = nByte;
   347         -  sendToServer(&msg);
   348         -  *ppStmt = msg.pStmt;
   349         -  if( pzTail ) *pzTail = msg.zOut;
   350         -  return msg.errCode;
   351         -}
   352         -int sqlite3_client_step(sqlite3_stmt *pStmt){
   353         -  SqlMessage msg;
   354         -  msg.op = MSG_Step;
   355         -  msg.pStmt = pStmt;
   356         -  sendToServer(&msg);
   357         -  return msg.errCode;
   358         -}
   359         -int sqlite3_client_reset(sqlite3_stmt *pStmt){
   360         -  SqlMessage msg;
   361         -  msg.op = MSG_Reset;
   362         -  msg.pStmt = pStmt;
   363         -  sendToServer(&msg);
   364         -  return msg.errCode;
   365         -}
   366         -int sqlite3_client_finalize(sqlite3_stmt *pStmt){
   367         -  SqlMessage msg;
   368         -  msg.op = MSG_Finalize;
   369         -  msg.pStmt = pStmt;
   370         -  sendToServer(&msg);
   371         -  return msg.errCode;
   372         -}
   373         -int sqlite3_client_close(sqlite3 *pDb){
   374         -  SqlMessage msg;
   375         -  msg.op = MSG_Close;
   376         -  msg.pDb = pDb;
   377         -  sendToServer(&msg);
   378         -  return msg.errCode;
   379         -}
   380         -
   381         -/*
   382         -** This routine implements the server.  To start the server, first
   383         -** make sure g.serverHalt is false, then create a new detached thread
   384         -** on this procedure.  See the sqlite3_server_start() routine below
   385         -** for an example.  This procedure loops until g.serverHalt becomes
   386         -** true.
   387         -*/
   388         -void *sqlite3_server(void *NotUsed){
   389         -  sqlite3_enable_shared_cache(1);
   390         -  if( pthread_mutex_trylock(&g.serverMutex) ){
   391         -    sqlite3_enable_shared_cache(0);
   392         -    return 0;  /* Another server is already running */
   393         -  }
   394         -  while( !g.serverHalt ){
   395         -    SqlMessage *pMsg;
   396         -
   397         -    /* Remove the last message from the message queue.
   398         -    */
   399         -    pthread_mutex_lock(&g.queueMutex);
   400         -    while( g.pQueueTail==0 && g.serverHalt==0 ){
   401         -      pthread_cond_wait(&g.serverWakeup, &g.queueMutex);
   402         -    }
   403         -    pMsg = g.pQueueTail;
   404         -    if( pMsg ){
   405         -      if( pMsg->pPrev ){
   406         -        pMsg->pPrev->pNext = 0;
   407         -      }else{
   408         -        g.pQueueHead = 0;
   409         -      }
   410         -      g.pQueueTail = pMsg->pPrev;
   411         -    }
   412         -    pthread_mutex_unlock(&g.queueMutex);
   413         -    if( pMsg==0 ) break;
   414         -
   415         -    /* Process the message just removed
   416         -    */
   417         -    pthread_mutex_lock(&pMsg->clientMutex);
   418         -    switch( pMsg->op ){
   419         -      case MSG_Open: {
   420         -        pMsg->errCode = sqlite3_open(pMsg->zIn, &pMsg->pDb);
   421         -        break;
   422         -      }
   423         -      case MSG_Prepare: {
   424         -        pMsg->errCode = sqlite3_prepare(pMsg->pDb, pMsg->zIn, pMsg->nByte,
   425         -                                        &pMsg->pStmt, &pMsg->zOut);
   426         -        break;
   427         -      }
   428         -      case MSG_Step: {
   429         -        pMsg->errCode = sqlite3_step(pMsg->pStmt);
   430         -        break;
   431         -      }
   432         -      case MSG_Reset: {
   433         -        pMsg->errCode = sqlite3_reset(pMsg->pStmt);
   434         -        break;
   435         -      }
   436         -      case MSG_Finalize: {
   437         -        pMsg->errCode = sqlite3_finalize(pMsg->pStmt);
   438         -        break;
   439         -      }
   440         -      case MSG_Close: {
   441         -        pMsg->errCode = sqlite3_close(pMsg->pDb);
   442         -        break;
   443         -      }
   444         -    }
   445         -
   446         -    /* Signal the client that the message has been processed.
   447         -    */
   448         -    pMsg->op = MSG_Done;
   449         -    pthread_mutex_unlock(&pMsg->clientMutex);
   450         -    pthread_cond_signal(&pMsg->clientWakeup);
   451         -  }
   452         -  pthread_mutex_unlock(&g.serverMutex);
   453         -  sqlite3_thread_cleanup();
   454         -  return 0;
   455         -}
   456         -
   457         -/*
   458         -** Start a server thread if one is not already running.  If there
   459         -** is aleady a server thread running, the new thread will quickly
   460         -** die and this routine is effectively a no-op.
   461         -*/
   462         -void sqlite3_server_start(void){
   463         -  pthread_t x;
   464         -  int rc;
   465         -  g.serverHalt = 0;
   466         -  rc = pthread_create(&x, 0, sqlite3_server, 0);
   467         -  if( rc==0 ){
   468         -    pthread_detach(x);
   469         -  }
   470         -}
   471         -
   472         -/*
   473         -** If a server thread is running, then stop it.  If no server is
   474         -** running, this routine is effectively a no-op.
   475         -**
   476         -** This routine returns immediately without waiting for the server
   477         -** thread to stop.  But be assured that the server will eventually stop.
   478         -*/
   479         -void sqlite3_server_stop(void){
   480         -  g.serverHalt = 1;
   481         -  pthread_cond_broadcast(&g.serverWakeup);
   482         -}
   483         -
   484         -#endif /* defined(OS_UNIX) && OS_UNIX && defined(THREADSAFE) && THREADSAFE */
   485         -#endif /* defined(SQLITE_SERVER) */

Deleted test/crashtest1.c.

     1         -/*
     2         -** This program tests the ability of SQLite database to recover from a crash.
     3         -** This program runs under Unix only, but the results are applicable to all
     4         -** systems.
     5         -**
     6         -** The main process first constructs a test database, then starts creating
     7         -** subprocesses that write to that database.  Each subprocess is killed off,
     8         -** without a chance to clean up its database connection, after a random
     9         -** delay.  This killing of the subprocesses simulates a crash or power
    10         -** failure.  The next subprocess to open the database should rollback
    11         -** whatever operation was in process at the time of the simulated crash.
    12         -**
    13         -** If any problems are encountered, an error is reported and the test stops.
    14         -** If no problems are seen after a large number of tests, we assume that
    15         -** the rollback mechanism is working.
    16         -*/
    17         -#include <stdio.h>
    18         -#include <unistd.h>
    19         -#include <sys/types.h>
    20         -#include <sys/wait.h>
    21         -#include <signal.h>
    22         -#include <stdlib.h>
    23         -#include <string.h>
    24         -#include <sched.h>
    25         -#include "sqlite.h"
    26         -
    27         -static void do_some_sql(int parent){
    28         -  char *zErr;
    29         -  int rc = SQLITE_OK;
    30         -  sqlite *db;
    31         -  int cnt = 0;
    32         -  static char zBig[] = 
    33         -    "-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
    34         -    "-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
    35         -
    36         -  if( access("./test.db-journal",0)==0 ){
    37         -    /*printf("pid %d: journal exists.  rollback will be required\n",getpid());*/    unlink("test.db-saved");
    38         -    system("cp test.db test.db-saved");
    39         -    unlink("test.db-journal-saved");
    40         -    system("cp test.db-journal test.db-journal-saved");
    41         -  }
    42         -  db = sqlite_open("./test.db", 0, &zErr);
    43         -  if( db==0 ){
    44         -    printf("ERROR: %s\n", zErr);
    45         -    if( strcmp(zErr,"database disk image is malformed")==0 ){
    46         -      kill(parent, SIGKILL);
    47         -    }
    48         -    exit(1);
    49         -  }
    50         -  srand(getpid());
    51         -  while( rc==SQLITE_OK ){
    52         -    cnt++;
    53         -    rc = sqlite_exec_printf(db, 
    54         -       "INSERT INTO t1 VALUES(%d,'%d%s')", 0, 0, &zErr,
    55         -       rand(), rand(), zBig);
    56         -  }
    57         -  if( rc!=SQLITE_OK ){
    58         -    printf("ERROR #%d: %s\n", rc, zErr);
    59         -    if( rc==SQLITE_CORRUPT ){
    60         -      kill(parent, SIGKILL);
    61         -    }
    62         -  }
    63         -  printf("pid %d: cnt=%d\n", getpid(), cnt);
    64         -}
    65         -
    66         -
    67         -int main(int argc, char **argv){
    68         -  int i;
    69         -  sqlite *db;
    70         -  char *zErr;
    71         -  int status;
    72         -  int parent = getpid();
    73         -
    74         -  unlink("test.db");
    75         -  unlink("test.db-journal");
    76         -  db = sqlite_open("test.db", 0, &zErr);
    77         -  if( db==0 ){
    78         -    printf("Cannot initialize: %s\n", zErr);
    79         -    return 1;
    80         -  }
    81         -  sqlite_exec(db, "CREATE TABLE t1(a,b)", 0, 0, 0);
    82         -  sqlite_close(db);
    83         -  for(i=0; i<10000; i++){
    84         -    int pid = fork();
    85         -    if( pid==0 ){
    86         -      sched_yield();
    87         -      do_some_sql(parent);
    88         -      return 0;
    89         -    }
    90         -    printf("test %d, pid=%d\n", i, pid);
    91         -    usleep(rand()%10000 + 1000);
    92         -    kill(pid, SIGKILL);
    93         -    waitpid(pid, &status, 0);
    94         -  }
    95         -  return 0;
    96         -}