SQLite

Check-in [7a9988d38c]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge the fixes and tests for the ieee754 extension.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | int-float-precision
Files: files | file ages | folders
SHA1: 7a9988d38cd7ab3df79eed97631ff23d5928b4ad
User & Date: drh 2015-11-06 17:59:33.872
Context
2015-11-06
19:53
Enforce consistent comparison results between integer and floating point values. Fix for ticket [38a97a87a6e4e83]. (check-in: 849e686da2 user: drh tags: trunk)
17:59
Merge the fixes and tests for the ieee754 extension. (Closed-Leaf check-in: 7a9988d38c user: drh tags: int-float-precision)
17:01
Test cases for the ieee754 extension. (check-in: 840cbda886 user: drh tags: trunk)
14:59
A different approach to handling integer/float comparisons. This seems to work better on systems where "long double" and "double" are the same type, but still needs refinement and testing. (check-in: a3f7614b20 user: drh tags: int-float-precision)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5_config.c.
703
704
705
706
707
708
709



710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736

737
738
739
740
741
742
743
  char *zRank = 0;
  char *zRankArgs = 0;
  int rc = SQLITE_OK;

  *pzRank = 0;
  *pzRankArgs = 0;




  p = fts5ConfigSkipWhitespace(p);
  pRank = p;
  p = fts5ConfigSkipBareword(p);

  if( p ){
    zRank = sqlite3Fts5MallocZero(&rc, 1 + p - pRank);
    if( zRank ) memcpy(zRank, pRank, p-pRank);
  }else{
    rc = SQLITE_ERROR;
  }

  if( rc==SQLITE_OK ){
    p = fts5ConfigSkipWhitespace(p);
    if( *p!='(' ) rc = SQLITE_ERROR;
    p++;
  }
  if( rc==SQLITE_OK ){
    const char *pArgs; 
    p = fts5ConfigSkipWhitespace(p);
    pArgs = p;
    if( *p!=')' ){
      p = fts5ConfigSkipArgs(p);
      if( p==0 ){
        rc = SQLITE_ERROR;
      }else{
        zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs);
        if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs);

      }
    }
  }

  if( rc!=SQLITE_OK ){
    sqlite3_free(zRank);
    assert( zRankArgs==0 );







>
>
>
|
|
|

|
|
|
|
|
|

|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>







703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
  char *zRank = 0;
  char *zRankArgs = 0;
  int rc = SQLITE_OK;

  *pzRank = 0;
  *pzRankArgs = 0;

  if( p==0 ){
    rc = SQLITE_ERROR;
  }else{
    p = fts5ConfigSkipWhitespace(p);
    pRank = p;
    p = fts5ConfigSkipBareword(p);

    if( p ){
      zRank = sqlite3Fts5MallocZero(&rc, 1 + p - pRank);
      if( zRank ) memcpy(zRank, pRank, p-pRank);
    }else{
      rc = SQLITE_ERROR;
    }

    if( rc==SQLITE_OK ){
      p = fts5ConfigSkipWhitespace(p);
      if( *p!='(' ) rc = SQLITE_ERROR;
      p++;
    }
    if( rc==SQLITE_OK ){
      const char *pArgs; 
      p = fts5ConfigSkipWhitespace(p);
      pArgs = p;
      if( *p!=')' ){
        p = fts5ConfigSkipArgs(p);
        if( p==0 ){
          rc = SQLITE_ERROR;
        }else{
          zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs);
          if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs);
        }
      }
    }
  }

  if( rc!=SQLITE_OK ){
    sqlite3_free(zRank);
    assert( zRankArgs==0 );
Changes to ext/fts5/fts5_index.c.
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
*/
int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){
  assert( p->rc==SQLITE_OK );

  /* Allocate the hash table if it has not already been allocated */
  if( p->pHash==0 ){
    p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData);

    /* Force the configuration to be loaded */
    fts5StructureRelease(fts5StructureRead(p));
  }

  /* Flush the hash table to disk if required */
  if( iRowid<p->iWriteRowid 
   || (iRowid==p->iWriteRowid && p->bDelete==0)
   || (p->nPendingData > p->pConfig->nHashSize) 
  ){







<
<
<







4443
4444
4445
4446
4447
4448
4449



4450
4451
4452
4453
4454
4455
4456
*/
int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){
  assert( p->rc==SQLITE_OK );

  /* Allocate the hash table if it has not already been allocated */
  if( p->pHash==0 ){
    p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData);



  }

  /* Flush the hash table to disk if required */
  if( iRowid<p->iWriteRowid 
   || (iRowid==p->iWriteRowid && p->bDelete==0)
   || (p->nPendingData > p->pConfig->nHashSize) 
  ){
Changes to ext/fts5/fts5_main.c.
390
391
392
393
394
395
396









397
398
399
400
401
402
403
    );
  }

  /* Call sqlite3_declare_vtab() */
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5ConfigDeclareVtab(pConfig);
  }










  if( rc!=SQLITE_OK ){
    fts5FreeVtab(pTab);
    pTab = 0;
  }else if( bCreate ){
    fts5CheckTransactionState(pTab, FTS5_BEGIN, 0);
  }







>
>
>
>
>
>
>
>
>







390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
    );
  }

  /* Call sqlite3_declare_vtab() */
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5ConfigDeclareVtab(pConfig);
  }

  /* Load the initial configuration */
  if( rc==SQLITE_OK ){
    assert( pConfig->pzErrmsg==0 );
    pConfig->pzErrmsg = pzErr;
    rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex);
    sqlite3Fts5IndexRollback(pTab->pIndex);
    pConfig->pzErrmsg = 0;
  }

  if( rc!=SQLITE_OK ){
    fts5FreeVtab(pTab);
    pTab = 0;
  }else if( bCreate ){
    fts5CheckTransactionState(pTab, FTS5_BEGIN, 0);
  }
825
826
827
828
829
830
831






























832
833
834
835
836
837
838
        break;
    }
  }
  
  return rc;
}































static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){
  Fts5Config *pConfig = pTab->pConfig;
  Fts5Sorter *pSorter;
  int nPhrase;
  int nByte;
  int rc = SQLITE_OK;
  char *zSql;







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
        break;
    }
  }
  
  return rc;
}


static sqlite3_stmt *fts5PrepareStatement(
  int *pRc,
  Fts5Config *pConfig, 
  const char *zFmt,
  ...
){
  sqlite3_stmt *pRet = 0;
  va_list ap;
  va_start(ap, zFmt);

  if( *pRc==SQLITE_OK ){
    int rc;
    char *zSql = sqlite3_vmprintf(zFmt, ap);
    if( zSql==0 ){
      rc = SQLITE_NOMEM; 
    }else{
      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
      if( rc!=SQLITE_OK ){
        *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
      }
      sqlite3_free(zSql);
    }
    *pRc = rc;
  }

  va_end(ap);
  return pRet;
} 

static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){
  Fts5Config *pConfig = pTab->pConfig;
  Fts5Sorter *pSorter;
  int nPhrase;
  int nByte;
  int rc = SQLITE_OK;
  char *zSql;
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
  /* TODO: It would be better to have some system for reusing statement
  ** handles here, rather than preparing a new one for each query. But that
  ** is not possible as SQLite reference counts the virtual table objects.
  ** And since the statement required here reads from this very virtual 
  ** table, saving it creates a circular reference.
  **
  ** If SQLite a built-in statement cache, this wouldn't be a problem. */
  zSql = sqlite3Fts5Mprintf(&rc, 
      "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s",
      pConfig->zDb, pConfig->zName, zRank, pConfig->zName,
      (zRankArgs ? ", " : ""),
      (zRankArgs ? zRankArgs : ""),
      bDesc ? "DESC" : "ASC"
  );
  if( zSql ){
    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pSorter->pStmt, 0);
    sqlite3_free(zSql);
  }

  pCsr->pSorter = pSorter;
  if( rc==SQLITE_OK ){
    assert( pTab->pSortCsr==0 );
    pTab->pSortCsr = pCsr;
    rc = fts5SorterNext(pCsr);
    pTab->pSortCsr = 0;







|






<
<
<
<







888
889
890
891
892
893
894
895
896
897
898
899
900
901




902
903
904
905
906
907
908
  /* TODO: It would be better to have some system for reusing statement
  ** handles here, rather than preparing a new one for each query. But that
  ** is not possible as SQLite reference counts the virtual table objects.
  ** And since the statement required here reads from this very virtual 
  ** table, saving it creates a circular reference.
  **
  ** If SQLite a built-in statement cache, this wouldn't be a problem. */
  pSorter->pStmt = fts5PrepareStatement(&rc, pConfig,
      "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s",
      pConfig->zDb, pConfig->zName, zRank, pConfig->zName,
      (zRankArgs ? ", " : ""),
      (zRankArgs ? zRankArgs : ""),
      bDesc ? "DESC" : "ASC"
  );





  pCsr->pSorter = pSorter;
  if( rc==SQLITE_OK ){
    assert( pTab->pSortCsr==0 );
    pTab->pSortCsr = pCsr;
    rc = fts5SorterNext(pCsr);
    pTab->pSortCsr = 0;
Changes to ext/fts5/test/fts5fault1.test.
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
do_faultsim_test 2 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { 
    INSERT INTO t1 VALUES('a b c', 'a bc def ghij klmno');
  }
} -test {
  faultsim_test_result {0 {}} 
}

reset_db
do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3');
  INSERT INTO t1 VALUES('a b c', 'a bc def ghij klmno');
}
faultsim_save_and_close
do_faultsim_test 3 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { DELETE FROM t1 }
} -test {
  faultsim_test_result {0 {}} 
}

reset_db
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE t2 USING fts5(a, b);
  INSERT INTO t2 VALUES('m f a jj th q jr ar',   'hj n h h sg j i m');
  INSERT INTO t2 VALUES('nr s t g od j kf h',    'sb h aq rg op rb n nl');







|













|







48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
do_faultsim_test 2 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { 
    INSERT INTO t1 VALUES('a b c', 'a bc def ghij klmno');
  }
} -test {
  faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}}
}

reset_db
do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3');
  INSERT INTO t1 VALUES('a b c', 'a bc def ghij klmno');
}
faultsim_save_and_close
do_faultsim_test 3 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { DELETE FROM t1 }
} -test {
  faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}}
}

reset_db
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE t2 USING fts5(a, b);
  INSERT INTO t2 VALUES('m f a jj th q jr ar',   'hj n h h sg j i m');
  INSERT INTO t2 VALUES('nr s t g od j kf h',    'sb h aq rg op rb n nl');
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
  9 { m* + f* }       {1 8}
} {
  do_faultsim_test 4.$tn -prep {
    faultsim_restore_and_reopen
  } -body "
    execsql { SELECT rowid FROM t2 WHERE t2 MATCH '$expr' }
  " -test "
    faultsim_test_result {[list 0 $res]}
  "
}


#-------------------------------------------------------------------------
# The following tests use a larger database populated with random data.
#







|







97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
  9 { m* + f* }       {1 8}
} {
  do_faultsim_test 4.$tn -prep {
    faultsim_restore_and_reopen
  } -body "
    execsql { SELECT rowid FROM t2 WHERE t2 MATCH '$expr' }
  " -test "
    faultsim_test_result {[list 0 $res]} {1 {vtable constructor failed: t2}}
  "
}


#-------------------------------------------------------------------------
# The following tests use a larger database populated with random data.
#
Changes to ext/fts5/test/fts5fault4.test.
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
}

set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}]

do_faultsim_test 4 -faults oom-* -body {
  db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'}
} -test {
  faultsim_test_result {0 {0 {} 3}}
}

#-------------------------------------------------------------------------
# An OOM within a query that uses a custom rank function.
#
reset_db
do_execsql_test 5.0 {







|







103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
}

set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}]

do_faultsim_test 4 -faults oom-* -body {
  db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'}
} -test {
  faultsim_test_result {0 {0 {} 4}}
}

#-------------------------------------------------------------------------
# An OOM within a query that uses a custom rank function.
#
reset_db
do_execsql_test 5.0 {
Changes to ext/fts5/test/fts5rank.test.
36
37
38
39
40
41
42























































43
44
45
  SELECT highlight(xyz, 0, '[', ']') FROM xyz WHERE xyz MATCH 'x' ORDER BY rank
} [list [string map {x [x]} $doc]]

do_execsql_test 1.3 {
  SELECT highlight(xyz, 0, '[', ']') FROM xyz
  WHERE xyz MATCH 'x AND y' ORDER BY rank
} [list [string map {x [x] y [y]} $doc]]
























































finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
  SELECT highlight(xyz, 0, '[', ']') FROM xyz WHERE xyz MATCH 'x' ORDER BY rank
} [list [string map {x [x]} $doc]]

do_execsql_test 1.3 {
  SELECT highlight(xyz, 0, '[', ']') FROM xyz
  WHERE xyz MATCH 'x AND y' ORDER BY rank
} [list [string map {x [x] y [y]} $doc]]

#-------------------------------------------------------------------------
# Check that the 'rank' option really is persistent.
#
do_execsql_test 2.0 {
  CREATE VIRTUAL TABLE tt USING fts5(a);
  INSERT INTO tt VALUES('a x x x x');
  INSERT INTO tt VALUES('x x a a a');
  INSERT INTO tt VALUES('x a a x x');
}

proc firstinst {cmd} { 
  foreach {p c o} [$cmd xInst 0] {}
  return $o
}
sqlite3_fts5_create_function db firstinst firstinst

do_execsql_test 2.1 {
  SELECT rowid FROM tt('a') ORDER BY rank;
} {2 3 1}

do_execsql_test 2.2 {
  SELECT rowid FROM tt('a', 'firstinst()') ORDER BY rank;
} {1 3 2}

do_execsql_test 2.3 {
  INSERT INTO tt(tt, rank) VALUES('rank', 'firstinst()');
  SELECT rowid FROM tt('a') ORDER BY rank;
} {1 3 2}

do_test 2.4 {
  sqlite3 db2 test.db
  catchsql { SELECT rowid FROM tt('a') ORDER BY rank; } db2
} {1 {no such function: firstinst}}

do_test 2.5 {
  db2 close
  sqlite3 db2 test.db
  sqlite3_fts5_create_function db2 firstinst firstinst
  execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db2
} {1 3 2}

do_test 2.6 {
  execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db2
} {1 3 2}

do_test 2.7 {
  execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db
} {1 3 2}







finish_test

Changes to ext/fts5/test/fts5simple.test.
331
332
333
334
335
336
337















338
339
340
    INSERT INTO ttt(rowid, x) VALUES(2, 'a b c');
    INSERT INTO ttt(rowid, x) VALUES(3, 'a b c');
  COMMIT;
}
do_test 14.2 { 
  fts5_level_segs ttt 
} {1}
















finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
    INSERT INTO ttt(rowid, x) VALUES(2, 'a b c');
    INSERT INTO ttt(rowid, x) VALUES(3, 'a b c');
  COMMIT;
}
do_test 14.2 { 
  fts5_level_segs ttt 
} {1}

#-------------------------------------------------------------------------
db func rnddoc fts5_rnddoc
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE x1 USING fts5(x);
  INSERT INTO x1(x1, rank) VALUES('pgsz', 32);

  WITH ii(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10 )
  INSERT INTO x1 SELECT rnddoc(5) FROM ii;
}

do_execsql_test 4.1 {
  SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'
} {0 {} 4}


finish_test

Changes to ext/misc/ieee754.c.
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
      sqlite3_result_double(context, 0.0);
      return;
    }
    while( (m>>32)&0xffe00000 ){
      m >>= 1;
      e++;
    }
    while( ((m>>32)&0xfff00000)==0 ){
      m <<= 1;
      e--;
    }
    e += 1075;
    if( e<0 ) e = m = 0;
    if( e>0x7ff ) m = 0;
    a = m & ((((sqlite3_int64)1)<<52)-1);
    a |= e<<52;
    if( isNeg ) a |= ((sqlite3_int64)1)<<63;
    memcpy(&r, &a, sizeof(r));
    sqlite3_result_double(context, r);
  }
}







|





|







90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
      sqlite3_result_double(context, 0.0);
      return;
    }
    while( (m>>32)&0xffe00000 ){
      m >>= 1;
      e++;
    }
    while( m!=0 && ((m>>32)&0xfff00000)==0 ){
      m <<= 1;
      e--;
    }
    e += 1075;
    if( e<0 ) e = m = 0;
    if( e>0x7ff ) e = 0x7ff;
    a = m & ((((sqlite3_int64)1)<<52)-1);
    a |= e<<52;
    if( isNeg ) a |= ((sqlite3_int64)1)<<63;
    memcpy(&r, &a, sizeof(r));
    sqlite3_result_double(context, r);
  }
}
Added test/ieee754.test.
















































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
# 2015-11-06
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# 
# Tests of the iee754 extension
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

load_static_extension db ieee754

foreach {id float rep} {
   1       1.0                            1,0
   2       2.0                            2,0
   3       0.5                            1,-1
   4       1.5                            3,-1
   5       0.0                            0,-1075
   6       4.9406564584124654e-324        4503599627370497,-1075
   7       2.2250738585072009e-308        9007199254740991,-1075
   8       2.2250738585072014e-308        1,-1022
} {
  do_test ieee754-100-$id-1 {
    db eval "SELECT ieee754($float);"
  } "ieee754($rep)"
  do_test ieee754-100-$id-2 {
    db eval "SELECT ieee754($rep)==$float;"
  } {1}
  if {$float!=0.0} {
    do_test ieee754-100-$id-3 {
      db eval "SELECT ieee754(-$float);"
    } "ieee754(-$rep)"
    do_test ieee754-100-$id-4 {
      db eval "SELECT ieee754(-$rep)==-$float;"
    } {1}
  }
}

do_execsql_test ieee754-110 {
  SELECT ieee754(1,1024), ieee754(4503599627370495,972);
} {Inf 1.79769313486232e+308}
do_execsql_test ieee754-111 {
  SELECT ieee754(-1,1024), ieee754(-4503599627370495,972);
} {-Inf -1.79769313486232e+308}
do_execsql_test ieee754-112 {
  SELECT ieee754(4503599627370495,973) is null;
} {1}

finish_test