SQLite

Check-in [ba39382ef5]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:If "PRAGMA page_size" commands are not authorized, the FTS module will assume a page size of 1024.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: ba39382ef546b4c4ae15ed90ef0b6c87762bffef
User & Date: drh 2011-06-23 00:59:41.395
Context
2011-06-23
01:07
Remove unused variables from test_vfs.c (check-in: 07884cc218 user: drh tags: trunk)
00:59
If "PRAGMA page_size" commands are not authorized, the FTS module will assume a page size of 1024. (check-in: ba39382ef5 user: drh tags: trunk)
2011-06-22
20:14
Added limited support to omittest.tcl for nmake makefile. (check-in: f5f46dc7b8 user: shaneh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3.c.
640
641
642
643
644
645
646



647
648
649
650
651
652
653
      rc = SQLITE_NOMEM;
    }else{
      rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0);
      if( rc==SQLITE_OK ){
        sqlite3_step(pStmt);
        p->nPgsz = sqlite3_column_int(pStmt, 0);
        rc = sqlite3_finalize(pStmt);



      }
    }
    assert( p->nPgsz>0 || rc!=SQLITE_OK );
    sqlite3_free(zSql);
    *pRc = rc;
  }
}







>
>
>







640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
      rc = SQLITE_NOMEM;
    }else{
      rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0);
      if( rc==SQLITE_OK ){
        sqlite3_step(pStmt);
        p->nPgsz = sqlite3_column_int(pStmt, 0);
        rc = sqlite3_finalize(pStmt);
      }else if( rc==SQLITE_AUTH ){
        p->nPgsz = 1024;
        rc = SQLITE_OK;
      }
    }
    assert( p->nPgsz>0 || rc!=SQLITE_OK );
    sqlite3_free(zSql);
    *pRc = rc;
  }
}
Changes to test/fts4aa.test.
18
19
20
21
22
23
24



25
26
27
28
29
30
31
32
33
34

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}




do_test fts4aa-1.0 {
db eval {
CREATE VIRTUAL TABLE t1 USING fts4(words, tokenize porter);
BEGIN TRANSACTION;
INSERT INTO t1(docid,words) VALUES(1001001,'In the beginning God created the heaven and the earth.');
INSERT INTO t1(docid,words) VALUES(1001002,'And the earth was without form, and void; and darkness was upon the face of the deep. And the Spirit of God moved upon the face of the waters.');
INSERT INTO t1(docid,words) VALUES(1001003,'And God said, Let there be light: and there was light.');
INSERT INTO t1(docid,words) VALUES(1001004,'And God saw the light, that it was good: and God divided the light from the darkness.');
INSERT INTO t1(docid,words) VALUES(1001005,'And God called the light Day, and the darkness he called Night. And the evening and the morning were the first day.');
INSERT INTO t1(docid,words) VALUES(1001006,'And God said, Let there be a firmament in the midst of the waters, and let it divide the waters from the waters.');







>
>
>
|

<







18
19
20
21
22
23
24
25
26
27
28
29

30
31
32
33
34
35
36

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}

# This procedure fills an existing FTS3/FTS4 table with many entries.
# The table needs to have a single column (other than docid) named "words".
#
proc fts4aa_fill_table {} {
db eval {

BEGIN TRANSACTION;
INSERT INTO t1(docid,words) VALUES(1001001,'In the beginning God created the heaven and the earth.');
INSERT INTO t1(docid,words) VALUES(1001002,'And the earth was without form, and void; and darkness was upon the face of the deep. And the Spirit of God moved upon the face of the waters.');
INSERT INTO t1(docid,words) VALUES(1001003,'And God said, Let there be light: and there was light.');
INSERT INTO t1(docid,words) VALUES(1001004,'And God saw the light, that it was good: and God divided the light from the darkness.');
INSERT INTO t1(docid,words) VALUES(1001005,'And God called the light Day, and the darkness he called Night. And the evening and the morning were the first day.');
INSERT INTO t1(docid,words) VALUES(1001006,'And God said, Let there be a firmament in the midst of the waters, and let it divide the waters from the waters.');
1557
1558
1559
1560
1561
1562
1563































1564
1565

1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632































































1633
INSERT INTO t1(docid,words) VALUES(1050022,'And Joseph dwelt in Egypt, he, and his father''s house: and Joseph lived an hundred and ten years.');
INSERT INTO t1(docid,words) VALUES(1050023,'And Joseph saw Ephraim''s children of the third generation: the children also of Machir the son of Manasseh were brought up upon Joseph''s knees.');
INSERT INTO t1(docid,words) VALUES(1050024,'And Joseph said unto his brethren, I die: and God will surely visit you, and bring you out of this land unto the land which he sware to Abraham, to Isaac, and to Jacob.');
INSERT INTO t1(docid,words) VALUES(1050025,'And Joseph took an oath of the children of Israel, saying, God will surely visit you, and ye shall carry up my bones from hence.');
INSERT INTO t1(docid,words) VALUES(1050026,'So Joseph died, being an hundred and ten years old: and they embalmed him, and he was put in a coffin in Egypt.');
COMMIT;
}































} {}



do_test fts4aa-1.1 {
  db eval {
    SELECT docid FROM t1 EXCEPT SELECT docid FROM t1_docsize
  }
} {}
do_test fts4aa-1.2 {
  db eval {
    SELECT docid FROM t1_docsize EXCEPT SELECT docid FROM t1
  }
} {}

proc mit {blob} {
  set scan(littleEndian) i*
  set scan(bigEndian) I*
  binary scan $blob $scan($::tcl_platform(byteOrder)) r
  return $r
}
db func mit mit

do_test fts4aa-2.1 {
  db eval {
    SELECT docid, mit(matchinfo(t1, 'pcxnal')) FROM t1 WHERE t1 MATCH 'melchizedek';
  }
} {1014018 {1 1 1 1 1 1533 25 20}}
do_test fts4aa-2.2 {
  db eval {
    SELECT docid, mit(matchinfo(t1, 'pcxnal')) FROM t1
     WHERE t1 MATCH 'spake hebrew'
     ORDER BY docid;
  }
} {1039014 {2 1 1 40 40 1 6 6 1533 25 42} 1039017 {2 1 1 40 40 1 6 6 1533 25 26}}
do_test fts4aa-2.3 {
  db eval {
    SELECT docid, mit(matchinfo(t1, 'pcxnal')) FROM t1
     WHERE t1 MATCH 'laban overtook jacob'
     ORDER BY docid;
  }
} {1031025 {3 1 2 54 46 1 3 3 2 181 160 1533 25 24}}

do_test fts4aa-9.1 {
  db eval {
    DELETE FROM t1 WHERE docid!=1050026;
    SELECT hex(size) FROM t1_docsize;
    SELECT hex(value) FROM t1_stat;
  }
} {17 01176F}

do_test fts4aa-9.2 {
  db eval {
    SELECT docid FROM t1 EXCEPT SELECT docid FROM t1_docsize
  }
} {}
do_test fts4aa-9.3 {
  db eval {
    SELECT docid FROM t1_docsize EXCEPT SELECT docid FROM t1
  }
} {}
do_test fts4aa-9.4 {
  # Note: Token 'in' is being deferred in the following query. 
  db eval {
    SELECT docid, mit(matchinfo(t1, 'pcxnal')) FROM t1
     WHERE t1 MATCH 'joseph died in egypt'
     ORDER BY docid;
  }
} {1050026 {4 1 1 1 1 1 1 1 2 1 1 1 1 1 1 23 23}}
































































finish_test







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


>
|



















|




|






|







|







|




|




|








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
INSERT INTO t1(docid,words) VALUES(1050022,'And Joseph dwelt in Egypt, he, and his father''s house: and Joseph lived an hundred and ten years.');
INSERT INTO t1(docid,words) VALUES(1050023,'And Joseph saw Ephraim''s children of the third generation: the children also of Machir the son of Manasseh were brought up upon Joseph''s knees.');
INSERT INTO t1(docid,words) VALUES(1050024,'And Joseph said unto his brethren, I die: and God will surely visit you, and bring you out of this land unto the land which he sware to Abraham, to Isaac, and to Jacob.');
INSERT INTO t1(docid,words) VALUES(1050025,'And Joseph took an oath of the children of Israel, saying, God will surely visit you, and ye shall carry up my bones from hence.');
INSERT INTO t1(docid,words) VALUES(1050026,'So Joseph died, being an hundred and ten years old: and they embalmed him, and he was put in a coffin in Egypt.');
COMMIT;
}
}

# The following is a list of queries to perform against the above
# FTS3/FTS4 database.  We will be trying these queries in various
# configurations to ensure that they always return the same answers.
#
set fts4aa_queries {
  {abraham}
  {the king}
  {"the king"}
  {abraham OR joseph}
  {ab* OR jos*}
  {lived t*}
  {spake hebrew}
  {melchizedek}
  {t* melchizedek}
  {melchizedek t*}
}
unset -nocomplain fts4aa_res

# Set up the baseline results
#
do_test fts4aa-1.0 {
  db eval {
    CREATE VIRTUAL TABLE t1 USING fts4(words, tokenize porter);
  }
  fts4aa_fill_table
  foreach q $::fts4aa_queries {
    set r [db eval {SELECT docid FROM t1 WHERE words MATCH $q ORDER BY docid}]
    set ::fts4aa_res($q) $r
  }
} {}

# Legacy test cases
#
do_test fts4aa-1.1 {
  db eval {
    SELECT docid FROM t1 EXCEPT SELECT docid FROM t1_docsize
  }
} {}
do_test fts4aa-1.2 {
  db eval {
    SELECT docid FROM t1_docsize EXCEPT SELECT docid FROM t1
  }
} {}

proc mit {blob} {
  set scan(littleEndian) i*
  set scan(bigEndian) I*
  binary scan $blob $scan($::tcl_platform(byteOrder)) r
  return $r
}
db func mit mit

do_test fts4aa-1.3 {
  db eval {
    SELECT docid, mit(matchinfo(t1, 'pcxnal')) FROM t1 WHERE t1 MATCH 'melchizedek';
  }
} {1014018 {1 1 1 1 1 1533 25 20}}
do_test fts4aa-1.4 {
  db eval {
    SELECT docid, mit(matchinfo(t1, 'pcxnal')) FROM t1
     WHERE t1 MATCH 'spake hebrew'
     ORDER BY docid;
  }
} {1039014 {2 1 1 40 40 1 6 6 1533 25 42} 1039017 {2 1 1 40 40 1 6 6 1533 25 26}}
do_test fts4aa-1.5 {
  db eval {
    SELECT docid, mit(matchinfo(t1, 'pcxnal')) FROM t1
     WHERE t1 MATCH 'laban overtook jacob'
     ORDER BY docid;
  }
} {1031025 {3 1 2 54 46 1 3 3 2 181 160 1533 25 24}}

do_test fts4aa-1.6 {
  db eval {
    DELETE FROM t1 WHERE docid!=1050026;
    SELECT hex(size) FROM t1_docsize;
    SELECT hex(value) FROM t1_stat;
  }
} {17 01176F}

do_test fts4aa-1.7 {
  db eval {
    SELECT docid FROM t1 EXCEPT SELECT docid FROM t1_docsize
  }
} {}
do_test fts4aa-1.8 {
  db eval {
    SELECT docid FROM t1_docsize EXCEPT SELECT docid FROM t1
  }
} {}
do_test fts4aa-1.9 {
  # Note: Token 'in' is being deferred in the following query. 
  db eval {
    SELECT docid, mit(matchinfo(t1, 'pcxnal')) FROM t1
     WHERE t1 MATCH 'joseph died in egypt'
     ORDER BY docid;
  }
} {1050026 {4 1 1 1 1 1 1 1 2 1 1 1 1 1 1 23 23}}

# Should get the same search results from FTS3
#
do_test fts4aa-2.0 {
  db eval {
    DROP TABLE t1;
    CREATE VIRTUAL TABLE t1 USING fts3(words, tokenize porter);
  }
  fts4aa_fill_table
} {}
unset -nocomplain ii
set ii 0
foreach {q r} [array get fts4aa_res] {
  incr ii
  do_test fts4aa-2.$ii {
    db eval {SELECT docid FROM t1 WHERE words MATCH $::q ORDER BY docid}
  } $r
}

# Should get the same search results when the page size is very large
#
do_test fts4aa-3.0 {
  db close
  file delete -force test.db
  sqlite3 db test.db
  db eval {
    PRAGMA page_size=65536;
    CREATE VIRTUAL TABLE t1 USING fts4(words, tokenize porter);
  }
  fts4aa_fill_table
} {}
unset -nocomplain ii
set ii 0
foreach {q r} [array get fts4aa_res] {
  incr ii
  do_test fts4aa-3.$ii {
    db eval {SELECT docid FROM t1 WHERE words MATCH $::q ORDER BY docid}
  } $r
}

# Should get the same search results when an authorizer prevents
# all PRAGMA statements.
#
proc no_pragma_auth {code arg1 arg2 arg3 arg4} {
  if {$code=="SQLITE_PRAGMA"} {return SQLITE_DENY}
  return SQLITE_OK;
}
do_test fts4aa-4.0 {
  db auth ::no_pragma_auth
  db eval {
    DROP TABLE t1;
    CREATE VIRTUAL TABLE t1 USING fts4(words, tokenize porter);
  }
  fts4aa_fill_table
} {}
unset -nocomplain ii
set ii 0
foreach {q r} [array get fts4aa_res] {
  incr ii
  do_test fts4aa-4.$ii {
    db eval {SELECT docid FROM t1 WHERE words MATCH $::q ORDER BY docid}
  } $r
}

finish_test