SQLite

Check-in [8f5e9c19]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a case where adjacent tokens are handled incorrectly by the fts5 snippet() function.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 8f5e9c192ff2820d8cfb076ab28f30697d10c22710583d6c7fd7019c4a0ea795
User & Date: dan 2023-11-03 17:20:20
Context
2023-11-03
18:45
Back out the ALWAYS inserted late yesterday. The fuzzer discovered a counter-example. (check-in: 57063557 user: drh tags: trunk)
17:20
Fix a case where adjacent tokens are handled incorrectly by the fts5 snippet() function. (check-in: 8f5e9c19 user: dan tags: trunk)
13:00
Wrap more of the stmt API behind the JNI wrapper1 API. (check-in: 8fea23dc user: stephan tags: trunk)
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_aux.c.

207
208
209
210
211
212
213








214
215
216
217
218
219
220

    if( rc==SQLITE_OK ){
      rc = fts5CInstIterNext(&p->iter);
    }
  }

  if( iPos==p->iRangeEnd ){








    fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
    p->iOff = iEndOff;
  }

  return rc;
}








>
>
>
>
>
>
>
>







207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228

    if( rc==SQLITE_OK ){
      rc = fts5CInstIterNext(&p->iter);
    }
  }

  if( iPos==p->iRangeEnd ){
    if( p->bOpen ){
      if( p->iter.iStart>=0 && iPos>=p->iter.iStart ){
        fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
        p->iOff = iEndOff;
      }
      fts5HighlightAppend(&rc, p, p->zClose, -1);
      p->bOpen = 0;
    }
    fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
    p->iOff = iEndOff;
  }

  return rc;
}

Added ext/fts5/test/fts5tokenizer2.test.



















































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
# 2023 Nov 03
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on the built-in fts5 tokenizers. 
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5tokenizer2

# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}

sqlite3_fts5_create_tokenizer db tst get_tst_tokenizer
proc get_tst_tokenizer {args} {
  return "tst_tokenizer"
}
proc tst_tokenizer {flags txt} {
  set token ""
  set lTok [list]

  foreach c [split $txt {}] {
    if {$token==""} {
      append token $c
    } else {
      set t1 [string is upper $token]
      set t2 [string is upper $c]

      if {$t1!=$t2} {
        lappend lTok $token
        set token ""
      }
      append token $c
    }
  }
  if {$token!=""} { lappend lTok $token }

  set iOff 0
  foreach t $lTok {
    set n [string length $t]
    sqlite3_fts5_token $t $iOff [expr $iOff+$n]
    incr iOff $n
  }
}

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(t, tokenize=tst);
}

do_execsql_test 1.1 {
  INSERT INTO t1 VALUES('AAdontBBmess');
}

do_execsql_test 1.2 {
  SELECT snippet(t1, 0, '>', '<', '...', 4) FROM t1('BB');
} {AAdont>BB<mess}

do_execsql_test 1.3 {
  SELECT highlight(t1, 0, '>', '<') FROM t1('BB');
} {AAdont>BB<mess}

do_execsql_test 1.4 {
  SELECT highlight(t1, 0, '>', '<') FROM t1('AA');
} {>AA<dontBBmess}

do_execsql_test 1.5 {
  SELECT highlight(t1, 0, '>', '<') FROM t1('dont');
} {AA>dont<BBmess}

do_execsql_test 1.6 {
  SELECT highlight(t1, 0, '>', '<') FROM t1('mess');
} {AAdontBB>mess<}

do_execsql_test 1.7 {
  SELECT highlight(t1, 0, '>', '<') FROM t1('BB mess');
} {AAdont>BBmess<}


finish_test