SQLite Forum

Two UTF-8 errors in Regexp
Login

Two UTF-8 errors in Regexp

(1) By jleedev on 2022-11-16 02:59:55 [source]

First, the UTF-8 decoding routing includes an off-by-one error when a character in the range U+10000..U+10FFFF is found at the end of the string:

 % ./sqlite3 --version 
3.40.0 2022-11-14 19:42:01 6ee57a2e4e3399481100c40da0229d2d33cbe3290016185c8a60839d14b209f6
 % ./sqlite3 
sqlite> SELECT regexp_bytecode(char(0x1f4a9));
INIT     efbfbdefbfbdefbfbdefbfbd
ANYSTAR     0
MATCH    65533
MATCH    65533
MATCH    65533
MATCH    65533
ACCEPT      0
sqlite> SELECT regexp_bytecode('('||char(0x1f4a9)||')');
ANYSTAR     0
MATCH    128169
ACCEPT      0

Second, the UTF-8 encoding routine that writes to zInit writes bad values when a character in the range U+0800..U+0FFF is found in this prefix (causing it to reject good matches):

sqlite> SELECT regexp_bytecode('\u0800');
INIT     e080  # should be e0a080
ANYSTAR     0
MATCH    2048
ACCEPT      0
sqlite> SELECT regexp_bytecode('\u0fff');
INIT     ffbf  # should be e0bfbf
ANYSTAR     0
MATCH    4095
ACCEPT      0

Patch:

diff --git c/ext/misc/regexp.c i/ext/misc/regexp.c
index d0c8ee5cf..50a8826b4 100644
--- c/ext/misc/regexp.c
+++ i/ext/misc/regexp.c
@@ -185,7 +185,7 @@ static unsigned re_next_char(ReInput *p){
       c = (c&0x0f)<<12 | ((p->z[p->i]&0x3f)<<6) | (p->z[p->i+1]&0x3f);
       p->i += 2;
       if( c<=0x7ff || (c>=0xd800 && c<=0xdfff) ) c = 0xfffd;
-    }else if( (c&0xf8)==0xf0 && p->i+3<p->mx && (p->z[p->i]&0xc0)==0x80
+    }else if( (c&0xf8)==0xf0 && p->i+2<p->mx && (p->z[p->i]&0xc0)==0x80
            && (p->z[p->i+1]&0xc0)==0x80 && (p->z[p->i+2]&0xc0)==0x80 ){
       c = (c&0x07)<<18 | ((p->z[p->i]&0x3f)<<12) | ((p->z[p->i+1]&0x3f)<<6)
                        | (p->z[p->i+2]&0x3f);
@@ -720,7 +720,7 @@ static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){
       unsigned x = pRe->aArg[i];
       if( x<=127 ){
         pRe->zInit[j++] = (unsigned char)x;
-      }else if( x<=0xfff ){
+      }else if( x<=0x7ff ){
         pRe->zInit[j++] = (unsigned char)(0xc0 | (x>>6));
         pRe->zInit[j++] = 0x80 | (x&0x3f);
       }else if( x<=0xffff ){

(2) By Larry Brasfield (larrybr) on 2022-11-17 19:30:39 in reply to 1 [link] [source]

Thanks for reporting this. Fixed here.