SQLite

Changes On Branch big-function-text
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch big-function-text Excluding Merge-Ins

This is equivalent to a diff from 0a745897 to 414a4d66

2023-07-21
22:28
Performance enhancements for queries using very large string values. (check-in: 3661b9cd user: drh tags: trunk)
22:22
Mark an unreachable branch as NEVER(). (Leaf check-in: 414a4d66 user: drh tags: big-function-text)
19:17
Fix harmless compiler warning introduced by the previous check-in. (check-in: 06f829e9 user: drh tags: big-function-text)
15:01
Multiple optimizations that try to preserve or infer the zero-terminated property of TEXT values. Avoid unnecessary copying of text values destined to become function parameters. All changes help improve performance of doing UPDATEs on large JSON values that are indexed multiple ways. (check-in: d0278cde user: drh tags: big-function-text)
2023-07-19
23:02
Performance enhancements to the JSON parser and code generator. (check-in: 0a745897 user: drh tags: trunk)
20:23
Revise the new JSON parser performance test to make the test run 25 times longer, and thus provide a more repeatable number. (check-in: bee9e403 user: drh tags: trunk)
17:24
Further improvement to JSON parser performance. (check-in: 144c8ccf user: drh tags: json-opt)

Changes to src/expr.c.

4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
            testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_LENGTHARG );
            testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_TYPEOFARG );
            testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_BYTELENARG);
            pFarg->a[0].pExpr->op2 = pDef->funcFlags & OPFLAG_BYTELENARG;
          }
        }

        sqlite3ExprCodeExprList(pParse, pFarg, r1, 0,
                                SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR);
      }else{
        r1 = 0;
      }
#ifndef SQLITE_OMIT_VIRTUALTABLE
      /* Possibly overload the function if the first argument is
      ** a virtual table column.
      **







|
<







4686
4687
4688
4689
4690
4691
4692
4693

4694
4695
4696
4697
4698
4699
4700
            testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_LENGTHARG );
            testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_TYPEOFARG );
            testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_BYTELENARG);
            pFarg->a[0].pExpr->op2 = pDef->funcFlags & OPFLAG_BYTELENARG;
          }
        }

        sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, SQLITE_ECEL_FACTOR);

      }else{
        r1 = 0;
      }
#ifndef SQLITE_OMIT_VIRTUALTABLE
      /* Possibly overload the function if the first argument is
      ** a virtual table column.
      **

Changes to src/json.c.

497
498
499
500
501
502
503

504
505
506
507
508
509
510
511
}


/* Make the JSON in p the result of the SQL function.
*/
static void jsonResult(JsonString *p){
  if( p->bErr==0 ){

    sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, 
                          p->bStatic ? SQLITE_TRANSIENT : sqlite3_free,
                          SQLITE_UTF8);
    jsonZero(p);
  }
  assert( p->bStatic );
}








>
|







497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
}


/* Make the JSON in p the result of the SQL function.
*/
static void jsonResult(JsonString *p){
  if( p->bErr==0 ){
    jsonAppendChar(p, 0);
    sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed-1,
                          p->bStatic ? SQLITE_TRANSIENT : sqlite3_free,
                          SQLITE_UTF8);
    jsonZero(p);
  }
  assert( p->bStatic );
}

Changes to src/util.c.

842
843
844
845
846
847
848


849
850
851
852
853
854
855
856
    memcpy(pOut, &u, 8);
    if( k-i>16 ) return 2;
    if( z[k]!=0 ) return 1;
    return 0;
  }else
#endif /* SQLITE_OMIT_HEX_INTEGER */
  {


    return sqlite3Atoi64(z, pOut, sqlite3Strlen30(z), SQLITE_UTF8);
  }
}

/*
** If zNum represents an integer that will fit in 32-bits, then set
** *pValue to that integer and return true.  Otherwise return false.
**







>
>
|







842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
    memcpy(pOut, &u, 8);
    if( k-i>16 ) return 2;
    if( z[k]!=0 ) return 1;
    return 0;
  }else
#endif /* SQLITE_OMIT_HEX_INTEGER */
  {
    int n = (int)(0x3fffffff&strspn(z,"+- \n\t0123456789"));
    if( z[n] ) n++;
    return sqlite3Atoi64(z, pOut, n, SQLITE_UTF8);
  }
}

/*
** If zNum represents an integer that will fit in 32-bits, then set
** *pValue to that integer and return true.  Otherwise return false.
**

Changes to src/vdbe.c.

552
553
554
555
556
557
558



559
560
561
562
563
564
565
    }
    sqlite3_str_appendf(pStr, " %c%d[", c, pMem->n);
    for(j=0; j<25 && j<pMem->n; j++){
      c = pMem->z[j];
      sqlite3_str_appendchar(pStr, 1, (c>=0x20&&c<=0x7f) ? c : '.');
    }
    sqlite3_str_appendf(pStr, "]%s", encnames[pMem->enc]);



  }
}
#endif

#ifdef SQLITE_DEBUG
/*
** Print the value of a register for tracing purposes:







>
>
>







552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
    }
    sqlite3_str_appendf(pStr, " %c%d[", c, pMem->n);
    for(j=0; j<25 && j<pMem->n; j++){
      c = pMem->z[j];
      sqlite3_str_appendchar(pStr, 1, (c>=0x20&&c<=0x7f) ? c : '.');
    }
    sqlite3_str_appendf(pStr, "]%s", encnames[pMem->enc]);
    if( f & MEM_Term ){
      sqlite3_str_appendf(pStr, "(0-term)");
    }
  }
}
#endif

#ifdef SQLITE_DEBUG
/*
** Print the value of a register for tracing purposes:
3081
3082
3083
3084
3085
3086
3087



3088
3089
3090
3091
3092
3093
3094
      */
      sqlite3VdbeSerialGet((u8*)sqlite3CtypeMap, t, pDest);
    }else{
      if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) goto too_big;
      rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, aOffset[p2], len, pDest);
      if( rc!=SQLITE_OK ) goto abort_due_to_error;
      sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest);



      pDest->flags &= ~MEM_Ephem;
    }
  }

op_column_out:
  UPDATE_MAX_BLOBSIZE(pDest);
  REGISTER_TRACE(pOp->p3, pDest);







>
>
>







3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
      */
      sqlite3VdbeSerialGet((u8*)sqlite3CtypeMap, t, pDest);
    }else{
      if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) goto too_big;
      rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, aOffset[p2], len, pDest);
      if( rc!=SQLITE_OK ) goto abort_due_to_error;
      sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest);
      if( (t&1)!=0 && encoding==SQLITE_UTF8 ){
        pDest->flags |= MEM_Term;
      }
      pDest->flags &= ~MEM_Ephem;
    }
  }

op_column_out:
  UPDATE_MAX_BLOBSIZE(pDest);
  REGISTER_TRACE(pOp->p3, pDest);

Changes to src/vdbeInt.h.

607
608
609
610
611
612
613

614
615
616
617
618
619
620
#else
int sqlite3VdbeMemSetZeroBlob(Mem*,int);
#endif
#ifdef SQLITE_DEBUG
int sqlite3VdbeMemIsRowSet(const Mem*);
#endif
int sqlite3VdbeMemSetRowSet(Mem*);

int sqlite3VdbeMemMakeWriteable(Mem*);
int sqlite3VdbeMemStringify(Mem*, u8, u8);
int sqlite3IntFloatCompare(i64,double);
i64 sqlite3VdbeIntValue(const Mem*);
int sqlite3VdbeMemIntegerify(Mem*);
double sqlite3VdbeRealValue(Mem*);
int sqlite3VdbeBooleanValue(Mem*, int ifNull);







>







607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
#else
int sqlite3VdbeMemSetZeroBlob(Mem*,int);
#endif
#ifdef SQLITE_DEBUG
int sqlite3VdbeMemIsRowSet(const Mem*);
#endif
int sqlite3VdbeMemSetRowSet(Mem*);
void sqlite3VdbeMemZeroTerminateIfAble(Mem*);
int sqlite3VdbeMemMakeWriteable(Mem*);
int sqlite3VdbeMemStringify(Mem*, u8, u8);
int sqlite3IntFloatCompare(i64,double);
i64 sqlite3VdbeIntValue(const Mem*);
int sqlite3VdbeMemIntegerify(Mem*);
double sqlite3VdbeRealValue(Mem*);
int sqlite3VdbeBooleanValue(Mem*, int ifNull);

Changes to src/vdbeapi.c.

510
511
512
513
514
515
516

517
518
519
520
521
522
523
    if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE;
    n &= ~(u64)1;
  }
  if( n>0x7fffffff ){
    (void)invokeValueDestructor(z, xDel, pCtx);
  }else{
    setResultStrOrError(pCtx, z, (int)n, enc, xDel);

  }
}
#ifndef SQLITE_OMIT_UTF16
void sqlite3_result_text16(
  sqlite3_context *pCtx,
  const void *z,
  int n,







>







510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
    if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE;
    n &= ~(u64)1;
  }
  if( n>0x7fffffff ){
    (void)invokeValueDestructor(z, xDel, pCtx);
  }else{
    setResultStrOrError(pCtx, z, (int)n, enc, xDel);
    sqlite3VdbeMemZeroTerminateIfAble(pCtx->pOut);
  }
}
#ifndef SQLITE_OMIT_UTF16
void sqlite3_result_text16(
  sqlite3_context *pCtx,
  const void *z,
  int n,

Changes to src/vdbemem.c.

309
310
311
312
313
314
315


























316
317
318
319
320
321
322
    return sqlite3VdbeMemGrow(pMem, szNew, 0);
  }
  assert( (pMem->flags & MEM_Dyn)==0 );
  pMem->z = pMem->zMalloc;
  pMem->flags &= (MEM_Null|MEM_Int|MEM_Real|MEM_IntReal);
  return SQLITE_OK;
}



























/*
** It is already known that pMem contains an unterminated string.
** Add the zero terminator.
**
** Three bytes of zero are added.  In this way, there is guaranteed
** to be a double-zero byte at an even byte boundary in order to







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
    return sqlite3VdbeMemGrow(pMem, szNew, 0);
  }
  assert( (pMem->flags & MEM_Dyn)==0 );
  pMem->z = pMem->zMalloc;
  pMem->flags &= (MEM_Null|MEM_Int|MEM_Real|MEM_IntReal);
  return SQLITE_OK;
}

/*
** If pMem is already a string, detect if it is a zero-terminated
** string, or make it into one if possible, and mark it as such.
**
** This is an optimization.  Correct operation continues even if
** this routine is a no-op.
*/
void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){
  if( (pMem->flags & (MEM_Str|MEM_Term))!=MEM_Str ) return;
  if( pMem->enc!=SQLITE_UTF8 ) return;
  if( NEVER(pMem->z==0) ) return;
  if( pMem->flags & MEM_Dyn ){
    if( pMem->xDel==sqlite3_free
     && sqlite3_msize(pMem->z) >= (u64)(pMem->n+1)
     && pMem->z[pMem->n]==0 
    ){
      pMem->flags |= MEM_Term;
      return;
    }
  }else if( pMem->szMalloc>0 && pMem->szMalloc >= pMem->n+1 ){
    pMem->z[pMem->n] = 0;
    pMem->flags |= MEM_Term;
    return;
  }
}

/*
** It is already known that pMem contains an unterminated string.
** Add the zero terminator.
**
** Three bytes of zero are added.  In this way, there is guaranteed
** to be a double-zero byte at an even byte boundary in order to
799
800
801
802
803
804
805

806
807
808
809
810
811
812
813


814
815
816
817
818
819
820
      break;
    }
    case SQLITE_AFF_REAL: {
      sqlite3VdbeMemRealify(pMem);
      break;
    }
    default: {

      assert( aff==SQLITE_AFF_TEXT );
      assert( MEM_Str==(MEM_Blob>>3) );
      pMem->flags |= (pMem->flags&MEM_Blob)>>3;
      sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding);
      assert( pMem->flags & MEM_Str || pMem->db->mallocFailed );
      pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero);
      if( encoding!=SQLITE_UTF8 ) pMem->n &= ~1;
      return sqlite3VdbeChangeEncoding(pMem, encoding);


    }
  }
  return SQLITE_OK;
}

/*
** Initialize bulk memory to be a consistent Mem object.







>







|
>
>







825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
      break;
    }
    case SQLITE_AFF_REAL: {
      sqlite3VdbeMemRealify(pMem);
      break;
    }
    default: {
      int rc;
      assert( aff==SQLITE_AFF_TEXT );
      assert( MEM_Str==(MEM_Blob>>3) );
      pMem->flags |= (pMem->flags&MEM_Blob)>>3;
      sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding);
      assert( pMem->flags & MEM_Str || pMem->db->mallocFailed );
      pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero);
      if( encoding!=SQLITE_UTF8 ) pMem->n &= ~1;
      rc = sqlite3VdbeChangeEncoding(pMem, encoding);
      if( rc ) return rc;
      sqlite3VdbeMemZeroTerminateIfAble(pMem);
    }
  }
  return SQLITE_OK;
}

/*
** Initialize bulk memory to be a consistent Mem object.