SQLite

#include <assert.h>

#ifndef SQLITE_AMALGAMATION

typedef unsigned char  u8;
typedef unsigned int   u32;
typedef unsigned short u16;
typedef short i16;
typedef sqlite3_int64 i64;
typedef sqlite3_uint64 u64;

#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))

#define testcase(x)
#define ALWAYS(x) 1

struct Fts5HashEntry {
  Fts5HashEntry *pHashNext;       /* Next hash entry with same hash-key */
  Fts5HashEntry *pScanNext;       /* Next entry in sorted order */
  
  int nAlloc;                     /* Total size of allocation */
  int iSzPoslist;                 /* Offset of space for 4-byte poslist size */
  int nData;                      /* Total bytes of data (incl. structure) */
  int nKey;                       /* Length of zKey[] in bytes */
  u8 bDel;                        /* Set delete-flag @ iSzPoslist */
  u8 bContent;                    /* Set content-flag (detail=none mode) */

  i16 iCol;                       /* Column of last value written */
  int iPos;                       /* Position of last value written */
  i64 iRowid;                     /* Rowid of last value written */
  char zKey[8];                   /* Nul-terminated entry key */
};

/*
** Size of Fts5HashEntry without the zKey[] array.

  
  bNew = (pHash->eDetail==FTS5_DETAIL_FULL);

  /* Attempt to locate an existing hash entry */
  iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
  for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
    if( p->zKey[0]==bByte 
     && p->nKey==nToken
     && memcmp(&p->zKey[1], pToken, nToken)==0 

    ){
      break;
    }
  }

  /* If an existing hash entry cannot be found, create a new one. */
  if( p==0 ){

    p = (Fts5HashEntry*)sqlite3_malloc(nByte);
    if( !p ) return SQLITE_NOMEM;
    memset(p, 0, FTS5_HASHENTRYSIZE);
    p->nAlloc = nByte;
    p->zKey[0] = bByte;
    memcpy(&p->zKey[1], pToken, nToken);
    assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
    p->nKey = nToken;
    p->zKey[nToken+1] = '\0';
    p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
    p->pHashNext = pHash->aSlot[iHash];
    pHash->aSlot[iHash] = p;
    pHash->nEntry++;

    /* Add the first rowid field to the hash-entry */

    lappend doc [lindex $vocab $j]
  }
  return $doc
}

foreach_detail_mode $testprefix {

  set vocab [build_vocab1]
  db func r random_doc 
  
  do_execsql_test 1.0 {
    CREATE VIRTUAL TABLE eee USING fts5(e, ee, detail=%DETAIL%);
    BEGIN;
      WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
      INSERT INTO eee SELECT r($vocab, 5), r($vocab, 7) FROM ii;
      INSERT INTO eee(eee) VALUES('integrity-check');
    COMMIT;
    INSERT INTO eee(eee) VALUES('integrity-check');
  }
  
  set hash [sqlite3_fts5_token_hash 1024 xyz]
  set vocab [build_vocab1 -prefix xyz -hash $hash]
  lappend vocab xyz
  
  do_execsql_test 1.1 {
    CREATE VIRTUAL TABLE vocab USING fts5vocab(eee, 'row'); 
    BEGIN;
  }
  do_test 1.2 {
    for {set i 1} {$i <= 100} {incr i} {
      execsql { INSERT INTO eee VALUES( r($vocab, 5), r($vocab, 7) ) }
    }
  } {}
    
  do_test 1.3 {
    db eval { SELECT term, doc FROM vocab } {
      set nRow [db one {SELECT count(*) FROM eee WHERE eee MATCH $term}]
      if {$nRow != $doc} {
        error "term=$term fts5vocab=$doc cnt=$nRow"
      }
    }
    set {} {}
  } {}
  
  do_execsql_test 1.4 {
    COMMIT;
    INSERT INTO eee(eee) VALUES('integrity-check');
  }

  #-----------------------------------------------------------------------
  # Add a small and very large token with the same hash value to an
  # empty table. At one point this would provoke an asan error.
  #
  do_test 2.0 {
    set big [string repeat 12345 40]
    set hash [sqlite3_fts5_token_hash 1024 $big]
    while {1} {
      set small [random_token]
      if {[sqlite3_fts5_token_hash 1024 $small]==$hash} break
    }

    execsql { CREATE VIRTUAL TABLE t2 USING fts5(x, detail=%DETAIL%) }
breakpoint
    execsql {
      INSERT INTO t2 VALUES($small || ' ' || $big);
    }
  } {}

} ;# foreach_detail_mode

finish_test