/ Check-in [a9a2aeab]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a bug causing an incorrect segment size value to be stored if both an old and new FTS version performed work on the same incremental merge operation.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts4-experimental
Files: files | file ages | folders
SHA1: a9a2aeab298ba2ac0b8835e61406e2d83bf7e39b
User & Date: dan 2014-05-16 10:30:44
Context
2014-05-16
15:48
Add extra test to backcompat.test to ensure that old and new versions of FTS may work together on the same incremental merge operation. Closed-Leaf check-in: 3997d47b user: dan tags: fts4-experimental
10:30
Fix a bug causing an incorrect segment size value to be stored if both an old and new FTS version performed work on the same incremental merge operation. check-in: a9a2aeab user: dan tags: fts4-experimental
2014-05-15
19:05
Merge latest trunk changes with this branch. check-in: 5809986f user: dan tags: fts4-experimental
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Changes to ext/fts3/fts3_write.c.

3632
3633
3634
3635
3636
3637
3638

3639
3640
3641
3642
3643
3644
3645
....
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
....
4176
4177
4178
4179
4180
4181
4182

4183
4184
4185
4186
4187
4188
4189
....
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
  int nLeafEst;                   /* Space allocated for leaf blocks */
  int nWork;                      /* Number of leaf pages flushed */
  sqlite3_int64 iAbsLevel;        /* Absolute level of input segments */
  int iIdx;                       /* Index of *output* segment in iAbsLevel+1 */
  sqlite3_int64 iStart;           /* Block number of first allocated block */
  sqlite3_int64 iEnd;             /* Block number of last allocated block */
  sqlite3_int64 nLeafData;        /* Bytes of leaf page data so far */

  NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT];
};

/*
** An object of the following type is used to read data from a single
** FTS segment node. See the following functions:
**
................................................................................
  if( rc==SQLITE_OK ){
    rc = fts3WriteSegdir(p, 
        pWriter->iAbsLevel+1,               /* level */
        pWriter->iIdx,                      /* idx */
        pWriter->iStart,                    /* start_block */
        pWriter->aNodeWriter[0].iBlock,     /* leaves_end_block */
        pWriter->iEnd,                      /* end_block */
        pWriter->nLeafData,                 /* end_block */
        pRoot->block.a, pRoot->block.n      /* root */
    );
  }
  sqlite3_free(pRoot->block.a);
  sqlite3_free(pRoot->key.a);

  *pRc = rc;
................................................................................
    if( sqlite3_step(pSelect)==SQLITE_ROW ){
      iStart = sqlite3_column_int64(pSelect, 1);
      iLeafEnd = sqlite3_column_int64(pSelect, 2);
      fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData);
      if( pWriter->nLeafData<0 ){
        pWriter->nLeafData = pWriter->nLeafData * -1;
      }

      nRoot = sqlite3_column_bytes(pSelect, 4);
      aRoot = sqlite3_column_blob(pSelect, 4);
    }else{
      return sqlite3_reset(pSelect);
    }

    /* Check for the zero-length marker in the %_segments table */
................................................................................
        }
      }

      if( nSeg!=0 ){
        pWriter->nLeafData = pWriter->nLeafData * -1;
      }
      fts3IncrmergeRelease(p, pWriter, &rc);
      if( nSeg==0 ){
        fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData);
      }
    }

    sqlite3Fts3SegReaderFinish(pCsr);
  }








>







 







|







 







>







 







|







3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
....
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
....
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
....
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
  int nLeafEst;                   /* Space allocated for leaf blocks */
  int nWork;                      /* Number of leaf pages flushed */
  sqlite3_int64 iAbsLevel;        /* Absolute level of input segments */
  int iIdx;                       /* Index of *output* segment in iAbsLevel+1 */
  sqlite3_int64 iStart;           /* Block number of first allocated block */
  sqlite3_int64 iEnd;             /* Block number of last allocated block */
  sqlite3_int64 nLeafData;        /* Bytes of leaf page data so far */
  u8 bNoLeafData;                 /* If true, store 0 for segment size */
  NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT];
};

/*
** An object of the following type is used to read data from a single
** FTS segment node. See the following functions:
**
................................................................................
  if( rc==SQLITE_OK ){
    rc = fts3WriteSegdir(p, 
        pWriter->iAbsLevel+1,               /* level */
        pWriter->iIdx,                      /* idx */
        pWriter->iStart,                    /* start_block */
        pWriter->aNodeWriter[0].iBlock,     /* leaves_end_block */
        pWriter->iEnd,                      /* end_block */
        (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0),   /* end_block */
        pRoot->block.a, pRoot->block.n      /* root */
    );
  }
  sqlite3_free(pRoot->block.a);
  sqlite3_free(pRoot->key.a);

  *pRc = rc;
................................................................................
    if( sqlite3_step(pSelect)==SQLITE_ROW ){
      iStart = sqlite3_column_int64(pSelect, 1);
      iLeafEnd = sqlite3_column_int64(pSelect, 2);
      fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData);
      if( pWriter->nLeafData<0 ){
        pWriter->nLeafData = pWriter->nLeafData * -1;
      }
      pWriter->bNoLeafData = (pWriter->nLeafData==0);
      nRoot = sqlite3_column_bytes(pSelect, 4);
      aRoot = sqlite3_column_blob(pSelect, 4);
    }else{
      return sqlite3_reset(pSelect);
    }

    /* Check for the zero-length marker in the %_segments table */
................................................................................
        }
      }

      if( nSeg!=0 ){
        pWriter->nLeafData = pWriter->nLeafData * -1;
      }
      fts3IncrmergeRelease(p, pWriter, &rc);
      if( nSeg==0 && pWriter->bNoLeafData==0 ){
        fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData);
      }
    }

    sqlite3Fts3SegReaderFinish(pCsr);
  }

Changes to test/fts4growth.test.

362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379



380
381
382







383











































384
385
386
do_execsql_test 6.5 {
  INSERT INTO x5 SELECT words FROM t1;
  SELECT level, idx, end_block FROM x5_segdir;
} {
  0 1 {1329 8297} 0 0 {1320 28009} 0 2 {1449 118006}
}

#do_test 3.2 {
  #t1_to_x2
  #execsql {SELECT level, count(*) FROM x2_segdir GROUP BY level}
#} {0 13 1 15 2 5}

#proc second {x} { lindex $x 1 }
#db func second second
#for {set i 0} {$i <1000} {incr i} {
#  t1_to_x2
#  db eval {
#    SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level;



#  } {
#    puts "$i.$level: $c"
#  }







#}












































finish_test








|
|
|
|
|
|
|
|
|
|
|
>
>
>
|
<
<
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383


384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
do_execsql_test 6.5 {
  INSERT INTO x5 SELECT words FROM t1;
  SELECT level, idx, end_block FROM x5_segdir;
} {
  0 1 {1329 8297} 0 0 {1320 28009} 0 2 {1449 118006}
}

#--------------------------------------------------------------------------
# Ensure that if part of an incremental merge is performed by an old
# version that does not support storing segment sizes in the end_block
# field, no size is stored in the final segment (as it would be incorrect).
#
do_execsql_test 7.1 {
  CREATE VIRTUAL TABLE x6 USING fts4;
  INSERT INTO x6 SELECT words FROM t1;
  INSERT INTO x6 SELECT words FROM t1;
  INSERT INTO x6 SELECT words FROM t1;
  INSERT INTO x6 SELECT words FROM t1;
  INSERT INTO x6 SELECT words FROM t1;
  INSERT INTO x6 SELECT words FROM t1;
  SELECT level, idx, end_block FROM x6_segdir;
} {


  0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
  0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
}

do_execsql_test 7.2 {
  INSERT INTO x6(x6) VALUES('merge=25,4');
  SELECT level, idx, end_block FROM x6_segdir;
} {
  0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
  0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
  1 0 {16014 -51226}
}

do_execsql_test 7.3 {
  UPDATE x6_segdir SET end_block = first(end_block) WHERE level=1;
  SELECT level, idx, end_block FROM x6_segdir;
} {
  0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
  0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
  1 0 16014
}

do_execsql_test 7.4 {
  INSERT INTO x6(x6) VALUES('merge=25,4');
  SELECT level, idx, end_block FROM x6_segdir;
} {
  0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
  0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
  1 0 16014
}

do_execsql_test 7.5 {
  INSERT INTO x6(x6) VALUES('merge=2500,4');
  SELECT level, idx, end_block FROM x6_segdir;
} {
  0 0 {598 118006} 0 1 {718 118006} 1 0 16014
}

do_execsql_test 7.6 {
  INSERT INTO x6(x6) VALUES('merge=2500,2');
  SELECT level, idx, start_block, leaves_end_block, end_block FROM x6_segdir;
} {
  2 0 23695 24147 {41262 633507}
}

do_execsql_test 7.7 {
  SELECT sum(length(block)) FROM x6_segments 
  WHERE blockid BETWEEN 23695 AND 24147
} {633507}



finish_test