/ Check-in [a9a2aeab]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a bug causing an incorrect segment size value to be stored if both an old and new FTS version performed work on the same incremental merge operation.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts4-experimental
Files: files | file ages | folders
SHA1: a9a2aeab298ba2ac0b8835e61406e2d83bf7e39b
User & Date: dan 2014-05-16 10:30:44
Context
2014-05-16
15:48
Add extra test to backcompat.test to ensure that old and new versions of FTS may work together on the same incremental merge operation. Closed-Leaf check-in: 3997d47b user: dan tags: fts4-experimental
10:30
Fix a bug causing an incorrect segment size value to be stored if both an old and new FTS version performed work on the same incremental merge operation. check-in: a9a2aeab user: dan tags: fts4-experimental
2014-05-15
19:05
Merge latest trunk changes with this branch. check-in: 5809986f user: dan tags: fts4-experimental
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3_write.c.

  3632   3632     int nLeafEst;                   /* Space allocated for leaf blocks */
  3633   3633     int nWork;                      /* Number of leaf pages flushed */
  3634   3634     sqlite3_int64 iAbsLevel;        /* Absolute level of input segments */
  3635   3635     int iIdx;                       /* Index of *output* segment in iAbsLevel+1 */
  3636   3636     sqlite3_int64 iStart;           /* Block number of first allocated block */
  3637   3637     sqlite3_int64 iEnd;             /* Block number of last allocated block */
  3638   3638     sqlite3_int64 nLeafData;        /* Bytes of leaf page data so far */
         3639  +  u8 bNoLeafData;                 /* If true, store 0 for segment size */
  3639   3640     NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT];
  3640   3641   };
  3641   3642   
  3642   3643   /*
  3643   3644   ** An object of the following type is used to read data from a single
  3644   3645   ** FTS segment node. See the following functions:
  3645   3646   **
................................................................................
  4070   4071     if( rc==SQLITE_OK ){
  4071   4072       rc = fts3WriteSegdir(p, 
  4072   4073           pWriter->iAbsLevel+1,               /* level */
  4073   4074           pWriter->iIdx,                      /* idx */
  4074   4075           pWriter->iStart,                    /* start_block */
  4075   4076           pWriter->aNodeWriter[0].iBlock,     /* leaves_end_block */
  4076   4077           pWriter->iEnd,                      /* end_block */
  4077         -        pWriter->nLeafData,                 /* end_block */
         4078  +        (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0),   /* end_block */
  4078   4079           pRoot->block.a, pRoot->block.n      /* root */
  4079   4080       );
  4080   4081     }
  4081   4082     sqlite3_free(pRoot->block.a);
  4082   4083     sqlite3_free(pRoot->key.a);
  4083   4084   
  4084   4085     *pRc = rc;
................................................................................
  4176   4177       if( sqlite3_step(pSelect)==SQLITE_ROW ){
  4177   4178         iStart = sqlite3_column_int64(pSelect, 1);
  4178   4179         iLeafEnd = sqlite3_column_int64(pSelect, 2);
  4179   4180         fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData);
  4180   4181         if( pWriter->nLeafData<0 ){
  4181   4182           pWriter->nLeafData = pWriter->nLeafData * -1;
  4182   4183         }
         4184  +      pWriter->bNoLeafData = (pWriter->nLeafData==0);
  4183   4185         nRoot = sqlite3_column_bytes(pSelect, 4);
  4184   4186         aRoot = sqlite3_column_blob(pSelect, 4);
  4185   4187       }else{
  4186   4188         return sqlite3_reset(pSelect);
  4187   4189       }
  4188   4190   
  4189   4191       /* Check for the zero-length marker in the %_segments table */
................................................................................
  4910   4912           }
  4911   4913         }
  4912   4914   
  4913   4915         if( nSeg!=0 ){
  4914   4916           pWriter->nLeafData = pWriter->nLeafData * -1;
  4915   4917         }
  4916   4918         fts3IncrmergeRelease(p, pWriter, &rc);
  4917         -      if( nSeg==0 ){
         4919  +      if( nSeg==0 && pWriter->bNoLeafData==0 ){
  4918   4920           fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData);
  4919   4921         }
  4920   4922       }
  4921   4923   
  4922   4924       sqlite3Fts3SegReaderFinish(pCsr);
  4923   4925     }
  4924   4926   

Changes to test/fts4growth.test.

   362    362   do_execsql_test 6.5 {
   363    363     INSERT INTO x5 SELECT words FROM t1;
   364    364     SELECT level, idx, end_block FROM x5_segdir;
   365    365   } {
   366    366     0 1 {1329 8297} 0 0 {1320 28009} 0 2 {1449 118006}
   367    367   }
   368    368   
   369         -#do_test 3.2 {
   370         -  #t1_to_x2
   371         -  #execsql {SELECT level, count(*) FROM x2_segdir GROUP BY level}
   372         -#} {0 13 1 15 2 5}
          369  +#--------------------------------------------------------------------------
          370  +# Ensure that if part of an incremental merge is performed by an old
          371  +# version that does not support storing segment sizes in the end_block
          372  +# field, no size is stored in the final segment (as it would be incorrect).
          373  +#
          374  +do_execsql_test 7.1 {
          375  +  CREATE VIRTUAL TABLE x6 USING fts4;
          376  +  INSERT INTO x6 SELECT words FROM t1;
          377  +  INSERT INTO x6 SELECT words FROM t1;
          378  +  INSERT INTO x6 SELECT words FROM t1;
          379  +  INSERT INTO x6 SELECT words FROM t1;
          380  +  INSERT INTO x6 SELECT words FROM t1;
          381  +  INSERT INTO x6 SELECT words FROM t1;
          382  +  SELECT level, idx, end_block FROM x6_segdir;
          383  +} {
          384  +  0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
          385  +  0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
          386  +}
          387  +
          388  +do_execsql_test 7.2 {
          389  +  INSERT INTO x6(x6) VALUES('merge=25,4');
          390  +  SELECT level, idx, end_block FROM x6_segdir;
          391  +} {
          392  +  0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
          393  +  0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
          394  +  1 0 {16014 -51226}
          395  +}
          396  +
          397  +do_execsql_test 7.3 {
          398  +  UPDATE x6_segdir SET end_block = first(end_block) WHERE level=1;
          399  +  SELECT level, idx, end_block FROM x6_segdir;
          400  +} {
          401  +  0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
          402  +  0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
          403  +  1 0 16014
          404  +}
          405  +
          406  +do_execsql_test 7.4 {
          407  +  INSERT INTO x6(x6) VALUES('merge=25,4');
          408  +  SELECT level, idx, end_block FROM x6_segdir;
          409  +} {
          410  +  0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
          411  +  0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
          412  +  1 0 16014
          413  +}
          414  +
          415  +do_execsql_test 7.5 {
          416  +  INSERT INTO x6(x6) VALUES('merge=2500,4');
          417  +  SELECT level, idx, end_block FROM x6_segdir;
          418  +} {
          419  +  0 0 {598 118006} 0 1 {718 118006} 1 0 16014
          420  +}
          421  +
          422  +do_execsql_test 7.6 {
          423  +  INSERT INTO x6(x6) VALUES('merge=2500,2');
          424  +  SELECT level, idx, start_block, leaves_end_block, end_block FROM x6_segdir;
          425  +} {
          426  +  2 0 23695 24147 {41262 633507}
          427  +}
          428  +
          429  +do_execsql_test 7.7 {
          430  +  SELECT sum(length(block)) FROM x6_segments 
          431  +  WHERE blockid BETWEEN 23695 AND 24147
          432  +} {633507}
   373    433   
   374         -#proc second {x} { lindex $x 1 }
   375         -#db func second second
   376         -#for {set i 0} {$i <1000} {incr i} {
   377         -#  t1_to_x2
   378         -#  db eval {
   379         -#    SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level;
   380         -#  } {
   381         -#    puts "$i.$level: $c"
   382         -#  }
   383         -#}
          434  +
   384    435   
   385    436   finish_test
   386    437