Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Allow multiple incremental merges to proceed concurrently. This is required to prevent a large crisis-merge from occuring while an even larger incremental-merge is underway. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts4-incr-merge-exp |
Files: | files | file ages | folders |
SHA1: |
7ed9d2f24a650b424b97dfc19b8042c4 |
User & Date: | dan 2012-03-27 11:48:02.399 |
Context
2012-03-27
| ||
13:44 | Merge the fts4-incr-merge-exp branch with fts4-incr-merge. (check-in: eb00b95885 user: dan tags: fts4-incr-merge) | |
11:48 | Allow multiple incremental merges to proceed concurrently. This is required to prevent a large crisis-merge from occuring while an even larger incremental-merge is underway. (Closed-Leaf check-in: 7ed9d2f24a user: dan tags: fts4-incr-merge-exp) | |
00:38 | Minor correct errors in the file format description for FTS3/4 contained in the fts3.c header comment. (check-in: fb8aacdd8f user: drh tags: fts4-incr-merge) | |
Changes
Changes to ext/fts3/fts3_write.c.
︙ | ︙ | |||
324 325 326 327 328 329 330 | /* This statement is used to determine which level to read the input from ** when performing an incremental merge. It returns the absolute level number ** of the oldest level in the db that contains at least ? segments. Or, ** if no level in the FTS index contains more than ? segments, the statement ** returns zero rows. */ /* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" | | | 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 | /* This statement is used to determine which level to read the input from ** when performing an incremental merge. It returns the absolute level number ** of the oldest level in the db that contains at least ? segments. Or, ** if no level in the FTS index contains more than ? segments, the statement ** returns zero rows. */ /* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" " ORDER BY (level %% 1024) ASC LIMIT 1", /* Estimate the upper limit on the number of leaf nodes in a new segment ** created by merging the oldest :2 segments from absolute level :1. See ** function sqlite3Fts3Incrmerge() for details. */ /* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", |
︙ | ︙ | |||
4444 4445 4446 4447 4448 4449 4450 | *pnRem = nRem; return rc; } /* ** Store an incr-merge hint in the database. */ | | < < < < < < < | < < < | | > < > > | | < < < < | < < > | | | | < | | > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | | > > < > > > > > > | < < < < < < | < < < < | > < < > > | > > > | | | < < < < < < < > > > | | > > > > > > > > > < | < > > > > > | > > | > > | > > > > > > | > | | | 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 | *pnRem = nRem; return rc; } /* ** Store an incr-merge hint in the database. */ static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ sqlite3_stmt *pReplace = 0; int rc; /* Return code */ rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT); sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); } return rc; } /* ** Load an incr-merge hint from the database. The incr-merge hint, if one ** exists, is stored in the rowid==1 row of the %_stat table. ** ** If successful, populate blob *pHint with the value read from the %_stat ** table and return SQLITE_OK. Otherwise, if an error occurs, return an ** SQLite error code. */ static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ sqlite3_stmt *pSelect = 0; int rc; pHint->n = 0; rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); if( rc==SQLITE_OK ){ int rc2; sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); if( SQLITE_ROW==sqlite3_step(pSelect) ){ const char *aHint = sqlite3_column_blob(pSelect, 0); int nHint = sqlite3_column_bytes(pSelect, 0); if( aHint ){ blobGrowBuffer(pHint, nHint, &rc); if( rc==SQLITE_OK ){ memcpy(pHint->a, aHint, nHint); pHint->n = nHint; } } } rc2 = sqlite3_reset(pSelect); if( rc==SQLITE_OK ) rc = rc2; } return rc; } /* ** If *pRc is not SQLITE_OK when this function is called, it is a no-op. ** Otherwise, append an entry to the hint stored in blob *pHint. Each entry ** consists of two varints, the absolute level number of the input segments ** and the number of input segments. ** ** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs, ** set *pRc to an SQLite error code before returning. */ static void fts3IncrmergeHintPush( Blob *pHint, /* Hint blob to append to */ i64 iAbsLevel, /* First varint to store in hint */ int nInput, /* Second varint to store in hint */ int *pRc /* IN/OUT: Error code */ ){ blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc); if( *pRc==SQLITE_OK ){ pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel); pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput); } } /* ** Read the last entry (most recently pushed) from the hint blob *pHint ** and then remove the entry. Write the two values read to *piAbsLevel and ** *pnInput before returning. ** ** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does ** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB. */ static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ const int nHint = pHint->n; int i; i = pHint->n-2; while( i>0 && (pHint->a[i-1] & 0x80) ) i--; while( i>0 && (pHint->a[i-1] & 0x80) ) i--; pHint->n = i; i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); i += sqlite3Fts3GetVarint32(&pHint->a[i], pnInput); if( i!=nHint ) return SQLITE_CORRUPT_VTAB; return SQLITE_OK; } /* ** Attempt an incremental merge that writes nMerge leaf blocks. ** ** Incremental merges happen nMin segments at a time. The two ** segments to be merged are the nMin oldest segments (the ones with ** the smallest indexes) in the highest level that contains at least ** nMin segments. Multiple merges might occur in an attempt to write the ** quota of nMerge leaf blocks. */ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ int rc; /* Return code */ int nRem = nMerge; /* Number of leaf pages yet to be written */ Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ IncrmergeWriter *pWriter; /* Writer object */ int nSeg = 0; /* Number of input segments */ sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ Blob hint = {0, 0, 0}; /* Hint read from %_stat table */ int bDirtyHint = 0; /* True if blob 'hint' has been modified */ /* Allocate space for the cursor, filter and writer objects */ const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); if( !pWriter ) return SQLITE_NOMEM; pFilter = (Fts3SegFilter *)&pWriter[1]; pCsr = (Fts3MultiSegReader *)&pFilter[1]; rc = fts3IncrmergeHintLoad(p, &hint); while( rc==SQLITE_OK && nRem>0 ){ const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ int bUseHint = 0; /* True if attempting to append */ /* Search the %_segdir table for the absolute level with the smallest ** relative level number that contains at least nMin segments, if any. ** If one is found, set iAbsLevel to the absolute level number and ** nSeg to nMin. If no level with at least nMin segments can be found, ** set nSeg to -1. */ rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); sqlite3_bind_int(pFindLevel, 1, nMin); if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ iAbsLevel = sqlite3_column_int64(pFindLevel, 0); nSeg = nMin; }else{ nSeg = -1; } rc = sqlite3_reset(pFindLevel); /* If the hint read from the %_stat table is not empty, check if the ** last entry in it specifies a relative level smaller than or equal ** to the level identified by the block above (if any). If so, this ** iteration of the loop will work on merging at the hinted level. */ if( rc==SQLITE_OK && hint.n ){ int nHint = hint.n; sqlite3_int64 iHintAbsLevel = 0; /* Hint level */ int nHintSeg = 0; /* Hint number of segments */ rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ iAbsLevel = iHintAbsLevel; nSeg = nHintSeg; bUseHint = 1; bDirtyHint = 1; }else{ /* This undoes the effect of the HintPop() above - so that no entry ** is removed from the hint blob. */ hint.n = nHint; } } /* If nSeg is less that zero, then there is no level with at least ** nMin segments and no hint in the %_stat table. No work to do. ** Exit early in this case. */ if( nSeg<0 ) break; /* Open a cursor to iterate through the contents of the oldest nSeg ** indexes of absolute level iAbsLevel. If this cursor is opened using ** the 'hint' parameters, it is possible that there are less than nSeg ** segments available in level iAbsLevel. In this case, no work is ** done on iAbsLevel - fall through to the next iteration of the loop ** to start work on some other level. */ memset(pWriter, 0, nAlloc); pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; if( rc==SQLITE_OK ){ rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); } if( SQLITE_OK==rc && pCsr->nSegment==nSeg && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){ int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); if( rc==SQLITE_OK ){ if( bUseHint && iIdx>0 ){ const char *zKey = pCsr->zTerm; |
︙ | ︙ | |||
4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 | if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; }while( rc==SQLITE_ROW ); /* Update or delete the input segments */ if( rc==SQLITE_OK ){ nRem -= (1 + pWriter->nWork); rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); } } fts3IncrmergeRelease(p, pWriter, &rc); } sqlite3Fts3SegReaderFinish(pCsr); | > > > > < < | | > > | 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 | if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; }while( rc==SQLITE_ROW ); /* Update or delete the input segments */ if( rc==SQLITE_OK ){ nRem -= (1 + pWriter->nWork); rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); if( nSeg!=0 ){ bDirtyHint = 1; fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); } } } fts3IncrmergeRelease(p, pWriter, &rc); } sqlite3Fts3SegReaderFinish(pCsr); } /* Write the hint values into the %_stat table for the next incr-merger */ if( bDirtyHint && rc==SQLITE_OK ){ rc = fts3IncrmergeHintStore(p, &hint); } sqlite3_free(pWriter); sqlite3_free(hint.a); return rc; } /* ** Convert the text beginning at *pz into an integer and return ** its value. Advance *pz to point to the first character past ** the integer. |
︙ | ︙ |
Changes to test/fts4merge.test.
︙ | ︙ | |||
192 193 194 195 196 197 198 | } { 0 {0 1 2 3 4 5 6 7} 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 2 {0 1 2} } do_execsql_test 5.3 { | | > | | | | | | | | > | | | < < > > | | < | | < | 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 | } { 0 {0 1 2 3 4 5 6 7} 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 2 {0 1 2} } do_execsql_test 5.3 { INSERT INTO t1(t1) VALUES('merge=1,5'); INSERT INTO t1(t1) VALUES('merge=1,5'); SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; } { 0 {0 1 2} 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14} 2 {0 1 2 3} } do_execsql_test 5.4 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'} do_test 5.5 { foreach docid [execsql {SELECT docid FROM t1}] { execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid} } } {} do_execsql_test 5.6 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'} do_execsql_test 5.7 { SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; SELECT quote(value) from t1_stat WHERE rowid=1; } { 0 {0 1 2 3 4 5 6 7 8 9 10} 1 {0 1 2 3 4 5 6 7 8 9 10 11 12} 2 {0 1 2 3 4 5 6 7} X'0105' } do_execsql_test 5.8 { INSERT INTO t1(t1) VALUES('merge=1,6'); INSERT INTO t1(t1) VALUES('merge=1,6'); SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; SELECT quote(value) from t1_stat WHERE rowid=1; } { 0 {0 1 2 3 4} 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 2 {0 1 2 3 4 5 6 7 8} X'0106' } do_test 5.8.1 { fts3_integrity_check t1 } ok do_test 5.9 { set L [expr 16*16*7 + 16*3 + 12] foreach docid [execsql { SELECT docid FROM t1 UNION ALL SELECT docid FROM t1 LIMIT $L }] { execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid} } } {} do_execsql_test 5.10 { SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; SELECT quote(value) from t1_stat WHERE rowid=1; } { 0 0 1 {0 1} 2 0 3 0 X'0106' } do_execsql_test 5.11 { INSERT INTO t1(t1) VALUES('merge=1,6'); SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; SELECT quote(value) from t1_stat WHERE rowid=1; } { 0 0 1 {0 1} 2 0 3 0 X'' } #------------------------------------------------------------------------- # Test cases 6.* # # At one point the following test caused an assert() to fail (because the # second 'merge=1,2' operation below actually "merges" a single input |
︙ | ︙ |