/ Check-in [e3305d4b]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Remove the SQLITE_CONFIG_WORKER_THREADS configuration parameter. The number of worker threads in the sorter is now determined only by the PRAGMA threads=N setting.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | threads
Files: files | file ages | folders
SHA1: e3305d4b4efcbe06945ce7f6ec0f2e864244aaf9
User & Date: drh 2014-08-25 23:44:44
Context
2014-08-29
14:40
Merge recent performance enhancements from trunk onto the threads branch. check-in: 35c44a3c user: drh tags: threads
2014-08-25
23:44
Remove the SQLITE_CONFIG_WORKER_THREADS configuration parameter. The number of worker threads in the sorter is now determined only by the PRAGMA threads=N setting. check-in: e3305d4b user: drh tags: threads
22:43
Merge the CAST operator enhancements from trunk. check-in: 6c8f86e4 user: drh tags: threads
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/global.c.

   163    163      0,                         /* szScratch */
   164    164      0,                         /* nScratch */
   165    165      (void*)0,                  /* pPage */
   166    166      0,                         /* szPage */
   167    167      0,                         /* nPage */
   168    168      0,                         /* mxParserStack */
   169    169      0,                         /* sharedCacheEnabled */
   170         -   SQLITE_DEFAULT_WORKER_THREADS,  /* nWorker */
   171    170      /* All the rest should always be initialized to zero */
   172    171      0,                         /* isInit */
   173    172      0,                         /* inProgress */
   174    173      0,                         /* isMutexInit */
   175    174      0,                         /* isMallocInit */
   176    175      0,                         /* isPCacheInit */
   177    176      0,                         /* nRefInitMutex */

Changes to src/main.c.

   511    511   #if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC)
   512    512       case SQLITE_CONFIG_WIN32_HEAPSIZE: {
   513    513         sqlite3GlobalConfig.nHeap = va_arg(ap, int);
   514    514         break;
   515    515       }
   516    516   #endif
   517    517   
   518         -    case SQLITE_CONFIG_WORKER_THREADS: {
   519         -#if SQLITE_MAX_WORKER_THREADS>0
   520         -      int n = va_arg(ap, int);
   521         -      if( n>SQLITE_MAX_WORKER_THREADS ) n = SQLITE_MAX_WORKER_THREADS;
   522         -      if( n>=0 ) sqlite3GlobalConfig.nWorker = n;
   523         -#endif
   524         -      break;
   525         -    }
   526         -
   527    518       default: {
   528    519         rc = SQLITE_ERROR;
   529    520         break;
   530    521       }
   531    522     }
   532    523     va_end(ap);
   533    524     return rc;

Changes to src/pragma.c.

  2288   2288     case PragTyp_THREADS: {
  2289   2289       sqlite3_int64 N;
  2290   2290       if( sqlite3GlobalConfig.bCoreMutex
  2291   2291        && zRight
  2292   2292        && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK
  2293   2293        && N>=0
  2294   2294       ){
  2295         -      if( N>sqlite3GlobalConfig.nWorker ) N = sqlite3GlobalConfig.nWorker;
         2295  +      if( N>SQLITE_MAX_WORKER_THREADS ) N = SQLITE_MAX_WORKER_THREADS;
  2296   2296         db->mxWorker = N&0xff;
  2297   2297       }
  2298   2298       returnSingleInt(pParse, "soft_heap_limit",  db->mxWorker);
  2299   2299       break;
  2300   2300     }
  2301   2301   
  2302   2302   #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)

Changes to src/shell.c.

  3814   3814     data->mode = MODE_List;
  3815   3815     memcpy(data->separator,"|", 2);
  3816   3816     memcpy(data->newline,"\r\n", 3);
  3817   3817     data->showHeader = 0;
  3818   3818     sqlite3_config(SQLITE_CONFIG_URI, 1);
  3819   3819     sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data);
  3820   3820     sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
  3821         -  sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, 64);
  3822   3821     sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> ");
  3823   3822     sqlite3_snprintf(sizeof(continuePrompt), continuePrompt,"   ...> ");
  3824   3823   }
  3825   3824   
  3826   3825   /*
  3827   3826   ** Output text to the console in a font that attracts extra attention.
  3828   3827   */

Changes to src/sqlite.h.in.

  1714   1714   ** [[SQLITE_CONFIG_WIN32_HEAPSIZE]]
  1715   1715   ** <dt>SQLITE_CONFIG_WIN32_HEAPSIZE
  1716   1716   ** <dd>^This option is only available if SQLite is compiled for Windows
  1717   1717   ** with the [SQLITE_WIN32_MALLOC] pre-processor macro defined.
  1718   1718   ** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value
  1719   1719   ** that specifies the maximum size of the created heap.
  1720   1720   ** </dl>
  1721         -**
  1722         -** [[SQLITE_CONFIG_WORKER_THREADS]]
  1723         -** <dt>SQLITE_CONFIG_WORKER_THREADS
  1724         -** <dd>^SQLITE_CONFIG_WORKER_THREADS takes a single argument of type int.
  1725         -** It is used to set the number of background worker threads that may be
  1726         -** launched when sorting large amounts of data. A value of 0 means launch 
  1727         -** no background threads at all. The maximum number of background threads 
  1728         -** allowed is configured at build-time by the SQLITE_MAX_WORKER_THREADS 
  1729         -** pre-processor option. 
  1730         -** </dl>
  1731   1721   */
  1732   1722   #define SQLITE_CONFIG_SINGLETHREAD  1  /* nil */
  1733   1723   #define SQLITE_CONFIG_MULTITHREAD   2  /* nil */
  1734   1724   #define SQLITE_CONFIG_SERIALIZED    3  /* nil */
  1735   1725   #define SQLITE_CONFIG_MALLOC        4  /* sqlite3_mem_methods* */
  1736   1726   #define SQLITE_CONFIG_GETMALLOC     5  /* sqlite3_mem_methods* */
  1737   1727   #define SQLITE_CONFIG_SCRATCH       6  /* void*, int sz, int N */
................................................................................
  1748   1738   #define SQLITE_CONFIG_URI          17  /* int */
  1749   1739   #define SQLITE_CONFIG_PCACHE2      18  /* sqlite3_pcache_methods2* */
  1750   1740   #define SQLITE_CONFIG_GETPCACHE2   19  /* sqlite3_pcache_methods2* */
  1751   1741   #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20  /* int */
  1752   1742   #define SQLITE_CONFIG_SQLLOG       21  /* xSqllog, void* */
  1753   1743   #define SQLITE_CONFIG_MMAP_SIZE    22  /* sqlite3_int64, sqlite3_int64 */
  1754   1744   #define SQLITE_CONFIG_WIN32_HEAPSIZE      23  /* int nByte */
  1755         -#define SQLITE_CONFIG_WORKER_THREADS      24  /* int nWorker */
  1756   1745   
  1757   1746   /*
  1758   1747   ** CAPI3REF: Database Connection Configuration Options
  1759   1748   **
  1760   1749   ** These constants are the available integer configuration options that
  1761   1750   ** can be passed as the second argument to the [sqlite3_db_config()] interface.
  1762   1751   **

Changes to src/sqliteInt.h.

   437    437   /*
   438    438   ** If no value has been provided for SQLITE_MAX_WORKER_THREADS, or if
   439    439   ** SQLITE_TEMP_STORE is set to 3 (never use temporary files), set it 
   440    440   ** to zero.
   441    441   */
   442    442   #if SQLITE_TEMP_STORE==3
   443    443   # undef SQLITE_MAX_WORKER_THREADS
          444  +# define SQLITE_MAX_WORKER_THREADS 0
   444    445   #endif
   445    446   #ifndef SQLITE_MAX_WORKER_THREADS
   446         -# define SQLITE_MAX_WORKER_THREADS 0
          447  +# define SQLITE_MAX_WORKER_THREADS 4
   447    448   #endif
   448    449   #ifndef SQLITE_DEFAULT_WORKER_THREADS
   449    450   # define SQLITE_DEFAULT_WORKER_THREADS 0
   450    451   #endif
   451    452   #if SQLITE_DEFAULT_WORKER_THREADS>SQLITE_MAX_WORKER_THREADS
   452    453   # undef SQLITE_MAX_WORKER_THREADS
   453    454   # define SQLITE_MAX_WORKER_THREADS SQLITE_DEFAULT_WORKER_THREADS
................................................................................
  2760   2761     int szScratch;                    /* Size of each scratch buffer */
  2761   2762     int nScratch;                     /* Number of scratch buffers */
  2762   2763     void *pPage;                      /* Page cache memory */
  2763   2764     int szPage;                       /* Size of each page in pPage[] */
  2764   2765     int nPage;                        /* Number of pages in pPage[] */
  2765   2766     int mxParserStack;                /* maximum depth of the parser stack */
  2766   2767     int sharedCacheEnabled;           /* true if shared-cache mode enabled */
  2767         -  int nWorker;                      /* Number of worker threads to use */
  2768   2768     /* The above might be initialized to non-zero.  The following need to always
  2769   2769     ** initially be zero, however. */
  2770   2770     int isInit;                       /* True after initialization has finished */
  2771   2771     int inProgress;                   /* True while initialization in progress */
  2772   2772     int isMutexInit;                  /* True after mutexes are initialized */
  2773   2773     int isMallocInit;                 /* True after malloc is initialized */
  2774   2774     int isPCacheInit;                 /* True after malloc is initialized */

Changes to src/test_malloc.c.

  1249   1249   
  1250   1250     rc = sqlite3_config(SQLITE_CONFIG_COVERING_INDEX_SCAN, bUseCis);
  1251   1251     Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE);
  1252   1252   
  1253   1253     return TCL_OK;
  1254   1254   }
  1255   1255   
  1256         -/*
  1257         -** Usage:    sqlite3_config_worker_threads N
  1258         -*/
  1259         -static int test_config_worker_threads(
  1260         -  void * clientData, 
  1261         -  Tcl_Interp *interp,
  1262         -  int objc,
  1263         -  Tcl_Obj *CONST objv[]
  1264         -){
  1265         -  int rc;
  1266         -  int nThread;
  1267         -
  1268         -  if( objc!=2 ){
  1269         -    Tcl_WrongNumArgs(interp, 1, objv, "N");
  1270         -    return TCL_ERROR;
  1271         -  }
  1272         -  if( Tcl_GetIntFromObj(interp, objv[1], &nThread) ){
  1273         -    return TCL_ERROR;
  1274         -  }
  1275         -
  1276         -  rc = sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, nThread);
  1277         -  Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE);
  1278         -
  1279         -  return TCL_OK;
  1280         -}
  1281   1256   
  1282   1257   /*
  1283   1258   ** Usage:    sqlite3_dump_memsys3  FILENAME
  1284   1259   **           sqlite3_dump_memsys5  FILENAME
  1285   1260   **
  1286   1261   ** Write a summary of unfreed memsys3 allocations to FILENAME.
  1287   1262   */
................................................................................
  1528   1503        { "sqlite3_config_heap",        test_config_heap              ,0 },
  1529   1504        { "sqlite3_config_heap_size",   test_config_heap_size         ,0 },
  1530   1505        { "sqlite3_config_memstatus",   test_config_memstatus         ,0 },
  1531   1506        { "sqlite3_config_lookaside",   test_config_lookaside         ,0 },
  1532   1507        { "sqlite3_config_error",       test_config_error             ,0 },
  1533   1508        { "sqlite3_config_uri",         test_config_uri               ,0 },
  1534   1509        { "sqlite3_config_cis",         test_config_cis               ,0 },
  1535         -     { "sqlite3_config_worker_threads", test_config_worker_threads ,0 },
  1536   1510        { "sqlite3_db_config_lookaside",test_db_config_lookaside      ,0 },
  1537   1511        { "sqlite3_dump_memsys3",       test_dump_memsys3             ,3 },
  1538   1512        { "sqlite3_dump_memsys5",       test_dump_memsys3             ,5 },
  1539   1513        { "sqlite3_install_memsys3",    test_install_memsys3          ,0 },
  1540   1514        { "sqlite3_memdebug_vfs_oom_test", test_vfs_oom_test          ,0 },
  1541   1515     };
  1542   1516     int i;

Changes to src/vdbesort.c.

   791    791     VdbeSorter *pSorter;            /* The new sorter */
   792    792     KeyInfo *pKeyInfo;              /* Copy of pCsr->pKeyInfo with db==0 */
   793    793     int szKeyInfo;                  /* Size of pCsr->pKeyInfo in bytes */
   794    794     int sz;                         /* Size of pSorter in bytes */
   795    795     int rc = SQLITE_OK;
   796    796   #if SQLITE_MAX_WORKER_THREADS==0
   797    797   # define nWorker 0
   798         -#elif SQLITE_MAX_WORKER_THREADS>=SORTER_MAX_MERGE_COUNT
   799         -  int nWorker = MIN(SORTER_MAX_MERGE_COUNT-1, db->mxWorker);
   800    798   #else
   801         -  int nWorker = db->mxWorker;
          799  +  int nWorker = sqlite3TempInMemory(db) ? 0 : db->mxWorker ;
          800  +#endif
          801  +
          802  +  /* Do not allow the total number of threads (main thread + all workers)
          803  +  ** to exceed the maximum merge count */
          804  +#if SQLITE_MAX_WORKER_THREADS>=SORTER_MAX_MERGE_COUNT
          805  +  if( nWorker>=SORTER_MAX_MERGE_COUNT ){
          806  +    nWorker = SORTER_MAX_MERGE_COUNT-1;
          807  +  }
   802    808   #endif
   803    809   
   804    810     assert( pCsr->pKeyInfo && pCsr->pBt==0 );
   805    811     szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*);
   806    812     sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask);
   807    813   
   808    814     pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo);

Changes to test/sort.test.

   540    540             4    1000000       3     file      true    false             0
   541    541             5          0       0   memory     false     true             0
   542    542             6          0       0     file     false     true       1000000     
   543    543             7          0       0     file     false     true         10000
   544    544   } {
   545    545     db close
   546    546     sqlite3_shutdown
   547         -  sqlite3_config_worker_threads $nWorker
   548    547     if {$coremutex} {
   549    548       sqlite3_config multithread
   550    549     } else {
   551    550       sqlite3_config singlethread
   552    551     }
   553    552     sqlite3_initialize
   554    553     sorter_test_fakeheap $fakeheap
   555    554     sqlite3_soft_heap_limit $softheaplimit
   556    555   
   557    556     reset_db
   558    557     sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit
   559         -  execsql "PRAGMA temp_store = $tmpstore"
          558  +  execsql "PRAGMA temp_store = $tmpstore; PRAGMA threads = $nWorker"
          559  +  
   560    560     
   561    561     set ten [string repeat X 10300]
   562    562     set one [string repeat y   200]
   563    563   
   564    564     if {$softheaplimit} {
   565    565       execsql { PRAGMA cache_size = 20 };
   566    566     } else {
................................................................................
   598    598     } [list 2 $one 4 $ten]
   599    599   
   600    600     sorter_test_fakeheap 0
   601    601   }
   602    602   
   603    603   db close
   604    604   sqlite3_shutdown
   605         -sqlite3_config_worker_threads 0
   606    605   set t(0) singlethread
   607    606   set t(1) multithread
   608    607   set t(2) serialized
   609    608   sqlite3_config $t($sqlite_options(threadsafe))
   610    609   sqlite3_initialize
   611    610   sqlite3_soft_heap_limit 0
   612    611   
................................................................................
   633    632   
   634    633   reset_db
   635    634   do_execsql_test 17.1 {
   636    635     SELECT * FROM sqlite_master ORDER BY sql;
   637    636   } {}
   638    637   
   639    638   finish_test
   640         -

Changes to test/sort2.test.

    18     18   source $testdir/tester.tcl
    19     19   set testprefix sort2
    20     20   
    21     21   foreach {tn script} {
    22     22     1 { }
    23     23     2 {
    24     24       catch { db close }
    25         -    sqlite3_shutdown
    26         -    sqlite3_config_worker_threads 7
    27     25       reset_db
           26  +    catch { db eval {PRAGMA threads=7} }
    28     27     }
    29     28   } {
    30     29   
    31     30     eval $script
    32     31   
    33     32     do_execsql_test $tn.1 {
    34     33       PRAGMA cache_size = 5;
................................................................................
    72     71         SELECT x+1, randomblob(100) FROM r
    73     72         LIMIT 1000000
    74     73       )
    75     74       SELECT count(x), length(y) FROM r GROUP BY (x%5)
    76     75     } {
    77     76       200000 100 200000 100 200000 100 200000 100 200000 100
    78     77     }
    79         -  
    80         -  db close
    81         -  sqlite3_shutdown
    82         -  sqlite3_config_worker_threads 0
    83         -  sqlite3_initialize
    84         -
    85     78   }
    86     79   
    87     80   finish_test
    88         -

Changes to test/sort4.test.

    15     15   #
    16     16   
    17     17   set testdir [file dirname $argv0]
    18     18   source $testdir/tester.tcl
    19     19   set testprefix sort4
    20     20   
    21     21   # Configure the sorter to use 3 background threads.
    22         -catch { db close }
    23         -sqlite3_shutdown
    24         -sqlite3_config_worker_threads 3
    25         -sqlite3_initialize
    26         -reset_db
           22  +db eval {PRAGMA threads=3}
    27     23   
    28     24   # Minimum number of seconds to run for. If the value is 0, each test
    29     25   # is run exactly once. Otherwise, tests are repeated until the timeout
    30     26   # expires.
    31     27   set SORT4TIMEOUT 0
    32     28   if {[permutation] == "multithread"} { set SORT4TIMEOUT 300 }
    33     29   
................................................................................
   186    182     do_sorter_test $t.8 -repeats 10 -rows 100000 -read 10000 -cachesize 250
   187    183   
   188    184     set iNow [clock_seconds]
   189    185     if {$iNow>=$iTimeLimit} break
   190    186     do_test "$testprefix-([expr $iTimeLimit-$iNow] seconds remain)" {} {}
   191    187   }
   192    188   
   193         -catch { db close }
   194         -sqlite3_shutdown
   195         -sqlite3_config_worker_threads 0
   196         -sqlite3_initialize
   197    189   finish_test
   198         -

Changes to test/sortfault.test.

    26     26             1          0       0     file multithread    false     false
    27     27             2     100000       0     file multithread    false     false
    28     28             3     100000       1     file multithread    false     false
    29     29             4    2000000       0     file singlethread   false      true
    30     30   } {
    31     31     if {$sqlite_options(threadsafe)} { set threadsmode singlethread }
    32     32   
    33         -  catch { db close }
    34         -  sqlite3_shutdown
    35         -  sqlite3_config_worker_threads $nWorker
           33  +  db eval "PRAGMA threads=$nWorker"
    36     34     sqlite3_config $threadsmode
    37     35     if { $lookaside } {
    38     36       sqlite3_config_lookaside 100 500
    39     37     } else {
    40     38       sqlite3_config_lookaside 0 0
    41     39     }
    42     40     sqlite3_initialize
................................................................................
   106    104   
   107    105       sqlite3_memdebug_vfs_oom_test 1
   108    106     }
   109    107   }
   110    108   
   111    109   catch { db close }
   112    110   sqlite3_shutdown
   113         -sqlite3_config_worker_threads 0
   114    111   set t(0) singlethread
   115    112   set t(1) multithread
   116    113   set t(2) serialized
   117    114   sqlite3_config $t($sqlite_options(threadsafe))
   118    115   sqlite3_config_lookaside 100 500
   119    116   sqlite3_initialize
   120    117   
................................................................................
   162    159   do_faultsim_test 5.1 -faults oom* -body {
   163    160     execsql { SELECT * FROM t1 ORDER BY a }
   164    161   } -test {
   165    162     faultsim_test_result [list 0 [list $::a $::b $::c $::c $::b $::a]]
   166    163   }
   167    164   
   168    165   finish_test
   169         -