Skip to content

web search done right

increase stack_size for indexers

Mikkel Denker44f67be57266
diff --git a/crates/core/src/entrypoint/indexer/mod.rs b/crates/core/src/entrypoint/indexer/mod.rs
index f1fd45c9..135351c2 100644
--- a/crates/core/src/entrypoint/indexer/mod.rs
+++ b/crates/core/src/entrypoint/indexer/mod.rs
@@ -47,32 +47,40 @@ pub fn run(config: &config::IndexerConfig) -> Result<()> {
let worker = crate::block_on(IndexingWorker::new(config.clone().into()));
- let indexes = warc_paths
- .into_par_iter()
- .skip(config.skip_warc_files.unwrap_or(0))
- .take(config.limit_warc_files.unwrap_or(usize::MAX))
- .map(|warc_path| Job {
- source_config: job_config.clone(),
- warc_path,
- base_path: config.output_path.clone(),
- settings: JobSettings {
- host_centrality_threshold: config.host_centrality_threshold,
- minimum_clean_words: config.minimum_clean_words,
- batch_size: config.batch_size,
- autocommit_after_num_inserts: config.autocommit_after_num_inserts,
- },
- })
- .map(|job| {
- IndexPointer(
- job.process(&worker)
- .path()
- .as_os_str()
- .to_str()
- .unwrap()
- .to_string(),
- )
- })
- .collect();
+ let thread_pool = rayon::ThreadPoolBuilder::new()
+ .num_threads(usize::from(std::thread::available_parallelism()?))
+ .stack_size(160_000_000)
+ .thread_name(move |num| format!("indexer-worker-{num}"))
+ .build()?;
+
+ let indexes = thread_pool.install(|| {
+ warc_paths
+ .into_par_iter()
+ .skip(config.skip_warc_files.unwrap_or(0))
+ .take(config.limit_warc_files.unwrap_or(usize::MAX))
+ .map(|warc_path| Job {
+ source_config: job_config.clone(),
+ warc_path,
+ base_path: config.output_path.clone(),
+ settings: JobSettings {
+ host_centrality_threshold: config.host_centrality_threshold,
+ minimum_clean_words: config.minimum_clean_words,
+ batch_size: config.batch_size,
+ autocommit_after_num_inserts: config.autocommit_after_num_inserts,
+ },
+ })
+ .map(|job| {
+ IndexPointer(
+ job.process(&worker)
+ .path()
+ .as_os_str()
+ .to_str()
+ .unwrap()
+ .to_string(),
+ )
+ })
+ .collect()
+ });
let index = merge(indexes)?;
crate::mv(index.path(), &config.output_path)?;