Skip to content

web search done right

Limit number of segments during indexing. Else we risk having too many open memmap's during merge

Mikkel Denker1aa8f97ec66b
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index bae04621..af1229f8 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -50,6 +50,7 @@ struct Job {
warc_paths: Vec<String>,
base_path: String,
host_centrality_threshold: Option<f64>,
+ max_num_segments: u32,
}
struct IndexingWorker {
@@ -191,6 +192,11 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
std::fs::remove_file(path).unwrap();
}
+ index
+ .inverted_index
+ .merge_into_segments(job.max_num_segments)
+ .unwrap();
+
info!("{} done", name);
index
@@ -292,6 +298,7 @@ impl Indexer {
.index_base_path
.clone()
.unwrap_or_else(|| "data/index".to_string()),
+ max_num_segments: config.final_num_segments.unwrap_or(20),
})
.collect_vec()
.into_iter(),
@@ -365,6 +372,7 @@ impl Indexer {
.output_path
.clone()
.unwrap_or_else(|| "data/index".to_string()),
+ max_num_segments: config.final_num_segments.unwrap_or(20),
})
.collect_vec()
.into_par_iter()
@@ -386,10 +394,9 @@ impl Indexer {
index = index.merge(other);
std::fs::remove_dir_all(other_path)?;
+ index.inverted_index.merge_into_segments(num_segments)?;
}
- index.inverted_index.merge_into_segments(num_segments)?;
-
Ok(())
}
}
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index 5e15fd1d..8a1adbe0 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -263,7 +263,7 @@ impl InvertedIndex {
.into_iter()
.collect();
- if segments.len() > 1 {
+ if segments.len() > num_segments as usize {
segments.sort_by_key(|b| std::cmp::Reverse(b.num_docs()));
for segment in segments {