Skip to content

web search done right

Merge index segments into num_segments/2 to only merge segments once every num_segments/2 index merges

Mikkel Denkera4c142f5f390
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 0f866c3d..697ac77c 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -230,7 +230,7 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
index
.inverted_index
- .merge_into_segments(job.max_num_segments)
+ .merge_into_max_segments(job.max_num_segments)
.unwrap();
info!("{} done", name);
@@ -351,7 +351,7 @@ impl Indexer {
index
.inverted_index
- .merge_into_segments(config.final_num_segments.unwrap_or(20))
+ .merge_into_max_segments(config.final_num_segments.unwrap_or(20))
.unwrap();
});
@@ -442,7 +442,7 @@ impl Indexer {
index = index.merge(other);
std::fs::remove_dir_all(other_path)?;
- index.inverted_index.merge_into_segments(num_segments)?;
+ index.inverted_index.merge_into_max_segments(num_segments)?;
}
Ok(())
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 20cbf5c0..d13138f0 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -271,8 +271,21 @@ impl InvertedIndex {
Ok(webpages)
}
- pub fn merge_into_segments(&mut self, num_segments: u32) -> Result<()> {
- assert!(num_segments > 0);
+ pub fn merge_into_max_segments(&mut self, max_num_segments: u32) -> Result<()> {
+ assert!(max_num_segments > 0);
+
+ let mut segments: Vec<_> = self
+ .tantivy_index
+ .load_metas()?
+ .segments
+ .into_iter()
+ .collect();
+
+ if segments.len() <= max_num_segments as usize {
+ return Ok(());
+ }
+
+ let num_segments = (max_num_segments + 1) / 2; // ceil(num_segments/2)
let mut merge_segments = Vec::new();
@@ -283,39 +296,29 @@ impl InvertedIndex {
});
}
- let mut segments: Vec<_> = self
- .tantivy_index
- .load_metas()?
- .segments
- .into_iter()
- .collect();
+ segments.sort_by_key(|b| std::cmp::Reverse(b.num_docs()));
- if segments.len() > num_segments as usize {
- segments.sort_by_key(|b| std::cmp::Reverse(b.num_docs()));
+ for segment in segments {
+ let best_candidate = merge_segments
+ .iter_mut()
+ .min_by(|a, b| a.num_docs.cmp(&b.num_docs))
+ .unwrap();
- for segment in segments {
- let best_candidate = merge_segments
- .iter_mut()
- .min_by(|a, b| a.num_docs.cmp(&b.num_docs))
- .unwrap();
+ best_candidate.num_docs += segment.num_docs();
+ best_candidate.segments.push(segment);
+ }
- best_candidate.num_docs += segment.num_docs();
- best_candidate.segments.push(segment);
- }
+ for merge in merge_segments
+ .into_iter()
+ .filter(|merge| !merge.segments.is_empty())
+ {
+ let segment_ids: Vec<_> = merge.segments.iter().map(|segment| segment.id()).collect();
+ self.writer.merge(&segment_ids[..]).wait()?;
- for merge in merge_segments
- .into_iter()
- .filter(|merge| !merge.segments.is_empty())
- {
- let segment_ids: Vec<_> =
- merge.segments.iter().map(|segment| segment.id()).collect();
- self.writer.merge(&segment_ids[..]).wait()?;
-
- let path = Path::new(&self.path);
- for segment in merge.segments {
- for file in segment.list_files() {
- std::fs::remove_file(path.join(file)).ok();
- }
+ let path = Path::new(&self.path);
+ for segment in merge.segments {
+ for file in segment.list_files() {
+ std::fs::remove_file(path.join(file)).ok();
}
}
}