web search done right
Merge index segments into num_segments/2 to only merge segments once every num_segments/2 index merges
Mikkel Denkera4c142f5f390
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 0f866c3d..697ac77c 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -230,7 +230,7 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {index.inverted_index- .merge_into_segments(job.max_num_segments)+ .merge_into_max_segments(job.max_num_segments).unwrap();info!("{} done", name);@@ -351,7 +351,7 @@ impl Indexer {index.inverted_index- .merge_into_segments(config.final_num_segments.unwrap_or(20))+ .merge_into_max_segments(config.final_num_segments.unwrap_or(20)).unwrap();});@@ -442,7 +442,7 @@ impl Indexer {index = index.merge(other);std::fs::remove_dir_all(other_path)?;- index.inverted_index.merge_into_segments(num_segments)?;+ index.inverted_index.merge_into_max_segments(num_segments)?;}Ok(())diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex 20cbf5c0..d13138f0 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -271,8 +271,21 @@ impl InvertedIndex {Ok(webpages)}- pub fn merge_into_segments(&mut self, num_segments: u32) -> Result<()> {- assert!(num_segments > 0);+ pub fn merge_into_max_segments(&mut self, max_num_segments: u32) -> Result<()> {+ assert!(max_num_segments > 0);++ let mut segments: Vec<_> = self+ .tantivy_index+ .load_metas()?+ .segments+ .into_iter()+ .collect();++ if segments.len() <= max_num_segments as usize {+ return Ok(());+ }++ let num_segments = (max_num_segments + 1) / 2; // ceil(num_segments/2)let mut merge_segments = Vec::new();@@ -283,39 +296,29 @@ impl InvertedIndex {});}- let mut segments: Vec<_> = self- .tantivy_index- .load_metas()?- .segments- .into_iter()- .collect();+ segments.sort_by_key(|b| std::cmp::Reverse(b.num_docs()));- if segments.len() > num_segments as usize {- segments.sort_by_key(|b| std::cmp::Reverse(b.num_docs()));+ for segment in segments {+ let best_candidate = merge_segments+ .iter_mut()+ .min_by(|a, b| a.num_docs.cmp(&b.num_docs))+ .unwrap();- for segment in segments {- let best_candidate = merge_segments- .iter_mut()- .min_by(|a, b| a.num_docs.cmp(&b.num_docs))- .unwrap();+ best_candidate.num_docs += segment.num_docs();+ best_candidate.segments.push(segment);+ }- best_candidate.num_docs += segment.num_docs();- best_candidate.segments.push(segment);- }+ for merge in merge_segments+ .into_iter()+ .filter(|merge| !merge.segments.is_empty())+ {+ let segment_ids: Vec<_> = merge.segments.iter().map(|segment| segment.id()).collect();+ self.writer.merge(&segment_ids[..]).wait()?;- for merge in merge_segments- .into_iter()- .filter(|merge| !merge.segments.is_empty())- {- let segment_ids: Vec<_> =- merge.segments.iter().map(|segment| segment.id()).collect();- self.writer.merge(&segment_ids[..]).wait()?;-- let path = Path::new(&self.path);- for segment in merge.segments {- for file in segment.list_files() {- std::fs::remove_file(path.join(file)).ok();- }+ let path = Path::new(&self.path);+ for segment in merge.segments {+ for file in segment.list_files() {+ std::fs::remove_file(path.join(file)).ok();}}}