web search done right
Control the number of segments after merge. (#54)
Mikkel Denker65cf8f9f5356
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 9168d4c0..6a151e2e 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -144,7 +144,6 @@ impl Map<IndexingWorker, FrozenIndex> for Job {std::fs::remove_file(file).ok();}index.commit().unwrap();- index.inverted_index.merge_all_segments().unwrap();info!("{} done", name);@@ -218,10 +217,14 @@ impl Indexer {warc_paths = Box::new(warc_paths.take(limit));}- let _index: Index = warc_paths+ let mut index: Index = warc_paths.map_reduce(&workers).expect("failed to build index");+ index+ .inverted_index+ .merge_into_segments(config.final_num_segments.unwrap_or(20))?;+Ok(())}@@ -287,7 +290,9 @@ impl Indexer {);if let Some(index) = index.as_mut() {- index.inverted_index.merge_all_segments().unwrap();+ index+ .inverted_index+ .merge_into_segments(config.final_num_segments.unwrap_or(20))?;}Ok(())diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex 2564d39c..47f5bbf8 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -20,7 +20,7 @@ use tantivy::collector::{Collector, Count};use tantivy::merge_policy::NoMergePolicy;use tantivy::schema::Schema;use tantivy::tokenizer::TokenizerManager;-use tantivy::{DocAddress, Document, IndexReader, IndexWriter};+use tantivy::{DocAddress, Document, IndexReader, IndexWriter, SegmentMeta};use crate::image_store::Image;use crate::query::Query;@@ -34,6 +34,11 @@ use std::fs;use std::path::Path;use std::sync::Arc;+struct SegmentMergeCandidate {+ num_docs: u32,+ segments: Vec<SegmentMeta>,+}+pub struct InvertedIndex {pub path: String,tantivy_index: tantivy::Index,@@ -149,8 +154,19 @@ impl InvertedIndex {})}- pub fn merge_all_segments(&mut self) -> Result<()> {- let segments: Vec<_> = self+ pub fn merge_into_segments(&mut self, num_segments: u32) -> Result<()> {+ assert!(num_segments > 0);++ let mut merge_segments = Vec::new();++ for _ in 0..num_segments {+ merge_segments.push(SegmentMergeCandidate {+ num_docs: 0,+ segments: Vec::new(),+ });+ }++ let mut segments: Vec<_> = self.tantivy_index.load_metas()?.segments@@ -158,13 +174,31 @@ impl InvertedIndex {.collect();if segments.len() > 1 {- let segment_ids: Vec<_> = segments.iter().map(|segment| segment.id()).collect();- self.writer.merge(&segment_ids[..]).wait()?;+ segments.sort_by_key(|b| std::cmp::Reverse(b.num_docs()));- let path = Path::new(&self.path);for segment in segments {- for file in segment.list_files() {- std::fs::remove_file(path.join(file)).ok();+ let best_candidate = merge_segments+ .iter_mut()+ .min_by(|a, b| a.num_docs.cmp(&b.num_docs))+ .unwrap();++ best_candidate.num_docs += segment.num_docs();+ best_candidate.segments.push(segment);+ }++ for merge in merge_segments+ .into_iter()+ .filter(|merge| !merge.segments.is_empty())+ {+ let segment_ids: Vec<_> =+ merge.segments.iter().map(|segment| segment.id()).collect();+ self.writer.merge(&segment_ids[..]).wait()?;++ let path = Path::new(&self.path);+ for segment in merge.segments {+ for file in segment.list_files() {+ std::fs::remove_file(path.join(file)).ok();+ }}}}diff --git a/src/lib.rs b/src/lib.rsindex 58828ed8..93e8557c 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -57,6 +57,7 @@ mod webpage;#[derive(Debug, Deserialize, Clone)]pub struct IndexingMasterConfig {limit_warc_files: Option<usize>,+ final_num_segments: Option<u32>,warc_source: WarcSource,workers: Vec<String>,batch_size: Option<usize>,@@ -67,6 +68,7 @@ pub struct IndexingMasterConfig {#[derive(Debug, Deserialize, Clone)]pub struct IndexingLocalConfig {limit_warc_files: Option<usize>,+ final_num_segments: Option<u32>,warc_source: WarcSource,batch_size: Option<usize>,webgraph_path: Option<String>,