web search done right
make sure webgraph merge doesn not exceed maximum number of allowed documents in tantivy
Mikkel Denker13a06b3820cf
diff --git a/crates/core/src/inverted_index/indexing.rs b/crates/core/src/inverted_index/indexing.rsindex 9c3c9f0f..890d2cdb 100644--- a/crates/core/src/inverted_index/indexing.rs+++ b/crates/core/src/inverted_index/indexing.rs@@ -347,4 +347,62 @@ mod test {search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");assert_eq!(result.documents.len(), 0);}++ #[test]+ fn test_merge_into_max_segments() {+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");++ index+ .insert(+ &Webpage::test_parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ TEST+ </body>+ </html>+ "#+ ),+ "https://www.example.com",+ )+ .unwrap(),+ )+ .expect("failed to insert webpage");+ index.commit().expect("failed to commit index");++ index+ .insert(+ &Webpage::test_parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ TEST+ </body>+ </html>+ "#+ ),+ "https://www.example.com",+ )+ .unwrap(),+ )+ .expect("failed to insert webpage");+ index.commit().expect("failed to commit index");++ let segments = index.segment_ids();++ assert_eq!(segments.len(), 2);++ index.merge_into_max_segments(1).unwrap();++ let segments = index.segment_ids();+ assert_eq!(segments.len(), 1);+ }}diff --git a/crates/core/src/main.rs b/crates/core/src/main.rsindex 006c9c38..11ccdda0 100644--- a/crates/core/src/main.rs+++ b/crates/core/src/main.rs@@ -241,9 +241,6 @@ enum WebgraphOptions {Merge {#[clap(required = true)]paths: Vec<String>,-- #[clap(default_value_t = stract::config::defaults::Webgraph::merge_all_segments())]- merge_all_segments: bool,},/// Deploy the webgraph server. The webgraph server is responsible for serving the webgraph to the search servers.@@ -348,10 +345,7 @@ fn main() -> Result<()> {let config = load_toml_config(config_path);entrypoint::Webgraph::run(&config)?;}- WebgraphOptions::Merge {- mut paths,- merge_all_segments,- } => {+ WebgraphOptions::Merge { mut paths } => {let mut webgraph = WebgraphBuilder::new(paths.remove(0), 0u64.into()).open()?;for other_path in paths {@@ -359,9 +353,7 @@ fn main() -> Result<()> {webgraph.merge(other)?;}- if merge_all_segments {- webgraph.optimize_read()?;- }+ webgraph.optimize_read()?;}WebgraphOptions::Server { config_path } => {let config: config::WebgraphServerConfig = load_toml_config(config_path);diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex 89cc7d16..90773fe6 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -91,16 +91,43 @@ impl EdgeStore {pub fn optimize_read(&mut self) -> Result<()> {self.prepare_writer()?;let base_path = Path::new(&self.path);- let segments: Vec<_> = self.index.load_metas()?.segments.into_iter().collect();-- tantivy::merge_segments(- self.writer- .as_mut()- .expect("writer should have been prepared"),- segments,- base_path,- 1,- )?;+ let mut segments: Vec<_> = self.index.load_metas()?.segments.into_iter().collect();+ segments.sort_by_key(|a| a.max_doc());++ let mut num_docs: tantivy::DocId = 0;+ let mut segments_to_merge = Vec::new();++ for segment in segments {+ if num_docs.saturating_add(segment.max_doc()) > tantivy::TERMINATED {+ let segments_to_merge = std::mem::take(&mut segments_to_merge);+ num_docs = 0;++ if segments_to_merge.len() > 1 {+ tantivy::merge_segments(+ self.writer+ .as_mut()+ .expect("writer should have been prepared"),+ segments_to_merge,+ base_path,+ 1,+ )?;+ }+ } else {+ num_docs += segment.max_doc();+ segments_to_merge.push(segment);+ }+ }++ if !segments_to_merge.is_empty() {+ tantivy::merge_segments(+ self.writer+ .as_mut()+ .expect("writer should have been prepared"),+ segments_to_merge,+ base_path,+ 1,+ )?;+ }Ok(())}@@ -431,4 +458,51 @@ mod tests {assert_eq!(edges[1].from, e3.from.id());assert_eq!(edges[2].from, e1.from.id());}++ #[test]+ fn test_optimize_read() {+ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut store = EdgeStore::open(&temp_dir, ShardId::new(0)).unwrap();++ store+ .insert(Edge {+ from: Node::from("https://www.first.com").into_host(),+ to: Node::from("https://www.second.com").into_host(),+ label: String::new(),+ rel_flags: RelFlags::default(),+ sort_score: 0.0,+ })+ .unwrap();+ store.commit().unwrap();++ store+ .insert(Edge {+ from: Node::from("https://www.second.com").into_host(),+ to: Node::from("https://www.first.com").into_host(),+ label: String::new(),+ rel_flags: RelFlags::default(),+ sort_score: 0.0,+ })+ .unwrap();+ store.commit().unwrap();++ store+ .insert(Edge {+ from: Node::from("https://www.third.com").into_host(),+ to: Node::from("https://www.first.com").into_host(),+ label: String::new(),+ rel_flags: RelFlags::default(),+ sort_score: 0.0,+ })+ .unwrap();+ store.commit().unwrap();++ let segments = store.index.load_metas().unwrap().segments;+ assert_eq!(segments.len(), 3);++ store.optimize_read().unwrap();++ let segments = store.index.load_metas().unwrap().segments;+ assert_eq!(segments.len(), 1);+ }}diff --git a/crates/tantivy/src/index/mod.rs b/crates/tantivy/src/index/mod.rsindex 309cc782..9f2f113e 100644--- a/crates/tantivy/src/index/mod.rs+++ b/crates/tantivy/src/index/mod.rs@@ -59,7 +59,10 @@ pub fn merge_segments<P: AsRef<Path>, D: crate::Document>(.min_by(|a, b| a.num_docs.cmp(&b.num_docs)).unwrap();- best_candidate.num_docs += segment.num_docs();+ best_candidate.num_docs = best_candidate+ .num_docs+ .checked_add(segment.num_docs())+ .expect("num docs must always be within u32::MAX");best_candidate.segments.push(segment);}