Skip to content

web search done right

make sure webgraph merge doesn not exceed maximum number of allowed documents in tantivy

Mikkel Denker13a06b3820cf
diff --git a/crates/core/src/inverted_index/indexing.rs b/crates/core/src/inverted_index/indexing.rs
index 9c3c9f0f..890d2cdb 100644
--- a/crates/core/src/inverted_index/indexing.rs
+++ b/crates/core/src/inverted_index/indexing.rs
@@ -347,4 +347,62 @@ mod test {
search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.documents.len(), 0);
}
+
+ #[test]
+ fn test_merge_into_max_segments() {
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
+
+ index
+ .insert(
+ &Webpage::test_parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ TEST
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.example.com",
+ )
+ .unwrap(),
+ )
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+
+ index
+ .insert(
+ &Webpage::test_parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ TEST
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.example.com",
+ )
+ .unwrap(),
+ )
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+
+ let segments = index.segment_ids();
+
+ assert_eq!(segments.len(), 2);
+
+ index.merge_into_max_segments(1).unwrap();
+
+ let segments = index.segment_ids();
+ assert_eq!(segments.len(), 1);
+ }
}
diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs
index 006c9c38..11ccdda0 100644
--- a/crates/core/src/main.rs
+++ b/crates/core/src/main.rs
@@ -241,9 +241,6 @@ enum WebgraphOptions {
Merge {
#[clap(required = true)]
paths: Vec<String>,
-
- #[clap(default_value_t = stract::config::defaults::Webgraph::merge_all_segments())]
- merge_all_segments: bool,
},
/// Deploy the webgraph server. The webgraph server is responsible for serving the webgraph to the search servers.
@@ -348,10 +345,7 @@ fn main() -> Result<()> {
let config = load_toml_config(config_path);
entrypoint::Webgraph::run(&config)?;
}
- WebgraphOptions::Merge {
- mut paths,
- merge_all_segments,
- } => {
+ WebgraphOptions::Merge { mut paths } => {
let mut webgraph = WebgraphBuilder::new(paths.remove(0), 0u64.into()).open()?;
for other_path in paths {
@@ -359,9 +353,7 @@ fn main() -> Result<()> {
webgraph.merge(other)?;
}
- if merge_all_segments {
- webgraph.optimize_read()?;
- }
+ webgraph.optimize_read()?;
}
WebgraphOptions::Server { config_path } => {
let config: config::WebgraphServerConfig = load_toml_config(config_path);
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index 89cc7d16..90773fe6 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -91,16 +91,43 @@ impl EdgeStore {
pub fn optimize_read(&mut self) -> Result<()> {
self.prepare_writer()?;
let base_path = Path::new(&self.path);
- let segments: Vec<_> = self.index.load_metas()?.segments.into_iter().collect();
-
- tantivy::merge_segments(
- self.writer
- .as_mut()
- .expect("writer should have been prepared"),
- segments,
- base_path,
- 1,
- )?;
+ let mut segments: Vec<_> = self.index.load_metas()?.segments.into_iter().collect();
+ segments.sort_by_key(|a| a.max_doc());
+
+ let mut num_docs: tantivy::DocId = 0;
+ let mut segments_to_merge = Vec::new();
+
+ for segment in segments {
+ if num_docs.saturating_add(segment.max_doc()) > tantivy::TERMINATED {
+ let segments_to_merge = std::mem::take(&mut segments_to_merge);
+ num_docs = 0;
+
+ if segments_to_merge.len() > 1 {
+ tantivy::merge_segments(
+ self.writer
+ .as_mut()
+ .expect("writer should have been prepared"),
+ segments_to_merge,
+ base_path,
+ 1,
+ )?;
+ }
+ } else {
+ num_docs += segment.max_doc();
+ segments_to_merge.push(segment);
+ }
+ }
+
+ if !segments_to_merge.is_empty() {
+ tantivy::merge_segments(
+ self.writer
+ .as_mut()
+ .expect("writer should have been prepared"),
+ segments_to_merge,
+ base_path,
+ 1,
+ )?;
+ }
Ok(())
}
@@ -431,4 +458,51 @@ mod tests {
assert_eq!(edges[1].from, e3.from.id());
assert_eq!(edges[2].from, e1.from.id());
}
+
+ #[test]
+ fn test_optimize_read() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut store = EdgeStore::open(&temp_dir, ShardId::new(0)).unwrap();
+
+ store
+ .insert(Edge {
+ from: Node::from("https://www.first.com").into_host(),
+ to: Node::from("https://www.second.com").into_host(),
+ label: String::new(),
+ rel_flags: RelFlags::default(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ store.commit().unwrap();
+
+ store
+ .insert(Edge {
+ from: Node::from("https://www.second.com").into_host(),
+ to: Node::from("https://www.first.com").into_host(),
+ label: String::new(),
+ rel_flags: RelFlags::default(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ store.commit().unwrap();
+
+ store
+ .insert(Edge {
+ from: Node::from("https://www.third.com").into_host(),
+ to: Node::from("https://www.first.com").into_host(),
+ label: String::new(),
+ rel_flags: RelFlags::default(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ store.commit().unwrap();
+
+ let segments = store.index.load_metas().unwrap().segments;
+ assert_eq!(segments.len(), 3);
+
+ store.optimize_read().unwrap();
+
+ let segments = store.index.load_metas().unwrap().segments;
+ assert_eq!(segments.len(), 1);
+ }
}
diff --git a/crates/tantivy/src/index/mod.rs b/crates/tantivy/src/index/mod.rs
index 309cc782..9f2f113e 100644
--- a/crates/tantivy/src/index/mod.rs
+++ b/crates/tantivy/src/index/mod.rs
@@ -59,7 +59,10 @@ pub fn merge_segments<P: AsRef<Path>, D: crate::Document>(
.min_by(|a, b| a.num_docs.cmp(&b.num_docs))
.unwrap();
- best_candidate.num_docs += segment.num_docs();
+ best_candidate.num_docs = best_candidate
+ .num_docs
+ .checked_add(segment.num_docs())
+ .expect("num docs must always be within u32::MAX");
best_candidate.segments.push(segment);
}