web search done right
optimize trivial segment merges
Mikkel Denkera7b61559db0e
diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rsindex da177773..e9f75cad 100644--- a/crates/core/src/entrypoint/webgraph.rs+++ b/crates/core/src/entrypoint/webgraph.rs@@ -267,6 +267,8 @@ impl Webgraph {}if config.merge_all_segments {+ host_graph.optimize_read(); // save space in id2node db+ page_graph.optimize_read(); // save space in id2node dbhost_graph.merge_all_segments(Default::default())?;page_graph.merge_all_segments(Default::default())?;}diff --git a/crates/core/src/main.rs b/crates/core/src/main.rsindex 36c3d0d9..1341cab5 100644--- a/crates/core/src/main.rs+++ b/crates/core/src/main.rs@@ -343,6 +343,7 @@ fn main() -> Result<()> {}if merge_all_segments {+ webgraph.optimize_read(); // save space in id2node dbwebgraph.merge_all_segments(Default::default())?;}diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rsindex a3497c76..34d6c4a9 100644--- a/crates/core/src/webgraph/segment.rs+++ b/crates/core/src/webgraph/segment.rs@@ -118,6 +118,16 @@ impl Segment {folder: P,id: String,) -> Result<()> {+ if segments.is_empty() {+ return Ok(());+ }++ if segments.len() == 1 {+ let segment = &segments[0];+ std::fs::rename(segment.path(), folder.as_ref().join(&id))?;+ return Ok(());+ }+let old_paths = segments.iter().map(|s| s.path()).collect::<Vec<_>>();let (adjacency, reversed_adjacency) = segments