Skip to content

web search done right

optimize trivial segment merges

Mikkel Denkera7b61559db0e
diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rs
index da177773..e9f75cad 100644
--- a/crates/core/src/entrypoint/webgraph.rs
+++ b/crates/core/src/entrypoint/webgraph.rs
@@ -267,6 +267,8 @@ impl Webgraph {
}
if config.merge_all_segments {
+ host_graph.optimize_read(); // save space in id2node db
+ page_graph.optimize_read(); // save space in id2node db
host_graph.merge_all_segments(Default::default())?;
page_graph.merge_all_segments(Default::default())?;
}
diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs
index 36c3d0d9..1341cab5 100644
--- a/crates/core/src/main.rs
+++ b/crates/core/src/main.rs
@@ -343,6 +343,7 @@ fn main() -> Result<()> {
}
if merge_all_segments {
+ webgraph.optimize_read(); // save space in id2node db
webgraph.merge_all_segments(Default::default())?;
}
diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rs
index a3497c76..34d6c4a9 100644
--- a/crates/core/src/webgraph/segment.rs
+++ b/crates/core/src/webgraph/segment.rs
@@ -118,6 +118,16 @@ impl Segment {
folder: P,
id: String,
) -> Result<()> {
+ if segments.is_empty() {
+ return Ok(());
+ }
+
+ if segments.len() == 1 {
+ let segment = &segments[0];
+ std::fs::rename(segment.path(), folder.as_ref().join(&id))?;
+ return Ok(());
+ }
+
let old_paths = segments.iter().map(|s| s.path()).collect::<Vec<_>>();
let (adjacency, reversed_adjacency) = segments