Skip to content

web search done right

merge id2node_db faster than batchputs

Mikkel Denker9ba4fb453184
diff --git a/crates/core/src/webgraph/id_node_db.rs b/crates/core/src/webgraph/id_node_db.rs
index 865f9742..712cf11d 100644
--- a/crates/core/src/webgraph/id_node_db.rs
+++ b/crates/core/src/webgraph/id_node_db.rs
@@ -50,18 +50,8 @@ impl Id2NodeDb {
self.db.iter()
}
- pub fn batch_put(&mut self, iter: impl Iterator<Item = (NodeID, Node)>) {
- for (id, node) in iter {
- self.put(&id, &node);
-
- if self.db.uncommitted_inserts() > 10_000_000 {
- self.flush();
- }
- }
-
- if self.db.uncommitted_inserts() > 10_000_000 {
- self.flush();
- }
+ pub fn merge(&mut self, other: Self) {
+ self.db.merge(other.db).unwrap();
}
pub fn flush(&mut self) {
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index 6ac45d2f..c3b98137 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -136,7 +136,8 @@ impl Webgraph {
pub fn merge(&mut self, other: Webgraph) -> io::Result<()> {
let other_folder = other.path.clone();
- self.id2node.batch_put(other.id2node.iter());
+ self.id2node.merge(other.id2node);
+ self.id2node.flush();
for segment in other.segments {
let id = segment.id();
@@ -152,7 +153,6 @@ impl Webgraph {
fs::remove_dir_all(other_folder)?;
self.save_metadata();
- self.id2node.flush();
Ok(())
}