web search done right
merge id2node_db faster than batchputs
Mikkel Denker9ba4fb453184
diff --git a/crates/core/src/webgraph/id_node_db.rs b/crates/core/src/webgraph/id_node_db.rsindex 865f9742..712cf11d 100644--- a/crates/core/src/webgraph/id_node_db.rs+++ b/crates/core/src/webgraph/id_node_db.rs@@ -50,18 +50,8 @@ impl Id2NodeDb {self.db.iter()}- pub fn batch_put(&mut self, iter: impl Iterator<Item = (NodeID, Node)>) {- for (id, node) in iter {- self.put(&id, &node);-- if self.db.uncommitted_inserts() > 10_000_000 {- self.flush();- }- }-- if self.db.uncommitted_inserts() > 10_000_000 {- self.flush();- }+ pub fn merge(&mut self, other: Self) {+ self.db.merge(other.db).unwrap();}pub fn flush(&mut self) {diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 6ac45d2f..c3b98137 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -136,7 +136,8 @@ impl Webgraph {pub fn merge(&mut self, other: Webgraph) -> io::Result<()> {let other_folder = other.path.clone();- self.id2node.batch_put(other.id2node.iter());+ self.id2node.merge(other.id2node);+ self.id2node.flush();for segment in other.segments {let id = segment.id();@@ -152,7 +153,6 @@ impl Webgraph {fs::remove_dir_all(other_folder)?;self.save_metadata();- self.id2node.flush();Ok(())}