web search done right
[webgraph] insert batches into id2node database automatically if the batch size becomes too large
Mikkel Denker3f141f55cac8
diff --git a/core/src/main.rs b/core/src/main.rsindex 815f26e2..c9089a97 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -279,10 +279,12 @@ fn main() -> Result<()> {entrypoint::Webgraph::run(&config)?;}WebgraphOptions::Merge { mut paths } => {- let mut webgraph = WebgraphBuilder::new(paths.remove(0)).open();+ let mut webgraph = WebgraphBuilder::new(paths.remove(0))+ .single_threaded()+ .open();for other_path in paths {- let other = WebgraphBuilder::new(&other_path).open();+ let other = WebgraphBuilder::new(&other_path).single_threaded().open();webgraph.merge(other);std::fs::remove_dir_all(other_path).unwrap();}diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 519dfe36..55159572 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -548,15 +548,25 @@ impl Id2NodeDb {fn batch_put(&mut self, iter: impl Iterator<Item = (NodeID, Node)>) {let mut batch = rocksdb::WriteBatch::default();+ let mut count = 0;for (id, node) in iter {batch.put(id.as_u64().to_le_bytes(),bincode::serialize(&node).unwrap(),);+ count += 1;++ if count > 10_000 {+ self.db.write(batch).unwrap();+ batch = rocksdb::WriteBatch::default();+ count = 0;+ }}- self.db.write(batch).unwrap();+ if count > 0 {+ self.db.write(batch).unwrap();+ }}fn flush(&self) {