Skip to content

web search done right

[webgraph] insert batches into id2node database automatically if the batch size becomes too large

Mikkel Denker3f141f55cac8
diff --git a/core/src/main.rs b/core/src/main.rs
index 815f26e2..c9089a97 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -279,10 +279,12 @@ fn main() -> Result<()> {
entrypoint::Webgraph::run(&config)?;
}
WebgraphOptions::Merge { mut paths } => {
- let mut webgraph = WebgraphBuilder::new(paths.remove(0)).open();
+ let mut webgraph = WebgraphBuilder::new(paths.remove(0))
+ .single_threaded()
+ .open();
for other_path in paths {
- let other = WebgraphBuilder::new(&other_path).open();
+ let other = WebgraphBuilder::new(&other_path).single_threaded().open();
webgraph.merge(other);
std::fs::remove_dir_all(other_path).unwrap();
}
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 519dfe36..55159572 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -548,15 +548,25 @@ impl Id2NodeDb {
fn batch_put(&mut self, iter: impl Iterator<Item = (NodeID, Node)>) {
let mut batch = rocksdb::WriteBatch::default();
+ let mut count = 0;
for (id, node) in iter {
batch.put(
id.as_u64().to_le_bytes(),
bincode::serialize(&node).unwrap(),
);
+ count += 1;
+
+ if count > 10_000 {
+ self.db.write(batch).unwrap();
+ batch = rocksdb::WriteBatch::default();
+ count = 0;
+ }
}
- self.db.write(batch).unwrap();
+ if count > 0 {
+ self.db.write(batch).unwrap();
+ }
}
fn flush(&self) {