Skip to content

web search done right

let number of webgraph segments depend on number of cores

Mikkel Denker913c5502c3a7
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index ac5b04da..c04c18d2 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -33,8 +33,6 @@ pub mod centrality;
mod store;
use self::segment::{LiveSegment, SegmentNodeID, StoredSegment};
-const MAX_NUM_SEGMENTS: usize = 400;
-
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct NodeID(pub u64);
@@ -430,8 +428,8 @@ impl Webgraph {
save_bin(&self.node2id, Path::new(&self.path).join("node2id.bin"));
save_bin(&self.id2node, Path::new(&self.path).join("id2node.bin"));
- if self.segments.len() > MAX_NUM_SEGMENTS {
- self.merge_segments((MAX_NUM_SEGMENTS / 2).max(1));
+ if self.segments.len() > 2 * num_cpus::get() {
+ self.merge_segments(num_cpus::get());
}
}