web search done right
use estimate of number of nodes during approx harmonic calculation
Mikkel Denkerfa6c48ffa9f2
diff --git a/core/src/webgraph/centrality/approx_harmonic.rs b/core/src/webgraph/centrality/approx_harmonic.rsindex e545c2a1..9cf9bc2c 100644--- a/core/src/webgraph/centrality/approx_harmonic.rs+++ b/core/src/webgraph/centrality/approx_harmonic.rs@@ -36,9 +36,9 @@ pub struct ApproxHarmonic {impl ApproxHarmonic {pub fn build<P: AsRef<Path>>(graph: &Webgraph, output: P) -> Self {- let num_nodes = graph.par_nodes().count();+ let num_nodes = graph.estimate_num_nodes();- tracing::info!("found {} nodes in graph", num_nodes);+ tracing::info!("found approximately {} nodes in graph", num_nodes);let num_samples = ((num_nodes as f64).log2() / EPSILON.powi(2)).ceil() as usize;diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 55159572..e8f74724 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -533,6 +533,14 @@ impl Id2NodeDb {})}+ fn estimate_num_keys(&self) -> usize {+ self.db+ .property_int_value("rocksdb.estimate-num-keys")+ .ok()+ .flatten()+ .unwrap_or_default() as usize+ }+fn iter(&self) -> impl Iterator<Item = (NodeID, Node)> + '_ {self.db.iterator(rocksdb::IteratorMode::Start)@@ -875,6 +883,10 @@ impl Webgraph {self.id2node.iter().map(|(id, node)| (node, id))}+ pub fn estimate_num_nodes(&self) -> usize {+ self.id2node.estimate_num_keys()+ }+/// Iterate all edges in the graph at least once./// Some edges may be returned multiple times./// This happens if they are present in more than one segment.