Skip to content

web search done right

use estimate of number of nodes during approx harmonic calculation

Mikkel Denkerfa6c48ffa9f2
diff --git a/core/src/webgraph/centrality/approx_harmonic.rs b/core/src/webgraph/centrality/approx_harmonic.rs
index e545c2a1..9cf9bc2c 100644
--- a/core/src/webgraph/centrality/approx_harmonic.rs
+++ b/core/src/webgraph/centrality/approx_harmonic.rs
@@ -36,9 +36,9 @@ pub struct ApproxHarmonic {
impl ApproxHarmonic {
pub fn build<P: AsRef<Path>>(graph: &Webgraph, output: P) -> Self {
- let num_nodes = graph.par_nodes().count();
+ let num_nodes = graph.estimate_num_nodes();
- tracing::info!("found {} nodes in graph", num_nodes);
+ tracing::info!("found approximately {} nodes in graph", num_nodes);
let num_samples = ((num_nodes as f64).log2() / EPSILON.powi(2)).ceil() as usize;
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 55159572..e8f74724 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -533,6 +533,14 @@ impl Id2NodeDb {
})
}
+ fn estimate_num_keys(&self) -> usize {
+ self.db
+ .property_int_value("rocksdb.estimate-num-keys")
+ .ok()
+ .flatten()
+ .unwrap_or_default() as usize
+ }
+
fn iter(&self) -> impl Iterator<Item = (NodeID, Node)> + '_ {
self.db
.iterator(rocksdb::IteratorMode::Start)
@@ -875,6 +883,10 @@ impl Webgraph {
self.id2node.iter().map(|(id, node)| (node, id))
}
+ pub fn estimate_num_nodes(&self) -> usize {
+ self.id2node.estimate_num_keys()
+ }
+
/// Iterate all edges in the graph at least once.
/// Some edges may be returned multiple times.
/// This happens if they are present in more than one segment.