web search done right
avoid inf values in approximate centrality by pushing norm inside the sum so the values don't get too big
Mikkel Denker215696ebebe4
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rsindex 686490ef..6bcaf3c7 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs@@ -88,6 +88,10 @@ impl ApproxCentralitySetup {}}+fn num_samples(num_nodes: u64, sample_rate: f64) -> u64 {+ ((num_nodes as f64).log2() / sample_rate.powi(2)).ceil() as u64+}+impl Setup for ApproxCentralitySetup {type DhtTables = ApproxCentralityTables;@@ -110,8 +114,7 @@ impl Setup for ApproxCentralitySetup {fn setup_first_round(&self, dht: &Self::DhtTables) {let upper_bound_num_nodes: u64 = self.workers.iter().map(|w| w.num_nodes()).sum();let num_samples_per_worker =- ((upper_bound_num_nodes as f64).log2() / self.sample_rate.powi(2)).ceil() as u64- / (self.workers.len() as u64);+ num_samples(upper_bound_num_nodes, self.sample_rate) / (self.workers.len() as u64);dht.meta.set((),@@ -209,12 +212,19 @@ pub fn run(config: ApproxHarmonicCoordinatorConfig) -> Result<()> {.build()?.block_on(setup_gossip(tokio_conf))?;+ let upper_bound_num_nodes: u64 = cluster.workers.iter().map(|w| w.num_nodes()).sum();++ let num_samples = num_samples(upper_bound_num_nodes, config.sample_rate);++ let norm = 1.0 / ((num_samples - 1) as f64);+let jobs = cluster.workers.iter().map(|worker| ApproxCentralityJob {shard: worker.shard(),max_distance: config.max_distance,+ norm,all_workers: cluster.workers.clone()@@ -227,17 +237,10 @@ pub fn run(config: ApproxHarmonicCoordinatorConfig) -> Result<()> {let coordinator = build(&cluster.dht, cluster.workers.clone(), config.sample_rate);let res = coordinator.run(jobs, ApproxCentralityFinish)?;- let num_nodes = res.centrality.num_keys();-- let num_samples =- res.meta.get(()).unwrap().num_samples_per_worker * cluster.workers.len() as u64;-let output_path = Path::new(&config.output_path);- let norm = num_nodes as f64 / (num_samples as f64 * (num_nodes as f64 - 1.0));-let store = store_harmonic(- res.centrality.iter().map(|(n, c)| (n, f64::from(c) * norm)),+ res.centrality.iter().map(|(n, c)| (n, f64::from(c))),output_path,);diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rsindex 640ca48a..f8942264 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs@@ -185,7 +185,13 @@ impl Mapper for ApproxCentralityMapper {for chunk in workers.dijkstra(node, job.max_distance).into_iter()- .map(|(n, d)| (n, KahanSum::from(1.0 / d as f64)))+ .filter_map(|(n, d)| {+ if d == 0 {+ None+ } else {+ Some((n, KahanSum::from((1.0 / d as f64) * job.norm)))+ }+ }).chunks(BATCH_SIZE).into_iter(){diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rsindex 8d5ffccd..bcacd3e9 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs@@ -48,6 +48,7 @@ impl_dht_tables!(ApproxCentralityTables, [centrality, meta]);pub struct ApproxCentralityJob {shard: ShardId,max_distance: u8,+ norm: f64,all_workers: Vec<(ShardId, SocketAddr)>,}