Skip to content

web search done right

avoid inf values in approximate centrality by pushing norm inside the sum so the values don't get too big

Mikkel Denker215696ebebe4
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
index 686490ef..6bcaf3c7 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
@@ -88,6 +88,10 @@ impl ApproxCentralitySetup {
}
}
+fn num_samples(num_nodes: u64, sample_rate: f64) -> u64 {
+ ((num_nodes as f64).log2() / sample_rate.powi(2)).ceil() as u64
+}
+
impl Setup for ApproxCentralitySetup {
type DhtTables = ApproxCentralityTables;
@@ -110,8 +114,7 @@ impl Setup for ApproxCentralitySetup {
fn setup_first_round(&self, dht: &Self::DhtTables) {
let upper_bound_num_nodes: u64 = self.workers.iter().map(|w| w.num_nodes()).sum();
let num_samples_per_worker =
- ((upper_bound_num_nodes as f64).log2() / self.sample_rate.powi(2)).ceil() as u64
- / (self.workers.len() as u64);
+ num_samples(upper_bound_num_nodes, self.sample_rate) / (self.workers.len() as u64);
dht.meta.set(
(),
@@ -209,12 +212,19 @@ pub fn run(config: ApproxHarmonicCoordinatorConfig) -> Result<()> {
.build()?
.block_on(setup_gossip(tokio_conf))?;
+ let upper_bound_num_nodes: u64 = cluster.workers.iter().map(|w| w.num_nodes()).sum();
+
+ let num_samples = num_samples(upper_bound_num_nodes, config.sample_rate);
+
+ let norm = 1.0 / ((num_samples - 1) as f64);
+
let jobs = cluster
.workers
.iter()
.map(|worker| ApproxCentralityJob {
shard: worker.shard(),
max_distance: config.max_distance,
+ norm,
all_workers: cluster
.workers
.clone()
@@ -227,17 +237,10 @@ pub fn run(config: ApproxHarmonicCoordinatorConfig) -> Result<()> {
let coordinator = build(&cluster.dht, cluster.workers.clone(), config.sample_rate);
let res = coordinator.run(jobs, ApproxCentralityFinish)?;
- let num_nodes = res.centrality.num_keys();
-
- let num_samples =
- res.meta.get(()).unwrap().num_samples_per_worker * cluster.workers.len() as u64;
-
let output_path = Path::new(&config.output_path);
- let norm = num_nodes as f64 / (num_samples as f64 * (num_nodes as f64 - 1.0));
-
let store = store_harmonic(
- res.centrality.iter().map(|(n, c)| (n, f64::from(c) * norm)),
+ res.centrality.iter().map(|(n, c)| (n, f64::from(c))),
output_path,
);
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
index 640ca48a..f8942264 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
@@ -185,7 +185,13 @@ impl Mapper for ApproxCentralityMapper {
for chunk in workers
.dijkstra(node, job.max_distance)
.into_iter()
- .map(|(n, d)| (n, KahanSum::from(1.0 / d as f64)))
+ .filter_map(|(n, d)| {
+ if d == 0 {
+ None
+ } else {
+ Some((n, KahanSum::from((1.0 / d as f64) * job.norm)))
+ }
+ })
.chunks(BATCH_SIZE)
.into_iter()
{
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs
index 8d5ffccd..bcacd3e9 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs
@@ -48,6 +48,7 @@ impl_dht_tables!(ApproxCentralityTables, [centrality, meta]);
pub struct ApproxCentralityJob {
shard: ShardId,
max_distance: u8,
+ norm: f64,
all_workers: Vec<(ShardId, SocketAddr)>,
}