Skip to content

web search done right

no need to run workers independently anymore as they don't need to communicate

Mikkel Denkerbc3fa6974c02
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
index 6bcaf3c7..3a2009ec 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
@@ -143,15 +143,8 @@ pub fn build(
) -> Coordinator<ApproxCentralityJob> {
let setup = ApproxCentralitySetup::new_for_dht_members(dht, workers.clone(), sample_rate);
- let mut coord = Coordinator::new(setup, workers.clone());
-
- for worker in &workers {
- coord = coord.with_mapper(ApproxCentralityMapper::ApproximateCentrality {
- worker_shard: worker.shard(),
- });
- }
-
- coord
+ Coordinator::new(setup, workers.clone())
+ .with_mapper(ApproxCentralityMapper::ApproximateCentrality)
}
struct ClusterInfo {
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
index 0043e3ea..f7d852d8 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
@@ -21,8 +21,8 @@ use std::{cmp, collections::BTreeMap};
use super::{worker::ApproxCentralityWorker, ApproxCentralityJob, Mapper};
use crate::{
- ampc::dht::upsert, distributed::member::ShardId,
- entrypoint::ampc::approximated_harmonic_centrality::DhtTable, kahan_sum::KahanSum, webgraph,
+ ampc::dht::upsert, entrypoint::ampc::approximated_harmonic_centrality::DhtTable,
+ kahan_sum::KahanSum, webgraph,
};
use rayon::prelude::*;
@@ -30,7 +30,7 @@ const BATCH_SIZE: usize = 1024;
#[derive(Debug, Clone, bincode::Decode, bincode::Encode)]
pub enum ApproxCentralityMapper {
- ApproximateCentrality { worker_shard: ShardId },
+ ApproximateCentrality,
}
struct Workers {
@@ -131,11 +131,7 @@ impl Mapper for ApproxCentralityMapper {
dht: &crate::ampc::DhtConn<<<Self as Mapper>::Job as super::Job>::DhtTables>,
) {
match self {
- ApproxCentralityMapper::ApproximateCentrality { worker_shard } => {
- if worker.shard() != *worker_shard {
- return;
- }
-
+ ApproxCentralityMapper::ApproximateCentrality => {
let workers = Workers::new(worker.clone());
let num_samples = dht.next().meta.get(()).unwrap().num_samples_per_worker;