web search done right
reduce communication between workers in approx centrality
Mikkel Denkerd7aee00f72e0
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rsindex f8942264..0043e3ea 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs@@ -21,13 +21,8 @@ use std::{cmp, collections::BTreeMap};use super::{worker::ApproxCentralityWorker, ApproxCentralityJob, Mapper};use crate::{- ampc::dht::upsert,- distributed::member::ShardId,- entrypoint::ampc::approximated_harmonic_centrality::{- worker::RemoteApproxCentralityWorker, DhtTable,- },- kahan_sum::KahanSum,- webgraph,+ ampc::dht::upsert, distributed::member::ShardId,+ entrypoint::ampc::approximated_harmonic_centrality::DhtTable, kahan_sum::KahanSum, webgraph,};use rayon::prelude::*;@@ -40,18 +35,11 @@ pub enum ApproxCentralityMapper {struct Workers {worker: ApproxCentralityWorker,- remote_workers: Vec<RemoteApproxCentralityWorker>,}impl Workers {- fn new(- worker: ApproxCentralityWorker,- remote_workers: Vec<RemoteApproxCentralityWorker>,- ) -> Self {- Self {- worker,- remote_workers,- }+ fn new(worker: ApproxCentralityWorker) -> Self {+ Self { worker }}fn outgoing_nodes(@@ -69,19 +57,6 @@ impl Workers {}}- // this does not retrieve outgoing edges from remote workers asynchronously,- // but instead waits for each worker to respond before moving on to the next one.- // this is not ideal, but it will only be a problem when we get way more workers- // in which case we should probably use an approximation that simply runs the- // normal harmonic centrality algorithm a fixed number of times.- for remote_worker in &self.remote_workers {- let nodes = remote_worker.outgoing_edge_nodes(nodes.to_vec(), limit);-- for (node, outgoing) in nodes {- res.entry(node).or_default().extend(outgoing);- }- }-res}@@ -161,20 +136,7 @@ impl Mapper for ApproxCentralityMapper {return;}- let remote_workers: Vec<_> = job- .all_workers- .iter()- .cloned()- .filter_map(|(shard, addr)| {- if shard == worker.shard() {- None- } else {- Some(RemoteApproxCentralityWorker::new(shard, addr).unwrap())- }- })- .collect();-- let workers = Workers::new(worker.clone(), remote_workers);+ let workers = Workers::new(worker.clone());let num_samples = dht.next().meta.get(()).unwrap().num_samples_per_worker;let sampled = workerdiff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rsindex 2b08d37e..ee67f684 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs@@ -14,8 +14,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>-use rustc_hash::{FxHashMap, FxHashSet};-use crate::{ampc::JobConn,config::ApproxHarmonicWorkerConfig,@@ -64,29 +62,6 @@ impl Message<ApproxCentralityWorker> for NumNodes {}}-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]-pub struct OutgoingEdgeNodes {- pub from: Vec<webgraph::NodeID>,- pub limit: webgraph::EdgeLimit,-}--impl Message<ApproxCentralityWorker> for OutgoingEdgeNodes {- type Response = FxHashMap<webgraph::NodeID, FxHashSet<webgraph::NodeID>>;-- fn handle(self, worker: &ApproxCentralityWorker) -> Self::Response {- let mut res: FxHashMap<webgraph::NodeID, FxHashSet<webgraph::NodeID>> =- FxHashMap::default();-- for node in &self.from {- for edge in worker.graph().raw_outgoing_edges(node, self.limit) {- res.entry(*node).or_default().insert(edge.to);- }- }-- res- }-}-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]pub struct BatchId2Node(Vec<webgraph::NodeID>);@@ -101,7 +76,7 @@ impl Message<ApproxCentralityWorker> for BatchId2Node {}}-impl_worker!(ApproxCentralityJob, RemoteApproxCentralityWorker => ApproxCentralityWorker, [NumNodes, OutgoingEdgeNodes, BatchId2Node]);+impl_worker!(ApproxCentralityJob, RemoteApproxCentralityWorker => ApproxCentralityWorker, [NumNodes, BatchId2Node]);#[derive(Clone)]pub struct RemoteApproxCentralityWorker {@@ -129,14 +104,6 @@ impl RemoteApproxCentralityWorker {self.send(NumNodes)}- pub fn outgoing_edge_nodes(- &self,- from: Vec<webgraph::NodeID>,- limit: webgraph::EdgeLimit,- ) -> FxHashMap<webgraph::NodeID, FxHashSet<webgraph::NodeID>> {- self.send(OutgoingEdgeNodes { from, limit })- }-pub fn batch_id2node(&self,ids: Vec<webgraph::NodeID>,