web search done right
use shortest paths as subroutine in approx centrality
Mikkel Denker852a10c08e78
diff --git a/crates/core/src/ampc/dht/value.rs b/crates/core/src/ampc/dht/value.rsindex 6fbcf3e6..417b0ff5 100644--- a/crates/core/src/ampc/dht/value.rs+++ b/crates/core/src/ampc/dht/value.rs@@ -22,9 +22,6 @@ pub trait ValueTrait: TryFrom<Value> + Into<Value> {}impl ValueTrait for String {}-type ApproxHarmonicMeta = crate::entrypoint::ampc::approximated_harmonic_centrality::Meta;-impl ValueTrait for ApproxHarmonicMeta {}-type F32 = f32;impl ValueTrait for F32 {}@@ -66,7 +63,6 @@ impl ValueTrait for Unit {})]pub enum Value {String(String),- ApproxHarmonicMeta(ApproxHarmonicMeta),F32(F32),F64(F64),U64(U64),@@ -105,7 +101,6 @@ macro_rules! impl_from_to_value {}impl_from_to_value!(String, String);-impl_from_to_value!(ApproxHarmonicMeta, ApproxHarmonicMeta);impl_from_to_value!(F32, F32);impl_from_to_value!(F64, F64);impl_from_to_value!(U64, U64);diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rsindex cc5b4e64..d4fe4314 100644--- a/crates/core/src/config/defaults.rs+++ b/crates/core/src/config/defaults.rs@@ -185,10 +185,6 @@ impl ApproxHarmonic {pub fn max_distance() -> u8 {7}-- pub fn save_centralities_with_zero() -> bool {- false- }}pub struct Webgraph;diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 4c5552da..ec8a2098 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -716,17 +716,6 @@ pub struct ApproxHarmonicCoordinatorConfig {#[serde(default = "defaults::ApproxHarmonic::max_distance")]pub max_distance: u8,-- #[serde(default = "defaults::ApproxHarmonic::save_centralities_with_zero")]- pub save_centralities_with_zero: bool,-}--#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]-pub struct ApproxHarmonicWorkerConfig {- pub gossip: GossipConfig,- pub shard: ShardId,- pub graph_path: String,- pub host: SocketAddr,}#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rsindex 38513301..2d6f7538 100644--- a/crates/core/src/distributed/member.rs+++ b/crates/core/src/distributed/member.rs@@ -125,23 +125,14 @@ pub enum Service {host: SocketAddr,shard: ShardId,},- HarmonicCoordinator {- host: SocketAddr,- },ApproxHarmonicWorker {host: SocketAddr,shard: ShardId,},- ApproxHarmonicCoordinator {- host: SocketAddr,- },ShortestPathWorker {host: SocketAddr,shard: ShardId,},- ShortestPathCoordinator {- host: SocketAddr,- },}impl std::fmt::Display for Service {@@ -163,19 +154,12 @@ impl std::fmt::Display for Service {}Self::Dht { host, shard } => write!(f, "Dht {} {}", host, shard),Self::HarmonicWorker { host, shard } => write!(f, "HarmonicWorker {} {}", host, shard),- Self::HarmonicCoordinator { host } => write!(f, "HarmonicCoordinator {}", host),Self::ApproxHarmonicWorker { host, shard } => {write!(f, "ApproxHarmonicWorker {} {}", host, shard)}- Self::ApproxHarmonicCoordinator { host } => {- write!(f, "ApproxHarmonicCoordinator {}", host)- }Self::ShortestPathWorker { host, shard } => {write!(f, "ShortestPathWorker {} {}", host, shard)}- Self::ShortestPathCoordinator { host } => {- write!(f, "ShortestPathCoordinator {}", host)- }}}}diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rsindex 99a5ea3f..b3f32d2d 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs@@ -14,155 +14,32 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>-use std::collections::BTreeMap;use std::net::SocketAddr;use std::path::Path;-use super::mapper::ApproxCentralityMapper;-use super::{worker::RemoteApproxCentralityWorker, ApproxCentralityTables};-use super::{ApproxCentralityJob, DhtTable as _, Finisher, Meta, Setup};-use crate::ampc::{Coordinator, DefaultDhtTable, DhtConn};+use rustc_hash::FxHashMap;++use crate::ampc::dht::ShardId;+use crate::ampc::{DhtTable as _, DhtTables as _};use crate::config::ApproxHarmonicCoordinatorConfig;use crate::distributed::cluster::Cluster;-use crate::distributed::member::{Member, Service, ShardId};-use crate::webgraph::centrality::{store_csv, store_harmonic, top_nodes, TopNodes};-use crate::Result;--pub struct ApproxCentralitySetup {- dht: DhtConn<ApproxCentralityTables>,- workers: Vec<RemoteApproxCentralityWorker>,- sample_rate: f64,-}--impl ApproxCentralitySetup {- pub async fn new(cluster: &Cluster, sample_rate: f64) -> Result<Self> {- let dht_members: Vec<_> = cluster- .members()- .await- .into_iter()- .filter_map(|member| {- if let Service::Dht { host, shard } = member.service {- Some((shard, host))- } else {- None- }- })- .collect();-- let workers = cluster- .members()- .await- .into_iter()- .filter_map(|member| {- if let Service::ApproxHarmonicWorker { host, shard } = member.service {- Some(RemoteApproxCentralityWorker::new(shard, host))- } else {- None- }- })- .collect::<Result<Vec<RemoteApproxCentralityWorker>>>()?;-- Ok(Self::new_for_dht_members(- &dht_members,- workers,- sample_rate,- ))- }- pub fn new_for_dht_members(- dht_members: &[(ShardId, SocketAddr)],- workers: Vec<RemoteApproxCentralityWorker>,- sample_rate: f64,- ) -> Self {- let initial = ApproxCentralityTables {- meta: DefaultDhtTable::new(dht_members, "meta"),- centrality: DefaultDhtTable::new(dht_members, "centrality"),- };-- let dht = DhtConn::new(initial);-- Self {- dht,- workers,- sample_rate,- }- }-}--fn num_samples(num_nodes: u64, sample_rate: f64) -> u64 {- ((num_nodes as f64).log2() / sample_rate.powi(2)).ceil() as u64-}--impl Setup for ApproxCentralitySetup {- type DhtTables = ApproxCentralityTables;-- fn init_dht(&self) -> DhtConn<Self::DhtTables> {- self.dht.clone()- }-- fn setup_round(&self, dht: &Self::DhtTables) {- let meta = dht.meta.get(()).unwrap();-- dht.meta.set(- (),- Meta {- round: meta.round + 1,- num_samples_per_worker: meta.num_samples_per_worker,- },- );- }-- fn setup_first_round(&self, dht: &Self::DhtTables) {- let upper_bound_num_nodes: u64 = self.workers.iter().map(|w| w.num_nodes()).sum();- let num_samples_per_worker =- num_samples(upper_bound_num_nodes, self.sample_rate) / (self.workers.len() as u64);-- dht.meta.set(- (),- Meta {- round: 0,- num_samples_per_worker,- },- );- }-}--pub struct ApproxCentralityFinish;--impl Finisher for ApproxCentralityFinish {- type Job = ApproxCentralityJob;-- fn is_finished(&self, dht: &ApproxCentralityTables) -> bool {- dht.meta.get(()).unwrap().round > 0- }-}--pub fn build(- dht: &[(ShardId, SocketAddr)],- workers: Vec<RemoteApproxCentralityWorker>,- sample_rate: f64,- save_centralities_with_zero: bool,-) -> Coordinator<ApproxCentralityJob> {- let setup = ApproxCentralitySetup::new_for_dht_members(dht, workers.clone(), sample_rate);-- let mut coord = Coordinator::new(setup, workers.clone());-- if save_centralities_with_zero {- coord = coord.with_mapper(ApproxCentralityMapper::InitCentrality)- }-- coord.with_mapper(ApproxCentralityMapper::ApproximateCentrality)-}+use crate::distributed::member::Service;+use crate::entrypoint::ampc::shortest_path::coordinator::ShortestPathFinish;+use crate::entrypoint::ampc::shortest_path::{self, RemoteShortestPathWorker, ShortestPathJob};+use crate::hyperloglog::HyperLogLog;+use crate::kahan_sum::KahanSum;+use crate::webgraph::centrality::{store_harmonic, TopNodes};+use crate::{webgraph, Result};struct ClusterInfo {// dropping the handle will leave the cluster_handle: Cluster,dht: Vec<(ShardId, SocketAddr)>,- workers: Vec<RemoteApproxCentralityWorker>,+ workers: Vec<RemoteShortestPathWorker>,}async fn setup_gossip(config: ApproxHarmonicCoordinatorConfig) -> Result<ClusterInfo> {- let handle = Cluster::join(- Member::new(Service::ApproxHarmonicCoordinator { host: config.host }),+ let handle = Cluster::join_as_spectator(config.gossip.addr,config.gossip.seed_nodes.unwrap_or_default(),)@@ -186,13 +63,13 @@ async fn setup_gossip(config: ApproxHarmonicCoordinatorConfig) -> Result<Clusterlet workers = members.iter().filter_map(|member| {- if let Service::ApproxHarmonicWorker { host, shard } = member.service {- Some(RemoteApproxCentralityWorker::new(shard, host))+ if let Service::ShortestPathWorker { host, shard } = member.service {+ Some(RemoteShortestPathWorker::new(shard, host))} else {None}})- .collect::<Result<Vec<RemoteApproxCentralityWorker>>>()?;+ .collect::<Result<Vec<RemoteShortestPathWorker>>>()?;Ok(ClusterInfo {_handle: handle,@@ -201,6 +78,10 @@ async fn setup_gossip(config: ApproxHarmonicCoordinatorConfig) -> Result<Cluster})}+fn num_samples(num_nodes: u64, sample_rate: f64) -> u64 {+ ((num_nodes as f64).log2() / sample_rate.powi(2)).ceil() as u64+}+pub fn run(config: ApproxHarmonicCoordinatorConfig) -> Result<()> {let tokio_conf = config.clone();let cluster = tokio::runtime::Builder::new_current_thread()@@ -208,58 +89,77 @@ pub fn run(config: ApproxHarmonicCoordinatorConfig) -> Result<()> {.build()?.block_on(setup_gossip(tokio_conf))?;- let upper_bound_num_nodes: u64 = cluster.workers.iter().map(|w| w.num_nodes()).sum();+ let sketch = cluster+ .workers+ .iter()+ .map(|worker| worker.get_node_sketch())+ .fold(HyperLogLog::default(), |mut acc, sketch| {+ acc.merge(&sketch);+ acc+ });++ let num_nodes = sketch.size() as u64;- let num_samples = num_samples(upper_bound_num_nodes, config.sample_rate);+ let num_samples = num_samples(num_nodes, config.sample_rate);+ tracing::info!("sampling {} nodes", num_samples);let norm = 1.0 / ((num_samples - 1) as f64);- let jobs: Vec<_> = cluster+ let num_samples_per_worker = num_samples.div_ceil(cluster.workers.len() as u64);++ let sampled_nodes: Vec<_> = cluster.workers.iter()- .map(|worker| ApproxCentralityJob {- shard: worker.shard(),- max_distance: config.max_distance,- norm,- all_workers: cluster- .workers- .clone()- .into_iter()- .map(|w| (w.shard(), w.addr()))- .collect(),- })+ .flat_map(|worker| worker.sample_nodes(num_samples_per_worker)).collect();- tracing::info!("starting {} jobs", jobs.len());+ let mut centralities: FxHashMap<webgraph::NodeID, KahanSum> = FxHashMap::default();- let coordinator = build(- &cluster.dht,- cluster.workers.clone(),- config.sample_rate,- config.save_centralities_with_zero,- );- let res = coordinator.run(jobs, ApproxCentralityFinish)?;+ for source in sampled_nodes {+ let jobs: Vec<_> = cluster+ .workers+ .iter()+ .map(|worker| ShortestPathJob {+ shard: worker.shard(),+ source,+ })+ .collect();++ tracing::info!("starting {} jobs", jobs.len());++ let coordinator =+ shortest_path::coordinator::build(&cluster.dht, cluster.workers.clone(), source);++ let res = coordinator.run(+ jobs,+ ShortestPathFinish {+ max_distance: Some(config.max_distance as u64),+ },+ )?;++ for (node, distance) in res.distances.iter() {+ let centrality = KahanSum::from((1.0 / distance as f64) * norm);+ centralities+ .entry(node)+ .and_modify(|sum| *sum += centrality)+ .or_insert(centrality);+ }++ res.drop_tables();+ }let output_path = Path::new(&config.output_path);let store = store_harmonic(- res.centrality- .iter()- .map(|(n, c)| (n, f64::from(c)))- .filter(|(_, c)| {- if config.save_centralities_with_zero {- true- } else {- *c > 0.0- }- }),+ centralities.iter().map(|(id, c)| (*id, f64::from(*c))),output_path,);- let top_nodes = top_nodes(&store, TopNodes::Top(1_000_000));+ let top_nodes = webgraph::centrality::top_nodes(&store, TopNodes::Top(1_000_000));let ids = top_nodes.iter().map(|(id, _)| *id).collect::<Vec<_>>();- let id2node: BTreeMap<_, _> = cluster++ let id2node: FxHashMap<_, _> = cluster.workers.iter().flat_map(|w| {@@ -275,7 +175,7 @@ pub fn run(config: ApproxHarmonicCoordinatorConfig) -> Result<()> {.filter_map(|(id, c)| id2node.get(id).map(|n| (n.clone(), *c))).collect::<Vec<_>>();- store_csv(top_nodes, output_path.join("harmonic.csv"));+ webgraph::centrality::store_csv(top_nodes, output_path.join("harmonic.csv"));Ok(())}diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rsdeleted file mode 100644index 03b95077..00000000--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs+++ /dev/null@@ -1,175 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2024 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>--use indicatif::ProgressIterator;-use itertools::Itertools;-use std::{cmp, collections::BTreeMap};--use super::{worker::ApproxCentralityWorker, ApproxCentralityJob, Mapper};-use crate::{- ampc::dht::upsert,- entrypoint::ampc::approximated_harmonic_centrality::DhtTable,- kahan_sum::KahanSum,- webgraph::{self, EdgeLimit},- webpage::html::links::RelFlags,-};-use rayon::prelude::*;--const BATCH_SIZE: usize = 1024;-const MAX_OUTGOING_EDGES: usize = 128;--pub static SKIPPED_REL: std::sync::LazyLock<RelFlags> = std::sync::LazyLock::new(|| {- RelFlags::TAG- | RelFlags::NOFOLLOW- | RelFlags::IS_IN_FOOTER- | RelFlags::IS_IN_NAVIGATION- | RelFlags::PRIVACY_POLICY- | RelFlags::TERMS_OF_SERVICE- | RelFlags::SEARCH- | RelFlags::LINK_TAG- | RelFlags::SCRIPT_TAG- | RelFlags::UGC-});--#[derive(Debug, Clone, bincode::Decode, bincode::Encode)]-pub enum ApproxCentralityMapper {- InitCentrality,- ApproximateCentrality,-}--struct Workers {- worker: ApproxCentralityWorker,-}--impl Workers {- fn new(worker: ApproxCentralityWorker) -> Self {- Self { worker }- }-- fn dijkstra(&self, source: webgraph::NodeID, max_dist: u8) -> BTreeMap<webgraph::NodeID, u8> {- let mut distances: BTreeMap<webgraph::NodeID, u8> = BTreeMap::default();- let mut queue = std::collections::BinaryHeap::new();-- queue.push(cmp::Reverse((0u8, source)));- distances.insert(source, 0u8);-- while let Some(cmp::Reverse((dist, node))) = queue.pop() {- if dist >= max_dist || dist > *distances.get(&node).unwrap_or(&u8::MAX) {- continue;- }-- for outgoing in self- .worker- .graph()- .search(- &webgraph::query::ForwardlinksQuery::new(node)- .with_limit(EdgeLimit::Limit(MAX_OUTGOING_EDGES)),- )- .unwrap_or_default()- .into_iter()- .filter(|e| !e.rel_flags.intersects(*SKIPPED_REL))- .map(|e| e.to)- {- let d = dist + 1;-- let current_dist = distances.entry(outgoing).or_insert(u8::MAX);-- if d < *current_dist {- *current_dist = d;-- if d < max_dist {- queue.push(cmp::Reverse((d, outgoing)));- }- }- }- }-- distances- }-}--impl Mapper for ApproxCentralityMapper {- type Job = ApproxCentralityJob;-- fn map(- &self,- job: Self::Job,- worker: &<<Self as Mapper>::Job as super::Job>::Worker,- dht: &crate::ampc::DhtConn<<<Self as Mapper>::Job as super::Job>::DhtTables>,- ) {- let pool = rayon::ThreadPoolBuilder::new()- .num_threads(usize::from(std::thread::available_parallelism().unwrap()))- .stack_size(80_000_000)- .thread_name(move |num| format!("approx-harmonic-mapper-{num}"))- .build()- .unwrap();-- pool.install(|| match self {- ApproxCentralityMapper::InitCentrality => {- let num_nodes = worker.num_nodes();- worker- .graph()- .page_nodes()- .chunks(BATCH_SIZE)- .into_iter()- .progress_count(num_nodes / BATCH_SIZE as u64)- .for_each(|chunk| {- let pairs: Vec<_> = chunk- .into_iter()- .map(|node| (node, KahanSum::from(0.0)))- .collect();-- dht.next().centrality.batch_set(pairs)- });- }- ApproxCentralityMapper::ApproximateCentrality => {- let workers = Workers::new(worker.clone());- let num_samples = dht.next().meta.get(()).unwrap().num_samples_per_worker;-- tracing::info!("Sampling {} nodes", num_samples);-- let sampled = worker- .graph()- .random_page_nodes_with_outgoing(num_samples as usize);-- let pb = indicatif::ProgressBar::new(sampled.len() as u64);-- sampled.into_par_iter().for_each(|node| {- for chunk in workers- .dijkstra(node, job.max_distance)- .into_iter()- .filter_map(|(n, d)| {- if d == 0 {- None- } else {- Some((n, KahanSum::from((1.0 / d as f64) * job.norm)))- }- })- .chunks(BATCH_SIZE)- .into_iter()- {- let pairs: Vec<_> = chunk.collect();- dht.next()- .centrality- .batch_upsert(upsert::KahanSumAdd, pairs);- }-- pb.inc(1);- });- }- })- }-}diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rsindex 064b35c3..5bf5edab 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs@@ -15,58 +15,3 @@// along with this program. If not, see <https://www.gnu.org/licenses/>pub mod coordinator;-mod mapper;-pub mod worker;--use std::net::SocketAddr;--use crate::distributed::member::ShardId;-use crate::kahan_sum::KahanSum;-use crate::{- ampc::{prelude::*, DefaultDhtTable},- webgraph,-};--use self::mapper::ApproxCentralityMapper;-use self::worker::{ApproxCentralityWorker, RemoteApproxCentralityWorker};--#[derive(- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,- Debug,- Clone,- PartialEq,- Eq,-)]-pub struct Meta {- round: u64,- num_samples_per_worker: u64,-}--#[derive(bincode::Encode, bincode::Decode, Debug, Clone)]-pub struct ApproxCentralityTables {- centrality: DefaultDhtTable<webgraph::NodeID, KahanSum>,- meta: DefaultDhtTable<(), Meta>,-}--impl_dht_tables!(ApproxCentralityTables, [centrality, meta]);--#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]-pub struct ApproxCentralityJob {- shard: ShardId,- max_distance: u8,- norm: f64,- all_workers: Vec<(ShardId, SocketAddr)>,-}--impl Job for ApproxCentralityJob {- type DhtTables = ApproxCentralityTables;- type Worker = ApproxCentralityWorker;- type Mapper = ApproxCentralityMapper;-- fn is_schedulable(&self, worker: &RemoteApproxCentralityWorker) -> bool {- self.shard == worker.shard()- }-}diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rsdeleted file mode 100644index ae67202c..00000000--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs+++ /dev/null@@ -1,158 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2024 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>--use crate::{- ampc::JobConn,- config::ApproxHarmonicWorkerConfig,- distributed::{- member::{Service, ShardId},- sonic,- },- hyperloglog::HyperLogLog,- webgraph::{self, Webgraph},- Result,-};-use std::{net::SocketAddr, sync::Arc};--use super::{impl_worker, ApproxCentralityJob, Message, RemoteWorker, Worker as _};--#[derive(Clone)]-pub struct ApproxCentralityWorker {- shard: ShardId,- graph: Arc<Webgraph>,- num_nodes: u64,-}--impl ApproxCentralityWorker {- pub fn new(graph: Webgraph, shard: ShardId) -> Self {- let num_nodes = graph- .page_nodes()- .map(|node| node.as_u128() as u64)- .collect::<HyperLogLog<2048>>()- .size() as u64;- Self {- graph: Arc::new(graph),- shard,- num_nodes,- }- }-- pub fn graph(&self) -> &Webgraph {- &self.graph- }-- pub fn shard(&self) -> ShardId {- self.shard- }-- pub fn num_nodes(&self) -> u64 {- self.num_nodes- }-}--#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]-pub struct NumNodes;--impl Message<ApproxCentralityWorker> for NumNodes {- type Response = u64;-- fn handle(self, worker: &ApproxCentralityWorker) -> Self::Response {- worker.num_nodes- }-}--#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]-pub struct BatchId2Node(Vec<webgraph::NodeID>);--impl Message<ApproxCentralityWorker> for BatchId2Node {- type Response = Vec<(webgraph::NodeID, webgraph::Node)>;-- fn handle(self, worker: &ApproxCentralityWorker) -> Self::Response {- self.0- .iter()- .filter_map(|id| {- worker- .graph- .search(&webgraph::query::Id2NodeQuery::Page(*id))- .ok()- .flatten()- .map(|node| (*id, node))- })- .collect()- }-}--impl_worker!(ApproxCentralityJob, RemoteApproxCentralityWorker => ApproxCentralityWorker, [NumNodes, BatchId2Node]);--#[derive(Clone)]-pub struct RemoteApproxCentralityWorker {- shard: ShardId,- pool: Arc<sonic::ConnectionPool<JobConn<ApproxCentralityJob>>>,-}--impl RemoteApproxCentralityWorker {- pub fn new(shard: ShardId, addr: SocketAddr) -> Result<Self> {- Ok(Self {- shard,- pool: Arc::new(sonic::ConnectionPool::new(addr)?),- })- }-- pub fn shard(&self) -> ShardId {- self.shard- }-- pub fn addr(&self) -> SocketAddr {- self.pool.addr()- }-- pub fn num_nodes(&self) -> u64 {- self.send(NumNodes)- }-- pub fn batch_id2node(- &self,- ids: Vec<webgraph::NodeID>,- ) -> Vec<(webgraph::NodeID, webgraph::Node)> {- self.send(BatchId2Node(ids))- }-}--impl RemoteWorker for RemoteApproxCentralityWorker {- type Job = ApproxCentralityJob;-- fn pool(&self) -> &sonic::ConnectionPool<JobConn<Self::Job>> {- &self.pool- }-}--pub fn run(config: ApproxHarmonicWorkerConfig) -> Result<()> {- tracing::info!("starting worker");- let tokio_conf = config.clone();-- let graph = Webgraph::builder(config.graph_path, config.shard).open()?;- let worker = ApproxCentralityWorker::new(graph, config.shard);- let service = Service::ApproxHarmonicWorker {- host: tokio_conf.host,- shard: tokio_conf.shard,- };- crate::start_gossip_cluster_thread(tokio_conf.gossip, Some(service));-- tracing::info!("worker ready");- worker.run(config.host)?;-- Ok(())-}diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rsindex 9c5d8321..2e0ddc82 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs@@ -17,7 +17,6 @@use crate::{ampc::{dht::ShardId, prelude::*, Coordinator, DhtConn},config::HarmonicCoordinatorConfig,- distributed::member::Member,webgraph::centrality::{store_csv, store_harmonic, top_nodes, TopNodes},Result,};@@ -143,8 +142,7 @@ struct ClusterInfo {}async fn setup_gossip(config: HarmonicCoordinatorConfig) -> Result<ClusterInfo> {- let handle = Cluster::join(- Member::new(Service::HarmonicCoordinator { host: config.host }),+ let handle = Cluster::join_as_spectator(config.gossip.addr,config.gossip.seed_nodes.unwrap_or_default(),)@@ -234,5 +232,7 @@ pub fn run(config: HarmonicCoordinatorConfig) -> Result<()> {store_csv(top_nodes, output_path.join("harmonic.csv"));+ res.drop_tables();+Ok(())}diff --git a/crates/core/src/entrypoint/ampc/shortest_path/coordinator.rs b/crates/core/src/entrypoint/ampc/shortest_path/coordinator.rsindex 375ab67e..b9b95094 100644--- a/crates/core/src/entrypoint/ampc/shortest_path/coordinator.rs+++ b/crates/core/src/entrypoint/ampc/shortest_path/coordinator.rs@@ -24,10 +24,10 @@ use url::Url;use super::mapper::ShortestPathMapper;use super::{worker::RemoteShortestPathWorker, ShortestPathTables};use super::{DhtTable as _, Finisher, Meta, Setup, ShortestPathJob};-use crate::ampc::{Coordinator, DefaultDhtTable, DhtConn};+use crate::ampc::{Coordinator, DefaultDhtTable, DhtConn, DhtTables};use crate::config::ShortestPathCoordinatorConfig;use crate::distributed::cluster::Cluster;-use crate::distributed::member::{Member, Service, ShardId};+use crate::distributed::member::{Service, ShardId};use crate::hyperloglog::HyperLogLog;use crate::webpage::url_ext::UrlExt;use crate::{webgraph, Result};@@ -102,7 +102,7 @@ impl Setup for ShortestPathSetup {}pub struct ShortestPathFinish {- max_distance: Option<u64>,+ pub max_distance: Option<u64>,}impl Finisher for ShortestPathFinish {@@ -140,8 +140,7 @@ struct ClusterInfo {}async fn setup_gossip(config: ShortestPathCoordinatorConfig) -> Result<ClusterInfo> {- let handle = Cluster::join(- Member::new(Service::ShortestPathCoordinator { host: config.host }),+ let handle = Cluster::join_as_spectator(config.gossip.addr,config.gossip.seed_nodes.unwrap_or_default(),)@@ -260,5 +259,7 @@ pub fn run(config: ShortestPathCoordinatorConfig) -> Result<()> {}}+ res.drop_tables();+Ok(())}diff --git a/crates/core/src/entrypoint/ampc/shortest_path/mod.rs b/crates/core/src/entrypoint/ampc/shortest_path/mod.rsindex 8ec6604e..3b4a8dab 100644--- a/crates/core/src/entrypoint/ampc/shortest_path/mod.rs+++ b/crates/core/src/entrypoint/ampc/shortest_path/mod.rs@@ -29,8 +29,8 @@ use crate::{webgraph,};-use self::mapper::ShortestPathMapper;-use self::worker::{RemoteShortestPathWorker, ShortestPathWorker};+pub use self::mapper::ShortestPathMapper;+pub use self::worker::{RemoteShortestPathWorker, ShortestPathWorker};#[derive(serde::Serialize,@@ -49,17 +49,17 @@ pub struct Meta {#[derive(bincode::Encode, bincode::Decode, Debug, Clone)]pub struct ShortestPathTables {- distances: DefaultDhtTable<webgraph::NodeID, u64>,- meta: DefaultDhtTable<(), Meta>,- changed_nodes: DefaultDhtTable<ShardId, UpdatedNodes>,+ pub distances: DefaultDhtTable<webgraph::NodeID, u64>,+ pub meta: DefaultDhtTable<(), Meta>,+ pub changed_nodes: DefaultDhtTable<ShardId, UpdatedNodes>,}impl_dht_tables!(ShortestPathTables, [distances, meta, changed_nodes]);#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]pub struct ShortestPathJob {- shard: ShardId,- source: webgraph::NodeID,+ pub shard: ShardId,+ pub source: webgraph::NodeID,}impl Job for ShortestPathJob {diff --git a/crates/core/src/entrypoint/ampc/shortest_path/worker.rs b/crates/core/src/entrypoint/ampc/shortest_path/worker.rsindex b202d392..3fdfc5e8 100644--- a/crates/core/src/entrypoint/ampc/shortest_path/worker.rs+++ b/crates/core/src/entrypoint/ampc/shortest_path/worker.rs@@ -41,7 +41,7 @@ pub struct ShortestPathWorker {}impl ShortestPathWorker {- pub fn new(graph: Webgraph, shard: ShardId) -> Self {+ pub fn new(graph: Arc<Webgraph>, shard: ShardId) -> Self {let mut nodes_sketch = HyperLogLog::default();for node in graph.page_nodes() {@@ -50,7 +50,7 @@ impl ShortestPathWorker {let num_nodes = nodes_sketch.size() as u64;Self {- graph: Arc::new(graph),+ graph,shard,changed_nodes: Arc::new(Mutex::new(UpdatedNodes::new(num_nodes))),nodes_sketch,@@ -101,6 +101,19 @@ impl Message<ShortestPathWorker> for UpdateChangedNodesPrecision {}}+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct SampleNodes(u64);++impl Message<ShortestPathWorker> for SampleNodes {+ type Response = Vec<webgraph::NodeID>;++ fn handle(self, worker: &ShortestPathWorker) -> Self::Response {+ worker+ .graph()+ .random_page_nodes_with_outgoing(self.0 as usize)+ }+}+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]pub struct BatchId2Node(Vec<webgraph::NodeID>);@@ -122,7 +135,7 @@ impl Message<ShortestPathWorker> for BatchId2Node {}}-impl_worker!(ShortestPathJob, RemoteShortestPathWorker => ShortestPathWorker, [BatchId2Node, GetNodeSketch, UpdateChangedNodesPrecision]);+impl_worker!(ShortestPathJob, RemoteShortestPathWorker => ShortestPathWorker, [BatchId2Node, GetNodeSketch, UpdateChangedNodesPrecision, SampleNodes]);#[derive(Clone)]pub struct RemoteShortestPathWorker {@@ -160,6 +173,10 @@ impl RemoteShortestPathWorker {pub fn update_changed_nodes_precision(&self, num_nodes: u64) {self.send(UpdateChangedNodesPrecision(num_nodes));}++ pub fn sample_nodes(&self, num_nodes: u64) -> Vec<webgraph::NodeID> {+ self.send(SampleNodes(num_nodes))+ }}impl RemoteWorker for RemoteShortestPathWorker {@@ -174,7 +191,7 @@ pub fn run(config: ShortestPathWorkerConfig) -> Result<()> {tracing::info!("starting worker");let tokio_conf = config.clone();- let graph = Webgraph::builder(config.graph_path, config.shard).open()?;+ let graph = Arc::new(Webgraph::builder(config.graph_path, config.shard).open()?);let worker = ShortestPathWorker::new(graph, config.shard);let service = Service::ShortestPathWorker {host: tokio_conf.host,diff --git a/crates/core/src/kahan_sum.rs b/crates/core/src/kahan_sum.rsindex e40119ae..9146e9a5 100644--- a/crates/core/src/kahan_sum.rs+++ b/crates/core/src/kahan_sum.rs@@ -62,6 +62,24 @@ impl Add<f64> for KahanSum {}}+impl AddAssign<KahanSum> for KahanSum {+ fn add_assign(&mut self, rhs: KahanSum) {+ let y = (rhs.sum + rhs.err) - self.err;+ let t = self.sum + y;+ self.err = (t - self.sum) - y;+ self.sum = t;+ }+}++impl Add<KahanSum> for KahanSum {+ type Output = Self;+ fn add(self, rhs: KahanSum) -> Self::Output {+ let mut k = self;+ k += rhs;+ k+ }+}+#[cfg(test)]mod tests {use super::*;@@ -85,4 +103,24 @@ mod tests {assert_eq!(10017.579623446147f64, f64::from(sum));}++ #[test]+ fn it_works_kahan() {+ let mut sum = KahanSum::default();+ assert_eq!(0.0, f64::from(sum));++ for elem in [+ 10000.0f64,+ std::f64::consts::PI,+ std::f64::consts::E,+ std::f64::consts::PI,+ std::f64::consts::E,+ std::f64::consts::PI,+ std::f64::consts::E,+ ] {+ sum += KahanSum::from(elem);+ }++ assert_eq!(10017.579623446147f64, f64::from(sum));+ }}diff --git a/crates/core/src/main.rs b/crates/core/src/main.rsindex 4264d440..054fd0db 100644--- a/crates/core/src/main.rs+++ b/crates/core/src/main.rs@@ -134,13 +134,8 @@ enum AmpcOptions {/// Workers needs to be started before the coordinator.HarmonicCoordinator { config_path: String },- /// Start a worker to compute an approximation of the harmonic centrality of a graph.- /// The approximation samples O(log n / sample_rate^2) nodes from the graph and computes- /// shortest paths from the sampled nodes.- ApproxHarmonicWorker { config_path: String },-/// Start a coordinator to distribute the approximation of the harmonic centrality computation.- /// Workers needs to be started before the coordinator.+ /// Uses shortest path workers. Workers needs to be started before the coordinator.ApproxHarmonicCoordinator { config_path: String },/// Start a worker to compute the shortest paths of a graph.@@ -469,27 +464,27 @@ fn main() -> Result<()> {.build()?.block_on(entrypoint::ampc::dht::run(config))?;}+AmpcOptions::HarmonicWorker { config_path } => {let config: config::HarmonicWorkerConfig = load_toml_config(config_path);entrypoint::ampc::harmonic_centrality::worker::run(config)?;}+AmpcOptions::HarmonicCoordinator { config_path } => {let config: config::HarmonicCoordinatorConfig = load_toml_config(config_path);entrypoint::ampc::harmonic_centrality::coordinator::run(config)?;}- AmpcOptions::ApproxHarmonicWorker { config_path } => {- let config: config::ApproxHarmonicWorkerConfig = load_toml_config(config_path);- entrypoint::ampc::approximated_harmonic_centrality::worker::run(config)?;- }AmpcOptions::ApproxHarmonicCoordinator { config_path } => {let config: config::ApproxHarmonicCoordinatorConfig = load_toml_config(config_path);entrypoint::ampc::approximated_harmonic_centrality::coordinator::run(config)?;}+AmpcOptions::ShortestPathWorker { config_path } => {let config: config::ShortestPathWorkerConfig = load_toml_config(config_path);entrypoint::ampc::shortest_path::worker::run(config)?;}+AmpcOptions::ShortestPathCoordinator { config_path } => {let config: config::ShortestPathCoordinatorConfig = load_toml_config(config_path);entrypoint::ampc::shortest_path::coordinator::run(config)?;