Skip to content

web search done right

use shortest paths as subroutine in approx centrality

Mikkel Denker852a10c08e78
diff --git a/crates/core/src/ampc/dht/value.rs b/crates/core/src/ampc/dht/value.rs
index 6fbcf3e6..417b0ff5 100644
--- a/crates/core/src/ampc/dht/value.rs
+++ b/crates/core/src/ampc/dht/value.rs
@@ -22,9 +22,6 @@ pub trait ValueTrait: TryFrom<Value> + Into<Value> {}
impl ValueTrait for String {}
-type ApproxHarmonicMeta = crate::entrypoint::ampc::approximated_harmonic_centrality::Meta;
-impl ValueTrait for ApproxHarmonicMeta {}
-
type F32 = f32;
impl ValueTrait for F32 {}
@@ -66,7 +63,6 @@ impl ValueTrait for Unit {}
)]
pub enum Value {
String(String),
- ApproxHarmonicMeta(ApproxHarmonicMeta),
F32(F32),
F64(F64),
U64(U64),
@@ -105,7 +101,6 @@ macro_rules! impl_from_to_value {
}
impl_from_to_value!(String, String);
-impl_from_to_value!(ApproxHarmonicMeta, ApproxHarmonicMeta);
impl_from_to_value!(F32, F32);
impl_from_to_value!(F64, F64);
impl_from_to_value!(U64, U64);
diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rs
index cc5b4e64..d4fe4314 100644
--- a/crates/core/src/config/defaults.rs
+++ b/crates/core/src/config/defaults.rs
@@ -185,10 +185,6 @@ impl ApproxHarmonic {
pub fn max_distance() -> u8 {
7
}
-
- pub fn save_centralities_with_zero() -> bool {
- false
- }
}
pub struct Webgraph;
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 4c5552da..ec8a2098 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -716,17 +716,6 @@ pub struct ApproxHarmonicCoordinatorConfig {
#[serde(default = "defaults::ApproxHarmonic::max_distance")]
pub max_distance: u8,
-
- #[serde(default = "defaults::ApproxHarmonic::save_centralities_with_zero")]
- pub save_centralities_with_zero: bool,
-}
-
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
-pub struct ApproxHarmonicWorkerConfig {
- pub gossip: GossipConfig,
- pub shard: ShardId,
- pub graph_path: String,
- pub host: SocketAddr,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rs
index 38513301..2d6f7538 100644
--- a/crates/core/src/distributed/member.rs
+++ b/crates/core/src/distributed/member.rs
@@ -125,23 +125,14 @@ pub enum Service {
host: SocketAddr,
shard: ShardId,
},
- HarmonicCoordinator {
- host: SocketAddr,
- },
ApproxHarmonicWorker {
host: SocketAddr,
shard: ShardId,
},
- ApproxHarmonicCoordinator {
- host: SocketAddr,
- },
ShortestPathWorker {
host: SocketAddr,
shard: ShardId,
},
- ShortestPathCoordinator {
- host: SocketAddr,
- },
}
impl std::fmt::Display for Service {
@@ -163,19 +154,12 @@ impl std::fmt::Display for Service {
}
Self::Dht { host, shard } => write!(f, "Dht {} {}", host, shard),
Self::HarmonicWorker { host, shard } => write!(f, "HarmonicWorker {} {}", host, shard),
- Self::HarmonicCoordinator { host } => write!(f, "HarmonicCoordinator {}", host),
Self::ApproxHarmonicWorker { host, shard } => {
write!(f, "ApproxHarmonicWorker {} {}", host, shard)
}
- Self::ApproxHarmonicCoordinator { host } => {
- write!(f, "ApproxHarmonicCoordinator {}", host)
- }
Self::ShortestPathWorker { host, shard } => {
write!(f, "ShortestPathWorker {} {}", host, shard)
}
- Self::ShortestPathCoordinator { host } => {
- write!(f, "ShortestPathCoordinator {}", host)
- }
}
}
}
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
index 99a5ea3f..b3f32d2d 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
@@ -14,155 +14,32 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>
-use std::collections::BTreeMap;
use std::net::SocketAddr;
use std::path::Path;
-use super::mapper::ApproxCentralityMapper;
-use super::{worker::RemoteApproxCentralityWorker, ApproxCentralityTables};
-use super::{ApproxCentralityJob, DhtTable as _, Finisher, Meta, Setup};
-use crate::ampc::{Coordinator, DefaultDhtTable, DhtConn};
+use rustc_hash::FxHashMap;
+
+use crate::ampc::dht::ShardId;
+use crate::ampc::{DhtTable as _, DhtTables as _};
use crate::config::ApproxHarmonicCoordinatorConfig;
use crate::distributed::cluster::Cluster;
-use crate::distributed::member::{Member, Service, ShardId};
-use crate::webgraph::centrality::{store_csv, store_harmonic, top_nodes, TopNodes};
-use crate::Result;
-
-pub struct ApproxCentralitySetup {
- dht: DhtConn<ApproxCentralityTables>,
- workers: Vec<RemoteApproxCentralityWorker>,
- sample_rate: f64,
-}
-
-impl ApproxCentralitySetup {
- pub async fn new(cluster: &Cluster, sample_rate: f64) -> Result<Self> {
- let dht_members: Vec<_> = cluster
- .members()
- .await
- .into_iter()
- .filter_map(|member| {
- if let Service::Dht { host, shard } = member.service {
- Some((shard, host))
- } else {
- None
- }
- })
- .collect();
-
- let workers = cluster
- .members()
- .await
- .into_iter()
- .filter_map(|member| {
- if let Service::ApproxHarmonicWorker { host, shard } = member.service {
- Some(RemoteApproxCentralityWorker::new(shard, host))
- } else {
- None
- }
- })
- .collect::<Result<Vec<RemoteApproxCentralityWorker>>>()?;
-
- Ok(Self::new_for_dht_members(
- &dht_members,
- workers,
- sample_rate,
- ))
- }
- pub fn new_for_dht_members(
- dht_members: &[(ShardId, SocketAddr)],
- workers: Vec<RemoteApproxCentralityWorker>,
- sample_rate: f64,
- ) -> Self {
- let initial = ApproxCentralityTables {
- meta: DefaultDhtTable::new(dht_members, "meta"),
- centrality: DefaultDhtTable::new(dht_members, "centrality"),
- };
-
- let dht = DhtConn::new(initial);
-
- Self {
- dht,
- workers,
- sample_rate,
- }
- }
-}
-
-fn num_samples(num_nodes: u64, sample_rate: f64) -> u64 {
- ((num_nodes as f64).log2() / sample_rate.powi(2)).ceil() as u64
-}
-
-impl Setup for ApproxCentralitySetup {
- type DhtTables = ApproxCentralityTables;
-
- fn init_dht(&self) -> DhtConn<Self::DhtTables> {
- self.dht.clone()
- }
-
- fn setup_round(&self, dht: &Self::DhtTables) {
- let meta = dht.meta.get(()).unwrap();
-
- dht.meta.set(
- (),
- Meta {
- round: meta.round + 1,
- num_samples_per_worker: meta.num_samples_per_worker,
- },
- );
- }
-
- fn setup_first_round(&self, dht: &Self::DhtTables) {
- let upper_bound_num_nodes: u64 = self.workers.iter().map(|w| w.num_nodes()).sum();
- let num_samples_per_worker =
- num_samples(upper_bound_num_nodes, self.sample_rate) / (self.workers.len() as u64);
-
- dht.meta.set(
- (),
- Meta {
- round: 0,
- num_samples_per_worker,
- },
- );
- }
-}
-
-pub struct ApproxCentralityFinish;
-
-impl Finisher for ApproxCentralityFinish {
- type Job = ApproxCentralityJob;
-
- fn is_finished(&self, dht: &ApproxCentralityTables) -> bool {
- dht.meta.get(()).unwrap().round > 0
- }
-}
-
-pub fn build(
- dht: &[(ShardId, SocketAddr)],
- workers: Vec<RemoteApproxCentralityWorker>,
- sample_rate: f64,
- save_centralities_with_zero: bool,
-) -> Coordinator<ApproxCentralityJob> {
- let setup = ApproxCentralitySetup::new_for_dht_members(dht, workers.clone(), sample_rate);
-
- let mut coord = Coordinator::new(setup, workers.clone());
-
- if save_centralities_with_zero {
- coord = coord.with_mapper(ApproxCentralityMapper::InitCentrality)
- }
-
- coord.with_mapper(ApproxCentralityMapper::ApproximateCentrality)
-}
+use crate::distributed::member::Service;
+use crate::entrypoint::ampc::shortest_path::coordinator::ShortestPathFinish;
+use crate::entrypoint::ampc::shortest_path::{self, RemoteShortestPathWorker, ShortestPathJob};
+use crate::hyperloglog::HyperLogLog;
+use crate::kahan_sum::KahanSum;
+use crate::webgraph::centrality::{store_harmonic, TopNodes};
+use crate::{webgraph, Result};
struct ClusterInfo {
// dropping the handle will leave the cluster
_handle: Cluster,
dht: Vec<(ShardId, SocketAddr)>,
- workers: Vec<RemoteApproxCentralityWorker>,
+ workers: Vec<RemoteShortestPathWorker>,
}
async fn setup_gossip(config: ApproxHarmonicCoordinatorConfig) -> Result<ClusterInfo> {
- let handle = Cluster::join(
- Member::new(Service::ApproxHarmonicCoordinator { host: config.host }),
+ let handle = Cluster::join_as_spectator(
config.gossip.addr,
config.gossip.seed_nodes.unwrap_or_default(),
)
@@ -186,13 +63,13 @@ async fn setup_gossip(config: ApproxHarmonicCoordinatorConfig) -> Result<Cluster
let workers = members
.iter()
.filter_map(|member| {
- if let Service::ApproxHarmonicWorker { host, shard } = member.service {
- Some(RemoteApproxCentralityWorker::new(shard, host))
+ if let Service::ShortestPathWorker { host, shard } = member.service {
+ Some(RemoteShortestPathWorker::new(shard, host))
} else {
None
}
})
- .collect::<Result<Vec<RemoteApproxCentralityWorker>>>()?;
+ .collect::<Result<Vec<RemoteShortestPathWorker>>>()?;
Ok(ClusterInfo {
_handle: handle,
@@ -201,6 +78,10 @@ async fn setup_gossip(config: ApproxHarmonicCoordinatorConfig) -> Result<Cluster
})
}
+fn num_samples(num_nodes: u64, sample_rate: f64) -> u64 {
+ ((num_nodes as f64).log2() / sample_rate.powi(2)).ceil() as u64
+}
+
pub fn run(config: ApproxHarmonicCoordinatorConfig) -> Result<()> {
let tokio_conf = config.clone();
let cluster = tokio::runtime::Builder::new_current_thread()
@@ -208,58 +89,77 @@ pub fn run(config: ApproxHarmonicCoordinatorConfig) -> Result<()> {
.build()?
.block_on(setup_gossip(tokio_conf))?;
- let upper_bound_num_nodes: u64 = cluster.workers.iter().map(|w| w.num_nodes()).sum();
+ let sketch = cluster
+ .workers
+ .iter()
+ .map(|worker| worker.get_node_sketch())
+ .fold(HyperLogLog::default(), |mut acc, sketch| {
+ acc.merge(&sketch);
+ acc
+ });
+
+ let num_nodes = sketch.size() as u64;
- let num_samples = num_samples(upper_bound_num_nodes, config.sample_rate);
+ let num_samples = num_samples(num_nodes, config.sample_rate);
+ tracing::info!("sampling {} nodes", num_samples);
let norm = 1.0 / ((num_samples - 1) as f64);
- let jobs: Vec<_> = cluster
+ let num_samples_per_worker = num_samples.div_ceil(cluster.workers.len() as u64);
+
+ let sampled_nodes: Vec<_> = cluster
.workers
.iter()
- .map(|worker| ApproxCentralityJob {
- shard: worker.shard(),
- max_distance: config.max_distance,
- norm,
- all_workers: cluster
- .workers
- .clone()
- .into_iter()
- .map(|w| (w.shard(), w.addr()))
- .collect(),
- })
+ .flat_map(|worker| worker.sample_nodes(num_samples_per_worker))
.collect();
- tracing::info!("starting {} jobs", jobs.len());
+ let mut centralities: FxHashMap<webgraph::NodeID, KahanSum> = FxHashMap::default();
- let coordinator = build(
- &cluster.dht,
- cluster.workers.clone(),
- config.sample_rate,
- config.save_centralities_with_zero,
- );
- let res = coordinator.run(jobs, ApproxCentralityFinish)?;
+ for source in sampled_nodes {
+ let jobs: Vec<_> = cluster
+ .workers
+ .iter()
+ .map(|worker| ShortestPathJob {
+ shard: worker.shard(),
+ source,
+ })
+ .collect();
+
+ tracing::info!("starting {} jobs", jobs.len());
+
+ let coordinator =
+ shortest_path::coordinator::build(&cluster.dht, cluster.workers.clone(), source);
+
+ let res = coordinator.run(
+ jobs,
+ ShortestPathFinish {
+ max_distance: Some(config.max_distance as u64),
+ },
+ )?;
+
+ for (node, distance) in res.distances.iter() {
+ let centrality = KahanSum::from((1.0 / distance as f64) * norm);
+ centralities
+ .entry(node)
+ .and_modify(|sum| *sum += centrality)
+ .or_insert(centrality);
+ }
+
+ res.drop_tables();
+ }
let output_path = Path::new(&config.output_path);
let store = store_harmonic(
- res.centrality
- .iter()
- .map(|(n, c)| (n, f64::from(c)))
- .filter(|(_, c)| {
- if config.save_centralities_with_zero {
- true
- } else {
- *c > 0.0
- }
- }),
+ centralities.iter().map(|(id, c)| (*id, f64::from(*c))),
output_path,
);
- let top_nodes = top_nodes(&store, TopNodes::Top(1_000_000));
+ let top_nodes = webgraph::centrality::top_nodes(&store, TopNodes::Top(1_000_000));
let ids = top_nodes.iter().map(|(id, _)| *id).collect::<Vec<_>>();
- let id2node: BTreeMap<_, _> = cluster
+
+ let id2node: FxHashMap<_, _> = cluster
.workers
.iter()
.flat_map(|w| {
@@ -275,7 +175,7 @@ pub fn run(config: ApproxHarmonicCoordinatorConfig) -> Result<()> {
.filter_map(|(id, c)| id2node.get(id).map(|n| (n.clone(), *c)))
.collect::<Vec<_>>();
- store_csv(top_nodes, output_path.join("harmonic.csv"));
+ webgraph::centrality::store_csv(top_nodes, output_path.join("harmonic.csv"));
Ok(())
}
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
deleted file mode 100644
index 03b95077..00000000
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
+++ /dev/null
@@ -1,175 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2024 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>
-
-use indicatif::ProgressIterator;
-use itertools::Itertools;
-use std::{cmp, collections::BTreeMap};
-
-use super::{worker::ApproxCentralityWorker, ApproxCentralityJob, Mapper};
-use crate::{
- ampc::dht::upsert,
- entrypoint::ampc::approximated_harmonic_centrality::DhtTable,
- kahan_sum::KahanSum,
- webgraph::{self, EdgeLimit},
- webpage::html::links::RelFlags,
-};
-use rayon::prelude::*;
-
-const BATCH_SIZE: usize = 1024;
-const MAX_OUTGOING_EDGES: usize = 128;
-
-pub static SKIPPED_REL: std::sync::LazyLock<RelFlags> = std::sync::LazyLock::new(|| {
- RelFlags::TAG
- | RelFlags::NOFOLLOW
- | RelFlags::IS_IN_FOOTER
- | RelFlags::IS_IN_NAVIGATION
- | RelFlags::PRIVACY_POLICY
- | RelFlags::TERMS_OF_SERVICE
- | RelFlags::SEARCH
- | RelFlags::LINK_TAG
- | RelFlags::SCRIPT_TAG
- | RelFlags::UGC
-});
-
-#[derive(Debug, Clone, bincode::Decode, bincode::Encode)]
-pub enum ApproxCentralityMapper {
- InitCentrality,
- ApproximateCentrality,
-}
-
-struct Workers {
- worker: ApproxCentralityWorker,
-}
-
-impl Workers {
- fn new(worker: ApproxCentralityWorker) -> Self {
- Self { worker }
- }
-
- fn dijkstra(&self, source: webgraph::NodeID, max_dist: u8) -> BTreeMap<webgraph::NodeID, u8> {
- let mut distances: BTreeMap<webgraph::NodeID, u8> = BTreeMap::default();
- let mut queue = std::collections::BinaryHeap::new();
-
- queue.push(cmp::Reverse((0u8, source)));
- distances.insert(source, 0u8);
-
- while let Some(cmp::Reverse((dist, node))) = queue.pop() {
- if dist >= max_dist || dist > *distances.get(&node).unwrap_or(&u8::MAX) {
- continue;
- }
-
- for outgoing in self
- .worker
- .graph()
- .search(
- &webgraph::query::ForwardlinksQuery::new(node)
- .with_limit(EdgeLimit::Limit(MAX_OUTGOING_EDGES)),
- )
- .unwrap_or_default()
- .into_iter()
- .filter(|e| !e.rel_flags.intersects(*SKIPPED_REL))
- .map(|e| e.to)
- {
- let d = dist + 1;
-
- let current_dist = distances.entry(outgoing).or_insert(u8::MAX);
-
- if d < *current_dist {
- *current_dist = d;
-
- if d < max_dist {
- queue.push(cmp::Reverse((d, outgoing)));
- }
- }
- }
- }
-
- distances
- }
-}
-
-impl Mapper for ApproxCentralityMapper {
- type Job = ApproxCentralityJob;
-
- fn map(
- &self,
- job: Self::Job,
- worker: &<<Self as Mapper>::Job as super::Job>::Worker,
- dht: &crate::ampc::DhtConn<<<Self as Mapper>::Job as super::Job>::DhtTables>,
- ) {
- let pool = rayon::ThreadPoolBuilder::new()
- .num_threads(usize::from(std::thread::available_parallelism().unwrap()))
- .stack_size(80_000_000)
- .thread_name(move |num| format!("approx-harmonic-mapper-{num}"))
- .build()
- .unwrap();
-
- pool.install(|| match self {
- ApproxCentralityMapper::InitCentrality => {
- let num_nodes = worker.num_nodes();
- worker
- .graph()
- .page_nodes()
- .chunks(BATCH_SIZE)
- .into_iter()
- .progress_count(num_nodes / BATCH_SIZE as u64)
- .for_each(|chunk| {
- let pairs: Vec<_> = chunk
- .into_iter()
- .map(|node| (node, KahanSum::from(0.0)))
- .collect();
-
- dht.next().centrality.batch_set(pairs)
- });
- }
- ApproxCentralityMapper::ApproximateCentrality => {
- let workers = Workers::new(worker.clone());
- let num_samples = dht.next().meta.get(()).unwrap().num_samples_per_worker;
-
- tracing::info!("Sampling {} nodes", num_samples);
-
- let sampled = worker
- .graph()
- .random_page_nodes_with_outgoing(num_samples as usize);
-
- let pb = indicatif::ProgressBar::new(sampled.len() as u64);
-
- sampled.into_par_iter().for_each(|node| {
- for chunk in workers
- .dijkstra(node, job.max_distance)
- .into_iter()
- .filter_map(|(n, d)| {
- if d == 0 {
- None
- } else {
- Some((n, KahanSum::from((1.0 / d as f64) * job.norm)))
- }
- })
- .chunks(BATCH_SIZE)
- .into_iter()
- {
- let pairs: Vec<_> = chunk.collect();
- dht.next()
- .centrality
- .batch_upsert(upsert::KahanSumAdd, pairs);
- }
-
- pb.inc(1);
- });
- }
- })
- }
-}
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs
index 064b35c3..5bf5edab 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs
@@ -15,58 +15,3 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>
pub mod coordinator;
-mod mapper;
-pub mod worker;
-
-use std::net::SocketAddr;
-
-use crate::distributed::member::ShardId;
-use crate::kahan_sum::KahanSum;
-use crate::{
- ampc::{prelude::*, DefaultDhtTable},
- webgraph,
-};
-
-use self::mapper::ApproxCentralityMapper;
-use self::worker::{ApproxCentralityWorker, RemoteApproxCentralityWorker};
-
-#[derive(
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
- Debug,
- Clone,
- PartialEq,
- Eq,
-)]
-pub struct Meta {
- round: u64,
- num_samples_per_worker: u64,
-}
-
-#[derive(bincode::Encode, bincode::Decode, Debug, Clone)]
-pub struct ApproxCentralityTables {
- centrality: DefaultDhtTable<webgraph::NodeID, KahanSum>,
- meta: DefaultDhtTable<(), Meta>,
-}
-
-impl_dht_tables!(ApproxCentralityTables, [centrality, meta]);
-
-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
-pub struct ApproxCentralityJob {
- shard: ShardId,
- max_distance: u8,
- norm: f64,
- all_workers: Vec<(ShardId, SocketAddr)>,
-}
-
-impl Job for ApproxCentralityJob {
- type DhtTables = ApproxCentralityTables;
- type Worker = ApproxCentralityWorker;
- type Mapper = ApproxCentralityMapper;
-
- fn is_schedulable(&self, worker: &RemoteApproxCentralityWorker) -> bool {
- self.shard == worker.shard()
- }
-}
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs
deleted file mode 100644
index ae67202c..00000000
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs
+++ /dev/null
@@ -1,158 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2024 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>
-
-use crate::{
- ampc::JobConn,
- config::ApproxHarmonicWorkerConfig,
- distributed::{
- member::{Service, ShardId},
- sonic,
- },
- hyperloglog::HyperLogLog,
- webgraph::{self, Webgraph},
- Result,
-};
-use std::{net::SocketAddr, sync::Arc};
-
-use super::{impl_worker, ApproxCentralityJob, Message, RemoteWorker, Worker as _};
-
-#[derive(Clone)]
-pub struct ApproxCentralityWorker {
- shard: ShardId,
- graph: Arc<Webgraph>,
- num_nodes: u64,
-}
-
-impl ApproxCentralityWorker {
- pub fn new(graph: Webgraph, shard: ShardId) -> Self {
- let num_nodes = graph
- .page_nodes()
- .map(|node| node.as_u128() as u64)
- .collect::<HyperLogLog<2048>>()
- .size() as u64;
- Self {
- graph: Arc::new(graph),
- shard,
- num_nodes,
- }
- }
-
- pub fn graph(&self) -> &Webgraph {
- &self.graph
- }
-
- pub fn shard(&self) -> ShardId {
- self.shard
- }
-
- pub fn num_nodes(&self) -> u64 {
- self.num_nodes
- }
-}
-
-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
-pub struct NumNodes;
-
-impl Message<ApproxCentralityWorker> for NumNodes {
- type Response = u64;
-
- fn handle(self, worker: &ApproxCentralityWorker) -> Self::Response {
- worker.num_nodes
- }
-}
-
-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
-pub struct BatchId2Node(Vec<webgraph::NodeID>);
-
-impl Message<ApproxCentralityWorker> for BatchId2Node {
- type Response = Vec<(webgraph::NodeID, webgraph::Node)>;
-
- fn handle(self, worker: &ApproxCentralityWorker) -> Self::Response {
- self.0
- .iter()
- .filter_map(|id| {
- worker
- .graph
- .search(&webgraph::query::Id2NodeQuery::Page(*id))
- .ok()
- .flatten()
- .map(|node| (*id, node))
- })
- .collect()
- }
-}
-
-impl_worker!(ApproxCentralityJob, RemoteApproxCentralityWorker => ApproxCentralityWorker, [NumNodes, BatchId2Node]);
-
-#[derive(Clone)]
-pub struct RemoteApproxCentralityWorker {
- shard: ShardId,
- pool: Arc<sonic::ConnectionPool<JobConn<ApproxCentralityJob>>>,
-}
-
-impl RemoteApproxCentralityWorker {
- pub fn new(shard: ShardId, addr: SocketAddr) -> Result<Self> {
- Ok(Self {
- shard,
- pool: Arc::new(sonic::ConnectionPool::new(addr)?),
- })
- }
-
- pub fn shard(&self) -> ShardId {
- self.shard
- }
-
- pub fn addr(&self) -> SocketAddr {
- self.pool.addr()
- }
-
- pub fn num_nodes(&self) -> u64 {
- self.send(NumNodes)
- }
-
- pub fn batch_id2node(
- &self,
- ids: Vec<webgraph::NodeID>,
- ) -> Vec<(webgraph::NodeID, webgraph::Node)> {
- self.send(BatchId2Node(ids))
- }
-}
-
-impl RemoteWorker for RemoteApproxCentralityWorker {
- type Job = ApproxCentralityJob;
-
- fn pool(&self) -> &sonic::ConnectionPool<JobConn<Self::Job>> {
- &self.pool
- }
-}
-
-pub fn run(config: ApproxHarmonicWorkerConfig) -> Result<()> {
- tracing::info!("starting worker");
- let tokio_conf = config.clone();
-
- let graph = Webgraph::builder(config.graph_path, config.shard).open()?;
- let worker = ApproxCentralityWorker::new(graph, config.shard);
- let service = Service::ApproxHarmonicWorker {
- host: tokio_conf.host,
- shard: tokio_conf.shard,
- };
- crate::start_gossip_cluster_thread(tokio_conf.gossip, Some(service));
-
- tracing::info!("worker ready");
- worker.run(config.host)?;
-
- Ok(())
-}
diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs
index 9c5d8321..2e0ddc82 100644
--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs
+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs
@@ -17,7 +17,6 @@
use crate::{
ampc::{dht::ShardId, prelude::*, Coordinator, DhtConn},
config::HarmonicCoordinatorConfig,
- distributed::member::Member,
webgraph::centrality::{store_csv, store_harmonic, top_nodes, TopNodes},
Result,
};
@@ -143,8 +142,7 @@ struct ClusterInfo {
}
async fn setup_gossip(config: HarmonicCoordinatorConfig) -> Result<ClusterInfo> {
- let handle = Cluster::join(
- Member::new(Service::HarmonicCoordinator { host: config.host }),
+ let handle = Cluster::join_as_spectator(
config.gossip.addr,
config.gossip.seed_nodes.unwrap_or_default(),
)
@@ -234,5 +232,7 @@ pub fn run(config: HarmonicCoordinatorConfig) -> Result<()> {
store_csv(top_nodes, output_path.join("harmonic.csv"));
+ res.drop_tables();
+
Ok(())
}
diff --git a/crates/core/src/entrypoint/ampc/shortest_path/coordinator.rs b/crates/core/src/entrypoint/ampc/shortest_path/coordinator.rs
index 375ab67e..b9b95094 100644
--- a/crates/core/src/entrypoint/ampc/shortest_path/coordinator.rs
+++ b/crates/core/src/entrypoint/ampc/shortest_path/coordinator.rs
@@ -24,10 +24,10 @@ use url::Url;
use super::mapper::ShortestPathMapper;
use super::{worker::RemoteShortestPathWorker, ShortestPathTables};
use super::{DhtTable as _, Finisher, Meta, Setup, ShortestPathJob};
-use crate::ampc::{Coordinator, DefaultDhtTable, DhtConn};
+use crate::ampc::{Coordinator, DefaultDhtTable, DhtConn, DhtTables};
use crate::config::ShortestPathCoordinatorConfig;
use crate::distributed::cluster::Cluster;
-use crate::distributed::member::{Member, Service, ShardId};
+use crate::distributed::member::{Service, ShardId};
use crate::hyperloglog::HyperLogLog;
use crate::webpage::url_ext::UrlExt;
use crate::{webgraph, Result};
@@ -102,7 +102,7 @@ impl Setup for ShortestPathSetup {
}
pub struct ShortestPathFinish {
- max_distance: Option<u64>,
+ pub max_distance: Option<u64>,
}
impl Finisher for ShortestPathFinish {
@@ -140,8 +140,7 @@ struct ClusterInfo {
}
async fn setup_gossip(config: ShortestPathCoordinatorConfig) -> Result<ClusterInfo> {
- let handle = Cluster::join(
- Member::new(Service::ShortestPathCoordinator { host: config.host }),
+ let handle = Cluster::join_as_spectator(
config.gossip.addr,
config.gossip.seed_nodes.unwrap_or_default(),
)
@@ -260,5 +259,7 @@ pub fn run(config: ShortestPathCoordinatorConfig) -> Result<()> {
}
}
+ res.drop_tables();
+
Ok(())
}
diff --git a/crates/core/src/entrypoint/ampc/shortest_path/mod.rs b/crates/core/src/entrypoint/ampc/shortest_path/mod.rs
index 8ec6604e..3b4a8dab 100644
--- a/crates/core/src/entrypoint/ampc/shortest_path/mod.rs
+++ b/crates/core/src/entrypoint/ampc/shortest_path/mod.rs
@@ -29,8 +29,8 @@ use crate::{
webgraph,
};
-use self::mapper::ShortestPathMapper;
-use self::worker::{RemoteShortestPathWorker, ShortestPathWorker};
+pub use self::mapper::ShortestPathMapper;
+pub use self::worker::{RemoteShortestPathWorker, ShortestPathWorker};
#[derive(
serde::Serialize,
@@ -49,17 +49,17 @@ pub struct Meta {
#[derive(bincode::Encode, bincode::Decode, Debug, Clone)]
pub struct ShortestPathTables {
- distances: DefaultDhtTable<webgraph::NodeID, u64>,
- meta: DefaultDhtTable<(), Meta>,
- changed_nodes: DefaultDhtTable<ShardId, UpdatedNodes>,
+ pub distances: DefaultDhtTable<webgraph::NodeID, u64>,
+ pub meta: DefaultDhtTable<(), Meta>,
+ pub changed_nodes: DefaultDhtTable<ShardId, UpdatedNodes>,
}
impl_dht_tables!(ShortestPathTables, [distances, meta, changed_nodes]);
#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
pub struct ShortestPathJob {
- shard: ShardId,
- source: webgraph::NodeID,
+ pub shard: ShardId,
+ pub source: webgraph::NodeID,
}
impl Job for ShortestPathJob {
diff --git a/crates/core/src/entrypoint/ampc/shortest_path/worker.rs b/crates/core/src/entrypoint/ampc/shortest_path/worker.rs
index b202d392..3fdfc5e8 100644
--- a/crates/core/src/entrypoint/ampc/shortest_path/worker.rs
+++ b/crates/core/src/entrypoint/ampc/shortest_path/worker.rs
@@ -41,7 +41,7 @@ pub struct ShortestPathWorker {
}
impl ShortestPathWorker {
- pub fn new(graph: Webgraph, shard: ShardId) -> Self {
+ pub fn new(graph: Arc<Webgraph>, shard: ShardId) -> Self {
let mut nodes_sketch = HyperLogLog::default();
for node in graph.page_nodes() {
@@ -50,7 +50,7 @@ impl ShortestPathWorker {
let num_nodes = nodes_sketch.size() as u64;
Self {
- graph: Arc::new(graph),
+ graph,
shard,
changed_nodes: Arc::new(Mutex::new(UpdatedNodes::new(num_nodes))),
nodes_sketch,
@@ -101,6 +101,19 @@ impl Message<ShortestPathWorker> for UpdateChangedNodesPrecision {
}
}
+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
+pub struct SampleNodes(u64);
+
+impl Message<ShortestPathWorker> for SampleNodes {
+ type Response = Vec<webgraph::NodeID>;
+
+ fn handle(self, worker: &ShortestPathWorker) -> Self::Response {
+ worker
+ .graph()
+ .random_page_nodes_with_outgoing(self.0 as usize)
+ }
+}
+
#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
pub struct BatchId2Node(Vec<webgraph::NodeID>);
@@ -122,7 +135,7 @@ impl Message<ShortestPathWorker> for BatchId2Node {
}
}
-impl_worker!(ShortestPathJob, RemoteShortestPathWorker => ShortestPathWorker, [BatchId2Node, GetNodeSketch, UpdateChangedNodesPrecision]);
+impl_worker!(ShortestPathJob, RemoteShortestPathWorker => ShortestPathWorker, [BatchId2Node, GetNodeSketch, UpdateChangedNodesPrecision, SampleNodes]);
#[derive(Clone)]
pub struct RemoteShortestPathWorker {
@@ -160,6 +173,10 @@ impl RemoteShortestPathWorker {
pub fn update_changed_nodes_precision(&self, num_nodes: u64) {
self.send(UpdateChangedNodesPrecision(num_nodes));
}
+
+ pub fn sample_nodes(&self, num_nodes: u64) -> Vec<webgraph::NodeID> {
+ self.send(SampleNodes(num_nodes))
+ }
}
impl RemoteWorker for RemoteShortestPathWorker {
@@ -174,7 +191,7 @@ pub fn run(config: ShortestPathWorkerConfig) -> Result<()> {
tracing::info!("starting worker");
let tokio_conf = config.clone();
- let graph = Webgraph::builder(config.graph_path, config.shard).open()?;
+ let graph = Arc::new(Webgraph::builder(config.graph_path, config.shard).open()?);
let worker = ShortestPathWorker::new(graph, config.shard);
let service = Service::ShortestPathWorker {
host: tokio_conf.host,
diff --git a/crates/core/src/kahan_sum.rs b/crates/core/src/kahan_sum.rs
index e40119ae..9146e9a5 100644
--- a/crates/core/src/kahan_sum.rs
+++ b/crates/core/src/kahan_sum.rs
@@ -62,6 +62,24 @@ impl Add<f64> for KahanSum {
}
}
+impl AddAssign<KahanSum> for KahanSum {
+ fn add_assign(&mut self, rhs: KahanSum) {
+ let y = (rhs.sum + rhs.err) - self.err;
+ let t = self.sum + y;
+ self.err = (t - self.sum) - y;
+ self.sum = t;
+ }
+}
+
+impl Add<KahanSum> for KahanSum {
+ type Output = Self;
+ fn add(self, rhs: KahanSum) -> Self::Output {
+ let mut k = self;
+ k += rhs;
+ k
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -85,4 +103,24 @@ mod tests {
assert_eq!(10017.579623446147f64, f64::from(sum));
}
+
+ #[test]
+ fn it_works_kahan() {
+ let mut sum = KahanSum::default();
+ assert_eq!(0.0, f64::from(sum));
+
+ for elem in [
+ 10000.0f64,
+ std::f64::consts::PI,
+ std::f64::consts::E,
+ std::f64::consts::PI,
+ std::f64::consts::E,
+ std::f64::consts::PI,
+ std::f64::consts::E,
+ ] {
+ sum += KahanSum::from(elem);
+ }
+
+ assert_eq!(10017.579623446147f64, f64::from(sum));
+ }
}
diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs
index 4264d440..054fd0db 100644
--- a/crates/core/src/main.rs
+++ b/crates/core/src/main.rs
@@ -134,13 +134,8 @@ enum AmpcOptions {
/// Workers needs to be started before the coordinator.
HarmonicCoordinator { config_path: String },
- /// Start a worker to compute an approximation of the harmonic centrality of a graph.
- /// The approximation samples O(log n / sample_rate^2) nodes from the graph and computes
- /// shortest paths from the sampled nodes.
- ApproxHarmonicWorker { config_path: String },
-
/// Start a coordinator to distribute the approximation of the harmonic centrality computation.
- /// Workers needs to be started before the coordinator.
+ /// Uses shortest path workers. Workers needs to be started before the coordinator.
ApproxHarmonicCoordinator { config_path: String },
/// Start a worker to compute the shortest paths of a graph.
@@ -469,27 +464,27 @@ fn main() -> Result<()> {
.build()?
.block_on(entrypoint::ampc::dht::run(config))?;
}
+
AmpcOptions::HarmonicWorker { config_path } => {
let config: config::HarmonicWorkerConfig = load_toml_config(config_path);
entrypoint::ampc::harmonic_centrality::worker::run(config)?;
}
+
AmpcOptions::HarmonicCoordinator { config_path } => {
let config: config::HarmonicCoordinatorConfig = load_toml_config(config_path);
entrypoint::ampc::harmonic_centrality::coordinator::run(config)?;
}
- AmpcOptions::ApproxHarmonicWorker { config_path } => {
- let config: config::ApproxHarmonicWorkerConfig = load_toml_config(config_path);
- entrypoint::ampc::approximated_harmonic_centrality::worker::run(config)?;
- }
AmpcOptions::ApproxHarmonicCoordinator { config_path } => {
let config: config::ApproxHarmonicCoordinatorConfig = load_toml_config(config_path);
entrypoint::ampc::approximated_harmonic_centrality::coordinator::run(config)?;
}
+
AmpcOptions::ShortestPathWorker { config_path } => {
let config: config::ShortestPathWorkerConfig = load_toml_config(config_path);
entrypoint::ampc::shortest_path::worker::run(config)?;
}
+
AmpcOptions::ShortestPathCoordinator { config_path } => {
let config: config::ShortestPathCoordinatorConfig = load_toml_config(config_path);
entrypoint::ampc::shortest_path::coordinator::run(config)?;