web search done right
bellman-ford inspired shortest path for distributed graph. works better than approach in approx centrality when graph is sharded. still need to implement some low-hanging fruits for optimisation
Mikkel Denker49abc7419fcb
diff --git a/configs/shortest_paths/coordinator.toml b/configs/shortest_paths/coordinator.tomlnew file mode 100644index 00000000..a04587cb--- /dev/null+++ b/configs/shortest_paths/coordinator.toml@@ -0,0 +1,7 @@+source = "https://www.cdc.gov/healthywater/swimming/"+host = "0.0.0.0:5000"+output_path = "data/shortest_paths"++[gossip]+addr = "0.0.0.0:5001"+seed_nodes = ["0.0.0.0:3102"]diff --git a/configs/shortest_paths/dht.toml b/configs/shortest_paths/dht.tomlnew file mode 100644index 00000000..48a39997--- /dev/null+++ b/configs/shortest_paths/dht.toml@@ -0,0 +1,7 @@+node_id = 0+shard = 0+host = "0.0.0.0:3101"++[gossip]+addr = "0.0.0.0:3102"+seed_nodes = []diff --git a/configs/shortest_paths/worker.toml b/configs/shortest_paths/worker.tomlnew file mode 100644index 00000000..c61f33fa--- /dev/null+++ b/configs/shortest_paths/worker.toml@@ -0,0 +1,7 @@+shard = 0+graph_path = "data/webgraph"+host = "0.0.0.0:5002"++[gossip]+addr = "0.0.0.0:5003"+seed_nodes = ["0.0.0.0:3102"]diff --git a/crates/core/src/ampc/dht/upsert.rs b/crates/core/src/ampc/dht/upsert.rsindex 69864c75..304428c6 100644--- a/crates/core/src/ampc/dht/upsert.rs+++ b/crates/core/src/ampc/dht/upsert.rs@@ -27,6 +27,12 @@ pub enum UpsertAction {Inserted,}+impl UpsertAction {+ pub fn is_changed(&self) -> bool {+ matches!(self, UpsertAction::Merged | UpsertAction::Inserted)+ }+}+#[enum_dispatch]pub trait UpsertFn {fn upsert(&self, old: Value, new: Value) -> Value;@@ -41,6 +47,7 @@ pub enum UpsertEnum {HyperLogLog64Upsert,HyperLogLog128Upsert,U64Add,+ U64Min,F32Add,F64Add,KahanSumAdd,@@ -94,6 +101,18 @@ impl UpsertFn for U64Add {}}+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct U64Min;++impl UpsertFn for U64Min {+ fn upsert(&self, old: Value, new: Value) -> Value {+ let old = unwrap_value!(old, U64);+ let new = unwrap_value!(new, U64);++ Value::U64(old.min(new))+ }+}+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct F32Add;diff --git a/crates/core/src/ampc/dht/value.rs b/crates/core/src/ampc/dht/value.rsindex 6736f197..311ff3d1 100644--- a/crates/core/src/ampc/dht/value.rs+++ b/crates/core/src/ampc/dht/value.rs@@ -50,6 +50,9 @@ impl ValueTrait for HyperLogLog128 {}type HarmonicMeta = crate::entrypoint::ampc::harmonic_centrality::Meta;impl ValueTrait for HarmonicMeta {}+type ShortestPathMeta = crate::entrypoint::ampc::shortest_path::Meta;+impl ValueTrait for ShortestPathMeta {}+impl ValueTrait for U64BloomFilter {}type Unit = ();@@ -71,6 +74,7 @@ pub enum Value {HyperLogLog64(HyperLogLog64),HyperLogLog128(HyperLogLog128),HarmonicMeta(HarmonicMeta),+ ShortestPathMeta(ShortestPathMeta),U64BloomFilter(U64BloomFilter),Unit(Unit),}@@ -108,5 +112,6 @@ impl_from_to_value!(HyperLogLog32, HyperLogLog32);impl_from_to_value!(HyperLogLog64, HyperLogLog64);impl_from_to_value!(HyperLogLog128, HyperLogLog128);impl_from_to_value!(HarmonicMeta, HarmonicMeta);+impl_from_to_value!(ShortestPathMeta, ShortestPathMeta);impl_from_to_value!(U64BloomFilter, U64BloomFilter);impl_from_to_value!(Unit, Unit);diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 8de655f8..cd881c9a 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -680,6 +680,14 @@ pub struct HarmonicCoordinatorConfig {pub output_path: String,}+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]+pub struct ShortestPathCoordinatorConfig {+ pub source: String,+ pub gossip: GossipConfig,+ pub host: SocketAddr,+ pub output_path: String,+}+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]pub struct HarmonicWorkerConfig {pub gossip: GossipConfig,@@ -688,6 +696,14 @@ pub struct HarmonicWorkerConfig {pub host: SocketAddr,}+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]+pub struct ShortestPathWorkerConfig {+ pub gossip: GossipConfig,+ pub shard: ShardId,+ pub graph_path: String,+ pub host: SocketAddr,+}+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]pub struct ApproxHarmonicCoordinatorConfig {pub gossip: GossipConfig,diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rsindex 894ee8d4..38513301 100644--- a/crates/core/src/distributed/member.rs+++ b/crates/core/src/distributed/member.rs@@ -135,6 +135,13 @@ pub enum Service {ApproxHarmonicCoordinator {host: SocketAddr,},+ ShortestPathWorker {+ host: SocketAddr,+ shard: ShardId,+ },+ ShortestPathCoordinator {+ host: SocketAddr,+ },}impl std::fmt::Display for Service {@@ -163,6 +170,12 @@ impl std::fmt::Display for Service {Self::ApproxHarmonicCoordinator { host } => {write!(f, "ApproxHarmonicCoordinator {}", host)}+ Self::ShortestPathWorker { host, shard } => {+ write!(f, "ShortestPathWorker {} {}", host, shard)+ }+ Self::ShortestPathCoordinator { host } => {+ write!(f, "ShortestPathCoordinator {}", host)+ }}}}diff --git a/crates/core/src/entrypoint/ampc/mod.rs b/crates/core/src/entrypoint/ampc/mod.rsindex ebfc0ade..f34918ec 100644--- a/crates/core/src/entrypoint/ampc/mod.rs+++ b/crates/core/src/entrypoint/ampc/mod.rs@@ -17,3 +17,4 @@pub mod approximated_harmonic_centrality;pub mod dht;pub mod harmonic_centrality;+pub mod shortest_path;diff --git a/crates/core/src/entrypoint/ampc/shortest_path/coordinator.rs b/crates/core/src/entrypoint/ampc/shortest_path/coordinator.rsnew file mode 100644index 00000000..43da0e2c--- /dev/null+++ b/crates/core/src/entrypoint/ampc/shortest_path/coordinator.rs@@ -0,0 +1,230 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use std::collections::BTreeMap;+use std::net::SocketAddr;+use std::path::Path;++use itertools::Itertools;+use url::Url;++use super::mapper::ShortestPathMapper;+use super::{worker::RemoteShortestPathWorker, ShortestPathTables};+use super::{DhtTable as _, Finisher, Meta, Setup, ShortestPathJob};+use crate::ampc::{Coordinator, DefaultDhtTable, DhtConn};+use crate::config::ShortestPathCoordinatorConfig;+use crate::distributed::cluster::Cluster;+use crate::distributed::member::{Member, Service, ShardId};+use crate::webpage::url_ext::UrlExt;+use crate::{webgraph, Result};++pub struct ShortestPathSetup {+ dht: DhtConn<ShortestPathTables>,+ source: webgraph::NodeID,+}++impl ShortestPathSetup {+ pub async fn new(cluster: &Cluster, source: webgraph::NodeID) -> Result<Self> {+ let dht_members: Vec<_> = cluster+ .members()+ .await+ .into_iter()+ .filter_map(|member| {+ if let Service::Dht { host, shard } = member.service {+ Some((shard, host))+ } else {+ None+ }+ })+ .collect();++ Ok(Self::new_for_dht_members(&dht_members, source))+ }+ pub fn new_for_dht_members(+ dht_members: &[(ShardId, SocketAddr)],+ source: webgraph::NodeID,+ ) -> Self {+ let initial = ShortestPathTables {+ distances: DefaultDhtTable::new(dht_members, "distances"),+ meta: DefaultDhtTable::new(dht_members, "meta"),+ changed_nodes: DefaultDhtTable::new(dht_members, "changed_nodes"),+ };++ let dht = DhtConn::new(initial);++ Self { dht, source }+ }+}++impl Setup for ShortestPathSetup {+ type DhtTables = ShortestPathTables;++ fn init_dht(&self) -> DhtConn<Self::DhtTables> {+ self.dht.clone()+ }++ fn setup_round(&self, dht: &Self::DhtTables) {+ dht.meta.set(+ (),+ Meta {+ round_had_changes: false,+ },+ );+ }++ fn setup_first_round(&self, dht: &Self::DhtTables) {+ dht.distances.set(self.source, 0);+ dht.meta.set(+ (),+ Meta {+ round_had_changes: true,+ },+ );+ }+}++pub struct ShortestPathFinish;++impl Finisher for ShortestPathFinish {+ type Job = ShortestPathJob;++ fn is_finished(&self, dht: &ShortestPathTables) -> bool {+ !dht.meta.get(()).unwrap().round_had_changes+ }+}++pub fn build(+ dht: &[(ShardId, SocketAddr)],+ workers: Vec<RemoteShortestPathWorker>,+ source: webgraph::NodeID,+) -> Coordinator<ShortestPathJob> {+ let setup = ShortestPathSetup::new_for_dht_members(dht, source);++ Coordinator::new(setup, workers.clone())+ .with_mapper(ShortestPathMapper::RelaxEdges)+ .with_mapper(ShortestPathMapper::UpdateChangedNodes)+}++struct ClusterInfo {+ // dropping the handle will leave the cluster+ _handle: Cluster,+ dht: Vec<(ShardId, SocketAddr)>,+ workers: Vec<RemoteShortestPathWorker>,+}++async fn setup_gossip(config: ShortestPathCoordinatorConfig) -> Result<ClusterInfo> {+ let handle = Cluster::join(+ Member::new(Service::ShortestPathCoordinator { host: config.host }),+ config.gossip.addr,+ config.gossip.seed_nodes.unwrap_or_default(),+ )+ .await?;++ tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;++ let members = handle.members().await;++ let dht = members+ .iter()+ .filter_map(|member| {+ if let Service::Dht { host, shard } = member.service {+ Some((shard, host))+ } else {+ None+ }+ })+ .collect();++ let workers = members+ .iter()+ .filter_map(|member| {+ if let Service::ShortestPathWorker { host, shard } = member.service {+ Some(RemoteShortestPathWorker::new(shard, host))+ } else {+ None+ }+ })+ .collect::<Result<Vec<RemoteShortestPathWorker>>>()?;++ Ok(ClusterInfo {+ _handle: handle,+ dht,+ workers,+ })+}++pub fn run(config: ShortestPathCoordinatorConfig) -> Result<()> {+ let source = webgraph::Node::from(Url::robust_parse(&config.source)?).id();+ let tokio_conf = config.clone();+ let cluster = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()?+ .block_on(setup_gossip(tokio_conf))?;++ let jobs: Vec<_> = cluster+ .workers+ .iter()+ .map(|worker| ShortestPathJob {+ shard: worker.shard(),+ source,+ })+ .collect();++ tracing::info!("starting {} jobs", jobs.len());++ let coordinator = build(&cluster.dht, cluster.workers.clone(), source);+ let res = coordinator.run(jobs, ShortestPathFinish)?;++ let output_path = Path::new(&config.output_path);++ if !output_path.exists() {+ std::fs::create_dir_all(output_path)?;+ }++ let mut writer = csv::Writer::from_writer(+ std::fs::OpenOptions::new()+ .create(true)+ .write(true)+ .truncate(true)+ .open(output_path.join("distances.csv"))?,+ );+ let mut distances = res.distances.iter().collect::<Vec<_>>();+ distances.sort_by_key(|(_id, distance)| *distance);++ let id2node: BTreeMap<_, _> = cluster+ .workers+ .iter()+ .flat_map(|w| {+ distances+ .iter()+ .chunks(10_000)+ .into_iter()+ .flat_map(move |c| {+ let ids = c.map(|(id, _)| *id).collect::<Vec<_>>();+ w.batch_id2node(ids)+ })+ .collect::<Vec<_>>()+ })+ .collect();++ for (id, distance) in distances {+ if let Some(node) = id2node.get(&id) {+ writer.write_record(&[node.as_str().to_string(), distance.to_string()])?;+ }+ }++ Ok(())+}diff --git a/crates/core/src/entrypoint/ampc/shortest_path/mapper.rs b/crates/core/src/entrypoint/ampc/shortest_path/mapper.rsnew file mode 100644index 00000000..8e030edd--- /dev/null+++ b/crates/core/src/entrypoint/ampc/shortest_path/mapper.rs@@ -0,0 +1,182 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use std::sync::{atomic::AtomicBool, Arc, Mutex};++use bloom::U64BloomFilter;+use rustc_hash::FxHashMap;++use super::{DhtTable as _, Mapper, Meta, ShortestPathJob, ShortestPathTables};+use crate::{+ ampc::{+ dht::{U64Min, UpsertAction},+ DhtConn,+ },+ webgraph,+ webpage::html::links::RelFlags,+};++const BATCH_SIZE: usize = 4096;++pub static SKIPPED_REL: std::sync::LazyLock<RelFlags> = std::sync::LazyLock::new(|| {+ RelFlags::TAG+ | RelFlags::NOFOLLOW+ | RelFlags::IS_IN_FOOTER+ | RelFlags::IS_IN_NAVIGATION+ | RelFlags::PRIVACY_POLICY+ | RelFlags::TERMS_OF_SERVICE+ | RelFlags::SEARCH+ | RelFlags::LINK_TAG+ | RelFlags::SCRIPT_TAG+ | RelFlags::UGC+});++#[derive(Debug, Clone, bincode::Decode, bincode::Encode)]+pub enum ShortestPathMapper {+ RelaxEdges,+ UpdateChangedNodes,+}++impl ShortestPathMapper {+ fn get_old_distances(+ batch: Vec<webgraph::NodeID>,+ dht: &DhtConn<ShortestPathTables>,+ ) -> FxHashMap<webgraph::NodeID, u64> {+ dht.prev().distances.batch_get(batch).into_iter().collect()+ }++ fn update_distances(+ batch: &[webgraph::SmallEdge],+ dht: &DhtConn<ShortestPathTables>,+ ) -> Vec<(webgraph::NodeID, UpsertAction)> {+ let old_distances = Self::get_old_distances(batch.iter().map(|e| e.from).collect(), dht);++ let mut new_distances = FxHashMap::default();++ for edge in batch {+ if let Some(old_distance) = old_distances.get(&edge.from) {+ let new_distance = old_distance + 1;+ let old_new = new_distances.entry(edge.to).or_insert(new_distance);++ if new_distance < *old_new {+ *old_new = new_distance;+ }+ }+ }++ dht.next()+ .distances+ .batch_upsert(U64Min, new_distances.into_iter().collect())+ }++ fn map_batch(+ batch: &[webgraph::SmallEdge],+ new_changed_nodes: &Mutex<U64BloomFilter>,+ round_had_changes: &AtomicBool,+ dht: &DhtConn<ShortestPathTables>,+ ) {+ let updates = Self::update_distances(batch, dht);+ let mut new_changed_nodes = new_changed_nodes.lock().unwrap();++ for (node, action) in updates {+ if action.is_changed() {+ new_changed_nodes.insert_u128(node.as_u128());+ round_had_changes.store(true, std::sync::atomic::Ordering::Relaxed);+ }+ }+ }+}++impl Mapper for ShortestPathMapper {+ type Job = ShortestPathJob;++ fn map(+ &self,+ job: Self::Job,+ worker: &<<Self as Mapper>::Job as super::Job>::Worker,+ dht: &crate::ampc::DhtConn<<<Self as Mapper>::Job as super::Job>::DhtTables>,+ ) {+ match self {+ ShortestPathMapper::RelaxEdges => {+ let round_had_changes = Arc::new(AtomicBool::new(false));+ let pool = rayon::ThreadPoolBuilder::new().build().unwrap();++ let new_changed_nodes = Arc::new(Mutex::new(U64BloomFilter::empty_from(+ &worker.changed_nodes().lock().unwrap(),+ )));++ pool.scope(|s| {+ let mut changed_nodes = worker.changed_nodes().lock().unwrap();+ changed_nodes.insert_u128(job.source.as_u128());++ let mut batch = Vec::with_capacity(BATCH_SIZE);++ for edge in worker.graph().page_edges() {+ if edge.rel_flags.intersects(*SKIPPED_REL) {+ continue;+ }++ if changed_nodes.contains_u128(edge.from.as_u128()) {+ batch.push(edge);+ }++ if batch.len() >= BATCH_SIZE {+ let update_batch = batch.clone();+ let update_new_changed_nodes = new_changed_nodes.clone();+ let update_round_had_changes = round_had_changes.clone();+ s.spawn(move |_| {+ Self::map_batch(+ &update_batch,+ &update_new_changed_nodes,+ &update_round_had_changes,+ dht,+ )+ });+ batch.clear();+ }+ }++ if !batch.is_empty() {+ Self::map_batch(&batch, &new_changed_nodes, &round_had_changes, dht);+ }+ });++ dht.next()+ .changed_nodes+ .set(worker.shard(), new_changed_nodes.lock().unwrap().clone());+ dht.next().meta.set(+ (),+ Meta {+ round_had_changes: round_had_changes+ .load(std::sync::atomic::Ordering::Relaxed),+ },+ );+ }+ ShortestPathMapper::UpdateChangedNodes => {+ let all_changed_nodes: Vec<_> =+ dht.next().changed_nodes.iter().map(|(_, v)| v).collect();+ let mut changed_nodes =+ U64BloomFilter::empty_from(&worker.changed_nodes().lock().unwrap());++ for bloom in all_changed_nodes {+ changed_nodes.union(bloom.clone());+ }++ *worker.changed_nodes().lock().unwrap() = changed_nodes;+ }+ }+ }+}diff --git a/crates/core/src/entrypoint/ampc/shortest_path/mod.rs b/crates/core/src/entrypoint/ampc/shortest_path/mod.rsnew file mode 100644index 00000000..8f345b49--- /dev/null+++ b/crates/core/src/entrypoint/ampc/shortest_path/mod.rs@@ -0,0 +1,71 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++//! Single-source shortest path.++pub mod coordinator;+mod mapper;+pub mod worker;++use bloom::U64BloomFilter;++use crate::distributed::member::ShardId;+use crate::{+ ampc::{prelude::*, DefaultDhtTable},+ webgraph,+};++use self::mapper::ShortestPathMapper;+use self::worker::{RemoteShortestPathWorker, ShortestPathWorker};++#[derive(+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Debug,+ Clone,+ PartialEq,+ Eq,+)]+pub struct Meta {+ round_had_changes: bool,+}++#[derive(bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct ShortestPathTables {+ distances: DefaultDhtTable<webgraph::NodeID, u64>,+ meta: DefaultDhtTable<(), Meta>,+ changed_nodes: DefaultDhtTable<ShardId, U64BloomFilter>,+}++impl_dht_tables!(ShortestPathTables, [distances, meta, changed_nodes]);++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct ShortestPathJob {+ shard: ShardId,+ source: webgraph::NodeID,+}++impl Job for ShortestPathJob {+ type DhtTables = ShortestPathTables;+ type Worker = ShortestPathWorker;+ type Mapper = ShortestPathMapper;++ fn is_schedulable(&self, worker: &RemoteShortestPathWorker) -> bool {+ self.shard == worker.shard()+ }+}diff --git a/crates/core/src/entrypoint/ampc/shortest_path/worker.rs b/crates/core/src/entrypoint/ampc/shortest_path/worker.rsnew file mode 100644index 00000000..e097b474--- /dev/null+++ b/crates/core/src/entrypoint/ampc/shortest_path/worker.rs@@ -0,0 +1,172 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use crate::{+ ampc::{JobConn, Worker},+ config::ShortestPathWorkerConfig,+ distributed::{+ member::{Service, ShardId},+ sonic,+ },+ hyperloglog::HyperLogLog,+ webgraph::{self, Webgraph},+ Result,+};+use std::{+ net::SocketAddr,+ sync::{Arc, Mutex},+};++use super::{impl_worker, Message, RemoteWorker, ShortestPathJob};+use bloom::U64BloomFilter;++#[derive(Clone)]+pub struct ShortestPathWorker {+ shard: ShardId,+ graph: Arc<Webgraph>,+ changed_nodes: Arc<Mutex<U64BloomFilter>>,+ nodes_sketch: HyperLogLog<4096>,+}++impl ShortestPathWorker {+ pub fn new(graph: Webgraph, shard: ShardId) -> Self {+ let mut nodes_sketch = HyperLogLog::default();++ for node in graph.page_nodes() {+ nodes_sketch.add_u128(node.as_u128());+ }++ Self {+ graph: Arc::new(graph),+ shard,+ changed_nodes: Arc::new(Mutex::new(U64BloomFilter::new(+ nodes_sketch.size() as u64,+ 0.01,+ ))),+ nodes_sketch,+ }+ }++ pub fn graph(&self) -> &Webgraph {+ &self.graph+ }++ pub fn shard(&self) -> ShardId {+ self.shard+ }++ pub fn changed_nodes(&self) -> &Arc<Mutex<U64BloomFilter>> {+ &self.changed_nodes+ }++ pub fn nodes_sketch(&self) -> &HyperLogLog<4096> {+ &self.nodes_sketch+ }+}++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct GetNodeSketch;++impl Message<ShortestPathWorker> for GetNodeSketch {+ type Response = HyperLogLog<4096>;++ fn handle(self, worker: &ShortestPathWorker) -> Self::Response {+ worker.nodes_sketch().clone()+ }+}++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct BatchId2Node(Vec<webgraph::NodeID>);++impl Message<ShortestPathWorker> for BatchId2Node {+ type Response = Vec<(webgraph::NodeID, webgraph::Node)>;++ fn handle(self, worker: &ShortestPathWorker) -> Self::Response {+ self.0+ .iter()+ .filter_map(|id| {+ worker+ .graph+ .search(&webgraph::query::Id2NodeQuery::Page(*id))+ .ok()+ .flatten()+ .map(|node| (*id, node))+ })+ .collect()+ }+}++impl_worker!(ShortestPathJob, RemoteShortestPathWorker => ShortestPathWorker, [BatchId2Node, GetNodeSketch]);++#[derive(Clone)]+pub struct RemoteShortestPathWorker {+ shard: ShardId,+ pool: Arc<sonic::ConnectionPool<JobConn<ShortestPathJob>>>,+}++impl RemoteShortestPathWorker {+ pub fn new(shard: ShardId, addr: SocketAddr) -> Result<Self> {+ Ok(Self {+ shard,+ pool: Arc::new(sonic::ConnectionPool::new(addr)?),+ })+ }++ pub fn shard(&self) -> ShardId {+ self.shard+ }++ pub fn addr(&self) -> SocketAddr {+ self.pool.addr()+ }++ pub fn batch_id2node(+ &self,+ ids: Vec<webgraph::NodeID>,+ ) -> Vec<(webgraph::NodeID, webgraph::Node)> {+ self.send(BatchId2Node(ids))+ }++ pub fn get_node_sketch(&self) -> HyperLogLog<4096> {+ self.send(GetNodeSketch)+ }+}++impl RemoteWorker for RemoteShortestPathWorker {+ type Job = ShortestPathJob;++ fn pool(&self) -> &sonic::ConnectionPool<JobConn<Self::Job>> {+ &self.pool+ }+}++pub fn run(config: ShortestPathWorkerConfig) -> Result<()> {+ tracing::info!("starting worker");+ let tokio_conf = config.clone();++ let graph = Webgraph::builder(config.graph_path, config.shard).open()?;+ let worker = ShortestPathWorker::new(graph, config.shard);+ let service = Service::ShortestPathWorker {+ host: tokio_conf.host,+ shard: tokio_conf.shard,+ };+ crate::start_gossip_cluster_thread(tokio_conf.gossip, Some(service));++ tracing::info!("worker ready");+ worker.run(config.host)?;++ Ok(())+}diff --git a/crates/core/src/main.rs b/crates/core/src/main.rsindex f636ab0d..4264d440 100644--- a/crates/core/src/main.rs+++ b/crates/core/src/main.rs@@ -13,6 +13,7 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use anyhow::{Context, Result};use clap::{Parser, Subcommand};use serde::de::DeserializeOwned;@@ -141,6 +142,13 @@ enum AmpcOptions {/// Start a coordinator to distribute the approximation of the harmonic centrality computation./// Workers needs to be started before the coordinator.ApproxHarmonicCoordinator { config_path: String },++ /// Start a worker to compute the shortest paths of a graph.+ ShortestPathWorker { config_path: String },++ /// Start a coordinator to distribute the shortest path computation.+ /// Workers needs to be started before the coordinator.+ ShortestPathCoordinator { config_path: String },}#[derive(Subcommand)]@@ -478,6 +486,14 @@ fn main() -> Result<()> {let config: config::ApproxHarmonicCoordinatorConfig = load_toml_config(config_path);entrypoint::ampc::approximated_harmonic_centrality::coordinator::run(config)?;}+ AmpcOptions::ShortestPathWorker { config_path } => {+ let config: config::ShortestPathWorkerConfig = load_toml_config(config_path);+ entrypoint::ampc::shortest_path::worker::run(config)?;+ }+ AmpcOptions::ShortestPathCoordinator { config_path } => {+ let config: config::ShortestPathCoordinatorConfig = load_toml_config(config_path);+ entrypoint::ampc::shortest_path::coordinator::run(config)?;+ }},Commands::Admin { options } => match options {