Skip to content

web search done right

bellman-ford inspired shortest path for distributed graph. works better than approach in approx centrality when graph is sharded. still need to implement some low-hanging fruits for optimisation

Mikkel Denker49abc7419fcb
diff --git a/configs/shortest_paths/coordinator.toml b/configs/shortest_paths/coordinator.toml
new file mode 100644
index 00000000..a04587cb
--- /dev/null
+++ b/configs/shortest_paths/coordinator.toml
@@ -0,0 +1,7 @@
+source = "https://www.cdc.gov/healthywater/swimming/"
+host = "0.0.0.0:5000"
+output_path = "data/shortest_paths"
+
+[gossip]
+addr = "0.0.0.0:5001"
+seed_nodes = ["0.0.0.0:3102"]
diff --git a/configs/shortest_paths/dht.toml b/configs/shortest_paths/dht.toml
new file mode 100644
index 00000000..48a39997
--- /dev/null
+++ b/configs/shortest_paths/dht.toml
@@ -0,0 +1,7 @@
+node_id = 0
+shard = 0
+host = "0.0.0.0:3101"
+
+[gossip]
+addr = "0.0.0.0:3102"
+seed_nodes = []
diff --git a/configs/shortest_paths/worker.toml b/configs/shortest_paths/worker.toml
new file mode 100644
index 00000000..c61f33fa
--- /dev/null
+++ b/configs/shortest_paths/worker.toml
@@ -0,0 +1,7 @@
+shard = 0
+graph_path = "data/webgraph"
+host = "0.0.0.0:5002"
+
+[gossip]
+addr = "0.0.0.0:5003"
+seed_nodes = ["0.0.0.0:3102"]
diff --git a/crates/core/src/ampc/dht/upsert.rs b/crates/core/src/ampc/dht/upsert.rs
index 69864c75..304428c6 100644
--- a/crates/core/src/ampc/dht/upsert.rs
+++ b/crates/core/src/ampc/dht/upsert.rs
@@ -27,6 +27,12 @@ pub enum UpsertAction {
Inserted,
}
+impl UpsertAction {
+ pub fn is_changed(&self) -> bool {
+ matches!(self, UpsertAction::Merged | UpsertAction::Inserted)
+ }
+}
+
#[enum_dispatch]
pub trait UpsertFn {
fn upsert(&self, old: Value, new: Value) -> Value;
@@ -41,6 +47,7 @@ pub enum UpsertEnum {
HyperLogLog64Upsert,
HyperLogLog128Upsert,
U64Add,
+ U64Min,
F32Add,
F64Add,
KahanSumAdd,
@@ -94,6 +101,18 @@ impl UpsertFn for U64Add {
}
}
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
+pub struct U64Min;
+
+impl UpsertFn for U64Min {
+ fn upsert(&self, old: Value, new: Value) -> Value {
+ let old = unwrap_value!(old, U64);
+ let new = unwrap_value!(new, U64);
+
+ Value::U64(old.min(new))
+ }
+}
+
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct F32Add;
diff --git a/crates/core/src/ampc/dht/value.rs b/crates/core/src/ampc/dht/value.rs
index 6736f197..311ff3d1 100644
--- a/crates/core/src/ampc/dht/value.rs
+++ b/crates/core/src/ampc/dht/value.rs
@@ -50,6 +50,9 @@ impl ValueTrait for HyperLogLog128 {}
type HarmonicMeta = crate::entrypoint::ampc::harmonic_centrality::Meta;
impl ValueTrait for HarmonicMeta {}
+type ShortestPathMeta = crate::entrypoint::ampc::shortest_path::Meta;
+impl ValueTrait for ShortestPathMeta {}
+
impl ValueTrait for U64BloomFilter {}
type Unit = ();
@@ -71,6 +74,7 @@ pub enum Value {
HyperLogLog64(HyperLogLog64),
HyperLogLog128(HyperLogLog128),
HarmonicMeta(HarmonicMeta),
+ ShortestPathMeta(ShortestPathMeta),
U64BloomFilter(U64BloomFilter),
Unit(Unit),
}
@@ -108,5 +112,6 @@ impl_from_to_value!(HyperLogLog32, HyperLogLog32);
impl_from_to_value!(HyperLogLog64, HyperLogLog64);
impl_from_to_value!(HyperLogLog128, HyperLogLog128);
impl_from_to_value!(HarmonicMeta, HarmonicMeta);
+impl_from_to_value!(ShortestPathMeta, ShortestPathMeta);
impl_from_to_value!(U64BloomFilter, U64BloomFilter);
impl_from_to_value!(Unit, Unit);
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 8de655f8..cd881c9a 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -680,6 +680,14 @@ pub struct HarmonicCoordinatorConfig {
pub output_path: String,
}
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub struct ShortestPathCoordinatorConfig {
+ pub source: String,
+ pub gossip: GossipConfig,
+ pub host: SocketAddr,
+ pub output_path: String,
+}
+
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub struct HarmonicWorkerConfig {
pub gossip: GossipConfig,
@@ -688,6 +696,14 @@ pub struct HarmonicWorkerConfig {
pub host: SocketAddr,
}
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub struct ShortestPathWorkerConfig {
+ pub gossip: GossipConfig,
+ pub shard: ShardId,
+ pub graph_path: String,
+ pub host: SocketAddr,
+}
+
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub struct ApproxHarmonicCoordinatorConfig {
pub gossip: GossipConfig,
diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rs
index 894ee8d4..38513301 100644
--- a/crates/core/src/distributed/member.rs
+++ b/crates/core/src/distributed/member.rs
@@ -135,6 +135,13 @@ pub enum Service {
ApproxHarmonicCoordinator {
host: SocketAddr,
},
+ ShortestPathWorker {
+ host: SocketAddr,
+ shard: ShardId,
+ },
+ ShortestPathCoordinator {
+ host: SocketAddr,
+ },
}
impl std::fmt::Display for Service {
@@ -163,6 +170,12 @@ impl std::fmt::Display for Service {
Self::ApproxHarmonicCoordinator { host } => {
write!(f, "ApproxHarmonicCoordinator {}", host)
}
+ Self::ShortestPathWorker { host, shard } => {
+ write!(f, "ShortestPathWorker {} {}", host, shard)
+ }
+ Self::ShortestPathCoordinator { host } => {
+ write!(f, "ShortestPathCoordinator {}", host)
+ }
}
}
}
diff --git a/crates/core/src/entrypoint/ampc/mod.rs b/crates/core/src/entrypoint/ampc/mod.rs
index ebfc0ade..f34918ec 100644
--- a/crates/core/src/entrypoint/ampc/mod.rs
+++ b/crates/core/src/entrypoint/ampc/mod.rs
@@ -17,3 +17,4 @@
pub mod approximated_harmonic_centrality;
pub mod dht;
pub mod harmonic_centrality;
+pub mod shortest_path;
diff --git a/crates/core/src/entrypoint/ampc/shortest_path/coordinator.rs b/crates/core/src/entrypoint/ampc/shortest_path/coordinator.rs
new file mode 100644
index 00000000..43da0e2c
--- /dev/null
+++ b/crates/core/src/entrypoint/ampc/shortest_path/coordinator.rs
@@ -0,0 +1,230 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+use std::collections::BTreeMap;
+use std::net::SocketAddr;
+use std::path::Path;
+
+use itertools::Itertools;
+use url::Url;
+
+use super::mapper::ShortestPathMapper;
+use super::{worker::RemoteShortestPathWorker, ShortestPathTables};
+use super::{DhtTable as _, Finisher, Meta, Setup, ShortestPathJob};
+use crate::ampc::{Coordinator, DefaultDhtTable, DhtConn};
+use crate::config::ShortestPathCoordinatorConfig;
+use crate::distributed::cluster::Cluster;
+use crate::distributed::member::{Member, Service, ShardId};
+use crate::webpage::url_ext::UrlExt;
+use crate::{webgraph, Result};
+
+pub struct ShortestPathSetup {
+ dht: DhtConn<ShortestPathTables>,
+ source: webgraph::NodeID,
+}
+
+impl ShortestPathSetup {
+ pub async fn new(cluster: &Cluster, source: webgraph::NodeID) -> Result<Self> {
+ let dht_members: Vec<_> = cluster
+ .members()
+ .await
+ .into_iter()
+ .filter_map(|member| {
+ if let Service::Dht { host, shard } = member.service {
+ Some((shard, host))
+ } else {
+ None
+ }
+ })
+ .collect();
+
+ Ok(Self::new_for_dht_members(&dht_members, source))
+ }
+ pub fn new_for_dht_members(
+ dht_members: &[(ShardId, SocketAddr)],
+ source: webgraph::NodeID,
+ ) -> Self {
+ let initial = ShortestPathTables {
+ distances: DefaultDhtTable::new(dht_members, "distances"),
+ meta: DefaultDhtTable::new(dht_members, "meta"),
+ changed_nodes: DefaultDhtTable::new(dht_members, "changed_nodes"),
+ };
+
+ let dht = DhtConn::new(initial);
+
+ Self { dht, source }
+ }
+}
+
+impl Setup for ShortestPathSetup {
+ type DhtTables = ShortestPathTables;
+
+ fn init_dht(&self) -> DhtConn<Self::DhtTables> {
+ self.dht.clone()
+ }
+
+ fn setup_round(&self, dht: &Self::DhtTables) {
+ dht.meta.set(
+ (),
+ Meta {
+ round_had_changes: false,
+ },
+ );
+ }
+
+ fn setup_first_round(&self, dht: &Self::DhtTables) {
+ dht.distances.set(self.source, 0);
+ dht.meta.set(
+ (),
+ Meta {
+ round_had_changes: true,
+ },
+ );
+ }
+}
+
+pub struct ShortestPathFinish;
+
+impl Finisher for ShortestPathFinish {
+ type Job = ShortestPathJob;
+
+ fn is_finished(&self, dht: &ShortestPathTables) -> bool {
+ !dht.meta.get(()).unwrap().round_had_changes
+ }
+}
+
+pub fn build(
+ dht: &[(ShardId, SocketAddr)],
+ workers: Vec<RemoteShortestPathWorker>,
+ source: webgraph::NodeID,
+) -> Coordinator<ShortestPathJob> {
+ let setup = ShortestPathSetup::new_for_dht_members(dht, source);
+
+ Coordinator::new(setup, workers.clone())
+ .with_mapper(ShortestPathMapper::RelaxEdges)
+ .with_mapper(ShortestPathMapper::UpdateChangedNodes)
+}
+
+struct ClusterInfo {
+ // dropping the handle will leave the cluster
+ _handle: Cluster,
+ dht: Vec<(ShardId, SocketAddr)>,
+ workers: Vec<RemoteShortestPathWorker>,
+}
+
+async fn setup_gossip(config: ShortestPathCoordinatorConfig) -> Result<ClusterInfo> {
+ let handle = Cluster::join(
+ Member::new(Service::ShortestPathCoordinator { host: config.host }),
+ config.gossip.addr,
+ config.gossip.seed_nodes.unwrap_or_default(),
+ )
+ .await?;
+
+ tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+
+ let members = handle.members().await;
+
+ let dht = members
+ .iter()
+ .filter_map(|member| {
+ if let Service::Dht { host, shard } = member.service {
+ Some((shard, host))
+ } else {
+ None
+ }
+ })
+ .collect();
+
+ let workers = members
+ .iter()
+ .filter_map(|member| {
+ if let Service::ShortestPathWorker { host, shard } = member.service {
+ Some(RemoteShortestPathWorker::new(shard, host))
+ } else {
+ None
+ }
+ })
+ .collect::<Result<Vec<RemoteShortestPathWorker>>>()?;
+
+ Ok(ClusterInfo {
+ _handle: handle,
+ dht,
+ workers,
+ })
+}
+
+pub fn run(config: ShortestPathCoordinatorConfig) -> Result<()> {
+ let source = webgraph::Node::from(Url::robust_parse(&config.source)?).id();
+ let tokio_conf = config.clone();
+ let cluster = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()?
+ .block_on(setup_gossip(tokio_conf))?;
+
+ let jobs: Vec<_> = cluster
+ .workers
+ .iter()
+ .map(|worker| ShortestPathJob {
+ shard: worker.shard(),
+ source,
+ })
+ .collect();
+
+ tracing::info!("starting {} jobs", jobs.len());
+
+ let coordinator = build(&cluster.dht, cluster.workers.clone(), source);
+ let res = coordinator.run(jobs, ShortestPathFinish)?;
+
+ let output_path = Path::new(&config.output_path);
+
+ if !output_path.exists() {
+ std::fs::create_dir_all(output_path)?;
+ }
+
+ let mut writer = csv::Writer::from_writer(
+ std::fs::OpenOptions::new()
+ .create(true)
+ .write(true)
+ .truncate(true)
+ .open(output_path.join("distances.csv"))?,
+ );
+ let mut distances = res.distances.iter().collect::<Vec<_>>();
+ distances.sort_by_key(|(_id, distance)| *distance);
+
+ let id2node: BTreeMap<_, _> = cluster
+ .workers
+ .iter()
+ .flat_map(|w| {
+ distances
+ .iter()
+ .chunks(10_000)
+ .into_iter()
+ .flat_map(move |c| {
+ let ids = c.map(|(id, _)| *id).collect::<Vec<_>>();
+ w.batch_id2node(ids)
+ })
+ .collect::<Vec<_>>()
+ })
+ .collect();
+
+ for (id, distance) in distances {
+ if let Some(node) = id2node.get(&id) {
+ writer.write_record(&[node.as_str().to_string(), distance.to_string()])?;
+ }
+ }
+
+ Ok(())
+}
diff --git a/crates/core/src/entrypoint/ampc/shortest_path/mapper.rs b/crates/core/src/entrypoint/ampc/shortest_path/mapper.rs
new file mode 100644
index 00000000..8e030edd
--- /dev/null
+++ b/crates/core/src/entrypoint/ampc/shortest_path/mapper.rs
@@ -0,0 +1,182 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+use std::sync::{atomic::AtomicBool, Arc, Mutex};
+
+use bloom::U64BloomFilter;
+use rustc_hash::FxHashMap;
+
+use super::{DhtTable as _, Mapper, Meta, ShortestPathJob, ShortestPathTables};
+use crate::{
+ ampc::{
+ dht::{U64Min, UpsertAction},
+ DhtConn,
+ },
+ webgraph,
+ webpage::html::links::RelFlags,
+};
+
+const BATCH_SIZE: usize = 4096;
+
+pub static SKIPPED_REL: std::sync::LazyLock<RelFlags> = std::sync::LazyLock::new(|| {
+ RelFlags::TAG
+ | RelFlags::NOFOLLOW
+ | RelFlags::IS_IN_FOOTER
+ | RelFlags::IS_IN_NAVIGATION
+ | RelFlags::PRIVACY_POLICY
+ | RelFlags::TERMS_OF_SERVICE
+ | RelFlags::SEARCH
+ | RelFlags::LINK_TAG
+ | RelFlags::SCRIPT_TAG
+ | RelFlags::UGC
+});
+
+#[derive(Debug, Clone, bincode::Decode, bincode::Encode)]
+pub enum ShortestPathMapper {
+ RelaxEdges,
+ UpdateChangedNodes,
+}
+
+impl ShortestPathMapper {
+ fn get_old_distances(
+ batch: Vec<webgraph::NodeID>,
+ dht: &DhtConn<ShortestPathTables>,
+ ) -> FxHashMap<webgraph::NodeID, u64> {
+ dht.prev().distances.batch_get(batch).into_iter().collect()
+ }
+
+ fn update_distances(
+ batch: &[webgraph::SmallEdge],
+ dht: &DhtConn<ShortestPathTables>,
+ ) -> Vec<(webgraph::NodeID, UpsertAction)> {
+ let old_distances = Self::get_old_distances(batch.iter().map(|e| e.from).collect(), dht);
+
+ let mut new_distances = FxHashMap::default();
+
+ for edge in batch {
+ if let Some(old_distance) = old_distances.get(&edge.from) {
+ let new_distance = old_distance + 1;
+ let old_new = new_distances.entry(edge.to).or_insert(new_distance);
+
+ if new_distance < *old_new {
+ *old_new = new_distance;
+ }
+ }
+ }
+
+ dht.next()
+ .distances
+ .batch_upsert(U64Min, new_distances.into_iter().collect())
+ }
+
+ fn map_batch(
+ batch: &[webgraph::SmallEdge],
+ new_changed_nodes: &Mutex<U64BloomFilter>,
+ round_had_changes: &AtomicBool,
+ dht: &DhtConn<ShortestPathTables>,
+ ) {
+ let updates = Self::update_distances(batch, dht);
+ let mut new_changed_nodes = new_changed_nodes.lock().unwrap();
+
+ for (node, action) in updates {
+ if action.is_changed() {
+ new_changed_nodes.insert_u128(node.as_u128());
+ round_had_changes.store(true, std::sync::atomic::Ordering::Relaxed);
+ }
+ }
+ }
+}
+
+impl Mapper for ShortestPathMapper {
+ type Job = ShortestPathJob;
+
+ fn map(
+ &self,
+ job: Self::Job,
+ worker: &<<Self as Mapper>::Job as super::Job>::Worker,
+ dht: &crate::ampc::DhtConn<<<Self as Mapper>::Job as super::Job>::DhtTables>,
+ ) {
+ match self {
+ ShortestPathMapper::RelaxEdges => {
+ let round_had_changes = Arc::new(AtomicBool::new(false));
+ let pool = rayon::ThreadPoolBuilder::new().build().unwrap();
+
+ let new_changed_nodes = Arc::new(Mutex::new(U64BloomFilter::empty_from(
+ &worker.changed_nodes().lock().unwrap(),
+ )));
+
+ pool.scope(|s| {
+ let mut changed_nodes = worker.changed_nodes().lock().unwrap();
+ changed_nodes.insert_u128(job.source.as_u128());
+
+ let mut batch = Vec::with_capacity(BATCH_SIZE);
+
+ for edge in worker.graph().page_edges() {
+ if edge.rel_flags.intersects(*SKIPPED_REL) {
+ continue;
+ }
+
+ if changed_nodes.contains_u128(edge.from.as_u128()) {
+ batch.push(edge);
+ }
+
+ if batch.len() >= BATCH_SIZE {
+ let update_batch = batch.clone();
+ let update_new_changed_nodes = new_changed_nodes.clone();
+ let update_round_had_changes = round_had_changes.clone();
+ s.spawn(move |_| {
+ Self::map_batch(
+ &update_batch,
+ &update_new_changed_nodes,
+ &update_round_had_changes,
+ dht,
+ )
+ });
+ batch.clear();
+ }
+ }
+
+ if !batch.is_empty() {
+ Self::map_batch(&batch, &new_changed_nodes, &round_had_changes, dht);
+ }
+ });
+
+ dht.next()
+ .changed_nodes
+ .set(worker.shard(), new_changed_nodes.lock().unwrap().clone());
+ dht.next().meta.set(
+ (),
+ Meta {
+ round_had_changes: round_had_changes
+ .load(std::sync::atomic::Ordering::Relaxed),
+ },
+ );
+ }
+ ShortestPathMapper::UpdateChangedNodes => {
+ let all_changed_nodes: Vec<_> =
+ dht.next().changed_nodes.iter().map(|(_, v)| v).collect();
+ let mut changed_nodes =
+ U64BloomFilter::empty_from(&worker.changed_nodes().lock().unwrap());
+
+ for bloom in all_changed_nodes {
+ changed_nodes.union(bloom.clone());
+ }
+
+ *worker.changed_nodes().lock().unwrap() = changed_nodes;
+ }
+ }
+ }
+}
diff --git a/crates/core/src/entrypoint/ampc/shortest_path/mod.rs b/crates/core/src/entrypoint/ampc/shortest_path/mod.rs
new file mode 100644
index 00000000..8f345b49
--- /dev/null
+++ b/crates/core/src/entrypoint/ampc/shortest_path/mod.rs
@@ -0,0 +1,71 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+//! Single-source shortest path.
+
+pub mod coordinator;
+mod mapper;
+pub mod worker;
+
+use bloom::U64BloomFilter;
+
+use crate::distributed::member::ShardId;
+use crate::{
+ ampc::{prelude::*, DefaultDhtTable},
+ webgraph,
+};
+
+use self::mapper::ShortestPathMapper;
+use self::worker::{RemoteShortestPathWorker, ShortestPathWorker};
+
+#[derive(
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+ Debug,
+ Clone,
+ PartialEq,
+ Eq,
+)]
+pub struct Meta {
+ round_had_changes: bool,
+}
+
+#[derive(bincode::Encode, bincode::Decode, Debug, Clone)]
+pub struct ShortestPathTables {
+ distances: DefaultDhtTable<webgraph::NodeID, u64>,
+ meta: DefaultDhtTable<(), Meta>,
+ changed_nodes: DefaultDhtTable<ShardId, U64BloomFilter>,
+}
+
+impl_dht_tables!(ShortestPathTables, [distances, meta, changed_nodes]);
+
+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
+pub struct ShortestPathJob {
+ shard: ShardId,
+ source: webgraph::NodeID,
+}
+
+impl Job for ShortestPathJob {
+ type DhtTables = ShortestPathTables;
+ type Worker = ShortestPathWorker;
+ type Mapper = ShortestPathMapper;
+
+ fn is_schedulable(&self, worker: &RemoteShortestPathWorker) -> bool {
+ self.shard == worker.shard()
+ }
+}
diff --git a/crates/core/src/entrypoint/ampc/shortest_path/worker.rs b/crates/core/src/entrypoint/ampc/shortest_path/worker.rs
new file mode 100644
index 00000000..e097b474
--- /dev/null
+++ b/crates/core/src/entrypoint/ampc/shortest_path/worker.rs
@@ -0,0 +1,172 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+use crate::{
+ ampc::{JobConn, Worker},
+ config::ShortestPathWorkerConfig,
+ distributed::{
+ member::{Service, ShardId},
+ sonic,
+ },
+ hyperloglog::HyperLogLog,
+ webgraph::{self, Webgraph},
+ Result,
+};
+use std::{
+ net::SocketAddr,
+ sync::{Arc, Mutex},
+};
+
+use super::{impl_worker, Message, RemoteWorker, ShortestPathJob};
+use bloom::U64BloomFilter;
+
+#[derive(Clone)]
+pub struct ShortestPathWorker {
+ shard: ShardId,
+ graph: Arc<Webgraph>,
+ changed_nodes: Arc<Mutex<U64BloomFilter>>,
+ nodes_sketch: HyperLogLog<4096>,
+}
+
+impl ShortestPathWorker {
+ pub fn new(graph: Webgraph, shard: ShardId) -> Self {
+ let mut nodes_sketch = HyperLogLog::default();
+
+ for node in graph.page_nodes() {
+ nodes_sketch.add_u128(node.as_u128());
+ }
+
+ Self {
+ graph: Arc::new(graph),
+ shard,
+ changed_nodes: Arc::new(Mutex::new(U64BloomFilter::new(
+ nodes_sketch.size() as u64,
+ 0.01,
+ ))),
+ nodes_sketch,
+ }
+ }
+
+ pub fn graph(&self) -> &Webgraph {
+ &self.graph
+ }
+
+ pub fn shard(&self) -> ShardId {
+ self.shard
+ }
+
+ pub fn changed_nodes(&self) -> &Arc<Mutex<U64BloomFilter>> {
+ &self.changed_nodes
+ }
+
+ pub fn nodes_sketch(&self) -> &HyperLogLog<4096> {
+ &self.nodes_sketch
+ }
+}
+
+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
+pub struct GetNodeSketch;
+
+impl Message<ShortestPathWorker> for GetNodeSketch {
+ type Response = HyperLogLog<4096>;
+
+ fn handle(self, worker: &ShortestPathWorker) -> Self::Response {
+ worker.nodes_sketch().clone()
+ }
+}
+
+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
+pub struct BatchId2Node(Vec<webgraph::NodeID>);
+
+impl Message<ShortestPathWorker> for BatchId2Node {
+ type Response = Vec<(webgraph::NodeID, webgraph::Node)>;
+
+ fn handle(self, worker: &ShortestPathWorker) -> Self::Response {
+ self.0
+ .iter()
+ .filter_map(|id| {
+ worker
+ .graph
+ .search(&webgraph::query::Id2NodeQuery::Page(*id))
+ .ok()
+ .flatten()
+ .map(|node| (*id, node))
+ })
+ .collect()
+ }
+}
+
+impl_worker!(ShortestPathJob, RemoteShortestPathWorker => ShortestPathWorker, [BatchId2Node, GetNodeSketch]);
+
+#[derive(Clone)]
+pub struct RemoteShortestPathWorker {
+ shard: ShardId,
+ pool: Arc<sonic::ConnectionPool<JobConn<ShortestPathJob>>>,
+}
+
+impl RemoteShortestPathWorker {
+ pub fn new(shard: ShardId, addr: SocketAddr) -> Result<Self> {
+ Ok(Self {
+ shard,
+ pool: Arc::new(sonic::ConnectionPool::new(addr)?),
+ })
+ }
+
+ pub fn shard(&self) -> ShardId {
+ self.shard
+ }
+
+ pub fn addr(&self) -> SocketAddr {
+ self.pool.addr()
+ }
+
+ pub fn batch_id2node(
+ &self,
+ ids: Vec<webgraph::NodeID>,
+ ) -> Vec<(webgraph::NodeID, webgraph::Node)> {
+ self.send(BatchId2Node(ids))
+ }
+
+ pub fn get_node_sketch(&self) -> HyperLogLog<4096> {
+ self.send(GetNodeSketch)
+ }
+}
+
+impl RemoteWorker for RemoteShortestPathWorker {
+ type Job = ShortestPathJob;
+
+ fn pool(&self) -> &sonic::ConnectionPool<JobConn<Self::Job>> {
+ &self.pool
+ }
+}
+
+pub fn run(config: ShortestPathWorkerConfig) -> Result<()> {
+ tracing::info!("starting worker");
+ let tokio_conf = config.clone();
+
+ let graph = Webgraph::builder(config.graph_path, config.shard).open()?;
+ let worker = ShortestPathWorker::new(graph, config.shard);
+ let service = Service::ShortestPathWorker {
+ host: tokio_conf.host,
+ shard: tokio_conf.shard,
+ };
+ crate::start_gossip_cluster_thread(tokio_conf.gossip, Some(service));
+
+ tracing::info!("worker ready");
+ worker.run(config.host)?;
+
+ Ok(())
+}
diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs
index f636ab0d..4264d440 100644
--- a/crates/core/src/main.rs
+++ b/crates/core/src/main.rs
@@ -13,6 +13,7 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
use anyhow::{Context, Result};
use clap::{Parser, Subcommand};
use serde::de::DeserializeOwned;
@@ -141,6 +142,13 @@ enum AmpcOptions {
/// Start a coordinator to distribute the approximation of the harmonic centrality computation.
/// Workers needs to be started before the coordinator.
ApproxHarmonicCoordinator { config_path: String },
+
+ /// Start a worker to compute the shortest paths of a graph.
+ ShortestPathWorker { config_path: String },
+
+ /// Start a coordinator to distribute the shortest path computation.
+ /// Workers needs to be started before the coordinator.
+ ShortestPathCoordinator { config_path: String },
}
#[derive(Subcommand)]
@@ -478,6 +486,14 @@ fn main() -> Result<()> {
let config: config::ApproxHarmonicCoordinatorConfig = load_toml_config(config_path);
entrypoint::ampc::approximated_harmonic_centrality::coordinator::run(config)?;
}
+ AmpcOptions::ShortestPathWorker { config_path } => {
+ let config: config::ShortestPathWorkerConfig = load_toml_config(config_path);
+ entrypoint::ampc::shortest_path::worker::run(config)?;
+ }
+ AmpcOptions::ShortestPathCoordinator { config_path } => {
+ let config: config::ShortestPathCoordinatorConfig = load_toml_config(config_path);
+ entrypoint::ampc::shortest_path::coordinator::run(config)?;
+ }
},
Commands::Admin { options } => match options {