web search done right
implement distributed version of approximated harmonic centrality the page graph still seems to be too big to calculate the exact centrality even when distributed across multiple workers (need more workers)
Mikkel Denker823efa671618
diff --git a/Cargo.lock b/Cargo.lockindex f3dccdc9..89074798 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4795,6 +4795,7 @@ dependencies = ["robotstxt-with-cache","rust-s3","rust-stemmers",+ "rustc-hash","safetensors 0.3.3","scylla","serde",diff --git a/Cargo.toml b/Cargo.tomlindex 794b2b51..6d2c2886 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -111,6 +111,7 @@ rusqlite = {version = "0.29.0", features = []}rust-s3 = {version = "0.33.0", features = ["blocking", "tokio"]}rust-stemmers = "1.2.0"+rustc-hash = "1.1"safetensors = "0.3.1"scylla = {version = "0.12.0", features = ["chrono"]}serde = {version = "1.0.137", features = ["rc", "derive"]}diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex b7257c67..66a63460 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -21,115 +21,116 @@ name = "stract"path = "src/main.rs"[dependencies]-aes-gcm = {workspace = true}-ahash = {workspace = true}-anyhow = {workspace = true}-async-channel = {workspace = true}-async-stream = {workspace = true}-axum = {workspace = true}-axum-extra = {workspace = true}-axum-macros = {workspace = true}-base64 = {workspace = true}-bincode = {workspace = true}-bitvec = {workspace = true}+aes-gcm.workspace = true+ahash.workspace = true+anyhow.workspace = true+async-channel.workspace = true+async-stream.workspace = true+axum-extra.workspace = true+axum-macros.workspace = true+axum.workspace = true+base64.workspace = true+bincode.workspace = true+bitvec.workspace = truebloom = {path = "../bloom"}-bytecount = {workspace = true}-bytemuck = {workspace = true}-byteorder = {workspace = true}-bzip2 = {workspace = true}-candle-core = {workspace = true}-candle-nn = {workspace = true}-candle-transformers = {workspace = true}-chardetng = {workspace = true}-chitchat = {workspace = true}-chrono = {workspace = true}-clap = {workspace = true}-crossbeam-channel = {workspace = true}-csv = {workspace = true}-dashmap = {workspace = true}-deadpool = {workspace = true}-encoding_rs = {workspace = true}-enum_dispatch = {workspace = true}-eventsource-stream = {workspace = true}-fend-core = {workspace = true}+bytecount.workspace = true+bytemuck.workspace = true+byteorder.workspace = true+bzip2.workspace = true+candle-core.workspace = true+candle-nn.workspace = true+candle-transformers.workspace = true+chardetng.workspace = true+chitchat.workspace = true+chrono.workspace = true+clap.workspace = true+crossbeam-channel.workspace = true+csv.workspace = true+dashmap.workspace = true+deadpool.workspace = true+encoding_rs.workspace = true+enum_dispatch.workspace = true+eventsource-stream.workspace = true+fend-core.workspace = truefile_store = {path = "../file-store"}-flate2 = {workspace = true}-fnv = {workspace = true}-fst = {workspace = true}-futures = {workspace = true}-half = {workspace = true}-hashbrown = {workspace = true}-http = {workspace = true}-image = {workspace = true}-indicatif = {workspace = true}-itertools = {workspace = true}+flate2.workspace = true+fnv.workspace = true+fst.workspace = true+futures.workspace = true+half.workspace = true+hashbrown.workspace = true+http.workspace = true+image.workspace = true+indicatif.workspace = true+itertools.workspace = truekuchiki = {path = "../kuchiki"}-libc = {workspace = true}-log = {workspace = true}-logos = {workspace = true}-lz-str = {workspace = true}-lz4_flex = {workspace = true}-md5 = {workspace = true}-memmap2 = {workspace = true}-mime = {workspace = true}-min-max-heap = {workspace = true}-nom = {workspace = true}-num-bigint = {workspace = true}-num-rational = {workspace = true}-once_cell = {workspace = true}-openraft = {workspace = true}+libc.workspace = true+log.workspace = true+logos.workspace = true+lz-str.workspace = true+lz4_flex.workspace = true+md5.workspace = true+memmap2.workspace = true+mime.workspace = true+min-max-heap.workspace = true+nom.workspace = true+num-bigint.workspace = true+num-rational.workspace = true+once_cell.workspace = true+openraft.workspace = trueoptics = {path = "../optics"}-postcard = {workspace = true}-publicsuffix = {workspace = true}-quick-xml = {workspace = true}-rand = {workspace = true}-rayon = {workspace = true}-redb = {workspace = true}-regex = {workspace = true}-reqwest = {workspace = true}-ring = {workspace = true}-rio_api = {workspace = true}-rio_turtle = {workspace = true}-rkyv = {workspace = true}-robotstxt-with-cache = {workspace = true}-rust-s3 = {workspace = true}-rust-stemmers = {workspace = true}-safetensors = {workspace = true}-scylla = {workspace = true}-serde = {workspace = true}-serde_json = {workspace = true}-serde_urlencoded = {workspace = true}+postcard.workspace = true+publicsuffix.workspace = true+quick-xml.workspace = true+rand.workspace = true+rayon.workspace = true+redb.workspace = true+regex.workspace = true+reqwest.workspace = true+ring.workspace = true+rio_api.workspace = true+rio_turtle.workspace = true+rkyv.workspace = true+robotstxt-with-cache.workspace = true+rust-s3.workspace = true+rust-stemmers.workspace = true+rustc-hash.workspace = true+safetensors.workspace = true+scylla.workspace = true+serde.workspace = true+serde_json.workspace = true+serde_urlencoded.workspace = truespeedy_kv = {path = "../speedy-kv"}-strum = {workspace = true}-tantivy = {workspace = true}-thiserror = {workspace = true}-tokenizers = {workspace = true}-tokio = {workspace = true}-tokio-stream = {workspace = true}-toml = {workspace = true}-tower = {workspace = true}-tower-http = {workspace = true}-tracing = {workspace = true}-tracing-subscriber = {workspace = true}-url = {workspace = true}-urlencoding = {workspace = true}-utoipa = {workspace = true}-utoipa-swagger-ui = {workspace = true}-uuid = {workspace = true}-whatlang = {workspace = true}-xxhash-rust = {workspace = true}+strum.workspace = true+tantivy.workspace = true+thiserror.workspace = true+tokenizers.workspace = true+tokio-stream.workspace = true+tokio.workspace = true+toml.workspace = true+tower-http.workspace = true+tower.workspace = true+tracing-subscriber.workspace = true+tracing.workspace = true+url.workspace = true+urlencoding.workspace = true+utoipa-swagger-ui.workspace = true+utoipa.workspace = true+uuid.workspace = true+whatlang.workspace = true+xxhash-rust.workspace = truezimba = {path = "../zimba"}[target.'cfg(not(target_env = "msvc"))'.dependencies]-tikv-jemallocator = {workspace = true}+tikv-jemallocator.workspace = true[dev-dependencies]-criterion = {workspace = true}-insta = {workspace = true}-maplit = {workspace = true}-proptest = {workspace = true}-proptest-derive = {workspace = true}-tracing-test = {workspace = true}+criterion.workspace = true+insta.workspace = true+maplit.workspace = true+proptest-derive.workspace = true+proptest.workspace = true+tracing-test.workspace = true[[bench]]harness = falsediff --git a/crates/core/src/ampc/dht/upsert.rs b/crates/core/src/ampc/dht/upsert.rsindex 757390e5..73c28816 100644--- a/crates/core/src/ampc/dht/upsert.rs+++ b/crates/core/src/ampc/dht/upsert.rs@@ -16,7 +16,7 @@use enum_dispatch::enum_dispatch;-use crate::hyperloglog::HyperLogLog;+use crate::{hyperloglog::HyperLogLog, kahan_sum::KahanSum};use super::Value;@@ -46,6 +46,7 @@ pub enum UpsertEnum {HyperLogLog128Upsert,U64Add,F64Add,+ KahanSumAdd,}macro_rules! hyperloglog_upsert {@@ -121,3 +122,25 @@ impl UpsertFn for F64Add {.into()}}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct KahanSumAdd;++impl UpsertFn for KahanSumAdd {+ fn upsert(&self, old: Value, new: Value) -> Value {+ let (old, _) =+ bincode::decode_from_slice::<KahanSum, _>(old.as_bytes(), bincode::config::standard())+ .unwrap();+ let (new, _) =+ bincode::decode_from_slice::<KahanSum, _>(new.as_bytes(), bincode::config::standard())+ .unwrap();++ let new: f64 = new.into();++ let res: KahanSum = old + new;++ bincode::encode_to_vec(res, bincode::config::standard())+ .unwrap()+ .into()+ }+}diff --git a/crates/core/src/ampc/mod.rs b/crates/core/src/ampc/mod.rsindex 8b58a223..006377d7 100644--- a/crates/core/src/ampc/mod.rs+++ b/crates/core/src/ampc/mod.rs@@ -68,7 +68,7 @@ pub enum Resp<J, R> {type JobDht<J> = DhtConn<<J as Job>::DhtTables>;-type JobConn<J> = sonic::Connection<JobReq<J>, JobResp<J>>;+pub type JobConn<J> = sonic::Connection<JobReq<J>, JobResp<J>>;#[must_use = "this `JobScheduled` may not have scheduled the job on any worker"]enum JobScheduled {diff --git a/crates/core/src/ampc/worker.rs b/crates/core/src/ampc/worker.rsindex 249371a6..469656c0 100644--- a/crates/core/src/ampc/worker.rs+++ b/crates/core/src/ampc/worker.rs@@ -14,10 +14,11 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>-use std::net::SocketAddr;+use std::ops::DerefMut;use super::{CoordReq, CoordResp, DhtConn, Job, JobConn, JobReq, JobResp, Req, Resp, Server};use crate::block_on;+use crate::distributed::sonic;use crate::Result;use anyhow::anyhow;use tokio::net::ToSocketAddrs;@@ -57,7 +58,7 @@ where{type Job: Job;- fn remote_addr(&self) -> SocketAddr;+ fn pool(&self) -> &sonic::ConnectionPool<JobConn<Self::Job>>;fn schedule_job(&self, job: &Self::Job, mapper: <Self::Job as Job>::Mapper) -> Result<()> {self.send_raw(&JobReq::Coordinator(CoordReq::ScheduleJob {@@ -84,8 +85,8 @@ where}}- fn conn(&self) -> Result<JobConn<Self::Job>> {- let conn = block_on(JobConn::connect(self.remote_addr()))?;+ fn conn(&self) -> Result<impl DerefMut<Target = JobConn<Self::Job>>> {+ let conn = block_on(self.pool().get())?;Ok(conn)}diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rsindex e690be5e..ec0af33c 100644--- a/crates/core/src/config/defaults.rs+++ b/crates/core/src/config/defaults.rs@@ -185,3 +185,14 @@ impl Indexing {25_000}}++pub struct ApproxHarmonic;+impl ApproxHarmonic {+ pub fn sample_rate() -> f64 {+ 0.3+ }++ pub fn max_distance() -> u8 {+ 7+ }+}diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex d2de173b..540b461e 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -584,6 +584,27 @@ pub struct HarmonicWorkerConfig {pub host: SocketAddr,}+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]+pub struct ApproxHarmonicCoordinatorConfig {+ pub gossip: GossipConfig,+ pub host: SocketAddr,+ pub output_path: String,++ #[serde(default = "defaults::ApproxHarmonic::sample_rate")]+ pub sample_rate: f64,++ #[serde(default = "defaults::ApproxHarmonic::max_distance")]+ pub max_distance: u8,+}++#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]+pub struct ApproxHarmonicWorkerConfig {+ pub gossip: GossipConfig,+ pub shard: ShardId,+ pub graph_path: String,+ pub host: SocketAddr,+}+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]pub struct CanonicalIndexConfig {pub output_path: String,diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rsindex ec10a189..aad69587 100644--- a/crates/core/src/distributed/member.rs+++ b/crates/core/src/distributed/member.rs@@ -94,6 +94,13 @@ pub enum Service {HarmonicCoordinator {host: SocketAddr,},+ ApproxHarmonicWorker {+ host: SocketAddr,+ shard: ShardId,+ },+ ApproxHarmonicCoordinator {+ host: SocketAddr,+ },}#[derive(PartialEq, Eq, Hash, Clone, Debug)]diff --git a/crates/core/src/distributed/sonic/connection_pool.rs b/crates/core/src/distributed/sonic/connection_pool.rsindex 1924bf82..db8b871d 100644--- a/crates/core/src/distributed/sonic/connection_pool.rs+++ b/crates/core/src/distributed/sonic/connection_pool.rs@@ -63,6 +63,10 @@ where.await.map_err(|_| anyhow::anyhow!("Failed to get connection from pool"))}++ pub fn addr(&self) -> SocketAddr {+ self.addr+ }}pub struct Manager<Req, Res> {diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rsnew file mode 100644index 00000000..23a7a73d--- /dev/null+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs@@ -0,0 +1,261 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use std::collections::BTreeMap;+use std::net::SocketAddr;+use std::path::Path;++use super::mapper::ApproxCentralityMapper;+use super::{worker::RemoteApproxCentralityWorker, ApproxCentralityTables};+use super::{ApproxCentralityJob, DhtTable as _, Finisher, Meta, Setup};+use crate::ampc::{Coordinator, DefaultDhtTable, DhtConn};+use crate::config::ApproxHarmonicCoordinatorConfig;+use crate::distributed::cluster::Cluster;+use crate::distributed::member::{Member, Service, ShardId};+use crate::webgraph::centrality::{store_csv, store_harmonic, top_nodes, TopNodes};+use crate::Result;++pub struct ApproxCentralitySetup {+ dht: DhtConn<ApproxCentralityTables>,+ workers: Vec<RemoteApproxCentralityWorker>,+ sample_rate: f64,+}++impl ApproxCentralitySetup {+ pub async fn new(cluster: &Cluster, sample_rate: f64) -> Result<Self> {+ let dht_members: Vec<_> = cluster+ .members()+ .await+ .into_iter()+ .filter_map(|member| {+ if let Service::Dht { host, shard } = member.service {+ Some((shard, host))+ } else {+ None+ }+ })+ .collect();++ let workers = cluster+ .members()+ .await+ .into_iter()+ .filter_map(|member| {+ if let Service::ApproxHarmonicWorker { host, shard } = member.service {+ Some(RemoteApproxCentralityWorker::new(shard, host))+ } else {+ None+ }+ })+ .collect::<Result<Vec<RemoteApproxCentralityWorker>>>()?;++ Ok(Self::new_for_dht_members(+ &dht_members,+ workers,+ sample_rate,+ ))+ }+ pub fn new_for_dht_members(+ dht_members: &[(ShardId, SocketAddr)],+ workers: Vec<RemoteApproxCentralityWorker>,+ sample_rate: f64,+ ) -> Self {+ let initial = ApproxCentralityTables {+ meta: DefaultDhtTable::new(dht_members, "meta"),+ centrality: DefaultDhtTable::new(dht_members, "centrality"),+ };++ let dht = DhtConn::new(initial);++ Self {+ dht,+ workers,+ sample_rate,+ }+ }+}++impl Setup for ApproxCentralitySetup {+ type DhtTables = ApproxCentralityTables;++ fn init_dht(&self) -> DhtConn<Self::DhtTables> {+ self.dht.clone()+ }++ fn setup_round(&self, dht: &Self::DhtTables) {+ let meta = dht.meta.get(()).unwrap();++ dht.meta.set(+ (),+ Meta {+ round: meta.round + 1,+ num_samples_per_worker: meta.num_samples_per_worker,+ },+ );+ }++ fn setup_first_round(&self, dht: &Self::DhtTables) {+ let upper_bound_num_nodes: u64 = self.workers.iter().map(|w| w.num_nodes()).sum();+ let num_samples_per_worker =+ ((upper_bound_num_nodes as f64).log2() / self.sample_rate.powi(2)).ceil() as u64+ / (self.workers.len() as u64);++ dht.meta.set(+ (),+ Meta {+ round: 0,+ num_samples_per_worker,+ },+ );+ }+}++pub struct ApproxCentralityFinish;++impl Finisher for ApproxCentralityFinish {+ type Job = ApproxCentralityJob;++ fn is_finished(&self, dht: &ApproxCentralityTables) -> bool {+ dht.meta.get(()).unwrap().round > 0+ }+}++pub fn build(+ dht: &[(ShardId, SocketAddr)],+ workers: Vec<RemoteApproxCentralityWorker>,+ sample_rate: f64,+) -> Coordinator<ApproxCentralityJob> {+ let setup = ApproxCentralitySetup::new_for_dht_members(dht, workers.clone(), sample_rate);++ let mut coord = Coordinator::new(setup, workers.clone());++ for worker in &workers {+ coord = coord.with_mapper(ApproxCentralityMapper::ApproximateCentrality {+ worker_shard: worker.shard(),+ });+ }++ coord+}++struct ClusterInfo {+ // dropping the handle will leave the cluster+ _handle: Cluster,+ dht: Vec<(ShardId, SocketAddr)>,+ workers: Vec<RemoteApproxCentralityWorker>,+}++async fn setup_gossip(config: ApproxHarmonicCoordinatorConfig) -> Result<ClusterInfo> {+ let handle = Cluster::join(+ Member {+ id: config.gossip.cluster_id,+ service: Service::ApproxHarmonicCoordinator { host: config.host },+ },+ config.gossip.addr,+ config.gossip.seed_nodes.unwrap_or_default(),+ )+ .await?;++ tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;++ let members = handle.members().await;++ let dht = members+ .iter()+ .filter_map(|member| {+ if let Service::Dht { host, shard } = member.service {+ Some((shard, host))+ } else {+ None+ }+ })+ .collect();++ let workers = members+ .iter()+ .filter_map(|member| {+ if let Service::ApproxHarmonicWorker { host, shard } = member.service {+ Some(RemoteApproxCentralityWorker::new(shard, host))+ } else {+ None+ }+ })+ .collect::<Result<Vec<RemoteApproxCentralityWorker>>>()?;++ Ok(ClusterInfo {+ _handle: handle,+ dht,+ workers,+ })+}++pub fn run(config: ApproxHarmonicCoordinatorConfig) -> Result<()> {+ let tokio_conf = config.clone();+ let cluster = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()?+ .block_on(setup_gossip(tokio_conf))?;++ let jobs = cluster+ .workers+ .iter()+ .map(|worker| ApproxCentralityJob {+ shard: worker.shard(),+ max_distance: config.max_distance,+ all_workers: cluster+ .workers+ .clone()+ .into_iter()+ .map(|w| (w.shard(), w.addr()))+ .collect(),+ })+ .collect();++ let coordinator = build(&cluster.dht, cluster.workers.clone(), config.sample_rate);+ let res = coordinator.run(jobs, ApproxCentralityFinish)?;++ let num_nodes = res.centrality.num_keys();++ let num_samples =+ res.meta.get(()).unwrap().num_samples_per_worker * cluster.workers.len() as u64;++ let output_path = Path::new(&config.output_path);++ let norm = num_nodes as f64 / (num_samples as f64 * (num_nodes as f64 - 1.0));++ let store = store_harmonic(+ res.centrality.iter().map(|(n, c)| (n, f64::from(c) * norm)),+ output_path,+ );++ let top_nodes = top_nodes(&store, TopNodes::Top(1_000_000));++ let ids = top_nodes.iter().map(|(id, _)| *id).collect::<Vec<_>>();+ let id2node: BTreeMap<_, _> = cluster+ .workers+ .iter()+ .flat_map(|w| w.batch_id2node(ids.clone()))+ .collect();++ let top_nodes = top_nodes+ .iter()+ .filter_map(|(id, c)| id2node.get(id).map(|n| (n.clone(), *c)))+ .collect::<Vec<_>>();++ store_csv(top_nodes, output_path.join("harmonic.csv"));++ Ok(())+}diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rsnew file mode 100644index 00000000..d8586947--- /dev/null+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs@@ -0,0 +1,210 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use itertools::Itertools;+use rustc_hash::{FxHashMap, FxHashSet};+use std::{cmp, collections::BTreeMap};++use super::{worker::ApproxCentralityWorker, ApproxCentralityJob, Mapper};+use crate::{+ ampc::dht::upsert,+ distributed::member::ShardId,+ entrypoint::ampc::approximated_harmonic_centrality::{+ worker::RemoteApproxCentralityWorker, DhtTable,+ },+ kahan_sum::KahanSum,+ webgraph,+};+use rayon::prelude::*;++const BATCH_SIZE: usize = 1024;++#[derive(Debug, Clone, bincode::Decode, bincode::Encode)]+pub enum ApproxCentralityMapper {+ ApproximateCentrality { worker_shard: ShardId },+}++struct Workers {+ worker: ApproxCentralityWorker,+ remote_workers: Vec<RemoteApproxCentralityWorker>,+}++impl Workers {+ fn new(+ worker: ApproxCentralityWorker,+ remote_workers: Vec<RemoteApproxCentralityWorker>,+ ) -> Self {+ Self {+ worker,+ remote_workers,+ }+ }++ fn outgoing_nodes(+ &self,+ nodes: &[webgraph::NodeID],+ ) -> FxHashMap<webgraph::NodeID, FxHashSet<webgraph::NodeID>> {+ let mut res: FxHashMap<webgraph::NodeID, FxHashSet<webgraph::NodeID>> =+ FxHashMap::default();++ let limit = webgraph::EdgeLimit::Unlimited;++ for node in nodes {+ for edge in self.worker.graph().raw_outgoing_edges(node, limit) {+ res.entry(*node).or_default().insert(edge.to);+ }+ }++ // this does not retrieve outgoing edges from remote workers asynchronously,+ // but instead waits for each worker to respond before moving on to the next one.+ // this is not ideal, but it will only be a problem when we get way more workers+ // in which case we should probably use an approximation that simply runs the+ // normal harmonic centrality algorithm a fixed number of times.+ for remote_worker in &self.remote_workers {+ let nodes = remote_worker.outgoing_edge_nodes(nodes.to_vec(), limit);++ for (node, outgoing) in nodes {+ res.entry(node).or_default().extend(outgoing);+ }+ }++ res+ }++ fn run_batch(&self, batch: &mut Vec<(u8, webgraph::NodeID)>) -> BTreeMap<webgraph::NodeID, u8> {+ let mut new_distances: BTreeMap<webgraph::NodeID, u8> = BTreeMap::default();++ let nodes: Vec<_> = batch.iter().map(|(_, node)| *node).collect();+ let outgoing_nodes = self.outgoing_nodes(&nodes);++ for (dist, node) in batch.iter().cloned() {+ if let Some(outgoing) = outgoing_nodes.get(&node) {+ for outgoing in outgoing {+ let d = dist + 1;++ let current_dist = new_distances.get(outgoing).unwrap_or(&u8::MAX);++ if d < *current_dist {+ new_distances.insert(*outgoing, d);+ }+ }+ }+ }++ batch.clear();++ new_distances+ }++ fn dijkstra(&self, source: webgraph::NodeID, max_dist: u8) -> BTreeMap<webgraph::NodeID, u8> {+ let mut distances: BTreeMap<webgraph::NodeID, u8> = BTreeMap::default();+ let mut queue = std::collections::BinaryHeap::new();+ let mut batch = Vec::with_capacity(BATCH_SIZE);++ queue.push(cmp::Reverse((0u8, source)));+ distances.insert(source, 0u8);++ loop {+ if queue.is_empty() && batch.is_empty() {+ break;+ }++ match queue.pop() {+ Some(cmp::Reverse((dist, node))) => batch.push((dist, node)),+ None => {+ for (node, dist) in self.run_batch(&mut batch) {+ let cur_dist = distances.get(&node).unwrap_or(&u8::MAX);++ if dist < *cur_dist {+ distances.insert(node, dist);++ if dist <= max_dist {+ queue.push(cmp::Reverse((dist, node)));+ }+ }+ }+ }+ }++ if batch.len() >= BATCH_SIZE {+ for (node, dist) in self.run_batch(&mut batch) {+ let cur_dist = distances.get(&node).unwrap_or(&u8::MAX);++ if dist < *cur_dist {+ distances.insert(node, dist);+ queue.push(cmp::Reverse((dist, node)));+ }+ }+ }+ }++ distances+ }+}++impl Mapper for ApproxCentralityMapper {+ type Job = ApproxCentralityJob;++ fn map(+ &self,+ job: Self::Job,+ worker: &<<Self as Mapper>::Job as super::Job>::Worker,+ dht: &crate::ampc::DhtConn<<<Self as Mapper>::Job as super::Job>::DhtTables>,+ ) {+ match self {+ ApproxCentralityMapper::ApproximateCentrality { worker_shard } => {+ if worker.shard() != *worker_shard {+ return;+ }++ let remote_workers: Vec<_> = job+ .all_workers+ .iter()+ .cloned()+ .filter_map(|(shard, addr)| {+ if shard == worker.shard() {+ None+ } else {+ Some(RemoteApproxCentralityWorker::new(shard, addr).unwrap())+ }+ })+ .collect();++ let workers = Workers::new(worker.clone(), remote_workers);+ let num_samples = dht.next().meta.get(()).unwrap().num_samples_per_worker;++ let sampled = worker+ .graph()+ .random_nodes_with_outgoing(num_samples as usize);++ sampled.into_par_iter().for_each(|node| {+ for chunk in workers+ .dijkstra(node, job.max_distance)+ .into_iter()+ .map(|(n, d)| (n, KahanSum::from(1.0 / d as f64)))+ .chunks(BATCH_SIZE)+ .into_iter()+ {+ let pairs: Vec<_> = chunk.collect();+ dht.next()+ .centrality+ .batch_upsert(upsert::KahanSumAdd, pairs);+ }+ });+ }+ }+ }+}diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rsnew file mode 100644index 00000000..8d5ffccd--- /dev/null+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs@@ -0,0 +1,62 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++pub mod coordinator;+mod mapper;+pub mod worker;++use std::net::SocketAddr;++use crate::distributed::member::ShardId;+use crate::{+ ampc::{prelude::*, DefaultDhtTable},+ kahan_sum::KahanSum,+ webgraph,+};++use self::mapper::ApproxCentralityMapper;+use self::worker::{ApproxCentralityWorker, RemoteApproxCentralityWorker};++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct Meta {+ round: u64,+ num_samples_per_worker: u64,+}++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct ApproxCentralityTables {+ centrality: DefaultDhtTable<webgraph::NodeID, KahanSum>,+ meta: DefaultDhtTable<(), Meta>,+}++impl_dht_tables!(ApproxCentralityTables, [centrality, meta]);++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct ApproxCentralityJob {+ shard: ShardId,+ max_distance: u8,+ all_workers: Vec<(ShardId, SocketAddr)>,+}++impl Job for ApproxCentralityJob {+ type DhtTables = ApproxCentralityTables;+ type Worker = ApproxCentralityWorker;+ type Mapper = ApproxCentralityMapper;++ fn is_schedulable(&self, worker: &RemoteApproxCentralityWorker) -> bool {+ self.shard == worker.shard()+ }+}diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rsnew file mode 100644index 00000000..2b08d37e--- /dev/null+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs@@ -0,0 +1,172 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use rustc_hash::{FxHashMap, FxHashSet};++use crate::{+ ampc::JobConn,+ config::ApproxHarmonicWorkerConfig,+ distributed::{+ member::{Service, ShardId},+ sonic,+ },+ webgraph::{self, Webgraph},+ Result,+};+use std::{net::SocketAddr, sync::Arc};++use super::{impl_worker, ApproxCentralityJob, Message, RemoteWorker, Worker as _};++#[derive(Clone)]+pub struct ApproxCentralityWorker {+ shard: ShardId,+ graph: Arc<Webgraph>,+}++impl ApproxCentralityWorker {+ pub fn new(graph: Webgraph, shard: ShardId) -> Self {+ Self {+ graph: Arc::new(graph),+ shard,+ }+ }++ pub fn graph(&self) -> &Webgraph {+ &self.graph+ }++ pub fn shard(&self) -> ShardId {+ self.shard+ }+}++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct NumNodes;++impl Message<ApproxCentralityWorker> for NumNodes {+ type Response = u64;++ fn handle(self, worker: &ApproxCentralityWorker) -> Self::Response {+ worker.graph.estimate_num_nodes() as u64+ }+}++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct OutgoingEdgeNodes {+ pub from: Vec<webgraph::NodeID>,+ pub limit: webgraph::EdgeLimit,+}++impl Message<ApproxCentralityWorker> for OutgoingEdgeNodes {+ type Response = FxHashMap<webgraph::NodeID, FxHashSet<webgraph::NodeID>>;++ fn handle(self, worker: &ApproxCentralityWorker) -> Self::Response {+ let mut res: FxHashMap<webgraph::NodeID, FxHashSet<webgraph::NodeID>> =+ FxHashMap::default();++ for node in &self.from {+ for edge in worker.graph().raw_outgoing_edges(node, self.limit) {+ res.entry(*node).or_default().insert(edge.to);+ }+ }++ res+ }+}++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct BatchId2Node(Vec<webgraph::NodeID>);++impl Message<ApproxCentralityWorker> for BatchId2Node {+ type Response = Vec<(webgraph::NodeID, webgraph::Node)>;++ fn handle(self, worker: &ApproxCentralityWorker) -> Self::Response {+ self.0+ .iter()+ .filter_map(|id| worker.graph.id2node(id).map(|node| (*id, node)))+ .collect()+ }+}++impl_worker!(ApproxCentralityJob, RemoteApproxCentralityWorker => ApproxCentralityWorker, [NumNodes, OutgoingEdgeNodes, BatchId2Node]);++#[derive(Clone)]+pub struct RemoteApproxCentralityWorker {+ shard: ShardId,+ pool: Arc<sonic::ConnectionPool<JobConn<ApproxCentralityJob>>>,+}++impl RemoteApproxCentralityWorker {+ pub fn new(shard: ShardId, addr: SocketAddr) -> Result<Self> {+ Ok(Self {+ shard,+ pool: Arc::new(sonic::ConnectionPool::new(addr)?),+ })+ }++ pub fn shard(&self) -> ShardId {+ self.shard+ }++ pub fn addr(&self) -> SocketAddr {+ self.pool.addr()+ }++ pub fn num_nodes(&self) -> u64 {+ self.send(NumNodes)+ }++ pub fn outgoing_edge_nodes(+ &self,+ from: Vec<webgraph::NodeID>,+ limit: webgraph::EdgeLimit,+ ) -> FxHashMap<webgraph::NodeID, FxHashSet<webgraph::NodeID>> {+ self.send(OutgoingEdgeNodes { from, limit })+ }++ pub fn batch_id2node(+ &self,+ ids: Vec<webgraph::NodeID>,+ ) -> Vec<(webgraph::NodeID, webgraph::Node)> {+ self.send(BatchId2Node(ids))+ }+}++impl RemoteWorker for RemoteApproxCentralityWorker {+ type Job = ApproxCentralityJob;++ fn pool(&self) -> &sonic::ConnectionPool<JobConn<Self::Job>> {+ &self.pool+ }+}++pub fn run(config: ApproxHarmonicWorkerConfig) -> Result<()> {+ let tokio_conf = config.clone();++ let graph = Webgraph::builder(config.graph_path)+ .single_threaded()+ .open();+ let worker = ApproxCentralityWorker::new(graph, config.shard);+ let service = Service::ApproxHarmonicWorker {+ host: tokio_conf.host,+ shard: tokio_conf.shard,+ };+ crate::start_gossip_cluster_thread(tokio_conf.gossip, Some(service));++ worker.run(config.host)?;++ Ok(())+}diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rsindex 95c503d7..3a13b577 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs@@ -36,7 +36,7 @@ pub struct CentralitySetup {}impl CentralitySetup {- pub async fn new(cluster: &Cluster) -> Self {+ pub async fn new(cluster: &Cluster) -> Result<Self> {let dht_members: Vec<_> = cluster.members().await@@ -50,7 +50,7 @@ impl CentralitySetup {}).collect();- let workers: Vec<_> = cluster+ let workers = cluster.members().await.into_iter()@@ -61,9 +61,9 @@ impl CentralitySetup {None}})- .collect();+ .collect::<Result<Vec<RemoteCentralityWorker>>>()?;- Self::new_for_dht_members(&dht_members, workers)+ Ok(Self::new_for_dht_members(&dht_members, workers))}pub fn new_for_dht_members(@@ -177,7 +177,7 @@ async fn setup_gossip(config: HarmonicCoordinatorConfig) -> Result<ClusterInfo>None}})- .collect();+ .collect::<Result<Vec<RemoteCentralityWorker>>>()?;Ok(ClusterInfo {_handle: handle,diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rsindex 7c6c8ecb..a4922891 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs@@ -134,7 +134,7 @@ mod tests {});std::thread::sleep(std::time::Duration::from_secs(2)); // Wait for worker to start- let a = RemoteCentralityWorker::new(1.into(), worker_addr);+ let a = RemoteCentralityWorker::new(1.into(), worker_addr).unwrap();let worker = CentralityWorker::new(2.into(), b);let worker_addr = free_socket_addr();@@ -144,7 +144,7 @@ mod tests {std::thread::sleep(std::time::Duration::from_secs(2)); // Wait for worker to start- let b = RemoteCentralityWorker::new(2.into(), worker_addr);+ let b = RemoteCentralityWorker::new(2.into(), worker_addr).unwrap();// assert_eq!(a.num_nodes() + b.num_nodes(), num_nodes as u64);diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rsindex 95e3a4f8..1662c91e 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs@@ -17,9 +17,12 @@use bloom::U64BloomFilter;use crate::{- ampc::prelude::*,+ ampc::{prelude::*, JobConn},config::HarmonicWorkerConfig,- distributed::member::{Service, ShardId},+ distributed::{+ member::{Service, ShardId},+ sonic,+ },webgraph::{self, Webgraph},Result,};@@ -97,16 +100,6 @@ impl CentralityWorker {}}-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]-pub struct GetShard;--impl Message<CentralityWorker> for GetShard {- type Response = ShardId;-- fn handle(self, worker: &CentralityWorker) -> Self::Response {- worker.shard- }-}#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]pub struct NumNodes;@@ -132,17 +125,20 @@ impl Message<CentralityWorker> for BatchId2Node {}}-impl_worker!(CentralityJob, RemoteCentralityWorker => CentralityWorker, [GetShard, NumNodes, BatchId2Node]);+impl_worker!(CentralityJob, RemoteCentralityWorker => CentralityWorker, [NumNodes, BatchId2Node]);#[derive(Clone)]pub struct RemoteCentralityWorker {shard: ShardId,- addr: SocketAddr,+ pool: Arc<sonic::ConnectionPool<JobConn<CentralityJob>>>,}impl RemoteCentralityWorker {- pub fn new(shard: ShardId, addr: SocketAddr) -> Self {- Self { shard, addr }+ pub fn new(shard: ShardId, addr: SocketAddr) -> Result<Self> {+ Ok(Self {+ shard,+ pool: Arc::new(sonic::ConnectionPool::new(addr)?),+ })}pub fn shard(&self) -> ShardId {@@ -164,8 +160,8 @@ impl RemoteCentralityWorker {impl RemoteWorker for RemoteCentralityWorker {type Job = CentralityJob;- fn remote_addr(&self) -> SocketAddr {- self.addr+ fn pool(&self) -> &crate::distributed::sonic::ConnectionPool<JobConn<Self::Job>> {+ &self.pool}}diff --git a/crates/core/src/entrypoint/ampc/mod.rs b/crates/core/src/entrypoint/ampc/mod.rsindex dd3bfb4d..ebfc0ade 100644--- a/crates/core/src/entrypoint/ampc/mod.rs+++ b/crates/core/src/entrypoint/ampc/mod.rs@@ -14,5 +14,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+pub mod approximated_harmonic_centrality;pub mod dht;pub mod harmonic_centrality;diff --git a/crates/core/src/main.rs b/crates/core/src/main.rsindex fa88e73e..64a1b497 100644--- a/crates/core/src/main.rs+++ b/crates/core/src/main.rs@@ -144,6 +144,14 @@ enum AmpcOptions {/// Start a coordinator to distribute the harmonic centrality computation./// Workers needs to be started before the coordinator.HarmonicCoordinator { config_path: String },++ /// Start a worker to compute an approximation of the harmonic centrality of a graph.+ /// The approximation samples a subset of the graph and computes shortest paths from the sampled nodes.+ ApproxHarmonicWorker { config_path: String },++ /// Start a coordinator to distribute the approximation of the harmonic centrality computation.+ /// Workers needs to be started before the coordinator.+ ApproxHarmonicCoordinator { config_path: String },}#[derive(Subcommand)]@@ -453,6 +461,15 @@ fn main() -> Result<()> {let config: config::HarmonicCoordinatorConfig = load_toml_config(config_path);entrypoint::ampc::harmonic_centrality::coordinator::run(config)?;}++ AmpcOptions::ApproxHarmonicWorker { config_path } => {+ let config: config::ApproxHarmonicWorkerConfig = load_toml_config(config_path);+ entrypoint::ampc::approximated_harmonic_centrality::worker::run(config)?;+ }+ AmpcOptions::ApproxHarmonicCoordinator { config_path } => {+ let config: config::ApproxHarmonicCoordinatorConfig = load_toml_config(config_path);+ entrypoint::ampc::approximated_harmonic_centrality::coordinator::run(config)?;+ }},}