web search done right
approx harmonic parallel graph executor
Mikkel Denker78c4676165fb
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rsindex a8abaab4..180c5f23 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs@@ -21,7 +21,8 @@ use std::{cmp, collections::BTreeMap};use super::{worker::ApproxCentralityWorker, ApproxCentralityJob, Mapper};use crate::{- ampc::dht::upsert, entrypoint::ampc::approximated_harmonic_centrality::DhtTable, webgraph,+ ampc::dht::upsert, entrypoint::ampc::approximated_harmonic_centrality::DhtTable,+ kahan_sum::KahanSum, webgraph,};use rayon::prelude::*;@@ -130,7 +131,14 @@ impl Mapper for ApproxCentralityMapper {worker: &<<Self as Mapper>::Job as super::Job>::Worker,dht: &crate::ampc::DhtConn<<<Self as Mapper>::Job as super::Job>::DhtTables>,) {- match self {+ let pool = rayon::ThreadPoolBuilder::new()+ .num_threads(usize::from(std::thread::available_parallelism().unwrap()))+ .stack_size(800_000_000)+ .thread_name(move |num| format!("approx-harmonic-mapper-{num}"))+ .build()+ .unwrap();++ pool.install(|| match self {ApproxCentralityMapper::InitCentrality => {let num_nodes = worker.graph().estimate_num_nodes();let nodes = worker.graph().nodes().collect::<Vec<_>>();@@ -138,7 +146,10 @@ impl Mapper for ApproxCentralityMapper {.par_chunks(BATCH_SIZE).progress_count(num_nodes as u64 / BATCH_SIZE as u64).for_each(|chunk| {- let pairs: Vec<_> = chunk.iter().map(|node| (*node, 0.0)).collect();+ let pairs: Vec<_> = chunk+ .iter()+ .map(|node| (*node, KahanSum::from(0.0)))+ .collect();dht.next().centrality.batch_set(pairs)});@@ -151,7 +162,9 @@ impl Mapper for ApproxCentralityMapper {.graph().random_nodes_with_outgoing(num_samples as usize);- sampled.into_par_iter().progress().for_each(|node| {+ let pb = indicatif::ProgressBar::new(sampled.len() as u64);++ sampled.into_par_iter().for_each(|node| {for chunk in workers.dijkstra(node, job.max_distance).into_iter()@@ -159,17 +172,21 @@ impl Mapper for ApproxCentralityMapper {if d == 0 {None} else {- Some((n, (1.0 / d as f32) * (job.norm as f32)))+ Some((n, KahanSum::from((1.0 / d as f64) * job.norm)))}}).chunks(BATCH_SIZE).into_iter(){let pairs: Vec<_> = chunk.collect();- dht.next().centrality.batch_upsert(upsert::F32Add, pairs);+ dht.next()+ .centrality+ .batch_upsert(upsert::KahanSumAdd, pairs);}++ pb.inc(1);});}- }+ })}}diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rsindex d8578f8d..064b35c3 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mod.rs@@ -21,6 +21,7 @@ pub mod worker;use std::net::SocketAddr;use crate::distributed::member::ShardId;+use crate::kahan_sum::KahanSum;use crate::{ampc::{prelude::*, DefaultDhtTable},webgraph,@@ -46,7 +47,7 @@ pub struct Meta {#[derive(bincode::Encode, bincode::Decode, Debug, Clone)]pub struct ApproxCentralityTables {- centrality: DefaultDhtTable<webgraph::NodeID, f32>,+ centrality: DefaultDhtTable<webgraph::NodeID, KahanSum>,meta: DefaultDhtTable<(), Meta>,}diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rsindex ee67f684..c2244109 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs@@ -123,9 +123,7 @@ impl RemoteWorker for RemoteApproxCentralityWorker {pub fn run(config: ApproxHarmonicWorkerConfig) -> Result<()> {let tokio_conf = config.clone();- let graph = Webgraph::builder(config.graph_path)- .single_threaded()- .open();+ let graph = Webgraph::builder(config.graph_path).open();let worker = ApproxCentralityWorker::new(graph, config.shard);let service = Service::ApproxHarmonicWorker {host: tokio_conf.host,diff --git a/crates/core/src/executor.rs b/crates/core/src/executor.rsindex fb0de0b3..1aa7454e 100644--- a/crates/core/src/executor.rs+++ b/crates/core/src/executor.rs@@ -40,7 +40,7 @@ impl Executor {pub fn with_threads(num_threads: usize, prefix: &'static str) -> Result<Executor> {let pool = ThreadPoolBuilder::new().num_threads(num_threads)- .stack_size(80_000_000)+ .stack_size(160_000_000).thread_name(move |num| format!("{prefix}{num}")).build()?;Ok(Executor::ThreadPool(pool))