Skip to content

web search done right

each worker in distributed harmonic should only use 1 thread for webgraph as they only need to traverse their edges which is not multithreaded anyway

Mikkel Denkerc9750a823ab6
diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs
index b06648cf..95c503d7 100644
--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs
+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs
@@ -225,7 +225,7 @@ pub fn run(config: HarmonicCoordinatorConfig) -> Result<()> {
let top_nodes = top_nodes
.iter()
- .map(|(id, c)| (id2node[id].clone(), *c))
+ .filter_map(|(id, c)| id2node.get(id).map(|n| (n.clone(), *c)))
.collect::<Vec<_>>();
store_csv(top_nodes, output_path.join("harmonic.csv"));
diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs
index 786114f4..169a0c8a 100644
--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs
+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs
@@ -176,7 +176,9 @@ impl RemoteWorker for RemoteCentralityWorker {
pub fn run(config: HarmonicWorkerConfig) -> Result<()> {
let tokio_conf = config.clone();
- let graph = Webgraph::builder(config.graph_path).open();
+ let graph = Webgraph::builder(config.graph_path)
+ .single_threaded()
+ .open();
let worker = CentralityWorker::new(config.shard, graph);
let service = Service::HarmonicWorker {
host: tokio_conf.host,