Skip to content

web search done right

simplify dijkstra to reduce allocations and btree searches

Mikkel Denker6bc7a5caf029
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
index 180c5f23..cf55b04f 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
@@ -16,7 +16,6 @@
use indicatif::ParallelProgressIterator;
use itertools::Itertools;
-use rustc_hash::{FxHashMap, FxHashSet};
use std::{cmp, collections::BTreeMap};
use super::{worker::ApproxCentralityWorker, ApproxCentralityJob, Mapper};
@@ -43,76 +42,30 @@ impl Workers {
Self { worker }
}
- fn outgoing_nodes(
- &self,
- nodes: &[webgraph::NodeID],
- ) -> FxHashMap<webgraph::NodeID, FxHashSet<webgraph::NodeID>> {
- let mut res: FxHashMap<webgraph::NodeID, FxHashSet<webgraph::NodeID>> =
- FxHashMap::default();
-
- let limit = webgraph::EdgeLimit::Unlimited;
-
- for node in nodes {
- for edge in self.worker.graph().raw_outgoing_edges(node, limit) {
- res.entry(*node).or_default().insert(edge.to);
- }
- }
-
- res
- }
-
- fn run_batch(&self, batch: &mut Vec<(u8, webgraph::NodeID)>) -> BTreeMap<webgraph::NodeID, u8> {
- let mut new_distances: BTreeMap<webgraph::NodeID, u8> = BTreeMap::default();
-
- let nodes: Vec<_> = batch.iter().map(|(_, node)| *node).collect();
- let outgoing_nodes = self.outgoing_nodes(&nodes);
-
- for (dist, node) in batch.iter().cloned() {
- if let Some(outgoing) = outgoing_nodes.get(&node) {
- for outgoing in outgoing {
- let d = dist + 1;
-
- let current_dist = new_distances.get(outgoing).unwrap_or(&u8::MAX);
-
- if d < *current_dist {
- new_distances.insert(*outgoing, d);
- }
- }
- }
- }
-
- batch.clear();
-
- new_distances
- }
-
fn dijkstra(&self, source: webgraph::NodeID, max_dist: u8) -> BTreeMap<webgraph::NodeID, u8> {
let mut distances: BTreeMap<webgraph::NodeID, u8> = BTreeMap::default();
let mut queue = std::collections::BinaryHeap::new();
- let mut batch = Vec::with_capacity(BATCH_SIZE);
queue.push(cmp::Reverse((0u8, source)));
distances.insert(source, 0u8);
- loop {
- if queue.is_empty() && batch.is_empty() {
- break;
- }
-
- if let Some(cmp::Reverse((dist, node))) = queue.pop() {
- batch.push((dist, node));
- }
+ while let Some(cmp::Reverse((dist, node))) = queue.pop() {
+ for outgoing in self
+ .worker
+ .graph()
+ .raw_outgoing_edges(&node, webgraph::EdgeLimit::Unlimited)
+ .into_iter()
+ .map(|e| e.to)
+ {
+ let d = dist + 1;
- if batch.len() >= BATCH_SIZE || queue.is_empty() {
- for (node, dist) in self.run_batch(&mut batch) {
- let cur_dist = distances.get(&node).unwrap_or(&u8::MAX);
+ let current_dist = distances.entry(outgoing).or_insert(u8::MAX);
- if dist < *cur_dist {
- distances.insert(node, dist);
+ if d < *current_dist {
+ *current_dist = d;
- if dist < max_dist {
- queue.push(cmp::Reverse((dist, node)));
- }
+ if d < max_dist {
+ queue.push(cmp::Reverse((d, outgoing)));
}
}
}