Skip to content

web search done right

chunk id2node requests into smaller parts to avoid timeout

Mikkel Denkerdf7ac44061c5
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
index 23a7a73d..686490ef 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
@@ -247,7 +247,12 @@ pub fn run(config: ApproxHarmonicCoordinatorConfig) -> Result<()> {
let id2node: BTreeMap<_, _> = cluster
.workers
.iter()
- .flat_map(|w| w.batch_id2node(ids.clone()))
+ .flat_map(|w| {
+ ids.clone()
+ .chunks(10_000)
+ .flat_map(move |c| w.batch_id2node(c.to_vec()))
+ .collect::<Vec<_>>()
+ })
.collect();
let top_nodes = top_nodes
diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs
index 3a13b577..84822151 100644
--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs
+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs
@@ -220,7 +220,12 @@ pub fn run(config: HarmonicCoordinatorConfig) -> Result<()> {
let id2node: BTreeMap<_, _> = cluster
.workers
.iter()
- .flat_map(|w| w.batch_id2node(ids.clone()))
+ .flat_map(|w| {
+ ids.clone()
+ .chunks(10_000)
+ .flat_map(move |c| w.batch_id2node(c.to_vec()))
+ .collect::<Vec<_>>()
+ })
.collect();
let top_nodes = top_nodes
diff --git a/crates/core/src/webgraph/centrality/mod.rs b/crates/core/src/webgraph/centrality/mod.rs
index ff012f3e..b3392fcb 100644
--- a/crates/core/src/webgraph/centrality/mod.rs
+++ b/crates/core/src/webgraph/centrality/mod.rs
@@ -80,7 +80,7 @@ where
for (node_id, centrality) in centralities {
store.insert(node_id, centrality).unwrap();
- if store.uncommitted_inserts() >= 1_000_000 {
+ if store.uncommitted_inserts() >= 100_000_000 {
store.commit().unwrap();
}
}
@@ -103,7 +103,7 @@ where
{
rank_store.insert(node_id, rank as u64).unwrap();
- if rank_store.uncommitted_inserts() >= 1_000_000 {
+ if rank_store.uncommitted_inserts() >= 100_000_000 {
rank_store.commit().unwrap();
}
}
@@ -120,7 +120,8 @@ mod tests {
#[test]
fn test_top_k() {
- let hits = [(SortableFloat(0.0), NodeID::from(0_u64)),
+ let hits = [
+ (SortableFloat(0.0), NodeID::from(0_u64)),
(SortableFloat(1.0), NodeID::from(1_u64)),
(SortableFloat(2.0), NodeID::from(2_u64)),
(SortableFloat(3.0), NodeID::from(3_u64)),
@@ -129,7 +130,8 @@ mod tests {
(SortableFloat(6.0), NodeID::from(6_u64)),
(SortableFloat(7.0), NodeID::from(7_u64)),
(SortableFloat(8.0), NodeID::from(8_u64)),
- (SortableFloat(9.0), NodeID::from(9_u64))];
+ (SortableFloat(9.0), NodeID::from(9_u64)),
+ ];
let top_5 = crate::sorted_k(hits.iter().copied(), 5);
assert_eq!(
@@ -159,7 +161,8 @@ mod tests {
#[test]
fn test_top_k_reversed() {
- let hits = [(SortableFloat(9.0), NodeID::from(9_u64)),
+ let hits = [
+ (SortableFloat(9.0), NodeID::from(9_u64)),
(SortableFloat(8.0), NodeID::from(8_u64)),
(SortableFloat(7.0), NodeID::from(7_u64)),
(SortableFloat(6.0), NodeID::from(6_u64)),
@@ -168,7 +171,8 @@ mod tests {
(SortableFloat(3.0), NodeID::from(3_u64)),
(SortableFloat(2.0), NodeID::from(2_u64)),
(SortableFloat(1.0), NodeID::from(1_u64)),
- (SortableFloat(0.0), NodeID::from(0_u64))];
+ (SortableFloat(0.0), NodeID::from(0_u64)),
+ ];
let top_5 = crate::sorted_k(hits.iter().copied().map(Reverse), 5)
.into_iter()