web search done right
chunk id2node requests into smaller parts to avoid timeout
Mikkel Denkerdf7ac44061c5
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rsindex 23a7a73d..686490ef 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs@@ -247,7 +247,12 @@ pub fn run(config: ApproxHarmonicCoordinatorConfig) -> Result<()> {let id2node: BTreeMap<_, _> = cluster.workers.iter()- .flat_map(|w| w.batch_id2node(ids.clone()))+ .flat_map(|w| {+ ids.clone()+ .chunks(10_000)+ .flat_map(move |c| w.batch_id2node(c.to_vec()))+ .collect::<Vec<_>>()+ }).collect();let top_nodes = top_nodesdiff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rsindex 3a13b577..84822151 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs@@ -220,7 +220,12 @@ pub fn run(config: HarmonicCoordinatorConfig) -> Result<()> {let id2node: BTreeMap<_, _> = cluster.workers.iter()- .flat_map(|w| w.batch_id2node(ids.clone()))+ .flat_map(|w| {+ ids.clone()+ .chunks(10_000)+ .flat_map(move |c| w.batch_id2node(c.to_vec()))+ .collect::<Vec<_>>()+ }).collect();let top_nodes = top_nodesdiff --git a/crates/core/src/webgraph/centrality/mod.rs b/crates/core/src/webgraph/centrality/mod.rsindex ff012f3e..b3392fcb 100644--- a/crates/core/src/webgraph/centrality/mod.rs+++ b/crates/core/src/webgraph/centrality/mod.rs@@ -80,7 +80,7 @@ wherefor (node_id, centrality) in centralities {store.insert(node_id, centrality).unwrap();- if store.uncommitted_inserts() >= 1_000_000 {+ if store.uncommitted_inserts() >= 100_000_000 {store.commit().unwrap();}}@@ -103,7 +103,7 @@ where{rank_store.insert(node_id, rank as u64).unwrap();- if rank_store.uncommitted_inserts() >= 1_000_000 {+ if rank_store.uncommitted_inserts() >= 100_000_000 {rank_store.commit().unwrap();}}@@ -120,7 +120,8 @@ mod tests {#[test]fn test_top_k() {- let hits = [(SortableFloat(0.0), NodeID::from(0_u64)),+ let hits = [+ (SortableFloat(0.0), NodeID::from(0_u64)),(SortableFloat(1.0), NodeID::from(1_u64)),(SortableFloat(2.0), NodeID::from(2_u64)),(SortableFloat(3.0), NodeID::from(3_u64)),@@ -129,7 +130,8 @@ mod tests {(SortableFloat(6.0), NodeID::from(6_u64)),(SortableFloat(7.0), NodeID::from(7_u64)),(SortableFloat(8.0), NodeID::from(8_u64)),- (SortableFloat(9.0), NodeID::from(9_u64))];+ (SortableFloat(9.0), NodeID::from(9_u64)),+ ];let top_5 = crate::sorted_k(hits.iter().copied(), 5);assert_eq!(@@ -159,7 +161,8 @@ mod tests {#[test]fn test_top_k_reversed() {- let hits = [(SortableFloat(9.0), NodeID::from(9_u64)),+ let hits = [+ (SortableFloat(9.0), NodeID::from(9_u64)),(SortableFloat(8.0), NodeID::from(8_u64)),(SortableFloat(7.0), NodeID::from(7_u64)),(SortableFloat(6.0), NodeID::from(6_u64)),@@ -168,7 +171,8 @@ mod tests {(SortableFloat(3.0), NodeID::from(3_u64)),(SortableFloat(2.0), NodeID::from(2_u64)),(SortableFloat(1.0), NodeID::from(1_u64)),- (SortableFloat(0.0), NodeID::from(0_u64))];+ (SortableFloat(0.0), NodeID::from(0_u64)),+ ];let top_5 = crate::sorted_k(hits.iter().copied().map(Reverse), 5).into_iter()