web search done right
parallelise webgraph write finalisation using a counting semaphore
Mikkel Denkere7f21c84eb16
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex cd1d325a..6d5ad890 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -176,6 +176,13 @@ impl Webgraph {let host_path = &config.host_graph_base_path;let page_path = &config.page_graph_base_path;+ const MAX_FINALIZE_CONCURRENT: usize = 8;+ let (s, r) = crossbeam_channel::bounded(MAX_FINALIZE_CONCURRENT);++ for _ in 0..MAX_FINALIZE_CONCURRENT {+ s.send(())?;+ }+for i in 0..num_workers {let host_path = host_path.clone();let host_path = Path::new(&host_path);@@ -191,19 +198,22 @@ impl Webgraph {};let jobs = jobs.clone();+ let (s, r) = (s.clone(), r.clone());handlers.push(std::thread::spawn(move || {for job in jobs.iter().skip(i).step_by(num_workers) {worker.process_job(job);}- worker+ r.recv().unwrap();+ let res = (worker.host_graph.finalize(), worker.page_graph.finalize());+ s.send(()).unwrap();+ res}));}let mut graphs = Vec::new();for handler in handlers {- let worker = handler.join().unwrap();- graphs.push((worker.host_graph.finalize(), worker.page_graph.finalize()));+ graphs.push(handler.join().unwrap());}let (mut host_graph, mut page_graph) = graphs.pop().unwrap();diff --git a/core/src/webgraph/centrality/approx_harmonic.rs b/core/src/webgraph/centrality/approx_harmonic.rsindex 4ac86024..e545c2a1 100644--- a/core/src/webgraph/centrality/approx_harmonic.rs+++ b/core/src/webgraph/centrality/approx_harmonic.rs@@ -36,9 +36,14 @@ pub struct ApproxHarmonic {impl ApproxHarmonic {pub fn build<P: AsRef<Path>>(graph: &Webgraph, output: P) -> Self {- let num_nodes = graph.nodes().count();+ let num_nodes = graph.par_nodes().count();++ tracing::info!("found {} nodes in graph", num_nodes);let num_samples = ((num_nodes as f64).log2() / EPSILON.powi(2)).ceil() as usize;++ tracing::info!("sampling {} nodes", num_samples);+let sampled = graph.random_nodes(num_samples);let res = Mutex::new(Self {