Skip to content

web search done right

parallelise webgraph write finalisation using a counting semaphore

Mikkel Denkere7f21c84eb16
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index cd1d325a..6d5ad890 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -176,6 +176,13 @@ impl Webgraph {
let host_path = &config.host_graph_base_path;
let page_path = &config.page_graph_base_path;
+ const MAX_FINALIZE_CONCURRENT: usize = 8;
+ let (s, r) = crossbeam_channel::bounded(MAX_FINALIZE_CONCURRENT);
+
+ for _ in 0..MAX_FINALIZE_CONCURRENT {
+ s.send(())?;
+ }
+
for i in 0..num_workers {
let host_path = host_path.clone();
let host_path = Path::new(&host_path);
@@ -191,19 +198,22 @@ impl Webgraph {
};
let jobs = jobs.clone();
+ let (s, r) = (s.clone(), r.clone());
handlers.push(std::thread::spawn(move || {
for job in jobs.iter().skip(i).step_by(num_workers) {
worker.process_job(job);
}
- worker
+ r.recv().unwrap();
+ let res = (worker.host_graph.finalize(), worker.page_graph.finalize());
+ s.send(()).unwrap();
+ res
}));
}
let mut graphs = Vec::new();
for handler in handlers {
- let worker = handler.join().unwrap();
- graphs.push((worker.host_graph.finalize(), worker.page_graph.finalize()));
+ graphs.push(handler.join().unwrap());
}
let (mut host_graph, mut page_graph) = graphs.pop().unwrap();
diff --git a/core/src/webgraph/centrality/approx_harmonic.rs b/core/src/webgraph/centrality/approx_harmonic.rs
index 4ac86024..e545c2a1 100644
--- a/core/src/webgraph/centrality/approx_harmonic.rs
+++ b/core/src/webgraph/centrality/approx_harmonic.rs
@@ -36,9 +36,14 @@ pub struct ApproxHarmonic {
impl ApproxHarmonic {
pub fn build<P: AsRef<Path>>(graph: &Webgraph, output: P) -> Self {
- let num_nodes = graph.nodes().count();
+ let num_nodes = graph.par_nodes().count();
+
+ tracing::info!("found {} nodes in graph", num_nodes);
let num_samples = ((num_nodes as f64).log2() / EPSILON.powi(2)).ceil() as usize;
+
+ tracing::info!("sampling {} nodes", num_samples);
+
let sampled = graph.random_nodes(num_samples);
let res = Mutex::new(Self {