Skip to content

web search done right

hopefully improve webgraph merges by merging in parallel

Mikkel Denkerb003d21c4fe5
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index a739c50f..2ca019f0 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -25,7 +25,7 @@ use futures::StreamExt;
use itertools::Itertools;
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
-use std::{net::SocketAddr, path::Path};
+use std::{net::SocketAddr, path::Path, thread};
use tokio::pin;
use tracing::{info, trace};
@@ -97,6 +97,7 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {
std::fs::remove_file(path).unwrap();
}
+ graph.merge_segments(1);
info!("{} done", name);
@@ -249,7 +250,7 @@ impl Webgraph {
};
let worker = StatelessWorker::default();
- let mut graphs: Vec<_> = warc_paths
+ let graphs: Vec<_> = warc_paths
.into_iter()
.take(config.limit_warc_files.unwrap_or(usize::MAX))
.chunks(config.batch_size.unwrap_or(1))
@@ -268,14 +269,51 @@ impl Webgraph {
.collect();
if graphs.len() > 1 {
- let pointer = graphs.pop().unwrap();
- let mut graph = open_graph(pointer.path);
+ Self::merge(graphs);
+ }
+
+ Ok(())
+ }
- for other in graphs {
- graph = graph.reduce(other);
+ fn merge(graphs: Vec<GraphPointer>) {
+ let num_graphs = graphs.len();
+ let mut it = graphs.into_iter();
+ let num_cores = num_cpus::get();
+
+ let mut threads = Vec::new();
+
+ for _ in 0..(num_cores + 1) {
+ let graphs = it
+ .by_ref()
+ .take(((num_graphs as f64) / (num_cores as f64)).ceil() as usize)
+ .collect_vec();
+
+ if graphs.is_empty() {
+ break;
}
+
+ threads.push(thread::spawn(move || {
+ let mut it = graphs.into_iter();
+ let mut graph = open_graph(it.next().unwrap().path);
+
+ for other in it {
+ graph = graph.reduce(other);
+ graph.merge_segments(1);
+ }
+
+ graph
+ }));
}
- Ok(())
+ let mut graphs = Vec::new();
+ for thread in threads {
+ graphs.push(thread.join().unwrap());
+ }
+
+ let mut graph = graphs.pop().unwrap();
+
+ for other in graphs {
+ graph = graph.reduce(other);
+ }
}
}