Skip to content

web search done right

Close each webgraph after creation. This ensures that we won't have too many open files.

Mikkel Denker2871d6bc8022
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 8a2e7309..7c0691e1 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -215,7 +215,15 @@ impl Indexer {
.panic_fuse()
.map(|job| job.map(&worker))
.map(Index::from)
- .reduce_with(|a, b| a.reduce(b));
+ .map(Some)
+ .reduce(
+ || None,
+ |a, b| match (a, b) {
+ (Some(a), Some(b)) => Some(a.reduce(b)),
+ (Some(graph), None) | (None, Some(graph)) => Some(graph),
+ (None, None) => None,
+ },
+ );
Ok(())
}
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index aeef16ba..5564bec9 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -27,6 +27,9 @@ use serde::{Deserialize, Serialize};
use std::{net::SocketAddr, path::Path};
use tracing::{debug, info, trace};
+#[derive(Debug, Serialize, Deserialize)]
+struct GraphPath(String);
+
#[derive(Debug, Serialize, Deserialize, Clone)]
enum JobConfig {
Http(HttpConfig),
@@ -40,49 +43,67 @@ struct Job {
graph_base_path: String,
}
-impl Map<StatelessWorker, FrozenWebgraph> for Job {
- fn map(self, _worker: &StatelessWorker) -> FrozenWebgraph {
- let name = self.warc_path.split('/').last().unwrap();
+fn open_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {
+ WebgraphBuilder::new(path)
+ .with_host_graph()
+ .with_full_graph()
+ .open()
+}
- info!("processing {}", name);
+fn process_job(job: &Job) -> webgraph::Webgraph {
+ let name = job.warc_path.split('/').last().unwrap();
- let mut graph = WebgraphBuilder::new(Path::new(&self.graph_base_path).join(name))
- .with_host_graph()
- .with_full_graph()
- .open();
+ info!("processing {}", name);
- let source = match self.config {
- JobConfig::Http(config) => WarcSource::HTTP(config),
- JobConfig::Local(config) => WarcSource::Local(config),
- };
+ let mut graph = open_graph(Path::new(&job.graph_base_path).join(name));
- debug!("downlooading warc file");
- let file = WarcFile::download(source, &self.warc_path).unwrap();
- debug!("finished downloading");
-
- for record in file.records().flatten() {
- let webpage = Html::parse(&record.response.body, &record.request.url);
- for link in webpage
- .links()
- .into_iter()
- .filter(|link| matches!(link.destination.protocol(), "http" | "https"))
- .filter(|link| link.source.domain() != link.destination.domain())
- {
- trace!("inserting link {:?}", link);
- graph.insert(
- Node::from(link.source),
- Node::from(link.destination),
- link.text,
- );
- }
+ let source = match job.config.clone() {
+ JobConfig::Http(config) => WarcSource::HTTP(config),
+ JobConfig::Local(config) => WarcSource::Local(config),
+ };
+
+ debug!("downlooading warc file");
+ let file = WarcFile::download(source, &job.warc_path).unwrap();
+ debug!("finished downloading");
+
+ for record in file.records().flatten() {
+ let webpage = Html::parse(&record.response.body, &record.request.url);
+ for link in webpage
+ .links()
+ .into_iter()
+ .filter(|link| matches!(link.destination.protocol(), "http" | "https"))
+ .filter(|link| link.source.domain() != link.destination.domain())
+ {
+ trace!("inserting link {:?}", link);
+ graph.insert(
+ Node::from(link.source),
+ Node::from(link.destination),
+ link.text,
+ );
}
+ }
- info!("{} done", name);
+ graph.flush();
+
+ info!("{} done", name);
+
+ graph
+}
+impl Map<StatelessWorker, FrozenWebgraph> for Job {
+ fn map(self, _worker: &StatelessWorker) -> FrozenWebgraph {
+ let graph = process_job(&self);
graph.into()
}
}
+impl Map<StatelessWorker, GraphPath> for Job {
+ fn map(self, _worker: &StatelessWorker) -> GraphPath {
+ let graph = process_job(&self);
+ GraphPath(graph.path)
+ }
+}
+
impl Reduce<FrozenWebgraph> for webgraph::Webgraph {
fn reduce(mut self, other: FrozenWebgraph) -> webgraph::Webgraph {
let other_path = match &other.root {
@@ -112,6 +133,24 @@ impl Reduce<webgraph::Webgraph> for webgraph::Webgraph {
}
}
+impl Reduce<GraphPath> for GraphPath {
+ fn reduce(self, other: GraphPath) -> Self {
+ let other_path = other.0.clone();
+ let self_path = self.0.clone();
+
+ {
+ let mut graph = open_graph(self.0);
+ let other_graph = open_graph(other.0);
+
+ graph.merge(other_graph);
+ }
+
+ std::fs::remove_dir_all(other_path).unwrap();
+
+ GraphPath(self_path)
+ }
+}
+
pub struct Webgraph {}
impl Webgraph {
@@ -147,8 +186,13 @@ impl Webgraph {
warc_paths = Box::new(warc_paths.take(limit));
}
- let _graph: webgraph::Webgraph = warc_paths
- .map_reduce(&workers)
+ let _graph: webgraph::Webgraph =
+ <Box<dyn Iterator<Item = Job> + std::marker::Send> as MapReduce<
+ StatelessWorker,
+ Job,
+ webgraph::FrozenWebgraph,
+ webgraph::Webgraph,
+ >>::map_reduce(warc_paths, &workers)
.expect("failed to build webgraph");
Ok(())
@@ -178,12 +222,22 @@ impl Webgraph {
.map(|path| Job {
config: job_config.clone(),
warc_path: path,
- graph_base_path: "data/webgraph".to_string(),
+ graph_base_path: config
+ .graph_base_path
+ .clone()
+ .unwrap_or_else(|| "data/webgraph".to_string()),
})
.par_bridge()
- .map(|job| job.map(&worker))
- .map(webgraph::Webgraph::from)
- .reduce_with(|a, b| a.reduce(b));
+ .map(|job| -> GraphPath { job.map(&worker) })
+ .map(Some)
+ .reduce(
+ || None,
+ |a, b| match (a, b) {
+ (Some(a), Some(b)) => Some(a.reduce(b)),
+ (Some(graph), None) | (None, Some(graph)) => Some(graph),
+ (None, None) => None,
+ },
+ );
Ok(())
}
diff --git a/src/lib.rs b/src/lib.rs
index 1fba47e3..ece3b966 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -80,6 +80,7 @@ pub struct WebgraphMasterConfig {
pub struct WebgraphLocalConfig {
limit_warc_files: Option<usize>,
warc_source: WarcSource,
+ graph_base_path: Option<String>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]