web search done right
Close each webgraph after creation. This ensures that we won't have too many open files.
Mikkel Denker2871d6bc8022
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 8a2e7309..7c0691e1 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -215,7 +215,15 @@ impl Indexer {.panic_fuse().map(|job| job.map(&worker)).map(Index::from)- .reduce_with(|a, b| a.reduce(b));+ .map(Some)+ .reduce(+ || None,+ |a, b| match (a, b) {+ (Some(a), Some(b)) => Some(a.reduce(b)),+ (Some(graph), None) | (None, Some(graph)) => Some(graph),+ (None, None) => None,+ },+ );Ok(())}diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex aeef16ba..5564bec9 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -27,6 +27,9 @@ use serde::{Deserialize, Serialize};use std::{net::SocketAddr, path::Path};use tracing::{debug, info, trace};+#[derive(Debug, Serialize, Deserialize)]+struct GraphPath(String);+#[derive(Debug, Serialize, Deserialize, Clone)]enum JobConfig {Http(HttpConfig),@@ -40,49 +43,67 @@ struct Job {graph_base_path: String,}-impl Map<StatelessWorker, FrozenWebgraph> for Job {- fn map(self, _worker: &StatelessWorker) -> FrozenWebgraph {- let name = self.warc_path.split('/').last().unwrap();+fn open_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {+ WebgraphBuilder::new(path)+ .with_host_graph()+ .with_full_graph()+ .open()+}- info!("processing {}", name);+fn process_job(job: &Job) -> webgraph::Webgraph {+ let name = job.warc_path.split('/').last().unwrap();- let mut graph = WebgraphBuilder::new(Path::new(&self.graph_base_path).join(name))- .with_host_graph()- .with_full_graph()- .open();+ info!("processing {}", name);- let source = match self.config {- JobConfig::Http(config) => WarcSource::HTTP(config),- JobConfig::Local(config) => WarcSource::Local(config),- };+ let mut graph = open_graph(Path::new(&job.graph_base_path).join(name));- debug!("downlooading warc file");- let file = WarcFile::download(source, &self.warc_path).unwrap();- debug!("finished downloading");-- for record in file.records().flatten() {- let webpage = Html::parse(&record.response.body, &record.request.url);- for link in webpage- .links()- .into_iter()- .filter(|link| matches!(link.destination.protocol(), "http" | "https"))- .filter(|link| link.source.domain() != link.destination.domain())- {- trace!("inserting link {:?}", link);- graph.insert(- Node::from(link.source),- Node::from(link.destination),- link.text,- );- }+ let source = match job.config.clone() {+ JobConfig::Http(config) => WarcSource::HTTP(config),+ JobConfig::Local(config) => WarcSource::Local(config),+ };++ debug!("downlooading warc file");+ let file = WarcFile::download(source, &job.warc_path).unwrap();+ debug!("finished downloading");++ for record in file.records().flatten() {+ let webpage = Html::parse(&record.response.body, &record.request.url);+ for link in webpage+ .links()+ .into_iter()+ .filter(|link| matches!(link.destination.protocol(), "http" | "https"))+ .filter(|link| link.source.domain() != link.destination.domain())+ {+ trace!("inserting link {:?}", link);+ graph.insert(+ Node::from(link.source),+ Node::from(link.destination),+ link.text,+ );}+ }- info!("{} done", name);+ graph.flush();++ info!("{} done", name);++ graph+}+impl Map<StatelessWorker, FrozenWebgraph> for Job {+ fn map(self, _worker: &StatelessWorker) -> FrozenWebgraph {+ let graph = process_job(&self);graph.into()}}+impl Map<StatelessWorker, GraphPath> for Job {+ fn map(self, _worker: &StatelessWorker) -> GraphPath {+ let graph = process_job(&self);+ GraphPath(graph.path)+ }+}+impl Reduce<FrozenWebgraph> for webgraph::Webgraph {fn reduce(mut self, other: FrozenWebgraph) -> webgraph::Webgraph {let other_path = match &other.root {@@ -112,6 +133,24 @@ impl Reduce<webgraph::Webgraph> for webgraph::Webgraph {}}+impl Reduce<GraphPath> for GraphPath {+ fn reduce(self, other: GraphPath) -> Self {+ let other_path = other.0.clone();+ let self_path = self.0.clone();++ {+ let mut graph = open_graph(self.0);+ let other_graph = open_graph(other.0);++ graph.merge(other_graph);+ }++ std::fs::remove_dir_all(other_path).unwrap();++ GraphPath(self_path)+ }+}+pub struct Webgraph {}impl Webgraph {@@ -147,8 +186,13 @@ impl Webgraph {warc_paths = Box::new(warc_paths.take(limit));}- let _graph: webgraph::Webgraph = warc_paths- .map_reduce(&workers)+ let _graph: webgraph::Webgraph =+ <Box<dyn Iterator<Item = Job> + std::marker::Send> as MapReduce<+ StatelessWorker,+ Job,+ webgraph::FrozenWebgraph,+ webgraph::Webgraph,+ >>::map_reduce(warc_paths, &workers).expect("failed to build webgraph");Ok(())@@ -178,12 +222,22 @@ impl Webgraph {.map(|path| Job {config: job_config.clone(),warc_path: path,- graph_base_path: "data/webgraph".to_string(),+ graph_base_path: config+ .graph_base_path+ .clone()+ .unwrap_or_else(|| "data/webgraph".to_string()),}).par_bridge()- .map(|job| job.map(&worker))- .map(webgraph::Webgraph::from)- .reduce_with(|a, b| a.reduce(b));+ .map(|job| -> GraphPath { job.map(&worker) })+ .map(Some)+ .reduce(+ || None,+ |a, b| match (a, b) {+ (Some(a), Some(b)) => Some(a.reduce(b)),+ (Some(graph), None) | (None, Some(graph)) => Some(graph),+ (None, None) => None,+ },+ );Ok(())}diff --git a/src/lib.rs b/src/lib.rsindex 1fba47e3..ece3b966 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -80,6 +80,7 @@ pub struct WebgraphMasterConfig {pub struct WebgraphLocalConfig {limit_warc_files: Option<usize>,warc_source: WarcSource,+ graph_base_path: Option<String>,}#[derive(Debug, Serialize, Deserialize, Clone)]