web search done right
limit warc files during webgraph building and other minor changes
Mikkel Denker4164cd3aebc6
diff --git a/src/bin/cuely.rs b/src/bin/cuely.rsindex 7d4cb2d6..494a0a47 100644--- a/src/bin/cuely.rs+++ b/src/bin/cuely.rs@@ -23,11 +23,12 @@ use tracing_subscriber::FmtSubscriber;#[derive(Parser, Debug)]struct Args {config_file: String,+ mapreduce_worker_addr: Option<String>,}fn main() {let subscriber = FmtSubscriber::builder()- .with_max_level(Level::TRACE)+ .with_max_level(Level::INFO).finish();tracing::subscriber::set_global_default(subscriber).unwrap();@@ -43,7 +44,7 @@ fn main() {.run().expect("Failed to index documents");}- Config::Webgraph(config) => WebgraphEntrypoint::from(config)+ Config::Webgraph(config) => WebgraphEntrypoint::new(config, args.mapreduce_worker_addr).run().expect("Failed to build webgraph"),}diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex 765d55ae..71258cf6 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -19,21 +19,11 @@ use crate::{webgraph::{FrozenWebgraph, Node, Webgraph, WebgraphBuilder},webpage::{self, Html},HttpConfig, LocalConfig, Result, WarcSource, WebgraphConfig, WebgraphLocalConfig,- WebgraphMasterConfig, WebgraphWorkerConfig,+ WebgraphMasterConfig,};use serde::{Deserialize, Serialize};use std::{net::SocketAddr, path::Path};-use tracing::{debug, info};--pub struct WebgraphEntrypoint {- config: WebgraphConfig,-}--impl From<WebgraphConfig> for WebgraphEntrypoint {- fn from(config: WebgraphConfig) -> Self {- Self { config }- }-}+use tracing::{debug, info, trace};#[derive(Debug, Serialize, Deserialize, Clone)]enum JobConfig {@@ -45,6 +35,7 @@ enum JobConfig {struct Job {config: JobConfig,warc_path: String,+ graph_base_path: String,}impl Map<FrozenWebgraph> for Job {@@ -53,7 +44,7 @@ impl Map<FrozenWebgraph> for Job {info!("processing {}", name);- let mut graph = WebgraphBuilder::new(Path::new("webgraph").join(name))+ let mut graph = WebgraphBuilder::new(Path::new(&self.graph_base_path).join(name)).with_host_graph().open();@@ -77,7 +68,7 @@ impl Map<FrozenWebgraph> for Job {}).filter(|link| webpage::domain(&link.source) != webpage::domain(&link.destination)){- debug!("inserting link {:?}", link);+ trace!("inserting link {:?}", link);graph.insert(Node::from(link.source),Node::from(link.destination),@@ -111,7 +102,18 @@ impl Reduce<FrozenWebgraph> for FrozenWebgraph {}}+pub struct WebgraphEntrypoint {+ config: WebgraphConfig,+ worker_addr: Option<String>,+}+impl WebgraphEntrypoint {+ pub fn new(config: WebgraphConfig, worker_addr: Option<String>) -> Self {+ Self {+ config,+ worker_addr,+ }+ }fn run_master(config: &WebgraphMasterConfig) -> Result<()> {info!("Running master for webgraph construction");@@ -129,21 +131,35 @@ impl WebgraphEntrypoint {WarcSource::Local(config) => JobConfig::Local(config),};+ let mut warc_paths: Box<dyn Iterator<Item = Job> + Send> =+ Box::new(warc_paths.into_iter().map(|warc_path| {+ Job {+ config: job_config.clone(),+ warc_path,+ graph_base_path: config+ .graph_base_path+ .clone()+ .unwrap_or_else(|| "webgraph".to_string()),+ }+ }));++ if let Some(limit) = config.limit_warc_files {+ warc_paths = Box::new(warc_paths.take(limit));+ }+warc_paths- .into_iter()- .take(15)- .map(|warc_path| Job {- config: job_config.clone(),- warc_path,- }).map_reduce(&workers).expect("failed to build webgraph");Ok(())}- fn run_worker(config: &WebgraphWorkerConfig) -> Result<()> {- Worker::run::<Job, FrozenWebgraph>(config.addr.parse::<SocketAddr>().unwrap())?;+ fn run_worker(worker_addr: String) -> Result<()> {+ Worker::run::<Job, FrozenWebgraph>(+ worker_addr+ .parse::<SocketAddr>()+ .expect("Could not parse worker address"),+ )?;Ok(())}@@ -161,6 +177,7 @@ impl WebgraphEntrypoint {.map(|path| Job {config: job_config.clone(),warc_path: path,+ graph_base_path: "webgraph".to_string(),}).map(|job| job.map()).fold(None, |acc: Option<FrozenWebgraph>, elem| match acc {@@ -174,7 +191,11 @@ impl WebgraphEntrypoint {pub fn run(&self) -> Result<()> {match &self.config {WebgraphConfig::Master(config) => WebgraphEntrypoint::run_master(config),- WebgraphConfig::Worker(config) => WebgraphEntrypoint::run_worker(config),+ WebgraphConfig::Worker => WebgraphEntrypoint::run_worker(+ self.worker_addr+ .clone()+ .expect("Worker address not specified"),+ ),WebgraphConfig::Local(config) => WebgraphEntrypoint::run_locally(config),}}diff --git a/src/lib.rs b/src/lib.rsindex cac32acb..3df89b4c 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -54,19 +54,16 @@ pub struct IndexingConfig {#[serde(tag = "mode")]pub enum WebgraphConfig {Master(WebgraphMasterConfig),- Worker(WebgraphWorkerConfig),+ Worker,Local(WebgraphLocalConfig),}#[derive(Debug, Deserialize, Clone)]pub struct WebgraphMasterConfig {+ limit_warc_files: Option<usize>,warc_source: WarcSource,workers: Vec<String>,-}--#[derive(Debug, Deserialize, Clone)]-pub struct WebgraphWorkerConfig {- addr: String,+ graph_base_path: Option<String>,}#[derive(Debug, Deserialize, Clone)]