web search done right
parallel local indexing
Mikkel Denkerd902e68d722b
diff --git a/configs/webgraph/local/local.toml b/configs/webgraph/local/local.tomlnew file mode 100644index 00000000..6b23e4aa--- /dev/null+++ b/configs/webgraph/local/local.toml@@ -0,0 +1,7 @@+limit_warc_files = 10+graph_base_path = "data/webgraph"++[warc_source]+type = "HTTP"+base_url = "https://data.commoncrawl.org/"+warc_paths_file = "warc.paths"\ No newline at end of filediff --git a/justfile b/justfileindex f2a197ee..d37f31db 100644--- a/justfile+++ b/justfile@@ -18,7 +18,7 @@@local:wget -nc -P data/warc_files https://data.commoncrawl.org/crawl-data/CC-MAIN-2022-05/segments/1642320299852.23/warc/CC-MAIN-20220116093137-20220116123137-00049.warc.gzrm -rf data/index/- cargo run --release -- indexer local configs/indexer/local.toml+ RUSTFLAGS="-C target-cpu=native" cargo run --release -- indexer local configs/indexer/local.tomlmv data/index/CC-MAIN-*/* data/index/just frontenddiff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 75e3b4f8..4be98569 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -16,6 +16,7 @@use std::net::SocketAddr;use std::path::Path;+use rayon::prelude::*;use serde::{Deserialize, Serialize};use tracing::{debug, info, trace};@@ -115,15 +116,28 @@ impl Map<IndexingWorker, FrozenIndex> for Job {impl Reduce<FrozenIndex> for Index {fn reduce(self, element: FrozenIndex) -> Self {- let other = element.into();+ let other: Index = element.into();- self.merge(other)+ let other_path = other.path.clone();++ let res = self.merge(other);++ std::fs::remove_dir_all(other_path).unwrap();++ res}}impl Reduce<Index> for Index {fn reduce(self, element: Index) -> Self {- self.merge(element)+ let other = element;+ let other_path = other.path.clone();++ let res = self.merge(other);++ std::fs::remove_dir_all(other_path).unwrap();++ res}}@@ -196,11 +210,11 @@ impl Indexer {warc_path: path,base_path: "data/index".to_string(),})+ .par_bridge()+ .panic_fuse().map(|job| job.map(&worker))- .fold(None, |acc: Option<Index>, elem: FrozenIndex| match acc {- Some(acc) => Some(acc.reduce(elem)),- None => Some(elem.into()),- });+ .map(Index::from)+ .reduce_with(|a, b| a.reduce(b));Ok(())}diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex b509d2a7..e3104592 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -20,6 +20,8 @@ use crate::{webpage::Html,HttpConfig, LocalConfig, Result, WarcSource, WebgraphLocalConfig, WebgraphMasterConfig,};+use rayon::prelude::ParallelBridge;+use rayon::prelude::*;use serde::{Deserialize, Serialize};use std::{net::SocketAddr, path::Path};use tracing::{debug, info, trace};@@ -171,19 +173,16 @@ impl WebgraphEntrypoint {warc_paths.into_iter()+ .take(config.limit_warc_files.unwrap_or(usize::MAX)).map(|path| Job {config: job_config.clone(),warc_path: path,graph_base_path: "data/webgraph".to_string(),})+ .par_bridge().map(|job| job.map(&worker))- .fold(- None,- |acc: Option<Webgraph>, elem: FrozenWebgraph| match acc {- Some(acc) => Some(acc.reduce(elem)),- None => Some(elem.into()),- },- );+ .map(Webgraph::from)+ .reduce_with(|a, b| a.reduce(b));Ok(())}diff --git a/src/lib.rs b/src/lib.rsindex d2acac35..da515e22 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -75,6 +75,7 @@ pub struct WebgraphMasterConfig {#[derive(Debug, Deserialize, Clone)]pub struct WebgraphLocalConfig {+ limit_warc_files: Option<usize>,warc_source: WarcSource,}diff --git a/src/warc.rs b/src/warc.rsindex a5d7d2d3..32d48930 100644--- a/src/warc.rs+++ b/src/warc.rs@@ -20,7 +20,6 @@ use std::fs::File;use std::io::{BufRead, BufReader, Read};use std::path::Path;use std::thread::sleep;-use std::time::Duration;use flate2::read::MultiGzDecoder;use tracing::debug;@@ -112,12 +111,10 @@ impl WarcFile {url += path;let client = reqwest::blocking::ClientBuilder::new()- .tcp_keepalive(Duration::from_secs(120))- .connect_timeout(Duration::from_secs(20))- .timeout(Duration::from_secs(300))- .pool_idle_timeout(Duration::from_secs(0))- .pool_max_idle_per_host(0)- .connection_verbose(true)+ .tcp_keepalive(None)+ .connect_timeout(None)+ .timeout(None)+ .pool_idle_timeout(None).build()?;let res = client.get(url).send()?;