Skip to content

web search done right

parallel local indexing

Mikkel Denkerd902e68d722b
diff --git a/configs/webgraph/local/local.toml b/configs/webgraph/local/local.toml
new file mode 100644
index 00000000..6b23e4aa
--- /dev/null
+++ b/configs/webgraph/local/local.toml
@@ -0,0 +1,7 @@
+limit_warc_files = 10
+graph_base_path = "data/webgraph"
+
+[warc_source]
+type = "HTTP"
+base_url = "https://data.commoncrawl.org/"
+warc_paths_file = "warc.paths"
\ No newline at end of file
diff --git a/justfile b/justfile
index f2a197ee..d37f31db 100644
--- a/justfile
+++ b/justfile
@@ -18,7 +18,7 @@
@local:
wget -nc -P data/warc_files https://data.commoncrawl.org/crawl-data/CC-MAIN-2022-05/segments/1642320299852.23/warc/CC-MAIN-20220116093137-20220116123137-00049.warc.gz
rm -rf data/index/
- cargo run --release -- indexer local configs/indexer/local.toml
+ RUSTFLAGS="-C target-cpu=native" cargo run --release -- indexer local configs/indexer/local.toml
mv data/index/CC-MAIN-*/* data/index/
just frontend
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 75e3b4f8..4be98569 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -16,6 +16,7 @@
use std::net::SocketAddr;
use std::path::Path;
+use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use tracing::{debug, info, trace};
@@ -115,15 +116,28 @@ impl Map<IndexingWorker, FrozenIndex> for Job {
impl Reduce<FrozenIndex> for Index {
fn reduce(self, element: FrozenIndex) -> Self {
- let other = element.into();
+ let other: Index = element.into();
- self.merge(other)
+ let other_path = other.path.clone();
+
+ let res = self.merge(other);
+
+ std::fs::remove_dir_all(other_path).unwrap();
+
+ res
}
}
impl Reduce<Index> for Index {
fn reduce(self, element: Index) -> Self {
- self.merge(element)
+ let other = element;
+ let other_path = other.path.clone();
+
+ let res = self.merge(other);
+
+ std::fs::remove_dir_all(other_path).unwrap();
+
+ res
}
}
@@ -196,11 +210,11 @@ impl Indexer {
warc_path: path,
base_path: "data/index".to_string(),
})
+ .par_bridge()
+ .panic_fuse()
.map(|job| job.map(&worker))
- .fold(None, |acc: Option<Index>, elem: FrozenIndex| match acc {
- Some(acc) => Some(acc.reduce(elem)),
- None => Some(elem.into()),
- });
+ .map(Index::from)
+ .reduce_with(|a, b| a.reduce(b));
Ok(())
}
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index b509d2a7..e3104592 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -20,6 +20,8 @@ use crate::{
webpage::Html,
HttpConfig, LocalConfig, Result, WarcSource, WebgraphLocalConfig, WebgraphMasterConfig,
};
+use rayon::prelude::ParallelBridge;
+use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use std::{net::SocketAddr, path::Path};
use tracing::{debug, info, trace};
@@ -171,19 +173,16 @@ impl WebgraphEntrypoint {
warc_paths
.into_iter()
+ .take(config.limit_warc_files.unwrap_or(usize::MAX))
.map(|path| Job {
config: job_config.clone(),
warc_path: path,
graph_base_path: "data/webgraph".to_string(),
})
+ .par_bridge()
.map(|job| job.map(&worker))
- .fold(
- None,
- |acc: Option<Webgraph>, elem: FrozenWebgraph| match acc {
- Some(acc) => Some(acc.reduce(elem)),
- None => Some(elem.into()),
- },
- );
+ .map(Webgraph::from)
+ .reduce_with(|a, b| a.reduce(b));
Ok(())
}
diff --git a/src/lib.rs b/src/lib.rs
index d2acac35..da515e22 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -75,6 +75,7 @@ pub struct WebgraphMasterConfig {
#[derive(Debug, Deserialize, Clone)]
pub struct WebgraphLocalConfig {
+ limit_warc_files: Option<usize>,
warc_source: WarcSource,
}
diff --git a/src/warc.rs b/src/warc.rs
index a5d7d2d3..32d48930 100644
--- a/src/warc.rs
+++ b/src/warc.rs
@@ -20,7 +20,6 @@ use std::fs::File;
use std::io::{BufRead, BufReader, Read};
use std::path::Path;
use std::thread::sleep;
-use std::time::Duration;
use flate2::read::MultiGzDecoder;
use tracing::debug;
@@ -112,12 +111,10 @@ impl WarcFile {
url += path;
let client = reqwest::blocking::ClientBuilder::new()
- .tcp_keepalive(Duration::from_secs(120))
- .connect_timeout(Duration::from_secs(20))
- .timeout(Duration::from_secs(300))
- .pool_idle_timeout(Duration::from_secs(0))
- .pool_max_idle_per_host(0)
- .connection_verbose(true)
+ .tcp_keepalive(None)
+ .connect_timeout(None)
+ .timeout(None)
+ .pool_idle_timeout(None)
.build()?;
let res = client.get(url).send()?;