Skip to content

web search done right

s3 warc source

Mikkel Denkerae79c7af90c9
diff --git a/Cargo.lock b/Cargo.lock
index c39c741f..86970877 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -549,6 +549,16 @@ dependencies = [
"generic-array",
]
+[[package]]
+name = "block_on_proc"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b872f3528eeeb4370ee73b51194dc1cd93680c2d0eb6c7a223889038d2c1a167"
+dependencies = [
+ "quote",
+ "syn 1.0.109",
+]
+
[[package]]
name = "bumpalo"
version = "3.12.1"
@@ -3687,6 +3697,7 @@ dependencies = [
"aws-creds",
"aws-region",
"base64 0.13.1",
+ "block_on_proc",
"bytes",
"cfg-if",
"futures",
diff --git a/Cargo.toml b/Cargo.toml
index 322db5be..c53e50a8 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -87,7 +87,7 @@ lz-str = "0.2.1"
safetensors = "0.3.1"
publicsuffix = "2"
robotstxt-with-cache = "0.4.0"
-rust-s3 = "0.33.0"
+rust-s3 = { version = "0.33.0", features = ["blocking", "tokio"] }
rusqlite = { version = "0.29.0", features = ["bundled", "modern-full", "buildtime_bindgen"] }
async-channel = "1.8.0"
lru = "0.10.0"
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 91e24cca..5b0bc6c3 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -34,7 +34,7 @@ use crate::webgraph::{Node, Webgraph, WebgraphBuilder};
use crate::webpage::{Html, Link, Webpage};
use crate::{
human_website_annotations, HttpConfig, IndexingLocalConfig, IndexingMasterConfig, LocalConfig,
- Result, WarcSource,
+ Result, S3Config, WarcSource,
};
pub struct Indexer {}
@@ -43,6 +43,27 @@ pub struct Indexer {}
pub enum JobConfig {
Http(HttpConfig),
Local(LocalConfig),
+ S3(S3Config),
+}
+
+impl From<WarcSource> for JobConfig {
+ fn from(value: WarcSource) -> Self {
+ match value {
+ WarcSource::HTTP(config) => JobConfig::Http(config),
+ WarcSource::Local(config) => JobConfig::Local(config),
+ WarcSource::S3(config) => JobConfig::S3(config),
+ }
+ }
+}
+
+impl From<JobConfig> for WarcSource {
+ fn from(value: JobConfig) -> Self {
+ match value {
+ JobConfig::Http(config) => WarcSource::HTTP(config),
+ JobConfig::Local(config) => WarcSource::Local(config),
+ JobConfig::S3(config) => WarcSource::S3(config),
+ }
+ }
}
#[derive(Debug, Serialize, Deserialize)]
@@ -81,10 +102,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
let mut index = Index::open(Path::new(&job.base_path).join(name)).unwrap();
- let source = match job.source_config.clone() {
- JobConfig::Http(config) => WarcSource::HTTP(config),
- JobConfig::Local(config) => WarcSource::Local(config),
- };
+ let source: WarcSource = job.source_config.clone().into();
let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.base_path);
pin!(warc_files);
@@ -288,10 +306,7 @@ impl Indexer {
.map(|worker| worker.parse().unwrap())
.collect();
- let job_config = match config.warc_source.clone() {
- WarcSource::HTTP(config) => JobConfig::Http(config),
- WarcSource::Local(config) => JobConfig::Local(config),
- };
+ let job_config: JobConfig = config.warc_source.clone().into();
let warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(
warc_paths
@@ -355,10 +370,7 @@ impl Indexer {
pub fn run_locally(config: &IndexingLocalConfig) -> Result<()> {
let warc_paths = config.warc_source.paths()?;
- let job_config = match config.warc_source.clone() {
- WarcSource::HTTP(config) => JobConfig::Http(config),
- WarcSource::Local(config) => JobConfig::Local(config),
- };
+ let job_config: JobConfig = config.warc_source.clone().into();
let worker = IndexingWorker::new(
config.centrality_store_path.clone(),
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index 0dc440df..22028373 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -19,7 +19,8 @@ use crate::{
warc::WarcFile,
webgraph::{self, FrozenWebgraph, Node, WebgraphBuilder},
webpage::Html,
- HttpConfig, LocalConfig, Result, WarcSource, WebgraphLocalConfig, WebgraphMasterConfig,
+ HttpConfig, LocalConfig, Result, S3Config, WarcSource, WebgraphLocalConfig,
+ WebgraphMasterConfig,
};
use itertools::Itertools;
use rayon::prelude::*;
@@ -37,6 +38,27 @@ struct GraphPointer {
pub enum JobConfig {
Http(HttpConfig),
Local(LocalConfig),
+ S3(S3Config),
+}
+
+impl From<WarcSource> for JobConfig {
+ fn from(value: WarcSource) -> Self {
+ match value {
+ WarcSource::HTTP(config) => JobConfig::Http(config),
+ WarcSource::Local(config) => JobConfig::Local(config),
+ WarcSource::S3(config) => JobConfig::S3(config),
+ }
+ }
+}
+
+impl From<JobConfig> for WarcSource {
+ fn from(value: JobConfig) -> Self {
+ match value {
+ JobConfig::Http(config) => WarcSource::HTTP(config),
+ JobConfig::Local(config) => WarcSource::Local(config),
+ JobConfig::S3(config) => WarcSource::S3(config),
+ }
+ }
}
#[derive(Debug, Serialize, Deserialize)]
@@ -57,10 +79,7 @@ pub fn process_job(job: &Job) -> webgraph::Webgraph {
let mut graph = open_graph(Path::new(&job.graph_base_path).join(name));
- let source = match job.config.clone() {
- JobConfig::Http(config) => WarcSource::HTTP(config),
- JobConfig::Local(config) => WarcSource::Local(config),
- };
+ let source = WarcSource::from(job.config.clone());
let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.graph_base_path);
pin!(warc_files);
@@ -175,10 +194,7 @@ impl Webgraph {
.map(|worker| worker.parse().unwrap())
.collect();
- let job_config = match config.warc_source.clone() {
- WarcSource::HTTP(config) => JobConfig::Http(config),
- WarcSource::Local(config) => JobConfig::Local(config),
- };
+ let job_config = JobConfig::from(config.warc_source.clone());
let mut warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(
warc_paths
@@ -234,10 +250,7 @@ impl Webgraph {
pub fn run_locally(config: &WebgraphLocalConfig) -> Result<()> {
let warc_paths = config.warc_source.paths()?;
- let job_config = match config.warc_source.clone() {
- WarcSource::HTTP(config) => JobConfig::Http(config),
- WarcSource::Local(config) => JobConfig::Local(config),
- };
+ let job_config = JobConfig::from(config.warc_source.clone());
let worker = StatelessWorker::default();
let graphs: Vec<_> = warc_paths
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 19174508..e49f0dfb 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -132,6 +132,7 @@ pub struct WebgraphLocalConfig {
pub enum WarcSource {
HTTP(HttpConfig),
Local(LocalConfig),
+ S3(S3Config),
}
impl WarcSource {
@@ -147,6 +148,35 @@ impl WarcSource {
WarcSource::Local(config) => {
warc_paths = config.names.clone();
}
+ WarcSource::S3(config) => {
+ let bucket = s3::Bucket::new(
+ &config.bucket,
+ s3::Region::Custom {
+ region: "".to_string(),
+ endpoint: config.endpoint.clone(),
+ },
+ s3::creds::Credentials {
+ access_key: Some(config.access_key.clone()),
+ secret_key: Some(config.secret_key.clone()),
+ security_token: None,
+ session_token: None,
+ expiration: None,
+ },
+ )?
+ .with_path_style();
+
+ let objects = bucket.list_blocking(config.folder.clone(), None)?;
+
+ for p in objects.into_iter().filter_map(|o| {
+ if o.name.ends_with("warc.gz") {
+ Some(o.name)
+ } else {
+ None
+ }
+ }) {
+ warc_paths.push(p);
+ }
+ }
}
Ok(warc_paths)
@@ -165,6 +195,15 @@ pub struct HttpConfig {
warc_paths_file: String,
}
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct S3Config {
+ pub bucket: String,
+ pub folder: String,
+ pub access_key: String,
+ pub secret_key: String,
+ pub endpoint: String,
+}
+
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct FrontendConfig {
pub queries_csv_path: String,
@@ -196,15 +235,6 @@ pub struct SearchServerConfig {
pub lambda_model_path: Option<String>,
pub host: SocketAddr,
}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct S3Config {
- pub bucket: String,
- pub folder: String,
- pub access_key: String,
- pub secret_key: String,
- pub endpoint: String,
-}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct CrawlCoordinatorConfig {
pub crawldb_folder: String,
@@ -377,6 +407,9 @@ pub enum Error {
#[error("Crawler")]
Crawler(#[from] crate::crawler::Error),
+
+ #[error("S3")]
+ S3(#[from] s3::error::S3Error),
}
pub(crate) type Result<T> = std::result::Result<T, Error>;
diff --git a/core/src/warc.rs b/core/src/warc.rs
index 19956f50..f3714133 100644
--- a/core/src/warc.rs
+++ b/core/src/warc.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::distributed::retry_strategy::ExponentialBackoff;
-use crate::{Error, Result, WarcSource};
+use crate::{Error, Result, S3Config, WarcSource};
use std::collections::BTreeMap;
use std::fs::File;
use std::io::{BufRead, BufReader, Cursor, Read, Seek, Write};
@@ -109,6 +109,7 @@ impl WarcFile {
WarcSource::Local(config) => {
WarcFile::load_from_folder(warc_path, &config.folder, buf)
}
+ WarcSource::S3(config) => WarcFile::download_from_s3(warc_path, &config, buf),
};
if res.is_ok() {
@@ -165,6 +166,35 @@ impl WarcFile {
Ok(())
}
+
+ fn download_from_s3<W: Write + Seek>(
+ warc_path: &str,
+ config: &S3Config,
+ buf: &mut W,
+ ) -> Result<()> {
+ let bucket = s3::Bucket::new(
+ &config.bucket,
+ s3::Region::Custom {
+ region: "".to_string(),
+ endpoint: config.endpoint.clone(),
+ },
+ s3::creds::Credentials {
+ access_key: Some(config.access_key.clone()),
+ secret_key: Some(config.secret_key.clone()),
+ security_token: None,
+ session_token: None,
+ expiration: None,
+ },
+ )?
+ .with_path_style();
+
+ let path = Path::new(&config.folder).join(warc_path);
+ let res = bucket.get_object_blocking(path.as_os_str().to_str().unwrap())?;
+
+ buf.write_all(res.bytes())?;
+
+ Ok(())
+ }
}
#[derive(Debug)]