web search done right
s3 warc source
Mikkel Denkerae79c7af90c9
diff --git a/Cargo.lock b/Cargo.lockindex c39c741f..86970877 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -549,6 +549,16 @@ dependencies = ["generic-array",]+[[package]]+name = "block_on_proc"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b872f3528eeeb4370ee73b51194dc1cd93680c2d0eb6c7a223889038d2c1a167"+dependencies = [+ "quote",+ "syn 1.0.109",+]+[[package]]name = "bumpalo"version = "3.12.1"@@ -3687,6 +3697,7 @@ dependencies = ["aws-creds","aws-region","base64 0.13.1",+ "block_on_proc","bytes","cfg-if","futures",diff --git a/Cargo.toml b/Cargo.tomlindex 322db5be..c53e50a8 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -87,7 +87,7 @@ lz-str = "0.2.1"safetensors = "0.3.1"publicsuffix = "2"robotstxt-with-cache = "0.4.0"-rust-s3 = "0.33.0"+rust-s3 = { version = "0.33.0", features = ["blocking", "tokio"] }rusqlite = { version = "0.29.0", features = ["bundled", "modern-full", "buildtime_bindgen"] }async-channel = "1.8.0"lru = "0.10.0"diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 91e24cca..5b0bc6c3 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -34,7 +34,7 @@ use crate::webgraph::{Node, Webgraph, WebgraphBuilder};use crate::webpage::{Html, Link, Webpage};use crate::{human_website_annotations, HttpConfig, IndexingLocalConfig, IndexingMasterConfig, LocalConfig,- Result, WarcSource,+ Result, S3Config, WarcSource,};pub struct Indexer {}@@ -43,6 +43,27 @@ pub struct Indexer {}pub enum JobConfig {Http(HttpConfig),Local(LocalConfig),+ S3(S3Config),+}++impl From<WarcSource> for JobConfig {+ fn from(value: WarcSource) -> Self {+ match value {+ WarcSource::HTTP(config) => JobConfig::Http(config),+ WarcSource::Local(config) => JobConfig::Local(config),+ WarcSource::S3(config) => JobConfig::S3(config),+ }+ }+}++impl From<JobConfig> for WarcSource {+ fn from(value: JobConfig) -> Self {+ match value {+ JobConfig::Http(config) => WarcSource::HTTP(config),+ JobConfig::Local(config) => WarcSource::Local(config),+ JobConfig::S3(config) => WarcSource::S3(config),+ }+ }}#[derive(Debug, Serialize, Deserialize)]@@ -81,10 +102,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {let mut index = Index::open(Path::new(&job.base_path).join(name)).unwrap();- let source = match job.source_config.clone() {- JobConfig::Http(config) => WarcSource::HTTP(config),- JobConfig::Local(config) => WarcSource::Local(config),- };+ let source: WarcSource = job.source_config.clone().into();let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.base_path);pin!(warc_files);@@ -288,10 +306,7 @@ impl Indexer {.map(|worker| worker.parse().unwrap()).collect();- let job_config = match config.warc_source.clone() {- WarcSource::HTTP(config) => JobConfig::Http(config),- WarcSource::Local(config) => JobConfig::Local(config),- };+ let job_config: JobConfig = config.warc_source.clone().into();let warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(warc_paths@@ -355,10 +370,7 @@ impl Indexer {pub fn run_locally(config: &IndexingLocalConfig) -> Result<()> {let warc_paths = config.warc_source.paths()?;- let job_config = match config.warc_source.clone() {- WarcSource::HTTP(config) => JobConfig::Http(config),- WarcSource::Local(config) => JobConfig::Local(config),- };+ let job_config: JobConfig = config.warc_source.clone().into();let worker = IndexingWorker::new(config.centrality_store_path.clone(),diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex 0dc440df..22028373 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -19,7 +19,8 @@ use crate::{warc::WarcFile,webgraph::{self, FrozenWebgraph, Node, WebgraphBuilder},webpage::Html,- HttpConfig, LocalConfig, Result, WarcSource, WebgraphLocalConfig, WebgraphMasterConfig,+ HttpConfig, LocalConfig, Result, S3Config, WarcSource, WebgraphLocalConfig,+ WebgraphMasterConfig,};use itertools::Itertools;use rayon::prelude::*;@@ -37,6 +38,27 @@ struct GraphPointer {pub enum JobConfig {Http(HttpConfig),Local(LocalConfig),+ S3(S3Config),+}++impl From<WarcSource> for JobConfig {+ fn from(value: WarcSource) -> Self {+ match value {+ WarcSource::HTTP(config) => JobConfig::Http(config),+ WarcSource::Local(config) => JobConfig::Local(config),+ WarcSource::S3(config) => JobConfig::S3(config),+ }+ }+}++impl From<JobConfig> for WarcSource {+ fn from(value: JobConfig) -> Self {+ match value {+ JobConfig::Http(config) => WarcSource::HTTP(config),+ JobConfig::Local(config) => WarcSource::Local(config),+ JobConfig::S3(config) => WarcSource::S3(config),+ }+ }}#[derive(Debug, Serialize, Deserialize)]@@ -57,10 +79,7 @@ pub fn process_job(job: &Job) -> webgraph::Webgraph {let mut graph = open_graph(Path::new(&job.graph_base_path).join(name));- let source = match job.config.clone() {- JobConfig::Http(config) => WarcSource::HTTP(config),- JobConfig::Local(config) => WarcSource::Local(config),- };+ let source = WarcSource::from(job.config.clone());let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.graph_base_path);pin!(warc_files);@@ -175,10 +194,7 @@ impl Webgraph {.map(|worker| worker.parse().unwrap()).collect();- let job_config = match config.warc_source.clone() {- WarcSource::HTTP(config) => JobConfig::Http(config),- WarcSource::Local(config) => JobConfig::Local(config),- };+ let job_config = JobConfig::from(config.warc_source.clone());let mut warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(warc_paths@@ -234,10 +250,7 @@ impl Webgraph {pub fn run_locally(config: &WebgraphLocalConfig) -> Result<()> {let warc_paths = config.warc_source.paths()?;- let job_config = match config.warc_source.clone() {- WarcSource::HTTP(config) => JobConfig::Http(config),- WarcSource::Local(config) => JobConfig::Local(config),- };+ let job_config = JobConfig::from(config.warc_source.clone());let worker = StatelessWorker::default();let graphs: Vec<_> = warc_pathsdiff --git a/core/src/lib.rs b/core/src/lib.rsindex 19174508..e49f0dfb 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -132,6 +132,7 @@ pub struct WebgraphLocalConfig {pub enum WarcSource {HTTP(HttpConfig),Local(LocalConfig),+ S3(S3Config),}impl WarcSource {@@ -147,6 +148,35 @@ impl WarcSource {WarcSource::Local(config) => {warc_paths = config.names.clone();}+ WarcSource::S3(config) => {+ let bucket = s3::Bucket::new(+ &config.bucket,+ s3::Region::Custom {+ region: "".to_string(),+ endpoint: config.endpoint.clone(),+ },+ s3::creds::Credentials {+ access_key: Some(config.access_key.clone()),+ secret_key: Some(config.secret_key.clone()),+ security_token: None,+ session_token: None,+ expiration: None,+ },+ )?+ .with_path_style();++ let objects = bucket.list_blocking(config.folder.clone(), None)?;++ for p in objects.into_iter().filter_map(|o| {+ if o.name.ends_with("warc.gz") {+ Some(o.name)+ } else {+ None+ }+ }) {+ warc_paths.push(p);+ }+ }}Ok(warc_paths)@@ -165,6 +195,15 @@ pub struct HttpConfig {warc_paths_file: String,}+#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct S3Config {+ pub bucket: String,+ pub folder: String,+ pub access_key: String,+ pub secret_key: String,+ pub endpoint: String,+}+#[derive(Debug, Serialize, Deserialize, Clone)]pub struct FrontendConfig {pub queries_csv_path: String,@@ -196,15 +235,6 @@ pub struct SearchServerConfig {pub lambda_model_path: Option<String>,pub host: SocketAddr,}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct S3Config {- pub bucket: String,- pub folder: String,- pub access_key: String,- pub secret_key: String,- pub endpoint: String,-}#[derive(Debug, Serialize, Deserialize, Clone)]pub struct CrawlCoordinatorConfig {pub crawldb_folder: String,@@ -377,6 +407,9 @@ pub enum Error {#[error("Crawler")]Crawler(#[from] crate::crawler::Error),++ #[error("S3")]+ S3(#[from] s3::error::S3Error),}pub(crate) type Result<T> = std::result::Result<T, Error>;diff --git a/core/src/warc.rs b/core/src/warc.rsindex 19956f50..f3714133 100644--- a/core/src/warc.rs+++ b/core/src/warc.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::distributed::retry_strategy::ExponentialBackoff;-use crate::{Error, Result, WarcSource};+use crate::{Error, Result, S3Config, WarcSource};use std::collections::BTreeMap;use std::fs::File;use std::io::{BufRead, BufReader, Cursor, Read, Seek, Write};@@ -109,6 +109,7 @@ impl WarcFile {WarcSource::Local(config) => {WarcFile::load_from_folder(warc_path, &config.folder, buf)}+ WarcSource::S3(config) => WarcFile::download_from_s3(warc_path, &config, buf),};if res.is_ok() {@@ -165,6 +166,35 @@ impl WarcFile {Ok(())}++ fn download_from_s3<W: Write + Seek>(+ warc_path: &str,+ config: &S3Config,+ buf: &mut W,+ ) -> Result<()> {+ let bucket = s3::Bucket::new(+ &config.bucket,+ s3::Region::Custom {+ region: "".to_string(),+ endpoint: config.endpoint.clone(),+ },+ s3::creds::Credentials {+ access_key: Some(config.access_key.clone()),+ secret_key: Some(config.secret_key.clone()),+ security_token: None,+ session_token: None,+ expiration: None,+ },+ )?+ .with_path_style();++ let path = Path::new(&config.folder).join(warc_path);+ let res = bucket.get_object_blocking(path.as_os_str().to_str().unwrap())?;++ buf.write_all(res.bytes())?;++ Ok(())+ }}#[derive(Debug)]