web search done right
download from http
Mikkel Denkerc6c200564cbb
diff --git a/Cargo.toml b/Cargo.tomlindex c02f8296..6e1eaf26 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -13,6 +13,7 @@ bincode = "1.3.3"encoding_rs = "0.8.31"flate2 = "1.0.23"itertools = "0.10.3"+reqwest = "0.11.10"rusoto_core = "0.48.0"rusoto_s3 = "0.48.0"scraper = "0.13.0"diff --git a/configs/indexer.toml b/configs/indexer.tomlindex f55b6fc5..108b529c 100644--- a/configs/indexer.toml+++ b/configs/indexer.toml@@ -1,7 +1,16 @@mode = "Indexer"warc_paths_file = "warc.paths"-[s3]-name = "us-east-1"-endpoint = "https://s3.us-east-1.amazonaws.com"-bucket = "commoncrawl"+# [warc_source]+# type = "S3"+#+# [warc_source.args]+# name = "us-east-1"+# endpoint = "https://s3.us-east-1.amazonaws.com"+# bucket = "commoncrawl"++[warc_source]+type = "HTTP"++[warc_source.args]+base_url = "https://data.commoncrawl.org/"diff --git a/src/indexer.rs b/src/indexer.rsindex e294c15c..108f21d1 100644--- a/src/indexer.rs+++ b/src/indexer.rs@@ -6,7 +6,7 @@ use tokio::io::AsyncReadExt;use crate::warc::WarcFile;use crate::webpage::Webpage;-use crate::{Config, Error, Result};+use crate::{Config, Error, Result, WarcSource};pub struct Indexer {warc_paths: Vec<String>,@@ -26,16 +26,27 @@ impl Indexer {}pub async fn run(self) -> Result<()> {+ let download_config = self+ .config+ .warc_source+ .expect("Indexing needs a warc source");for warc_s3_path in self.warc_paths {println!("{}", warc_s3_path);- let raw_object = Indexer::download_from_s3(- warc_s3_path,- self.config.s3.name.clone(),- self.config.s3.endpoint.clone(),- self.config.s3.bucket.clone(),- )- .await?;+ let raw_object = match download_config {+ WarcSource::S3(config) => {+ Indexer::download_from_s3(+ warc_s3_path,+ config.name.clone(),+ config.endpoint.clone(),+ config.bucket.clone(),+ )+ .await?+ }+ WarcSource::HTTP(config) => {+ Indexer::download_from_http(warc_s3_path, config.base_url.clone()).await?+ }+ };println!("Downloaded {} bytes", raw_object.len());let warc = WarcFile::new(&raw_object[..]);@@ -51,6 +62,19 @@ impl Indexer {Ok(())}+ async fn download_from_http(warc_path: String, base_url: String) -> Result<Vec<u8>> {+ let mut url = base_url;+ if !url.ends_with('/') {+ url += "/";+ }+ url += &warc_path;++ let client = reqwest::Client::new();+ let res = client.get(url).send().await?;++ Ok(Vec::from(&res.bytes().await?[..]))+ }+async fn download_from_s3(key: String,region_name: String,diff --git a/src/lib.rs b/src/lib.rsindex 9a938697..16b7cfbc 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -13,10 +13,17 @@ pub use indexer::Indexer;#[derive(Debug, Deserialize)]pub struct Config {pub mode: Mode,- s3: S3Config,+ warc_source: Option<WarcSource>,warc_paths_file: String,}+#[derive(Debug, Deserialize)]+#[serde(tag = "type", content = "args")]+pub enum WarcSource {+ S3(S3Config),+ HTTP(HttpConfig),+}+#[derive(Debug, Deserialize)]pub enum Mode {Indexer,@@ -29,11 +36,19 @@ pub struct S3Config {bucket: String,}+#[derive(Debug, Deserialize)]+pub struct HttpConfig {+ base_url: String,+}+#[derive(Error, Debug)]pub enum Error {#[error("Failed to download object from S3")]S3DownloadError,+ #[error("Failed to download object from HTTP")]+ HTTPDownloadERror(#[from] reqwest::Error),+#[error("Failed to get the object from S3")]GetObjectError(#[from] rusoto_core::RusotoError<rusoto_s3::GetObjectError>),