Skip to content

web search done right

download from http

Mikkel Denkerc6c200564cbb
diff --git a/Cargo.toml b/Cargo.toml
index c02f8296..6e1eaf26 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,6 +13,7 @@ bincode = "1.3.3"
encoding_rs = "0.8.31"
flate2 = "1.0.23"
itertools = "0.10.3"
+reqwest = "0.11.10"
rusoto_core = "0.48.0"
rusoto_s3 = "0.48.0"
scraper = "0.13.0"
diff --git a/configs/indexer.toml b/configs/indexer.toml
index f55b6fc5..108b529c 100644
--- a/configs/indexer.toml
+++ b/configs/indexer.toml
@@ -1,7 +1,16 @@
mode = "Indexer"
warc_paths_file = "warc.paths"
-[s3]
-name = "us-east-1"
-endpoint = "https://s3.us-east-1.amazonaws.com"
-bucket = "commoncrawl"
+# [warc_source]
+# type = "S3"
+#
+# [warc_source.args]
+# name = "us-east-1"
+# endpoint = "https://s3.us-east-1.amazonaws.com"
+# bucket = "commoncrawl"
+
+[warc_source]
+type = "HTTP"
+
+[warc_source.args]
+base_url = "https://data.commoncrawl.org/"
diff --git a/src/indexer.rs b/src/indexer.rs
index e294c15c..108f21d1 100644
--- a/src/indexer.rs
+++ b/src/indexer.rs
@@ -6,7 +6,7 @@ use tokio::io::AsyncReadExt;
use crate::warc::WarcFile;
use crate::webpage::Webpage;
-use crate::{Config, Error, Result};
+use crate::{Config, Error, Result, WarcSource};
pub struct Indexer {
warc_paths: Vec<String>,
@@ -26,16 +26,27 @@ impl Indexer {
}
pub async fn run(self) -> Result<()> {
+ let download_config = self
+ .config
+ .warc_source
+ .expect("Indexing needs a warc source");
for warc_s3_path in self.warc_paths {
println!("{}", warc_s3_path);
- let raw_object = Indexer::download_from_s3(
- warc_s3_path,
- self.config.s3.name.clone(),
- self.config.s3.endpoint.clone(),
- self.config.s3.bucket.clone(),
- )
- .await?;
+ let raw_object = match download_config {
+ WarcSource::S3(config) => {
+ Indexer::download_from_s3(
+ warc_s3_path,
+ config.name.clone(),
+ config.endpoint.clone(),
+ config.bucket.clone(),
+ )
+ .await?
+ }
+ WarcSource::HTTP(config) => {
+ Indexer::download_from_http(warc_s3_path, config.base_url.clone()).await?
+ }
+ };
println!("Downloaded {} bytes", raw_object.len());
let warc = WarcFile::new(&raw_object[..]);
@@ -51,6 +62,19 @@ impl Indexer {
Ok(())
}
+ async fn download_from_http(warc_path: String, base_url: String) -> Result<Vec<u8>> {
+ let mut url = base_url;
+ if !url.ends_with('/') {
+ url += "/";
+ }
+ url += &warc_path;
+
+ let client = reqwest::Client::new();
+ let res = client.get(url).send().await?;
+
+ Ok(Vec::from(&res.bytes().await?[..]))
+ }
+
async fn download_from_s3(
key: String,
region_name: String,
diff --git a/src/lib.rs b/src/lib.rs
index 9a938697..16b7cfbc 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -13,10 +13,17 @@ pub use indexer::Indexer;
#[derive(Debug, Deserialize)]
pub struct Config {
pub mode: Mode,
- s3: S3Config,
+ warc_source: Option<WarcSource>,
warc_paths_file: String,
}
+#[derive(Debug, Deserialize)]
+#[serde(tag = "type", content = "args")]
+pub enum WarcSource {
+ S3(S3Config),
+ HTTP(HttpConfig),
+}
+
#[derive(Debug, Deserialize)]
pub enum Mode {
Indexer,
@@ -29,11 +36,19 @@ pub struct S3Config {
bucket: String,
}
+#[derive(Debug, Deserialize)]
+pub struct HttpConfig {
+ base_url: String,
+}
+
#[derive(Error, Debug)]
pub enum Error {
#[error("Failed to download object from S3")]
S3DownloadError,
+ #[error("Failed to download object from HTTP")]
+ HTTPDownloadERror(#[from] reqwest::Error),
+
#[error("Failed to get the object from S3")]
GetObjectError(#[from] rusoto_core::RusotoError<rusoto_s3::GetObjectError>),