Skip to content

web search done right

process warc files concurrently

Mikkel Denkerc55af35f011a
diff --git a/Cargo.toml b/Cargo.toml
index 6e1eaf26..cabd9585 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -12,6 +12,7 @@ name = "cuely"
bincode = "1.3.3"
encoding_rs = "0.8.31"
flate2 = "1.0.23"
+futures = "0.3.21"
itertools = "0.10.3"
reqwest = "0.11.10"
rusoto_core = "0.48.0"
diff --git a/src/indexer.rs b/src/indexer.rs
index 108f21d1..e7f458e3 100644
--- a/src/indexer.rs
+++ b/src/indexer.rs
@@ -1,3 +1,4 @@
+use futures::prelude::*;
use rusoto_core::Region;
use rusoto_s3::{GetObjectRequest, S3Client, S3};
use std::fs::File;
@@ -26,38 +27,46 @@ impl Indexer {
}
pub async fn run(self) -> Result<()> {
- let download_config = self
- .config
- .warc_source
- .expect("Indexing needs a warc source");
- for warc_s3_path in self.warc_paths {
- println!("{}", warc_s3_path);
-
- let raw_object = match download_config {
- WarcSource::S3(config) => {
- Indexer::download_from_s3(
- warc_s3_path,
- config.name.clone(),
- config.endpoint.clone(),
- config.bucket.clone(),
- )
- .await?
+ let warc_source = &self.config.warc_source;
+
+ stream::iter(self.warc_paths)
+ .map(|warc_s3_path| async move {
+ println!("{}", warc_s3_path);
+ let download_config = warc_source.as_ref().expect("Indexing needs a warc source");
+
+ let raw_object = match download_config {
+ WarcSource::S3(config) => {
+ Indexer::download_from_s3(
+ warc_s3_path,
+ config.name.clone(),
+ config.endpoint.clone(),
+ config.bucket.clone(),
+ )
+ .await
+ }
+ WarcSource::HTTP(config) => {
+ Indexer::download_from_http(warc_s3_path, config.base_url.clone()).await
+ }
+ };
+
+ if raw_object.is_err() {
+ return;
}
- WarcSource::HTTP(config) => {
- Indexer::download_from_http(warc_s3_path, config.base_url.clone()).await?
+ let raw_object = raw_object.unwrap();
+
+ println!("Downloaded {} bytes", raw_object.len());
+ let warc = WarcFile::new(&raw_object[..]);
+ for record in warc.flatten() {
+ let webpage = Webpage::parse(&record.response.body);
+ println!("TEST: {:?}", webpage.text());
+ println!();
}
- };
- println!("Downloaded {} bytes", raw_object.len());
- let warc = WarcFile::new(&raw_object[..]);
- for record in warc.flatten() {
- let webpage = Webpage::parse(&record.response.body);
- println!("TEST: {:?}", webpage.text());
- println!();
- }
-
- panic!();
- }
+ panic!();
+ })
+ .buffer_unordered(1)
+ .collect::<Vec<_>>()
+ .await;
Ok(())
}
diff --git a/src/lib.rs b/src/lib.rs
index 16b7cfbc..b4f23127 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -17,7 +17,7 @@ pub struct Config {
warc_paths_file: String,
}
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Clone)]
#[serde(tag = "type", content = "args")]
pub enum WarcSource {
S3(S3Config),
@@ -29,14 +29,14 @@ pub enum Mode {
Indexer,
}
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Clone)]
pub struct S3Config {
name: String,
endpoint: String,
bucket: String,
}
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Clone)]
pub struct HttpConfig {
base_url: String,
}