web search done right
process warc files concurrently
Mikkel Denkerc55af35f011a
diff --git a/Cargo.toml b/Cargo.tomlindex 6e1eaf26..cabd9585 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -12,6 +12,7 @@ name = "cuely"bincode = "1.3.3"encoding_rs = "0.8.31"flate2 = "1.0.23"+futures = "0.3.21"itertools = "0.10.3"reqwest = "0.11.10"rusoto_core = "0.48.0"diff --git a/src/indexer.rs b/src/indexer.rsindex 108f21d1..e7f458e3 100644--- a/src/indexer.rs+++ b/src/indexer.rs@@ -1,3 +1,4 @@+use futures::prelude::*;use rusoto_core::Region;use rusoto_s3::{GetObjectRequest, S3Client, S3};use std::fs::File;@@ -26,38 +27,46 @@ impl Indexer {}pub async fn run(self) -> Result<()> {- let download_config = self- .config- .warc_source- .expect("Indexing needs a warc source");- for warc_s3_path in self.warc_paths {- println!("{}", warc_s3_path);-- let raw_object = match download_config {- WarcSource::S3(config) => {- Indexer::download_from_s3(- warc_s3_path,- config.name.clone(),- config.endpoint.clone(),- config.bucket.clone(),- )- .await?+ let warc_source = &self.config.warc_source;++ stream::iter(self.warc_paths)+ .map(|warc_s3_path| async move {+ println!("{}", warc_s3_path);+ let download_config = warc_source.as_ref().expect("Indexing needs a warc source");++ let raw_object = match download_config {+ WarcSource::S3(config) => {+ Indexer::download_from_s3(+ warc_s3_path,+ config.name.clone(),+ config.endpoint.clone(),+ config.bucket.clone(),+ )+ .await+ }+ WarcSource::HTTP(config) => {+ Indexer::download_from_http(warc_s3_path, config.base_url.clone()).await+ }+ };++ if raw_object.is_err() {+ return;}- WarcSource::HTTP(config) => {- Indexer::download_from_http(warc_s3_path, config.base_url.clone()).await?+ let raw_object = raw_object.unwrap();++ println!("Downloaded {} bytes", raw_object.len());+ let warc = WarcFile::new(&raw_object[..]);+ for record in warc.flatten() {+ let webpage = Webpage::parse(&record.response.body);+ println!("TEST: {:?}", webpage.text());+ println!();}- };- println!("Downloaded {} bytes", raw_object.len());- let warc = WarcFile::new(&raw_object[..]);- for record in warc.flatten() {- let webpage = Webpage::parse(&record.response.body);- println!("TEST: {:?}", webpage.text());- println!();- }-- panic!();- }+ panic!();+ })+ .buffer_unordered(1)+ .collect::<Vec<_>>()+ .await;Ok(())}diff --git a/src/lib.rs b/src/lib.rsindex 16b7cfbc..b4f23127 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -17,7 +17,7 @@ pub struct Config {warc_paths_file: String,}-#[derive(Debug, Deserialize)]+#[derive(Debug, Deserialize, Clone)]#[serde(tag = "type", content = "args")]pub enum WarcSource {S3(S3Config),@@ -29,14 +29,14 @@ pub enum Mode {Indexer,}-#[derive(Debug, Deserialize)]+#[derive(Debug, Deserialize, Clone)]pub struct S3Config {name: String,endpoint: String,bucket: String,}-#[derive(Debug, Deserialize)]+#[derive(Debug, Deserialize, Clone)]pub struct HttpConfig {base_url: String,}