web search done right
fix s3 warc download
Mikkel Denkerfb3700dfafe1
diff --git a/core/src/lib.rs b/core/src/lib.rsindex e49f0dfb..c953d146 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -165,11 +165,22 @@ impl WarcSource {)?.with_path_style();- let objects = bucket.list_blocking(config.folder.clone(), None)?;+ let mut folder = config.folder.clone();++ if !folder.ends_with('/') {+ folder.push('/');+ }++ let pages = bucket.list_blocking(folder, Some("/".to_string()))?;++ let objects = pages+ .into_iter()+ .flat_map(|p| p.contents.into_iter())+ .collect::<Vec<_>>();for p in objects.into_iter().filter_map(|o| {- if o.name.ends_with("warc.gz") {- Some(o.name)+ if o.key.ends_with("warc.gz") {+ Some(o.key)} else {None}diff --git a/core/src/warc.rs b/core/src/warc.rsindex f3714133..e12818d8 100644--- a/core/src/warc.rs+++ b/core/src/warc.rs@@ -188,8 +188,7 @@ impl WarcFile {)?.with_path_style();- let path = Path::new(&config.folder).join(warc_path);- let res = bucket.get_object_blocking(path.as_os_str().to_str().unwrap())?;+ let res = bucket.get_object_blocking(warc_path)?;buf.write_all(res.bytes())?;