Skip to content

web search done right

fix s3 warc download

Mikkel Denkerfb3700dfafe1
diff --git a/core/src/lib.rs b/core/src/lib.rs
index e49f0dfb..c953d146 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -165,11 +165,22 @@ impl WarcSource {
)?
.with_path_style();
- let objects = bucket.list_blocking(config.folder.clone(), None)?;
+ let mut folder = config.folder.clone();
+
+ if !folder.ends_with('/') {
+ folder.push('/');
+ }
+
+ let pages = bucket.list_blocking(folder, Some("/".to_string()))?;
+
+ let objects = pages
+ .into_iter()
+ .flat_map(|p| p.contents.into_iter())
+ .collect::<Vec<_>>();
for p in objects.into_iter().filter_map(|o| {
- if o.name.ends_with("warc.gz") {
- Some(o.name)
+ if o.key.ends_with("warc.gz") {
+ Some(o.key)
} else {
None
}
diff --git a/core/src/warc.rs b/core/src/warc.rs
index f3714133..e12818d8 100644
--- a/core/src/warc.rs
+++ b/core/src/warc.rs
@@ -188,8 +188,7 @@ impl WarcFile {
)?
.with_path_style();
- let path = Path::new(&config.folder).join(warc_path);
- let res = bucket.get_object_blocking(path.as_os_str().to_str().unwrap())?;
+ let res = bucket.get_object_blocking(warc_path)?;
buf.write_all(res.bytes())?;