Skip to content

web search done right

treat non-200 response as a download failed when downloading warc files

Mikkel Denkerd2caac61c66d
diff --git a/core/src/warc.rs b/core/src/warc.rs
index 7bee7682..db4e4d49 100644
--- a/core/src/warc.rs
+++ b/core/src/warc.rs
@@ -149,6 +149,10 @@ impl WarcFile {
.build()?;
let res = client.get(url).send().await?;
+ if res.status().as_u16() != 200 {
+ return Err(Error::DownloadFailed);
+ }
+
let mut stream = res.bytes_stream();
buf.rewind()?;