Skip to content

web search done right

Bug/retry image download (#48)

Mikkel Denker7bf0dc844f73
diff --git a/src/image_downloader.rs b/src/image_downloader.rs
index 07135b5a..344ccfa0 100644
--- a/src/image_downloader.rs
+++ b/src/image_downloader.rs
@@ -19,6 +19,7 @@ use std::{collections::HashSet, hash::Hash, time::Duration};
use serde::Serialize;
use crate::{
+ exponential_backoff::ExponentialBackoff,
image_store::{Image, ImageStore},
webpage::Url,
};
@@ -43,16 +44,20 @@ where
continue;
}
- if let Some(image) = url
- .download_bytes(self.timeout.unwrap_or_else(|| Duration::from_secs(20)))
- .await
- .and_then(|bytes| Image::from_bytes(bytes).ok())
- .map(|image| DownloadedImage {
- image,
- key: self.key.clone(),
- })
- {
- return Some(image);
+ for duration in ExponentialBackoff::from_millis(10).take(5) {
+ if let Some(image) = url
+ .download_bytes(self.timeout.unwrap_or_else(|| Duration::from_secs(20)))
+ .await
+ .and_then(|bytes| Image::from_bytes(bytes).ok())
+ .map(|image| DownloadedImage {
+ image,
+ key: self.key.clone(),
+ })
+ {
+ return Some(image);
+ }
+
+ tokio::time::sleep(duration).await;
}
}
diff --git a/src/warc.rs b/src/warc.rs
index da742d5d..c8632d39 100644
--- a/src/warc.rs
+++ b/src/warc.rs
@@ -1,4 +1,3 @@
-use crate::exponential_backoff::ExponentialBackoff;
// Cuely is an open source web search engine.
// Copyright (C) 2022 Cuely ApS
//
@@ -14,6 +13,8 @@ use crate::exponential_backoff::ExponentialBackoff;
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::exponential_backoff::ExponentialBackoff;
use crate::{Error, Result, WarcSource};
use std::collections::BTreeMap;
use std::fs::File;