web search done right
Bug/retry image download (#48)
Mikkel Denker7bf0dc844f73
diff --git a/src/image_downloader.rs b/src/image_downloader.rsindex 07135b5a..344ccfa0 100644--- a/src/image_downloader.rs+++ b/src/image_downloader.rs@@ -19,6 +19,7 @@ use std::{collections::HashSet, hash::Hash, time::Duration};use serde::Serialize;use crate::{+ exponential_backoff::ExponentialBackoff,image_store::{Image, ImageStore},webpage::Url,};@@ -43,16 +44,20 @@ wherecontinue;}- if let Some(image) = url- .download_bytes(self.timeout.unwrap_or_else(|| Duration::from_secs(20)))- .await- .and_then(|bytes| Image::from_bytes(bytes).ok())- .map(|image| DownloadedImage {- image,- key: self.key.clone(),- })- {- return Some(image);+ for duration in ExponentialBackoff::from_millis(10).take(5) {+ if let Some(image) = url+ .download_bytes(self.timeout.unwrap_or_else(|| Duration::from_secs(20)))+ .await+ .and_then(|bytes| Image::from_bytes(bytes).ok())+ .map(|image| DownloadedImage {+ image,+ key: self.key.clone(),+ })+ {+ return Some(image);+ }++ tokio::time::sleep(duration).await;}}diff --git a/src/warc.rs b/src/warc.rsindex da742d5d..c8632d39 100644--- a/src/warc.rs+++ b/src/warc.rs@@ -1,4 +1,3 @@-use crate::exponential_backoff::ExponentialBackoff;// Cuely is an open source web search engine.// Copyright (C) 2022 Cuely ApS//@@ -14,6 +13,8 @@ use crate::exponential_backoff::ExponentialBackoff;//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::exponential_backoff::ExponentialBackoff;use crate::{Error, Result, WarcSource};use std::collections::BTreeMap;use std::fs::File;