Skip to content

web search done right

Retry if warc download fails

Mikkel Denker9fa16c6d9f16
diff --git a/Cargo.toml b/Cargo.toml
index 603166e0..934e588f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -33,7 +33,6 @@ sled = { version = "0.34.7", features = ["compression"] }
tantivy = "0.18.0"
thiserror = "1.0.31"
tl = "0.7.5"
-tokio-retry = "0.3.0"
toml = "0.5.9"
tracing = "0.1.34"
tracing-subscriber = "0.3.11"
diff --git a/src/exponential_backoff.rs b/src/exponential_backoff.rs
new file mode 100644
index 00000000..ff98216c
--- /dev/null
+++ b/src/exponential_backoff.rs
@@ -0,0 +1,94 @@
+/*
+ * This file is copied from [tokio_retry](https://github.com/srijs/rust-tokio-retry/blob/master/src/strategy/exponential_backoff.rs)
+ * as it seems silly to pull-in a new dependency for a single file.
+ * */
+use std::iter::Iterator;
+use std::time::Duration;
+use std::u64::MAX as U64_MAX;
+
+/// A retry strategy driven by exponential back-off.
+///
+/// The power corresponds to the number of past attempts.
+#[derive(Debug, Clone)]
+pub struct ExponentialBackoff {
+ current: u64,
+ base: u64,
+ factor: u64,
+ max_delay: Option<Duration>,
+}
+
+impl ExponentialBackoff {
+ /// Constructs a new exponential back-off strategy,
+ /// given a base duration in milliseconds.
+ ///
+ /// The resulting duration is calculated by taking the base to the `n`-th power,
+ /// where `n` denotes the number of past attempts.
+ pub fn from_millis(base: u64) -> ExponentialBackoff {
+ ExponentialBackoff {
+ current: base,
+ base,
+ factor: 1u64,
+ max_delay: None,
+ }
+ }
+}
+
+impl Iterator for ExponentialBackoff {
+ type Item = Duration;
+
+ fn next(&mut self) -> Option<Duration> {
+ // set delay duration by applying factor
+ let duration = if let Some(duration) = self.current.checked_mul(self.factor) {
+ Duration::from_millis(duration)
+ } else {
+ Duration::from_millis(U64_MAX)
+ };
+
+ // check if we reached max delay
+ if let Some(ref max_delay) = self.max_delay {
+ if duration > *max_delay {
+ return Some(*max_delay);
+ }
+ }
+
+ if let Some(next) = self.current.checked_mul(self.base) {
+ self.current = next;
+ } else {
+ self.current = U64_MAX;
+ }
+
+ Some(duration)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn returns_some_exponential_base_10() {
+ let mut s = ExponentialBackoff::from_millis(10);
+
+ assert_eq!(s.next(), Some(Duration::from_millis(10)));
+ assert_eq!(s.next(), Some(Duration::from_millis(100)));
+ assert_eq!(s.next(), Some(Duration::from_millis(1000)));
+ }
+
+ #[test]
+ fn returns_some_exponential_base_2() {
+ let mut s = ExponentialBackoff::from_millis(2);
+
+ assert_eq!(s.next(), Some(Duration::from_millis(2)));
+ assert_eq!(s.next(), Some(Duration::from_millis(4)));
+ assert_eq!(s.next(), Some(Duration::from_millis(8)));
+ }
+
+ #[test]
+ fn saturates_at_maximum_value() {
+ let mut s = ExponentialBackoff::from_millis(U64_MAX - 1);
+
+ assert_eq!(s.next(), Some(Duration::from_millis(U64_MAX - 1)));
+ assert_eq!(s.next(), Some(Duration::from_millis(U64_MAX)));
+ assert_eq!(s.next(), Some(Duration::from_millis(U64_MAX)));
+ }
+}
diff --git a/src/lib.rs b/src/lib.rs
index 1c4c8131..e31d8ecc 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -26,6 +26,7 @@ pub mod index;
pub mod mapreduce;
mod directory;
+mod exponential_backoff;
mod query;
pub mod ranking;
mod schema;
@@ -132,7 +133,7 @@ pub enum Error {
S3DownloadError,
#[error("Failed to download object from HTTP")]
- HTTPDownloadERror(#[from] reqwest::Error),
+ HTTPDownloadError(#[from] reqwest::Error),
#[error("Failed to get the object from S3")]
GetObjectError(#[from] rusoto_core::RusotoError<rusoto_s3::GetObjectError>),
@@ -163,6 +164,9 @@ pub enum Error {
#[error("Error executing distributed jobs")]
MapReduce(#[from] mapreduce::Error),
+
+ #[error("Failed to download warc files after all retries")]
+ DownloadFailed,
}
pub(crate) type Result<T> = std::result::Result<T, Error>;
diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rs
index 0cc2c587..888ed7a0 100644
--- a/src/mapreduce/manager.rs
+++ b/src/mapreduce/manager.rs
@@ -1,5 +1,6 @@
use super::{Error, Result};
use super::{Map, Reduce};
+use crate::exponential_backoff::ExponentialBackoff;
use crate::mapreduce::Task;
use indicatif::{ParallelProgressIterator, ProgressBar, ProgressStyle};
use rayon::iter::ParallelBridge;
@@ -11,7 +12,6 @@ use std::ops::Deref;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
-use tokio_retry::strategy::ExponentialBackoff;
use tracing::{debug, warn};
#[derive(Debug)]
diff --git a/src/warc.rs b/src/warc.rs
index 74c7013d..118850db 100644
--- a/src/warc.rs
+++ b/src/warc.rs
@@ -1,3 +1,4 @@
+use crate::exponential_backoff::ExponentialBackoff;
// Cuely is an open source web search engine.
// Copyright (C) 2022 Cuely ApS
//
@@ -18,9 +19,11 @@ use std::collections::BTreeMap;
use std::fs::File;
use std::io::{BufRead, BufReader, Read};
use std::path::Path;
+use std::thread::sleep;
use std::time::Duration;
use flate2::read::MultiGzDecoder;
+use tracing::debug;
pub(crate) struct WarcFile {
bytes: Vec<u8>,
@@ -67,16 +70,28 @@ impl WarcFile {
}
pub(crate) fn download(source: WarcSource, path: &str) -> Result<Self> {
- match source {
- WarcSource::S3(config) => WarcFile::download_from_s3(
- path,
- config.name.clone(),
- config.endpoint.clone(),
- config.bucket,
- ),
- WarcSource::HTTP(config) => WarcFile::download_from_http(path, config.base_url),
- WarcSource::Local(config) => WarcFile::load_from_folder(path, &config.folder),
+ for dur in ExponentialBackoff::from_millis(10).take(5) {
+ let res = match source.clone() {
+ WarcSource::S3(config) => WarcFile::download_from_s3(
+ path,
+ config.name.clone(),
+ config.endpoint.clone(),
+ config.bucket,
+ ),
+ WarcSource::HTTP(config) => WarcFile::download_from_http(path, config.base_url),
+ WarcSource::Local(config) => WarcFile::load_from_folder(path, &config.folder),
+ };
+
+ if res.is_ok() {
+ return res;
+ }
+
+ debug!("warc download failed, retrying in {} ms", dur.as_millis());
+
+ sleep(dur);
}
+
+ Err(Error::DownloadFailed)
}
fn load_from_folder(name: &str, folder: &str) -> Result<Self> {