web search done right
Retry if warc download fails
Mikkel Denker9fa16c6d9f16
diff --git a/Cargo.toml b/Cargo.tomlindex 603166e0..934e588f 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -33,7 +33,6 @@ sled = { version = "0.34.7", features = ["compression"] }tantivy = "0.18.0"thiserror = "1.0.31"tl = "0.7.5"-tokio-retry = "0.3.0"toml = "0.5.9"tracing = "0.1.34"tracing-subscriber = "0.3.11"diff --git a/src/exponential_backoff.rs b/src/exponential_backoff.rsnew file mode 100644index 00000000..ff98216c--- /dev/null+++ b/src/exponential_backoff.rs@@ -0,0 +1,94 @@+/*+ * This file is copied from [tokio_retry](https://github.com/srijs/rust-tokio-retry/blob/master/src/strategy/exponential_backoff.rs)+ * as it seems silly to pull-in a new dependency for a single file.+ * */+use std::iter::Iterator;+use std::time::Duration;+use std::u64::MAX as U64_MAX;++/// A retry strategy driven by exponential back-off.+///+/// The power corresponds to the number of past attempts.+#[derive(Debug, Clone)]+pub struct ExponentialBackoff {+ current: u64,+ base: u64,+ factor: u64,+ max_delay: Option<Duration>,+}++impl ExponentialBackoff {+ /// Constructs a new exponential back-off strategy,+ /// given a base duration in milliseconds.+ ///+ /// The resulting duration is calculated by taking the base to the `n`-th power,+ /// where `n` denotes the number of past attempts.+ pub fn from_millis(base: u64) -> ExponentialBackoff {+ ExponentialBackoff {+ current: base,+ base,+ factor: 1u64,+ max_delay: None,+ }+ }+}++impl Iterator for ExponentialBackoff {+ type Item = Duration;++ fn next(&mut self) -> Option<Duration> {+ // set delay duration by applying factor+ let duration = if let Some(duration) = self.current.checked_mul(self.factor) {+ Duration::from_millis(duration)+ } else {+ Duration::from_millis(U64_MAX)+ };++ // check if we reached max delay+ if let Some(ref max_delay) = self.max_delay {+ if duration > *max_delay {+ return Some(*max_delay);+ }+ }++ if let Some(next) = self.current.checked_mul(self.base) {+ self.current = next;+ } else {+ self.current = U64_MAX;+ }++ Some(duration)+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn returns_some_exponential_base_10() {+ let mut s = ExponentialBackoff::from_millis(10);++ assert_eq!(s.next(), Some(Duration::from_millis(10)));+ assert_eq!(s.next(), Some(Duration::from_millis(100)));+ assert_eq!(s.next(), Some(Duration::from_millis(1000)));+ }++ #[test]+ fn returns_some_exponential_base_2() {+ let mut s = ExponentialBackoff::from_millis(2);++ assert_eq!(s.next(), Some(Duration::from_millis(2)));+ assert_eq!(s.next(), Some(Duration::from_millis(4)));+ assert_eq!(s.next(), Some(Duration::from_millis(8)));+ }++ #[test]+ fn saturates_at_maximum_value() {+ let mut s = ExponentialBackoff::from_millis(U64_MAX - 1);++ assert_eq!(s.next(), Some(Duration::from_millis(U64_MAX - 1)));+ assert_eq!(s.next(), Some(Duration::from_millis(U64_MAX)));+ assert_eq!(s.next(), Some(Duration::from_millis(U64_MAX)));+ }+}diff --git a/src/lib.rs b/src/lib.rsindex 1c4c8131..e31d8ecc 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -26,6 +26,7 @@ pub mod index;pub mod mapreduce;mod directory;+mod exponential_backoff;mod query;pub mod ranking;mod schema;@@ -132,7 +133,7 @@ pub enum Error {S3DownloadError,#[error("Failed to download object from HTTP")]- HTTPDownloadERror(#[from] reqwest::Error),+ HTTPDownloadError(#[from] reqwest::Error),#[error("Failed to get the object from S3")]GetObjectError(#[from] rusoto_core::RusotoError<rusoto_s3::GetObjectError>),@@ -163,6 +164,9 @@ pub enum Error {#[error("Error executing distributed jobs")]MapReduce(#[from] mapreduce::Error),++ #[error("Failed to download warc files after all retries")]+ DownloadFailed,}pub(crate) type Result<T> = std::result::Result<T, Error>;diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rsindex 0cc2c587..888ed7a0 100644--- a/src/mapreduce/manager.rs+++ b/src/mapreduce/manager.rs@@ -1,5 +1,6 @@use super::{Error, Result};use super::{Map, Reduce};+use crate::exponential_backoff::ExponentialBackoff;use crate::mapreduce::Task;use indicatif::{ParallelProgressIterator, ProgressBar, ProgressStyle};use rayon::iter::ParallelBridge;@@ -11,7 +12,6 @@ use std::ops::Deref;use std::sync::atomic::{AtomicU32, Ordering};use std::sync::{Arc, Mutex};use std::time::Duration;-use tokio_retry::strategy::ExponentialBackoff;use tracing::{debug, warn};#[derive(Debug)]diff --git a/src/warc.rs b/src/warc.rsindex 74c7013d..118850db 100644--- a/src/warc.rs+++ b/src/warc.rs@@ -1,3 +1,4 @@+use crate::exponential_backoff::ExponentialBackoff;// Cuely is an open source web search engine.// Copyright (C) 2022 Cuely ApS//@@ -18,9 +19,11 @@ use std::collections::BTreeMap;use std::fs::File;use std::io::{BufRead, BufReader, Read};use std::path::Path;+use std::thread::sleep;use std::time::Duration;use flate2::read::MultiGzDecoder;+use tracing::debug;pub(crate) struct WarcFile {bytes: Vec<u8>,@@ -67,16 +70,28 @@ impl WarcFile {}pub(crate) fn download(source: WarcSource, path: &str) -> Result<Self> {- match source {- WarcSource::S3(config) => WarcFile::download_from_s3(- path,- config.name.clone(),- config.endpoint.clone(),- config.bucket,- ),- WarcSource::HTTP(config) => WarcFile::download_from_http(path, config.base_url),- WarcSource::Local(config) => WarcFile::load_from_folder(path, &config.folder),+ for dur in ExponentialBackoff::from_millis(10).take(5) {+ let res = match source.clone() {+ WarcSource::S3(config) => WarcFile::download_from_s3(+ path,+ config.name.clone(),+ config.endpoint.clone(),+ config.bucket,+ ),+ WarcSource::HTTP(config) => WarcFile::download_from_http(path, config.base_url),+ WarcSource::Local(config) => WarcFile::load_from_folder(path, &config.folder),+ };++ if res.is_ok() {+ return res;+ }++ debug!("warc download failed, retrying in {} ms", dur.as_millis());++ sleep(dur);}++ Err(Error::DownloadFailed)}fn load_from_folder(name: &str, folder: &str) -> Result<Self> {