Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Centralize asset download retry policy

Manav Rathi4afff60c785c
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
index fc9da118fb..95f7c74d9b 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
@@ -356,7 +356,6 @@ internal class ModelSettingsActions(
companion object {
private const val MAX_DOWNLOAD_RETRIES = 5
- private val NON_RETRYABLE_HTTP = setOf(401, 403, 404)
private const val RETRY_DELAY_BASE_MS = 1500L
private const val RETRY_DELAY_MAX_MS = 12000L
private const val DEFAULT_TEMPERATURE = 0.5f
@@ -371,8 +370,7 @@ internal class ModelSettingsActions(
if (err is RequiredModelValidationError) return false
when (err) {
is AssetDownloadException.Validation -> return false
- is AssetDownloadException.Http ->
- if (err.status.toInt() in NON_RETRYABLE_HTTP) return false
+ is AssetDownloadException.Http -> return err.retryable
else -> {}
}
return true
diff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
index 78fb826bc0..397d0ea388 100644
--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
@@ -1902,8 +1902,8 @@ final class ChatViewModel: ObservableObject {
switch error {
case AssetDownloadError.Validation:
return false
- case let AssetDownloadError.Http(status):
- if status == 401 || status == 403 || status == 404 { return false }
+ case let AssetDownloadError.Http(_, retryable):
+ return retryable
default:
break
}
diff --git a/rust/bindings/uniffi/ensu/src/assets.rs b/rust/bindings/uniffi/ensu/src/assets.rs
index a0998de1f3..e9908946a6 100644
--- a/rust/bindings/uniffi/ensu/src/assets.rs
+++ b/rust/bindings/uniffi/ensu/src/assets.rs
@@ -22,7 +22,7 @@ pub enum AssetDownloadError {
#[error("{detail}")]
Validation { detail: String },
#[error("HTTP {status}")]
- Http { status: u16 },
+ Http { status: u16, retryable: bool },
#[error("network: {detail}")]
Network { detail: String },
#[error("{detail}")]
@@ -35,12 +35,13 @@ pub enum AssetDownloadError {
impl From<download::Error> for AssetDownloadError {
fn from(value: download::Error) -> Self {
+ let retryable = value.is_retryable();
match value {
download::Error::Cancelled => Self::Cancelled,
download::Error::Target { source, .. } => Self::from(*source),
download::Error::Fallback { single, .. } => Self::from(*single),
download::Error::Validation(detail) => Self::Validation { detail },
- download::Error::Http(status) => Self::Http { status },
+ download::Error::Http(status) => Self::Http { status, retryable },
download::Error::Network(detail) => Self::Network { detail },
download::Error::SizeMismatch { expected, actual } => Self::InvalidDownload {
detail: format!("size mismatch: expected {expected} bytes, got {actual}"),
diff --git a/rust/crates/assets/src/download.rs b/rust/crates/assets/src/download.rs
index d50ce1d76c..301ee2f3a5 100644
--- a/rust/crates/assets/src/download.rs
+++ b/rust/crates/assets/src/download.rs
@@ -54,6 +54,19 @@ pub enum Error {
Json(#[from] serde_json::Error),
}
+impl Error {
+ pub fn is_retryable(&self) -> bool {
+ match self {
+ Self::Target { source, .. } | Self::Fallback { single: source, .. } => {
+ source.is_retryable()
+ }
+ Self::Cancelled | Self::Validation(_) | Self::StorageFull => false,
+ Self::Http(401 | 403 | 404) => false,
+ _ => true,
+ }
+ }
+}
+
impl From<std::io::Error> for Error {
fn from(err: std::io::Error) -> Self {
if err.kind() == std::io::ErrorKind::StorageFull {
@@ -523,8 +536,9 @@ async fn download_file_single(
}
if !response.status().is_success() {
- if attempt == MAX_ATTEMPTS {
- return Err(Error::Http(response.status().as_u16()));
+ let error = Error::Http(response.status().as_u16());
+ if attempt == MAX_ATTEMPTS || !error.is_retryable() {
+ return Err(error);
}
retry_count = retry_count.saturating_add(1);
continue;
@@ -837,13 +851,13 @@ impl RangeDownloadContext<'_, '_> {
};
if response.status() != StatusCode::PARTIAL_CONTENT {
+ let error = Error::Http(response.status().as_u16());
if response.status() == StatusCode::OK
|| response.status() == StatusCode::RANGE_NOT_SATISFIABLE
+ || attempt == MAX_ATTEMPTS
+ || !error.is_retryable()
{
- return Err(Error::Http(response.status().as_u16()));
- }
- if attempt == MAX_ATTEMPTS {
- return Err(Error::Http(response.status().as_u16()));
+ return Err(error);
}
self.increment_retry(part_index);
self.emit_progress();
@@ -2588,6 +2602,214 @@ mod tests {
let _ = fs::remove_dir_all(test_dir);
}
+ #[tokio::test]
+ async fn permanent_http_errors_stop_single_stream_retries() {
+ for status in [401, 403, 404] {
+ let requests = http_failure_requests(status, false).await;
+ assert_eq!(requests.get(&None), Some(&1), "HTTP {status}");
+ }
+ }
+
+ #[tokio::test]
+ async fn permanent_http_errors_allow_one_full_stream_fallback() {
+ for status in [401, 403, 404] {
+ let requests = http_failure_requests(status, true).await;
+ assert!(requests.keys().any(Option::is_some));
+ assert_eq!(requests.get(&None), Some(&1), "HTTP {status}");
+ assert!(
+ requests.values().all(|count| *count == 1),
+ "HTTP {status}: {requests:?}"
+ );
+ }
+ }
+
+ #[tokio::test]
+ async fn other_http_errors_keep_single_stream_retries() {
+ for status in [400, 416, 429, 500] {
+ let requests = http_failure_requests(status, false).await;
+ assert_eq!(requests.get(&None), Some(&MAX_ATTEMPTS), "HTTP {status}");
+ }
+ }
+
+ #[tokio::test]
+ async fn recoverable_http_failures_still_download() {
+ for (ranged, status) in [
+ (false, 429),
+ (false, 500),
+ (true, 200),
+ (true, 403),
+ (true, 416),
+ ] {
+ let bytes = Arc::new(sample_bytes(MIN_RANGE_DOWNLOAD_BYTES as usize));
+ let full_gets = Arc::new(AtomicUsize::new(0));
+ let server = {
+ let bytes = Arc::clone(&bytes);
+ let full_gets = Arc::clone(&full_gets);
+ TestServer::spawn(move |mut stream| {
+ let Some(request) = read_test_request(&stream) else {
+ return;
+ };
+ if request.line.starts_with("HEAD ") {
+ let ranges = if ranged {
+ "Accept-Ranges: bytes\r\n"
+ } else {
+ ""
+ };
+ let response = format!(
+ "HTTP/1.1 200 OK\r\nContent-Length: {}\r\n{ranges}Connection: close\r\n\r\n",
+ bytes.len()
+ );
+ let _ = stream.write_all(response.as_bytes());
+ return;
+ }
+ if request.range.is_some()
+ || (!ranged && full_gets.fetch_add(1, Ordering::SeqCst) == 0)
+ {
+ let response = format!(
+ "HTTP/1.1 {status} Test\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
+ );
+ let _ = stream.write_all(response.as_bytes());
+ return;
+ }
+ if ranged {
+ full_gets.fetch_add(1, Ordering::SeqCst);
+ }
+ let response = format!(
+ "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
+ bytes.len()
+ );
+ let _ = stream.write_all(response.as_bytes());
+ let _ = stream.write_all(bytes.as_slice());
+ })
+ };
+ let test_dir = scratch_dir(&format!("recoverable-{status}-{ranged}"));
+ let destination = test_dir.join("model.bin");
+ download(
+ vec![DownloadTarget {
+ label: "Model".to_string(),
+ url: server.url("/model.bin"),
+ expected_size: bytes.len() as u64,
+ sha256: sha_hex(&bytes),
+ destination: destination.clone(),
+ }],
+ |_| {},
+ false,
+ )
+ .await
+ .expect("recoverable download");
+ assert_eq!(fs::read(&destination).unwrap(), *bytes);
+ assert_eq!(full_gets.load(Ordering::SeqCst), if ranged { 1 } else { 2 });
+ let _ = fs::remove_dir_all(test_dir);
+ }
+ }
+
+ #[tokio::test]
+ async fn batch_http_errors_prefer_non_retryable_failure() {
+ use crate::{Asset, AssetFile, AssetStore};
+
+ let server = TestServer::spawn(|mut stream| {
+ let Some(request) = read_test_request(&stream) else {
+ return;
+ };
+ let (status, size) = if request.line.starts_with("HEAD ") {
+ (200, 1)
+ } else {
+ let path = request.line.split_whitespace().nth(1).unwrap();
+ (path[1..].parse::<u16>().unwrap(), 0)
+ };
+ let response = format!(
+ "HTTP/1.1 {status} Test\r\nContent-Length: {size}\r\nConnection: close\r\n\r\n"
+ );
+ let _ = stream.write_all(response.as_bytes());
+ });
+
+ for (statuses, expected) in [([500, 403], 403), ([403, 500], 403), ([500, 502], 500)] {
+ let assets = statuses.map(|status| {
+ Asset::file(
+ vec![status.to_string()],
+ AssetFile {
+ name: "model".to_string(),
+ url: server.url(&format!("/{status}")),
+ size: 1,
+ sha256: "0".repeat(64),
+ },
+ )
+ .unwrap()
+ });
+ let root = scratch_dir("batch-retryability");
+ let error = AssetStore::new(&root)
+ .download(&assets, |_| {}, CancellationToken::new())
+ .await
+ .unwrap_err();
+ assert!(
+ matches!(
+ &error,
+ Error::Target { source, .. }
+ if matches!(**source, Error::Http(status) if status == expected)
+ ),
+ "{statuses:?}: {error}"
+ );
+ let _ = fs::remove_dir_all(root);
+ }
+ }
+
+ async fn http_failure_requests(
+ status: u16,
+ ranged: bool,
+ ) -> std::collections::HashMap<Option<String>, usize> {
+ let requests = Arc::new(Mutex::new(std::collections::HashMap::new()));
+ let server = {
+ let requests = Arc::clone(&requests);
+ TestServer::spawn(move |mut stream| {
+ let Some(request) = read_test_request(&stream) else {
+ return;
+ };
+ let response = if request.line.starts_with("HEAD ") {
+ let ranges = if ranged {
+ "Accept-Ranges: bytes\r\n"
+ } else {
+ ""
+ };
+ format!(
+ "HTTP/1.1 200 OK\r\nContent-Length: {MIN_RANGE_DOWNLOAD_BYTES}\r\n{ranges}Connection: close\r\n\r\n"
+ )
+ } else {
+ *requests.lock().unwrap().entry(request.range).or_insert(0) += 1;
+ format!(
+ "HTTP/1.1 {status} Failure\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
+ )
+ };
+ let _ = stream.write_all(response.as_bytes());
+ })
+ };
+ let test_dir = scratch_dir(&format!("http-{status}-{ranged}"));
+ let destination = test_dir.join("model.bin");
+ let error = download(
+ vec![DownloadTarget {
+ label: "Model".to_string(),
+ url: server.url("/model.bin"),
+ expected_size: MIN_RANGE_DOWNLOAD_BYTES,
+ destination: destination.clone(),
+ sha256: "0".repeat(64),
+ }],
+ |_| {},
+ false,
+ )
+ .await
+ .expect_err("HTTP failure");
+ let Error::Target { source, .. } = error else {
+ panic!("missing download context")
+ };
+ let error = match *source {
+ Error::Fallback { single, .. } => *single,
+ error => error,
+ };
+ assert!(matches!(error, Error::Http(actual) if actual == status));
+ assert!(!destination.exists());
+ let _ = fs::remove_dir_all(test_dir);
+ requests.lock().unwrap().clone()
+ }
+
#[tokio::test]
async fn sha_pinned_target_adopts_cached_file_regardless_of_url() {
let test_dir = scratch_dir("sha-cached");
diff --git a/rust/crates/assets/src/store.rs b/rust/crates/assets/src/store.rs
index 18e5878e9a..0842732ce5 100644
--- a/rust/crates/assets/src/store.rs
+++ b/rust/crates/assets/src/store.rs
@@ -189,7 +189,11 @@ impl AssetStore {
if cancellation.is_cancelled() {
return Err(Error::Cancelled);
}
- if let Some(error) = results.into_iter().find_map(Result::err) {
+ if let Some(error) = results
+ .into_iter()
+ .filter_map(Result::err)
+ .min_by_key(Error::is_retryable)
+ {
return Err(error);
}
Ok(())