π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Centralize asset download retry policy
Manav Rathi4afff60c785c
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.ktindex fc9da118fb..95f7c74d9b 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt@@ -356,7 +356,6 @@ internal class ModelSettingsActions(companion object {private const val MAX_DOWNLOAD_RETRIES = 5- private val NON_RETRYABLE_HTTP = setOf(401, 403, 404)private const val RETRY_DELAY_BASE_MS = 1500Lprivate const val RETRY_DELAY_MAX_MS = 12000Lprivate const val DEFAULT_TEMPERATURE = 0.5f@@ -371,8 +370,7 @@ internal class ModelSettingsActions(if (err is RequiredModelValidationError) return falsewhen (err) {is AssetDownloadException.Validation -> return false- is AssetDownloadException.Http ->- if (err.status.toInt() in NON_RETRYABLE_HTTP) return false+ is AssetDownloadException.Http -> return err.retryableelse -> {}}return truediff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swiftindex 78fb826bc0..397d0ea388 100644--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift@@ -1902,8 +1902,8 @@ final class ChatViewModel: ObservableObject {switch error {case AssetDownloadError.Validation:return false- case let AssetDownloadError.Http(status):- if status == 401 || status == 403 || status == 404 { return false }+ case let AssetDownloadError.Http(_, retryable):+ return retryabledefault:break}diff --git a/rust/bindings/uniffi/ensu/src/assets.rs b/rust/bindings/uniffi/ensu/src/assets.rsindex a0998de1f3..e9908946a6 100644--- a/rust/bindings/uniffi/ensu/src/assets.rs+++ b/rust/bindings/uniffi/ensu/src/assets.rs@@ -22,7 +22,7 @@ pub enum AssetDownloadError {#[error("{detail}")]Validation { detail: String },#[error("HTTP {status}")]- Http { status: u16 },+ Http { status: u16, retryable: bool },#[error("network: {detail}")]Network { detail: String },#[error("{detail}")]@@ -35,12 +35,13 @@ pub enum AssetDownloadError {impl From<download::Error> for AssetDownloadError {fn from(value: download::Error) -> Self {+ let retryable = value.is_retryable();match value {download::Error::Cancelled => Self::Cancelled,download::Error::Target { source, .. } => Self::from(*source),download::Error::Fallback { single, .. } => Self::from(*single),download::Error::Validation(detail) => Self::Validation { detail },- download::Error::Http(status) => Self::Http { status },+ download::Error::Http(status) => Self::Http { status, retryable },download::Error::Network(detail) => Self::Network { detail },download::Error::SizeMismatch { expected, actual } => Self::InvalidDownload {detail: format!("size mismatch: expected {expected} bytes, got {actual}"),diff --git a/rust/crates/assets/src/download.rs b/rust/crates/assets/src/download.rsindex d50ce1d76c..301ee2f3a5 100644--- a/rust/crates/assets/src/download.rs+++ b/rust/crates/assets/src/download.rs@@ -54,6 +54,19 @@ pub enum Error {Json(#[from] serde_json::Error),}+impl Error {+ pub fn is_retryable(&self) -> bool {+ match self {+ Self::Target { source, .. } | Self::Fallback { single: source, .. } => {+ source.is_retryable()+ }+ Self::Cancelled | Self::Validation(_) | Self::StorageFull => false,+ Self::Http(401 | 403 | 404) => false,+ _ => true,+ }+ }+}+impl From<std::io::Error> for Error {fn from(err: std::io::Error) -> Self {if err.kind() == std::io::ErrorKind::StorageFull {@@ -523,8 +536,9 @@ async fn download_file_single(}if !response.status().is_success() {- if attempt == MAX_ATTEMPTS {- return Err(Error::Http(response.status().as_u16()));+ let error = Error::Http(response.status().as_u16());+ if attempt == MAX_ATTEMPTS || !error.is_retryable() {+ return Err(error);}retry_count = retry_count.saturating_add(1);continue;@@ -837,13 +851,13 @@ impl RangeDownloadContext<'_, '_> {};if response.status() != StatusCode::PARTIAL_CONTENT {+ let error = Error::Http(response.status().as_u16());if response.status() == StatusCode::OK|| response.status() == StatusCode::RANGE_NOT_SATISFIABLE+ || attempt == MAX_ATTEMPTS+ || !error.is_retryable(){- return Err(Error::Http(response.status().as_u16()));- }- if attempt == MAX_ATTEMPTS {- return Err(Error::Http(response.status().as_u16()));+ return Err(error);}self.increment_retry(part_index);self.emit_progress();@@ -2588,6 +2602,214 @@ mod tests {let _ = fs::remove_dir_all(test_dir);}+ #[tokio::test]+ async fn permanent_http_errors_stop_single_stream_retries() {+ for status in [401, 403, 404] {+ let requests = http_failure_requests(status, false).await;+ assert_eq!(requests.get(&None), Some(&1), "HTTP {status}");+ }+ }++ #[tokio::test]+ async fn permanent_http_errors_allow_one_full_stream_fallback() {+ for status in [401, 403, 404] {+ let requests = http_failure_requests(status, true).await;+ assert!(requests.keys().any(Option::is_some));+ assert_eq!(requests.get(&None), Some(&1), "HTTP {status}");+ assert!(+ requests.values().all(|count| *count == 1),+ "HTTP {status}: {requests:?}"+ );+ }+ }++ #[tokio::test]+ async fn other_http_errors_keep_single_stream_retries() {+ for status in [400, 416, 429, 500] {+ let requests = http_failure_requests(status, false).await;+ assert_eq!(requests.get(&None), Some(&MAX_ATTEMPTS), "HTTP {status}");+ }+ }++ #[tokio::test]+ async fn recoverable_http_failures_still_download() {+ for (ranged, status) in [+ (false, 429),+ (false, 500),+ (true, 200),+ (true, 403),+ (true, 416),+ ] {+ let bytes = Arc::new(sample_bytes(MIN_RANGE_DOWNLOAD_BYTES as usize));+ let full_gets = Arc::new(AtomicUsize::new(0));+ let server = {+ let bytes = Arc::clone(&bytes);+ let full_gets = Arc::clone(&full_gets);+ TestServer::spawn(move |mut stream| {+ let Some(request) = read_test_request(&stream) else {+ return;+ };+ if request.line.starts_with("HEAD ") {+ let ranges = if ranged {+ "Accept-Ranges: bytes\r\n"+ } else {+ ""+ };+ let response = format!(+ "HTTP/1.1 200 OK\r\nContent-Length: {}\r\n{ranges}Connection: close\r\n\r\n",+ bytes.len()+ );+ let _ = stream.write_all(response.as_bytes());+ return;+ }+ if request.range.is_some()+ || (!ranged && full_gets.fetch_add(1, Ordering::SeqCst) == 0)+ {+ let response = format!(+ "HTTP/1.1 {status} Test\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"+ );+ let _ = stream.write_all(response.as_bytes());+ return;+ }+ if ranged {+ full_gets.fetch_add(1, Ordering::SeqCst);+ }+ let response = format!(+ "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",+ bytes.len()+ );+ let _ = stream.write_all(response.as_bytes());+ let _ = stream.write_all(bytes.as_slice());+ })+ };+ let test_dir = scratch_dir(&format!("recoverable-{status}-{ranged}"));+ let destination = test_dir.join("model.bin");+ download(+ vec![DownloadTarget {+ label: "Model".to_string(),+ url: server.url("/model.bin"),+ expected_size: bytes.len() as u64,+ sha256: sha_hex(&bytes),+ destination: destination.clone(),+ }],+ |_| {},+ false,+ )+ .await+ .expect("recoverable download");+ assert_eq!(fs::read(&destination).unwrap(), *bytes);+ assert_eq!(full_gets.load(Ordering::SeqCst), if ranged { 1 } else { 2 });+ let _ = fs::remove_dir_all(test_dir);+ }+ }++ #[tokio::test]+ async fn batch_http_errors_prefer_non_retryable_failure() {+ use crate::{Asset, AssetFile, AssetStore};++ let server = TestServer::spawn(|mut stream| {+ let Some(request) = read_test_request(&stream) else {+ return;+ };+ let (status, size) = if request.line.starts_with("HEAD ") {+ (200, 1)+ } else {+ let path = request.line.split_whitespace().nth(1).unwrap();+ (path[1..].parse::<u16>().unwrap(), 0)+ };+ let response = format!(+ "HTTP/1.1 {status} Test\r\nContent-Length: {size}\r\nConnection: close\r\n\r\n"+ );+ let _ = stream.write_all(response.as_bytes());+ });++ for (statuses, expected) in [([500, 403], 403), ([403, 500], 403), ([500, 502], 500)] {+ let assets = statuses.map(|status| {+ Asset::file(+ vec![status.to_string()],+ AssetFile {+ name: "model".to_string(),+ url: server.url(&format!("/{status}")),+ size: 1,+ sha256: "0".repeat(64),+ },+ )+ .unwrap()+ });+ let root = scratch_dir("batch-retryability");+ let error = AssetStore::new(&root)+ .download(&assets, |_| {}, CancellationToken::new())+ .await+ .unwrap_err();+ assert!(+ matches!(+ &error,+ Error::Target { source, .. }+ if matches!(**source, Error::Http(status) if status == expected)+ ),+ "{statuses:?}: {error}"+ );+ let _ = fs::remove_dir_all(root);+ }+ }++ async fn http_failure_requests(+ status: u16,+ ranged: bool,+ ) -> std::collections::HashMap<Option<String>, usize> {+ let requests = Arc::new(Mutex::new(std::collections::HashMap::new()));+ let server = {+ let requests = Arc::clone(&requests);+ TestServer::spawn(move |mut stream| {+ let Some(request) = read_test_request(&stream) else {+ return;+ };+ let response = if request.line.starts_with("HEAD ") {+ let ranges = if ranged {+ "Accept-Ranges: bytes\r\n"+ } else {+ ""+ };+ format!(+ "HTTP/1.1 200 OK\r\nContent-Length: {MIN_RANGE_DOWNLOAD_BYTES}\r\n{ranges}Connection: close\r\n\r\n"+ )+ } else {+ *requests.lock().unwrap().entry(request.range).or_insert(0) += 1;+ format!(+ "HTTP/1.1 {status} Failure\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"+ )+ };+ let _ = stream.write_all(response.as_bytes());+ })+ };+ let test_dir = scratch_dir(&format!("http-{status}-{ranged}"));+ let destination = test_dir.join("model.bin");+ let error = download(+ vec![DownloadTarget {+ label: "Model".to_string(),+ url: server.url("/model.bin"),+ expected_size: MIN_RANGE_DOWNLOAD_BYTES,+ destination: destination.clone(),+ sha256: "0".repeat(64),+ }],+ |_| {},+ false,+ )+ .await+ .expect_err("HTTP failure");+ let Error::Target { source, .. } = error else {+ panic!("missing download context")+ };+ let error = match *source {+ Error::Fallback { single, .. } => *single,+ error => error,+ };+ assert!(matches!(error, Error::Http(actual) if actual == status));+ assert!(!destination.exists());+ let _ = fs::remove_dir_all(test_dir);+ requests.lock().unwrap().clone()+ }+#[tokio::test]async fn sha_pinned_target_adopts_cached_file_regardless_of_url() {let test_dir = scratch_dir("sha-cached");diff --git a/rust/crates/assets/src/store.rs b/rust/crates/assets/src/store.rsindex 18e5878e9a..0842732ce5 100644--- a/rust/crates/assets/src/store.rs+++ b/rust/crates/assets/src/store.rs@@ -189,7 +189,11 @@ impl AssetStore {if cancellation.is_cancelled() {return Err(Error::Cancelled);}- if let Some(error) = results.into_iter().find_map(Result::err) {+ if let Some(error) = results+ .into_iter()+ .filter_map(Result::err)+ .min_by_key(Error::is_retryable)+ {return Err(error);}Ok(())