π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Add checksum verification to the model downloader and use it for ML test assets
Manav Rathid50f968282b1
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex c689e59641..11ce76b1dc 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2454,6 +2454,7 @@ version = "0.0.0"dependencies = ["anyhow","ente-image",+ "ente-model-download","fast_image_resize","flate2","half",@@ -2466,13 +2467,12 @@ dependencies = ["plsfix","quick-xml","regex",- "reqwest 0.12.28","serde","serde_json",- "sha2 0.10.9","tar","tempfile","thiserror 2.0.18",+ "url","usearch",]@@ -7134,7 +7134,6 @@ dependencies = ["base64 0.22.1","bytes","encoding_rs",- "futures-channel","futures-core","futures-util","h2",diff --git a/rust/crates/model-download/src/download.rs b/rust/crates/model-download/src/download.rsindex be8984fa8a..2afd97a63b 100644--- a/rust/crates/model-download/src/download.rs+++ b/rust/crates/model-download/src/download.rs@@ -9,6 +9,7 @@ use futures_util::future::try_join_all;use reqwest::header::{ACCEPT_RANGES, CONTENT_RANGE, ETAG, IF_RANGE, LAST_MODIFIED, RANGE};use reqwest::{Client, Response, StatusCode};use serde::{Deserialize, Serialize};+use sha2::{Digest, Sha256};use tokio::runtime::Builder;use tokio::time::timeout;@@ -70,6 +71,7 @@ pub struct Target {pub label: String,pub url: String,pub destination_path: String,+ pub sha256: Option<String>,}#[derive(Debug, Clone, Serialize, Deserialize)]@@ -432,7 +434,7 @@ async fn download_file(.await{Ok(report) => return Ok(report),- Err(Error::Cancelled) => return Err(Error::Cancelled),+ Err(err @ (Error::Cancelled | Error::Validation(_))) => return Err(err),Err(err) => {range_error = Some(err);cleanup_range_download(destination);@@ -660,6 +662,14 @@ async fn download_file_single(continue;}+ if let Some(expected) = &target.sha256+ && let Err(err) = check_sha256(&tmp_path, expected, &target.label)+ {+ let _ = fs::remove_file(&tmp_path);+ let _ = fs::remove_file(&partial_metadata_path);+ return Err(err);+ }+if let Err(err) = validate(target, &tmp_path) {let _ = fs::remove_file(&tmp_path);let _ = fs::remove_file(&partial_metadata_path);@@ -781,6 +791,14 @@ async fn download_file_ranged(emit_range_file_progress(total, file_started_at, &range_states, &on_progress);+ if let Some(expected) = &target.sha256+ && let Err(err) = check_sha256(&tmp_path, expected, &target.label)+ {+ let _ = fs::remove_file(&tmp_path);+ let _ = fs::remove_file(&range_metadata_path);+ return Err(err);+ }+if let Err(err) = validate(target, &tmp_path) {let _ = fs::remove_file(&tmp_path);let _ = fs::remove_file(&range_metadata_path);@@ -1518,6 +1536,13 @@ fn prepare_cached_download(if !destination.exists() || validate(target, destination).is_err() {return false;}+ if let Some(expected) = &target.sha256 {+ if check_sha256(destination, expected, &target.label).is_err() {+ return false;+ }+ cleanup_range_download(destination);+ return true;+ }let adopted = if !metadata_path_for(destination).exists() {let size = file_size(destination).unwrap_or(0);let _ = write_download_metadata(destination, target, size, None);@@ -1604,6 +1629,35 @@ pub fn metadata_path_for(path: &Path) -> PathBuf {PathBuf::from(format!("{}.metadata.json", path.display()))}+pub fn sha256_file(path: &Path) -> Result<String, Error> {+ use std::io::Read;+ let mut file = File::open(path)?;+ let mut hasher = Sha256::new();+ let mut buffer = [0u8; 1024 * 1024];+ loop {+ let read = file.read(&mut buffer)?;+ if read == 0 {+ break;+ }+ hasher.update(&buffer[..read]);+ }+ let mut hex = String::with_capacity(64);+ for byte in hasher.finalize() {+ hex.push_str(&format!("{byte:02x}"));+ }+ Ok(hex)+}++fn check_sha256(path: &Path, expected: &str, label: &str) -> Result<(), Error> {+ let actual = sha256_file(path)?;+ if actual != expected.trim().to_ascii_lowercase() {+ return Err(Error::Validation(format!(+ "{label} checksum mismatch: expected {expected}, got {actual}"+ )));+ }+ Ok(())+}+fn range_metadata_path_for(path: &Path) -> PathBuf {PathBuf::from(format!("{}.tmp.ranges.json", path.display()))}@@ -1892,6 +1946,7 @@ mod tests {label: "Model".to_string(),url,destination_path: destination.display().to_string(),+ sha256: None,}],|_, _| Ok(()),|_| {},@@ -1941,6 +1996,7 @@ mod tests {label: "Model".to_string(),url: server.url("/model.bin"),destination_path: destination.display().to_string(),+ sha256: None,}],|_, _| Ok(()),|_| {},@@ -1978,6 +2034,7 @@ mod tests {label: "Model".to_string(),url: server.url("/model.bin"),destination_path: destination.display().to_string(),+ sha256: None,}],|_, _| Err(Error::Validation("rejected".to_string())),|_| {},@@ -2031,6 +2088,7 @@ mod tests {label: "Model".to_string(),url: server.url("/model.bin"),destination_path: destination.display().to_string(),+ sha256: None,};fetch(vec![target.clone()], |_, _| Ok(()), |_| {}, || false)@@ -2069,6 +2127,7 @@ mod tests {label: "Model".to_string(),url: server.url("/model.bin"),destination_path: destination.display().to_string(),+ sha256: None,};let require_zero_header = |_: &Target, path: &Path| match fs::read(path).ok()@@ -2120,6 +2179,7 @@ mod tests {label: "Model".to_string(),url: server.url("/model.bin"),destination_path: destination.display().to_string(),+ sha256: None,};fetch(vec![target], |_, _| Ok(()), |_| {}, || false).expect("fetch adopts existing file");@@ -2160,6 +2220,7 @@ mod tests {label: "Model".to_string(),url: server.url("/model.bin"),destination_path: destination.display().to_string(),+ sha256: None,};fetch(vec![target], |_, _| Ok(()), |_| {}, || false).expect("fetch redownloads");@@ -2197,6 +2258,7 @@ mod tests {label: "Model".to_string(),url: "http://127.0.0.1:1/old-model.bin".to_string(),destination_path: destination.display().to_string(),+ sha256: None,};write_download_metadata(&destination, &stale_target, bytes.len() as u64, None).expect("write stale sidecar");@@ -2205,6 +2267,7 @@ mod tests {label: "Model".to_string(),url: server.url("/model.bin"),destination_path: destination.display().to_string(),+ sha256: None,};fetch(vec![target], |_, _| Ok(()), |_| {}, || false).expect("fetch redownloads");@@ -2251,6 +2314,7 @@ mod tests {label: "Model".to_string(),url: server.url("/model.bin"),destination_path: destination.display().to_string(),+ sha256: None,};fs::write(tmp_path_for(&destination), &bytes[..256]).expect("place partial tmp");write_partial_download_metadata(@@ -2310,6 +2374,7 @@ mod tests {label: "Model".to_string(),url: server.url("/model.bin"),destination_path: destination.display().to_string(),+ sha256: None,};fs::write(tmp_path_for(&destination), vec![0xAAu8; 256]).expect("place stale tmp");write_partial_download_metadata(@@ -2359,6 +2424,7 @@ mod tests {label: "Model".to_string(),url: server.url("/model.bin"),destination_path: destination.display().to_string(),+ sha256: None,};fetch(vec![target], |_, _| Ok(()), |_| {}, || false).expect("fresh download succeeds");@@ -2402,6 +2468,7 @@ mod tests {label: "Model".to_string(),url: server.url("/model.bin"),destination_path: destination.display().to_string(),+ sha256: None,};fs::write(tmp_path_for(&destination), &bytes[..256]).expect("place partial tmp");write_partial_download_metadata(@@ -2495,6 +2562,7 @@ mod tests {label: "Model".to_string(),url: "http://127.0.0.1:1/model.bin".to_string(),destination_path: destination.display().to_string(),+ sha256: None,}],|_, _| Ok(()),|_| {},@@ -2520,6 +2588,7 @@ mod tests {label: "Model".to_string(),url: "http://127.0.0.1:1/model.bin".to_string(),destination_path: destination.display().to_string(),+ sha256: None,}],|_, _| Ok(()),|_| {},@@ -2535,6 +2604,168 @@ mod tests {let _ = fs::remove_dir_all(test_dir);}+ #[test]+ fn sha_pinned_download_verifies_content() {+ let bytes = Arc::new(sample_bytes(2048));+ let get_count = Arc::new(AtomicUsize::new(0));+ let server = {+ let bytes = Arc::clone(&bytes);+ let get_count = Arc::clone(&get_count);+ TestServer::spawn(move |stream| {+ handle_no_range_test_request(stream, Arc::clone(&bytes), Arc::clone(&get_count));+ })+ };++ let test_dir = scratch_dir("sha-download");+ let destination = test_dir.join("model.bin");+ let mut expected = String::new();+ for byte in Sha256::digest(bytes.as_slice()) {+ expected.push_str(&format!("{byte:02x}"));+ }++ fetch(+ vec![Target {+ label: "Model".to_string(),+ url: server.url("/model.bin"),+ destination_path: destination.display().to_string(),+ sha256: Some(expected),+ }],+ |_, _| Ok(()),+ |_| {},+ || false,+ )+ .expect("sha-pinned download succeeds");+ assert_eq!(+ fs::read(&destination).expect("read downloaded file"),+ *bytes+ );++ let _ = fs::remove_dir_all(test_dir);+ }++ #[test]+ fn sha_mismatch_fails_immediately_without_installing() {+ let bytes = Arc::new(sample_bytes(2048));+ let get_count = Arc::new(AtomicUsize::new(0));+ let server = {+ let bytes = Arc::clone(&bytes);+ let get_count = Arc::clone(&get_count);+ TestServer::spawn(move |stream| {+ handle_no_range_test_request(stream, Arc::clone(&bytes), Arc::clone(&get_count));+ })+ };++ let test_dir = scratch_dir("sha-mismatch");+ let destination = test_dir.join("model.bin");++ let result = fetch(+ vec![Target {+ label: "Model".to_string(),+ url: server.url("/model.bin"),+ destination_path: destination.display().to_string(),+ sha256: Some("0".repeat(64)),+ }],+ |_, _| Ok(()),+ |_| {},+ || false,+ );++ assert!(result.is_err());+ assert!(!destination.exists());+ assert_eq!(get_count.load(Ordering::SeqCst), 1);++ let _ = fs::remove_dir_all(test_dir);+ }++ #[test]+ fn ranged_sha_mismatch_fails_without_single_stream_fallback() {+ let bytes = Arc::new(sample_bytes(MIN_RANGE_DOWNLOAD_BYTES as usize + 123));+ let head_count = Arc::new(AtomicUsize::new(0));+ let range_get_count = Arc::new(AtomicUsize::new(0));+ let full_get_count = Arc::new(AtomicUsize::new(0));+ let server = {+ let bytes = Arc::clone(&bytes);+ let head_count = Arc::clone(&head_count);+ let range_get_count = Arc::clone(&range_get_count);+ let full_get_count = Arc::clone(&full_get_count);+ TestServer::spawn(move |stream| {+ handle_range_test_request(+ stream,+ Arc::clone(&bytes),+ Arc::clone(&head_count),+ Arc::clone(&range_get_count),+ Arc::clone(&full_get_count),+ );+ })+ };++ let test_dir = scratch_dir("ranged-sha-mismatch");+ let destination = test_dir.join("model.bin");++ let result = fetch(+ vec![Target {+ label: "Model".to_string(),+ url: server.url("/model.bin"),+ destination_path: destination.display().to_string(),+ sha256: Some("0".repeat(64)),+ }],+ |_, _| Ok(()),+ |_| {},+ || false,+ );++ assert!(result.is_err());+ assert!(!destination.exists());+ assert!(!tmp_path_for(&destination).exists());+ assert_eq!(+ range_get_count.load(Ordering::SeqCst),+ RANGE_DOWNLOAD_CONCURRENCY+ );+ assert_eq!(full_get_count.load(Ordering::SeqCst), 0);++ let _ = fs::remove_dir_all(test_dir);+ }++ #[test]+ fn sha_pinned_target_adopts_cached_file_regardless_of_url() {+ let test_dir = scratch_dir("sha-cached");+ let destination = test_dir.join("model.bin");+ fs::write(&destination, b"model-bytes").expect("write cached file");+ let sha = sha256_file(&destination).expect("hash cached file");+ let old_target = Target {+ label: "Model".to_string(),+ url: "http://cache.example.org/old-location".to_string(),+ destination_path: destination.display().to_string(),+ sha256: Some(sha.clone()),+ };+ write_download_metadata(&destination, &old_target, 11, None).expect("write stale metadata");++ fetch(+ vec![Target {+ url: "http://127.0.0.1:9/new-location".to_string(),+ ..old_target.clone()+ }],+ |_, _| Ok(()),+ |_| {},+ || false,+ )+ .expect("cached file adopted by checksum despite changed URL");++ fs::write(&destination, b"corrupted").expect("corrupt cached file");+ let result = fetch(+ vec![Target {+ url: "http://127.0.0.1:9/new-location".to_string(),+ ..old_target+ }],+ |_, _| Ok(()),+ |_| {},+ || false,+ );+ assert!(result.is_err());++ let _ = fs::remove_dir_all(test_dir);+ }+fn sample_bytes(len: usize) -> Vec<u8> {(0..len).map(|index| (index % 251) as u8).collect()}diff --git a/rust/crates/model-download/src/downloader.rs b/rust/crates/model-download/src/downloader.rsindex 8b010c5fcf..695d19cb7c 100644--- a/rust/crates/model-download/src/downloader.rs+++ b/rust/crates/model-download/src/downloader.rs@@ -224,11 +224,13 @@ impl ModelDownloader {.join(format!("{id}.tar.gz")).display().to_string(),+ sha256: None,}],ModelDownloadTarget::Onnx { url, .. } => vec![Target {label: "Model".to_string(),url: url.clone(),destination_path: self.model_path(target).display().to_string(),+ sha256: None,}],}}diff --git a/rust/crates/model-download/src/gguf.rs b/rust/crates/model-download/src/gguf.rsindex ad7558fe79..93038996ca 100644--- a/rust/crates/model-download/src/gguf.rs+++ b/rust/crates/model-download/src/gguf.rs@@ -20,12 +20,14 @@ pub(crate) fn expected_targets(destination_path: model_path(models_dir, id, url, mmproj_url).display().to_string(),+ sha256: None,}];if let Some(mmproj) = mmproj_path(models_dir, id, url, mmproj_url) {targets.push(Target {label: "Mmproj".to_string(),url: trimmed(mmproj_url).unwrap().to_string(),destination_path: mmproj.display().to_string(),+ sha256: None,});}targetsdiff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.tomlindex 5e2937fc50..78b4f27c1b 100644--- a/rust/crates/photos/Cargo.toml+++ b/rust/crates/photos/Cargo.toml@@ -56,13 +56,13 @@ ort = { workspace = true, features = [[dev-dependencies]anyhow = "1"+ente-model-download.workspace = trueflate2 = "1.1"-reqwest = { version = "0.12", default-features = false, features = ["blocking", "rustls-tls-webpki-roots"] }serde = { version = "1", features = ["derive"] }serde_json.workspace = true-sha2 = "0.10"tar = "0.4"tempfile = "3.12"+url = "2.5.8"[[test]]name = "ml_indexing"diff --git a/rust/crates/photos/tests/support/ml_indexing.rs b/rust/crates/photos/tests/support/ml_indexing.rsindex 14b8d5bc88..47ca04c51f 100644--- a/rust/crates/photos/tests/support/ml_indexing.rs+++ b/rust/crates/photos/tests/support/ml_indexing.rs@@ -1,12 +1,11 @@use std::{collections::{BTreeSet, HashMap, HashSet},fs,- io::{Read, Write},path::{Path, PathBuf},- time::Duration,};-use anyhow::{Context, Result, anyhow, bail};+use anyhow::{Context, Result, bail};+use ente_model_download::download;use ente_photos::ml::{error::MlError,indexing::{@@ -16,17 +15,14 @@ use ente_photos::ml::{types::FaceResult as RustFaceResult,};use flate2::read::GzDecoder;-use reqwest::{StatusCode, Url, blocking::Client};use serde::Deserialize;-use sha2::{Digest, Sha256};+use url::Url;const ASSET_LOCK_PATH: &str = "infra/ml/test/ml_indexing/assets.json";const CLIP_EMBEDDING_DIM: usize = 512;const FACE_EMBEDDING_DIM: usize = 192;const FLOAT_TOLERANCE: f64 = 1e-8;const PRINT_STATS_ENV: &str = "ENTE_ML_INDEXING_PRINT_STATS";-const DOWNLOAD_MAX_ATTEMPTS: usize = 4;-const DOWNLOAD_RETRY_BASE_DELAY_MS: u64 = 500;pub(crate) fn run_with_large_stack(name: &str, test: fn() -> Result<()>) {let result = std::thread::Builder::new()@@ -168,7 +164,6 @@ fn should_print_stats() -> bool {pub(crate) struct MlIndexingTestContext {asset_lock: AssetLock,cache_dir: PathBuf,- client: Client,golden_results: HashMap<String, ComparableResult>,manifest: FixtureManifest,runtime_config: MlRuntimeConfig,@@ -179,30 +174,21 @@ impl MlIndexingTestContext {let repo_root = repo_root()?;let asset_lock = load_asset_lock(&repo_root)?;let cache_dir = cache_dir(&repo_root);- let client = Client::builder()- .user_agent("ente-rust-ml-indexing-test")- .build()- .context("build HTTP client")?;-- let manifest_path =- resolve_document_asset(&client, &cache_dir, "manifest", &asset_lock.manifest)?;- let golden_path = resolve_document_asset(- &client,- &cache_dir,- "python-golden",- &asset_lock.python_golden,- )?;++ let manifest_path = resolve_document_asset(&cache_dir, "manifest", &asset_lock.manifest)?;+ let golden_path =+ resolve_document_asset(&cache_dir, "python-golden", &asset_lock.python_golden)?;let manifest = load_manifest(&manifest_path)?;let golden_results = load_golden_results(&golden_path)?;let onnx_runtime_library =- resolve_onnx_runtime_library(&client, &cache_dir, &asset_lock.onnx_runtime)?;+ resolve_onnx_runtime_library(&cache_dir, &asset_lock.onnx_runtime)?;let _ = ort::init_from(&onnx_runtime_library).context("load ONNX Runtime dynamic library")?.commit();let runtime_config = MlRuntimeConfig {- model_paths: resolve_model_paths(&client, &cache_dir, &asset_lock.models)?,+ model_paths: resolve_model_paths(&cache_dir, &asset_lock.models)?,provider_policy: ExecutionProviderPolicy {prefer_coreml: false,prefer_nnapi: false,@@ -214,7 +200,6 @@ impl MlIndexingTestContext {Ok(Self {asset_lock,cache_dir,- client,golden_results,manifest,runtime_config,@@ -353,12 +338,7 @@ impl MlIndexingTestContext {}fn resolve_fixture(&self, fixture: &FixtureFile) -> Result<PathBuf> {- resolve_fixture_asset(- &self.client,- &self.cache_dir,- &self.asset_lock.fixture_base_url,- fixture,- )+ resolve_fixture_asset(&self.cache_dir, &self.asset_lock.fixture_base_url, fixture)}fn unsupported_decode_file_ids(&self) -> HashSet<String> {@@ -539,18 +519,12 @@ fn file_id_for_manifest_path(path: &str) -> Result<String> {.with_context(|| format!("derive file_id from manifest path '{path}'"))}-fn resolve_document_asset(- client: &Client,- cache_dir: &Path,- label: &str,- asset: &DocumentAsset,-) -> Result<PathBuf> {+fn resolve_document_asset(cache_dir: &Path, label: &str, asset: &DocumentAsset) -> Result<PathBuf> {let target = cache_path_for(cache_dir, "documents", &asset.path, &asset.sha256)?;- ensure_remote_asset(client, &asset.url, &asset.sha256, &target, label)+ ensure_remote_asset(&asset.url, &asset.sha256, &target, label)}fn resolve_fixture_asset(- client: &Client,cache_dir: &Path,fixture_base_url: &str,fixture: &FixtureFile,@@ -562,14 +536,10 @@ fn resolve_fixture_asset(.with_context(|| format!("join fixture URL for {}", fixture.path))?.to_string();let target = cache_path_for(cache_dir, "fixtures", &fixture.path, &fixture.sha256)?;- ensure_remote_asset(client, &url, &fixture.sha256, &target, &file_id)+ ensure_remote_asset(&url, &fixture.sha256, &target, &file_id)}-fn resolve_onnx_runtime_library(- client: &Client,- cache_dir: &Path,- assets: &OnnxRuntimeAssets,-) -> Result<PathBuf> {+fn resolve_onnx_runtime_library(cache_dir: &Path, assets: &OnnxRuntimeAssets) -> Result<PathBuf> {let target_key = onnx_runtime_target_key()?;let archive = assets.archives.get(&target_key).with_context(|| {let supported = assets@@ -591,7 +561,6 @@ fn resolve_onnx_runtime_library(&archive.sha256,)?;let archive_path = ensure_remote_asset(- client,&archive.url,&archive.sha256,&archive_path,@@ -682,16 +651,10 @@ fn extract_tgz_member(archive_path: &Path, member_path: &str, target: &Path) ->);}-fn resolve_model_paths(- client: &Client,- cache_dir: &Path,- models: &ModelAssets,-) -> Result<ModelPaths> {- let face_detection =- resolve_model_asset(client, cache_dir, "face-detection", &models.face_detection)?;- let face_embedding =- resolve_model_asset(client, cache_dir, "face-embedding", &models.face_embedding)?;- let clip_image = resolve_model_asset(client, cache_dir, "clip-image", &models.clip_image)?;+fn resolve_model_paths(cache_dir: &Path, models: &ModelAssets) -> Result<ModelPaths> {+ let face_detection = resolve_model_asset(cache_dir, "face-detection", &models.face_detection)?;+ let face_embedding = resolve_model_asset(cache_dir, "face-embedding", &models.face_embedding)?;+ let clip_image = resolve_model_asset(cache_dir, "clip-image", &models.clip_image)?;Ok(ModelPaths {face_detection: face_detection.to_string_lossy().into_owned(),@@ -707,142 +670,32 @@ fn resolve_model_paths(})}-fn resolve_model_asset(- client: &Client,- cache_dir: &Path,- label: &str,- asset: &ModelAsset,-) -> Result<PathBuf> {+fn resolve_model_asset(cache_dir: &Path, label: &str, asset: &ModelAsset) -> Result<PathBuf> {let target = cache_path_for(cache_dir, "models", &asset.file_name, &asset.sha256)?;- ensure_remote_asset(client, &asset.url, &asset.sha256, &target, label)-}--enum DownloadAttemptError {- Fatal(anyhow::Error),- Retryable(anyhow::Error),+ ensure_remote_asset(&asset.url, &asset.sha256, &target, label)}fn ensure_remote_asset(- client: &Client,url: &str,expected_sha256: &str,target: &Path,label: &str,) -> Result<PathBuf> {- if target.is_file() {- verify_file_sha256(target, expected_sha256, label)?;- return Ok(target.to_path_buf());- }-- let parent = target- .parent()- .with_context(|| format!("resolve parent for {}", target.display()))?;- fs::create_dir_all(parent)- .with_context(|| format!("create cache directory {}", parent.display()))?;-- let mut last_error = None;- for attempt in 1..=DOWNLOAD_MAX_ATTEMPTS {- match download_asset_once(client, url, expected_sha256, target, parent, label) {- Ok(path) => return Ok(path),- Err(DownloadAttemptError::Fatal(error)) => return Err(error),- Err(DownloadAttemptError::Retryable(error)) => {- last_error = Some(error);- if attempt < DOWNLOAD_MAX_ATTEMPTS {- std::thread::sleep(download_retry_delay(attempt));- }- }- }- }-- Err(last_error.expect("retry loop records an error before exhausting attempts")).with_context(- || format!("download {label} from {url} after {DOWNLOAD_MAX_ATTEMPTS} attempt(s)"),+ download::fetch(+ vec![download::Target {+ label: label.to_string(),+ url: url.to_string(),+ destination_path: target.display().to_string(),+ sha256: Some(normalize_sha256(expected_sha256)),+ }],+ |_, _| Ok(()),+ |_| {},+ || false,)-}--fn download_asset_once(- client: &Client,- url: &str,- expected_sha256: &str,- target: &Path,- parent: &Path,- label: &str,-) -> Result<PathBuf, DownloadAttemptError> {- let mut response = match client.get(url).send() {- Ok(response) if response.status().is_success() => response,- Ok(response) => {- let status = response.status();- let error = anyhow!("download {label} from {url}: HTTP status {status}");- return Err(if is_retryable_download_status(status) {- DownloadAttemptError::Retryable(error)- } else {- DownloadAttemptError::Fatal(error)- });- }- Err(error) => {- return Err(DownloadAttemptError::Retryable(- anyhow::Error::new(error).context(format!("download {label} from {url}")),- ));- }- };-- let mut temp_file = tempfile::NamedTempFile::new_in(parent)- .map_err(|error| fatal_io(error, format!("create temp file in {}", parent.display())))?;- let mut hasher = Sha256::new();- let mut buffer = [0_u8; 1024 * 1024];- loop {- let read = match response.read(&mut buffer) {- Ok(read) => read,- Err(error) => {- return Err(DownloadAttemptError::Retryable(- anyhow::Error::new(error)- .context(format!("read {label} response body from {url}")),- ));- }- };- if read == 0 {- break;- }- hasher.update(&buffer[..read]);- temp_file- .write_all(&buffer[..read])- .map_err(|error| fatal_io(error, format!("write cached {label}")))?;- }-- let actual_sha256 = hex_digest(&hasher.finalize());- if actual_sha256 != normalize_sha256(expected_sha256) {- return Err(DownloadAttemptError::Retryable(anyhow!(- "{label} SHA-256 mismatch after download: expected {}, got {actual_sha256}",- normalize_sha256(expected_sha256)- )));- }-- temp_file- .flush()- .map_err(|error| fatal_io(error, format!("flush cached {label}")))?;- temp_file.persist(target).map_err(|error| {- DownloadAttemptError::Fatal(anyhow!(- "persist cached {label} to {}: {}",- target.display(),- error.error- ))- })?;+ .with_context(|| format!("download {label} from {url}"))?;Ok(target.to_path_buf())}-fn fatal_io(error: std::io::Error, context: String) -> DownloadAttemptError {- DownloadAttemptError::Fatal(anyhow::Error::new(error).context(context))-}--fn is_retryable_download_status(status: StatusCode) -> bool {- status == StatusCode::TOO_MANY_REQUESTS- || status == StatusCode::REQUEST_TIMEOUT- || status.is_server_error()-}--fn download_retry_delay(attempt: usize) -> Duration {- Duration::from_millis(DOWNLOAD_RETRY_BASE_DELAY_MS * 2_u64.pow((attempt - 1) as u32))-}-fn cache_path_for(cache_dir: &Path,namespace: &str,@@ -861,7 +714,8 @@ fn cache_path_for(}fn verify_file_sha256(path: &Path, expected_sha256: &str, label: &str) -> Result<()> {- let actual_sha256 = sha256_file(path)?;+ let actual_sha256 =+ download::sha256_file(path).with_context(|| format!("hash {}", path.display()))?;let expected_sha256 = normalize_sha256(expected_sha256);if actual_sha256 != expected_sha256 {bail!(@@ -874,35 +728,10 @@ fn verify_file_sha256(path: &Path, expected_sha256: &str, label: &str) -> ResultOk(())}-fn sha256_file(path: &Path) -> Result<String> {- let mut file = fs::File::open(path).with_context(|| format!("open {}", path.display()))?;- let mut hasher = Sha256::new();- let mut buffer = [0_u8; 1024 * 1024];- loop {- let read = file- .read(&mut buffer)- .with_context(|| format!("read {}", path.display()))?;- if read == 0 {- break;- }- hasher.update(&buffer[..read]);- }- let digest = hasher.finalize();- Ok(hex_digest(&digest))-}-fn normalize_sha256(value: &str) -> String {value.trim().to_ascii_lowercase()}-fn hex_digest(bytes: &[u8]) -> String {- let mut hex = String::with_capacity(bytes.len() * 2);- for byte in bytes {- hex.push_str(&format!("{byte:02x}"));- }- hex-}-fn expect_decode_error(file_id: &str, req: AnalyzeImageRequest, failures: &mut Vec<String>) {match analyze_image(req) {Err(MlError::Decode(_)) => {}