Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Add checksum verification to the model downloader and use it for ML test assets

Manav Rathid50f968282b1
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index c689e59641..11ce76b1dc 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2454,6 +2454,7 @@ version = "0.0.0"
dependencies = [
"anyhow",
"ente-image",
+ "ente-model-download",
"fast_image_resize",
"flate2",
"half",
@@ -2466,13 +2467,12 @@ dependencies = [
"plsfix",
"quick-xml",
"regex",
- "reqwest 0.12.28",
"serde",
"serde_json",
- "sha2 0.10.9",
"tar",
"tempfile",
"thiserror 2.0.18",
+ "url",
"usearch",
]
@@ -7134,7 +7134,6 @@ dependencies = [
"base64 0.22.1",
"bytes",
"encoding_rs",
- "futures-channel",
"futures-core",
"futures-util",
"h2",
diff --git a/rust/crates/model-download/src/download.rs b/rust/crates/model-download/src/download.rs
index be8984fa8a..2afd97a63b 100644
--- a/rust/crates/model-download/src/download.rs
+++ b/rust/crates/model-download/src/download.rs
@@ -9,6 +9,7 @@ use futures_util::future::try_join_all;
use reqwest::header::{ACCEPT_RANGES, CONTENT_RANGE, ETAG, IF_RANGE, LAST_MODIFIED, RANGE};
use reqwest::{Client, Response, StatusCode};
use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
use tokio::runtime::Builder;
use tokio::time::timeout;
@@ -70,6 +71,7 @@ pub struct Target {
pub label: String,
pub url: String,
pub destination_path: String,
+ pub sha256: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -432,7 +434,7 @@ async fn download_file(
.await
{
Ok(report) => return Ok(report),
- Err(Error::Cancelled) => return Err(Error::Cancelled),
+ Err(err @ (Error::Cancelled | Error::Validation(_))) => return Err(err),
Err(err) => {
range_error = Some(err);
cleanup_range_download(destination);
@@ -660,6 +662,14 @@ async fn download_file_single(
continue;
}
+ if let Some(expected) = &target.sha256
+ && let Err(err) = check_sha256(&tmp_path, expected, &target.label)
+ {
+ let _ = fs::remove_file(&tmp_path);
+ let _ = fs::remove_file(&partial_metadata_path);
+ return Err(err);
+ }
+
if let Err(err) = validate(target, &tmp_path) {
let _ = fs::remove_file(&tmp_path);
let _ = fs::remove_file(&partial_metadata_path);
@@ -781,6 +791,14 @@ async fn download_file_ranged(
emit_range_file_progress(total, file_started_at, &range_states, &on_progress);
+ if let Some(expected) = &target.sha256
+ && let Err(err) = check_sha256(&tmp_path, expected, &target.label)
+ {
+ let _ = fs::remove_file(&tmp_path);
+ let _ = fs::remove_file(&range_metadata_path);
+ return Err(err);
+ }
+
if let Err(err) = validate(target, &tmp_path) {
let _ = fs::remove_file(&tmp_path);
let _ = fs::remove_file(&range_metadata_path);
@@ -1518,6 +1536,13 @@ fn prepare_cached_download(
if !destination.exists() || validate(target, destination).is_err() {
return false;
}
+ if let Some(expected) = &target.sha256 {
+ if check_sha256(destination, expected, &target.label).is_err() {
+ return false;
+ }
+ cleanup_range_download(destination);
+ return true;
+ }
let adopted = if !metadata_path_for(destination).exists() {
let size = file_size(destination).unwrap_or(0);
let _ = write_download_metadata(destination, target, size, None);
@@ -1604,6 +1629,35 @@ pub fn metadata_path_for(path: &Path) -> PathBuf {
PathBuf::from(format!("{}.metadata.json", path.display()))
}
+pub fn sha256_file(path: &Path) -> Result<String, Error> {
+ use std::io::Read;
+ let mut file = File::open(path)?;
+ let mut hasher = Sha256::new();
+ let mut buffer = [0u8; 1024 * 1024];
+ loop {
+ let read = file.read(&mut buffer)?;
+ if read == 0 {
+ break;
+ }
+ hasher.update(&buffer[..read]);
+ }
+ let mut hex = String::with_capacity(64);
+ for byte in hasher.finalize() {
+ hex.push_str(&format!("{byte:02x}"));
+ }
+ Ok(hex)
+}
+
+fn check_sha256(path: &Path, expected: &str, label: &str) -> Result<(), Error> {
+ let actual = sha256_file(path)?;
+ if actual != expected.trim().to_ascii_lowercase() {
+ return Err(Error::Validation(format!(
+ "{label} checksum mismatch: expected {expected}, got {actual}"
+ )));
+ }
+ Ok(())
+}
+
fn range_metadata_path_for(path: &Path) -> PathBuf {
PathBuf::from(format!("{}.tmp.ranges.json", path.display()))
}
@@ -1892,6 +1946,7 @@ mod tests {
label: "Model".to_string(),
url,
destination_path: destination.display().to_string(),
+ sha256: None,
}],
|_, _| Ok(()),
|_| {},
@@ -1941,6 +1996,7 @@ mod tests {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination_path: destination.display().to_string(),
+ sha256: None,
}],
|_, _| Ok(()),
|_| {},
@@ -1978,6 +2034,7 @@ mod tests {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination_path: destination.display().to_string(),
+ sha256: None,
}],
|_, _| Err(Error::Validation("rejected".to_string())),
|_| {},
@@ -2031,6 +2088,7 @@ mod tests {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination_path: destination.display().to_string(),
+ sha256: None,
};
fetch(vec![target.clone()], |_, _| Ok(()), |_| {}, || false)
@@ -2069,6 +2127,7 @@ mod tests {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination_path: destination.display().to_string(),
+ sha256: None,
};
let require_zero_header = |_: &Target, path: &Path| match fs::read(path)
.ok()
@@ -2120,6 +2179,7 @@ mod tests {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination_path: destination.display().to_string(),
+ sha256: None,
};
fetch(vec![target], |_, _| Ok(()), |_| {}, || false).expect("fetch adopts existing file");
@@ -2160,6 +2220,7 @@ mod tests {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination_path: destination.display().to_string(),
+ sha256: None,
};
fetch(vec![target], |_, _| Ok(()), |_| {}, || false).expect("fetch redownloads");
@@ -2197,6 +2258,7 @@ mod tests {
label: "Model".to_string(),
url: "http://127.0.0.1:1/old-model.bin".to_string(),
destination_path: destination.display().to_string(),
+ sha256: None,
};
write_download_metadata(&destination, &stale_target, bytes.len() as u64, None)
.expect("write stale sidecar");
@@ -2205,6 +2267,7 @@ mod tests {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination_path: destination.display().to_string(),
+ sha256: None,
};
fetch(vec![target], |_, _| Ok(()), |_| {}, || false).expect("fetch redownloads");
@@ -2251,6 +2314,7 @@ mod tests {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination_path: destination.display().to_string(),
+ sha256: None,
};
fs::write(tmp_path_for(&destination), &bytes[..256]).expect("place partial tmp");
write_partial_download_metadata(
@@ -2310,6 +2374,7 @@ mod tests {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination_path: destination.display().to_string(),
+ sha256: None,
};
fs::write(tmp_path_for(&destination), vec![0xAAu8; 256]).expect("place stale tmp");
write_partial_download_metadata(
@@ -2359,6 +2424,7 @@ mod tests {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination_path: destination.display().to_string(),
+ sha256: None,
};
fetch(vec![target], |_, _| Ok(()), |_| {}, || false).expect("fresh download succeeds");
@@ -2402,6 +2468,7 @@ mod tests {
label: "Model".to_string(),
url: server.url("/model.bin"),
destination_path: destination.display().to_string(),
+ sha256: None,
};
fs::write(tmp_path_for(&destination), &bytes[..256]).expect("place partial tmp");
write_partial_download_metadata(
@@ -2495,6 +2562,7 @@ mod tests {
label: "Model".to_string(),
url: "http://127.0.0.1:1/model.bin".to_string(),
destination_path: destination.display().to_string(),
+ sha256: None,
}],
|_, _| Ok(()),
|_| {},
@@ -2520,6 +2588,7 @@ mod tests {
label: "Model".to_string(),
url: "http://127.0.0.1:1/model.bin".to_string(),
destination_path: destination.display().to_string(),
+ sha256: None,
}],
|_, _| Ok(()),
|_| {},
@@ -2535,6 +2604,168 @@ mod tests {
let _ = fs::remove_dir_all(test_dir);
}
+ #[test]
+ fn sha_pinned_download_verifies_content() {
+ let bytes = Arc::new(sample_bytes(2048));
+ let get_count = Arc::new(AtomicUsize::new(0));
+ let server = {
+ let bytes = Arc::clone(&bytes);
+ let get_count = Arc::clone(&get_count);
+ TestServer::spawn(move |stream| {
+ handle_no_range_test_request(stream, Arc::clone(&bytes), Arc::clone(&get_count));
+ })
+ };
+
+ let test_dir = scratch_dir("sha-download");
+ let destination = test_dir.join("model.bin");
+ let mut expected = String::new();
+ for byte in Sha256::digest(bytes.as_slice()) {
+ expected.push_str(&format!("{byte:02x}"));
+ }
+
+ fetch(
+ vec![Target {
+ label: "Model".to_string(),
+ url: server.url("/model.bin"),
+ destination_path: destination.display().to_string(),
+ sha256: Some(expected),
+ }],
+ |_, _| Ok(()),
+ |_| {},
+ || false,
+ )
+ .expect("sha-pinned download succeeds");
+ assert_eq!(
+ fs::read(&destination).expect("read downloaded file"),
+ *bytes
+ );
+
+ let _ = fs::remove_dir_all(test_dir);
+ }
+
+ #[test]
+ fn sha_mismatch_fails_immediately_without_installing() {
+ let bytes = Arc::new(sample_bytes(2048));
+ let get_count = Arc::new(AtomicUsize::new(0));
+ let server = {
+ let bytes = Arc::clone(&bytes);
+ let get_count = Arc::clone(&get_count);
+ TestServer::spawn(move |stream| {
+ handle_no_range_test_request(stream, Arc::clone(&bytes), Arc::clone(&get_count));
+ })
+ };
+
+ let test_dir = scratch_dir("sha-mismatch");
+ let destination = test_dir.join("model.bin");
+
+ let result = fetch(
+ vec![Target {
+ label: "Model".to_string(),
+ url: server.url("/model.bin"),
+ destination_path: destination.display().to_string(),
+ sha256: Some("0".repeat(64)),
+ }],
+ |_, _| Ok(()),
+ |_| {},
+ || false,
+ );
+
+ assert!(result.is_err());
+ assert!(!destination.exists());
+ assert_eq!(get_count.load(Ordering::SeqCst), 1);
+
+ let _ = fs::remove_dir_all(test_dir);
+ }
+
+ #[test]
+ fn ranged_sha_mismatch_fails_without_single_stream_fallback() {
+ let bytes = Arc::new(sample_bytes(MIN_RANGE_DOWNLOAD_BYTES as usize + 123));
+ let head_count = Arc::new(AtomicUsize::new(0));
+ let range_get_count = Arc::new(AtomicUsize::new(0));
+ let full_get_count = Arc::new(AtomicUsize::new(0));
+ let server = {
+ let bytes = Arc::clone(&bytes);
+ let head_count = Arc::clone(&head_count);
+ let range_get_count = Arc::clone(&range_get_count);
+ let full_get_count = Arc::clone(&full_get_count);
+ TestServer::spawn(move |stream| {
+ handle_range_test_request(
+ stream,
+ Arc::clone(&bytes),
+ Arc::clone(&head_count),
+ Arc::clone(&range_get_count),
+ Arc::clone(&full_get_count),
+ );
+ })
+ };
+
+ let test_dir = scratch_dir("ranged-sha-mismatch");
+ let destination = test_dir.join("model.bin");
+
+ let result = fetch(
+ vec![Target {
+ label: "Model".to_string(),
+ url: server.url("/model.bin"),
+ destination_path: destination.display().to_string(),
+ sha256: Some("0".repeat(64)),
+ }],
+ |_, _| Ok(()),
+ |_| {},
+ || false,
+ );
+
+ assert!(result.is_err());
+ assert!(!destination.exists());
+ assert!(!tmp_path_for(&destination).exists());
+ assert_eq!(
+ range_get_count.load(Ordering::SeqCst),
+ RANGE_DOWNLOAD_CONCURRENCY
+ );
+ assert_eq!(full_get_count.load(Ordering::SeqCst), 0);
+
+ let _ = fs::remove_dir_all(test_dir);
+ }
+
+ #[test]
+ fn sha_pinned_target_adopts_cached_file_regardless_of_url() {
+ let test_dir = scratch_dir("sha-cached");
+ let destination = test_dir.join("model.bin");
+ fs::write(&destination, b"model-bytes").expect("write cached file");
+ let sha = sha256_file(&destination).expect("hash cached file");
+ let old_target = Target {
+ label: "Model".to_string(),
+ url: "http://cache.example.org/old-location".to_string(),
+ destination_path: destination.display().to_string(),
+ sha256: Some(sha.clone()),
+ };
+ write_download_metadata(&destination, &old_target, 11, None).expect("write stale metadata");
+
+ fetch(
+ vec![Target {
+ url: "http://127.0.0.1:9/new-location".to_string(),
+ ..old_target.clone()
+ }],
+ |_, _| Ok(()),
+ |_| {},
+ || false,
+ )
+ .expect("cached file adopted by checksum despite changed URL");
+
+ fs::write(&destination, b"corrupted").expect("corrupt cached file");
+ let result = fetch(
+ vec![Target {
+ url: "http://127.0.0.1:9/new-location".to_string(),
+ ..old_target
+ }],
+ |_, _| Ok(()),
+ |_| {},
+ || false,
+ );
+ assert!(result.is_err());
+
+ let _ = fs::remove_dir_all(test_dir);
+ }
+
fn sample_bytes(len: usize) -> Vec<u8> {
(0..len).map(|index| (index % 251) as u8).collect()
}
diff --git a/rust/crates/model-download/src/downloader.rs b/rust/crates/model-download/src/downloader.rs
index 8b010c5fcf..695d19cb7c 100644
--- a/rust/crates/model-download/src/downloader.rs
+++ b/rust/crates/model-download/src/downloader.rs
@@ -224,11 +224,13 @@ impl ModelDownloader {
.join(format!("{id}.tar.gz"))
.display()
.to_string(),
+ sha256: None,
}],
ModelDownloadTarget::Onnx { url, .. } => vec![Target {
label: "Model".to_string(),
url: url.clone(),
destination_path: self.model_path(target).display().to_string(),
+ sha256: None,
}],
}
}
diff --git a/rust/crates/model-download/src/gguf.rs b/rust/crates/model-download/src/gguf.rs
index ad7558fe79..93038996ca 100644
--- a/rust/crates/model-download/src/gguf.rs
+++ b/rust/crates/model-download/src/gguf.rs
@@ -20,12 +20,14 @@ pub(crate) fn expected_targets(
destination_path: model_path(models_dir, id, url, mmproj_url)
.display()
.to_string(),
+ sha256: None,
}];
if let Some(mmproj) = mmproj_path(models_dir, id, url, mmproj_url) {
targets.push(Target {
label: "Mmproj".to_string(),
url: trimmed(mmproj_url).unwrap().to_string(),
destination_path: mmproj.display().to_string(),
+ sha256: None,
});
}
targets
diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.toml
index 5e2937fc50..78b4f27c1b 100644
--- a/rust/crates/photos/Cargo.toml
+++ b/rust/crates/photos/Cargo.toml
@@ -56,13 +56,13 @@ ort = { workspace = true, features = [
[dev-dependencies]
anyhow = "1"
+ente-model-download.workspace = true
flate2 = "1.1"
-reqwest = { version = "0.12", default-features = false, features = ["blocking", "rustls-tls-webpki-roots"] }
serde = { version = "1", features = ["derive"] }
serde_json.workspace = true
-sha2 = "0.10"
tar = "0.4"
tempfile = "3.12"
+url = "2.5.8"
[[test]]
name = "ml_indexing"
diff --git a/rust/crates/photos/tests/support/ml_indexing.rs b/rust/crates/photos/tests/support/ml_indexing.rs
index 14b8d5bc88..47ca04c51f 100644
--- a/rust/crates/photos/tests/support/ml_indexing.rs
+++ b/rust/crates/photos/tests/support/ml_indexing.rs
@@ -1,12 +1,11 @@
use std::{
collections::{BTreeSet, HashMap, HashSet},
fs,
- io::{Read, Write},
path::{Path, PathBuf},
- time::Duration,
};
-use anyhow::{Context, Result, anyhow, bail};
+use anyhow::{Context, Result, bail};
+use ente_model_download::download;
use ente_photos::ml::{
error::MlError,
indexing::{
@@ -16,17 +15,14 @@ use ente_photos::ml::{
types::FaceResult as RustFaceResult,
};
use flate2::read::GzDecoder;
-use reqwest::{StatusCode, Url, blocking::Client};
use serde::Deserialize;
-use sha2::{Digest, Sha256};
+use url::Url;
const ASSET_LOCK_PATH: &str = "infra/ml/test/ml_indexing/assets.json";
const CLIP_EMBEDDING_DIM: usize = 512;
const FACE_EMBEDDING_DIM: usize = 192;
const FLOAT_TOLERANCE: f64 = 1e-8;
const PRINT_STATS_ENV: &str = "ENTE_ML_INDEXING_PRINT_STATS";
-const DOWNLOAD_MAX_ATTEMPTS: usize = 4;
-const DOWNLOAD_RETRY_BASE_DELAY_MS: u64 = 500;
pub(crate) fn run_with_large_stack(name: &str, test: fn() -> Result<()>) {
let result = std::thread::Builder::new()
@@ -168,7 +164,6 @@ fn should_print_stats() -> bool {
pub(crate) struct MlIndexingTestContext {
asset_lock: AssetLock,
cache_dir: PathBuf,
- client: Client,
golden_results: HashMap<String, ComparableResult>,
manifest: FixtureManifest,
runtime_config: MlRuntimeConfig,
@@ -179,30 +174,21 @@ impl MlIndexingTestContext {
let repo_root = repo_root()?;
let asset_lock = load_asset_lock(&repo_root)?;
let cache_dir = cache_dir(&repo_root);
- let client = Client::builder()
- .user_agent("ente-rust-ml-indexing-test")
- .build()
- .context("build HTTP client")?;
-
- let manifest_path =
- resolve_document_asset(&client, &cache_dir, "manifest", &asset_lock.manifest)?;
- let golden_path = resolve_document_asset(
- &client,
- &cache_dir,
- "python-golden",
- &asset_lock.python_golden,
- )?;
+
+ let manifest_path = resolve_document_asset(&cache_dir, "manifest", &asset_lock.manifest)?;
+ let golden_path =
+ resolve_document_asset(&cache_dir, "python-golden", &asset_lock.python_golden)?;
let manifest = load_manifest(&manifest_path)?;
let golden_results = load_golden_results(&golden_path)?;
let onnx_runtime_library =
- resolve_onnx_runtime_library(&client, &cache_dir, &asset_lock.onnx_runtime)?;
+ resolve_onnx_runtime_library(&cache_dir, &asset_lock.onnx_runtime)?;
let _ = ort::init_from(&onnx_runtime_library)
.context("load ONNX Runtime dynamic library")?
.commit();
let runtime_config = MlRuntimeConfig {
- model_paths: resolve_model_paths(&client, &cache_dir, &asset_lock.models)?,
+ model_paths: resolve_model_paths(&cache_dir, &asset_lock.models)?,
provider_policy: ExecutionProviderPolicy {
prefer_coreml: false,
prefer_nnapi: false,
@@ -214,7 +200,6 @@ impl MlIndexingTestContext {
Ok(Self {
asset_lock,
cache_dir,
- client,
golden_results,
manifest,
runtime_config,
@@ -353,12 +338,7 @@ impl MlIndexingTestContext {
}
fn resolve_fixture(&self, fixture: &FixtureFile) -> Result<PathBuf> {
- resolve_fixture_asset(
- &self.client,
- &self.cache_dir,
- &self.asset_lock.fixture_base_url,
- fixture,
- )
+ resolve_fixture_asset(&self.cache_dir, &self.asset_lock.fixture_base_url, fixture)
}
fn unsupported_decode_file_ids(&self) -> HashSet<String> {
@@ -539,18 +519,12 @@ fn file_id_for_manifest_path(path: &str) -> Result<String> {
.with_context(|| format!("derive file_id from manifest path '{path}'"))
}
-fn resolve_document_asset(
- client: &Client,
- cache_dir: &Path,
- label: &str,
- asset: &DocumentAsset,
-) -> Result<PathBuf> {
+fn resolve_document_asset(cache_dir: &Path, label: &str, asset: &DocumentAsset) -> Result<PathBuf> {
let target = cache_path_for(cache_dir, "documents", &asset.path, &asset.sha256)?;
- ensure_remote_asset(client, &asset.url, &asset.sha256, &target, label)
+ ensure_remote_asset(&asset.url, &asset.sha256, &target, label)
}
fn resolve_fixture_asset(
- client: &Client,
cache_dir: &Path,
fixture_base_url: &str,
fixture: &FixtureFile,
@@ -562,14 +536,10 @@ fn resolve_fixture_asset(
.with_context(|| format!("join fixture URL for {}", fixture.path))?
.to_string();
let target = cache_path_for(cache_dir, "fixtures", &fixture.path, &fixture.sha256)?;
- ensure_remote_asset(client, &url, &fixture.sha256, &target, &file_id)
+ ensure_remote_asset(&url, &fixture.sha256, &target, &file_id)
}
-fn resolve_onnx_runtime_library(
- client: &Client,
- cache_dir: &Path,
- assets: &OnnxRuntimeAssets,
-) -> Result<PathBuf> {
+fn resolve_onnx_runtime_library(cache_dir: &Path, assets: &OnnxRuntimeAssets) -> Result<PathBuf> {
let target_key = onnx_runtime_target_key()?;
let archive = assets.archives.get(&target_key).with_context(|| {
let supported = assets
@@ -591,7 +561,6 @@ fn resolve_onnx_runtime_library(
&archive.sha256,
)?;
let archive_path = ensure_remote_asset(
- client,
&archive.url,
&archive.sha256,
&archive_path,
@@ -682,16 +651,10 @@ fn extract_tgz_member(archive_path: &Path, member_path: &str, target: &Path) ->
);
}
-fn resolve_model_paths(
- client: &Client,
- cache_dir: &Path,
- models: &ModelAssets,
-) -> Result<ModelPaths> {
- let face_detection =
- resolve_model_asset(client, cache_dir, "face-detection", &models.face_detection)?;
- let face_embedding =
- resolve_model_asset(client, cache_dir, "face-embedding", &models.face_embedding)?;
- let clip_image = resolve_model_asset(client, cache_dir, "clip-image", &models.clip_image)?;
+fn resolve_model_paths(cache_dir: &Path, models: &ModelAssets) -> Result<ModelPaths> {
+ let face_detection = resolve_model_asset(cache_dir, "face-detection", &models.face_detection)?;
+ let face_embedding = resolve_model_asset(cache_dir, "face-embedding", &models.face_embedding)?;
+ let clip_image = resolve_model_asset(cache_dir, "clip-image", &models.clip_image)?;
Ok(ModelPaths {
face_detection: face_detection.to_string_lossy().into_owned(),
@@ -707,142 +670,32 @@ fn resolve_model_paths(
})
}
-fn resolve_model_asset(
- client: &Client,
- cache_dir: &Path,
- label: &str,
- asset: &ModelAsset,
-) -> Result<PathBuf> {
+fn resolve_model_asset(cache_dir: &Path, label: &str, asset: &ModelAsset) -> Result<PathBuf> {
let target = cache_path_for(cache_dir, "models", &asset.file_name, &asset.sha256)?;
- ensure_remote_asset(client, &asset.url, &asset.sha256, &target, label)
-}
-
-enum DownloadAttemptError {
- Fatal(anyhow::Error),
- Retryable(anyhow::Error),
+ ensure_remote_asset(&asset.url, &asset.sha256, &target, label)
}
fn ensure_remote_asset(
- client: &Client,
url: &str,
expected_sha256: &str,
target: &Path,
label: &str,
) -> Result<PathBuf> {
- if target.is_file() {
- verify_file_sha256(target, expected_sha256, label)?;
- return Ok(target.to_path_buf());
- }
-
- let parent = target
- .parent()
- .with_context(|| format!("resolve parent for {}", target.display()))?;
- fs::create_dir_all(parent)
- .with_context(|| format!("create cache directory {}", parent.display()))?;
-
- let mut last_error = None;
- for attempt in 1..=DOWNLOAD_MAX_ATTEMPTS {
- match download_asset_once(client, url, expected_sha256, target, parent, label) {
- Ok(path) => return Ok(path),
- Err(DownloadAttemptError::Fatal(error)) => return Err(error),
- Err(DownloadAttemptError::Retryable(error)) => {
- last_error = Some(error);
- if attempt < DOWNLOAD_MAX_ATTEMPTS {
- std::thread::sleep(download_retry_delay(attempt));
- }
- }
- }
- }
-
- Err(last_error.expect("retry loop records an error before exhausting attempts")).with_context(
- || format!("download {label} from {url} after {DOWNLOAD_MAX_ATTEMPTS} attempt(s)"),
+ download::fetch(
+ vec![download::Target {
+ label: label.to_string(),
+ url: url.to_string(),
+ destination_path: target.display().to_string(),
+ sha256: Some(normalize_sha256(expected_sha256)),
+ }],
+ |_, _| Ok(()),
+ |_| {},
+ || false,
)
-}
-
-fn download_asset_once(
- client: &Client,
- url: &str,
- expected_sha256: &str,
- target: &Path,
- parent: &Path,
- label: &str,
-) -> Result<PathBuf, DownloadAttemptError> {
- let mut response = match client.get(url).send() {
- Ok(response) if response.status().is_success() => response,
- Ok(response) => {
- let status = response.status();
- let error = anyhow!("download {label} from {url}: HTTP status {status}");
- return Err(if is_retryable_download_status(status) {
- DownloadAttemptError::Retryable(error)
- } else {
- DownloadAttemptError::Fatal(error)
- });
- }
- Err(error) => {
- return Err(DownloadAttemptError::Retryable(
- anyhow::Error::new(error).context(format!("download {label} from {url}")),
- ));
- }
- };
-
- let mut temp_file = tempfile::NamedTempFile::new_in(parent)
- .map_err(|error| fatal_io(error, format!("create temp file in {}", parent.display())))?;
- let mut hasher = Sha256::new();
- let mut buffer = [0_u8; 1024 * 1024];
- loop {
- let read = match response.read(&mut buffer) {
- Ok(read) => read,
- Err(error) => {
- return Err(DownloadAttemptError::Retryable(
- anyhow::Error::new(error)
- .context(format!("read {label} response body from {url}")),
- ));
- }
- };
- if read == 0 {
- break;
- }
- hasher.update(&buffer[..read]);
- temp_file
- .write_all(&buffer[..read])
- .map_err(|error| fatal_io(error, format!("write cached {label}")))?;
- }
-
- let actual_sha256 = hex_digest(&hasher.finalize());
- if actual_sha256 != normalize_sha256(expected_sha256) {
- return Err(DownloadAttemptError::Retryable(anyhow!(
- "{label} SHA-256 mismatch after download: expected {}, got {actual_sha256}",
- normalize_sha256(expected_sha256)
- )));
- }
-
- temp_file
- .flush()
- .map_err(|error| fatal_io(error, format!("flush cached {label}")))?;
- temp_file.persist(target).map_err(|error| {
- DownloadAttemptError::Fatal(anyhow!(
- "persist cached {label} to {}: {}",
- target.display(),
- error.error
- ))
- })?;
+ .with_context(|| format!("download {label} from {url}"))?;
Ok(target.to_path_buf())
}
-fn fatal_io(error: std::io::Error, context: String) -> DownloadAttemptError {
- DownloadAttemptError::Fatal(anyhow::Error::new(error).context(context))
-}
-
-fn is_retryable_download_status(status: StatusCode) -> bool {
- status == StatusCode::TOO_MANY_REQUESTS
- || status == StatusCode::REQUEST_TIMEOUT
- || status.is_server_error()
-}
-
-fn download_retry_delay(attempt: usize) -> Duration {
- Duration::from_millis(DOWNLOAD_RETRY_BASE_DELAY_MS * 2_u64.pow((attempt - 1) as u32))
-}
-
fn cache_path_for(
cache_dir: &Path,
namespace: &str,
@@ -861,7 +714,8 @@ fn cache_path_for(
}
fn verify_file_sha256(path: &Path, expected_sha256: &str, label: &str) -> Result<()> {
- let actual_sha256 = sha256_file(path)?;
+ let actual_sha256 =
+ download::sha256_file(path).with_context(|| format!("hash {}", path.display()))?;
let expected_sha256 = normalize_sha256(expected_sha256);
if actual_sha256 != expected_sha256 {
bail!(
@@ -874,35 +728,10 @@ fn verify_file_sha256(path: &Path, expected_sha256: &str, label: &str) -> Result
Ok(())
}
-fn sha256_file(path: &Path) -> Result<String> {
- let mut file = fs::File::open(path).with_context(|| format!("open {}", path.display()))?;
- let mut hasher = Sha256::new();
- let mut buffer = [0_u8; 1024 * 1024];
- loop {
- let read = file
- .read(&mut buffer)
- .with_context(|| format!("read {}", path.display()))?;
- if read == 0 {
- break;
- }
- hasher.update(&buffer[..read]);
- }
- let digest = hasher.finalize();
- Ok(hex_digest(&digest))
-}
-
fn normalize_sha256(value: &str) -> String {
value.trim().to_ascii_lowercase()
}
-fn hex_digest(bytes: &[u8]) -> String {
- let mut hex = String::with_capacity(bytes.len() * 2);
- for byte in bytes {
- hex.push_str(&format!("{byte:02x}"));
- }
- hex
-}
-
fn expect_decode_error(file_id: &str, req: AnalyzeImageRequest, failures: &mut Vec<String>) {
match analyze_image(req) {
Err(MlError::Decode(_)) => {}