π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Verify asset download sizes
Neeraj Gupta7363141d4ae2
diff --git a/infra/ml/test/ml_indexing/assets.json b/infra/ml/test/ml_indexing/assets.jsonindex 956e12e4c9..1868d18e31 100644--- a/infra/ml/test/ml_indexing/assets.json+++ b/infra/ml/test/ml_indexing/assets.json@@ -5,11 +5,13 @@"manifest": {"path": "manifest.json","url": "https://raw.githubusercontent.com/ente/test-fixtures/13c7cf83717140be20b32e6fbb178d5f15ea09ea/ml/indexing/v1/manifest.json",+ "size": 8662,"sha256": "fa4a2dcd1787825a77b85343aace954572e0f4d9080e4cb3031179159af9011c"},"python_golden": {"path": "goldens/python/results.json","url": "https://raw.githubusercontent.com/ente/test-fixtures/13c7cf83717140be20b32e6fbb178d5f15ea09ea/ml/indexing/v1/goldens/python/results.json",+ "size": 440894,"sha256": "2def6506e098234abc2e6e10febd820b8e3e66d6ad4c9525cb4f1cec3418f9b6"},"onnx_runtime": {@@ -17,18 +19,21 @@"archives": {"aarch64-apple-darwin": {"url": "https://github.com/ente/ort-packaging/releases/download/ort-1.28.1-r1/onnxruntime-coreml-macos-arm64-1.28.1-r1.tar.gz",+ "size": 7997883,"sha256": "12890a871d0e658dd064be187fa63d27c6b692e69f94d4750532fd8f4d179c1e","library_path": "./libonnxruntime.1.28.1.dylib","library_sha256": "11a4bb659860e56448aae6dfac2cd31eaddf2218588d435eb26d6f79a07b91bc"},"x86_64-unknown-linux-gnu": {"url": "https://github.com/ente/ort-packaging/releases/download/ort-1.28.1-r1/onnxruntime-webgpu-linux-x64-1.28.1-r1.tar.gz",+ "size": 12484422,"sha256": "94e55336214e891e9f648901f0c815f113d1534cecae82ecef9c3828cd73d4f4","library_path": "./libonnxruntime.so.1.28.1","library_sha256": "1750d82f5b8c3c4924fd0301de7ac92747e33d852f4748e1e9da97406f257bed"},"aarch64-unknown-linux-gnu": {"url": "https://github.com/ente/ort-packaging/releases/download/ort-1.28.1-r1/onnxruntime-webgpu-linux-arm64-1.28.1-r1.tar.gz",+ "size": 12985661,"sha256": "2eb71ced50e16b4e2d4ab20b924276d50d123417962a7d874a0cdb7f0171c63d","library_path": "./libonnxruntime.so.1.28.1","library_sha256": "5d801e125ca434b5b699e900ee841d44bf5dbb5a660645265f2fcf71d7dd7bec"@@ -39,26 +44,31 @@"face_detection": {"file_name": "yolov5s_face_640_640_static_b1.onnx","url": "https://models.ente.com/yolov5s_face_640_640_static_b1.onnx",+ "size": 32355091,"sha256": "e047647409403d52696035ecd445792173e50d7fbdcccac97b958a585db9aa3d"},"face_embedding": {"file_name": "mobilefacenet_portable_static_b1.onnx","url": "https://models.ente.com/mobilefacenet_portable_static_b1.onnx",+ "size": 5278803,"sha256": "0763fc33f54e138476194da95987e133b3e976075a6b1d3e1b2caedb251b1a36"},"clip_image": {"file_name": "mobileclip_s2_image_gelu_opset20.onnx","url": "https://models.ente.com/mobileclip_s2_image_gelu_opset20.onnx",+ "size": 143057352,"sha256": "205a430af825e501c5138e5bb9abea942482a7a4fd4a680e98e47cf0830dce7e"},"clip_text": {"file_name": "mobileclip_s2_text_opset18_quant.onnx","url": "https://models.ente.com/mobileclip_s2_text_opset18_quant.onnx",+ "size": 67144712,"sha256": "d92f33dfcff83077fc2e0d3414250710efbb51795dfd89767bdbefb5fdc47322"},"clip_text_vocab": {"file_name": "bpe_simple_vocab_16e6.txt","url": "https://models.ente.com/bpe_simple_vocab_16e6.txt",+ "size": 3194984,"sha256": "67603cfda2e032ad77b5f8808af37789d590db664b26df8705d2bf8b3c553fc8"}},diff --git a/rust/crates/assets/src/download.rs b/rust/crates/assets/src/download.rsindex 26a5d336b0..d50ce1d76c 100644--- a/rust/crates/assets/src/download.rs+++ b/rust/crates/assets/src/download.rs@@ -68,6 +68,7 @@ impl From<std::io::Error> for Error {pub struct DownloadTarget {pub label: String,pub url: String,+ pub expected_size: u64,pub sha256: String,pub destination: PathBuf,}@@ -237,10 +238,9 @@ impl Downloader {continue;}let probe = fetch_download_probe(&self.client, &target.url).await;- let total = probe.content_length?;- let downloaded =- existing_download_bytes(target, &target.destination, &probe, false).min(total);- remaining = remaining.saturating_add(total - downloaded);+ let downloaded = existing_download_bytes(target, &target.destination, &probe, false)+ .min(target.expected_size);+ remaining = remaining.checked_add(target.expected_size - downloaded)?;}Some(remaining)}@@ -281,7 +281,7 @@ async fn fetch_async(cached.push(is_cached);if is_cached {download_probes.push(DownloadProbe {- content_length: file_size(destination),+ content_length: Some(target.expected_size),supports_ranges: false,response_metadata: None,});@@ -290,30 +290,22 @@ async fn fetch_async(}}- let total_bytes = if download_probes+ let total_bytes = targets.iter()- .all(|probe| probe.content_length.is_some())- {- let total = download_probes- .iter()- .filter_map(|probe| probe.content_length)- .sum::<u64>();- (total > 0).then_some(total)- } else {- None- };+ .try_fold(0u64, |total, target| {+ total.checked_add(target.expected_size)+ })+ .filter(|total| *total > 0);let mut file_states = targets.iter()- .zip(&download_probes).zip(&cached)- .map(|((target, probe), cached)| {+ .zip(&download_probes)+ .map(|((target, cached), probe)| {let existing = existing_download_bytes(target, &target.destination, probe, *cached);FileProgress {- downloaded_bytes: probe- .content_length- .map_or(existing, |value| existing.min(value)),- total_bytes: probe.content_length,+ downloaded_bytes: existing.min(target.expected_size),+ total_bytes: Some(target.expected_size),network_downloaded_bytes: 0,elapsed: Duration::ZERO,retry_count: 0,@@ -339,7 +331,7 @@ async fn fetch_async(.get(index).cloned().unwrap_or_else(DownloadProbe::default);- let expected_file_total = download_probe.content_length;+ let expected_file_total = target.expected_size;let target_label = target.label.clone();let file_report = download_file(client,@@ -374,7 +366,7 @@ async fn fetch_async(if let Some(state) = file_states.get_mut(index) {state.downloaded_bytes = file_report.final_size;- state.total_bytes = expected_file_total.or(Some(file_report.final_size));+ state.total_bytes = Some(expected_file_total);state.network_downloaded_bytes = file_report.network_downloaded_bytes;state.elapsed = file_report.elapsed;state.retry_count = file_report.retry_count;@@ -426,46 +418,36 @@ async fn download_file(.parent().ok_or_else(|| Error::InvalidTarget(destination.display().to_string()))?;fs::create_dir_all(parent)?;+ validate_reported_size(download_probe.content_length, target.expected_size)?;let mut range_error = None;- if let Some(total) = download_probe.content_length {- if should_use_range_download(total, download_probe) {- match download_file_ranged(- client,- target,- destination,- total,- download_probe.response_metadata.clone(),- &mut on_progress,- is_cancelled,- )- .await- {- Ok(report) => return Ok(report),- Err(err @ (Error::Cancelled | Error::Validation(_) | Error::StorageFull)) => {- return Err(err);- }- Err(err) => {- range_error = Some(err);- cleanup_range_download(destination);- }+ if should_use_range_download(target.expected_size, download_probe) {+ match download_file_ranged(+ client,+ target,+ destination,+ target.expected_size,+ download_probe.response_metadata.clone(),+ &mut on_progress,+ is_cancelled,+ )+ .await+ {+ Ok(report) => return Ok(report),+ Err(err @ (Error::Cancelled | Error::Validation(_) | Error::StorageFull)) => {+ return Err(err);+ }+ Err(err) => {+ range_error = Some(err);+ cleanup_range_download(destination);}- } else if range_metadata_path_for(destination).exists() {- cleanup_range_download(destination);}} else if range_metadata_path_for(destination).exists() {cleanup_range_download(destination);}- let single_result = download_file_single(- client,- target,- destination,- download_probe.content_length,- &mut on_progress,- is_cancelled,- )- .await;+ let single_result =+ download_file_single(client, target, destination, &mut on_progress, is_cancelled).await;match (single_result, range_error) {(Ok(report), _) => Ok(report),@@ -481,10 +463,10 @@ async fn download_file_single(client: &Client,target: &DownloadTarget,destination: &Path,- expected_file_total: Option<u64>,on_progress: &mut (dyn FnMut(FileProgress) + Send),is_cancelled: &(impl Fn() -> bool + Sync),) -> Result<FileDownloadReport, Error> {+ let expected_size = target.expected_size;let tmp_path = tmp_path_for(destination);let partial_metadata_path = partial_metadata_path_for(destination);let file_started_at = Instant::now();@@ -496,7 +478,11 @@ async fn download_file_single(return Err(Error::Cancelled);}- let tmp_size = file_size(&tmp_path).unwrap_or(0);+ let mut tmp_size = file_size(&tmp_path).unwrap_or(0);+ if tmp_size > expected_size {+ cleanup_range_download(destination);+ tmp_size = 0;+ }let resume_validator = if tmp_size > 0 {partial_download_validator(destination, target)} else {@@ -558,7 +544,12 @@ async fn download_file_single(}let resume_from = if append { resume_from } else { 0 };let response_metadata = response_metadata(&response);- let file_total = content_total(&response, resume_from).or(expected_file_total);+ if let Err(error) =+ validate_reported_size(content_total(&response, resume_from), expected_size)+ {+ cleanup_range_download(destination);+ return Err(error);+ }if !append {if if_range_header_value(Some(&response_metadata)).is_some() {@@ -581,7 +572,7 @@ async fn download_file_single(on_progress(FileProgress {downloaded_bytes: downloaded,- total_bytes: file_total,+ total_bytes: Some(expected_size),network_downloaded_bytes,elapsed: file_started_at.elapsed(),retry_count,@@ -609,14 +600,23 @@ async fn download_file_single(break;};+ let chunk_len = chunk.len() as u64;+ if chunk_len > expected_size - downloaded {+ drop(file);+ cleanup_range_download(destination);+ return Err(Error::SizeMismatch {+ expected: expected_size,+ actual: downloaded.saturating_add(chunk_len),+ });+ }file.write_all(&chunk)?;- downloaded = downloaded.saturating_add(chunk.len() as u64);- network_downloaded_bytes = network_downloaded_bytes.saturating_add(chunk.len() as u64);+ downloaded += chunk_len;+ network_downloaded_bytes = network_downloaded_bytes.saturating_add(chunk_len);if last_progress.elapsed() >= PROGRESS_INTERVAL {on_progress(FileProgress {downloaded_bytes: downloaded,- total_bytes: file_total,+ total_bytes: Some(expected_size),network_downloaded_bytes,elapsed: file_started_at.elapsed(),retry_count,@@ -635,22 +635,16 @@ async fn download_file_single(on_progress(FileProgress {downloaded_bytes: downloaded,- total_bytes: file_total,+ total_bytes: Some(expected_size),network_downloaded_bytes,elapsed: file_started_at.elapsed(),retry_count,});- if let Some(total) = file_total- && downloaded != total- {- if downloaded > total {- let _ = fs::remove_file(&tmp_path);- let _ = fs::remove_file(&partial_metadata_path);- }+ if downloaded != expected_size {if attempt == MAX_ATTEMPTS {return Err(Error::SizeMismatch {- expected: total,+ expected: expected_size,actual: downloaded,});}@@ -883,14 +877,14 @@ impl RangeDownloadContext<'_, '_> {};let chunk_len = chunk.len() as u64;- if downloaded_in_range.saturating_add(chunk_len) > range_len {+ if chunk_len > range_len - downloaded_in_range {return Err(Error::Protocol("received more bytes than requested".to_string(),));}write_all_at(file, chunk.as_ref(), range.start + downloaded_in_range)?;- downloaded_in_range = downloaded_in_range.saturating_add(chunk_len);+ downloaded_in_range += chunk_len;{let mut states = self.range_states.lock().expect("range state lock");@@ -1056,6 +1050,15 @@ fn should_use_range_download(total: u64, probe: &DownloadProbe) -> bool {total >= MIN_RANGE_DOWNLOAD_BYTES && probe.supports_ranges}+fn validate_reported_size(reported: Option<u64>, expected: u64) -> Result<(), Error> {+ if let Some(actual) = reported+ && actual != expected+ {+ return Err(Error::SizeMismatch { expected, actual });+ }+ Ok(())+}+fn prepare_range_download_metadata(target: &DownloadTarget,destination: &Path,@@ -1334,7 +1337,7 @@ fn bytes_per_second(bytes: u64, elapsed: Duration) -> f64 {}async fn prepare_cached_download(target: &DownloadTarget, destination: &Path) -> bool {- if !destination.exists()+ if file_size(destination) != Some(target.expected_size)|| check_sha256(destination, &target.sha256, &target.label).await.is_err()@@ -1474,7 +1477,11 @@ fn existing_download_bytes(}if range_metadata_path_for(destination).exists() {- if let Some(total) = probe.content_length {+ let total = target.expected_size;+ if probe+ .content_length+ .is_none_or(|reported| reported == total)+ {let ranges = range_download_parts(total, RANGE_DOWNLOAD_CONCURRENCY);if let Some(metadata) = read_range_download_metadata(destination)&& file_size(&tmp_path_for(destination)) == Some(total)@@ -1499,6 +1506,7 @@ fn existing_download_bytes(partial_download_validator(destination, target).and_then(|_| file_size(&tmp_path_for(destination)))+ .filter(|size| *size <= target.expected_size).unwrap_or(0)}@@ -1708,6 +1716,7 @@ mod tests {vec![DownloadTarget {label: "Model".to_string(),url,+ expected_size: bytes.len() as u64,destination: destination.clone(),sha256: sha_hex(&bytes),}],@@ -1758,6 +1767,7 @@ mod tests {vec![DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),+ expected_size: bytes.len() as u64,destination: destination.clone(),sha256: sha_hex(&bytes),}],@@ -1776,6 +1786,135 @@ mod tests {let _ = fs::remove_dir_all(test_dir);}+ #[tokio::test]+ async fn reported_size_mismatch_is_rejected_before_preallocation() {+ let expected_size = MIN_RANGE_DOWNLOAD_BYTES;+ let reported_size = expected_size * 2;+ let get_count = Arc::new(AtomicUsize::new(0));+ let server = {+ let get_count = Arc::clone(&get_count);+ TestServer::spawn(move |mut stream| {+ let Some(request) = read_test_request(&stream) else {+ return;+ };+ if request.line.starts_with("HEAD ") {+ let response = format!(+ "HTTP/1.1 200 OK\r\nContent-Length: {reported_size}\r\nAccept-Ranges: bytes\r\nConnection: close\r\n\r\n"+ );+ let _ = stream.write_all(response.as_bytes());+ } else {+ get_count.fetch_add(1, Ordering::SeqCst);+ }+ })+ };++ let test_dir = scratch_dir("reported-size-mismatch");+ let destination = test_dir.join("model.bin");+ let result = download(+ vec![DownloadTarget {+ label: "Model".to_string(),+ url: server.url("/model.bin"),+ expected_size,+ destination: destination.clone(),+ sha256: "0".repeat(64),+ }],+ |_| {},+ false,+ )+ .await;++ assert!(matches!(+ result,+ Err(Error::Target { source, .. })+ if matches!(*source, Error::SizeMismatch { expected, actual }+ if expected == expected_size && actual == reported_size)+ ));+ assert_eq!(get_count.load(Ordering::SeqCst), 0);+ assert!(!tmp_path_for(&destination).exists());++ let _ = fs::remove_dir_all(test_dir);+ }++ #[tokio::test]+ async fn headerless_stream_is_bounded_by_expected_size() {+ let expected = Arc::new(sample_bytes(512));+ let served = Arc::new(sample_bytes(513));+ let expected_size = expected.len() as u64;+ let served_size = served.len() as u64;+ let get_count = Arc::new(AtomicUsize::new(0));+ let server = {+ let served = Arc::clone(&served);+ let get_count = Arc::clone(&get_count);+ TestServer::spawn(move |stream| {+ handle_headerless_test_request(stream, Arc::clone(&served), Arc::clone(&get_count));+ })+ };++ let test_dir = scratch_dir("bounded-headerless-download");+ let destination = test_dir.join("model.bin");+ let result = download(+ vec![DownloadTarget {+ label: "Model".to_string(),+ url: server.url("/model.bin"),+ expected_size,+ destination: destination.clone(),+ sha256: sha_hex(&expected),+ }],+ |_| {},+ false,+ )+ .await;++ assert!(matches!(+ result,+ Err(Error::Target { source, .. })+ if matches!(*source, Error::SizeMismatch { expected, actual }+ if expected == expected_size && actual == served_size)+ ));+ assert_eq!(get_count.load(Ordering::SeqCst), 1);+ assert!(!destination.exists());+ assert!(!tmp_path_for(&destination).exists());++ let _ = fs::remove_dir_all(test_dir);+ }++ #[tokio::test]+ async fn headerless_stream_with_expected_size_succeeds() {+ let bytes = Arc::new(sample_bytes(512));+ let get_count = Arc::new(AtomicUsize::new(0));+ let server = {+ let bytes = Arc::clone(&bytes);+ let get_count = Arc::clone(&get_count);+ TestServer::spawn(move |stream| {+ handle_headerless_test_request(stream, Arc::clone(&bytes), Arc::clone(&get_count));+ })+ };++ let test_dir = scratch_dir("expected-headerless-download");+ let destination = test_dir.join("model.bin");+ download(+ vec![DownloadTarget {+ label: "Model".to_string(),+ url: server.url("/model.bin"),+ expected_size: bytes.len() as u64,+ destination: destination.clone(),+ sha256: sha_hex(&bytes),+ }],+ |_| {},+ false,+ )+ .await+ .expect("headerless download succeeds");++ assert_eq!(+ fs::read(&destination).expect("read downloaded file"),+ *bytes+ );+ assert_eq!(get_count.load(Ordering::SeqCst), 1);++ let _ = fs::remove_dir_all(test_dir);+ }+#[tokio::test]async fn downloads_files_sequentially() {let bytes = Arc::new(sample_bytes(512));@@ -1803,6 +1942,7 @@ mod tests {let targets = ["model", "projector"].map(|label| DownloadTarget {label: label.to_string(),url: server.url(&format!("/{label}.bin")),+ expected_size: bytes.len() as u64,destination: test_dir.join(format!("{label}.bin")),sha256: sha_hex(&bytes),});@@ -1843,6 +1983,7 @@ mod tests {let target = DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),+ expected_size: bytes.len() as u64,destination: destination.clone(),sha256: sha_hex(&bytes),};@@ -1885,6 +2026,7 @@ mod tests {let target = DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),+ expected_size: bytes.len() as u64,destination: destination.clone(),sha256: sha_hex(&bytes),};@@ -1931,6 +2073,7 @@ mod tests {let target = DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),+ expected_size: bytes.len() as u64,destination: destination.clone(),sha256: sha_hex(&bytes),};@@ -1977,6 +2120,7 @@ mod tests {let target = DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),+ expected_size: bytes.len() as u64,destination: destination.clone(),sha256: sha_hex(&bytes),};@@ -2039,6 +2183,7 @@ mod tests {let target = DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),+ expected_size: bytes.len() as u64,destination: destination.clone(),sha256: sha_hex(&bytes),};@@ -2091,6 +2236,7 @@ mod tests {let target = DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),+ expected_size: bytes.len() as u64,destination: destination.clone(),sha256: sha_hex(&bytes),};@@ -2137,6 +2283,7 @@ mod tests {let target = DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),+ expected_size: bytes.len() as u64,destination: destination.clone(),sha256: sha_hex(&bytes),};@@ -2232,6 +2379,7 @@ mod tests {vec![DownloadTarget {label: "Model".to_string(),url: "http://127.0.0.1:1/model.bin".to_string(),+ expected_size: 1,destination: destination.clone(),sha256: "0".repeat(64),}],@@ -2262,6 +2410,7 @@ mod tests {let target = DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),+ expected_size: bytes.len() as u64,destination: destination.clone(),sha256: sha_hex(&bytes),};@@ -2296,6 +2445,7 @@ mod tests {vec![DownloadTarget {label: "Model".to_string(),url: "http://127.0.0.1:1/model.bin".to_string(),+ expected_size: b"complete-model".len() as u64,destination: destination.clone(),sha256: sha_hex(b"complete-model"),}],@@ -2336,6 +2486,7 @@ mod tests {vec![DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),+ expected_size: bytes.len() as u64,destination: destination.clone(),sha256: expected,}],@@ -2371,6 +2522,7 @@ mod tests {vec![DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),+ expected_size: bytes.len() as u64,destination: destination.clone(),sha256: "0".repeat(64),}],@@ -2415,6 +2567,7 @@ mod tests {vec![DownloadTarget {label: "Model".to_string(),url: server.url("/model.bin"),+ expected_size: bytes.len() as u64,destination: destination.clone(),sha256: "0".repeat(64),}],@@ -2444,6 +2597,7 @@ mod tests {let old_target = DownloadTarget {label: "Model".to_string(),url: "http://cache.example.org/old-location".to_string(),+ expected_size: b"model-bytes".len() as u64,destination: destination.clone(),sha256: sha.clone(),};@@ -2568,6 +2722,22 @@ mod tests {let _ = stream.write_all(bytes.as_slice());}+ fn handle_headerless_test_request(+ mut stream: TcpStream,+ bytes: Arc<Vec<u8>>,+ get_count: Arc<AtomicUsize>,+ ) {+ let Some(request) = read_test_request(&stream) else {+ return;+ };+ let _ = stream.write_all(b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n");+ if request.line.starts_with("HEAD ") {+ return;+ }+ get_count.fetch_add(1, Ordering::SeqCst);+ let _ = stream.write_all(bytes.as_slice());+ }+fn handle_stalled_test_request(mut stream: TcpStream,bytes: Arc<Vec<u8>>,diff --git a/rust/crates/assets/src/store.rs b/rust/crates/assets/src/store.rsindex 4b7fadf454..18e5878e9a 100644--- a/rust/crates/assets/src/store.rs+++ b/rust/crates/assets/src/store.rs@@ -23,13 +23,18 @@ pub struct Asset {#[derive(Debug, Clone, PartialEq, Eq)]enum AssetKind {Files(Vec<AssetFile>),- TarGz { url: String, sha256: String },+ TarGz {+ url: String,+ size: u64,+ sha256: String,+ },}#[derive(Debug, Clone, PartialEq, Eq)]pub struct AssetFile {pub name: String,pub url: String,+ pub size: u64,pub sha256: String,}@@ -77,12 +82,17 @@ impl Asset {})}- pub fn tar_gz(components: Vec<String>, url: String, sha256: String) -> Result<Self, Error> {+ pub fn tar_gz(+ components: Vec<String>,+ url: String,+ size: u64,+ sha256: String,+ ) -> Result<Self, Error> {validate_components(&components)?;download::validate_sha256(&sha256)?;Ok(Self {components,- kind: AssetKind::TarGz { url, sha256 },+ kind: AssetKind::TarGz { url, size, sha256 },})}}@@ -286,13 +296,15 @@ impl AssetStore {.map(|file| DownloadTarget {label: file.name.clone(),url: file.url.clone(),+ expected_size: file.size,sha256: file.sha256.clone(),destination: staging.join(&file.name),}).collect(),- AssetKind::TarGz { url, sha256 } => vec![DownloadTarget {+ AssetKind::TarGz { url, size, sha256 } => vec![DownloadTarget {label: asset.components.last().cloned().unwrap_or_default(),url: url.clone(),+ expected_size: *size,sha256: sha256.clone(),destination: staging.join(ARCHIVE_FILE),}],@@ -763,6 +775,7 @@ mod tests {let asset = Asset::tar_gz(key(&["models", "parakeet"]),"https://example.invalid/parakeet.tar.gz".to_string(),+ archive.len() as u64,sha(&archive),).unwrap();@@ -797,6 +810,7 @@ mod tests {let asset = Asset::tar_gz(key(&["models", "linked"]),"https://example.invalid/linked.tar.gz".to_string(),+ archive.len() as u64,sha(&archive),).unwrap();@@ -855,6 +869,7 @@ mod tests {AssetFile {name: name.to_string(),url: format!("https://example.invalid/{name}"),+ size: bytes.len() as u64,sha256: sha(bytes),}}diff --git a/rust/crates/ensu/src/config.rs b/rust/crates/ensu/src/config.rsindex f499d8a3c2..639bcf4684 100644--- a/rust/crates/ensu/src/config.rs+++ b/rust/crates/ensu/src/config.rs@@ -14,8 +14,10 @@ pub struct ModelPreset {pub id: String,pub title: String,pub url: String,+ pub size: u64,pub sha256: String,pub mmproj_url: Option<String>,+ pub mmproj_size: Option<u64>,pub mmproj_sha256: Option<String>,}@@ -62,11 +64,13 @@ fn lfm_vl_1_6b() -> ModelPreset {id: "lfm-vl-1.6b".to_string(),title: "LFM 2.5 VL 1.6B (Q4_0)".to_string(),url: "https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B-GGUF/resolve/main/LFM2.5-VL-1.6B-Q4_0.gguf?download=true".to_string(),+ size: 695_752_480,sha256: "8186364a4e7c3ad30f6dd3d3b7a4e0074c77dd91eed6cad5d8be9090ce285804".to_string(),mmproj_url: Some("https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B-GGUF/resolve/main/mmproj-LFM2.5-VL-1.6b-Q8_0.gguf".to_string(),),+ mmproj_size: Some(583_109_888),mmproj_sha256: Some("2ce89e610c56f3198ece2b86cf61743a08b9307279c89125eb2412ebb908689d".to_string()),}}@@ -76,11 +80,13 @@ fn qwen_0_8b() -> ModelPreset {id: "qwen-0.8b".to_string(),title: "Qwen 3.5 0.8B (Q4_K_M)".to_string(),url: "https://huggingface.co/unsloth/Qwen3.5-0.8B-GGUF/resolve/main/Qwen3.5-0.8B-Q4_K_M.gguf?download=true".to_string(),+ size: 532_517_120,sha256: "bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517".to_string(),mmproj_url: Some("https://huggingface.co/unsloth/Qwen3.5-0.8B-GGUF/resolve/main/mmproj-F16.gguf".to_string(),),+ mmproj_size: Some(204_987_232),mmproj_sha256: Some("56e4c6cfe73b0c82e3e82bc518d7591997e61d81f723fc41a586f4fa69ea2453".to_string()),}}@@ -90,11 +96,13 @@ fn qwen_2b_q8() -> ModelPreset {id: "qwen-2b-q8".to_string(),title: "Qwen 3.5 2B (Q8_0)".to_string(),url: "https://huggingface.co/unsloth/Qwen3.5-2B-GGUF/resolve/main/Qwen3.5-2B-Q8_0.gguf?download=true".to_string(),+ size: 2_012_012_800,sha256: "1b04acba824817554f4ce23639bc8495ff70453b8fcb047900c731521021f2c1".to_string(),mmproj_url: Some("https://huggingface.co/unsloth/Qwen3.5-2B-GGUF/resolve/main/mmproj-F16.gguf".to_string(),),+ mmproj_size: Some(668_227_264),mmproj_sha256: Some("7035e9cb8d7c6a9681d07eef9a364783e86ea4cd73faab2eabb4f43a101830c7".to_string()),}}@@ -104,11 +112,13 @@ fn qwen_4b_q4km() -> ModelPreset {id: "qwen-4b-q4km".to_string(),title: "Qwen 3.5 4B (Q4_K_M)".to_string(),url: "https://huggingface.co/unsloth/Qwen3.5-4B-GGUF/resolve/main/Qwen3.5-4B-Q4_K_M.gguf?download=true".to_string(),+ size: 2_740_937_888,sha256: "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4".to_string(),mmproj_url: Some("https://huggingface.co/unsloth/Qwen3.5-4B-GGUF/resolve/main/mmproj-F16.gguf".to_string(),),+ mmproj_size: Some(672_423_616),mmproj_sha256: Some("cd88edcf8d031894960bb0c9c5b9b7e1fea6ebee02b9f7ce925a00d12891f864".to_string()),}}@@ -118,11 +128,13 @@ fn gemma_4_e4b_q4km() -> ModelPreset {id: "gemma-4-e4b-q4km".to_string(),title: "Gemma 4 E4B (Q4_K_M)".to_string(),url: "https://huggingface.co/ente-ai/gemma-4-E4B-it-GGUF/resolve/f0089e04ac8494e513619d18b44c829c6b815440/gemma-4-E4B-it-Q4_K_M.gguf?download=true".to_string(),+ size: 4_977_171_584,sha256: "85a896a047553e842f25297ee5b031d64ff30147d9c4af17b1e4b394cd1fab87".to_string(),mmproj_url: Some("https://huggingface.co/ente-ai/gemma-4-E4B-it-GGUF/resolve/f0089e04ac8494e513619d18b44c829c6b815440/mmproj-F16.gguf".to_string(),),+ mmproj_size: Some(990_372_672),mmproj_sha256: Some("ddf46c21d7078e95338cfc22306b19b276a29a5ad089023449dd54d4b6170a51".to_string()),}}@@ -132,11 +144,13 @@ fn gemma_4_e2b_q4km() -> ModelPreset {id: "gemma-4-e2b-q4km".to_string(),title: "Gemma 4 E2B (Q4_K_M)".to_string(),url: "https://huggingface.co/ente-ai/gemma-4-E2B-it-GGUF/resolve/d9f70b02c9a2193b7263daee865dfa93276fd99a/gemma-4-E2B-it-Q4_K_M.gguf?download=true".to_string(),+ size: 3_106_738_272,sha256: "740185b21d22ceb83a11c3aa62ad5842ef32c70f6096d756bbee85a1e4ec34b8".to_string(),mmproj_url: Some("https://huggingface.co/ente-ai/gemma-4-E2B-it-GGUF/resolve/d9f70b02c9a2193b7263daee865dfa93276fd99a/mmproj-F16.gguf".to_string(),),+ mmproj_size: Some(985_654_080),mmproj_sha256: Some("140be8d7849741f88c50757d529b84373ee8e27052cc2236855b537f4a8215fa".to_string()),}}@@ -146,8 +160,10 @@ fn parakeet_v3_int8() -> ModelPreset {id: "parakeet-v3-int8".to_string(),title: "Transcription model".to_string(),url: "https://models.ente.com/parakeet-v3-int8.tar.gz".to_string(),+ size: 478_517_071,sha256: "43d37191602727524a7d8c6da0eef11c4ba24320f5b4730f1a2497befc2efa77".to_string(),mmproj_url: None,+ mmproj_size: None,mmproj_sha256: None,}}@@ -157,8 +173,10 @@ fn silero_vad_v4() -> ModelPreset {id: "silero-vad-v4".to_string(),title: "Voice activity model".to_string(),url: "https://models.ente.com/silero_vad_v4.onnx".to_string(),+ size: 1_807_522,sha256: "a35ebf52fd3ce5f1469b2a36158dba761bc47b973ea3382b3186ca15b1f5af28".to_string(),mmproj_url: None,+ mmproj_size: None,mmproj_sha256: None,}}@@ -295,19 +313,20 @@ mod tests {}#[test]- fn catalog_presets_pair_mmproj_url_with_checksum() {+ fn catalog_presets_pair_mmproj_metadata() {for preset in llm_catalog() {let has_url = preset.mmproj_url.as_deref().is_some_and(|u| !u.trim().is_empty());+ let has_size = preset.mmproj_size.is_some();let has_sha = preset.mmproj_sha256.as_deref().is_some_and(|s| !s.trim().is_empty());- assert_eq!(- has_url, has_sha,- "preset {} must pair mmproj URL with its checksum",+ assert!(+ has_url == has_size && has_size == has_sha,+ "preset {} must pair mmproj URL, size, and checksum",preset.id);}diff --git a/rust/crates/ensu/src/config/knowledge.rs b/rust/crates/ensu/src/config/knowledge.rsindex 9435446002..b4fda45325 100644--- a/rust/crates/ensu/src/config/knowledge.rs+++ b/rust/crates/ensu/src/config/knowledge.rs@@ -30,6 +30,7 @@ const MODIFICATION_NOTICE: &str = "Adapted by Ente";#[derive(Debug, Clone, PartialEq)]pub struct KnowledgeEmbeddingConfig {pub model_url: String,+ pub model_size: u64,pub model_sha256: String,pub target_id: String,pub source_dim: u32,@@ -57,6 +58,7 @@ pub struct KnowledgeDatasetConfig {pub label: String,pub current_download_identity: String,pub artifact_base_url: String,+ pub artifact_sizes: Vec<u64>,pub artifact_sha256: Vec<String>,pub download_size_bytes: i64,pub max_chars: u32,@@ -97,6 +99,7 @@ pub(crate) fn knowledge_index_contract() -> KnowledgeIndexContract {pub(crate) fn knowledge_embedding_config() -> KnowledgeEmbeddingConfig {KnowledgeEmbeddingConfig {model_url: EMBEDDING_MODEL_URL.to_owned(),+ model_size: 333_590_944,model_sha256: EMBEDDING_MODEL_SHA256.to_owned(),target_id: EMBEDDING_TARGET_ID.to_owned(),source_dim: SOURCE_DIM,@@ -117,6 +120,7 @@ pub(crate) fn knowledge_datasets() -> Vec<KnowledgeDatasetConfig> {label: "Simple English Wikipedia".to_owned(),current_download_identity: "simplewiki-2026-07-02".to_owned(),artifact_base_url: "https://huggingface.co/datasets/ente-ai/ensu-knowledge-packs/resolve/a13b90e443dcdc1561ac777ea17ee6ed4703e35f/simplewiki/data/".to_owned(),+ artifact_sizes: vec![554, 134_535_168, 33_297_292, 16_432],artifact_sha256: vec!["118dfc77aef7c7186c8b15551ec94e8b8878fde0b3ca20c97cc8222e5b70b309".to_owned(),"ed90f60e757784fd4752652139299be792948628be5d7455847333f362d30cd5".to_owned(),@@ -140,6 +144,7 @@ pub(crate) fn knowledge_datasets() -> Vec<KnowledgeDatasetConfig> {label: "Wikibooks".to_owned(),current_download_identity: "enwikibooks-2026-07-02".to_owned(),artifact_base_url: "https://huggingface.co/datasets/ente-ai/ensu-knowledge-packs/resolve/a13b90e443dcdc1561ac777ea17ee6ed4703e35f/wikibooks/data/".to_owned(),+ artifact_sizes: vec![556, 128_000_000, 74_579_279, 15_640],artifact_sha256: vec!["2076386cdc215ca4950aa92c4b6bcac605a63e76fda4989677eb5f9a637bba14".to_owned(),"bcf4f5d15b78470def1908aaddcb2abae17fbe7752d60c90d11fd4d951b0b591".to_owned(),@@ -163,6 +168,7 @@ pub(crate) fn knowledge_datasets() -> Vec<KnowledgeDatasetConfig> {label: "English Wikipedia".to_owned(),current_download_identity: "enwiki-2026-07-07".to_owned(),artifact_base_url: "https://huggingface.co/datasets/ente-ai/ensu-knowledge-packs/resolve/61f12c75fe451efe6c0d06af3903add4292e06af/fullwiki/data/".to_owned(),+ artifact_sizes: vec![608, 269_391_360, 93_626_839, 32_896],artifact_sha256: vec!["0c000cf313b642c9a25695c645c9cef2062e3d4c9e33d0cb05f19bf3145c8084".to_owned(),"36fcb1045e20aac98f8477cbd4f15b79ffb337ced756a457d55180fa862c5453".to_owned(),@@ -248,11 +254,16 @@ mod tests {assert_eq!(datasets[0].download_size_bytes, 167_849_446);assert_eq!(datasets[1].download_size_bytes, 202_595_475);assert_eq!(datasets[2].download_size_bytes, 363_051_703);+ assert!(datasets.iter().all(|dataset| {+ dataset.artifact_sizes.iter().sum::<u64>()+ == u64::try_from(dataset.download_size_bytes).unwrap()+ }));assert_eq!(datasets[0].relevance_threshold, 0.55);assert_eq!(datasets[1].relevance_threshold, 0.58);assert_eq!(datasets[2].relevance_threshold, 0.55);assert!(datasets.iter().all(|dataset| {- dataset.artifact_sha256.len() == KNOWLEDGE_ARTIFACT_FILENAMES.len()+ dataset.artifact_sizes.len() == KNOWLEDGE_ARTIFACT_FILENAMES.len()+ && dataset.artifact_sha256.len() == KNOWLEDGE_ARTIFACT_FILENAMES.len()&& dataset.artifact_sha256.iter()diff --git a/rust/crates/ensu/src/model/migrations.rs b/rust/crates/ensu/src/model/migrations.rsindex a3cf18af35..d5d7bfb30d 100644--- a/rust/crates/ensu/src/model/migrations.rs+++ b/rust/crates/ensu/src/model/migrations.rs@@ -258,8 +258,10 @@ mod tests {id: id.to_string(),title: String::new(),url: model_url.to_string(),+ size: 1,sha256: "0".repeat(64),mmproj_url: Some(mmproj_url.to_string()),+ mmproj_size: Some(1),mmproj_sha256: Some("0".repeat(64)),};LlmMigration {diff --git a/rust/crates/ensu/src/model/mod.rs b/rust/crates/ensu/src/model/mod.rsindex 27b80eb8e5..0f56a21052 100644--- a/rust/crates/ensu/src/model/mod.rs+++ b/rust/crates/ensu/src/model/mod.rs@@ -27,22 +27,25 @@ pub(crate) fn llm_asset(preset: &ModelPreset) -> Result<Asset, InvalidPreset> {let mut files = vec![AssetFile {name: LLM_MODEL_FILE.to_string(),url: preset.url.clone(),+ size: preset.size,sha256: preset.sha256.clone(),}];match (trimmed(preset.mmproj_url.as_deref()),+ preset.mmproj_size,trimmed(preset.mmproj_sha256.as_deref()),) {- (Some(url), Some(sha256)) => files.push(AssetFile {+ (Some(url), Some(size), Some(sha256)) => files.push(AssetFile {name: LLM_MMPROJ_FILE.to_string(),url: url.to_string(),+ size,sha256: sha256.to_string(),}),- (None, None) => {}+ (None, None, None) => {}_ => {return Err(invalid_preset(preset,- "mmproj URL and checksum must be paired",+ "mmproj URL, size, and checksum must be paired",));}}@@ -68,8 +71,13 @@ pub fn desktop_llm_asset(model_id: &str) -> Result<Asset, InvalidPreset> {pub fn transcription_model_asset() -> Asset {let preset = config::defaults().transcription_model;- Asset::tar_gz(model_key(&preset.id), preset.url, preset.sha256)- .expect("transcription asset config")+ Asset::tar_gz(+ model_key(&preset.id),+ preset.url,+ preset.size,+ preset.sha256,+ )+ .expect("transcription asset config")}pub fn voice_activity_model_asset() -> Asset {@@ -79,6 +87,7 @@ pub fn voice_activity_model_asset() -> Asset {vec![AssetFile {name: VOICE_ACTIVITY_MODEL_FILE.to_string(),url: preset.url,+ size: preset.size,sha256: preset.sha256,}],)@@ -92,6 +101,7 @@ pub fn knowledge_embedding_model_asset() -> Asset {vec![AssetFile {name: LLM_MODEL_FILE.to_string(),url: embedding.model_url,+ size: embedding.model_size,sha256: embedding.model_sha256,}],)@@ -188,8 +198,10 @@ mod tests {id: "qwen-2b-q8".to_string(),title: "Qwen".to_string(),url: "https://example.org/main.gguf".to_string(),+ size: 1,sha256: "0".repeat(64),mmproj_url: mmproj_url.map(Into::into),+ mmproj_size: mmproj_url.map(|_| 1),mmproj_sha256: mmproj_sha256.map(Into::into),}}diff --git a/rust/crates/ensu/src/retrieval/pack.rs b/rust/crates/ensu/src/retrieval/pack.rsindex 65a9fc4572..31ef7e850f 100644--- a/rust/crates/ensu/src/retrieval/pack.rs+++ b/rust/crates/ensu/src/retrieval/pack.rs@@ -15,10 +15,12 @@ pub fn knowledge_asset(expected_pack: &KnowledgeDatasetConfig) -> Result<Asset,let files = KNOWLEDGE_ARTIFACT_FILENAMES.into_iter().zip(urls)+ .zip(expected_pack.artifact_sizes.iter().copied()).zip(expected_pack.artifact_sha256.iter().cloned())- .map(|((filename, url), sha256)| AssetFile {+ .map(|(((filename, url), size), sha256)| AssetFile {name: filename.to_owned(),url,+ size,sha256,}).collect();diff --git a/rust/crates/ml/src/assets.rs b/rust/crates/ml/src/assets.rsindex a8758ea2dc..faf35f4c7d 100644--- a/rust/crates/ml/src/assets.rs+++ b/rust/crates/ml/src/assets.rs@@ -36,6 +36,7 @@ fn model_asset(model: Model) -> Asset {.map(|file| AssetFile {name: file.name.to_string(),url: format!("https://models.ente.com/{}", file.name),+ size: file.size,sha256: file.sha256.to_string(),}).collect(),@@ -317,12 +318,14 @@ mod tests {let lock: serde_json::Value = serde_json::from_str(&contents).unwrap();for model in lock["models"].as_object().unwrap().values() {let name = model["file_name"].as_str().unwrap();+ let size = model["size"].as_u64().unwrap();let sha256 = model["sha256"].as_str().unwrap();let catalog_file = Model::ALL.iter().flat_map(|model| model_asset_spec(*model).files).find(|file| file.name == name).unwrap_or_else(|| panic!("{name} is missing from the Photos model catalog"));+ assert_eq!(catalog_file.size, size, "{name}");assert_eq!(catalog_file.sha256, sha256, "{name}");}}diff --git a/rust/crates/ml/src/scan/mod.rs b/rust/crates/ml/src/scan/mod.rsindex 86f4d7b60a..82da7c2115 100644--- a/rust/crates/ml/src/scan/mod.rs+++ b/rust/crates/ml/src/scan/mod.rs@@ -34,6 +34,7 @@ fn segmentation_model_asset() -> Result<Asset, ScanError> {AssetFile {name: SEGMENTATION_MODEL_FILE.to_string(),url: SEGMENTATION_MODEL_URL.to_string(),+ size: 5_085_426,sha256: SEGMENTATION_MODEL_SHA256.to_string(),},)diff --git a/rust/crates/ml/tests/support/ml_indexing.rs b/rust/crates/ml/tests/support/ml_indexing.rsindex 08bb9e6350..e5f80fbd0f 100644--- a/rust/crates/ml/tests/support/ml_indexing.rs+++ b/rust/crates/ml/tests/support/ml_indexing.rs@@ -472,6 +472,7 @@ struct AssetLock {struct DocumentAsset {path: String,url: String,+ size: u64,sha256: String,}@@ -483,6 +484,7 @@ struct OnnxRuntimeAssets {#[derive(Debug, Deserialize)]struct OnnxRuntimeArchive {url: String,+ size: u64,sha256: String,library_path: String,library_sha256: String,@@ -503,6 +505,7 @@ struct ModelAssets {struct ModelAsset {file_name: String,url: String,+ size: u64,sha256: String,}@@ -527,6 +530,7 @@ struct FixtureManifest {#[derive(Debug, Deserialize)]struct FixtureFile {path: String,+ size: u64,sha256: String,}@@ -625,6 +629,7 @@ async fn resolve_document_asset(label,&file_id_for_manifest_path(&asset.path)?,&asset.url,+ asset.size,&asset.sha256,).await@@ -653,6 +658,7 @@ async fn fetch_fixtures(&label,&label,&fixture_url(fixture_base_url, &fixture.path)?,+ fixture.size,&fixture.sha256,).await?,@@ -685,6 +691,7 @@ async fn resolve_onnx_runtime_library(&target_key,&archive_name,&archive.url,+ archive.size,&archive.sha256,).await?;@@ -830,6 +837,7 @@ async fn golden_model(label,&model.file_name,&model.url,+ model.size,&model.sha256,).await?,@@ -848,6 +856,7 @@ async fn resolve_model_asset(label,&asset.file_name,&asset.url,+ asset.size,&asset.sha256,).await@@ -859,6 +868,7 @@ async fn download_file(key: &str,name: &str,url: &str,+ size: u64,expected_sha256: &str,) -> Result<PathBuf> {let sha256 = normalize_sha256(expected_sha256);@@ -867,6 +877,7 @@ async fn download_file(AssetFile {name: name.to_string(),url: url.to_string(),+ size,sha256,},)diff --git a/rust/crates/photos/src/location.rs b/rust/crates/photos/src/location.rsindex 11444df519..b44bc3dc33 100644--- a/rust/crates/photos/src/location.rs+++ b/rust/crates/photos/src/location.rs@@ -74,28 +74,33 @@ fn asset() -> Result<Asset, ente_assets::download::Error> {vec![file(CITY_FILE,+ 2_809_615,"898dca892a71fd601ae8e75e5c55fd6d4591e4c98de335d9b33e7f076aa668f5",),file(URBAN_CENTER_FILE,+ 1_825_064,"e981b0383e6502ede56b52d5be96cda66f53d7352492e677d8978467c106dab0",),file(COUNTRY_FILE,+ 2_504_017,"3ead1ef2bb03b4ffa95813df4316cf149d37c82d16a322352bd6e3b19218c796",),file(DISPUTE_FILE,+ 61_543,"c44d0af6ad40a3b195bc10cb87512c16998a20e837af190ff1a46b00170dfd7b",),],)}-fn file(name: &str, sha256: &str) -> AssetFile {+fn file(name: &str, size: u64, sha256: &str) -> AssetFile {AssetFile {name: name.into(),url: format!("{ASSET_URL}/{name}"),+ size,sha256: sha256.into(),}}