π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Download files sequentially and preserve resumable state on failures
Manav Rathicf2819790d03
diff --git a/rust/crates/model-download/src/download.rs b/rust/crates/model-download/src/download.rsindex d0b7e5a0d0..7f8d127538 100644--- a/rust/crates/model-download/src/download.rs+++ b/rust/crates/model-download/src/download.rs@@ -280,106 +280,91 @@ async fn fetch_async(&on_progress,);- let mut downloads = Vec::new();-for (index, target) in targets.iter().enumerate() {if cached[index] {continue;}+ if is_cancelled() {+ return Err(Error::Cancelled);+ }let destination = PathBuf::from(&target.destination_path);-let download_probe = download_probes.get(index).cloned().unwrap_or_else(DownloadProbe::default);let expected_file_total = download_probe.content_length;- let progress_states = Rc::clone(&file_states);- let progress_callback = Rc::clone(&on_progress);let target_label = target.label.clone();- let client = &client;- let is_cancelled = &is_cancelled;-- downloads.push(async move {- if is_cancelled() {- return Err(Error::Cancelled);- }-- let file_report = download_file(- client,- target,- &destination,- &download_probe,- |file_progress| {- {- let mut states = progress_states.borrow_mut();- if let Some(state) = states.get_mut(index) {- state.downloaded_bytes = file_progress.downloaded_bytes;- state.total_bytes = file_progress.total_bytes;- state.network_downloaded_bytes = file_progress.network_downloaded_bytes;- state.elapsed = file_progress.elapsed;- state.retry_count = file_progress.retry_count;- }+ let file_report = download_file(+ &client,+ target,+ &destination,+ &download_probe,+ |file_progress| {+ {+ let mut states = file_states.borrow_mut();+ if let Some(state) = states.get_mut(index) {+ state.downloaded_bytes = file_progress.downloaded_bytes;+ state.total_bytes = file_progress.total_bytes;+ state.network_downloaded_bytes = file_progress.network_downloaded_bytes;+ state.elapsed = file_progress.elapsed;+ state.retry_count = file_progress.retry_count;}+ }- let metrics = aggregate_progress_metrics(- download_started_at.elapsed(),- &progress_states,- index,- false,- false,- );- emit_progress_from_states(- &target_label,- total_bytes,- metrics,- Some(index),- &progress_states,- &progress_callback,- );- },- is_cancelled,- )- .await- .map_err(|err| match err {- Error::Cancelled => Error::Cancelled,- err => Error::Target {- label: target.label.clone(),- source: Box::new(err),- },- })?;+ let metrics = aggregate_progress_metrics(+ download_started_at.elapsed(),+ &file_states,+ index,+ false,+ false,+ );+ emit_progress_from_states(+ &target_label,+ total_bytes,+ metrics,+ Some(index),+ &file_states,+ &on_progress,+ );+ },+ &is_cancelled,+ )+ .await+ .map_err(|err| match err {+ Error::Cancelled => Error::Cancelled,+ err => Error::Target {+ label: target.label.clone(),+ source: Box::new(err),+ },+ })?;- {- let mut states = progress_states.borrow_mut();- if let Some(state) = states.get_mut(index) {- state.downloaded_bytes = file_report.final_size;- state.total_bytes = expected_file_total.or(Some(file_report.final_size));- state.network_downloaded_bytes = file_report.network_downloaded_bytes;- state.elapsed = file_report.elapsed;- state.retry_count = file_report.retry_count;- }+ {+ let mut states = file_states.borrow_mut();+ if let Some(state) = states.get_mut(index) {+ state.downloaded_bytes = file_report.final_size;+ state.total_bytes = expected_file_total.or(Some(file_report.final_size));+ state.network_downloaded_bytes = file_report.network_downloaded_bytes;+ state.elapsed = file_report.elapsed;+ state.retry_count = file_report.retry_count;}+ }- let metrics = aggregate_progress_metrics(- download_started_at.elapsed(),- &progress_states,- index,- true,- false,- );- emit_progress_from_states(- &target_label,- total_bytes,- metrics,- Some(index),- &progress_states,- &progress_callback,- );-- Ok(file_report)- });+ let metrics = aggregate_progress_metrics(+ download_started_at.elapsed(),+ &file_states,+ index,+ true,+ false,+ );+ emit_progress_from_states(+ &target_label,+ total_bytes,+ metrics,+ Some(index),+ &file_states,+ &on_progress,+ );}-- let _reports = try_join_all(downloads).await?;let complete_metrics = aggregate_complete_metrics(download_started_at.elapsed(), &file_states);emit_progress_from_states(@@ -422,7 +407,9 @@ async fn download_file(.await{Ok(report) => return Ok(report),- Err(err @ (Error::Cancelled | Error::Validation(_))) => return Err(err),+ Err(err @ (Error::Cancelled | Error::Validation(_) | Error::StorageFull)) => {+ return Err(err);+ }Err(err) => {range_error = Some(err);cleanup_range_download(destination);@@ -1913,6 +1900,43 @@ mod tests {let _ = fs::remove_dir_all(test_dir);}+ #[test]+ fn downloads_files_sequentially() {+ let bytes = Arc::new(sample_bytes(512));+ let active = Arc::new(AtomicUsize::new(0));+ let max_active = Arc::new(AtomicUsize::new(0));++ let server = {+ let bytes = Arc::clone(&bytes);+ let active = Arc::clone(&active);+ let max_active = Arc::clone(&max_active);+ TestServer::spawn(move |stream| {+ let current = active.fetch_add(1, Ordering::SeqCst) + 1;+ max_active.fetch_max(current, Ordering::SeqCst);+ thread::sleep(Duration::from_millis(50));+ handle_no_range_test_request(+ stream,+ Arc::clone(&bytes),+ Arc::new(AtomicUsize::new(0)),+ );+ active.fetch_sub(1, Ordering::SeqCst);+ })+ };++ let test_dir = scratch_dir("sequential-downloads");+ let targets = ["model", "projector"].map(|label| Target {+ label: label.to_string(),+ url: server.url(&format!("/{label}.bin")),+ destination_path: test_dir.join(format!("{label}.bin")).display().to_string(),+ sha256: sha_hex(&bytes),+ });++ fetch(targets.into(), |_| {}, || false).expect("downloads succeed");+ assert_eq!(max_active.load(Ordering::SeqCst), 1);++ let _ = fs::remove_dir_all(test_dir);+ }+#[test]fn completed_download_is_served_from_cache() {let bytes = Arc::new(sample_bytes(MIN_RANGE_DOWNLOAD_BYTES as usize + 123));diff --git a/rust/crates/model-download/src/downloader.rs b/rust/crates/model-download/src/downloader.rsindex c86693334b..c327abd0fb 100644--- a/rust/crates/model-download/src/downloader.rs+++ b/rust/crates/model-download/src/downloader.rs@@ -13,15 +13,15 @@ use crate::gguf;#[derive(Debug, Clone)]pub enum ModelDownloadTarget {+ Files {+ id: String,+ files: Vec<ModelFile>,+ },TarGz {id: String,url: String,sha256: String,},- Files {- id: String,- files: Vec<ModelFile>,- },}impl ModelDownloadTarget {@@ -190,12 +190,14 @@ impl ModelDownloader {log_line: Some(format!("Extracting model archive id={id}")),});let archive_path = self.staging_dir().join(format!("{id}.tar.gz"));- let result = archive::extract_tar_gz(- &archive_path,- &self.staging_dir().join(id),- &self.model_path(target),- );- let _ = fs::remove_file(&archive_path);+ let extraction_dir = self.staging_dir().join(id);+ let result =+ archive::extract_tar_gz(&archive_path, &extraction_dir, &self.model_path(target));+ if result.is_err() {+ let _ = fs::remove_dir_all(extraction_dir);+ } else {+ let _ = fs::remove_file(&archive_path);+ }result}@@ -235,7 +237,7 @@ impl ModelDownloader {fn storage_key(target: &ModelDownloadTarget) -> &str {match target {- ModelDownloadTarget::TarGz { id, .. } | ModelDownloadTarget::Files { id, .. } => id,+ ModelDownloadTarget::Files { id, .. } | ModelDownloadTarget::TarGz { id, .. } => id,}}@@ -784,6 +786,22 @@ mod tests {let _ = fs::remove_dir_all(dir);}+ #[test]+ fn failed_archive_extraction_cleans_staging() {+ let dir = scratch_dir("variant-extract-failure");+ let downloader = ModelDownloader::new(&dir);+ let target = tar_gz_target("parakeet-v3-int8");+ let staging_dir = dir.join(".staging");+ fs::create_dir_all(&staging_dir).unwrap();+ fs::write(staging_dir.join("parakeet-v3-int8.tar.gz"), b"invalid").unwrap();++ assert!(downloader.extract_if_archive(&target, &mut |_| {}).is_err());+ assert!(!staging_dir.join("parakeet-v3-int8").exists());+ assert!(staging_dir.join("parakeet-v3-int8.tar.gz").exists());++ let _ = fs::remove_dir_all(dir);+ }+#[test]fn onnx_target_downloaded_and_removed_via_model_file() {let dir = scratch_dir("variant-onnx");