Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Download files sequentially and preserve resumable state on failures

Manav Rathicf2819790d03
diff --git a/rust/crates/model-download/src/download.rs b/rust/crates/model-download/src/download.rs
index d0b7e5a0d0..7f8d127538 100644
--- a/rust/crates/model-download/src/download.rs
+++ b/rust/crates/model-download/src/download.rs
@@ -280,106 +280,91 @@ async fn fetch_async(
&on_progress,
);
- let mut downloads = Vec::new();
-
for (index, target) in targets.iter().enumerate() {
if cached[index] {
continue;
}
+ if is_cancelled() {
+ return Err(Error::Cancelled);
+ }
let destination = PathBuf::from(&target.destination_path);
-
let download_probe = download_probes
.get(index)
.cloned()
.unwrap_or_else(DownloadProbe::default);
let expected_file_total = download_probe.content_length;
- let progress_states = Rc::clone(&file_states);
- let progress_callback = Rc::clone(&on_progress);
let target_label = target.label.clone();
- let client = &client;
- let is_cancelled = &is_cancelled;
-
- downloads.push(async move {
- if is_cancelled() {
- return Err(Error::Cancelled);
- }
-
- let file_report = download_file(
- client,
- target,
- &destination,
- &download_probe,
- |file_progress| {
- {
- let mut states = progress_states.borrow_mut();
- if let Some(state) = states.get_mut(index) {
- state.downloaded_bytes = file_progress.downloaded_bytes;
- state.total_bytes = file_progress.total_bytes;
- state.network_downloaded_bytes = file_progress.network_downloaded_bytes;
- state.elapsed = file_progress.elapsed;
- state.retry_count = file_progress.retry_count;
- }
+ let file_report = download_file(
+ &client,
+ target,
+ &destination,
+ &download_probe,
+ |file_progress| {
+ {
+ let mut states = file_states.borrow_mut();
+ if let Some(state) = states.get_mut(index) {
+ state.downloaded_bytes = file_progress.downloaded_bytes;
+ state.total_bytes = file_progress.total_bytes;
+ state.network_downloaded_bytes = file_progress.network_downloaded_bytes;
+ state.elapsed = file_progress.elapsed;
+ state.retry_count = file_progress.retry_count;
}
+ }
- let metrics = aggregate_progress_metrics(
- download_started_at.elapsed(),
- &progress_states,
- index,
- false,
- false,
- );
- emit_progress_from_states(
- &target_label,
- total_bytes,
- metrics,
- Some(index),
- &progress_states,
- &progress_callback,
- );
- },
- is_cancelled,
- )
- .await
- .map_err(|err| match err {
- Error::Cancelled => Error::Cancelled,
- err => Error::Target {
- label: target.label.clone(),
- source: Box::new(err),
- },
- })?;
+ let metrics = aggregate_progress_metrics(
+ download_started_at.elapsed(),
+ &file_states,
+ index,
+ false,
+ false,
+ );
+ emit_progress_from_states(
+ &target_label,
+ total_bytes,
+ metrics,
+ Some(index),
+ &file_states,
+ &on_progress,
+ );
+ },
+ &is_cancelled,
+ )
+ .await
+ .map_err(|err| match err {
+ Error::Cancelled => Error::Cancelled,
+ err => Error::Target {
+ label: target.label.clone(),
+ source: Box::new(err),
+ },
+ })?;
- {
- let mut states = progress_states.borrow_mut();
- if let Some(state) = states.get_mut(index) {
- state.downloaded_bytes = file_report.final_size;
- state.total_bytes = expected_file_total.or(Some(file_report.final_size));
- state.network_downloaded_bytes = file_report.network_downloaded_bytes;
- state.elapsed = file_report.elapsed;
- state.retry_count = file_report.retry_count;
- }
+ {
+ let mut states = file_states.borrow_mut();
+ if let Some(state) = states.get_mut(index) {
+ state.downloaded_bytes = file_report.final_size;
+ state.total_bytes = expected_file_total.or(Some(file_report.final_size));
+ state.network_downloaded_bytes = file_report.network_downloaded_bytes;
+ state.elapsed = file_report.elapsed;
+ state.retry_count = file_report.retry_count;
}
+ }
- let metrics = aggregate_progress_metrics(
- download_started_at.elapsed(),
- &progress_states,
- index,
- true,
- false,
- );
- emit_progress_from_states(
- &target_label,
- total_bytes,
- metrics,
- Some(index),
- &progress_states,
- &progress_callback,
- );
-
- Ok(file_report)
- });
+ let metrics = aggregate_progress_metrics(
+ download_started_at.elapsed(),
+ &file_states,
+ index,
+ true,
+ false,
+ );
+ emit_progress_from_states(
+ &target_label,
+ total_bytes,
+ metrics,
+ Some(index),
+ &file_states,
+ &on_progress,
+ );
}
-
- let _reports = try_join_all(downloads).await?;
let complete_metrics = aggregate_complete_metrics(download_started_at.elapsed(), &file_states);
emit_progress_from_states(
@@ -422,7 +407,9 @@ async fn download_file(
.await
{
Ok(report) => return Ok(report),
- Err(err @ (Error::Cancelled | Error::Validation(_))) => return Err(err),
+ Err(err @ (Error::Cancelled | Error::Validation(_) | Error::StorageFull)) => {
+ return Err(err);
+ }
Err(err) => {
range_error = Some(err);
cleanup_range_download(destination);
@@ -1913,6 +1900,43 @@ mod tests {
let _ = fs::remove_dir_all(test_dir);
}
+ #[test]
+ fn downloads_files_sequentially() {
+ let bytes = Arc::new(sample_bytes(512));
+ let active = Arc::new(AtomicUsize::new(0));
+ let max_active = Arc::new(AtomicUsize::new(0));
+
+ let server = {
+ let bytes = Arc::clone(&bytes);
+ let active = Arc::clone(&active);
+ let max_active = Arc::clone(&max_active);
+ TestServer::spawn(move |stream| {
+ let current = active.fetch_add(1, Ordering::SeqCst) + 1;
+ max_active.fetch_max(current, Ordering::SeqCst);
+ thread::sleep(Duration::from_millis(50));
+ handle_no_range_test_request(
+ stream,
+ Arc::clone(&bytes),
+ Arc::new(AtomicUsize::new(0)),
+ );
+ active.fetch_sub(1, Ordering::SeqCst);
+ })
+ };
+
+ let test_dir = scratch_dir("sequential-downloads");
+ let targets = ["model", "projector"].map(|label| Target {
+ label: label.to_string(),
+ url: server.url(&format!("/{label}.bin")),
+ destination_path: test_dir.join(format!("{label}.bin")).display().to_string(),
+ sha256: sha_hex(&bytes),
+ });
+
+ fetch(targets.into(), |_| {}, || false).expect("downloads succeed");
+ assert_eq!(max_active.load(Ordering::SeqCst), 1);
+
+ let _ = fs::remove_dir_all(test_dir);
+ }
+
#[test]
fn completed_download_is_served_from_cache() {
let bytes = Arc::new(sample_bytes(MIN_RANGE_DOWNLOAD_BYTES as usize + 123));
diff --git a/rust/crates/model-download/src/downloader.rs b/rust/crates/model-download/src/downloader.rs
index c86693334b..c327abd0fb 100644
--- a/rust/crates/model-download/src/downloader.rs
+++ b/rust/crates/model-download/src/downloader.rs
@@ -13,15 +13,15 @@ use crate::gguf;
#[derive(Debug, Clone)]
pub enum ModelDownloadTarget {
+ Files {
+ id: String,
+ files: Vec<ModelFile>,
+ },
TarGz {
id: String,
url: String,
sha256: String,
},
- Files {
- id: String,
- files: Vec<ModelFile>,
- },
}
impl ModelDownloadTarget {
@@ -190,12 +190,14 @@ impl ModelDownloader {
log_line: Some(format!("Extracting model archive id={id}")),
});
let archive_path = self.staging_dir().join(format!("{id}.tar.gz"));
- let result = archive::extract_tar_gz(
- &archive_path,
- &self.staging_dir().join(id),
- &self.model_path(target),
- );
- let _ = fs::remove_file(&archive_path);
+ let extraction_dir = self.staging_dir().join(id);
+ let result =
+ archive::extract_tar_gz(&archive_path, &extraction_dir, &self.model_path(target));
+ if result.is_err() {
+ let _ = fs::remove_dir_all(extraction_dir);
+ } else {
+ let _ = fs::remove_file(&archive_path);
+ }
result
}
@@ -235,7 +237,7 @@ impl ModelDownloader {
fn storage_key(target: &ModelDownloadTarget) -> &str {
match target {
- ModelDownloadTarget::TarGz { id, .. } | ModelDownloadTarget::Files { id, .. } => id,
+ ModelDownloadTarget::Files { id, .. } | ModelDownloadTarget::TarGz { id, .. } => id,
}
}
@@ -784,6 +786,22 @@ mod tests {
let _ = fs::remove_dir_all(dir);
}
+ #[test]
+ fn failed_archive_extraction_cleans_staging() {
+ let dir = scratch_dir("variant-extract-failure");
+ let downloader = ModelDownloader::new(&dir);
+ let target = tar_gz_target("parakeet-v3-int8");
+ let staging_dir = dir.join(".staging");
+ fs::create_dir_all(&staging_dir).unwrap();
+ fs::write(staging_dir.join("parakeet-v3-int8.tar.gz"), b"invalid").unwrap();
+
+ assert!(downloader.extract_if_archive(&target, &mut |_| {}).is_err());
+ assert!(!staging_dir.join("parakeet-v3-int8").exists());
+ assert!(staging_dir.join("parakeet-v3-int8.tar.gz").exists());
+
+ let _ = fs::remove_dir_all(dir);
+ }
+
#[test]
fn onnx_target_downloaded_and_removed_via_model_file() {
let dir = scratch_dir("variant-onnx");