A high-performance inference engine for AI models
Revert "hf downloading support" (#799)
uuuvn2d355aa90959
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex 027fa96db..636254475 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -81,7 +81,7 @@ jobs:tests: 'kernels'- host: 'linux'target: 'wasm32-unknown-unknown'- crates: ['kiban', 'download-manager']+ crates: ['kiban']env:CARGO_ENCODED_RUSTFLAGS: "-Dwarnings"diff --git a/Cargo.lock b/Cargo.lockindex 4a053599a..f409446a1 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1750,13 +1750,10 @@ version = "0.5.23"dependencies = ["async-trait","base64 0.22.1",- "bitflags 2.13.0","block2","chrono","crc32c","futures-util",- "hex",- "http 1.4.2","kiban","libc","mock-registry",@@ -1767,8 +1764,6 @@ dependencies = ["rstest","serde","serde_json",- "sha1",- "sha2","tempfile","thiserror 2.0.18","tokio",@@ -7537,6 +7532,7 @@ dependencies = ["backend-remote","bindings","bindings-types",+ "bitflags 2.13.0","chrono","confy","dirs",diff --git a/Cargo.toml b/Cargo.tomlindex 1b9479ca7..df5e7a80f 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -152,9 +152,6 @@ crc32c = "0.6.8"base64 = "0.22.1"bitflags = "2.11.1"chrono = { version = "0.4.42", features = ["serde"] }-hex = "0.4.3"-sha1 = "0.10.6"-sha2 = "0.10.9"jsonschema = "0.46.0"# Please make sure to use compatible versions of the cargo-swift / uniffi pair.# See: https://github.com/antoniusnaumann/cargo-swift?tab=readme-ov-file#installing-for-a-different-uniffi-versiondiff --git a/apps/playground/Sources/Views/ModelListView.swift b/apps/playground/Sources/Views/ModelListView.swiftindex e83e55d72..437b626db 100644--- a/apps/playground/Sources/Views/ModelListView.swift+++ b/apps/playground/Sources/Views/ModelListView.swift@@ -130,7 +130,7 @@ struct ModelListView: View {guardlet selectedIdentifier,let model = group.modelByIdentifier[selectedIdentifier],- model.isOnDevice(),+ model.isLocal(),let state = engineWrapper.downloadStates[model.identifier]else { return }@@ -347,7 +347,7 @@ struct ModelListView: View {private var selectedModelState: DownloadState? {guard let selectedIdentifier else { return nil }guard let model = models()- .filter({ $0.isOnDevice() })+ .filter({ $0.isLocal() }).first(where: { $0.identifier == selectedIdentifier }) else {return nil}@@ -358,7 +358,7 @@ struct ModelListView: View {private func select(identifier: String) {selectedIdentifier = identifierif let model = models()- .filter({ $0.isOnDevice() })+ .filter({ $0.isLocal() }).first(where: { $0.identifier == selectedIdentifier }),let state = engineWrapper.downloadStates[model.identifier] {selectedModelSection = section(for: state)@@ -488,7 +488,7 @@ struct ModelListView: View {}}let cloud = all.filter { $0.isRemote() }.sorted { $0.name() < $1.name() }- let local = all.filter { $0.isOnDevice() }+ let local = all.filter { $0.isLocal() }let modelByIdentifier = Dictionary(uniqueKeysWithValues: all.map { ($0.identifier, $0) })var bySection: [ModelSection: [String]] = [:]diff --git a/crates/legacy/cli/src/bench/runner.rs b/crates/legacy/cli/src/bench/runner.rsindex e1d4aef9e..f46695698 100644--- a/crates/legacy/cli/src/bench/runner.rs+++ b/crates/legacy/cli/src/bench/runner.rs@@ -4,7 +4,7 @@ use std::{};use anyhow::{Context, Result};-use shoji::types::model::{ModelFamily, ModelSource};+use shoji::types::model::{ModelFamily, ModelReference};use sysinfo::System;use uzu::{engine::{Engine, EngineConfig},@@ -166,9 +166,9 @@ impl BenchRunner {.to_lowercase();let all_models = engine.models().await?;for model in all_models {- if let ModelAccessibility::OnDevice {- source:- ModelSource::Registry {+ if let ModelAccessibility::Local {+ reference:+ ModelReference::Mirai {toolchain_version: _toolchain_version,repository,source_repository,diff --git a/crates/legacy/cli/src/common/thinking.rs b/crates/legacy/cli/src/common/thinking.rsindex 864f7481a..b4520c10a 100644--- a/crates/legacy/cli/src/common/thinking.rs+++ b/crates/legacy/cli/src/common/thinking.rs@@ -204,7 +204,7 @@ impl ThinkingSupport {mod tests {use shoji::types::{basic::Value,- model::{ModelAccessibility, ModelSource, ModelSpecialization},+ model::{ModelAccessibility, ModelReference, ModelSpecialization},};use super::*;@@ -236,8 +236,8 @@ mod tests {"uzu".to_string(),"0.0".to_string(),vec![ModelSpecialization::Chat {}],- ModelAccessibility::OnDevice {- source: ModelSource::Filesystem {+ ModelAccessibility::Local {+ reference: ModelReference::Local {path: "/nonexistent".to_string(),},},diff --git a/crates/legacy/cli/src/interactive/components/application.rs b/crates/legacy/cli/src/interactive/components/application.rsindex 1b7656e23..3a93f0bb2 100644--- a/crates/legacy/cli/src/interactive/components/application.rs+++ b/crates/legacy/cli/src/interactive/components/application.rs@@ -162,7 +162,7 @@ pub fn Application(match engine.model(identifier.clone()).await {Ok(Some(model)) => {let model_exists = model.is_downloadable()- || !model.is_on_device()+ || !model.is_local()|| matches!(engine.model_path(&model).await,Some(path) if std::path::Path::new(&path).exists()diff --git a/crates/legacy/cli/src/interactive/model.rs b/crates/legacy/cli/src/interactive/model.rsindex 5a3ef2dd2..cd124bb39 100644--- a/crates/legacy/cli/src/interactive/model.rs+++ b/crates/legacy/cli/src/interactive/model.rs@@ -1,4 +1,4 @@-use shoji::types::model::{Model, ModelAccessibility, ModelSource};+use shoji::types::model::{Model, ModelAccessibility, ModelReference};use sysinfo::System;use uzu::engine::{Engine, EngineError};@@ -116,8 +116,8 @@ fn model_shorthand_matches(}fn checkpoint_size_bytes(model: &Model) -> Option<u64> {- if let ModelAccessibility::OnDevice {- source: ModelSource::Registry {+ if let ModelAccessibility::Local {+ reference: ModelReference::Mirai {files,..},diff --git a/crates/legacy/cli/unit/interactive/model_test.rs b/crates/legacy/cli/unit/interactive/model_test.rsindex 844dddb65..960a9824d 100644--- a/crates/legacy/cli/unit/interactive/model_test.rs+++ b/crates/legacy/cli/unit/interactive/model_test.rs@@ -124,8 +124,8 @@ fn checkpoint(metadata: metadata(quantization_method),}),specializations: vec![],- accessibility: ModelAccessibility::OnDevice {- source: ModelSource::Registry {+ accessibility: ModelAccessibility::Local {+ reference: ModelReference::Mirai {toolchain_version: "test".to_string(),repository: None,source_repository: None,diff --git a/crates/legacy/download-manager/Cargo.toml b/crates/legacy/download-manager/Cargo.tomlindex be3e58d76..12d78a82b 100644--- a/crates/legacy/download-manager/Cargo.toml+++ b/crates/legacy/download-manager/Cargo.toml@@ -11,20 +11,15 @@ license.workspace = true[dependencies]kiban.workspace = truefutures-util.workspace = true-http.workspace = trueserde.workspace = trueserde_json.workspace = trueasync-trait.workspace = truethiserror.workspace = truecrc32c.workspace = truebase64.workspace = true-bitflags.workspace = truetracing.workspace = truechrono.workspace = truereqwest.workspace = true-hex.workspace = true-sha1.workspace = true-sha2.workspace = truetokio = { workspace = true, features = ["macros", "sync"] }[target.'cfg(target_vendor = "apple")'.dependencies]diff --git a/crates/legacy/download-manager/docs/STATE_REDUCTION_DECISION_MATRIX.md b/crates/legacy/download-manager/docs/STATE_REDUCTION_DECISION_MATRIX.mdindex b129a406c..ec256fab7 100644--- a/crates/legacy/download-manager/docs/STATE_REDUCTION_DECISION_MATRIX.md+++ b/crates/legacy/download-manager/docs/STATE_REDUCTION_DECISION_MATRIX.md@@ -33,7 +33,7 @@ Determines if a file on disk is valid, invalid, or missing based on CRC validati**CRC Caching Optimization:**-- Valid files get an `.integrity` receipt to avoid recalculation on next launch+- Valid files get a `.crc` cache file to avoid recalculation on next launch- Makes subsequent launches ~100x faster for large files- Cache is invalidated when file changes@@ -219,7 +219,7 @@ if task.state() == Completed && checked_file_state == Valid {```rustif file_size < expected_bytes {// Partial download - preserve file- remove_integrity_receipt();+ remove_crc_cache();return NotDownloaded;} else {// Corrupted full-size file - delete@@ -337,7 +337,7 @@ Initial: file=Exists(100MB), crc=Missing, resume=Missing, task=None↓Checked: Invalid (file incomplete, fails CRC)Reconcile: Detect: 100MB < 1.1GB (partial)-Action: PRESERVE file, remove integrity receipt+Action: PRESERVE file, remove CRC cacheInternal: NotDownloadedDisplay: NotDownloaded (0 / 1.1GB)↓ Future download can resume from partial file if URLSession supports itdiff --git a/crates/legacy/download-manager/examples/cross_platform_download.rs b/crates/legacy/download-manager/examples/cross_platform_download.rsindex e3be5e496..0f8b81dcb 100644--- a/crates/legacy/download-manager/examples/cross_platform_download.rs+++ b/crates/legacy/download-manager/examples/cross_platform_download.rs@@ -20,7 +20,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {}println!("Creating download task...");- let task = manager.file_download_task((&url).into(), &destination, FileCheck::None, None).await?;+ let task = manager.file_download_task(&url, &destination, FileCheck::None, None).await?;println!("Starting download...");let mut progress_stream = task.progress().await?;diff --git a/crates/legacy/download-manager/examples/wasm/src/lib.rs b/crates/legacy/download-manager/examples/wasm/src/lib.rsindex 479d0c9ab..474369046 100644--- a/crates/legacy/download-manager/examples/wasm/src/lib.rs+++ b/crates/legacy/download-manager/examples/wasm/src/lib.rs@@ -74,7 +74,7 @@ async fn download_internal(fs::asyn::remove_file(&file_path).await?;}- let task = manager.file_download_task((&url).into(), &file_path, FileCheck::None, None).await?;+ let task = manager.file_download_task(&url, &file_path, FileCheck::None, None).await?;let mut progress_stream = task.progress().await?;task.download().await?;diff --git a/crates/legacy/download-manager/src/backends/apple/active_task.rs b/crates/legacy/download-manager/src/backends/apple/active_task.rsindex 9cbccc565..764147991 100644--- a/crates/legacy/download-manager/src/backends/apple/active_task.rs+++ b/crates/legacy/download-manager/src/backends/apple/active_task.rs@@ -5,7 +5,7 @@ use std::{use objc2::rc::Retained;use objc2_foundation::NSURLSessionDownloadTask;-use tokio::{fs::write as tokio_write, sync::oneshot::channel as tokio_oneshot_channel};+use tokio::sync::oneshot::channel as tokio_oneshot_channel;use crate::{DownloadId,@@ -21,7 +21,6 @@ pub struct AppleActiveTask {task: Retained<NSURLSessionDownloadTask>,event_registry: AppleEventRegistry,sink_key: AppleSinkKey,- authenticated: bool,}impl AppleActiveTask {@@ -29,14 +28,12 @@ impl AppleActiveTask {task: Retained<NSURLSessionDownloadTask>,event_registry: AppleEventRegistry,download_id: DownloadId,- authenticated: bool,) -> Self {let sink_key = (download_id, task.task_identifier());Self {task,event_registry,sink_key,- authenticated,}}@@ -62,12 +59,7 @@ impl ActiveTask for AppleActiveTask {self,destination: &Path,) -> Result<PathBuf, <Self::Backend as DownloadBackend>::Error> {- let resume_artifact_path = destination.with_added_extension("resume_data");- if self.authenticated {- drop(self);- tokio_write(&resume_artifact_path, []).await.map_err(|error| AppleBackendError::Io(error.to_string()))?;- return Ok(resume_artifact_path);- }+ let resume_artifact_path = destination.with_extension("resume_data");let (resume_data_sender, resume_data_receiver) = tokio_oneshot_channel::<Box<[u8]>>();let pending_resume_data_sender = Arc::new(Mutex::new(Some(resume_data_sender)));self.unregister_event_sink();@@ -91,7 +83,7 @@ impl ActiveTask for AppleActiveTask {}let resume_data_bytes =resume_data_receiver.await.map_err(|error| AppleBackendError::ResumeData(error.to_string()))?;- tokio_write(&resume_artifact_path, resume_data_bytes)+ tokio::fs::write(&resume_artifact_path, resume_data_bytes).await.map_err(|error| AppleBackendError::Io(error.to_string()))?;Ok(resume_artifact_path)@@ -101,7 +93,8 @@ impl ActiveTask for AppleActiveTask {self,_destination: &Path,) -> CancelOutcome {- drop(self);+ self.unregister_event_sink();+ self.task.cancel();CancelOutcome::BestEffort}}diff --git a/crates/legacy/download-manager/src/backends/apple/backend.rs b/crates/legacy/download-manager/src/backends/apple/backend.rsindex d8b3b3c3f..63d9db66b 100644--- a/crates/legacy/download-manager/src/backends/apple/backend.rs+++ b/crates/legacy/download-manager/src/backends/apple/backend.rs@@ -12,7 +12,7 @@ use crate::{common::{self, InitialTaskAttachment},},lock_manager::DestinationLockLease,- traits::{ActiveDownloadGeneration, BackendEventSender, Capabilities, DownloadBackend, DownloadConfig},+ traits::{ActiveDownloadGeneration, BackendEventSender, DownloadBackend, DownloadConfig},};#[derive(Clone, Debug, Default)]@@ -22,7 +22,6 @@ impl DownloadBackend for AppleBackend {type Context = AppleBackendContext;type ActiveTask = AppleActiveTask;type Error = AppleBackendError;- const CAPABILITIES: Capabilities = Capabilities::CACHES_REDIRECTED_URL_IN_RESUME_DATA;async fn read_resume_progress(part_path: &Path) -> Option<u64> {resume_data_parser::read_resume_progress(part_path)@@ -69,12 +68,7 @@ impl common::Backend for AppleBackend {task.resume();}Ok(InitialTaskAttachment::Downloading {- active_task: AppleActiveTask::new(- task,- context.event_registry(),- config.download_id,- config.request.is_authenticated(),- ),+ active_task: AppleActiveTask::new(task, context.event_registry(), config.download_id),initial_downloaded_bytes,total_bytes,})diff --git a/crates/legacy/download-manager/src/backends/apple/context.rs b/crates/legacy/download-manager/src/backends/apple/context.rsindex 383b92b04..4d59921ff 100644--- a/crates/legacy/download-manager/src/backends/apple/context.rs+++ b/crates/legacy/download-manager/src/backends/apple/context.rs@@ -1,23 +1,19 @@use std::{collections::HashMap,- fmt::{Debug, Formatter, Result as FmtResult},path::Path,sync::{Arc, Mutex},};-use http::header::{ACCEPT_ENCODING, AUTHORIZATION};use kiban::rt::RuntimeHandle;-use objc2::rc::Retained;+use objc2::{rc::Retained, runtime::ProtocolObject};use objc2_foundation::{- NSBundle, NSData, NSMutableURLRequest, NSString, NSURL, NSURLSession, NSURLSessionConfiguration,+ NSBundle, NSData, NSString, NSURL, NSURLSession, NSURLSessionConfiguration, NSURLSessionDelegate,NSURLSessionDownloadTask, NSURLSessionTaskState,};-use tokio::{fs::read as tokio_read, sync::oneshot::channel as tokio_oneshot_channel};+use tokio::sync::oneshot::channel as tokio_oneshot_channel;-#[cfg(test)]-use crate::DownloadId;use crate::{- DownloadInfo, HttpDownloadRequest,+ DownloadInfo, FileCheck,backends::apple::{AppleActiveTask, AppleBackend, AppleBackendError, AppleEventRegistry, AppleEventSink, AppleGetTasksHandler,AppleSessionDelegate, task_ext::AppleDownloadTaskExt,@@ -28,15 +24,17 @@ use crate::{pub struct AppleBackendContext {session: Retained<NSURLSession>,+ _delegate: Retained<AppleSessionDelegate>,+ _delegate_protocol_object: Retained<ProtocolObject<dyn NSURLSessionDelegate>>,event_registry: AppleEventRegistry,runtime_handle: RuntimeHandle,}-impl Debug for AppleBackendContext {+impl std::fmt::Debug for AppleBackendContext {fn fmt(&self,- formatter: &mut Formatter<'_>,- ) -> FmtResult {+ formatter: &mut std::fmt::Formatter<'_>,+ ) -> std::fmt::Result {formatter.debug_struct("AppleBackendContext").finish_non_exhaustive()}}@@ -45,17 +43,19 @@ impl AppleBackendContext {pub fn new(runtime_handle: RuntimeHandle) -> Self {let event_registry = Arc::new(Mutex::new(HashMap::new()));let delegate = AppleSessionDelegate::new(Arc::clone(&event_registry));- let delegate = AppleSessionDelegate::protocol_object(delegate);+ let delegate_protocol_object = AppleSessionDelegate::protocol_object(delegate.clone());let session = unsafe {NSURLSession::sessionWithConfiguration_delegate_delegateQueue(&automatic_session_configuration(),- Some(&delegate),+ Some(&delegate_protocol_object),None,)};Self {session,+ _delegate: delegate,+ _delegate_protocol_object: delegate_protocol_object,event_registry,runtime_handle,}@@ -65,7 +65,7 @@ impl AppleBackendContext {&self,config: &DownloadConfig,) -> Result<Option<Retained<NSURLSessionDownloadTask>>, AppleBackendError> {- select_matching_download_task(self.download_tasks().await?, config)+ self.find_download_task(config).await}pub(crate) async fn has_download_task_to_claim(@@ -78,11 +78,23 @@ impl AppleBackendContext {.any(|task| task.download_id() == Some(config.download_id) && is_live_task_state(task.state())))}+ #[cfg(test)]+ #[allow(dead_code)]+ pub(crate) fn event_sink_count_for_download(+ &self,+ download_id: crate::DownloadId,+ ) -> usize {+ self.event_registry+ .lock()+ .map(|registry| registry.keys().filter(|(id, _)| *id == download_id).count())+ .unwrap_or(0)+ }+#[cfg(test)]#[allow(dead_code)]pub(crate) fn event_sink_task_identifiers_for_download(&self,- download_id: DownloadId,+ download_id: crate::DownloadId,) -> Vec<u64> {self.event_registry.lock()@@ -95,6 +107,14 @@ impl AppleBackendContext {.unwrap_or_default()}+ async fn find_download_task(+ &self,+ config: &DownloadConfig,+ ) -> Result<Option<Retained<NSURLSessionDownloadTask>>, AppleBackendError> {+ let download_tasks = self.download_tasks().await?;+ Ok(select_matching_download_task(download_tasks, config))+ }+async fn download_tasks(&self) -> Result<Box<[Retained<NSURLSessionDownloadTask>]>, AppleBackendError> {let (download_tasks_sender, download_tasks_receiver) = tokio_oneshot_channel();let pending_download_tasks_sender = Arc::new(Mutex::new(Some(download_tasks_sender)));@@ -139,9 +159,7 @@ impl AppleBackendContext {fn select_matching_download_task(download_tasks: Box<[Retained<NSURLSessionDownloadTask>]>,config: &DownloadConfig,-) -> Result<Option<Retained<NSURLSessionDownloadTask>>, AppleBackendError> {- let authorization = authorization_header(&config.request)?;- let authorization_field = NSString::from_str(AUTHORIZATION.as_str());+) -> Option<Retained<NSURLSessionDownloadTask>> {let mut live_match = None;for task in download_tasks {if task.download_id() != Some(config.download_id) {@@ -152,14 +170,9 @@ fn select_matching_download_task(continue;}let info = task.download_info();- let source_url_matches = info.as_ref().map(|info| info.source_url == config.request.url).unwrap_or(false);- let file_check_matches = info.as_ref().is_some_and(|info| info.resolved_file_check() == config.file_check);- let authorization_matches = task- .originalRequest()- .and_then(|request| request.valueForHTTPHeaderField(&authorization_field))- .map(|value| value.to_string())- == authorization;- if !(source_url_matches && file_check_matches && authorization_matches) {+ let source_url_matches = info.as_ref().map(|info| info.source_url == config.source_url).unwrap_or(false);+ let crc_matches = info.as_ref().map(|info| info.crc32c == config.file_check.expected_crc()).unwrap_or(false);+ if !(source_url_matches && crc_matches) {task.cancel();continue;}@@ -169,7 +182,7 @@ fn select_matching_download_task(task.cancel();}}- Ok(live_match)+ live_match}fn is_live_task_state(state: NSURLSessionTaskState) -> bool {@@ -204,16 +217,11 @@ impl BackendContext for AppleBackendContext {backend_event_sender: BackendEventSender,_destination_lease: &DestinationLockLease,) -> Result<AppleActiveTask, AppleBackendError> {- let request = apple_request(&config)?;- let task = self.session.downloadTaskWithRequest(&request);+ let ns_url = NSURL::URLWithString(&NSString::from_str(&config.source_url)).ok_or(AppleBackendError::BadUrl)?;+ let task = self.session.downloadTaskWithURL(&ns_url);self.prepare_task(&task, Arc::clone(&config), generation, backend_event_sender);task.resume();- Ok(AppleActiveTask::new(- task,- Arc::clone(&self.event_registry),- config.download_id,- config.request.is_authenticated(),- ))+ Ok(AppleActiveTask::new(task, Arc::clone(&self.event_registry), config.download_id))}async fn resume(@@ -222,23 +230,21 @@ impl BackendContext for AppleBackendContext {generation: ActiveDownloadGeneration,resume_artifact_path: &Path,backend_event_sender: BackendEventSender,- destination_lease: &DestinationLockLease,+ _destination_lease: &DestinationLockLease,) -> Result<AppleActiveTask, AppleBackendError> {- if config.request.is_authenticated() {- return self.download(config, generation, backend_event_sender, destination_lease).await;- }let resume_data =- tokio_read(resume_artifact_path).await.map_err(|error| AppleBackendError::Io(error.to_string()))?;+ tokio::fs::read(resume_artifact_path).await.map_err(|error| AppleBackendError::Io(error.to_string()))?;let task = if resume_data.is_empty() {- let request = apple_request(&config)?;- self.session.downloadTaskWithRequest(&request)+ let ns_url =+ NSURL::URLWithString(&NSString::from_str(&config.source_url)).ok_or(AppleBackendError::BadUrl)?;+ self.session.downloadTaskWithURL(&ns_url)} else {let ns_data = NSData::with_bytes(&resume_data);self.session.downloadTaskWithResumeData(&ns_data)};self.prepare_task(&task, Arc::clone(&config), generation, backend_event_sender);task.resume();- Ok(AppleActiveTask::new(task, Arc::clone(&self.event_registry), config.download_id, false))+ Ok(AppleActiveTask::new(task, Arc::clone(&self.event_registry), config.download_id))}}@@ -250,11 +256,16 @@ impl AppleBackendContext {generation: ActiveDownloadGeneration,backend_event_sender: BackendEventSender,) {- let download_info = DownloadInfo::new(- config.request.url.clone(),- config.destination.to_string_lossy().to_string(),- config.file_check.clone(),- );+ let download_info = match &config.file_check {+ FileCheck::CRC(crc) => DownloadInfo::with_crc(+ config.source_url.clone(),+ config.destination.to_string_lossy().to_string(),+ crc.clone(),+ ),+ FileCheck::None => {+ DownloadInfo::new(config.source_url.clone(), config.destination.to_string_lossy().to_string())+ },+ };task.set_download_info(&download_info);if let Ok(mut registry) = self.event_registry.lock() {registry.insert(@@ -262,7 +273,6 @@ impl AppleBackendContext {AppleEventSink {generation,destination: config.destination.clone(),- expected_bytes: config.expected_bytes,backend_event_sender,runtime_handle: self.runtime_handle.clone(),},@@ -270,26 +280,3 @@ impl AppleBackendContext {}}}--fn apple_request(config: &DownloadConfig) -> Result<Retained<NSMutableURLRequest>, AppleBackendError> {- let url = NSURL::URLWithString(&NSString::from_str(&config.request.url)).ok_or(AppleBackendError::BadUrl)?;- let request = NSMutableURLRequest::requestWithURL(&url);- if let Some(authorization) = authorization_header(&config.request)? {- request.setValue_forHTTPHeaderField(- Some(&NSString::from_str(&authorization)),- &NSString::from_str(AUTHORIZATION.as_str()),- );- }- request.setValue_forHTTPHeaderField(- Some(&NSString::from_str("identity")),- &NSString::from_str(ACCEPT_ENCODING.as_str()),- );- Ok(request)-}--fn authorization_header(request: &HttpDownloadRequest) -> Result<Option<String>, AppleBackendError> {- request- .bearer_token()- .map(|token| token.map(|token| format!("Bearer {token}")))- .map_err(|error| AppleBackendError::Authentication(error.to_string()))-}diff --git a/crates/legacy/download-manager/src/backends/apple/delegate.rs b/crates/legacy/download-manager/src/backends/apple/delegate.rsindex 223f8bf4b..2f272c8e7 100644--- a/crates/legacy/download-manager/src/backends/apple/delegate.rs+++ b/crates/legacy/download-manager/src/backends/apple/delegate.rs@@ -1,14 +1,9 @@use std::{collections::HashMap,- fs::{copy, create_dir_all, remove_file, rename},- io::Error as IoError,path::PathBuf,- ptr::{from_ref, null_mut},sync::{Arc, Mutex},};-use block2::DynBlock;-use http::{StatusCode, uri::Scheme};use kiban::rt::RuntimeHandle;use objc2::{ClassType, DefinedClass, define_class, msg_send,@@ -16,11 +11,9 @@ use objc2::{runtime::ProtocolObject,};use objc2_foundation::{- NSError, NSHTTPURLResponse, NSMutableCopying, NSObject, NSObjectProtocol, NSString, NSURL, NSURLRequest,- NSURLSession, NSURLSessionDelegate, NSURLSessionDownloadDelegate, NSURLSessionDownloadTask, NSURLSessionTask,- NSURLSessionTaskDelegate,+ NSError, NSObject, NSObjectProtocol, NSURL, NSURLSession, NSURLSessionDelegate, NSURLSessionDownloadDelegate,+ NSURLSessionDownloadTask, NSURLSessionTask, NSURLSessionTaskDelegate,};-use reqwest::Url;use crate::{DownloadId,@@ -33,22 +26,10 @@ use crate::{pub struct AppleEventSink {pub generation: ActiveDownloadGeneration,pub destination: PathBuf,- pub expected_bytes: Option<u64>,pub backend_event_sender: BackendEventSender,pub runtime_handle: RuntimeHandle,}-impl AppleEventSink {- fn send_terminal(- self,- event: BackendEvent,- ) {- self.runtime_handle.clone().spawn(async move {- let _ = self.backend_event_sender.send_terminal(event).await;- });- }-}-pub type AppleSinkKey = (DownloadId, u64);pub type AppleEventRegistry = Arc<Mutex<HashMap<AppleSinkKey, AppleEventSink>>>;@@ -65,48 +46,17 @@ define_class!(unsafe impl NSObjectProtocol for AppleSessionDelegate {}- unsafe impl NSURLSessionDelegate for AppleSessionDelegate {}-- unsafe impl NSURLSessionTaskDelegate for AppleSessionDelegate {- #[unsafe(method(URLSession:task:willPerformHTTPRedirection:newRequest:completionHandler:))]- unsafe fn will_perform_http_redirection(+ unsafe impl NSURLSessionDelegate for AppleSessionDelegate {+ #[unsafe(method(URLSession:didBecomeInvalidWithError:))]+ fn did_become_invalid_with_error(&self,_session: &NSURLSession,- _task: &NSURLSessionTask,- response: &NSHTTPURLResponse,- request: &NSURLRequest,- completion_handler: &DynBlock<dyn Fn(*mut NSURLRequest)>,+ _error: Option<&NSError>,) {- let current_url =- response.URL().and_then(|url| url.absoluteString()).and_then(|url| Url::parse(&url.to_string()).ok());- let next_url =- request.URL().and_then(|url| url.absoluteString()).and_then(|url| Url::parse(&url.to_string()).ok());- let redirected_request = match (current_url, next_url) {- (Some(current), Some(next)) => {- match (Scheme::try_from(current.scheme()), Scheme::try_from(next.scheme())) {- (Ok(current_scheme), Ok(next_scheme))- if (current_scheme == Scheme::HTTP- && (next_scheme == Scheme::HTTP || next_scheme == Scheme::HTTPS))- || (current_scheme == Scheme::HTTPS && next_scheme == Scheme::HTTPS) =>- {- if current.origin() == next.origin() {- from_ref(request).cast_mut()- } else {- let redirected = request.mutableCopy();- redirected.setValue_forHTTPHeaderField(None, &NSString::from_str("Authorization"));- let pointer: *const _ = &*redirected;- completion_handler.call((pointer.cast::<NSURLRequest>().cast_mut(),));- return;- }- },- _ => null_mut(),- }- },- _ => null_mut(),- };- completion_handler.call((redirected_request,));}+ }+ unsafe impl NSURLSessionTaskDelegate for AppleSessionDelegate {#[unsafe(method(URLSession:task:didCompleteWithError:))]fn did_complete_with_error(&self,@@ -130,8 +80,9 @@ define_class!(return;};let message = error.localizedDescription().to_string();- let event = BackendEvent::error(sink.generation, message);- sink.send_terminal(event);+ sink.runtime_handle.clone().spawn(async move {+ let _ = sink.backend_event_sender.send_terminal(BackendEvent::error(sink.generation, message)).await;+ });}}@@ -152,42 +103,27 @@ define_class!(else {return;};- let status_code = download_task- .response()- .and_then(|response| response.downcast_ref::<NSHTTPURLResponse>().map(|response| response.statusCode()))- .and_then(|status_code| u16::try_from(status_code).ok())- .and_then(|status_code| StatusCode::from_u16(status_code).ok());- if !status_code.is_some_and(|status_code| status_code.is_success()) {- let message = status_code.map_or_else(- || "download response was not HTTP".to_string(),- |status_code| format!("download failed with HTTP status {}", status_code.as_u16()),- );- let event = BackendEvent::error(sink.generation, message);- sink.send_terminal(event);- return;- }let Some(temporary_path) = location.path().map(|path| PathBuf::from(path.to_string())) else {- let event =- BackendEvent::error(sink.generation, "download temporary location was unavailable".to_string());- sink.send_terminal(event);return;};// Move synchronously: NSURLSession deletes the temp file as soon as this method returns.if let Some(parent) = sink.destination.parent() {- let _ = create_dir_all(parent);+ let _ = std::fs::create_dir_all(parent);}- let move_result = rename(&temporary_path, &sink.destination).or_else(|_| {- copy(&temporary_path, &sink.destination)?;- let _ = remove_file(&temporary_path);- Ok::<(), IoError>(())+ let move_result = std::fs::rename(&temporary_path, &sink.destination).or_else(|_| {+ std::fs::copy(&temporary_path, &sink.destination)?;+ let _ = std::fs::remove_file(&temporary_path);+ Ok::<(), std::io::Error>(())});let terminal_event = match move_result {Ok(()) => BackendEvent::completed(sink.generation),Err(error) => BackendEvent::error(sink.generation, format!("move into destination failed: {error}")),};- sink.send_terminal(terminal_event);+ sink.runtime_handle.clone().spawn(async move {+ let _ = sink.backend_event_sender.send_terminal(terminal_event).await;+ });}#[unsafe(method(URLSession:downloadTask:didWriteData:totalBytesWritten:totalBytesExpectedToWrite:))]@@ -208,22 +144,8 @@ define_class!(else {return;};- let downloaded_bytes = u64::try_from(cumulative_bytes_written).unwrap_or_default();- let total_bytes = u64::try_from(total_expected_bytes_to_write).ok().filter(|total_bytes| *total_bytes > 0);- if let Some(expected_bytes) = sink.expected_bytes- && downloaded_bytes > expected_bytes- {- download_task.cancel();- if let Ok(mut registry) = Self::ivars(self).event_registry.lock() {- registry.remove(&key);- }- let event = BackendEvent::error(- sink.generation,- format!("response exceeded the registry size of {expected_bytes} bytes"),- );- sink.send_terminal(event);- return;- }+ let downloaded_bytes = cumulative_bytes_written.max(0) as u64;+ let total_bytes = (total_expected_bytes_to_write > 0).then_some(total_expected_bytes_to_write as u64);sink.runtime_handle.clone().spawn(async move {sink.backend_event_sender.send_progress(sink.generation, downloaded_bytes, total_bytes).await;diff --git a/crates/legacy/download-manager/src/backends/apple/error.rs b/crates/legacy/download-manager/src/backends/apple/error.rsindex fcd52d16b..3efbc188f 100644--- a/crates/legacy/download-manager/src/backends/apple/error.rs+++ b/crates/legacy/download-manager/src/backends/apple/error.rs@@ -8,6 +8,4 @@ pub enum AppleBackendError {Io(String),#[error("task enumeration error: {0}")]TaskEnumeration(String),- #[error("authentication error: {0}")]- Authentication(String),}diff --git a/crates/legacy/download-manager/src/backends/apple/task_ext.rs b/crates/legacy/download-manager/src/backends/apple/task_ext.rsindex f5c71c1d7..2b50fca55 100644--- a/crates/legacy/download-manager/src/backends/apple/task_ext.rs+++ b/crates/legacy/download-manager/src/backends/apple/task_ext.rs@@ -45,12 +45,12 @@ impl AppleDownloadTaskExt for NSURLSessionDownloadTask {fn count_of_bytes_expected_to_receive(&self) -> u64 {let bytes_expected: i64 = unsafe { msg_send![self, countOfBytesExpectedToReceive] };- u64::try_from(bytes_expected).unwrap_or_default()+ bytes_expected.max(0) as u64}fn count_of_bytes_received(&self) -> u64 {let bytes_received: i64 = unsafe { msg_send![self, countOfBytesReceived] };- u64::try_from(bytes_received).unwrap_or_default()+ bytes_received.max(0) as u64}fn task_identifier(&self) -> u64 {diff --git a/crates/legacy/download-manager/src/backends/common/action_executor.rs b/crates/legacy/download-manager/src/backends/common/action_executor.rsindex f24b144d9..19f836b01 100644--- a/crates/legacy/download-manager/src/backends/common/action_executor.rs+++ b/crates/legacy/download-manager/src/backends/common/action_executor.rs@@ -1,10 +1,10 @@-use std::{io::ErrorKind, path::Path};+use std::path::Path;use kiban::fs;use crate::{DownloadError,- integrity::save_integrity_cache,+ crc_utils::save_crc_file,lock_manager::DestinationLockLease,reducer::{Action, ActionPlan},};@@ -18,7 +18,7 @@ pub async fn apply_actions(Action::DeleteFile {path,}- | Action::DeleteIntegrityCache {+ | Action::DeleteCrcCache {path,}| Action::DeleteResumeArtifact {@@ -26,11 +26,11 @@ pub async fn apply_actions(} => {remove_file_if_present(path).await?;},- Action::SaveIntegrityCache {+ Action::SaveCrcCache {destination,- file_check,+ crc,} => {- save_integrity_cache(destination, file_check).await?;+ save_crc_file(destination, crc).await?;},}}@@ -41,7 +41,7 @@ pub async fn apply_actions(async fn remove_file_if_present(path: &Path) -> Result<(), DownloadError> {match fs::asyn::remove_file(path).await {Ok(()) => Ok(()),- Err(error) if error.kind() == ErrorKind::NotFound => Ok(()),+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),Err(error) => Err(DownloadError::from(error)),}}diff --git a/crates/legacy/download-manager/src/backends/common/manager.rs b/crates/legacy/download-manager/src/backends/common/manager.rsindex bcc918ad5..423a9d668 100644--- a/crates/legacy/download-manager/src/backends/common/manager.rs+++ b/crates/legacy/download-manager/src/backends/common/manager.rs@@ -1,12 +1,10 @@-use std::{io::ErrorKind, path::Path, sync::Arc};+use std::sync::Arc;use kiban::rt::RuntimeHandle;use tokio_stream::wrappers::BroadcastStream as TokioBroadcastStream;-use uuid::Uuid;use crate::{- DownloadError, DownloadEvent, DownloadId, FileCheck, FileDownloadManager, FileDownloadTask, HttpDownloadRequest,- LockFileState, SharedDownloadEventSender,+ DownloadError, DownloadEvent, FileCheck, FileDownloadManager, FileDownloadTask, LockFileState,backends::common::{Backend, DownloadManagerState, Startup},check_lock_file, compute_download_id,download_log_event::{DownloadLogEvent, log},@@ -47,7 +45,7 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {self.state.subscribe_to_all_downloads()}- fn global_broadcast_sender(&self) -> SharedDownloadEventSender {+ fn global_broadcast_sender(&self) -> crate::SharedDownloadEventSender {self.state.global_broadcast_sender()}@@ -57,7 +55,7 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {async fn remove_file_task(&self,- download_id: DownloadId,+ download_id: crate::DownloadId,) -> Result<(), DownloadError> {let construction_lock = self.state.construction_lock(download_id).await;let _construction_guard = construction_lock.lock().await;@@ -74,8 +72,8 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {async fn file_download_task(&self,- request: HttpDownloadRequest,- destination_path: &Path,+ source_url: &str,+ destination_path: &std::path::Path,file_check: FileCheck,expected_bytes: Option<u64>,) -> Result<Arc<dyn FileDownloadTask>, DownloadError> {@@ -84,7 +82,7 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {&& !cached_task.is_stopped(){let task = cached_task.public();- ensure_cached_task_matches(&cached_task, &task, &request, &file_check, expected_bytes)?;+ ensure_cached_task_matches(&task, source_url, &file_check, expected_bytes)?;return Ok(task);}@@ -94,7 +92,7 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {let cached_task_result = match self.state.get_task(download_id).await {Some(cached_task) if !cached_task.is_stopped() => {let task = cached_task.public();- ensure_cached_task_matches(&cached_task, &task, &request, &file_check, expected_bytes)?;+ ensure_cached_task_matches(&task, source_url, &file_check, expected_bytes)?;Some(Ok(task))},Some(_) => {@@ -108,7 +106,7 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {} else {let startup = observe_startup::<B>(download_id,- request.clone(),+ source_url,destination_path,file_check.clone(),expected_bytes,@@ -120,7 +118,7 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {startup,self.context.as_ref(),download_id,- request.clone(),+ source_url,destination_path,file_check,expected_bytes,@@ -152,10 +150,7 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {let public_task: Arc<dyn FileDownloadTask> = task.clone();let managed_task = task;self.state- .insert_task(- download_id,- CachedFileDownloadTask::new(Arc::clone(&public_task), managed_task, request),- )+ .insert_task(download_id, CachedFileDownloadTask::new(Arc::clone(&public_task), managed_task)).await;Ok(public_task)}@@ -169,7 +164,7 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {async fn destination_foreign_lock(&self,- destination_path: &Path,+ destination_path: &std::path::Path,) -> Option<String> {let lock_path = lock_path_for_destination(destination_path);match check_lock_file(&lock_path, &self.state.manager_id, self.state.instance_id, kiban::process::id()).await {@@ -180,17 +175,17 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {}pub(crate) async fn observe_startup<B: Backend>(- download_id: DownloadId,- request: HttpDownloadRequest,- destination_path: &Path,+ download_id: crate::DownloadId,+ source_url: &str,+ destination_path: &std::path::Path,file_check: FileCheck,expected_bytes: Option<u64>,manager_id: &str,- manager_instance_id: Uuid,+ manager_instance_id: uuid::Uuid,) -> Result<Startup, DownloadError> {Startup::observe::<B>(download_id,- request,+ source_url,destination_path,file_check,expected_bytes,@@ -203,13 +198,13 @@ pub(crate) async fn observe_startup<B: Backend>(pub(crate) async fn prepare_startup<B: Backend>(startup: Startup,context: &B::Context,- download_id: DownloadId,- request: HttpDownloadRequest,- destination_path: &Path,+ download_id: crate::DownloadId,+ source_url: &str,+ destination_path: &std::path::Path,file_check: FileCheck,expected_bytes: Option<u64>,manager_id: &str,- manager_instance_id: Uuid,+ manager_instance_id: uuid::Uuid,) -> Result<(Startup, Option<DestinationLockLease>), DownloadError> {if !startup_requires_destination_lease::<B>(&startup, context).await? {return Ok((startup, None));@@ -218,10 +213,10 @@ pub(crate) async fn prepare_startup<B: Backend>(let lease =match DestinationLockLease::acquire_for_destination(destination_path, manager_id, manager_instance_id).await {Ok(lease) => lease,- Err(error) if error.kind() == ErrorKind::AlreadyExists => {+ Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {let startup = observe_startup::<B>(download_id,- request,+ source_url,destination_path,file_check,expected_bytes,@@ -238,7 +233,7 @@ pub(crate) async fn prepare_startup<B: Backend>(};let startup = match observe_startup::<B>(download_id,- request,+ source_url,destination_path,file_check,expected_bytes,@@ -289,29 +284,30 @@ fn startup_can_attach_initial_task<B: Backend>(startup: &Startup) -> bool {}fn ensure_cached_task_matches(- cached: &CachedFileDownloadTask,- task: &Arc<dyn FileDownloadTask>,- request: &HttpDownloadRequest,+ cached: &Arc<dyn FileDownloadTask>,+ source_url: &str,file_check: &FileCheck,expected_bytes: Option<u64>,) -> Result<(), DownloadError> {- if !cached.request_matches(request) {+ if cached.source_url() != source_url {return Err(DownloadError::ConflictingConfig(format!(- "{} already requested with a different HTTP request",- task.destination().display(),+ "{} already requested with source_url {:?}; cannot reuse for {:?}",+ cached.destination().display(),+ cached.source_url(),+ source_url,)));}- if task.file_check() != file_check {+ if cached.file_check() != file_check {return Err(DownloadError::ConflictingConfig(format!("{} already requested with a different file_check",- task.destination().display(),+ cached.destination().display(),)));}- if task.expected_bytes() != expected_bytes {+ if cached.expected_bytes() != expected_bytes {return Err(DownloadError::ConflictingConfig(format!("{} already requested with expected_bytes {:?}; got {:?}",- task.destination().display(),- task.expected_bytes(),+ cached.destination().display(),+ cached.expected_bytes(),expected_bytes,)));}diff --git a/crates/legacy/download-manager/src/backends/common/startup.rs b/crates/legacy/download-manager/src/backends/common/startup.rsindex 9aca42d10..f8629828e 100644--- a/crates/legacy/download-manager/src/backends/common/startup.rs+++ b/crates/legacy/download-manager/src/backends/common/startup.rs@@ -1,14 +1,13 @@use std::{path::Path, sync::Arc};use kiban::fs;-use uuid::Uuid;use crate::{- DownloadError, DownloadId, FileCheck, FileState, HttpDownloadRequest, LockFileState,+ DownloadError, DownloadId, FileCheck, FileState, LockFileState,backends::common::{Backend, action_executor::apply_actions},check_lock_file,+ crc_utils::crc_path_for_file,file_download_task_actor::{ProgressCounters, PublicProjection},- integrity::integrity_receipt_path,lock_manager::{DestinationLockLease, lock_path_for_destination},reducer::{ActionPlan, DiskObservation, InitialLifecycleState, LockObservation, decide, validate},traits::DownloadConfig,@@ -27,16 +26,19 @@ pub struct Startup {impl Startup {pub async fn observe<B: Backend>(download_id: DownloadId,- request: HttpDownloadRequest,+ source_url: &str,destination_path: &Path,file_check: FileCheck,expected_bytes: Option<u64>,manager_id: &str,- manager_instance_id: Uuid,+ manager_instance_id: uuid::Uuid,) -> Result<Self, DownloadError> {- request.validate()?;- let resume_artifact_path = destination_path.with_added_extension(B::RESUME_ARTIFACT_EXTENSION);- let integrity_path = integrity_receipt_path(destination_path);+ let resume_artifact_path = destination_path.with_extension(B::RESUME_ARTIFACT_EXTENSION);+ let expected_crc = match &file_check {+ FileCheck::CRC(crc) => Some(crc.clone()),+ FileCheck::None => None,+ };+ let crc_path = crc_path_for_file(destination_path);let resume_state = file_state(&resume_artifact_path).await;let resume_size = match resume_state {FileState::Exists => B::read_resume_progress(&resume_artifact_path).await,@@ -44,14 +46,14 @@ impl Startup {};let observation = DiskObservation {destination_state: file_state(destination_path).await,- integrity_state: file_state(&integrity_path).await,+ crc_state: file_state(&crc_path).await,resume_state,destination_size: fs::asyn::file_length(destination_path).await.ok(),resume_size,- file_check: file_check.clone(),+ expected_crc,expected_bytes,destination_path: destination_path.to_path_buf(),- integrity_path,+ crc_path: Some(crc_path),resume_artifact_path: Some(resume_artifact_path),};let lock_state = check_lock_file(@@ -64,11 +66,11 @@ impl Startup {let lock_observation = LockObservation {state: lock_state.clone(),};- let validation = validate(&observation).await?;+ let validation = validate(&observation).await;let decision = decide(&observation, &lock_observation, &validation);let config = Arc::new(DownloadConfig {download_id,- request,+ source_url: source_url.to_string(),destination: destination_path.to_path_buf(),file_check,expected_bytes,diff --git a/crates/legacy/download-manager/src/backends/universal/context.rs b/crates/legacy/download-manager/src/backends/universal/context.rsindex ef996a79c..361b62e38 100644--- a/crates/legacy/download-manager/src/backends/universal/context.rs+++ b/crates/legacy/download-manager/src/backends/universal/context.rs@@ -5,18 +5,13 @@ use std::{};use futures_util::StreamExt;-use http::StatusCode;-#[cfg(not(target_family = "wasm"))]-use http::uri::Scheme;use kiban::{fs, fs::PartFile, rt::RuntimeHandle, time::Instant};use reqwest::{- Client,- header::{CONTENT_RANGE, RANGE},+ StatusCode,+ header::{CONTENT_LENGTH, CONTENT_RANGE, RANGE},};-#[cfg(not(target_family = "wasm"))]-use reqwest::{header::ACCEPT_ENCODING, redirect::Policy};use tokio::sync::{- oneshot::{Sender as TokioOneshotSender, channel as tokio_oneshot_channel},+ oneshot::channel as tokio_oneshot_channel,watch::{Receiver as TokioWatchReceiver, channel as tokio_watch_channel},};@@ -56,7 +51,7 @@ impl BackendContext for UniversalBackendContext {backend_event_sender: BackendEventSender,_destination_lease: &DestinationLockLease,) -> Result<UniversalActiveTask, UniversalBackendError> {- let resume_artifact_path = config.destination.with_added_extension("part");+ let resume_artifact_path = config.destination.with_extension("part");self.start(config, generation, resume_artifact_path, backend_event_sender).await}@@ -117,7 +112,7 @@ async fn download_streaming(retry_count: u16,progress_interval: Duration,pause_receiver: TokioWatchReceiver<bool>,- completion_sender: TokioOneshotSender<()>,+ completion_sender: tokio::sync::oneshot::Sender<()>,) {let mut pause_receiver = pause_receiver;let result = download_streaming_with_retries(@@ -152,26 +147,9 @@ async fn download_streaming_with_retries(progress_interval: Duration,pause_receiver: &mut TokioWatchReceiver<bool>,) -> Result<DownloadStreamCompletion, String> {- let client = Client::builder();- #[cfg(not(target_family = "wasm"))]- let client = client.redirect(Policy::custom(|attempt| {- let current_scheme = attempt.previous().last().and_then(|url| Scheme::try_from(url.scheme()).ok());- let next_scheme = Scheme::try_from(attempt.url().scheme()).ok();- let downgrade = matches!(- (current_scheme, next_scheme),- (Some(current), Some(next)) if current == Scheme::HTTPS && next != Scheme::HTTPS- );- if downgrade || attempt.previous().len() >= 10 {- attempt.error("unsafe redirect")- } else {- attempt.follow()- }- }));- let client = client.build().map_err(|error| error.to_string())?;let mut attempt = 0_u16;loop {match download_once(- &client,Arc::clone(&config),generation,resume_artifact_path,@@ -193,7 +171,6 @@ async fn download_streaming_with_retries(}async fn download_once(- client: &Client,config: Arc<DownloadConfig>,generation: ActiveDownloadGeneration,resume_artifact_path: &Path,@@ -201,33 +178,33 @@ async fn download_once(progress_interval: Duration,pause_receiver: &mut TokioWatchReceiver<bool>,) -> Result<DownloadStreamCompletion, String> {+ let client = reqwest::Client::new();let resume_from_bytes = fs::asyn::file_length(resume_artifact_path).await.unwrap_or(0);- let request = config.request.apply(client.get(&config.request.url)).map_err(|error| error.to_string())?;- #[cfg(not(target_family = "wasm"))]- let request = request.header(ACCEPT_ENCODING, "identity");- let mut request = request;+ let mut request = client.get(&config.source_url);if resume_from_bytes > 0 {request = request.header(RANGE, format!("bytes={resume_from_bytes}-"));}let response = tokio::select! {_ = wait_for_pause(pause_receiver) => return Ok(DownloadStreamCompletion::Paused),- response = request.send() => response.map_err(|error| error.without_url().to_string())?,+ response = request.send() => response.map_err(|error| error.to_string())?,};let status = response.status();let content_range = response.headers().get(CONTENT_RANGE).cloned();let resume_from_bytes = if resume_from_bytes > 0 {match status {StatusCode::PARTIAL_CONTENT => {- let validation = content_range+ let header = content_range.as_ref()- .ok_or_else(|| "server returned 206 without Content-Range header".to_string())- .and_then(|header| {- header.to_str().map_err(|error| format!("non-utf8 Content-Range header: {error}"))- })- .and_then(|header| validate_content_range(header, resume_from_bytes, config.expected_bytes));- if let Err(message) = validation {- return Err(discard_invalid_resume(resume_artifact_path, message).await);+ .ok_or_else(|| "server returned 206 without Content-Range header".to_string())?;+ let header_value =+ header.to_str().map_err(|error| format!("non-utf8 Content-Range header: {error}"))?;+ let advertised_start = parse_content_range_start(header_value)+ .ok_or_else(|| format!("server returned malformed Content-Range header: {header_value}"))?;+ if advertised_start != resume_from_bytes {+ return Err(format!(+ "server returned bytes starting at {advertised_start} but client requested {resume_from_bytes}"+ ));}resume_from_bytes},@@ -250,8 +227,12 @@ async fn download_once(} else {resume_from_bytes};- let response = response.error_for_status().map_err(|error| error.without_url().to_string())?;- let remaining_bytes = response.content_length();+ let response = response.error_for_status().map_err(|error| error.to_string())?;+ let remaining_bytes = response+ .headers()+ .get(CONTENT_LENGTH)+ .and_then(|value| value.to_str().ok())+ .and_then(|value| value.parse::<u64>().ok());let total_bytes = remaining_bytes.map(|remaining_bytes| remaining_bytes.saturating_add(resume_from_bytes)).or(config.expected_bytes);@@ -275,17 +256,9 @@ async fn download_once(let Some(chunk) = chunk else {break;};- let chunk = chunk.map_err(|error| error.without_url().to_string())?;- let chunk_len = u64::try_from(chunk.len()).map_err(|_| "response chunk length overflow".to_string())?;- let next_downloaded_bytes =- downloaded_bytes.checked_add(chunk_len).ok_or_else(|| "downloaded byte count overflow".to_string())?;- if let Some(expected_bytes) = config.expected_bytes- && next_downloaded_bytes > expected_bytes- {- return Err(format!("response exceeded the registry size of {expected_bytes} bytes"));- }+ let chunk = chunk.map_err(|error| error.to_string())?;file.write_all(&chunk).await.map_err(|error| error.to_string())?;- downloaded_bytes = next_downloaded_bytes;+ downloaded_bytes = downloaded_bytes.saturating_add(chunk.len() as u64);if last_progress_emit.elapsed() >= progress_interval {backend_event_sender.send_progress(generation, downloaded_bytes, total_bytes).await;@@ -308,34 +281,11 @@ async fn wait_for_pause(pause_receiver: &mut TokioWatchReceiver<bool>) {let _ = pause_receiver.changed().await;}-async fn discard_invalid_resume(- resume_artifact_path: &Path,- message: impl Into<String>,-) -> String {- let _ = fs::asyn::remove_file(resume_artifact_path).await;- message.into()-}--fn validate_content_range(- header: &str,- requested_start: u64,- expected_total: Option<u64>,-) -> Result<(), String> {- let value = header.strip_prefix("bytes ").ok_or_else(|| format!("malformed Content-Range: {header}"))?;- let (range, _) = value.split_once('/').ok_or_else(|| format!("malformed Content-Range: {header}"))?;- let (start, _) = range.split_once('-').ok_or_else(|| format!("malformed Content-Range: {header}"))?;- let start = start.parse::<u64>().map_err(|_| format!("malformed Content-Range: {header}"))?;- if start != requested_start {- return Err(format!("server returned bytes starting at {start} but client requested {requested_start}"));- }- if let Some(expected) = expected_total {- let total = parse_content_range_total(header)- .ok_or_else(|| format!("server returned Content-Range without a total: {header}"))?;- if total != expected {- return Err(format!("server advertised {total} bytes but registry declared {expected}"));- }- }- Ok(())+fn parse_content_range_start(header_value: &str) -> Option<u64> {+ let value = header_value.strip_prefix("bytes ")?.trim_start();+ let (range, _) = value.split_once('/')?;+ let (start, _) = range.split_once('-')?;+ start.parse::<u64>().ok()}fn parse_content_range_total(header_value: &str) -> Option<u64> {diff --git a/crates/legacy/download-manager/src/crc_utils.rs b/crates/legacy/download-manager/src/crc_utils.rsnew file mode 100644index 000000000..12583efbd--- /dev/null+++ b/crates/legacy/download-manager/src/crc_utils.rs@@ -0,0 +1,111 @@+use std::{+ io::{Error as IoError, ErrorKind},+ path::{Path, PathBuf},+};++use base64::Engine;+use kiban::{fs, time::SystemTime};+use serde::{Deserialize, Serialize};++const CRC_CACHE_VERSION: u8 = 1;+const CRC_READ_CHUNK_SIZE: u64 = 8 * 1024 * 1024;++#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)]+struct CrcCacheReceipt {+ version: u8,+ crc: String,+ file_size: u64,+ modified_unix_seconds: u64,+ modified_nanos: u32,+}++pub async fn calculate_and_verify_crc(+ file_path: &Path,+ expected_crc32c_base64: &str,+) -> Result<bool, IoError> {+ let Some(expected_crc) = decode_crc32c(expected_crc32c_base64) else {+ return Ok(false);+ };++ let file_len = kiban::fs::asyn::file_length(file_path).await?;+ let mut offset = 0;+ let mut actual_crc = 0;+ while offset < file_len {+ let end = offset.saturating_add(CRC_READ_CHUNK_SIZE).min(file_len);+ let chunk = kiban::fs::asyn::read_range(file_path, offset..end).await?;+ if chunk.len() != (end - offset) as usize {+ return Err(IoError::new(ErrorKind::UnexpectedEof, "file changed during CRC verification"));+ }+ actual_crc = crc32c::crc32c_append(actual_crc, &chunk);+ offset = end;+ }++ Ok(actual_crc == expected_crc)+}++pub async fn crc_cache_matches(+ file_path: &Path,+ expected_crc32c_base64: &str,+) -> bool {+ if decode_crc32c(expected_crc32c_base64).is_none() {+ return false;+ }++ let Ok(cache_contents) = kiban::fs::asyn::read_to_string(crc_path_for_file(file_path)).await else {+ return false;+ };+ let Ok(cached_receipt) = serde_json::from_str::<CrcCacheReceipt>(&cache_contents) else {+ return false;+ };+ let Some(current_receipt) = CrcCacheReceipt::from_file(file_path, expected_crc32c_base64).await else {+ return false;+ };++ cached_receipt == current_receipt+}++pub async fn save_crc_file(+ file_path: &Path,+ crc_value: &str,+) -> Result<(), IoError> {+ let Some(receipt) = CrcCacheReceipt::from_file(file_path, crc_value).await else {+ return Ok(());+ };+ let receipt_json = serde_json::to_vec(&receipt).map_err(IoError::other)?;+ fs::asyn::write(crc_path_for_file(file_path), receipt_json).await+}++pub fn crc_path_for_file(file_path: &Path) -> PathBuf {+ PathBuf::from(format!("{}.crc", file_path.display()))+}++impl CrcCacheReceipt {+ async fn from_file(+ file_path: &Path,+ crc: &str,+ ) -> Option<Self> {+ if !fs::asyn::is_file(file_path).await {+ return None;+ }++ let file_size = fs::asyn::file_length(file_path).await.ok()?;+ let modified = fs::asyn::file_modified(file_path).await.ok()?.duration_since(SystemTime::UNIX_EPOCH).ok()?;+ Some(Self {+ version: CRC_CACHE_VERSION,+ crc: crc.to_string(),+ file_size,+ modified_unix_seconds: modified.as_secs(),+ modified_nanos: modified.subsec_nanos(),+ })+ }+}++fn decode_crc32c(expected_crc32c_base64: &str) -> Option<u32> {+ let expected_bytes = base64::engine::general_purpose::STANDARD.decode(expected_crc32c_base64).ok()?;++ if expected_bytes.len() != 4 {+ return None;+ }++ Some(u32::from_be_bytes([expected_bytes[0], expected_bytes[1], expected_bytes[2], expected_bytes[3]]))+}diff --git a/crates/legacy/download-manager/src/download_error.rs b/crates/legacy/download-manager/src/download_error.rsindex 792fe68a8..169ce3a23 100644--- a/crates/legacy/download-manager/src/download_error.rs+++ b/crates/legacy/download-manager/src/download_error.rs@@ -1,7 +1,3 @@-use std::io::Error as IoError;--use serde_json::Error as JsonError;-#[derive(thiserror::Error, Clone, Debug, PartialEq, Eq)]pub enum DownloadError {#[error("io error: {0}")]@@ -16,10 +12,6 @@ pub enum DownloadError {ResumeUnsupported,#[error("bad url")]BadUrl,- #[error("authenticated downloads require HTTPS")]- InsecureAuthenticatedRequest,- #[error("download authentication is no longer available")]- AuthenticationUnavailable,#[error("missing download info for task")]MissingDownloadInfo,#[error("resume data read failed")]@@ -48,25 +40,16 @@ pub enum DownloadError {ChannelClosed,#[error("backend error: {0}")]Backend(String),- #[error("invalid expected {algorithm} digest")]- InvalidDigest {- algorithm: &'static str,- },- #[error("integrity verification I/O failed for {path}: {message}")]- IntegrityIo {- path: String,- message: String,- },}-impl From<IoError> for DownloadError {- fn from(error: IoError) -> Self {+impl From<std::io::Error> for DownloadError {+ fn from(error: std::io::Error) -> Self {Self::Io(error.to_string())}}-impl From<JsonError> for DownloadError {- fn from(error: JsonError) -> Self {+impl From<serde_json::Error> for DownloadError {+ fn from(error: serde_json::Error) -> Self {Self::SerdeJson(error.to_string())}}diff --git a/crates/legacy/download-manager/src/download_info.rs b/crates/legacy/download-manager/src/download_info.rsindex 93270a472..6dfa71d44 100644--- a/crates/legacy/download-manager/src/download_info.rs+++ b/crates/legacy/download-manager/src/download_info.rs@@ -1,7 +1,4 @@use serde::{Deserialize, Serialize};-use serde_json::{Error as JsonError, from_str, to_string};--use crate::FileCheck;#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)]pub struct DownloadInfo {@@ -9,34 +6,37 @@ pub struct DownloadInfo {pub destination_path: String,#[serde(default)]pub crc32c: Option<String>,- #[serde(default, skip_serializing_if = "Option::is_none")]- pub file_check: Option<FileCheck>,}impl DownloadInfo {pub fn new<U: Into<String>, P: Into<String>>(source_url: U,destination_path: P,- file_check: FileCheck,) -> Self {- let crc32c = file_check.expected_crc();Self {source_url: source_url.into(),destination_path: destination_path.into(),- crc32c,- file_check: Some(file_check),+ crc32c: None,}}- pub fn resolved_file_check(&self) -> FileCheck {- self.file_check.clone().or_else(|| self.crc32c.clone().map(FileCheck::CRC)).unwrap_or(FileCheck::None)+ pub fn with_crc<U: Into<String>, P: Into<String>, C: Into<String>>(+ source_url: U,+ destination_path: P,+ crc32c: C,+ ) -> Self {+ Self {+ source_url: source_url.into(),+ destination_path: destination_path.into(),+ crc32c: Some(crc32c.into()),+ }}- pub fn to_json(&self) -> Result<String, JsonError> {- to_string(self)+ pub fn to_json(&self) -> Result<String, serde_json::Error> {+ serde_json::to_string(self)}- pub fn from_json(json: &str) -> Result<Self, JsonError> {- from_str(json)+ pub fn from_json(json: &str) -> Result<Self, serde_json::Error> {+ serde_json::from_str(json)}}diff --git a/crates/legacy/download-manager/src/file_check.rs b/crates/legacy/download-manager/src/file_check.rsindex 1c13e8ccc..21857f77a 100644--- a/crates/legacy/download-manager/src/file_check.rs+++ b/crates/legacy/download-manager/src/file_check.rs@@ -3,8 +3,6 @@ use serde::{Deserialize, Serialize};#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)]pub enum FileCheck {CRC(String),- Sha256(String),- GitBlobSha1(String),None,}@@ -12,16 +10,7 @@ impl FileCheck {pub fn expected_crc(&self) -> Option<String> {match self {Self::CRC(crc) => Some(crc.clone()),- Self::Sha256(_) | Self::GitBlobSha1(_) | Self::None => None,- }- }-- pub fn verification_failure_message(&self) -> &'static str {- match self {- Self::CRC(_) => "CRC verification failed",- Self::Sha256(_) => "SHA-256 verification failed",- Self::GitBlobSha1(_) => "Git blob SHA-1 verification failed",- Self::None => "integrity verification failed",+ Self::None => None,}}}diff --git a/crates/legacy/download-manager/src/file_download_manager.rs b/crates/legacy/download-manager/src/file_download_manager.rsindex eed7c1d96..db5554aa4 100644--- a/crates/legacy/download-manager/src/file_download_manager.rs+++ b/crates/legacy/download-manager/src/file_download_manager.rs@@ -7,7 +7,7 @@ use tokio_stream::wrappers::BroadcastStream as TokioBroadcastStream;#[cfg(target_vendor = "apple")]use crate::backends::apple::AppleDownloadManager;use crate::{- DownloadError, DownloadId, FileCheck, FileDownloadEvent, FileDownloadTask, HttpDownloadRequest,+ DownloadError, DownloadId, FileCheck, FileDownloadEvent, FileDownloadTask,backends::universal::UniversalDownloadManager,};@@ -48,7 +48,7 @@ pub trait FileDownloadManager: Send + Sync + 'static {async fn file_download_task(&self,- request: HttpDownloadRequest,+ source_url: &str,destination_path: &Path,file_check: FileCheck,expected_bytes: Option<u64>,diff --git a/crates/legacy/download-manager/src/file_download_task.rs b/crates/legacy/download-manager/src/file_download_task.rsindex b8591d45b..b699588f9 100644--- a/crates/legacy/download-manager/src/file_download_task.rs+++ b/crates/legacy/download-manager/src/file_download_task.rs@@ -3,7 +3,7 @@ use std::{fmt::Debug, path::Path, sync::Arc};use tokio::sync::broadcast::Sender as TokioBroadcastSender;use tokio_stream::wrappers::BroadcastStream as TokioBroadcastStream;-use crate::{DownloadError, DownloadEventSender, DownloadId, FileCheck, FileDownloadState, HttpDownloadRequest};+use crate::{DownloadError, DownloadEventSender, DownloadId, FileCheck, FileDownloadState};#[cfg_attr(not(target_family = "wasm"), async_trait::async_trait)]#[cfg_attr(target_family = "wasm", async_trait::async_trait(?Send))]@@ -41,19 +41,16 @@ pub(crate) trait ManagedFileDownloadTask: FileDownloadTask {pub(crate) struct CachedFileDownloadTask {public: Arc<dyn FileDownloadTask>,managed: Arc<dyn ManagedFileDownloadTask>,- request: HttpDownloadRequest,}impl CachedFileDownloadTask {pub(crate) fn new(public: Arc<dyn FileDownloadTask>,managed: Arc<dyn ManagedFileDownloadTask>,- request: HttpDownloadRequest,) -> Self {Self {public,managed,- request,}}@@ -68,11 +65,4 @@ impl CachedFileDownloadTask {pub(crate) fn is_stopped(&self) -> bool {self.managed.is_stopped()}-- pub fn request_matches(- &self,- request: &HttpDownloadRequest,- ) -> bool {- self.request == *request- }}diff --git a/crates/legacy/download-manager/src/file_download_task_actor/actor_state.rs b/crates/legacy/download-manager/src/file_download_task_actor/actor_state.rsindex 46de5c55f..f685eba71 100644--- a/crates/legacy/download-manager/src/file_download_task_actor/actor_state.rs+++ b/crates/legacy/download-manager/src/file_download_task_actor/actor_state.rs@@ -15,7 +15,6 @@ pub(crate) enum DownloadActorState<B: DownloadBackend> {active_task: B::ActiveTask,generation: ActiveDownloadGeneration,destination_lease: DestinationLockLease,- resumed: bool,},Downloaded,}diff --git a/crates/legacy/download-manager/src/file_download_task_actor/download_task_actor.rs b/crates/legacy/download-manager/src/file_download_task_actor/download_task_actor.rsindex 97eba1e23..f83822e0c 100644--- a/crates/legacy/download-manager/src/file_download_task_actor/download_task_actor.rs+++ b/crates/legacy/download-manager/src/file_download_task_actor/download_task_actor.rs@@ -15,18 +15,18 @@ use tokio::sync::{};use crate::{- DownloadError, FileDownloadState, LockFileState, check_lock_file,+ DownloadError, FileCheck, FileDownloadState, LockFileState, check_lock_file,+ crc_utils::{calculate_and_verify_crc, crc_path_for_file, save_crc_file},download_log_event::{DownloadLogEvent, log},file_download_task_actor::{BackendEvent, DownloadActorState, PendingProgressSlot, ProgressCounters, PublicProjection, TaskCommand,TerminalOutcome, project_runtime_public_state,},- integrity::{integrity_receipt_path, save_integrity_cache, verify_file_integrity},lock_manager::{DestinationLockLease, lock_path_for_destination},release_lock_if_owned,traits::{ActiveDownloadGeneration, ActiveDownloadGenerationCounter, ActiveTask, BackendContext, BackendEventSender,- Capabilities, DownloadBackend, DownloadConfig,+ DownloadBackend, DownloadConfig,},};@@ -347,7 +347,6 @@ impl<B: DownloadBackend> DownloadTaskActor<B> {let from_state = current_state.name();if let DownloadActorState::Downloading {destination_lease,- resumed,..} = current_state{@@ -366,14 +365,11 @@ impl<B: DownloadBackend> DownloadTaskActor<B> {Err(message) => {remove_file(&self.config.destination).await;remove_resume_artifact(&self.config.destination).await;- remove_file(&integrity_receipt_path(&self.config.destination)).await;+ remove_file(&crc_path_for_file(&self.config.destination)).await;self.progress_counters = ProgressCounters::default();+ self.projection = PublicProjection::StickyError(message.clone());release_destination_lease(destination_lease).await;self.finish_transition(from_state, DownloadActorState::NotDownloaded);- if self.restart_after_stale_resume(resumed).await {- return;- }- self.projection = PublicProjection::StickyError(message.clone());self.pending_terminal_outcome = Some(TerminalOutcome::Error(message));},}@@ -401,19 +397,15 @@ impl<B: DownloadBackend> DownloadTaskActor<B> {if let DownloadActorState::Downloading {active_task,destination_lease,- resumed,..} = current_state{let _ = active_task.cancel(&self.config.destination).await;remove_resume_artifact(&self.config.destination).await;+ self.projection = PublicProjection::StickyError(message.clone());self.progress_counters = ProgressCounters::default();release_destination_lease(destination_lease).await;self.finish_transition(from_state, DownloadActorState::NotDownloaded);- if self.restart_after_stale_resume(resumed).await {- return;- }- self.projection = PublicProjection::StickyError(message.clone());self.pending_terminal_outcome = Some(TerminalOutcome::Error(message));}}@@ -437,25 +429,6 @@ impl<B: DownloadBackend> DownloadTaskActor<B> {}}- async fn restart_after_stale_resume(- &mut self,- resumed: bool,- ) -> bool {- if !resumed- || self.config.request.is_authenticated()- || !B::CAPABILITIES.contains(Capabilities::CACHES_REDIRECTED_URL_IN_RESUME_DATA)- {- return false;- }- self.projection = PublicProjection::None;- if let Err(error) = self.start_fresh_download().await {- let message = error.to_string();- self.projection = PublicProjection::StickyError(message.clone());- self.pending_terminal_outcome = Some(TerminalOutcome::Error(message));- }- true- }-async fn start_fresh_download(&mut self) -> Result<(), DownloadError> {let lease = self.acquire_destination_lease().await?;let generation = self.generation_counter.allocate_next();@@ -480,7 +453,6 @@ impl<B: DownloadBackend> DownloadTaskActor<B> {active_task,generation,destination_lease: lease,- resumed: false,});Ok(())}@@ -524,7 +496,6 @@ impl<B: DownloadBackend> DownloadTaskActor<B> {active_task,generation,destination_lease: lease,- resumed: true,});Ok(())}@@ -632,8 +603,8 @@ async fn remove_file(path: &Path) {}async fn remove_resume_artifact(destination: &Path) {- remove_file(&destination.with_added_extension("part")).await;- remove_file(&destination.with_added_extension("resume_data")).await;+ remove_file(&destination.with_extension("part")).await;+ remove_file(&destination.with_extension("resume_data")).await;}async fn validate_completed_file(config: &DownloadConfig) -> Result<u64, String> {@@ -668,12 +639,20 @@ async fn validate_completed_file(config: &DownloadConfig) -> Result<u64, String>let total_bytes = config.expected_bytes.unwrap_or(actual_bytes);- match verify_file_integrity(&config.destination, &config.file_check).await {- Ok(true) => {- let _ = save_integrity_cache(&config.destination, &config.file_check).await;- Ok(total_bytes)+ match &config.file_check {+ FileCheck::None => Ok(total_bytes),+ FileCheck::CRC(expected_crc) => {+ let crc_result = calculate_and_verify_crc(&config.destination, expected_crc).await;+ match crc_result {+ Ok(true) => {+ let destination = config.destination.clone();+ let expected = expected_crc.clone();+ let _ = save_crc_file(&destination, &expected).await;+ Ok(total_bytes)+ },+ Ok(false) => Err("CRC verification failed".to_string()),+ Err(error) => Err(format!("CRC verification error: {error}")),+ }},- Ok(false) => Err(config.file_check.verification_failure_message().to_string()),- Err(error) => Err(format!("integrity verification error: {error}")),}}diff --git a/crates/legacy/download-manager/src/file_download_task_actor/generic_file_download_task.rs b/crates/legacy/download-manager/src/file_download_task_actor/generic_file_download_task.rsindex 06dee5dc1..44b53aafb 100644--- a/crates/legacy/download-manager/src/file_download_task_actor/generic_file_download_task.rs+++ b/crates/legacy/download-manager/src/file_download_task_actor/generic_file_download_task.rs@@ -13,7 +13,7 @@ use tokio::sync::{Mutex as TokioMutex,broadcast::{Sender as TokioBroadcastSender, channel as tokio_broadcast_channel},mpsc::{Sender as TokioMpscSender, channel as tokio_mpsc_channel},- oneshot::{Sender as TokioOneshotSender, channel as tokio_oneshot_channel},+ oneshot::channel as tokio_oneshot_channel,watch::{Receiver as TokioWatchReceiver, channel as tokio_watch_channel},};use tokio_stream::{StreamExt as TokioStreamExt, wrappers::BroadcastStream as TokioBroadcastStream};@@ -117,7 +117,6 @@ impl<B: DownloadBackend> GenericFileDownloadTask<B> {active_task,generation: attachment_generation,destination_lease,- resumed: false,},ProgressCounters {downloaded_bytes: initial_downloaded_bytes,@@ -167,7 +166,7 @@ impl<B: DownloadBackend> GenericFileDownloadTask<B> {async fn send_command(&self,- command_builder: impl FnOnce(TokioOneshotSender<Result<(), DownloadError>>) -> TaskCommand,+ command_builder: impl FnOnce(tokio::sync::oneshot::Sender<Result<(), DownloadError>>) -> TaskCommand,) -> Result<(), DownloadError> {if self.is_stopped.load(Ordering::SeqCst) {return Err(DownloadError::TaskStopped);@@ -199,7 +198,7 @@ impl<B: DownloadBackend> fmt::Debug for GenericFileDownloadTask<B> {formatter.debug_struct("GenericFileDownloadTask").field("download_id", &self.config.download_id)- .field("source_url", &self.config.request.url)+ .field("source_url", &self.config.source_url).field("destination", &self.config.destination).finish()}@@ -207,13 +206,13 @@ impl<B: DownloadBackend> fmt::Debug for GenericFileDownloadTask<B> {#[cfg_attr(not(target_family = "wasm"), async_trait::async_trait)]#[cfg_attr(target_family = "wasm", async_trait::async_trait(?Send))]-impl<B: DownloadBackend> FileDownloadTask for GenericFileDownloadTask<B> {+impl<B: DownloadBackend> crate::FileDownloadTask for GenericFileDownloadTask<B> {fn download_id(&self) -> DownloadId {self.config.download_id}fn source_url(&self) -> &str {- &self.config.request.url+ &self.config.source_url}fn destination(&self) -> &Path {diff --git a/crates/legacy/download-manager/src/http_download_request.rs b/crates/legacy/download-manager/src/http_download_request.rsdeleted file mode 100644index d24be885c..000000000--- a/crates/legacy/download-manager/src/http_download_request.rs+++ /dev/null@@ -1,120 +0,0 @@-use std::{- fmt,- net::IpAddr,- sync::{Arc, Weak},-};--use http::uri::Scheme;-use reqwest::{RequestBuilder, Url};--use crate::DownloadError;--#[derive(Clone)]-pub struct HttpDownloadRequest {- pub url: String,- bearer_token: Option<Weak<str>>,-}--impl HttpDownloadRequest {- pub fn get(url: impl Into<String>) -> Self {- Self {- url: url.into(),- bearer_token: None,- }- }-- pub fn with_bearer_token(- url: impl Into<String>,- bearer_token: &Arc<str>,- ) -> Self {- Self {- url: url.into(),- bearer_token: Some(Arc::downgrade(bearer_token)),- }- }-- pub fn is_authenticated(&self) -> bool {- self.bearer_token.is_some()- }-- pub fn bearer_token(&self) -> Result<Option<Arc<str>>, DownloadError> {- self.bearer_token- .as_ref()- .map(|token| token.upgrade().ok_or(DownloadError::AuthenticationUnavailable))- .transpose()- }-- pub fn apply(- &self,- request: RequestBuilder,- ) -> Result<RequestBuilder, DownloadError> {- Ok(match self.bearer_token()? {- Some(token) => request.bearer_auth(token.as_ref()),- None => request,- })- }-- pub fn validate(&self) -> Result<(), DownloadError> {- let url = Url::parse(&self.url).map_err(|_| DownloadError::BadUrl)?;- let scheme = Scheme::try_from(url.scheme()).map_err(|_| DownloadError::BadUrl)?;- if self.is_authenticated() && scheme != Scheme::HTTPS && !is_loopback(&url) {- return Err(DownloadError::InsecureAuthenticatedRequest);- }- Ok(())- }-}--impl fmt::Debug for HttpDownloadRequest {- fn fmt(- &self,- formatter: &mut fmt::Formatter<'_>,- ) -> fmt::Result {- let end = self.url.find(['?', '#']).unwrap_or(self.url.len());- formatter- .debug_struct("HttpDownloadRequest")- .field("url", &&self.url[..end])- .field("authenticated", &self.is_authenticated())- .finish()- }-}--impl PartialEq for HttpDownloadRequest {- fn eq(- &self,- other: &Self,- ) -> bool {- self.url == other.url- && match (&self.bearer_token, &other.bearer_token) {- (Some(left), Some(right)) => Weak::ptr_eq(left, right),- (None, None) => true,- _ => false,- }- }-}--impl Eq for HttpDownloadRequest {}--impl From<String> for HttpDownloadRequest {- fn from(url: String) -> Self {- Self::get(url)- }-}--impl From<&str> for HttpDownloadRequest {- fn from(url: &str) -> Self {- Self::get(url)- }-}--impl From<&String> for HttpDownloadRequest {- fn from(url: &String) -> Self {- Self::get(url)- }-}--fn is_loopback(url: &Url) -> bool {- let Some(host) = url.host_str() else {- return false;- };- host.eq_ignore_ascii_case("localhost") || host.parse::<IpAddr>().is_ok_and(|address| address.is_loopback())-}diff --git a/crates/legacy/download-manager/src/integrity.rs b/crates/legacy/download-manager/src/integrity.rsdeleted file mode 100644index d5f613e7f..000000000--- a/crates/legacy/download-manager/src/integrity.rs+++ /dev/null@@ -1,239 +0,0 @@-use std::{- io::{Error as IoError, ErrorKind},- path::{Path, PathBuf},-};--use base64::{Engine, engine::general_purpose::STANDARD};-use crc32c::crc32c_append;-use hex::decode_to_slice;-use kiban::{fs, time::SystemTime};-use serde::{Deserialize, Serialize};-use serde_json::{from_str, to_vec};-use sha1::Sha1;-use sha2::{Digest, Sha256};--use crate::FileCheck;--const INTEGRITY_RECEIPT_VERSION: u8 = 1;-const READ_CHUNK_SIZE: u64 = 8 * 1024 * 1024;--#[derive(Debug, thiserror::Error)]-pub enum VerificationError {- #[error("invalid expected {algorithm} digest")]- InvalidExpectedDigest {- algorithm: &'static str,- },- #[error(transparent)]- Io(#[from] IoError),-}--#[derive(Serialize, Deserialize, PartialEq, Eq)]-struct IntegrityReceipt {- version: u8,- file_check: FileCheck,- #[serde(flatten)]- fingerprint: FileFingerprint,-}--#[derive(Serialize, Deserialize, PartialEq, Eq)]-struct FileFingerprint {- file_size: u64,- modified_unix_seconds: u64,- modified_nanos: u32,-}--enum StreamingVerifier {- Crc32c {- expected: u32,- actual: u32,- },- Sha256 {- expected: [u8; 32],- actual: Sha256,- },- GitBlobSha1 {- expected: [u8; 20],- actual: Sha1,- },-}--pub async fn verify_file_integrity(- file_path: &Path,- file_check: &FileCheck,-) -> Result<bool, VerificationError> {- if matches!(file_check, FileCheck::None) {- return Ok(true);- }-- let file_len = fs::asyn::file_length(file_path).await?;- let mut verifier = StreamingVerifier::new(file_check, file_len)?;- let mut offset = 0;- while offset < file_len {- let end = offset.saturating_add(READ_CHUNK_SIZE).min(file_len);- let chunk = fs::asyn::read_range(file_path, offset..end).await?;- let expected_len = usize::try_from(end - offset)- .map_err(|_| IoError::new(ErrorKind::InvalidData, "integrity chunk size overflow"))?;- if chunk.len() != expected_len {- return Err(IoError::new(ErrorKind::UnexpectedEof, "file changed during integrity verification").into());- }- verifier.update(&chunk);- offset = end;- }- if fs::asyn::file_length(file_path).await? != file_len {- return Err(IoError::new(ErrorKind::UnexpectedEof, "file changed during integrity verification").into());- }-- Ok(verifier.matches_expected())-}--pub async fn integrity_cache_matches(- file_path: &Path,- file_check: &FileCheck,-) -> bool {- if !expected_digest_is_well_formed(file_check) {- return false;- }- let Ok(cache_contents) = fs::asyn::read_to_string(integrity_receipt_path(file_path)).await else {- return false;- };- let Ok(fingerprint) = FileFingerprint::from_file(file_path).await else {- return false;- };-- from_str::<IntegrityReceipt>(&cache_contents).is_ok_and(|cached| {- cached- == IntegrityReceipt {- version: INTEGRITY_RECEIPT_VERSION,- file_check: file_check.clone(),- fingerprint,- }- })-}--pub async fn save_integrity_cache(- file_path: &Path,- file_check: &FileCheck,-) -> Result<(), IoError> {- if matches!(file_check, FileCheck::None) {- return Ok(());- }- let fingerprint = FileFingerprint::from_file(file_path).await?;- let receipt = to_vec(&IntegrityReceipt {- version: INTEGRITY_RECEIPT_VERSION,- file_check: file_check.clone(),- fingerprint,- })- .map_err(IoError::other)?;- fs::asyn::write(integrity_receipt_path(file_path), receipt).await-}--pub fn integrity_receipt_path(file_path: &Path) -> PathBuf {- file_path.with_added_extension("integrity")-}--impl StreamingVerifier {- fn new(- file_check: &FileCheck,- file_len: u64,- ) -> Result<Self, VerificationError> {- match file_check {- FileCheck::CRC(expected) => Ok(Self::Crc32c {- expected: decode_crc32c(expected).ok_or(VerificationError::InvalidExpectedDigest {- algorithm: "crc32c",- })?,- actual: 0,- }),- FileCheck::Sha256(expected) => Ok(Self::Sha256 {- expected: decode_hex(expected, "sha256")?,- actual: Sha256::new(),- }),- FileCheck::GitBlobSha1(expected) => {- let mut actual = Sha1::new();- actual.update(format!("blob {file_len}\0").as_bytes());- Ok(Self::GitBlobSha1 {- expected: decode_hex(expected, "git_blob_sha1")?,- actual,- })- },- FileCheck::None => unreachable!("handled before constructing verifier"),- }- }-- fn update(- &mut self,- chunk: &[u8],- ) {- match self {- Self::Crc32c {- actual,- ..- } => *actual = crc32c_append(*actual, chunk),- Self::Sha256 {- actual,- ..- } => actual.update(chunk),- Self::GitBlobSha1 {- actual,- ..- } => actual.update(chunk),- }- }-- fn matches_expected(self) -> bool {- match self {- Self::Crc32c {- expected,- actual,- } => actual == expected,- Self::Sha256 {- expected,- actual,- } => actual.finalize().as_slice() == expected,- Self::GitBlobSha1 {- expected,- actual,- } => actual.finalize().as_slice() == expected,- }- }-}--impl FileFingerprint {- async fn from_file(file_path: &Path) -> Result<Self, IoError> {- let file_size = fs::asyn::file_length(file_path).await?;- let modified = fs::asyn::file_modified(file_path)- .await?- .duration_since(SystemTime::UNIX_EPOCH)- .map_err(|error| IoError::new(ErrorKind::InvalidData, error))?;- Ok(Self {- file_size,- modified_unix_seconds: modified.as_secs(),- modified_nanos: modified.subsec_nanos(),- })- }-}--fn expected_digest_is_well_formed(file_check: &FileCheck) -> bool {- match file_check {- FileCheck::CRC(expected) => decode_crc32c(expected).is_some(),- FileCheck::Sha256(expected) => decode_hex::<32>(expected, "sha256").is_ok(),- FileCheck::GitBlobSha1(expected) => decode_hex::<20>(expected, "git_blob_sha1").is_ok(),- FileCheck::None => false,- }-}--fn decode_crc32c(expected_crc32c_base64: &str) -> Option<u32> {- let expected_bytes = STANDARD.decode(expected_crc32c_base64).ok()?;- let expected_bytes: [u8; 4] = expected_bytes.try_into().ok()?;- Some(u32::from_be_bytes(expected_bytes))-}--fn decode_hex<const LENGTH: usize>(- encoded: &str,- algorithm: &'static str,-) -> Result<[u8; LENGTH], VerificationError> {- let mut decoded = [0; LENGTH];- decode_to_slice(encoded, &mut decoded).map_err(|_| VerificationError::InvalidExpectedDigest {- algorithm,- })?;- Ok(decoded)-}diff --git a/crates/legacy/download-manager/src/lib.rs b/crates/legacy/download-manager/src/lib.rsindex a42d474e0..844189b23 100644--- a/crates/legacy/download-manager/src/lib.rs+++ b/crates/legacy/download-manager/src/lib.rs@@ -1,8 +1,5 @@-use std::path::Path;--use uuid::Uuid;-mod checked_file_state;+mod crc_utils;mod download_error;mod download_info;mod download_log_event;@@ -14,8 +11,6 @@ mod file_download_phase;mod file_download_state;mod file_download_task;mod file_state;-mod http_download_request;-mod integrity;mod lock_file_info;mod lock_file_state;mod lock_manager;@@ -38,12 +33,11 @@ pub use file_download_phase::FileDownloadPhase;pub use file_download_state::FileDownloadState;pub use file_download_task::FileDownloadTask;pub use file_state::FileState;-pub use http_download_request::HttpDownloadRequest;pub use lock_file_info::LockFileInfo;pub use lock_file_state::LockFileState;pub use lock_manager::{acquire_lock, check_lock_file, release_lock_if_owned, try_acquire_lock};-pub type DownloadId = Uuid;-pub fn compute_download_id(destination_path: &Path) -> DownloadId {- Uuid::new_v5(&Uuid::NAMESPACE_URL, destination_path.display().to_string().as_bytes())+pub type DownloadId = uuid::Uuid;+pub fn compute_download_id(destination_path: &std::path::Path) -> DownloadId {+ uuid::Uuid::new_v5(&uuid::Uuid::NAMESPACE_URL, destination_path.display().to_string().as_bytes())}diff --git a/crates/legacy/download-manager/src/reducer/action.rs b/crates/legacy/download-manager/src/reducer/action.rsindex a9cc606af..e2421db93 100644--- a/crates/legacy/download-manager/src/reducer/action.rs+++ b/crates/legacy/download-manager/src/reducer/action.rs@@ -2,21 +2,19 @@ use std::path::PathBuf;use serde::{Deserialize, Serialize};-use crate::FileCheck;-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)]pub enum Action {DeleteFile {path: PathBuf,},- DeleteIntegrityCache {+ DeleteCrcCache {path: PathBuf,},DeleteResumeArtifact {path: PathBuf,},- SaveIntegrityCache {+ SaveCrcCache {destination: PathBuf,- file_check: FileCheck,+ crc: String,},}diff --git a/crates/legacy/download-manager/src/reducer/decision.rs b/crates/legacy/download-manager/src/reducer/decision.rsindex b28b77439..a6acae665 100644--- a/crates/legacy/download-manager/src/reducer/decision.rs+++ b/crates/legacy/download-manager/src/reducer/decision.rs@@ -31,7 +31,7 @@ pub fn decide(part_path: observation.resume_artifact_path.clone()- .unwrap_or_else(|| observation.destination_path.with_added_extension("part")),+ .unwrap_or_else(|| observation.destination_path.with_extension("part")),}},CheckedFileState::Invalid | CheckedFileState::Missing => InitialLifecycleState::NotDownloaded,@@ -96,9 +96,11 @@ fn decide_actions(path: observation.destination_path.clone(),});- if observation.integrity_state == FileState::Exists {- actions.push(Action::DeleteIntegrityCache {- path: observation.integrity_path.clone(),+ if observation.crc_state == FileState::Exists+ && let Some(path) = observation.crc_path.clone()+ {+ actions.push(Action::DeleteCrcCache {+ path,});}}diff --git a/crates/legacy/download-manager/src/reducer/disk_observation.rs b/crates/legacy/download-manager/src/reducer/disk_observation.rsindex fbe454fdf..ac75f6693 100644--- a/crates/legacy/download-manager/src/reducer/disk_observation.rs+++ b/crates/legacy/download-manager/src/reducer/disk_observation.rs@@ -2,18 +2,18 @@ use std::path::PathBuf;use serde::{Deserialize, Serialize};-use crate::{FileCheck, FileState};+use crate::FileState;#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)]pub struct DiskObservation {pub destination_state: FileState,- pub integrity_state: FileState,+ pub crc_state: FileState,pub resume_state: FileState,pub destination_size: Option<u64>,pub resume_size: Option<u64>,- pub file_check: FileCheck,+ pub expected_crc: Option<String>,pub expected_bytes: Option<u64>,pub destination_path: PathBuf,- pub integrity_path: PathBuf,+ pub crc_path: Option<PathBuf>,pub resume_artifact_path: Option<PathBuf>,}diff --git a/crates/legacy/download-manager/src/reducer/validation_outcome.rs b/crates/legacy/download-manager/src/reducer/validation_outcome.rsindex 9bc0c18b3..fba6749ab 100644--- a/crates/legacy/download-manager/src/reducer/validation_outcome.rs+++ b/crates/legacy/download-manager/src/reducer/validation_outcome.rs@@ -1,8 +1,6 @@-use std::path::Path;-use crate::{- CheckedFileState, DownloadError, FileCheck, FileState,- integrity::{VerificationError, integrity_cache_matches, verify_file_integrity},+ CheckedFileState, FileState,+ crc_utils::{calculate_and_verify_crc, crc_cache_matches},reducer::{Action, ActionPlan, DiskObservation},};@@ -12,67 +10,56 @@ pub struct ValidationOutcome {pub action_plan: ActionPlan,}-pub async fn validate(observation: &DiskObservation) -> Result<ValidationOutcome, DownloadError> {+pub async fn validate(observation: &DiskObservation) -> ValidationOutcome {let length_matches = match (observation.destination_size, observation.expected_bytes) {(Some(actual), Some(expected)) => actual == expected,_ => true,};- let (checked, mut actions) = match (observation.destination_state, &observation.file_check) {+ let (checked, mut actions) = match (observation.destination_state, observation.expected_crc.as_deref()) {(FileState::Missing, _) => (CheckedFileState::Missing, Vec::new()),(FileState::Exists, _) if !length_matches => (CheckedFileState::Invalid, Vec::new()),- (FileState::Exists, FileCheck::None) => (CheckedFileState::Valid, Vec::new()),- (FileState::Exists, file_check) => validate_integrity_with_cache(observation, file_check).await?,+ (FileState::Exists, None) => (CheckedFileState::Valid, Vec::new()),+ (FileState::Exists, Some(expected_crc)) => validate_crc_with_cache(observation, expected_crc).await,};- if checked == CheckedFileState::Missing && observation.integrity_state == FileState::Exists {- actions.push(Action::DeleteIntegrityCache {- path: observation.integrity_path.clone(),+ if checked == CheckedFileState::Missing+ && observation.crc_state == FileState::Exists+ && let Some(path) = observation.crc_path.clone()+ {+ actions.push(Action::DeleteCrcCache {+ path,});}- Ok(ValidationOutcome {+ let action_plan = ActionPlan::from_ordered_actions(actions);++ ValidationOutcome {checked,- action_plan: ActionPlan::from_ordered_actions(actions),- })+ action_plan,+ }}-async fn validate_integrity_with_cache(+async fn validate_crc_with_cache(observation: &DiskObservation,- file_check: &FileCheck,-) -> Result<(CheckedFileState, Vec<Action>), DownloadError> {- if observation.integrity_state == FileState::Exists- && integrity_cache_matches(&observation.destination_path, file_check).await+ expected_crc: &str,+) -> (CheckedFileState, Vec<Action>) {+ if observation.crc_state == FileState::Exists+ && crc_cache_matches(&observation.destination_path, expected_crc).await{- return Ok((CheckedFileState::Valid, Vec::new()));+ return (CheckedFileState::Valid, Vec::new());}- match verify_file_integrity(&observation.destination_path, file_check).await {- Ok(true) => Ok((+ let crc_check = calculate_and_verify_crc(&observation.destination_path, expected_crc).await.ok();++ match crc_check {+ Some(true) => (CheckedFileState::Valid,- vec![Action::SaveIntegrityCache {+ vec![Action::SaveCrcCache {destination: observation.destination_path.clone(),- file_check: file_check.clone(),+ crc: expected_crc.to_string(),}],- )),- Ok(false) => Ok((CheckedFileState::Invalid, Vec::new())),- Err(error) => Err(map_verification_error(&observation.destination_path, error)),- }-}--fn map_verification_error(- path: &Path,- error: VerificationError,-) -> DownloadError {- match error {- VerificationError::InvalidExpectedDigest {- algorithm,- } => DownloadError::InvalidDigest {- algorithm,- },- VerificationError::Io(error) => DownloadError::IntegrityIo {- path: path.display().to_string(),- message: error.to_string(),- },+ ),+ _ => (CheckedFileState::Invalid, Vec::new()),}}diff --git a/crates/legacy/download-manager/src/traits/capabilities.rs b/crates/legacy/download-manager/src/traits/capabilities.rsdeleted file mode 100644index 857cf2fd4..000000000--- a/crates/legacy/download-manager/src/traits/capabilities.rs+++ /dev/null@@ -1,10 +0,0 @@-use bitflags::bitflags;--bitflags! {- #[repr(transparent)]- #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]- pub struct Capabilities: u32 {- /// Resume data can retain a redirected CDN URL that may expire before reuse.- const CACHES_REDIRECTED_URL_IN_RESUME_DATA = 1 << 0;- }-}diff --git a/crates/legacy/download-manager/src/traits/download_backend.rs b/crates/legacy/download-manager/src/traits/download_backend.rsindex face76fd8..35de4e90c 100644--- a/crates/legacy/download-manager/src/traits/download_backend.rs+++ b/crates/legacy/download-manager/src/traits/download_backend.rs@@ -2,13 +2,12 @@ use std::{fmt::Debug, future::Future, path::Path};use kiban::{fs, maybe::MaybeSend};-use crate::traits::{ActiveTask, BackendContext, Capabilities};+use crate::traits::{ActiveTask, BackendContext};pub trait DownloadBackend: Debug + Clone + Send + Sync + 'static {type Context: BackendContext<Backend = Self>;type ActiveTask: ActiveTask<Backend = Self>;type Error: std::error::Error + Send + Sync + 'static;- const CAPABILITIES: Capabilities = Capabilities::empty();// Default = file size (correct for `.part`-style artifacts). Apple must override:// `.resume_data` is a small metadata blob, not the downloaded bytes.diff --git a/crates/legacy/download-manager/src/traits/download_config.rs b/crates/legacy/download-manager/src/traits/download_config.rsindex 311707fd9..041d50a7e 100644--- a/crates/legacy/download-manager/src/traits/download_config.rs+++ b/crates/legacy/download-manager/src/traits/download_config.rs@@ -1,13 +1,14 @@use std::path::PathBuf;+use serde::{Deserialize, Serialize};use uuid::Uuid;-use crate::{DownloadId, FileCheck, HttpDownloadRequest};+use crate::{DownloadId, FileCheck};-#[derive(Clone, Debug, PartialEq, Eq)]+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)]pub struct DownloadConfig {pub download_id: DownloadId,- pub request: HttpDownloadRequest,+ pub source_url: String,pub destination: PathBuf,pub file_check: FileCheck,pub expected_bytes: Option<u64>,diff --git a/crates/legacy/download-manager/src/traits/mod.rs b/crates/legacy/download-manager/src/traits/mod.rsindex 67e2f6414..007ea3a41 100644--- a/crates/legacy/download-manager/src/traits/mod.rs+++ b/crates/legacy/download-manager/src/traits/mod.rs@@ -4,7 +4,6 @@ mod active_task;mod backend_context;mod backend_event_sender;mod cancel_outcome;-mod capabilities;mod download_backend;mod download_config;@@ -14,6 +13,5 @@ pub use active_task::ActiveTask;pub use backend_context::BackendContext;pub use backend_event_sender::BackendEventSender;pub use cancel_outcome::CancelOutcome;-pub use capabilities::Capabilities;pub use download_backend::DownloadBackend;pub use download_config::DownloadConfig;diff --git a/crates/legacy/download-manager/tests/unit/backends/apple/test_context.rs b/crates/legacy/download-manager/tests/unit/backends/apple/test_context.rsindex 03dfa8194..47ecbc516 100644--- a/crates/legacy/download-manager/tests/unit/backends/apple/test_context.rs+++ b/crates/legacy/download-manager/tests/unit/backends/apple/test_context.rs@@ -1,40 +1,21 @@-use std::{- error::Error,- io::Error as IoError,- path::PathBuf,- sync::{- Arc,- atomic::{AtomicUsize, Ordering},- },- time::Duration,-};+use std::sync::Arc;use kiban::rt::RuntimeHandle;use mock_registry::{Behavior, MockRegistry};-use tempfile::tempdir;-use tokio::{- fs::{read as tokio_read, write as tokio_write},- sync::{- Mutex as TokioMutex,- mpsc::{Receiver as TokioMpscReceiver, channel as tokio_mpsc_channel},- watch::channel as tokio_watch_channel,- },- time::{sleep as tokio_sleep, timeout as tokio_timeout},+use tokio::sync::{+ Mutex as TokioMutex,+ mpsc::{Receiver as TokioMpscReceiver, channel as tokio_mpsc_channel},+ watch::channel as tokio_watch_channel,};use uuid::Uuid;-use wiremock::{- Mock, MockServer, Request, ResponseTemplate,- matchers::{header, method, path},-};use crate::{- FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase, HttpDownloadRequest,+ FileCheck,backends::apple::AppleBackendContext,- common::wait_for_phase,compute_download_id,file_download_task_actor::{BackendEvent, PendingProgressSlot},lock_manager::DestinationLockLease,- traits::{ActiveDownloadGeneration, ActiveTask, BackendContext, BackendEventSender, DownloadConfig},+ traits::{ActiveDownloadGeneration, BackendContext, BackendEventSender, DownloadConfig},};fn backend_event_sender() -> (BackendEventSender, TokioMpscReceiver<BackendEvent>) {@@ -47,102 +28,48 @@ fn backend_event_sender() -> (BackendEventSender, TokioMpscReceiver<BackendEvent)}-fn download_config(- request: impl Into<HttpDownloadRequest>,- destination: PathBuf,- expected_bytes: u64,-) -> Arc<DownloadConfig> {- Arc::new(DownloadConfig {+#[tokio::test(flavor = "multi_thread")]+async fn test_apple_resume_empty_resume_data_starts_fresh_download() -> Result<(), Box<dyn std::error::Error>> {+ let registry = MockRegistry::start().await?;+ let served_file = registry.file("config.json")?;+ let temporary_directory = tempfile::tempdir()?;+ let destination = temporary_directory.path().join(&served_file.file.name);+ let resume_artifact_path = destination.with_extension("resume_data");+ tokio::fs::write(&resume_artifact_path, b"").await?;++ let context = AppleBackendContext::new(RuntimeHandle::current());+ let config = Arc::new(DownloadConfig {download_id: compute_download_id(&destination),- request: request.into(),- destination,+ source_url: served_file.file.url.clone(),+ destination: destination.clone(),file_check: FileCheck::None,- expected_bytes: Some(expected_bytes),+ expected_bytes: Some(served_file.file.size as u64),manager_id: "test-manager".to_string(),manager_instance_id: Uuid::new_v4(),- })-}--async fn destination_lease(config: &DownloadConfig) -> Result<DestinationLockLease, IoError> {- DestinationLockLease::acquire_for_destination(&config.destination, &config.manager_id, config.manager_instance_id)- .await-}--#[tokio::test(flavor = "multi_thread")]-async fn apple_authenticated_redirect_strips_authorization_cross_origin() -> Result<(), Box<dyn Error>> {- let bytes = b"private redirected file";- let target = MockServer::start().await;- Mock::given(method("GET"))- .and(path("/cdn/model.bin"))- .and(header("accept-encoding", "identity"))- .respond_with(ResponseTemplate::new(200).set_body_bytes(bytes.as_slice()))- .expect(1)- .mount(&target)- .await;- let origin = MockServer::start().await;- Mock::given(method("GET"))- .and(path("/model.bin"))- .and(header("authorization", "Bearer test-token"))- .and(header("accept-encoding", "identity"))- .respond_with(ResponseTemplate::new(302).insert_header("Location", format!("{}/cdn/model.bin", target.uri())))- .expect(1)- .mount(&origin)- .await;- let directory = tempdir()?;- let destination = directory.path().join("model.bin");- let token: Arc<str> = Arc::from("test-token");- let config = download_config(- HttpDownloadRequest::with_bearer_token(format!("{}/model.bin", origin.uri()), &token),- destination.clone(),- bytes.len() as u64,- );- let lease = destination_lease(&config).await?;- let generation = ActiveDownloadGeneration::new(0);- let (sender, mut receiver) = backend_event_sender();- let context = AppleBackendContext::new(RuntimeHandle::current());-- let active_task = context.download(Arc::clone(&config), generation, sender, &lease).await?;- let event = tokio_timeout(Duration::from_secs(5), receiver.recv())- .await?- .ok_or_else(|| IoError::other("backend event channel closed"))?;-- assert_eq!(event, BackendEvent::completed(generation));- let target_requests = target.received_requests().await.expect("requests should be available");- assert_eq!(target_requests.len(), 1);- assert!(target_requests[0].headers.get("authorization").is_none());- assert_eq!(tokio_read(destination).await?, bytes);- drop(active_task);- lease.release().await?;- Ok(())-}--#[tokio::test(flavor = "multi_thread")]-async fn apple_http_error_body_is_not_installed() -> Result<(), Box<dyn Error>> {- let error_body = b"denied";- let server = MockServer::start().await;- Mock::given(method("GET"))- .and(path("/model.bin"))- .respond_with(ResponseTemplate::new(404).set_body_bytes(error_body.as_slice()))- .expect(1)- .mount(&server)- .await;-- let temporary_directory = tempdir()?;- let destination = temporary_directory.path().join("model.bin");- let context = AppleBackendContext::new(RuntimeHandle::current());- let config = download_config(format!("{}/model.bin", server.uri()), destination.clone(), error_body.len() as u64);- let destination_lease = destination_lease(&config).await?;+ });+ let destination_lease = DestinationLockLease::acquire_for_destination(+ &config.destination,+ &config.manager_id,+ config.manager_instance_id,+ )+ .await?;let generation = ActiveDownloadGeneration::new(0);let (backend_event_sender, mut backend_event_receiver) = backend_event_sender();- let active_task =- context.download(Arc::clone(&config), generation, backend_event_sender, &destination_lease).await?;- let event = tokio_timeout(Duration::from_secs(5), backend_event_receiver.recv())+ let active_task = context+ .resume(Arc::clone(&config), generation, &resume_artifact_path, backend_event_sender, &destination_lease)+ .await?;+ let event = tokio::time::timeout(std::time::Duration::from_secs(5), backend_event_receiver.recv()).await?- .ok_or_else(|| IoError::other("backend event channel closed"))?;+ .ok_or_else(|| {+ std::io::Error::new(+ std::io::ErrorKind::UnexpectedEof,+ "backend event channel closed before empty resume-data fallback completed",+ )+ })?;- assert_eq!(event, BackendEvent::error(generation, "download failed with HTTP status 404".to_string()));- assert!(!destination.exists());+ assert_eq!(event, BackendEvent::completed(generation));+ assert_eq!(tokio::fs::read(&destination).await?, served_file.bytes.to_vec());drop(active_task);destination_lease.release().await?;@@ -150,138 +77,75 @@ async fn apple_http_error_body_is_not_installed() -> Result<(), Box<dyn Error>>}#[tokio::test(flavor = "multi_thread")]-async fn apple_download_can_pause_and_resume() -> Result<(), Box<dyn Error>> {+async fn dropping_apple_active_task_unregisters_event_sink() -> Result<(), Box<dyn std::error::Error>> {let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;- let served_file = registry.file("tokenizer.json")?;- let temporary_directory = tempdir()?;+ let served_file = registry.file("config.json")?;+ let temporary_directory = tempfile::tempdir()?;let destination = temporary_directory.path().join(&served_file.file.name);let context = AppleBackendContext::new(RuntimeHandle::current());- let config = download_config(served_file.file.url.clone(), destination.clone(), served_file.file.size as u64);- let destination_lease = destination_lease(&config).await?;- let initial_generation = ActiveDownloadGeneration::new(0);- let (initial_event_sender, _initial_event_receiver) = backend_event_sender();- let active_task =- context.download(Arc::clone(&config), initial_generation, initial_event_sender, &destination_lease).await?;-- tokio_sleep(Duration::from_millis(100)).await;- let resume_artifact_path = active_task.pause(&destination).await?;- assert!(!tokio_read(&resume_artifact_path).await?.is_empty());+ let download_id = compute_download_id(&destination);+ let config = Arc::new(DownloadConfig {+ download_id,+ source_url: served_file.file.url.clone(),+ destination,+ file_check: FileCheck::None,+ expected_bytes: Some(served_file.file.size as u64),+ manager_id: "test-manager".to_string(),+ manager_instance_id: Uuid::new_v4(),+ });+ let destination_lease = DestinationLockLease::acquire_for_destination(+ &config.destination,+ &config.manager_id,+ config.manager_instance_id,+ )+ .await?;- let resumed_generation = ActiveDownloadGeneration::new(1);- let (resumed_event_sender, mut resumed_event_receiver) = backend_event_sender();- let resumed_task = context- .resume(- Arc::clone(&config),- resumed_generation,- &resume_artifact_path,- resumed_event_sender,- &destination_lease,- )+ let (backend_event_sender, _backend_event_receiver) = backend_event_sender();+ let active_task = context+ .download(Arc::clone(&config), ActiveDownloadGeneration::new(0), backend_event_sender, &destination_lease).await?;- let event = tokio_timeout(Duration::from_secs(10), resumed_event_receiver.recv())- .await?- .ok_or_else(|| IoError::other("backend event channel closed before resumed download completed"))?;-- assert_eq!(event, BackendEvent::completed(resumed_generation));- assert_eq!(tokio_read(&destination).await?, served_file.bytes.to_vec());+ assert_eq!(+ context.event_sink_count_for_download(download_id),+ 1,+ "precondition: exactly one event sink should be registered for download_id after starting download",+ );- drop(resumed_task);+ drop(active_task);destination_lease.release().await?;++ assert_eq!(+ context.event_sink_count_for_download(download_id),+ 0,+ "dropping AppleActiveTask must unregister its event sink",+ );Ok(())}#[tokio::test(flavor = "multi_thread")]-async fn apple_failed_resume_restarts_from_source() -> Result<(), Box<dyn Error>> {+async fn registry_distinguishes_generations_for_same_download_id() -> Result<(), Box<dyn std::error::Error>> {let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;- let served_file = registry.file("tokenizer.json")?;- let temporary_directory = tempdir()?;+ let served_file = registry.file("config.json")?;+ let temporary_directory = tempfile::tempdir()?;let destination = temporary_directory.path().join(&served_file.file.name);- let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Apple, RuntimeHandle::current()).await?;- let task = manager- .file_download_task(- (&served_file.file.url).into(),- &destination,- FileCheck::None,- Some(served_file.file.size as u64),- )- .await?;-- task.download().await?;- tokio_sleep(Duration::from_millis(100)).await;- task.pause().await?;- tokio_write(destination.with_added_extension("resume_data"), b"invalid resume data").await?;-- let mut progress = task.progress().await?;- task.download().await?;- let state = wait_for_phase(&task, &mut progress, |phase| matches!(phase, FileDownloadPhase::Downloaded)).await;-- assert_eq!(state.downloaded_bytes, served_file.file.size as u64);- assert_eq!(tokio_read(destination).await?, served_file.bytes.to_vec());- Ok(())-}--#[tokio::test(flavor = "multi_thread")]-async fn apple_authenticated_pause_restarts_once_without_resume_data() -> Result<(), Box<dyn Error>> {- let bytes = b"private model";- let attempts = Arc::new(AtomicUsize::new(0));- let server = MockServer::start().await;- Mock::given(method("GET"))- .and(path("/model.bin"))- .and(header("authorization", "Bearer test-token"))- .respond_with({- let attempts = Arc::clone(&attempts);- move |_: &Request| {- if attempts.fetch_add(1, Ordering::Relaxed) == 0 {- ResponseTemplate::new(200).set_body_bytes(bytes.as_slice()).set_delay(Duration::from_secs(1))- } else {- ResponseTemplate::new(500)- }- }- })- .mount(&server)- .await;- let directory = tempdir()?;- let destination = directory.path().join("model.bin");- let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Apple, RuntimeHandle::current()).await?;- let token: Arc<str> = Arc::from("test-token");- let request = HttpDownloadRequest::with_bearer_token(format!("{}/model.bin", server.uri()), &token);- let task = manager.file_download_task(request, &destination, FileCheck::None, Some(bytes.len() as u64)).await?;-- task.download().await?;- tokio_timeout(Duration::from_secs(5), async {- while attempts.load(Ordering::Relaxed) == 0 {- tokio_sleep(Duration::from_millis(10)).await;- }- })- .await?;- task.pause().await?;-- assert!(tokio_read(destination.with_added_extension("resume_data")).await?.is_empty());- let mut progress = task.progress().await?;- task.download().await?;- wait_for_phase(&task, &mut progress, |phase| matches!(phase, FileDownloadPhase::Error(_))).await;-- assert_eq!(attempts.load(Ordering::Relaxed), 2, "authenticated fresh resume must not be retried");- assert!(!destination.exists());- Ok(())-}--#[tokio::test(flavor = "multi_thread")]-async fn registry_distinguishes_generations_for_same_download_id() -> Result<(), Box<dyn Error>> {- let bytes = b"model";- let server = MockServer::start().await;- Mock::given(method("GET"))- .and(path("/model.bin"))- .respond_with(ResponseTemplate::new(200).set_body_bytes(bytes.as_slice()).set_delay(Duration::from_secs(5)))- .mount(&server)- .await;- let temporary_directory = tempdir()?;- let destination = temporary_directory.path().join("model.bin");let download_id = compute_download_id(&destination);+let context = AppleBackendContext::new(RuntimeHandle::current());- let config = download_config(format!("{}/model.bin", server.uri()), destination, bytes.len() as u64);- let destination_lease = destination_lease(&config).await?;+ let config = Arc::new(DownloadConfig {+ download_id,+ source_url: served_file.file.url.clone(),+ destination,+ file_check: FileCheck::None,+ expected_bytes: Some(served_file.file.size as u64),+ manager_id: "test-manager".to_string(),+ manager_instance_id: Uuid::new_v4(),+ });+ let destination_lease = DestinationLockLease::acquire_for_destination(+ &config.destination,+ &config.manager_id,+ config.manager_instance_id,+ )+ .await?;let (backend_event_sender_first, _backend_event_receiver_first) = backend_event_sender();let first_task = context@@ -308,28 +172,35 @@ async fn registry_distinguishes_generations_for_same_download_id() -> Result<(),assert_ne!(both_keys[0], both_keys[1], "the two generations must have distinct task identifiers");drop(first_task);- assert_eq!(context.event_sink_task_identifiers_for_download(download_id).len(), 1);drop(second_task);- assert!(context.event_sink_task_identifiers_for_download(download_id).is_empty());destination_lease.release().await?;Ok(())}#[tokio::test(flavor = "multi_thread")]-async fn claim_cancels_task_with_different_credentials() -> Result<(), Box<dyn Error>> {+async fn claim_cancels_mismatched_task() -> Result<(), Box<dyn std::error::Error>> {let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;let served_file = registry.file("tokenizer.json")?;- let temporary_directory = tempdir()?;+ let temporary_directory = tempfile::tempdir()?;let destination = temporary_directory.path().join(&served_file.file.name);+ let download_id = compute_download_id(&destination);let context = AppleBackendContext::new(RuntimeHandle::current());- let original_token: Arc<str> = Arc::from("original-token");- let original_config = download_config(- HttpDownloadRequest::with_bearer_token(served_file.file.url.clone(), &original_token),- destination,- served_file.file.size as u64,- );- let destination_lease = destination_lease(&original_config).await?;+ let original_config = Arc::new(DownloadConfig {+ download_id,+ source_url: served_file.file.url.clone(),+ destination: destination.clone(),+ file_check: FileCheck::None,+ expected_bytes: Some(served_file.file.size as u64),+ manager_id: "test-manager".to_string(),+ manager_instance_id: Uuid::new_v4(),+ });+ let destination_lease = DestinationLockLease::acquire_for_destination(+ &original_config.destination,+ &original_config.manager_id,+ original_config.manager_instance_id,+ )+ .await?;let (backend_event_sender, _backend_event_receiver) = backend_event_sender();let original_task = context@@ -346,9 +217,8 @@ async fn claim_cancels_task_with_different_credentials() -> Result<(), Box<dyn E"precondition: original task should be visible before the mismatched claim",);- let replacement_token: Arc<str> = Arc::from("replacement-token");let mismatched_config = DownloadConfig {- request: HttpDownloadRequest::with_bearer_token(served_file.file.url.clone(), &replacement_token),+ source_url: "http://example.invalid/different-url".to_string(),..(*original_config).clone()};assert!(@@ -363,7 +233,7 @@ async fn claim_cancels_task_with_different_credentials() -> Result<(), Box<dyn Ecancelled = true;break;}- tokio_sleep(Duration::from_millis(20)).await;+ tokio::time::sleep(std::time::Duration::from_millis(20)).await;}assert!(diff --git a/crates/legacy/download-manager/tests/unit/backends/common/test_actor_task_drop.rs b/crates/legacy/download-manager/tests/unit/backends/common/test_actor_task_drop.rsindex 1115497ca..ef2deb6a3 100644--- a/crates/legacy/download-manager/tests/unit/backends/common/test_actor_task_drop.rs+++ b/crates/legacy/download-manager/tests/unit/backends/common/test_actor_task_drop.rs@@ -1,10 +1,8 @@-use std::{error::Error, sync::Arc, time::Duration};+use std::{path::PathBuf, sync::Arc};use download_manager::{DownloadError, FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase};use kiban::rt::RuntimeHandle;use rstest::rstest;-use tempfile::tempdir;-use tokio::{fs::write as tokio_write, time::sleep as tokio_sleep};use crate::common::{Behavior, MockRegistry, wait_for_phase};@@ -15,15 +13,15 @@ use crate::common::{Behavior, MockRegistry, wait_for_phase};async fn remove_paused_task_deletes_resume_artifact(#[case] download_manager_type: FileDownloadManagerType,#[case] resume_artifact_extension: &str,-) -> Result<(), Box<dyn Error>> {- let temporary_directory = tempdir()?;+) -> Result<(), Box<dyn std::error::Error>> {+ let temporary_directory = tempfile::tempdir()?;let destination = temporary_directory.path().join("model.bin");- let resume_artifact = destination.with_added_extension(resume_artifact_extension);- tokio_write(&resume_artifact, b"partial").await?;+ let resume_artifact = destination.with_extension(resume_artifact_extension);+ tokio::fs::write(&resume_artifact, b"partial").await?;let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await?;let task = manager- .file_download_task("http://example.invalid/model.bin".into(), &destination, FileCheck::None, Some(100))+ .file_download_task("http://example.invalid/model.bin", &destination, FileCheck::None, Some(100)).await?;assert!(matches!(task.state().await.phase, FileDownloadPhase::Paused));@@ -42,17 +40,17 @@ async fn remove_paused_task_deletes_resume_artifact(async fn dropping_active_download_cancels_backend_before_releasing_lock(#[case] download_manager_type: FileDownloadManagerType,#[case] resume_artifact_extension: &str,-) -> Result<(), Box<dyn Error>> {+) -> Result<(), Box<dyn std::error::Error>> {let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;let tokenizer = registry.file("tokenizer.json")?;- let temp_dir = tempdir()?;+ let temp_dir = tempfile::tempdir()?;let destination = temp_dir.path().join(&tokenizer.file.name);- let lock_path = destination.with_added_extension("lock");- let resume_artifact = destination.with_added_extension(resume_artifact_extension);+ let lock_path = PathBuf::from(format!("{}.lock", destination.display()));+ let resume_artifact = destination.with_extension(resume_artifact_extension);let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();let task = manager.file_download_task(- (&tokenizer.file.url).into(),+ &tokenizer.file.url,&destination,FileCheck::CRC(tokenizer.crc32c()?),Some(tokenizer.file.size as u64),@@ -69,7 +67,7 @@ async fn dropping_active_download_cancels_backend_before_releasing_lock(assert!(matches!(task.download().await, Err(DownloadError::TaskStopped)));let replacement_task = manager.file_download_task(- (&tokenizer.file.url).into(),+ &tokenizer.file.url,&destination,FileCheck::CRC(tokenizer.crc32c()?),Some(tokenizer.file.size as u64),@@ -85,13 +83,13 @@ async fn dropping_active_download_cancels_backend_before_releasing_lock(lock_released = true;break;}- tokio_sleep(Duration::from_millis(20)).await;+ tokio::time::sleep(std::time::Duration::from_millis(20)).await;}assert!(lock_released);assert!(!resume_artifact.exists());let destination_size_at_release = destination.metadata().map(|m| m.len()).unwrap_or(0);- tokio_sleep(Duration::from_millis(200)).await;+ tokio::time::sleep(std::time::Duration::from_millis(200)).await;let destination_size_after = destination.metadata().map(|m| m.len()).unwrap_or(0);assert_eq!(destination_size_at_release, destination_size_after);Ok(())diff --git a/crates/legacy/download-manager/tests/unit/backends/common/test_corrupt_body.rs b/crates/legacy/download-manager/tests/unit/backends/common/test_corrupt_body.rsindex 0f86fde99..a036eb5ab 100644--- a/crates/legacy/download-manager/tests/unit/backends/common/test_corrupt_body.rs+++ b/crates/legacy/download-manager/tests/unit/backends/common/test_corrupt_body.rs@@ -18,7 +18,7 @@ async fn test_corrupt_body_fails_crc(let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();let task = manager.file_download_task(- (&tokenizer.file.url).into(),+ &tokenizer.file.url,&destination,FileCheck::CRC(tokenizer.crc32c()?),Some(tokenizer.file.size as u64),@@ -48,7 +48,7 @@ async fn test_corrupt_body_cancel_resets_error_state(let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await?;let task = manager.file_download_task(- (&tokenizer.file.url).into(),+ &tokenizer.file.url,&destination,FileCheck::CRC(tokenizer.crc32c()?),Some(tokenizer.file.size as u64),diff --git a/crates/legacy/download-manager/tests/unit/backends/common/test_lifecycle.rs b/crates/legacy/download-manager/tests/unit/backends/common/test_lifecycle.rsindex f834351bc..8edeff37c 100644--- a/crates/legacy/download-manager/tests/unit/backends/common/test_lifecycle.rs+++ b/crates/legacy/download-manager/tests/unit/backends/common/test_lifecycle.rs@@ -1,10 +1,6 @@use download_manager::{FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase};-use hex::encode as hex_encode;use kiban::rt::RuntimeHandle;use rstest::rstest;-use sha2::{Digest, Sha256};-use tempfile::tempdir;-use tokio::fs::read as tokio_read;use crate::common::{MockRegistry, wait_for_phase};@@ -17,13 +13,13 @@ async fn test_download_fresh_completes() -> Result<(), Box<dyn std::error::Error>> {let registry = MockRegistry::start().await?;let tokenizer = registry.file("tokenizer.json")?;- let temp_dir = tempdir().unwrap();+ let temp_dir = tempfile::tempdir().unwrap();let destination = temp_dir.path().join(&tokenizer.file.name);let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();let task = manager.file_download_task(- (&tokenizer.file.url).into(),+ &tokenizer.file.url,&destination,FileCheck::CRC(tokenizer.crc32c()?),Some(tokenizer.file.size as u64),@@ -37,32 +33,7 @@ async fn test_download_fresh_completes(assert_eq!(state.downloaded_bytes, tokenizer.file.size as u64);assert_eq!(state.total_bytes, tokenizer.file.size as u64);- assert_eq!(tokio_read(&destination).await.unwrap(), tokenizer.bytes.to_vec());- assert!(destination.with_added_extension("integrity").is_file());- Ok(())-}--#[tokio::test(flavor = "multi_thread")]-async fn test_sha256_download_completes() -> Result<(), Box<dyn std::error::Error>> {- let registry = MockRegistry::start().await?;- let tokenizer = registry.file("tokenizer.json")?;- let temporary_directory = tempdir()?;- let destination = temporary_directory.path().join("tokenizer.json");- let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Universal, RuntimeHandle::current()).await?;- let check = FileCheck::Sha256(hex_encode(Sha256::digest(&tokenizer.bytes)));- let task = manager- .file_download_task(- (&tokenizer.file.url).into(),- &destination,- check,- Some(tokenizer.file.size as u64),- )- .await?;- let mut progress = task.progress().await?;-- task.download().await?;- wait_for_phase(&task, &mut progress, |phase| matches!(phase, FileDownloadPhase::Downloaded)).await;-- assert_eq!(tokio_read(destination).await?, tokenizer.bytes.as_ref());+ assert_eq!(tokio::fs::read(&destination).await.unwrap(), tokenizer.bytes.to_vec());+ assert!(std::path::PathBuf::from(format!("{}.crc", destination.display())).is_file());Ok(())}diff --git a/crates/legacy/download-manager/tests/unit/backends/common/test_manager_startup.rs b/crates/legacy/download-manager/tests/unit/backends/common/test_manager_startup.rsindex e5beb4d5d..ae2c29040 100644--- a/crates/legacy/download-manager/tests/unit/backends/common/test_manager_startup.rs+++ b/crates/legacy/download-manager/tests/unit/backends/common/test_manager_startup.rs@@ -1,27 +1,21 @@-use std::{error::Error, path::Path, process::id, sync::Arc};-use chrono::Utc;-use download_manager::{- DownloadError, FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase, FileDownloadTask,-};+use download_manager::{FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase};use kiban::rt::RuntimeHandle;-use serde_json::{json, to_vec};use tempfile::tempdir;-use tokio::fs::{create_dir as tokio_create_dir, write as tokio_write};use crate::common::MockRegistry;#[tokio::test(flavor = "multi_thread")]-async fn test_manager_startup_valid_existing_file_is_downloaded() -> Result<(), Box<dyn Error>> {+async fn test_manager_startup_valid_existing_file_is_downloaded() -> Result<(), Box<dyn std::error::Error>> {let registry = MockRegistry::start().await?;let served_file = registry.file("config.json")?;let expected_crc = served_file.crc32c()?;let temporary_directory = tempdir()?;let destination = temporary_directory.path().join(&served_file.file.name);- let integrity_path = destination.with_added_extension("integrity");- let resume_artifact = destination.with_added_extension("part");- tokio_write(&destination, served_file.bytes.as_ref()).await?;- tokio_write(&resume_artifact, b"partial").await?;+ let crc_path = std::path::PathBuf::from(format!("{}.crc", destination.display()));+ let resume_artifact = destination.with_extension("part");+ tokio::fs::write(&destination, served_file.bytes.as_ref()).await?;+ tokio::fs::write(&resume_artifact, b"partial").await?;let task = manager_task(&served_file.file.url,@@ -33,26 +27,57 @@ async fn test_manager_startup_valid_existing_file_is_downloaded() -> Result<(),assert_eq!(task.state().await.phase, FileDownloadPhase::Downloaded);assert!(!resume_artifact.exists());- assert!(integrity_path.is_file());+ let crc_receipt: serde_json::Value = serde_json::from_str(&tokio::fs::read_to_string(crc_path).await?)?;+ assert_eq!(crc_receipt["version"].as_u64(), Some(1));+ assert_eq!(crc_receipt["crc"].as_str(), Some(expected_crc.as_str()));+ assert_eq!(crc_receipt["file_size"].as_u64(), Some(served_file.file.size as u64));+ assert!(crc_receipt["modified_unix_seconds"].as_u64().is_some());+ assert!(crc_receipt["modified_nanos"].as_u64().is_some());+ Ok(())+}++#[tokio::test(flavor = "multi_thread")]+async fn test_manager_startup_rejects_legacy_crc_cache_after_file_changes() -> Result<(), Box<dyn std::error::Error>> {+ let registry = MockRegistry::start().await?;+ let served_file = registry.file("config.json")?;+ let expected_crc = served_file.crc32c()?;+ let temporary_directory = tempdir()?;+ let destination = temporary_directory.path().join(&served_file.file.name);+ let crc_path = std::path::PathBuf::from(format!("{}.crc", destination.display()));+ let mut stale_destination_bytes = served_file.bytes.to_vec();+ stale_destination_bytes[0] = stale_destination_bytes[0].wrapping_add(1);+ tokio::fs::write(&destination, stale_destination_bytes).await?;+ tokio::fs::write(&crc_path, &expected_crc).await?;++ let task = manager_task(+ &served_file.file.url,+ &destination,+ FileCheck::CRC(expected_crc),+ Some(served_file.file.size as u64),+ )+ .await?;++ assert_eq!(task.state().await.phase, FileDownloadPhase::NotDownloaded);+ assert!(!destination.exists());Ok(())}#[tokio::test(flavor = "multi_thread")]-async fn test_manager_startup_revalidates_stale_integrity_receipt() -> Result<(), Box<dyn Error>> {+async fn test_manager_startup_revalidates_stale_metadata_crc_cache() -> Result<(), Box<dyn std::error::Error>> {let registry = MockRegistry::start().await?;let served_file = registry.file("config.json")?;let expected_crc = served_file.crc32c()?;let temporary_directory = tempdir()?;let destination = temporary_directory.path().join(&served_file.file.name);- let integrity_path = destination.with_added_extension("integrity");+ let crc_path = std::path::PathBuf::from(format!("{}.crc", destination.display()));let mut stale_destination_bytes = served_file.bytes.to_vec();stale_destination_bytes[0] = stale_destination_bytes[0].wrapping_add(1);- tokio_write(&destination, stale_destination_bytes).await?;- tokio_write(- &integrity_path,- to_vec(&json!({+ tokio::fs::write(&destination, stale_destination_bytes).await?;+ tokio::fs::write(+ &crc_path,+ serde_json::to_vec(&serde_json::json!({"version": 1,- "file_check": { "CRC": expected_crc.clone() },+ "crc": expected_crc.clone(),"file_size": served_file.file.size,"modified_unix_seconds": 0,"modified_nanos": 0,@@ -74,12 +99,12 @@ async fn test_manager_startup_revalidates_stale_integrity_receipt() -> Result<()}#[tokio::test(flavor = "multi_thread")]-async fn test_manager_startup_directory_destination_is_not_downloaded() -> Result<(), Box<dyn Error>> {+async fn test_manager_startup_directory_destination_is_not_downloaded() -> Result<(), Box<dyn std::error::Error>> {let registry = MockRegistry::start().await?;let served_file = registry.file("config.json")?;let temporary_directory = tempdir()?;let destination = temporary_directory.path().join(&served_file.file.name);- tokio_create_dir(&destination).await?;+ tokio::fs::create_dir(&destination).await?;let task = manager_task(&served_file.file.url, &destination, FileCheck::None, None).await?;@@ -88,23 +113,23 @@ async fn test_manager_startup_directory_destination_is_not_downloaded() -> Resul}#[tokio::test(flavor = "multi_thread")]-async fn test_manager_startup_foreign_lock_preserves_files() -> Result<(), Box<dyn Error>> {+async fn test_manager_startup_foreign_lock_preserves_files() -> Result<(), Box<dyn std::error::Error>> {let registry = MockRegistry::start().await?;let served_file = registry.file("config.json")?;let temporary_directory = tempdir()?;let destination = temporary_directory.path().join(&served_file.file.name);- let integrity_path = destination.with_added_extension("integrity");- let resume_artifact = destination.with_added_extension("part");- let lock_path = destination.with_added_extension("lock");- tokio_write(&destination, b"corrupt").await?;- tokio_write(&integrity_path, served_file.crc32c()?).await?;- tokio_write(&resume_artifact, b"partial").await?;- tokio_write(+ let crc_path = std::path::PathBuf::from(format!("{}.crc", destination.display()));+ let resume_artifact = destination.with_extension("part");+ let lock_path = std::path::PathBuf::from(format!("{}.lock", destination.display()));+ tokio::fs::write(&destination, b"corrupt").await?;+ tokio::fs::write(&crc_path, served_file.crc32c()?).await?;+ tokio::fs::write(&resume_artifact, b"partial").await?;+ tokio::fs::write(&lock_path,- to_vec(&json!({+ serde_json::to_vec(&serde_json::json!({"manager_id": "foreign-manager","acquired_at": Utc::now(),- "process_id": id(),+ "process_id": std::process::id(),}))?,).await?;@@ -119,17 +144,17 @@ async fn test_manager_startup_foreign_lock_preserves_files() -> Result<(), Box<dassert!(matches!(task.state().await.phase, FileDownloadPhase::LockedByOther(_)));assert!(destination.exists());- assert!(integrity_path.exists());+ assert!(crc_path.exists());assert!(resume_artifact.exists());Ok(())}async fn manager_task(source_url: &str,- destination: &Path,+ destination: &std::path::Path,file_check: FileCheck,expected_bytes: Option<u64>,-) -> Result<Arc<dyn FileDownloadTask>, DownloadError> {+) -> Result<std::sync::Arc<dyn download_manager::FileDownloadTask>, download_manager::DownloadError> {let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Universal, RuntimeHandle::current()).await?;- manager.file_download_task(source_url.into(), destination, file_check, expected_bytes).await+ manager.file_download_task(source_url, destination, file_check, expected_bytes).await}diff --git a/crates/legacy/download-manager/tests/unit/backends/common/test_state_and_lock_regression.rs b/crates/legacy/download-manager/tests/unit/backends/common/test_state_and_lock_regression.rsindex c4a0d23be..d5d486054 100644--- a/crates/legacy/download-manager/tests/unit/backends/common/test_state_and_lock_regression.rs+++ b/crates/legacy/download-manager/tests/unit/backends/common/test_state_and_lock_regression.rs@@ -1,9 +1,8 @@-use std::{error::Error, sync::Arc};+use std::sync::Arc;use download_manager::{DownloadError, FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase};use kiban::rt::RuntimeHandle;use rstest::rstest;-use tempfile::tempdir;use crate::common::{Behavior, MockRegistry, wait_for_phase};@@ -13,22 +12,22 @@ use crate::common::{Behavior, MockRegistry, wait_for_phase};#[tokio::test(flavor = "multi_thread")]async fn test_concurrent_task_creation_returns_same_task(#[case] download_manager_type: FileDownloadManagerType-) -> Result<(), Box<dyn Error>> {+) -> Result<(), Box<dyn std::error::Error>> {let registry = MockRegistry::start().await?;let tokenizer = registry.file("tokenizer.json")?;- let temp_dir = tempdir().unwrap();+ let temp_dir = tempfile::tempdir().unwrap();let destination = temp_dir.path().join(&tokenizer.file.name);let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();let (first, second) = tokio::join!(manager.file_download_task(- (&tokenizer.file.url).into(),+ &tokenizer.file.url,&destination,FileCheck::CRC(tokenizer.crc32c()?),Some(tokenizer.file.size as u64),),manager.file_download_task(- (&tokenizer.file.url).into(),+ &tokenizer.file.url,&destination,FileCheck::CRC(tokenizer.crc32c()?),Some(tokenizer.file.size as u64),@@ -41,41 +40,73 @@ async fn test_concurrent_task_creation_returns_same_task(Ok(())}+#[rstest]+#[case::universal(FileDownloadManagerType::Universal)]+#[cfg_attr(target_vendor = "apple", case::apple(FileDownloadManagerType::Apple))]#[tokio::test(flavor = "multi_thread")]-async fn conflicting_task_configuration_is_rejected() -> Result<(), Box<dyn Error>> {+async fn different_urls_targeting_same_destination_are_rejected(+ #[case] download_manager_type: FileDownloadManagerType+) -> Result<(), Box<dyn std::error::Error>> {let registry = MockRegistry::start().await?;let tokenizer = registry.file("tokenizer.json")?;- let temp_dir = tempdir().unwrap();+ let temp_dir = tempfile::tempdir().unwrap();let destination = temp_dir.path().join(&tokenizer.file.name);- let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Universal, RuntimeHandle::current()).await?;+ let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();- manager+ let _first = manager.file_download_task(- (&tokenizer.file.url).into(),+ &tokenizer.file.url,&destination,FileCheck::CRC(tokenizer.crc32c()?),Some(tokenizer.file.size as u64),)- .await?;- let different_url = manager+ .await+ .unwrap();+ let second = manager.file_download_task(- "http://example.invalid/different-url".into(),+ "http://example.invalid/different-url",&destination,FileCheck::None,Some(tokenizer.file.size as u64),).await;- let different_size = manager++ assert!(matches!(second, Err(DownloadError::ConflictingConfig(_))));+ Ok(())+}++#[rstest]+#[case::universal(FileDownloadManagerType::Universal)]+#[cfg_attr(target_vendor = "apple", case::apple(FileDownloadManagerType::Apple))]+#[tokio::test(flavor = "multi_thread")]+async fn different_expected_bytes_targeting_same_destination_are_rejected(+ #[case] download_manager_type: FileDownloadManagerType+) -> Result<(), Box<dyn std::error::Error>> {+ let registry = MockRegistry::start().await?;+ let tokenizer = registry.file("tokenizer.json")?;+ let temp_dir = tempfile::tempdir().unwrap();+ let destination = temp_dir.path().join(&tokenizer.file.name);+ let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();++ let _first = manager+ .file_download_task(+ &tokenizer.file.url,+ &destination,+ FileCheck::CRC(tokenizer.crc32c()?),+ Some(tokenizer.file.size as u64),+ )+ .await+ .unwrap();+ let second = manager.file_download_task(- (&tokenizer.file.url).into(),+ &tokenizer.file.url,&destination,FileCheck::CRC(tokenizer.crc32c()?),Some(tokenizer.file.size as u64 + 1),).await;- assert!(matches!(different_url, Err(DownloadError::ConflictingConfig(_))));- assert!(matches!(different_size, Err(DownloadError::ConflictingConfig(_))));+ assert!(matches!(second, Err(DownloadError::ConflictingConfig(_))));Ok(())}@@ -85,16 +116,16 @@ async fn conflicting_task_configuration_is_rejected() -> Result<(), Box<dyn Erro#[tokio::test(flavor = "multi_thread")]async fn separate_managers_in_same_process_cannot_share_destination_lock(#[case] download_manager_type: FileDownloadManagerType-) -> Result<(), Box<dyn Error>> {+) -> Result<(), Box<dyn std::error::Error>> {let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;let tokenizer = registry.file("tokenizer.json")?;- let temp_dir = tempdir().unwrap();+ let temp_dir = tempfile::tempdir().unwrap();let destination = temp_dir.path().join(&tokenizer.file.name);let manager_a = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();let task_a = manager_a.file_download_task(- (&tokenizer.file.url).into(),+ &tokenizer.file.url,&destination,FileCheck::CRC(tokenizer.crc32c()?),Some(tokenizer.file.size as u64),@@ -108,7 +139,7 @@ async fn separate_managers_in_same_process_cannot_share_destination_lock(let manager_b = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();let task_b = manager_b.file_download_task(- (&tokenizer.file.url).into(),+ &tokenizer.file.url,&destination,FileCheck::CRC(tokenizer.crc32c()?),Some(tokenizer.file.size as u64),diff --git a/crates/legacy/download-manager/tests/unit/backends/common/test_truncated_body.rs b/crates/legacy/download-manager/tests/unit/backends/common/test_truncated_body.rsindex 8c370e59a..a33df50cf 100644--- a/crates/legacy/download-manager/tests/unit/backends/common/test_truncated_body.rs+++ b/crates/legacy/download-manager/tests/unit/backends/common/test_truncated_body.rs@@ -1,7 +1,6 @@use download_manager::{FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase};use kiban::rt::RuntimeHandle;use rstest::rstest;-use tempfile::tempdir;use crate::common::{Behavior, MockRegistry, wait_for_phase};@@ -14,17 +13,12 @@ async fn truncated_body_without_crc_fails_length_check() -> Result<(), Box<dyn std::error::Error>> {let registry = MockRegistry::start_with(Behavior::TRUNCATE_BODY).await?;let served_file = registry.file("config.json")?;- let temp_dir = tempdir().unwrap();+ let temp_dir = tempfile::tempdir().unwrap();let destination = temp_dir.path().join(&served_file.file.name);let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();let task = manager- .file_download_task(- (&served_file.file.url).into(),- &destination,- FileCheck::None,- Some(served_file.file.size as u64),- )+ .file_download_task(&served_file.file.url, &destination, FileCheck::None, Some(served_file.file.size as u64)).await.unwrap();let mut progress = task.progress().await.unwrap();@@ -42,27 +36,3 @@ async fn truncated_body_without_crc_fails_length_check();Ok(())}--#[rstest]-#[case::universal(FileDownloadManagerType::Universal)]-#[cfg_attr(target_vendor = "apple", case::apple(FileDownloadManagerType::Apple))]-#[tokio::test(flavor = "multi_thread")]-async fn oversized_body_is_rejected_before_install(- #[case] download_manager_type: FileDownloadManagerType-) -> Result<(), Box<dyn std::error::Error>> {- let registry = MockRegistry::start().await?;- let tokenizer = registry.file("tokenizer.json")?;- let temporary_directory = tempdir()?;- let destination = temporary_directory.path().join(&tokenizer.file.name);- let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await?;- let task = manager- .file_download_task((&tokenizer.file.url).into(), &destination, FileCheck::None, Some(1))- .await?;- let mut progress = task.progress().await?;-- task.download().await?;- wait_for_phase(&task, &mut progress, |phase| matches!(phase, FileDownloadPhase::Error(_))).await;-- assert!(!destination.exists());- Ok(())-}diff --git a/crates/legacy/download-manager/tests/unit/backends/universal/mod.rs b/crates/legacy/download-manager/tests/unit/backends/universal/mod.rsindex af7de0b39..522185cd3 100644--- a/crates/legacy/download-manager/tests/unit/backends/universal/mod.rs+++ b/crates/legacy/download-manager/tests/unit/backends/universal/mod.rs@@ -1,3 +1,2 @@mod test_resume;mod test_resume_content_range_validation;-mod test_size_limit;diff --git a/crates/legacy/download-manager/tests/unit/backends/universal/test_resume.rs b/crates/legacy/download-manager/tests/unit/backends/universal/test_resume.rsindex 1bbe04b72..6a47c2f51 100644--- a/crates/legacy/download-manager/tests/unit/backends/universal/test_resume.rs+++ b/crates/legacy/download-manager/tests/unit/backends/universal/test_resume.rs@@ -1,39 +1,33 @@-use std::{error::Error, sync::Arc, time::Duration};+use std::time::Duration;-use download_manager::{FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase, HttpDownloadRequest};+use download_manager::{FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase};use kiban::rt::RuntimeHandle;use tempfile::tempdir;-use tokio::{- fs::{read as tokio_read, write as tokio_write},- time::timeout as tokio_timeout,-};+use tokio::time::timeout;use wiremock::{Mock, MockServer, ResponseTemplate,- matchers::{header, method, path},+ matchers::{method, path},};#[tokio::test(flavor = "multi_thread")]-async fn resume_sends_range_and_restarts_when_ignored() -> Result<(), Box<dyn Error>> {+async fn test_universal_resume_restarts_when_server_ignores_range() -> Result<(), Box<dyn std::error::Error>> {let bytes = b"complete file body";let server = MockServer::start().await;Mock::given(method("GET")).and(path("/model.bin"))- .and(header("accept-encoding", "identity"))- .and(header("range", "bytes=7-")).respond_with(ResponseTemplate::new(200).set_body_bytes(bytes.as_slice()))- .expect(1).mount(&server).await;let temporary_directory = tempdir().unwrap();let destination = temporary_directory.path().join("model.bin");- let part_path = destination.with_added_extension("part");- tokio_write(&part_path, b"partial").await?;+ let part_path = destination.with_extension("part");+ tokio::fs::write(&part_path, b"partial").await?;let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Universal, RuntimeHandle::current()).await?;let task = manager.file_download_task(- (&format!("{}/model.bin", server.uri())).into(),+ &format!("{}/model.bin", server.uri()),&destination,FileCheck::None,Some(bytes.len() as u64),@@ -42,109 +36,9 @@ async fn resume_sends_range_and_restarts_when_ignored() -> Result<(), Box<dyn Erassert_eq!(task.state().await.phase, FileDownloadPhase::Paused);task.download().await?;- tokio_timeout(Duration::from_secs(10), task.wait()).await?;+ timeout(Duration::from_secs(10), task.wait()).await?;assert_eq!(task.state().await.phase, FileDownloadPhase::Downloaded);- assert_eq!(tokio_read(&destination).await?, bytes);- Ok(())-}--#[tokio::test(flavor = "multi_thread")]-async fn authenticated_redirect_strips_authorization_cross_origin() -> Result<(), Box<dyn Error>> {- let bytes = b"private redirected file";- let target = MockServer::start().await;- Mock::given(method("GET"))- .and(path("/cdn/model.bin"))- .and(header("accept-encoding", "identity"))- .respond_with(ResponseTemplate::new(200).set_body_bytes(bytes.as_slice()))- .expect(1)- .mount(&target)- .await;- let origin = MockServer::start().await;- Mock::given(method("GET"))- .and(path("/model.bin"))- .and(header("authorization", "Bearer test-token"))- .and(header("accept-encoding", "identity"))- .respond_with(ResponseTemplate::new(302).insert_header("Location", format!("{}/cdn/model.bin", target.uri())))- .expect(1)- .mount(&origin)- .await;- let directory = tempdir()?;- let destination = directory.path().join("model.bin");- let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Universal, RuntimeHandle::current()).await?;- let token: Arc<str> = Arc::from("test-token");- let request = HttpDownloadRequest::with_bearer_token(format!("{}/model.bin", origin.uri()), &token);-- let task = manager.file_download_task(request, &destination, FileCheck::None, Some(bytes.len() as u64)).await?;- task.download().await?;- tokio_timeout(Duration::from_secs(10), task.wait()).await?;-- let target_requests = target.received_requests().await.expect("requests should be available");- assert_eq!(target_requests.len(), 1);- assert!(target_requests[0].headers.get("authorization").is_none());- assert_eq!(tokio_read(destination).await?, bytes);- Ok(())-}--#[tokio::test(flavor = "multi_thread")]-async fn authenticated_resume_sends_bearer_and_range() -> Result<(), Box<dyn Error>> {- let bytes = b"private file body";- let resume_from = 7;- let server = MockServer::start().await;- Mock::given(method("GET"))- .and(path("/model.bin"))- .and(header("authorization", "Bearer test-token"))- .and(header("range", "bytes=7-"))- .respond_with(- ResponseTemplate::new(206)- .insert_header(- "Content-Range",- format!("bytes {resume_from}-{}/{}", bytes.len() - 1, bytes.len()),- )- .set_body_bytes(&bytes[resume_from..]),- )- .expect(1)- .mount(&server)- .await;- let directory = tempdir()?;- let destination = directory.path().join("model.bin");- tokio_write(destination.with_added_extension("part"), &bytes[..resume_from]).await?;- let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Universal, RuntimeHandle::current()).await?;- let token: Arc<str> = Arc::from("test-token");- let request = HttpDownloadRequest::with_bearer_token(format!("{}/model.bin", server.uri()), &token);- let task = manager.file_download_task(request, &destination, FileCheck::None, Some(bytes.len() as u64)).await?;-- task.download().await?;- tokio_timeout(Duration::from_secs(10), task.wait()).await?;-- assert_eq!(task.state().await.phase, FileDownloadPhase::Downloaded);- assert_eq!(tokio_read(destination).await?, bytes);- Ok(())-}--#[tokio::test(flavor = "multi_thread")]-async fn signed_redirect_url_is_redacted_from_errors() -> Result<(), Box<dyn Error>> {- let server = MockServer::start().await;- Mock::given(method("GET")).and(path("/model.bin")).respond_with(ResponseTemplate::new(500)).mount(&server).await;- let directory = tempdir()?;- let destination = directory.path().join("model.bin");- let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Universal, RuntimeHandle::current()).await?;- let task = manager- .file_download_task(- (&format!("{}/model.bin?X-Amz-Signature=super-secret", server.uri())).into(),- &destination,- FileCheck::None,- Some(1),- )- .await?;-- task.download().await?;- tokio_timeout(Duration::from_secs(10), task.wait()).await?;-- let FileDownloadPhase::Error(message) = task.state().await.phase else {- panic!("request should fail");- };- assert!(!message.contains("super-secret"));- assert!(!message.contains("X-Amz-Signature"));+ assert_eq!(tokio::fs::read(&destination).await?, bytes);Ok(())}diff --git a/crates/legacy/download-manager/tests/unit/backends/universal/test_resume_content_range_validation.rs b/crates/legacy/download-manager/tests/unit/backends/universal/test_resume_content_range_validation.rsindex c244558a1..b5de0614f 100644--- a/crates/legacy/download-manager/tests/unit/backends/universal/test_resume_content_range_validation.rs+++ b/crates/legacy/download-manager/tests/unit/backends/universal/test_resume_content_range_validation.rs@@ -1,23 +1,16 @@-use std::{error::Error, time::Duration};+use std::time::Duration;use download_manager::{FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase};use kiban::rt::RuntimeHandle;-use rstest::rstest;use tempfile::tempdir;-use tokio::{- fs::{read as tokio_read, write as tokio_write},- time::timeout as tokio_timeout,-};+use tokio::time::timeout;use wiremock::{Mock, MockServer, ResponseTemplate,- matchers::{header, method, path},+ matchers::{method, path},};-#[rstest]-#[case::offset("bytes 0-4/10")]-#[case::total("bytes 5-9/11")]#[tokio::test(flavor = "multi_thread")]-async fn invalid_content_range_restarts_from_zero(#[case] content_range: &str) -> Result<(), Box<dyn Error>> {+async fn resume_rejects_206_when_content_range_offset_mismatches_request() -> Result<(), Box<dyn std::error::Error>> {let full_bytes: &[u8] = b"abcdefghij";let partial_bytes: &[u8] = b"abcde";let total = full_bytes.len();@@ -25,44 +18,39 @@ async fn invalid_content_range_restarts_from_zero(#[case] content_range: &str) -let server = MockServer::start().await;Mock::given(method("GET")).and(path("/model.bin"))- .and(header("range", "bytes=5-")).respond_with(ResponseTemplate::new(206)- .set_body_bytes(&full_bytes[partial_bytes.len()..])- .insert_header("Content-Range", content_range),+ .set_body_bytes(partial_bytes)+ .insert_header("Content-Range", format!("bytes 0-{}/{}", total - 1, total)),)- .with_priority(1)- .expect(1)- .mount(&server)- .await;- Mock::given(method("GET"))- .and(path("/model.bin"))- .respond_with(ResponseTemplate::new(200).set_body_bytes(full_bytes))- .with_priority(2)- .expect(1).mount(&server).await;let temporary_directory = tempdir().unwrap();let destination = temporary_directory.path().join("model.bin");- let part_path = destination.with_added_extension("part");- tokio_write(&part_path, partial_bytes).await?;+ let part_path = destination.with_extension("part");+ tokio::fs::write(&part_path, partial_bytes).await?;let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Universal, RuntimeHandle::current()).await?;let task = manager- .file_download_task(- (&format!("{}/model.bin", server.uri())).into(),- &destination,- FileCheck::None,- Some(total as u64),- )+ .file_download_task(&format!("{}/model.bin", server.uri()), &destination, FileCheck::None, Some(total as u64)).await?;assert_eq!(task.state().await.phase, FileDownloadPhase::Paused);task.download().await?;- tokio_timeout(Duration::from_secs(10), task.wait()).await?;+ timeout(Duration::from_secs(10), task.wait()).await?;- assert_eq!(task.state().await.phase, FileDownloadPhase::Downloaded);- assert_eq!(tokio_read(destination).await?, full_bytes);- Ok(())+ let final_phase = task.state().await.phase;+ match &final_phase {+ FileDownloadPhase::Error(_) => Ok(()),+ FileDownloadPhase::Downloaded => {+ assert_eq!(+ tokio::fs::read(&destination).await?,+ full_bytes,+ "phase reached Downloaded but file content was corrupted by accepting a misaligned Content-Range",+ );+ Ok(())+ },+ other => panic!("unexpected final phase: {:?}", other),+ }}diff --git a/crates/legacy/download-manager/tests/unit/backends/universal/test_size_limit.rs b/crates/legacy/download-manager/tests/unit/backends/universal/test_size_limit.rsdeleted file mode 100644index 252fcd6b9..000000000--- a/crates/legacy/download-manager/tests/unit/backends/universal/test_size_limit.rs+++ /dev/null@@ -1,77 +0,0 @@-use std::{error::Error, sync::Arc, time::Duration};--use kiban::{fs, rt::RuntimeHandle};-use tempfile::tempdir;-use tokio::{- sync::{- Mutex as TokioMutex,- mpsc::{Receiver as TokioMpscReceiver, channel as tokio_mpsc_channel},- watch::channel as tokio_watch_channel,- },- time::timeout as tokio_timeout,-};-use uuid::Uuid;-use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method};--use crate::{- FileCheck,- backends::universal::UniversalBackendContext,- compute_download_id,- file_download_task_actor::{BackendEvent, PendingProgressSlot},- lock_manager::DestinationLockLease,- traits::{ActiveDownloadGeneration, BackendContext, BackendEventSender, DownloadConfig},-};--fn backend_event_sender() -> (BackendEventSender, TokioMpscReceiver<BackendEvent>) {- let (sender, receiver) = tokio_mpsc_channel(8);- let pending_progress = Arc::new(TokioMutex::new(PendingProgressSlot::default()));- let (progress_waker, _) = tokio_watch_channel(());- (BackendEventSender::new(Uuid::new_v4(), sender, pending_progress, progress_waker), receiver)-}--#[tokio::test(flavor = "multi_thread")]-async fn response_body_cannot_grow_part_beyond_expected_size() -> Result<(), Box<dyn Error>> {- let server = MockServer::start().await;- Mock::given(method("GET"))- .respond_with(ResponseTemplate::new(200).set_body_bytes(vec![0; 256 * 1024]))- .expect(1)- .mount(&server)- .await;- let directory = tempdir()?;- let destination = directory.path().join("model.bin");- let config = Arc::new(DownloadConfig {- download_id: compute_download_id(&destination),- request: server.uri().into(),- destination: destination.clone(),- file_check: FileCheck::None,- expected_bytes: Some(1),- manager_id: "test-manager".to_string(),- manager_instance_id: Uuid::new_v4(),- });- let destination_lease = DestinationLockLease::acquire_for_destination(- &destination,- &config.manager_id,- config.manager_instance_id,- )- .await?;- let mut context = UniversalBackendContext::new(RuntimeHandle::current());- context.retries = 0;- let (sender, mut receiver) = backend_event_sender();-- let active_task = context- .download(- Arc::clone(&config),- ActiveDownloadGeneration::new(0),- sender,- &destination_lease,- )- .await?;- let event = tokio_timeout(Duration::from_secs(5), receiver.recv()).await?;-- assert!(matches!(event, Some(BackendEvent::Error { .. })));- assert!(fs::asyn::file_length(destination.with_added_extension("part")).await.unwrap_or(0) <= 1);- assert!(!destination.exists());- drop(active_task);- destination_lease.release().await?;- Ok(())-}diff --git a/crates/legacy/download-manager/tests/unit/mod.rs b/crates/legacy/download-manager/tests/unit/mod.rsindex 5712649e9..0637eea96 100644--- a/crates/legacy/download-manager/tests/unit/mod.rs+++ b/crates/legacy/download-manager/tests/unit/mod.rs@@ -1,4 +1,2 @@mod backends;-mod test_http_download_request;-mod test_integrity;mod test_lock_manager;diff --git a/crates/legacy/download-manager/tests/unit/test_http_download_request.rs b/crates/legacy/download-manager/tests/unit/test_http_download_request.rsdeleted file mode 100644index e93c1d311..000000000--- a/crates/legacy/download-manager/tests/unit/test_http_download_request.rs+++ /dev/null@@ -1,45 +0,0 @@-use std::sync::Arc;--use crate::{DownloadError, HttpDownloadRequest};--#[test]-fn request_debug_redacts_bearer_token() {- let token: Arc<str> = Arc::from("secret-token");- let request = HttpDownloadRequest::with_bearer_token("https://example.com/model?signature=signed-secret", &token);- let debug = format!("{request:?}");-- assert!(debug.contains("authenticated: true"));- assert!(!debug.contains("secret-token"));- assert!(!debug.contains("signed-secret"));-}--#[test]-fn authenticated_request_requires_https_except_for_loopback() {- let token: Arc<str> = Arc::from("token");- let insecure = HttpDownloadRequest::with_bearer_token("http://example.com/model", &token);- let loopback = HttpDownloadRequest::with_bearer_token("http://127.0.0.1/model", &token);-- assert_eq!(insecure.validate(), Err(DownloadError::InsecureAuthenticatedRequest));- assert_eq!(loopback.validate(), Ok(()));-}--#[test]-fn request_does_not_keep_credentials_alive() {- let token: Arc<str> = Arc::from("token");- let request = HttpDownloadRequest::with_bearer_token("https://example.com/model", &token);- drop(token);-- assert_eq!(request.bearer_token(), Err(DownloadError::AuthenticationUnavailable));-}--#[test]-fn authenticated_requests_match_only_the_same_credential_owner() {- let first_token: Arc<str> = Arc::from("token");- let second_token: Arc<str> = Arc::from("token");- let first = HttpDownloadRequest::with_bearer_token("https://example.com/model", &first_token);- let first_clone = HttpDownloadRequest::with_bearer_token("https://example.com/model", &first_token);- let second = HttpDownloadRequest::with_bearer_token("https://example.com/model", &second_token);-- assert_eq!(first, first_clone);- assert_ne!(first, second);-}diff --git a/crates/legacy/download-manager/tests/unit/test_integrity.rs b/crates/legacy/download-manager/tests/unit/test_integrity.rsdeleted file mode 100644index 145bbe706..000000000--- a/crates/legacy/download-manager/tests/unit/test_integrity.rs+++ /dev/null@@ -1,87 +0,0 @@-use std::error::Error;--use base64::{Engine, engine::general_purpose::STANDARD};-use crc32c::crc32c;-use tempfile::tempdir;-use tokio::fs::write as tokio_write;--use crate::{- DownloadInfo, FileCheck,- integrity::{VerificationError, integrity_cache_matches, save_integrity_cache, verify_file_integrity},-};--const HELLO: &[u8] = b"hello\n";-const HELLO_SHA256: &str = "5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03";-const HELLO_GIT_BLOB_SHA1: &str = "ce013625030ba8dba906f756967f9e9ca394464a";--#[tokio::test]-async fn sha256_detects_same_size_corruption() -> Result<(), Box<dyn Error>> {- let directory = tempdir()?;- let path = directory.path().join("model.bin");- let check = FileCheck::Sha256(HELLO_SHA256.to_string());- tokio_write(&path, HELLO).await?;- assert!(verify_file_integrity(&path, &check).await?);-- tokio_write(&path, b"jello\n").await?;- assert!(!verify_file_integrity(&path, &check).await?);- Ok(())-}--#[tokio::test]-async fn git_blob_sha1_includes_the_blob_header() -> Result<(), Box<dyn Error>> {- let directory = tempdir()?;- let path = directory.path().join("config.json");- let check = FileCheck::GitBlobSha1(HELLO_GIT_BLOB_SHA1.to_string());- tokio_write(&path, HELLO).await?;-- assert!(verify_file_integrity(&path, &check).await?);- Ok(())-}--#[tokio::test]-async fn digest_cache_is_invalidated_when_file_changes() -> Result<(), Box<dyn Error>> {- let directory = tempdir()?;- let path = directory.path().join("weights.bin");- let check = FileCheck::Sha256(HELLO_SHA256.to_string());- tokio_write(&path, HELLO).await?;- save_integrity_cache(&path, &check).await?;- assert!(integrity_cache_matches(&path, &check).await);-- tokio_write(&path, b"jello\n").await?;- assert!(!integrity_cache_matches(&path, &check).await);- Ok(())-}--#[tokio::test]-async fn crc_and_invalid_digest_behavior_remain_explicit() -> Result<(), Box<dyn Error>> {- let directory = tempdir()?;- let path = directory.path().join("tokenizer.json");- tokio_write(&path, HELLO).await?;- let crc = STANDARD.encode(crc32c(HELLO).to_be_bytes());- assert!(verify_file_integrity(&path, &FileCheck::CRC(crc)).await?);-- let error = verify_file_integrity(&path, &FileCheck::Sha256("invalid".to_string()))- .await- .expect_err("invalid digest must not be treated as corrupt content");- assert!(matches!(- error,- VerificationError::InvalidExpectedDigest {- algorithm: "sha256"- }- ));- Ok(())-}--#[test]-fn apple_task_info_preserves_new_and_legacy_checks() -> Result<(), Box<dyn Error>> {- let check = FileCheck::Sha256(HELLO_SHA256.to_string());- let info = DownloadInfo::new("https://example.com/model", "/tmp/model", check.clone());- let restored = DownloadInfo::from_json(&info.to_json()?)?;- assert_eq!(restored.resolved_file_check(), check);-- let legacy = DownloadInfo::from_json(- r#"{"source_url":"https://example.com/model","destination_path":"/tmp/model","crc32c":"AAAAAA=="}"#,- )?;- assert_eq!(legacy.resolved_file_check(), FileCheck::CRC("AAAAAA==".to_string()));- Ok(())-}diff --git a/crates/legacy/mock-registry/src/model.rs b/crates/legacy/mock-registry/src/model.rsindex a07146481..8b59bdca0 100644--- a/crates/legacy/mock-registry/src/model.rs+++ b/crates/legacy/mock-registry/src/model.rs@@ -1,6 +1,6 @@use shoji::types::{basic::Metadata,- model::{Model, ModelAccessibility, ModelBackend, ModelRegistry, ModelSource},+ model::{Model, ModelAccessibility, ModelBackend, ModelReference, ModelRegistry},};use crate::ServedFile;@@ -36,8 +36,8 @@ pub fn mock_model(properties: None,quantization: None,specializations: vec![],- accessibility: ModelAccessibility::OnDevice {- source: ModelSource::Registry {+ accessibility: ModelAccessibility::Local {+ reference: ModelReference::Mirai {toolchain_version: "1.0".to_string(),repository: None,source_repository: None,diff --git a/crates/legacy/shoji/src/types/basic/file.rs b/crates/legacy/shoji/src/types/basic/file.rsindex 7ab0e454c..c6f5ca40c 100644--- a/crates/legacy/shoji/src/types/basic/file.rs+++ b/crates/legacy/shoji/src/types/basic/file.rs@@ -18,4 +18,9 @@ impl File {pub fn crc32c(&self) -> Option<String> {self.hashes.iter().find(|hash| hash.method == HashMethod::CRC32C).map(|hash| hash.value.clone())}++ #[bindings::export(Method(Getter))]+ pub fn md5(&self) -> Option<String> {+ self.hashes.iter().find(|hash| hash.method == HashMethod::MD5).map(|hash| hash.value.clone())+ }}diff --git a/crates/legacy/shoji/src/types/basic/hash.rs b/crates/legacy/shoji/src/types/basic/hash.rsindex 3c6581b94..7a8032e83 100644--- a/crates/legacy/shoji/src/types/basic/hash.rs+++ b/crates/legacy/shoji/src/types/basic/hash.rs@@ -6,6 +6,8 @@ use serde::{Deserialize, Serialize};pub enum HashMethod {#[serde(rename = "crc32c")]CRC32C,+ #[serde(rename = "md5")]+ MD5,}#[bindings::export(Structure(Class))]diff --git a/crates/legacy/shoji/src/types/model/accessibility.rs b/crates/legacy/shoji/src/types/model/accessibility.rsindex 85392e835..82d01870d 100644--- a/crates/legacy/shoji/src/types/model/accessibility.rs+++ b/crates/legacy/shoji/src/types/model/accessibility.rs@@ -1,13 +1,13 @@use serde::{Deserialize, Serialize};-use crate::types::{basic::Repository, model::ModelSource};+use crate::types::{basic::Repository, model::ModelReference};#[bindings::export(Enumeration)]#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]#[serde(tag = "type", rename_all = "snake_case")]pub enum ModelAccessibility {- OnDevice {- source: ModelSource,+ Local {+ reference: ModelReference,},Remote {repository: Option<Repository>,diff --git a/crates/legacy/shoji/src/types/model/mod.rs b/crates/legacy/shoji/src/types/model/mod.rsindex 69fdf1941..a30a19526 100644--- a/crates/legacy/shoji/src/types/model/mod.rs+++ b/crates/legacy/shoji/src/types/model/mod.rs@@ -4,8 +4,8 @@ mod family;mod model;mod properties;mod quantization;+mod reference;mod registry;-mod source;mod specialization;mod vendor;@@ -15,7 +15,7 @@ pub use family::ModelFamily;pub use model::Model;pub use properties::ModelProperties;pub use quantization::ModelQuantization;+pub use reference::ModelReference;pub use registry::ModelRegistry;-pub use source::ModelSource;pub use specialization::ModelSpecialization;pub use vendor::ModelVendor;diff --git a/crates/legacy/shoji/src/types/model/model.rs b/crates/legacy/shoji/src/types/model/model.rsindex a3ca910a8..31fa14c3a 100644--- a/crates/legacy/shoji/src/types/model/model.rs+++ b/crates/legacy/shoji/src/types/model/model.rs@@ -3,8 +3,8 @@ use serde::{Deserialize, Serialize};use crate::types::{basic::{Metadata, Value},model::{- ModelAccessibility, ModelBackend, ModelFamily, ModelProperties, ModelQuantization, ModelRegistry, ModelSource,- ModelSpecialization,+ ModelAccessibility, ModelBackend, ModelFamily, ModelProperties, ModelQuantization, ModelReference,+ ModelRegistry, ModelSpecialization,},};@@ -43,8 +43,8 @@ impl Model {}#[bindings::export(Method(Getter))]- pub fn is_on_device(&self) -> bool {- matches!(self.accessibility, ModelAccessibility::OnDevice { .. })+ pub fn is_local(&self) -> bool {+ matches!(self.accessibility, ModelAccessibility::Local { .. })}#[bindings::export(Method(Getter))]@@ -56,8 +56,8 @@ impl Model {pub fn is_downloadable(&self) -> bool {matches!(self.accessibility,- ModelAccessibility::OnDevice {- source: ModelSource::Registry { .. }+ ModelAccessibility::Local {+ reference: ModelReference::Mirai { .. } | ModelReference::HuggingFace { .. }})}@@ -75,11 +75,11 @@ impl Model {#[bindings::export(Method(Getter))]pub fn repo_ids(&self) -> Vec<String> {match &self.accessibility {- ModelAccessibility::OnDevice {- source,+ ModelAccessibility::Local {+ reference,..- } => match source {- ModelSource::Registry {+ } => match reference {+ ModelReference::Mirai {repository,source_repository,..@@ -93,7 +93,10 @@ impl Model {}result},- ModelSource::Filesystem {+ ModelReference::HuggingFace {+ repository,+ } => vec![repository.identifier.clone()],+ ModelReference::Local {..} => vec![],},@@ -111,16 +114,19 @@ impl Model {}#[bindings::export(Method(Getter))]- pub fn filesystem_path(&self) -> Option<String> {+ pub fn local_external_path(&self) -> Option<String> {match &self.accessibility {- ModelAccessibility::OnDevice {- source,+ ModelAccessibility::Local {+ reference,..- } => match source {- ModelSource::Registry {+ } => match reference {+ ModelReference::Mirai {+ ..+ } => None,+ ModelReference::HuggingFace {..} => None,- ModelSource::Filesystem {+ ModelReference::Local {path,} => Some(path.clone()),},@@ -133,10 +139,10 @@ impl Model {#[bindings::export(Method(Getter))]pub fn reference_name(&self) -> Option<String> {match &self.accessibility {- ModelAccessibility::OnDevice {- source,+ ModelAccessibility::Local {+ reference,..- } => Some(source.name()),+ } => Some(reference.name()),ModelAccessibility::Remote {..} => None,@@ -146,19 +152,18 @@ impl Model {#[bindings::export(Method(Getter))]pub fn checkpoint_version(&self) -> Option<String> {match &self.accessibility {- ModelAccessibility::OnDevice {- source,+ ModelAccessibility::Local {+ reference,..- } => match source {- ModelSource::Registry {+ } => match reference {+ ModelReference::Mirai {toolchain_version,- repository,..- } => repository- .as_ref()- .and_then(|repository| repository.commit_hash.clone())- .or_else(|| Some(toolchain_version.clone())),- ModelSource::Filesystem {+ } => Some(toolchain_version.clone()),+ ModelReference::HuggingFace {+ repository,+ } => repository.commit_hash.clone(),+ ModelReference::Local {..} => None,},diff --git a/crates/legacy/shoji/src/types/model/source.rs b/crates/legacy/shoji/src/types/model/reference.rssimilarity index 58%rename from crates/legacy/shoji/src/types/model/source.rsrename to crates/legacy/shoji/src/types/model/reference.rsindex dc16e1325..019e93a8d 100644--- a/crates/legacy/shoji/src/types/model/source.rs+++ b/crates/legacy/shoji/src/types/model/reference.rs@@ -5,28 +5,33 @@ use crate::types::basic::{File, Repository};#[bindings::export(Enumeration)]#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]#[serde(tag = "type", rename_all = "snake_case")]-pub enum ModelSource {- #[serde(rename = "managed")]- Registry {+pub enum ModelReference {+ Mirai {toolchain_version: String,repository: Option<Repository>,source_repository: Option<Repository>,files: Vec<File>,},- Filesystem {+ HuggingFace {+ repository: Repository,+ },+ Local {path: String,},}-impl ModelSource {+impl ModelReference {pub fn name(&self) -> String {match self {- ModelSource::Registry {+ ModelReference::Mirai {+ ..+ } => "mirai".to_string(),+ ModelReference::HuggingFace {..- } => "registry".to_string(),- ModelSource::Filesystem {+ } => "huggingface".to_string(),+ ModelReference::Local {..- } => "filesystem".to_string(),+ } => "local".to_string(),}}}diff --git a/crates/legacy/uzu/Cargo.toml b/crates/legacy/uzu/Cargo.tomlindex aae7731de..3317e73d2 100644--- a/crates/legacy/uzu/Cargo.toml+++ b/crates/legacy/uzu/Cargo.toml@@ -63,6 +63,7 @@ uzu-engine.workspace = truekiban.workspace = trueshoji.workspace = trueanyhow.workspace = true+bitflags.workspace = trueuniffi = { workspace = true, optional = true }napi = { workspace = true, optional = true }napi-derive = { workspace = true, optional = true }@@ -133,11 +134,11 @@ rstest.workspace = true[[test]]name = "tool"-path = "tests/integration/tool/mod.rs"+path = "tests/tool/mod.rs"[[test]]-name = "integration"-path = "tests/integration/test_entry.rs"+name = "unit"+path = "tests/unit/test_entry.rs"[build-dependencies]napi-build = { workspace = true, optional = true }diff --git a/crates/legacy/uzu/bindings/python/tests/test_engine.py b/crates/legacy/uzu/bindings/python/tests/test_engine.pyindex cf7ed5d7f..b765195ee 100644--- a/crates/legacy/uzu/bindings/python/tests/test_engine.py+++ b/crates/legacy/uzu/bindings/python/tests/test_engine.py@@ -1,6 +1,6 @@import asyncio-from uzu import ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfig, ReasoningEffort, SamplingMethod+from uzu import ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfigdef test_chat_reply_produces_text() -> None:@@ -16,17 +16,15 @@ def test_chat_reply_produces_text() -> None:session = await engine.chat(model, ChatConfig.create())messages = [- ChatMessage.system()- .with_text("You are a helpful assistant")- .with_reasoning_effort(ReasoningEffort.Disabled),+ ChatMessage.system().with_text("You are a helpful assistant"),ChatMessage.user().with_text("Hi"),]- config = ChatReplyConfig.create().with_token_limit(64).with_sampling_method(SamplingMethod.Greedy())- replies = await session.reply(messages, config)+ replies = await session.reply(messages, ChatReplyConfig.create())assert replies, "Reply has no messages"message = replies[-1].message+ assert message.reasoning is not Noneassert message.text is not Noneasyncio.run(run())diff --git a/crates/legacy/uzu/bindings/python/uzu/__init__.pyi b/crates/legacy/uzu/bindings/python/uzu/__init__.pyiindex fca414766..95c6d64c3 100644--- a/crates/legacy/uzu/bindings/python/uzu/__init__.pyi+++ b/crates/legacy/uzu/bindings/python/uzu/__init__.pyi@@ -61,8 +61,8 @@ __all__ = ["ModelFamily","ModelProperties","ModelQuantization",+ "ModelReference","ModelRegistry",- "ModelSource","ModelSpecialization","ModelVendor","PcmBatch",@@ -696,7 +696,7 @@ class Engine:def remove_registry(self, registry_identifier: builtins.str) -> collections.abc.Awaitable[None]: ...def remove_backend(self, identifier: builtins.str) -> collections.abc.Awaitable[None]: ...def models(self) -> collections.abc.Awaitable[builtins.list[Model]]: ...- def models_on_device(self) -> collections.abc.Awaitable[builtins.list[Model]]: ...+ def models_local(self) -> collections.abc.Awaitable[builtins.list[Model]]: ...def models_remote(self) -> collections.abc.Awaitable[builtins.list[Model]]: ...def models_downloadable(self) -> collections.abc.Awaitable[builtins.list[Model]]: ...def models_for_chat(self) -> collections.abc.Awaitable[builtins.list[Model]]: ...@@ -798,6 +798,8 @@ class File:def hashes(self) -> builtins.list[Hash]: ...@propertydef crc32c(self) -> typing.Optional[builtins.str]: ...+ @property+ def md5(self) -> typing.Optional[builtins.str]: ...def __new__(cls, url: builtins.str, name: builtins.str, size: builtins.int, hashes: typing.Sequence[Hash]) -> File: ...class Grammar:@@ -885,7 +887,7 @@ class Model:@propertydef name(self) -> builtins.str: ...@property- def is_on_device(self) -> builtins.bool: ...+ def is_local(self) -> builtins.bool: ...@propertydef is_remote(self) -> builtins.bool: ...@property@@ -897,7 +899,7 @@ class Model:@propertydef repo_ids(self) -> builtins.list[builtins.str]: ...@property- def filesystem_path(self) -> typing.Optional[builtins.str]: ...+ def local_external_path(self) -> typing.Optional[builtins.str]: ...@propertydef reference_name(self) -> typing.Optional[builtins.str]: ...@property@@ -918,11 +920,11 @@ class Model:class ModelAccessibility:@typing.final- class OnDevice(ModelAccessibility):- __match_args__ = ("source",)+ class Local(ModelAccessibility):+ __match_args__ = ("reference",)@property- def source(self) -> ModelSource: ...- def __new__(cls, source: ModelSource) -> ModelAccessibility.OnDevice: ...+ def reference(self) -> ModelReference: ...+ def __new__(cls, reference: ModelReference) -> ModelAccessibility.Local: ...@typing.finalclass Remote(ModelAccessibility):@@ -987,19 +989,9 @@ class ModelQuantization:def name(self) -> builtins.str: ...def __new__(cls, identifier: builtins.str, method: builtins.str, bits_per_weight: builtins.int, vendor: ModelVendor, metadata: Metadata) -> ModelQuantization: ...-class ModelRegistry:- @property- def identifier(self) -> builtins.str: ...- @property- def metadata(self) -> Metadata: ...- @property- def name(self) -> builtins.str: ...- def __new__(cls, identifier: builtins.str, metadata: Metadata) -> ModelRegistry: ...--class ModelSource:+class ModelReference:@typing.final- class Registry(ModelSource):+ class Mirai(ModelReference):__match_args__ = ("toolchain_version", "repository", "source_repository", "files",)@propertydef toolchain_version(self) -> builtins.str: ...@@ -1009,17 +1001,34 @@ class ModelSource:def source_repository(self) -> typing.Optional[Repository]: ...@propertydef files(self) -> builtins.list[File]: ...- def __new__(cls, toolchain_version: builtins.str, repository: typing.Optional[Repository], source_repository: typing.Optional[Repository], files: typing.Sequence[File]) -> ModelSource.Registry: ...+ def __new__(cls, toolchain_version: builtins.str, repository: typing.Optional[Repository], source_repository: typing.Optional[Repository], files: typing.Sequence[File]) -> ModelReference.Mirai: ...@typing.final- class Filesystem(ModelSource):+ class HuggingFace(ModelReference):+ __match_args__ = ("repository",)+ @property+ def repository(self) -> Repository: ...+ def __new__(cls, repository: Repository) -> ModelReference.HuggingFace: ...++ @typing.final+ class Local(ModelReference):__match_args__ = ("path",)@propertydef path(self) -> builtins.str: ...- def __new__(cls, path: builtins.str) -> ModelSource.Filesystem: ...+ def __new__(cls, path: builtins.str) -> ModelReference.Local: ......+class ModelRegistry:+ @property+ def identifier(self) -> builtins.str: ...+ @property+ def metadata(self) -> Metadata: ...+ @property+ def name(self) -> builtins.str: ...+ def __new__(cls, identifier: builtins.str, metadata: Metadata) -> ModelRegistry: ...+class ModelSpecialization:@typing.finalclass Chat(ModelSpecialization):@@ -1352,6 +1361,7 @@ class DownloadManagerType(enum.Enum):@typing.finalclass HashMethod(enum.Enum):CRC32C = ...+ MD5 = ...@typing.finalclass ImageFormat(enum.Enum):diff --git a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swiftindex 762a75aeb..e095452fc 100644--- a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift+++ b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift@@ -1741,6 +1741,14 @@ public func crc32c() -> String? {})}+public func md5() -> String? {+ return try! FfiConverterOptionString.lift(try! rustCall() {+ uniffi_shoji_fn_method_file_md5(+ FfiConverterTypeFile_lower(self),$0+ )+})+}+}@@ -2019,14 +2027,6 @@ public func checkpointVersion() -> String? {})}-public func filesystemPath() -> String? {- return try! FfiConverterOptionString.lift(try! rustCall() {- uniffi_shoji_fn_method_model_filesystem_path(- FfiConverterTypeModel_lower(self),$0- )-})-}-public func isChatCapable() -> Bool {return try! FfiConverterBool.lift(try! rustCall() {uniffi_shoji_fn_method_model_is_chat_capable(@@ -2051,9 +2051,9 @@ public func isDownloadable() -> Bool {})}-public func isOnDevice() -> Bool {+public func isLocal() -> Bool {return try! FfiConverterBool.lift(try! rustCall() {- uniffi_shoji_fn_method_model_is_on_device(+ uniffi_shoji_fn_method_model_is_local(FfiConverterTypeModel_lower(self),$0)})@@ -2099,6 +2099,14 @@ public func isTranslationCapable() -> Bool {})}+public func localExternalPath() -> String? {+ return try! FfiConverterOptionString.lift(try! rustCall() {+ uniffi_shoji_fn_method_model_local_external_path(+ FfiConverterTypeModel_lower(self),$0+ )+})+}+public func name() -> String {return try! FfiConverterString.lift(try! rustCall() {uniffi_shoji_fn_method_model_name(@@ -4192,6 +4200,7 @@ public func FfiConverterTypeGrammar_lower(_ value: Grammar) -> RustBuffer {public enum HashMethod: Equatable, Hashable, Codable {case crc32c+ case md5@@ -4215,6 +4224,8 @@ public struct FfiConverterTypeHashMethod: FfiConverterRustBuffer {case 1: return .crc32c+ case 2: return .md5+default: throw UniffiInternalError.unexpectedEnumCase}}@@ -4226,6 +4237,10 @@ public struct FfiConverterTypeHashMethod: FfiConverterRustBuffer {case .crc32c:writeInt(&buf, Int32(1))++ case .md5:+ writeInt(&buf, Int32(2))+}}}@@ -4392,7 +4407,7 @@ public func FfiConverterTypeImageTheme_lower(_ value: ImageTheme) -> RustBufferpublic enum ModelAccessibility: Equatable, Hashable, Codable {- case onDevice(source: ModelSource+ case local(reference: ModelReference)case remote(repository: Repository?)@@ -4417,7 +4432,7 @@ public struct FfiConverterTypeModelAccessibility: FfiConverterRustBuffer {let variant: Int32 = try readInt(&buf)switch variant {- case 1: return .onDevice(source: try FfiConverterTypeModelSource.read(from: &buf)+ case 1: return .local(reference: try FfiConverterTypeModelReference.read(from: &buf))case 2: return .remote(repository: try FfiConverterOptionTypeRepository.read(from: &buf)@@ -4431,9 +4446,9 @@ public struct FfiConverterTypeModelAccessibility: FfiConverterRustBuffer {switch value {- case let .onDevice(source):+ case let .local(reference):writeInt(&buf, Int32(1))- FfiConverterTypeModelSource.write(source, into: &buf)+ FfiConverterTypeModelReference.write(reference, into: &buf)case let .remote(repository):@@ -4463,11 +4478,13 @@ public func FfiConverterTypeModelAccessibility_lower(_ value: ModelAccessibility// Note that we don't yet support `indirect` for enums.// See https://github.com/mozilla/uniffi-rs/issues/396 for further discussion.-public enum ModelSource: Equatable, Hashable, Codable {+public enum ModelReference: Equatable, Hashable, Codable {- case registry(toolchainVersion: String, repository: Repository?, sourceRepository: Repository?, files: [File]+ case mirai(toolchainVersion: String, repository: Repository?, sourceRepository: Repository?, files: [File]+ )+ case huggingFace(repository: Repository)- case filesystem(path: String+ case local(path: String)@@ -4477,34 +4494,37 @@ public enum ModelSource: Equatable, Hashable, Codable {}#if compiler(>=6)-extension ModelSource: Sendable {}+extension ModelReference: Sendable {}#endif#if swift(>=5.8)@_documentation(visibility: private)#endif-public struct FfiConverterTypeModelSource: FfiConverterRustBuffer {- typealias SwiftType = ModelSource+public struct FfiConverterTypeModelReference: FfiConverterRustBuffer {+ typealias SwiftType = ModelReference- public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ModelSource {+ public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ModelReference {let variant: Int32 = try readInt(&buf)switch variant {- case 1: return .registry(toolchainVersion: try FfiConverterString.read(from: &buf), repository: try FfiConverterOptionTypeRepository.read(from: &buf), sourceRepository: try FfiConverterOptionTypeRepository.read(from: &buf), files: try FfiConverterSequenceTypeFile.read(from: &buf)+ case 1: return .mirai(toolchainVersion: try FfiConverterString.read(from: &buf), repository: try FfiConverterOptionTypeRepository.read(from: &buf), sourceRepository: try FfiConverterOptionTypeRepository.read(from: &buf), files: try FfiConverterSequenceTypeFile.read(from: &buf)+ )++ case 2: return .huggingFace(repository: try FfiConverterTypeRepository.read(from: &buf))- case 2: return .filesystem(path: try FfiConverterString.read(from: &buf)+ case 3: return .local(path: try FfiConverterString.read(from: &buf))default: throw UniffiInternalError.unexpectedEnumCase}}- public static func write(_ value: ModelSource, into buf: inout [UInt8]) {+ public static func write(_ value: ModelReference, into buf: inout [UInt8]) {switch value {- case let .registry(toolchainVersion,repository,sourceRepository,files):+ case let .mirai(toolchainVersion,repository,sourceRepository,files):writeInt(&buf, Int32(1))FfiConverterString.write(toolchainVersion, into: &buf)FfiConverterOptionTypeRepository.write(repository, into: &buf)@@ -4512,8 +4532,13 @@ public struct FfiConverterTypeModelSource: FfiConverterRustBuffer {FfiConverterSequenceTypeFile.write(files, into: &buf)- case let .filesystem(path):+ case let .huggingFace(repository):writeInt(&buf, Int32(2))+ FfiConverterTypeRepository.write(repository, into: &buf)+++ case let .local(path):+ writeInt(&buf, Int32(3))FfiConverterString.write(path, into: &buf)}@@ -4524,15 +4549,15 @@ public struct FfiConverterTypeModelSource: FfiConverterRustBuffer {#if swift(>=5.8)@_documentation(visibility: private)#endif-public func FfiConverterTypeModelSource_lift(_ buf: RustBuffer) throws -> ModelSource {- return try FfiConverterTypeModelSource.lift(buf)+public func FfiConverterTypeModelReference_lift(_ buf: RustBuffer) throws -> ModelReference {+ return try FfiConverterTypeModelReference.lift(buf)}#if swift(>=5.8)@_documentation(visibility: private)#endif-public func FfiConverterTypeModelSource_lower(_ value: ModelSource) -> RustBuffer {- return FfiConverterTypeModelSource.lower(value)+public func FfiConverterTypeModelReference_lower(_ value: ModelReference) -> RustBuffer {+ return FfiConverterTypeModelReference.lower(value)}diff --git a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swift b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swiftindex 472aae101..fea4d4fbe 100644--- a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swift+++ b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swift@@ -902,7 +902,7 @@ public protocol EngineProtocol: AnyObject, Sendable {func modelsForTranslation() async throws -> [Model]- func modelsOnDevice() async throws -> [Model]+ func modelsLocal() async throws -> [Model]func modelsRemote() async throws -> [Model]@@ -1390,11 +1390,11 @@ open func modelsForTranslation()async throws -> [Model] {)}-open func modelsOnDevice()async throws -> [Model] {+open func modelsLocal()async throws -> [Model] {returntry await uniffiRustCallAsync(rustFutureFunc: {- uniffi_uzu_fn_method_engine_models_on_device(+ uniffi_uzu_fn_method_engine_models_local(self.uniffiCloneHandle())@@ -3114,14 +3114,20 @@ public enum StorageError: Swift.Error, Equatable, Hashable, Codable, Foundation.case UnableToCreateDirectory(path: String)+ case UnableToCreateDownloadManager(message: String+ )case DownloadManager(message: String)+ case HashNotFound(identifier: String, name: String+ )case InvalidStateTransition(from: DownloadPhase, to: DownloadPhase)case Io(message: String)case ItemNotFound(identifier: String)+ case Registry(RegistryError+ )case UnsupportedItem(identifier: String)@@ -3156,20 +3162,30 @@ public struct FfiConverterTypeStorageError: FfiConverterRustBuffer {case 1: return .UnableToCreateDirectory(path: try FfiConverterString.read(from: &buf))- case 2: return .DownloadManager(+ case 2: return .UnableToCreateDownloadManager(message: try FfiConverterString.read(from: &buf))- case 3: return .InvalidStateTransition(+ case 3: return .DownloadManager(+ message: try FfiConverterString.read(from: &buf)+ )+ case 4: return .HashNotFound(+ identifier: try FfiConverterString.read(from: &buf),+ name: try FfiConverterString.read(from: &buf)+ )+ case 5: return .InvalidStateTransition(from: try FfiConverterTypeDownloadPhase.read(from: &buf),to: try FfiConverterTypeDownloadPhase.read(from: &buf))- case 4: return .Io(+ case 6: return .Io(message: try FfiConverterString.read(from: &buf))- case 5: return .ItemNotFound(+ case 7: return .ItemNotFound(identifier: try FfiConverterString.read(from: &buf))- case 6: return .UnsupportedItem(+ case 8: return .Registry(+ try FfiConverterTypeRegistryError.read(from: &buf)+ )+ case 9: return .UnsupportedItem(identifier: try FfiConverterString.read(from: &buf))@@ -3189,29 +3205,45 @@ public struct FfiConverterTypeStorageError: FfiConverterRustBuffer {FfiConverterString.write(path, into: &buf)- case let .DownloadManager(message):+ case let .UnableToCreateDownloadManager(message):writeInt(&buf, Int32(2))FfiConverterString.write(message, into: &buf)- case let .InvalidStateTransition(from,to):+ case let .DownloadManager(message):writeInt(&buf, Int32(3))+ FfiConverterString.write(message, into: &buf)+++ case let .HashNotFound(identifier,name):+ writeInt(&buf, Int32(4))+ FfiConverterString.write(identifier, into: &buf)+ FfiConverterString.write(name, into: &buf)+++ case let .InvalidStateTransition(from,to):+ writeInt(&buf, Int32(5))FfiConverterTypeDownloadPhase.write(from, into: &buf)FfiConverterTypeDownloadPhase.write(to, into: &buf)case let .Io(message):- writeInt(&buf, Int32(4))+ writeInt(&buf, Int32(6))FfiConverterString.write(message, into: &buf)case let .ItemNotFound(identifier):- writeInt(&buf, Int32(5))+ writeInt(&buf, Int32(7))FfiConverterString.write(identifier, into: &buf)+ case let .Registry(v1):+ writeInt(&buf, Int32(8))+ FfiConverterTypeRegistryError.write(v1, into: &buf)++case let .UnsupportedItem(identifier):- writeInt(&buf, Int32(6))+ writeInt(&buf, Int32(9))FfiConverterString.write(identifier, into: &buf)}@@ -3767,7 +3799,7 @@ private let initializationResult: InitializationResult = {if (uniffi_uzu_checksum_method_engine_models_for_translation() != 56970) {return InitializationResult.apiChecksumMismatch}- if (uniffi_uzu_checksum_method_engine_models_on_device() != 11803) {+ if (uniffi_uzu_checksum_method_engine_models_local() != 2150) {return InitializationResult.apiChecksumMismatch}if (uniffi_uzu_checksum_method_engine_models_remote() != 36115) {diff --git a/crates/legacy/uzu/bindings/swift/Tests/UzuTests/EngineTests.swift b/crates/legacy/uzu/bindings/swift/Tests/UzuTests/EngineTests.swiftindex c4aee2336..895d53626 100644--- a/crates/legacy/uzu/bindings/swift/Tests/UzuTests/EngineTests.swift+++ b/crates/legacy/uzu/bindings/swift/Tests/UzuTests/EngineTests.swift@@ -20,20 +20,14 @@ final class EngineTests: XCTestCase {let session = try await engine.chat(model: model, config: .create())let messages = [- ChatMessage.system()- .withText(text: "You are a helpful assistant")- .withReasoningEffort(reasoningEffort: .disabled),+ ChatMessage.system().withText(text: "You are a helpful assistant"),ChatMessage.user().withText(text: "Hi"),]- let reply = try await session.reply(- input: messages,- config: .create()- .withTokenLimit(tokenLimit: 64)- .withSamplingMethod(samplingMethod: .greedy)- )+ let reply = try await session.reply(input: messages, config: .create())let message = try XCTUnwrap(reply.last?.message, "Reply has no messages")+ XCTAssertNotNil(message.reasoning())XCTAssertNotNil(message.text())}}diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mtsindex a662c9d67..96d38b508 100644--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts@@ -435,6 +435,7 @@ export declare class File {hashes: Array<Hash>constructor(url: string, name: string, size: number, hashes: Array<Hash>)get crc32C(): string | null+ get md5(): string | null}export declare class GrammarJsonAny {@@ -491,13 +492,13 @@ export declare class Model {encoding?: anyconstructor(identifier: string, registry: ModelRegistry, backends: Array<ModelBackend>, family?: ModelFamily, properties?: ModelProperties, quantization?: ModelQuantization, specializations: Array<ModelSpecialization>, accessibility: ModelAccessibility, encoding?: any)get name(): string- get isOnDevice(): boolean+ get isLocal(): booleanget isRemote(): booleanget isDownloadable(): booleanget isQuantized(): booleanget cacheIdentifier(): stringget repoIds(): Array<string>- get filesystemPath(): string | null+ get localExternalPath(): string | nullget referenceName(): string | nullget checkpointVersion(): string | null@@ -509,9 +510,9 @@ export declare class Model {get isSpeculationCapable(): boolean}-export declare class ModelAccessibilityOnDevice {- source: ModelSource- constructor(source: ModelSource)+export declare class ModelAccessibilityLocal {+ reference: ModelReference+ constructor(reference: ModelReference)}export declare class ModelAccessibilityRemote {@@ -554,19 +555,17 @@ export declare class ModelQuantization {get name(): string}-export declare class ModelRegistry {- identifier: string- metadata: Metadata- constructor(identifier: string, metadata: Metadata)- get name(): string+export declare class ModelReferenceHuggingFace {+ repository: Repository+ constructor(repository: Repository)}-export declare class ModelSourceFilesystem {+export declare class ModelReferenceLocal {path: stringconstructor(path: string)}-export declare class ModelSourceRegistry {+export declare class ModelReferenceMirai {toolchainVersion: stringrepository?: RepositorysourceRepository?: Repository@@ -574,6 +573,13 @@ export declare class ModelSourceRegistry {constructor(toolchainVersion: string, repository?: Repository, sourceRepository?: Repository, files: Array<File>)}+export declare class ModelRegistry {+ identifier: string+ metadata: Metadata+ constructor(identifier: string, metadata: Metadata)+ get name(): string+}+export declare class ModelSpecializationChat {constructor()@@ -794,7 +800,8 @@ export type Grammar =GrammarJsonAny | GrammarJsonSchema | GrammarRegexexport declare const enum HashMethod {- CRC32C = 'CRC32C'+ CRC32C = 'CRC32C',+ MD5 = 'MD5'}export declare const enum ImageFormat {@@ -809,10 +816,10 @@ export declare const enum ImageTheme {}export type ModelAccessibility =- ModelAccessibilityOnDevice | ModelAccessibilityRemote+ ModelAccessibilityLocal | ModelAccessibilityRemote-export type ModelSource =- ModelSourceRegistry | ModelSourceFilesystem+export type ModelReference =+ ModelReferenceMirai | ModelReferenceHuggingFace | ModelReferenceLocalexport type ModelSpecialization =ModelSpecializationChat | ModelSpecializationClassification | ModelSpecializationTextToSpeech | ModelSpecializationTranslation | ModelSpecializationSpeculation@@ -933,7 +940,7 @@ export declare class Engine {removeRegistry(registryIdentifier: string): Promise<void>removeBackend(identifier: string): Promise<void>get models(): Promise<Array<Model>>- get modelsOnDevice(): Promise<Array<Model>>+ get modelsLocal(): Promise<Array<Model>>get modelsRemote(): Promise<Array<Model>>get modelsDownloadable(): Promise<Array<Model>>get modelsForChat(): Promise<Array<Model>>@@ -1057,8 +1064,11 @@ export type SettingsError =export type StorageError =| { type: 'UnableToCreateDirectory', path: string }+ | { type: 'UnableToCreateDownloadManager', message: string }| { type: 'DownloadManager', message: string }+ | { type: 'HashNotFound', identifier: string, name: string }| { type: 'InvalidStateTransition', from: DownloadPhase, to: DownloadPhase }| { type: 'IO', message: string }| { type: 'ItemNotFound', identifier: string }+ | { type: 'Registry', field0: RegistryError }| { type: 'UnsupportedItem', identifier: string }diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.tsindex a662c9d67..96d38b508 100644--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts@@ -435,6 +435,7 @@ export declare class File {hashes: Array<Hash>constructor(url: string, name: string, size: number, hashes: Array<Hash>)get crc32C(): string | null+ get md5(): string | null}export declare class GrammarJsonAny {@@ -491,13 +492,13 @@ export declare class Model {encoding?: anyconstructor(identifier: string, registry: ModelRegistry, backends: Array<ModelBackend>, family?: ModelFamily, properties?: ModelProperties, quantization?: ModelQuantization, specializations: Array<ModelSpecialization>, accessibility: ModelAccessibility, encoding?: any)get name(): string- get isOnDevice(): boolean+ get isLocal(): booleanget isRemote(): booleanget isDownloadable(): booleanget isQuantized(): booleanget cacheIdentifier(): stringget repoIds(): Array<string>- get filesystemPath(): string | null+ get localExternalPath(): string | nullget referenceName(): string | nullget checkpointVersion(): string | null@@ -509,9 +510,9 @@ export declare class Model {get isSpeculationCapable(): boolean}-export declare class ModelAccessibilityOnDevice {- source: ModelSource- constructor(source: ModelSource)+export declare class ModelAccessibilityLocal {+ reference: ModelReference+ constructor(reference: ModelReference)}export declare class ModelAccessibilityRemote {@@ -554,19 +555,17 @@ export declare class ModelQuantization {get name(): string}-export declare class ModelRegistry {- identifier: string- metadata: Metadata- constructor(identifier: string, metadata: Metadata)- get name(): string+export declare class ModelReferenceHuggingFace {+ repository: Repository+ constructor(repository: Repository)}-export declare class ModelSourceFilesystem {+export declare class ModelReferenceLocal {path: stringconstructor(path: string)}-export declare class ModelSourceRegistry {+export declare class ModelReferenceMirai {toolchainVersion: stringrepository?: RepositorysourceRepository?: Repository@@ -574,6 +573,13 @@ export declare class ModelSourceRegistry {constructor(toolchainVersion: string, repository?: Repository, sourceRepository?: Repository, files: Array<File>)}+export declare class ModelRegistry {+ identifier: string+ metadata: Metadata+ constructor(identifier: string, metadata: Metadata)+ get name(): string+}+export declare class ModelSpecializationChat {constructor()@@ -794,7 +800,8 @@ export type Grammar =GrammarJsonAny | GrammarJsonSchema | GrammarRegexexport declare const enum HashMethod {- CRC32C = 'CRC32C'+ CRC32C = 'CRC32C',+ MD5 = 'MD5'}export declare const enum ImageFormat {@@ -809,10 +816,10 @@ export declare const enum ImageTheme {}export type ModelAccessibility =- ModelAccessibilityOnDevice | ModelAccessibilityRemote+ ModelAccessibilityLocal | ModelAccessibilityRemote-export type ModelSource =- ModelSourceRegistry | ModelSourceFilesystem+export type ModelReference =+ ModelReferenceMirai | ModelReferenceHuggingFace | ModelReferenceLocalexport type ModelSpecialization =ModelSpecializationChat | ModelSpecializationClassification | ModelSpecializationTextToSpeech | ModelSpecializationTranslation | ModelSpecializationSpeculation@@ -933,7 +940,7 @@ export declare class Engine {removeRegistry(registryIdentifier: string): Promise<void>removeBackend(identifier: string): Promise<void>get models(): Promise<Array<Model>>- get modelsOnDevice(): Promise<Array<Model>>+ get modelsLocal(): Promise<Array<Model>>get modelsRemote(): Promise<Array<Model>>get modelsDownloadable(): Promise<Array<Model>>get modelsForChat(): Promise<Array<Model>>@@ -1057,8 +1064,11 @@ export type SettingsError =export type StorageError =| { type: 'UnableToCreateDirectory', path: string }+ | { type: 'UnableToCreateDownloadManager', message: string }| { type: 'DownloadManager', message: string }+ | { type: 'HashNotFound', identifier: string, name: string }| { type: 'InvalidStateTransition', from: DownloadPhase, to: DownloadPhase }| { type: 'IO', message: string }| { type: 'ItemNotFound', identifier: string }+ | { type: 'Registry', field0: RegistryError }| { type: 'UnsupportedItem', identifier: string }diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.js b/crates/legacy/uzu/bindings/typescript/src/napi/index.jsindex 1e4ed47fa..a919f8a45 100644--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.js+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.js@@ -573,15 +573,16 @@ module.exports.Hash = nativeBinding.Hashmodule.exports.Image = nativeBinding.Imagemodule.exports.Metadata = nativeBinding.Metadatamodule.exports.Model = nativeBinding.Model-module.exports.ModelAccessibilityOnDevice = nativeBinding.ModelAccessibilityOnDevice+module.exports.ModelAccessibilityLocal = nativeBinding.ModelAccessibilityLocalmodule.exports.ModelAccessibilityRemote = nativeBinding.ModelAccessibilityRemotemodule.exports.ModelBackend = nativeBinding.ModelBackendmodule.exports.ModelFamily = nativeBinding.ModelFamilymodule.exports.ModelProperties = nativeBinding.ModelPropertiesmodule.exports.ModelQuantization = nativeBinding.ModelQuantization+module.exports.ModelReferenceHuggingFace = nativeBinding.ModelReferenceHuggingFace+module.exports.ModelReferenceLocal = nativeBinding.ModelReferenceLocal+module.exports.ModelReferenceMirai = nativeBinding.ModelReferenceMiraimodule.exports.ModelRegistry = nativeBinding.ModelRegistry-module.exports.ModelSourceFilesystem = nativeBinding.ModelSourceFilesystem-module.exports.ModelSourceRegistry = nativeBinding.ModelSourceRegistrymodule.exports.ModelSpecializationChat = nativeBinding.ModelSpecializationChatmodule.exports.ModelSpecializationClassification = nativeBinding.ModelSpecializationClassificationmodule.exports.ModelSpecializationSpeculation = nativeBinding.ModelSpecializationSpeculationdiff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs b/crates/legacy/uzu/bindings/typescript/src/napi/index.mjsindex 07d34a3ce..624762937 100644--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs@@ -70,15 +70,16 @@ export const Hash = cjs.Hash;export const Image = cjs.Image;export const Metadata = cjs.Metadata;export const Model = cjs.Model;-export const ModelAccessibilityOnDevice = cjs.ModelAccessibilityOnDevice;+export const ModelAccessibilityLocal = cjs.ModelAccessibilityLocal;export const ModelAccessibilityRemote = cjs.ModelAccessibilityRemote;export const ModelBackend = cjs.ModelBackend;export const ModelFamily = cjs.ModelFamily;export const ModelProperties = cjs.ModelProperties;export const ModelQuantization = cjs.ModelQuantization;+export const ModelReferenceHuggingFace = cjs.ModelReferenceHuggingFace;+export const ModelReferenceLocal = cjs.ModelReferenceLocal;+export const ModelReferenceMirai = cjs.ModelReferenceMirai;export const ModelRegistry = cjs.ModelRegistry;-export const ModelSourceFilesystem = cjs.ModelSourceFilesystem;-export const ModelSourceRegistry = cjs.ModelSourceRegistry;export const ModelSpecializationChat = cjs.ModelSpecializationChat;export const ModelSpecializationClassification = cjs.ModelSpecializationClassification;export const ModelSpecializationSpeculation = cjs.ModelSpecializationSpeculation;diff --git a/crates/legacy/uzu/bindings/typescript/tests/engine.test.ts b/crates/legacy/uzu/bindings/typescript/tests/engine.test.tsindex 898c69854..d03bd7e05 100644--- a/crates/legacy/uzu/bindings/typescript/tests/engine.test.ts+++ b/crates/legacy/uzu/bindings/typescript/tests/engine.test.ts@@ -1,12 +1,4 @@-import {- ChatConfig,- ChatMessage,- ChatReplyConfig,- Engine,- EngineConfig,- ReasoningEffort,- SamplingMethodGreedy,-} from '@trymirai/uzu';+import { ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfig } from '@trymirai/uzu';jest.setTimeout(600_000);@@ -23,18 +15,14 @@ test('chat reply produces text', async () => {const session = await engine.chat(model!, ChatConfig.create());const messages = [- ChatMessage.system()- .withText('You are a helpful assistant')- .withReasoningEffort('Disabled' as ReasoningEffort),+ ChatMessage.system().withText('You are a helpful assistant'),ChatMessage.user().withText('Hi'),];- const reply = await session.reply(- messages,- ChatReplyConfig.create().withTokenLimit(64).withSamplingMethod(new SamplingMethodGreedy()),- );+ const reply = await session.reply(messages, ChatReplyConfig.create());const message = reply[reply.length - 1]?.message;expect(message).toBeDefined();+ expect(message!.reasoning).not.toBeNull();expect(message!.text).not.toBeNull();});diff --git a/crates/legacy/uzu/src/api/mod.rs b/crates/legacy/uzu/src/api/mod.rsdeleted file mode 100644index 7dbc13c43..000000000--- a/crates/legacy/uzu/src/api/mod.rs+++ /dev/null@@ -1,5 +0,0 @@-mod request;-mod response;--pub use request::FetchModelsRequest;-pub use response::{FetchedModels, HuggingFaceFileResponse, HuggingFaceModelResponse, OpenAIModelsResponse};diff --git a/crates/legacy/uzu/src/api/request.rs b/crates/legacy/uzu/src/api/request.rsdeleted file mode 100644index d67697f09..000000000--- a/crates/legacy/uzu/src/api/request.rs+++ /dev/null@@ -1,59 +0,0 @@-use indexmap::IndexMap;-use nagare::api::{Config, Endpoint, Payload};-use reqwest::{Method, header::CONTENT_TYPE};-use serde::Serialize;-use serde_json::json;--use crate::{device::Device, registry::mirai::Backend};--#[derive(Serialize)]-pub struct FetchModelsRequest {- device: Device,- backends: Vec<Backend>,- include_traces: bool,- show_all: bool,-}--impl FetchModelsRequest {- pub fn new(- device: Device,- backends: Vec<Backend>,- include_traces: bool,- show_all: bool,- ) -> Self {- Self {- device,- backends,- include_traces,- show_all,- }- }-}--impl Endpoint for FetchModelsRequest {- fn method(&self) -> Method {- Method::POST- }-- fn path(&self) -> String {- "fetch/models".to_string()- }-- fn headers(&self) -> IndexMap<String, String> {- IndexMap::from([(CONTENT_TYPE.to_string(), "application/json".to_string())])- }-- fn payload(- &self,- _: &Config,- ) -> Payload {- Payload {- query: None,- body: Some(json!(self)),- }- }-}--#[cfg(test)]-#[path = "../../tests/unit/api/request_test.rs"]-mod tests;diff --git a/crates/legacy/uzu/src/api/response.rs b/crates/legacy/uzu/src/api/response.rsdeleted file mode 100644index a620dedb2..000000000--- a/crates/legacy/uzu/src/api/response.rs+++ /dev/null@@ -1,344 +0,0 @@-use hanashi::chat::EncodingConfig;-use serde::{Deserialize, Deserializer};-use shoji::types::{- basic::{File, Metadata, Repository, Value},- model::{- Model, ModelAccessibility, ModelBackend, ModelFamily, ModelProperties, ModelQuantization, ModelRegistry,- ModelSource, ModelSpecialization, ModelVendor,- },-};--fn get_metadata(- metadatas: &[Metadata],- identifier: &str,-) -> Result<Metadata, String> {- metadatas- .iter()- .find(|metadata| metadata.identifier == identifier)- .cloned()- .ok_or_else(|| format!("missing metadata {identifier}"))-}--#[derive(Deserialize)]-#[serde(rename_all = "snake_case")]-struct Registry {- id: String,- metadata_id: String,-}--impl Registry {- fn to(- &self,- metadatas: &[Metadata],- ) -> Result<ModelRegistry, String> {- let metadata = get_metadata(metadatas, &self.metadata_id)?;- Ok(ModelRegistry {- identifier: self.id.clone(),- metadata,- })- }-}--#[derive(Deserialize)]-#[serde(rename_all = "snake_case")]-struct Backend {- id: String,- version: String,- metadata_id: String,-}--impl Backend {- fn to(- &self,- metadatas: &[Metadata],- ) -> Result<ModelBackend, String> {- let metadata = get_metadata(metadatas, &self.metadata_id)?;- Ok(ModelBackend {- identifier: self.id.clone(),- version: self.version.clone(),- metadata,- })- }-}--#[derive(Deserialize)]-#[serde(rename_all = "snake_case")]-struct Vendor {- id: String,- metadata_id: String,-}--impl Vendor {- fn to(- &self,- metadatas: &[Metadata],- ) -> Result<ModelVendor, String> {- let metadata = get_metadata(metadatas, &self.metadata_id)?;- Ok(ModelVendor {- identifier: self.id.clone(),- metadata,- })- }-}--#[derive(Deserialize)]-#[serde(rename_all = "snake_case")]-struct Family {- id: String,- vendor: Vendor,- metadata_id: String,-}--impl Family {- fn to(- &self,- metadatas: &[Metadata],- ) -> Result<ModelFamily, String> {- let vendor = self.vendor.to(metadatas)?;- let metadata = get_metadata(metadatas, &self.metadata_id)?;- Ok(ModelFamily {- identifier: self.id.clone(),- vendor,- metadata,- })- }-}--#[derive(Deserialize)]-#[serde(rename_all = "snake_case")]-struct Properties {- id: String,- size: i64,- version: Option<String>,- metadata_id: String,-}--impl Properties {- fn to(- &self,- metadatas: &[Metadata],- ) -> Result<ModelProperties, String> {- let metadata = get_metadata(metadatas, &self.metadata_id)?;- Ok(ModelProperties {- identifier: self.id.clone(),- size: self.size,- version: self.version.clone(),- metadata,- })- }-}--#[derive(Deserialize)]-#[serde(rename_all = "snake_case")]-struct Quantization {- id: String,- method: String,- bits_per_weight: u32,- vendor: Vendor,- metadata_id: String,-}--impl Quantization {- fn to(- &self,- metadatas: &[Metadata],- ) -> Result<ModelQuantization, String> {- let vendor = self.vendor.to(metadatas)?;- let metadata = get_metadata(metadatas, &self.metadata_id)?;- Ok(ModelQuantization {- identifier: self.id.clone(),- method: self.method.clone(),- bits_per_weight: self.bits_per_weight,- vendor,- metadata,- })- }-}--#[derive(Deserialize)]-#[serde(tag = "type", rename_all = "snake_case")]-enum FetchedAccessibility {- Local {- reference: FetchedReference,- },- Remote {- repository: Option<Repository>,- },-}--impl From<&FetchedAccessibility> for ModelAccessibility {- fn from(accessibility: &FetchedAccessibility) -> Self {- match accessibility {- FetchedAccessibility::Local {- reference,- } => Self::OnDevice {- source: reference.into(),- },- FetchedAccessibility::Remote {- repository,- } => Self::Remote {- repository: repository.clone(),- },- }- }-}--#[derive(Deserialize)]-#[serde(tag = "type", rename_all = "snake_case")]-enum FetchedReference {- Mirai {- toolchain_version: String,- repository: Option<Repository>,- source_repository: Option<Repository>,- files: Vec<File>,- },- Local {- path: String,- },-}--impl From<&FetchedReference> for ModelSource {- fn from(reference: &FetchedReference) -> Self {- match reference {- FetchedReference::Mirai {- toolchain_version,- repository,- source_repository,- files,- } => Self::Registry {- toolchain_version: toolchain_version.clone(),- repository: repository.clone(),- source_repository: source_repository.clone(),- files: files.clone(),- },- FetchedReference::Local {- path,- } => Self::Filesystem {- path: path.clone(),- },- }- }-}--#[derive(Deserialize)]-#[serde(rename_all = "snake_case")]-struct FetchedModel {- id: String,- registry: Registry,- backends: Vec<Backend>,- family: Option<Family>,- properties: Option<Properties>,- quantization: Option<Quantization>,- specializations: Vec<ModelSpecialization>,- accessibility: FetchedAccessibility,- #[serde(rename = "encodings", default, deserialize_with = "deserialize_encoding")]- encoding: Option<Value>,-}--fn deserialize_encoding<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Option<Value>, D::Error> {- let encodings = Vec::<Value>::deserialize(deserializer)?;- Ok(EncodingConfig::select(&encodings))-}--impl FetchedModel {- fn to(- &self,- metadatas: &[Metadata],- ) -> Result<Model, String> {- let registry = self.registry.to(metadatas)?;- let backends = self.backends.iter().map(|backend| backend.to(metadatas)).collect::<Result<Vec<_>, _>>()?;- let family = self.family.as_ref().map(|family| family.to(metadatas)).transpose()?;- let properties = self.properties.as_ref().map(|properties| properties.to(metadatas)).transpose()?;- let quantization = self.quantization.as_ref().map(|quantization| quantization.to(metadatas)).transpose()?;- Ok(Model {- identifier: self.id.clone(),- registry,- backends,- family,- properties,- quantization,- specializations: self.specializations.clone(),- accessibility: (&self.accessibility).into(),- encoding: self.encoding.clone(),- })- }-}--#[derive(Deserialize)]-#[serde(rename_all = "snake_case")]-pub struct FetchedModels {- models: Vec<FetchedModel>,- metadatas: Vec<Metadata>,-}--impl FetchedModels {- pub fn models(&self) -> Result<Vec<Model>, String> {- self.models.iter().map(|model| model.to(&self.metadatas)).collect()- }-}--#[derive(Deserialize)]-pub struct HuggingFaceModelResponse {- pub sha: String,- pub private: Option<bool>,- pub gated: Option<HuggingFaceGatedResponse>,- pub siblings: Vec<HuggingFaceFileResponse>,-}--impl HuggingFaceModelResponse {- pub fn requires_authentication(&self) -> bool {- if self.private != Some(false) {- return true;- }- match &self.gated {- Some(HuggingFaceGatedResponse::Boolean(false)) => false,- Some(HuggingFaceGatedResponse::Approval(approval)) => {- matches!(approval, HuggingFaceGatedApproval::Auto | HuggingFaceGatedApproval::Manual)- },- Some(HuggingFaceGatedResponse::Boolean(true)) | None => true,- }- }-}--#[derive(Deserialize)]-#[serde(untagged)]-pub enum HuggingFaceGatedResponse {- Boolean(bool),- Approval(HuggingFaceGatedApproval),-}--#[derive(Deserialize)]-#[serde(rename_all = "lowercase")]-pub enum HuggingFaceGatedApproval {- Auto,- Manual,-}--#[derive(Deserialize)]-pub struct HuggingFaceFileResponse {- pub rfilename: String,- pub size: Option<u64>,- #[serde(rename = "blobId")]- pub blob_id: Option<String>,- pub lfs: Option<HuggingFaceLfsResponse>,-}--#[derive(Deserialize)]-pub struct HuggingFaceLfsResponse {- pub sha256: Option<String>,- pub size: Option<u64>,-}--#[derive(Deserialize)]-pub struct OpenAIModelsResponse {- pub data: Vec<OpenAIModelResponse>,-}--#[derive(Deserialize)]-pub struct OpenAIModelResponse {- pub id: String,-}--#[cfg(test)]-#[path = "../../tests/unit/api/response_test.rs"]-mod tests;diff --git a/crates/legacy/uzu/src/engine/bridge/helpers.rs b/crates/legacy/uzu/src/engine/bridge/helpers.rsindex 6e612f036..1208b2b82 100644--- a/crates/legacy/uzu/src/engine/bridge/helpers.rs+++ b/crates/legacy/uzu/src/engine/bridge/helpers.rs@@ -133,5 +133,41 @@ pub fn get_sampling_method<B: Backend>(}#[cfg(all(test, feature = "capability-grammar"))]-#[path = "../../../tests/unit/engine/bridge/helpers_test.rs"]-mod tests;+mod tests {+ use tokenizers::{Tokenizer, decoders::fuse::Fuse, models::wordlevel::WordLevel};++ use super::grammar_trigger_token_sequence_for_prompt;++ fn tokenizer() -> Tokenizer {+ let model = WordLevel::builder()+ .vocab(+ [+ ("<unk>".to_string(), 0),+ ("</think>".to_string(), 1),+ ("\n\n".to_string(), 2),+ ("next".to_string(), 3),+ ]+ .into_iter()+ .collect(),+ )+ .build()+ .unwrap();+ let mut tokenizer = Tokenizer::new(model);+ tokenizer.with_decoder(Some(Fuse::new()));+ tokenizer+ }++ #[test]+ fn grammar_engages_immediately_when_prompt_closes_reasoning() {+ let tokenizer = tokenizer();++ assert_eq!(grammar_trigger_token_sequence_for_prompt(Some(&[1]), &[1, 2], &tokenizer), None);+ }++ #[test]+ fn grammar_waits_for_trigger_when_only_history_contains_it() {+ let tokenizer = tokenizer();++ assert_eq!(grammar_trigger_token_sequence_for_prompt(Some(&[1]), &[1, 2, 3], &tokenizer), Some(vec![1]));+ }+}diff --git a/crates/legacy/uzu/src/engine/config.rs b/crates/legacy/uzu/src/engine/config.rsindex 6875a9120..4c63aec0e 100644--- a/crates/legacy/uzu/src/engine/config.rs+++ b/crates/legacy/uzu/src/engine/config.rs@@ -19,7 +19,7 @@ pub const KEY_BASETEN_API_KEY: &str = "BASETEN_API_KEY";pub const KEY_OPENROUTER_API_KEY: &str = "OPENROUTER_API_KEY";#[bindings::export(Structure(Class))]-#[derive(Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]#[serde(rename_all = "snake_case")]pub struct EngineConfig {pub application_identifier: Option<String>,diff --git a/crates/legacy/uzu/src/engine/downloader.rs b/crates/legacy/uzu/src/engine/downloader.rsindex e9bcf0d5f..feb6c2214 100644--- a/crates/legacy/uzu/src/engine/downloader.rs+++ b/crates/legacy/uzu/src/engine/downloader.rs@@ -1,8 +1,5 @@-use std::{sync::Weak, time::Duration};--use kiban::time::sleep as kiban_sleep;use serde::{Deserialize, Serialize};-use tokio_stream::{StreamExt as TokioStreamExt, wrappers::BroadcastStream as TokioBroadcastStream};+use tokio_stream::{StreamExt, wrappers::BroadcastStream};use crate::{engine::EngineError,@@ -18,25 +15,22 @@ use crate::{pub struct Downloader {identifier: String,storage: SharedAccess<Storage>,- huggingface_api_key: Option<Weak<str>>,}impl Downloader {pub fn new(identifier: String,storage: SharedAccess<Storage>,- huggingface_api_key: Option<Weak<str>>,) -> Self {Self {identifier,storage,- huggingface_api_key,}}async fn wait_for_resume_to_be_observable(&self,- mut stream: TokioBroadcastStream<(String, DownloadState)>,+ mut stream: BroadcastStream<(String, DownloadState)>,) -> Result<(), EngineError> {if self.state().await.is_some_and(|state| Self::is_resume_observable_phase(&state.phase)) {return Ok(());@@ -84,22 +78,25 @@ impl Downloader {let state = self.state().await.ok_or(StorageError::ItemNotFound {identifier: self.identifier.clone(),})?;- let reset = match state.phase {- DownloadPhase::Downloading {} | DownloadPhase::Downloaded {} => return Ok(()),++ match state.phase {+ DownloadPhase::Downloading {} | DownloadPhase::Downloaded {} => Ok(()),+ DownloadPhase::NotDownloaded {} | DownloadPhase::Paused {} | DownloadPhase::Locked {} => {+ let stream = self.storage.lock().await.subscribe();+ self.storage.lock().await.download(&self.identifier).await?;+ self.wait_for_resume_to_be_observable(stream).await+ },DownloadPhase::Error {..- } => true,- DownloadPhase::NotDownloaded {} | DownloadPhase::Paused {} | DownloadPhase::Locked {} => false,- };- let stream = self.storage.lock().await.subscribe();- let storage = self.storage.lock().await;- if reset {- storage.delete(&self.identifier).await?;+ } => {+ let stream = self.storage.lock().await.subscribe();+ let storage = self.storage.lock().await;+ storage.delete(&self.identifier).await?;+ storage.download(&self.identifier).await?;+ drop(storage);+ self.wait_for_resume_to_be_observable(stream).await+ },}- let huggingface_api_key = self.huggingface_api_key.as_ref().and_then(Weak::upgrade);- storage.download(&self.identifier, huggingface_api_key).await?;- drop(storage);- self.wait_for_resume_to_be_observable(stream).await}#[bindings::export(Method)]@@ -129,14 +126,14 @@ impl Downloader {#[derive(Clone)]pub struct DownloaderStream {identifier: String,- stream: SharedAccess<Option<TokioBroadcastStream<(String, DownloadState)>>>,+ stream: SharedAccess<Option<BroadcastStream<(String, DownloadState)>>>,storage: Option<SharedAccess<Storage>>,}impl DownloaderStream {pub(crate) fn new(identifier: String,- stream: TokioBroadcastStream<(String, DownloadState)>,+ stream: BroadcastStream<(String, DownloadState)>,storage: SharedAccess<Storage>,) -> Self {Self {@@ -170,7 +167,7 @@ impl DownloaderStream {Some(_) => {},None => return true,}- kiban_sleep(Duration::from_millis(20)).await;+ tokio::time::sleep(std::time::Duration::from_millis(20)).await;}true}diff --git a/crates/legacy/uzu/src/engine/mod.rs b/crates/legacy/uzu/src/engine/mod.rsindex 0d3a362c6..ea53ea698 100644--- a/crates/legacy/uzu/src/engine/mod.rs+++ b/crates/legacy/uzu/src/engine/mod.rs@@ -14,7 +14,7 @@ pub use download_manager::DownloadManagerType;pub use downloader::{Downloader, DownloaderStream, DownloaderStreamUpdate};pub use error::EngineError;use indexmap::{IndexMap, IndexSet};-use kiban::rt::{RuntimeHandle as KibanRuntimeHandle, spawn as kiban_spawn};+use kiban::rt::RuntimeHandle;use nagare::{api::Config as ClientConfig,chat::{ChatInstance, ChatSession},@@ -29,21 +29,17 @@ use shoji::{session::chat::ChatConfig,},};-use tokio_stream::{StreamExt as TokioStreamExt, wrappers::BroadcastStream as TokioBroadcastStream};+use tokio_stream::{StreamExt, wrappers::BroadcastStream};use crate::{device::Device,engine::bridge::UzuLlmBackend,helpers::{SharedAccess, is_endpoint_reachable},logs,- models::ModelCatalog,registry::{CachedRegistry, MergedRegistry, RegistryError,local::{Config as LocalRegistryConfig, Registry as LocalRegistry},- mirai::{- Backend as MiraiBackend, Config as MiraiRegistryConfig, MIRAI_API_HOST, MIRAI_API_SCHEME,- Registry as MiraiRegistry,- },+ mirai::{Backend as MiraiBackend, Config as MiraiRegistryConfig, Registry as MiraiRegistry},openai::{Config as OpenAIConfig, Registry as OpenAIRegistry},},settings::Settings,@@ -58,18 +54,15 @@ use crate::{pub struct Engine {settings: SharedAccess<Option<Settings>>,registry: SharedAccess<MergedRegistry>,- catalog: ModelCatalog,- catalog_refresh_lock: SharedAccess<()>,storage: SharedAccess<Storage>,backends: SharedAccess<HashMap<String, Arc<dyn Backend>>>,callback: SharedAccess<Option<Arc<EngineCallback>>>,telemetry: SharedAccess<Telemetry>,- huggingface_api_key: Option<Arc<str>>,}impl Engine {pub async fn new(config: EngineConfig) -> Result<Self, EngineError> {- let runtime_handle = KibanRuntimeHandle::try_current().map_err(|error| EngineError::TokioError {+ let runtime_handle = RuntimeHandle::try_current().map_err(|error| EngineError::TokioError {message: error.to_string(),})?;@@ -87,7 +80,7 @@ impl Engine {let telemetry = SharedAccess::new({let client_config = ClientConfig::new(- format!("{MIRAI_API_SCHEME}://{MIRAI_API_HOST}/api/v2"),+ "https://sdk.trymirai.com/api/v2".to_string(),Duration::from_secs(10),IndexMap::new(),);@@ -110,21 +103,15 @@ impl Engine {let storage_cache_path = storage_config.cache_path();logs::start(storage_config.cache_path(), &storage_config.log_name(), false);- let huggingface_api_key = config.huggingface_api_key.clone().map(Arc::<str>::from);- let catalog =- ModelCatalog::new(huggingface_api_key.clone(), storage_cache_path.join("resolved-models.json")).await?;let storage = SharedAccess::new(Storage::new(runtime_handle, storage_config).await?);let engine = Self {settings: SharedAccess::new(settings),storage,registry,- catalog,- catalog_refresh_lock: SharedAccess::new(()),backends: SharedAccess::new(HashMap::new()),callback: SharedAccess::new(None),telemetry,- huggingface_api_key,};engine.spawn_storage_listener().await;@@ -141,11 +128,12 @@ impl Engine {version: uzu_backend_version.clone(),}],include_traces: false,+ cache_path: storage_cache_path,};let mirai_registry = Box::new(MiraiRegistry::new(mirai_registry_config)?);engine.add_backend(Arc::new(uzu_backend) as Arc<dyn Backend>).await;- engine.registry.lock().await.add(Box::new(CachedRegistry::new(mirai_registry)))?;+ engine.add_registry(mirai_registry).await?;if let Some(lalamo_path) = config.lalamo_path {let lalamo_registry = LocalRegistry::new(LocalRegistryConfig::lalamo(@@ -153,7 +141,7 @@ impl Engine {uzu_backend_version.clone(),lalamo_path,))?;- engine.registry.lock().await.add(Box::new(CachedRegistry::new(Box::new(lalamo_registry))))?;+ engine.add_registry(Box::new(lalamo_registry)).await?;}if let Some(local_path) = config.local_path {let local_registry = LocalRegistry::new(LocalRegistryConfig::local(@@ -161,7 +149,7 @@ impl Engine {uzu_backend_version.clone(),local_path,))?;- engine.registry.lock().await.add(Box::new(CachedRegistry::new(Box::new(local_registry))))?;+ engine.add_registry(Box::new(local_registry)).await?;}}@@ -199,19 +187,14 @@ impl Engine {for config in openai_configs {let registry = OpenAIRegistry::new(config.clone())?;let backend = OpenAIBackend::new(config.into()).map_err(|_| EngineError::UnableToCreateBackend {})?;- engine.registry.lock().await.add(Box::new(CachedRegistry::new(Box::new(registry))))?;+ engine.add_registry(Box::new(registry)).await?;engine.add_backend(Arc::new(backend) as Arc<dyn Backend>).await;}- engine.handle_initial_catalog_refresh().await?;Ok(engine)}}-#[cfg(test)]-#[path = "../../tests/unit/engine/cache_test.rs"]-mod tests;-#[bindings::export(Implementation)]impl Engine {#[bindings::export(Method(Factory))]@@ -237,15 +220,8 @@ impl Engine {&self,registry: Box<dyn Registry<Error = RegistryError>>,) -> Result<(), EngineError> {- let identifier = registry.indentifier();- let refresh = self.catalog_refresh_lock.lock().await;self.registry.lock().await.add(Box::new(CachedRegistry::new(registry)))?;- if let Err(error) = self.refresh_catalog().await {- self.registry.lock().await.remove(&identifier);- return Err(error);- }- drop(refresh);- self.notify_callback().await;+ self.handle_registry_refresh().await?;Ok(())}@@ -264,16 +240,8 @@ impl Engine {&self,registry_identifier: String,) -> Result<(), EngineError> {- let refresh = self.catalog_refresh_lock.lock().await;- let removed = self.registry.lock().await.remove(®istry_identifier);- if let Err(error) = self.refresh_catalog().await {- if let Some((index, registry)) = removed {- self.registry.lock().await.restore(index, registry);- }- return Err(error);- }- drop(refresh);- self.notify_callback().await;+ self.registry.lock().await.remove(®istry_identifier)?;+ self.handle_registry_refresh().await?;Ok(())}@@ -290,12 +258,12 @@ impl Engine {impl Engine {#[bindings::export(Method(Getter))]pub async fn models(&self) -> Result<Vec<Model>, EngineError> {- Ok(self.catalog.models().await)+ self.registry.lock().await.models().await.map_err(EngineError::from)}#[bindings::export(Method(Getter))]- pub async fn models_on_device(&self) -> Result<Vec<Model>, EngineError> {- Ok(self.models().await?.into_iter().filter(|model| model.is_on_device()).collect())+ pub async fn models_local(&self) -> Result<Vec<Model>, EngineError> {+ Ok(self.models().await?.into_iter().filter(|model| model.is_local()).collect())}#[bindings::export(Method(Getter))]@@ -422,7 +390,7 @@ impl Engine {&self,identifier: String,) -> Result<Option<Model>, EngineError> {- Ok(self.models().await?.into_iter().find(|model| model.identifier == identifier))+ self.registry.lock().await.model_by_identifier(&identifier).await.map_err(EngineError::from)}#[bindings::export(Method)]@@ -430,7 +398,7 @@ impl Engine {&self,repo_id: String,) -> Result<Option<Model>, EngineError> {- Ok(self.models().await?.into_iter().find(|model| model.repo_ids().contains(&repo_id)))+ self.registry.lock().await.model_by_repo_id(&repo_id).await.map_err(EngineError::from)}#[bindings::export(Method)]@@ -455,11 +423,11 @@ impl Engine {&self,model: &Model,) -> Option<String> {- if !model.is_on_device() {+ if !model.is_local() {return None;}- if let Some(filesystem_path) = model.filesystem_path() {- return Some(filesystem_path);+ if let Some(local_external_path) = model.local_external_path() {+ return Some(local_external_path);}let storage = self.storage.lock().await;let state = storage.state(&model.identifier).await?;@@ -482,11 +450,7 @@ impl Engine {&self,model: &Model,) -> Downloader {- Downloader::new(- model.identifier.clone(),- self.storage.clone(),- self.huggingface_api_key.as_ref().map(Arc::downgrade),- )+ Downloader::new(model.identifier.clone(), self.storage.clone())}#[bindings::export(Method)]@@ -603,45 +567,24 @@ impl Engine {}impl Engine {- pub async fn storage_subscribe(&self) -> TokioBroadcastStream<(String, DownloadState)> {+ pub async fn storage_subscribe(&self) -> BroadcastStream<(String, DownloadState)> {self.storage.lock().await.subscribe()}- async fn refresh_catalog(&self) -> Result<(), EngineError> {+ async fn handle_registry_refresh(&self) -> Result<(), EngineError> {let models = self.registry.lock().await.models().await?;- let resolved = self.catalog.resolve(models).await?;- self.storage.lock().await.refresh(&resolved, self.huggingface_api_key.clone()).await?;- self.catalog.commit(resolved).await;- Ok(())- }-- async fn handle_initial_catalog_refresh(&self) -> Result<(), EngineError> {- let _refresh = self.catalog_refresh_lock.lock().await;- match self.refresh_catalog().await {- Ok(()) => Ok(()),- Err(error) => {- let previous = self.catalog.snapshot().await;- if previous.is_empty() {- return Err(error);- }- tracing::warn!(?error, "serving previous resolved models after initial refresh failure");- self.storage.lock().await.refresh(&previous, self.huggingface_api_key.clone()).await?;- Ok(())- },- }- }-- async fn notify_callback(&self) {+ self.storage.lock().await.refresh(models).await?;if let Some(callback) = self.callback.lock().await.as_ref().cloned() {callback.on_event();- }+ };+ Ok(())}async fn spawn_storage_listener(&self) {let mut stream = self.storage_subscribe().await;let callback = self.callback.clone();let telemetry = self.telemetry.lock().await.clone();- kiban_spawn(async move {+ tokio::spawn(async move {let mut last_phase: HashMap<String, DownloadPhase> = HashMap::new();while let Some(update) = stream.next().await {let Ok((id, state)) = update else {diff --git a/crates/legacy/uzu/src/lib.rs b/crates/legacy/uzu/src/lib.rsindex 41226d96a..6db09af94 100644--- a/crates/legacy/uzu/src/lib.rs+++ b/crates/legacy/uzu/src/lib.rs@@ -31,8 +31,6 @@ pyo3_stub_gen::define_stub_info_gatherer!(pyo3_bindings_annotations);#[cfg(feature = "bindings-pyo3")]pyo3_stub_gen::reexport_module_members!("uzu" from "uzu._tool"; "UzuToolFunction", "uzu_tool_function");-#[cfg(not(target_family = "wasm"))]-mod api;#[cfg(not(target_family = "wasm"))]pub mod device;#[cfg(not(target_family = "wasm"))]@@ -42,8 +40,6 @@ pub mod helpers;#[cfg(not(target_family = "wasm"))]pub mod logs;#[cfg(not(target_family = "wasm"))]-mod models;-#[cfg(not(target_family = "wasm"))]pub mod player;#[cfg(not(target_family = "wasm"))]pub mod registry;diff --git a/crates/legacy/uzu/src/models/catalog.rs b/crates/legacy/uzu/src/models/catalog.rsdeleted file mode 100644index a4f80c0da..000000000--- a/crates/legacy/uzu/src/models/catalog.rs+++ /dev/null@@ -1,59 +0,0 @@-use std::{path::PathBuf, sync::Arc};--use shoji::types::model::Model;--use super::{ModelsResolver, ResolvedModels};-use crate::{helpers::SharedAccess, registry::RegistryError};--#[derive(Clone)]-pub struct ModelCatalog {- resolver: ModelsResolver,- published: SharedAccess<ResolvedModels>,-}--impl ModelCatalog {- pub async fn new(- api_key: Option<Arc<str>>,- cache_path: PathBuf,- ) -> Result<Self, RegistryError> {- let resolver = ModelsResolver::new(api_key, cache_path)?;- let published = match resolver.load_cache().await {- Ok(Some(models)) => models,- Ok(None) => ResolvedModels::default(),- Err(error) => {- tracing::warn!(?error, "failed to load resolved models cache");- ResolvedModels::default()- },- };- Ok(Self {- resolver,- published: SharedAccess::new(published),- })- }-- pub async fn models(&self) -> Vec<Model> {- self.published.lock().await.models()- }-- pub async fn snapshot(&self) -> ResolvedModels {- self.published.lock().await.clone()- }-- pub async fn resolve(- &self,- models: Vec<Model>,- ) -> Result<ResolvedModels, RegistryError> {- let previous = self.snapshot().await;- self.resolver.resolve(models, &previous).await- }-- pub async fn commit(- &self,- models: ResolvedModels,- ) {- if let Err(error) = self.resolver.save_cache(&models).await {- tracing::warn!(?error, "failed to save resolved models cache");- }- *self.published.lock().await = models;- }-}diff --git a/crates/legacy/uzu/src/models/mod.rs b/crates/legacy/uzu/src/models/mod.rsdeleted file mode 100644index 631aa0401..000000000--- a/crates/legacy/uzu/src/models/mod.rs+++ /dev/null@@ -1,7 +0,0 @@-mod catalog;-mod resolved;-mod resolver;--pub use catalog::ModelCatalog;-pub use resolved::{ResolvedFile, ResolvedModel, ResolvedModels};-pub use resolver::ModelsResolver;diff --git a/crates/legacy/uzu/src/models/resolved.rs b/crates/legacy/uzu/src/models/resolved.rsdeleted file mode 100644index d181e239a..000000000--- a/crates/legacy/uzu/src/models/resolved.rs+++ /dev/null@@ -1,104 +0,0 @@-use download_manager::FileCheck;-use serde::{Deserialize, Serialize};-use shoji::types::{basic::File, model::Model};--const CACHE_VERSION: u8 = 2;--#[derive(Clone, Serialize, Deserialize)]-pub struct ResolvedModels {- version: u8,- models: Vec<ResolvedModel>,- #[serde(skip)]- reusable_hugging_face_files: bool,-}--impl Default for ResolvedModels {- fn default() -> Self {- Self {- version: CACHE_VERSION,- models: Vec::new(),- reusable_hugging_face_files: false,- }- }-}--#[derive(Clone, Serialize, Deserialize)]-pub struct ResolvedModel {- model: Model,- files: Option<Vec<ResolvedFile>>,-}--#[derive(Clone, PartialEq, Serialize, Deserialize)]-pub struct ResolvedFile {- pub file: File,- pub check: FileCheck,- pub requires_authentication: bool,-}--impl ResolvedModels {- pub fn new(models: Vec<ResolvedModel>) -> Self {- Self {- version: CACHE_VERSION,- models,- reusable_hugging_face_files: true,- }- }-- pub fn models(&self) -> Vec<Model> {- self.models.iter().map(|resolved| resolved.model.clone()).collect()- }-- pub fn iter(&self) -> impl Iterator<Item = &ResolvedModel> {- self.models.iter()- }-- pub fn is_empty(&self) -> bool {- self.models.is_empty()- }-- pub fn validate_cache(mut self) -> Option<Self> {- if self.version != CACHE_VERSION {- return None;- }- self.reusable_hugging_face_files = false;- Some(self)- }-- pub fn reusable_hugging_face_files(- &self,- model: &Model,- ) -> Option<Vec<ResolvedFile>> {- if !self.reusable_hugging_face_files {- return None;- }- self.models- .iter()- .find(|resolved| {- resolved.model.identifier == model.identifier && resolved.model.accessibility == model.accessibility- })- .and_then(|resolved| resolved.files.clone())- }-}--impl ResolvedModel {- pub fn downloadable(- model: Model,- files: Vec<ResolvedFile>,- ) -> Self {- Self {- model,- files: Some(files),- }- }-- pub fn passthrough(model: Model) -> Self {- Self {- model,- files: None,- }- }-- pub fn parts(&self) -> (&Model, Option<&[ResolvedFile]>) {- (&self.model, self.files.as_deref())- }-}diff --git a/crates/legacy/uzu/src/models/resolver.rs b/crates/legacy/uzu/src/models/resolver.rsdeleted file mode 100644index 36d304802..000000000--- a/crates/legacy/uzu/src/models/resolver.rs+++ /dev/null@@ -1,276 +0,0 @@-use std::{collections::HashSet, io::ErrorKind, path::PathBuf, sync::Arc, time::Duration};--use download_manager::FileCheck;-use futures_util::{StreamExt, TryStreamExt, stream};-use kiban::fs::asyn::{- create_dir_all as kiban_create_dir_all, read as kiban_read, rename as kiban_rename, write as kiban_write,-};-use reqwest::{Client, Url};-use serde_json::{from_slice, to_vec};-use shoji::types::{- basic::{File, Repository},- model::{Model, ModelAccessibility, ModelSource},-};-use uuid::Uuid;--use super::{ResolvedFile, ResolvedModel, ResolvedModels};-use crate::{- api::{HuggingFaceFileResponse, HuggingFaceModelResponse},- registry::RegistryError,-};--const HUGGING_FACE_URL: &str = "https://huggingface.co";-const MAX_CONCURRENT_REQUESTS: usize = 8;-const SIDECAR_EXTENSIONS: [&str; 4] = ["part", "resume_data", "lock", "integrity"];--#[derive(Clone)]-pub struct ModelsResolver {- client: Client,- endpoint: Url,- api_key: Option<Arc<str>>,- cache_path: PathBuf,-}--impl ModelsResolver {- pub fn new(- api_key: Option<Arc<str>>,- cache_path: PathBuf,- ) -> Result<Self, RegistryError> {- let client =- Client::builder().https_only(true).timeout(Duration::from_secs(30)).build().map_err(resolution_error)?;- let endpoint = Url::parse(HUGGING_FACE_URL).map_err(resolution_error)?;- Ok(Self {- client,- endpoint,- api_key,- cache_path,- })- }-- pub async fn load_cache(&self) -> Result<Option<ResolvedModels>, RegistryError> {- let contents = match kiban_read(&self.cache_path).await {- Ok(contents) => contents,- Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None),- Err(error) => return Err(resolution_error(error)),- };- let models = from_slice::<ResolvedModels>(&contents).map_err(resolution_error)?;- Ok(models.validate_cache())- }-- pub async fn save_cache(- &self,- models: &ResolvedModels,- ) -> Result<(), RegistryError> {- let parent = self.cache_path.parent().ok_or_else(|| resolution_error("invalid resolved models cache path"))?;- kiban_create_dir_all(parent).await.map_err(resolution_error)?;- let temporary = self.cache_path.with_added_extension(format!("{}.tmp", Uuid::new_v4()));- kiban_write(&temporary, to_vec(models).map_err(resolution_error)?).await.map_err(resolution_error)?;- kiban_rename(temporary, &self.cache_path).await.map_err(resolution_error)- }-- pub async fn resolve(- &self,- models: Vec<Model>,- previous: &ResolvedModels,- ) -> Result<ResolvedModels, RegistryError> {- let futures = models.into_iter().map(|model| self.resolve_model(model, previous));- let resolved = stream::iter(futures).buffered(MAX_CONCURRENT_REQUESTS).try_collect::<Vec<_>>().await?;- Ok(ResolvedModels::new(resolved))- }-- async fn resolve_model(- &self,- model: Model,- previous: &ResolvedModels,- ) -> Result<ResolvedModel, RegistryError> {- let ModelAccessibility::OnDevice {- source:- ModelSource::Registry {- repository,- files,- ..- },- } = &model.accessibility- else {- return Ok(ResolvedModel::passthrough(model));- };- let Some((repository, revision)) = repository- .as_ref()- .and_then(|repository| repository.commit_hash.as_deref().map(|revision| (repository, revision)))- else {- let files = files- .iter()- .cloned()- .map(|file| {- let check = file.crc32c().map(FileCheck::CRC).ok_or_else(|| {- resolution_error(format!("missing CRC32C for {}/{}", model.identifier, file.name))- })?;- Ok(ResolvedFile {- file,- check,- requires_authentication: false,- })- })- .collect::<Result<Vec<_>, RegistryError>>()?;- return Ok(ResolvedModel::downloadable(model, files));- };- if let Some(files) = previous.reusable_hugging_face_files(&model) {- return Ok(ResolvedModel::downloadable(model, files));- }- let files = self.resolve_hugging_face(repository, revision).await?;- Ok(ResolvedModel::downloadable(model, files))- }-- async fn resolve_hugging_face(- &self,- repository: &Repository,- revision: &str,- ) -> Result<Vec<ResolvedFile>, RegistryError> {- if !is_lower_hex(revision, 40) {- return Err(resolution_error("invalid commit"));- }- let mut metadata_url = self.endpoint.clone();- metadata_url- .path_segments_mut()- .map_err(|_| resolution_error("invalid Hugging Face endpoint"))?- .clear()- .extend(["api", "models"])- .extend(repository.identifier.split('/'))- .extend(["revision", revision]);- metadata_url.query_pairs_mut().append_pair("blobs", "true");- let mut request = self.client.get(metadata_url);- if let Some(api_key) = &self.api_key {- request = request.bearer_auth(api_key.as_ref());- }- let response = request.send().await.map_err(resolution_error)?;- if !response.status().is_success() {- return Err(resolution_error(format!("Hugging Face returned HTTP {}", response.status().as_u16())));- }- let response: HuggingFaceModelResponse = response.json().await.map_err(resolution_error)?;- if response.sha != revision {- return Err(resolution_error(format!(- "Hugging Face returned revision {} instead of {revision}",- response.sha- )));- }- let requires_authentication = response.requires_authentication();- if requires_authentication && self.api_key.is_none() {- return Err(resolution_error("Hugging Face authentication is required"));- }- self.resolve_files(repository, revision, response.siblings, requires_authentication)- }-- fn resolve_files(- &self,- repository: &Repository,- revision: &str,- siblings: Vec<HuggingFaceFileResponse>,- requires_authentication: bool,- ) -> Result<Vec<ResolvedFile>, RegistryError> {- let mut resolve_base = self.endpoint.clone();- resolve_base- .path_segments_mut()- .map_err(|_| resolution_error("invalid Hugging Face endpoint"))?- .clear()- .extend(repository.identifier.split('/'))- .extend(["resolve", revision]);- let mut names = HashSet::with_capacity(siblings.len());- let mut files = Vec::with_capacity(siblings.len());- for sibling in siblings- .iter()- .filter(|sibling| repository.paths.as_ref().is_none_or(|paths| paths.contains(&sibling.rfilename)))- {- validate_path(&sibling.rfilename)?;- let normalized_name = PathBuf::from(sibling.rfilename.to_lowercase());- if !names.insert(normalized_name) {- return Err(invalid_file(&sibling.rfilename));- }- let (size, check) = file_size_and_check(sibling)?;- let mut url = resolve_base.clone();- url.path_segments_mut()- .map_err(|_| resolution_error("invalid Hugging Face endpoint"))?- .extend(sibling.rfilename.split('/'));- files.push(ResolvedFile {- file: File {- url: url.into(),- name: sibling.rfilename.clone(),- size: i64::try_from(size).map_err(|_| invalid_file("file size overflow"))?,- hashes: Vec::new(),- },- check,- requires_authentication,- });- }- let sidecars = names- .iter()- .flat_map(|name| SIDECAR_EXTENSIONS.iter().map(|extension| name.with_added_extension(extension)))- .collect::<HashSet<_>>();- let collides = names.iter().any(|name| {- name.ancestors().skip(1).any(|parent| names.contains(parent))- || name.ancestors().any(|path| sidecars.contains(path))- });- if files.is_empty() || collides {- return Err(invalid_file("empty or colliding manifest"));- }- Ok(files)- }-}--fn file_size_and_check(file: &HuggingFaceFileResponse) -> Result<(u64, FileCheck), RegistryError> {- if let Some(lfs) = &file.lfs {- if let (Some(size), Some(lfs_size)) = (file.size, lfs.size)- && size != lfs_size- {- return Err(resolution_error(format!("Hugging Face size mismatch for {}", file.rfilename)));- }- let size = lfs- .size- .or(file.size)- .ok_or_else(|| resolution_error(format!("Hugging Face is missing size for {}", file.rfilename)))?;- let digest = lfs- .sha256- .as_deref()- .filter(|digest| is_lower_hex(digest, 64))- .ok_or_else(|| resolution_error(format!("missing or invalid SHA-256 for {}", file.rfilename)))?;- Ok((size, FileCheck::Sha256(digest.to_string())))- } else {- let size = file- .size- .ok_or_else(|| resolution_error(format!("Hugging Face is missing size for {}", file.rfilename)))?;- let digest = file- .blob_id- .as_deref()- .filter(|digest| is_lower_hex(digest, 40))- .ok_or_else(|| resolution_error(format!("missing or invalid Git blob SHA-1 for {}", file.rfilename)))?;- Ok((size, FileCheck::GitBlobSha1(digest.to_string())))- }-}--fn validate_path(path: &str) -> Result<(), RegistryError> {- if path.contains(['\\', ':']) || path.split('/').any(|segment| segment.is_empty() || matches!(segment, "." | ".."))- {- return Err(invalid_file(path));- }- Ok(())-}--fn is_lower_hex(- value: &str,- length: usize,-) -> bool {- value.len() == length && value.bytes().all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f'))-}--fn invalid_file(path: &str) -> RegistryError {- resolution_error(format!("invalid file: {path}"))-}--fn resolution_error(message: impl ToString) -> RegistryError {- RegistryError::UnableToGetModels {- message: message.to_string(),- }-}--#[cfg(test)]-#[path = "../../tests/unit/models/resolver_test.rs"]-mod tests;diff --git a/crates/legacy/uzu/src/registry/local/config.rs b/crates/legacy/uzu/src/registry/local/config.rsindex 6b6ae2291..b6c189a6a 100644--- a/crates/legacy/uzu/src/registry/local/config.rs+++ b/crates/legacy/uzu/src/registry/local/config.rs@@ -77,8 +77,8 @@ impl Config {}fn resolve_model(mut model: Model) -> Result<Model, RegistryError> {- let model_path = model.filesystem_path().ok_or_else(|| RegistryError::UnableToGetModels {- message: format!("Filesystem model {} has no path", model.identifier),+ let model_path = model.local_external_path().ok_or_else(|| RegistryError::UnableToGetModels {+ message: format!("Local model {} has no local path", model.identifier),})?;let model_path = PathBuf::from(model_path);let specialization = Self::resolve_model_specialization(&model_path)?;diff --git a/crates/legacy/uzu/src/registry/local/mod.rs b/crates/legacy/uzu/src/registry/local/mod.rsindex 331d87adc..15c0842c2 100644--- a/crates/legacy/uzu/src/registry/local/mod.rs+++ b/crates/legacy/uzu/src/registry/local/mod.rs@@ -8,7 +8,7 @@ use shoji::{traits::Registry as RegistryTrait,types::{basic::Value,- model::{Model, ModelAccessibility, ModelSource, ModelSpecialization},+ model::{Model, ModelAccessibility, ModelReference, ModelSpecialization},},};@@ -98,8 +98,8 @@ impl Registry {self.config.backend_identifier.clone(),self.config.backend_version.clone(),vec![ModelSpecialization::Chat {}],- ModelAccessibility::OnDevice {- source: ModelSource::Filesystem {+ ModelAccessibility::Local {+ reference: ModelReference::Local {path: path.to_string_lossy().to_string(),},},diff --git a/crates/legacy/uzu/src/registry/merged.rs b/crates/legacy/uzu/src/registry/merged.rsindex 32622ad70..57aa502ac 100644--- a/crates/legacy/uzu/src/registry/merged.rs+++ b/crates/legacy/uzu/src/registry/merged.rs@@ -19,7 +19,7 @@ impl MergedRegistry {&mut self,registry: Box<dyn Registry<Error = RegistryError>>,) -> Result<(), RegistryError> {- if self.registries.iter().any(|current| current.indentifier() == registry.indentifier()) {+ if self.registries.iter().any(|current_registry| current_registry.indentifier() == registry.indentifier()) {return Err(RegistryError::UnableToAddRegistry {identifier: registry.indentifier(),});@@ -31,17 +31,9 @@ impl MergedRegistry {pub fn remove(&mut self,identifier: &str,- ) -> Option<(usize, Box<dyn Registry<Error = RegistryError>>)> {- let index = self.registries.iter().position(|registry| registry.indentifier() == identifier)?;- Some((index, self.registries.remove(index)))- }-- pub fn restore(- &mut self,- index: usize,- registry: Box<dyn Registry<Error = RegistryError>>,- ) {- self.registries.insert(index, registry);+ ) -> Result<(), RegistryError> {+ self.registries.retain(|registry| registry.indentifier() != identifier);+ Ok(())}}@@ -57,8 +49,13 @@ impl Registry for MergedRegistry {let results = futures::future::join_all(self.registries.iter().map(|registry| registry.models())).await;let mut models = Vec::new();- for result in results {- models.extend(result?);+ for (registry, result) in self.registries.iter().zip(results) {+ match result {+ Ok(registry_models) => models.extend(registry_models),+ Err(error) => {+ tracing::warn!(?error, registry = %registry.indentifier(), "skipping registry that failed to list models");+ },+ }}Ok(models)})diff --git a/crates/legacy/uzu/src/registry/mirai/api.rs b/crates/legacy/uzu/src/registry/mirai/api.rsnew file mode 100644index 000000000..64a907019--- /dev/null+++ b/crates/legacy/uzu/src/registry/mirai/api.rs@@ -0,0 +1,59 @@+use indexmap::IndexMap;+use nagare::api::{Config, Endpoint as EndpointTrait, Payload};+use reqwest::{Method, header::CONTENT_TYPE};+use serde_json::json;++use crate::{device::Device, registry::mirai::config::Backend};++pub enum Endpoint {+ FetchModels {+ device: Device,+ backends: Vec<Backend>,+ include_traces: bool,+ show_all: bool,+ },+}++impl EndpointTrait for Endpoint {+ fn method(&self) -> Method {+ match self {+ Endpoint::FetchModels {+ ..+ } => Method::POST,+ }+ }++ fn path(&self) -> String {+ match self {+ Endpoint::FetchModels {+ ..+ } => "fetch/models".to_string(),+ }+ }++ fn headers(&self) -> IndexMap<String, String> {+ IndexMap::from([(CONTENT_TYPE.to_string(), "application/json".to_string())])+ }++ fn payload(+ &self,+ _: &Config,+ ) -> Payload {+ match self {+ Endpoint::FetchModels {+ device,+ backends,+ include_traces,+ show_all,+ } => Payload {+ query: None,+ body: Some(json!({+ "device": device,+ "backends": backends,+ "include_traces": include_traces,+ "show_all": show_all+ })),+ },+ }+ }+}diff --git a/crates/legacy/uzu/src/registry/mirai/config.rs b/crates/legacy/uzu/src/registry/mirai/config.rsindex 75c8e432e..0c264a58c 100644--- a/crates/legacy/uzu/src/registry/mirai/config.rs+++ b/crates/legacy/uzu/src/registry/mirai/config.rs@@ -1,3 +1,5 @@+use std::path::PathBuf;+use serde::{Deserialize, Serialize};use crate::device::Device;@@ -16,6 +18,7 @@ pub struct Config {pub device: Device,pub backends: Vec<Backend>,pub include_traces: bool,+ pub cache_path: PathBuf,}impl Config {@@ -24,12 +27,14 @@ impl Config {device: Device,backends: Vec<Backend>,include_traces: bool,+ cache_path: PathBuf,) -> Self {Self {api_key,device,backends,include_traces,+ cache_path,}}}diff --git a/crates/legacy/uzu/src/registry/mirai/mod.rs b/crates/legacy/uzu/src/registry/mirai/mod.rsindex b9a277a12..978fddac2 100644--- a/crates/legacy/uzu/src/registry/mirai/mod.rs+++ b/crates/legacy/uzu/src/registry/mirai/mod.rs@@ -1,20 +1,24 @@+mod api;mod config;+mod types;-use std::{future::Future, pin::Pin, time::Duration};+use std::{+ fs::{read_to_string, write},+ future::Future,+ path::PathBuf,+ pin::Pin,+ time::Duration,+};+pub use api::Endpoint;pub use config::{Backend, Config};use indexmap::IndexMap;use nagare::api::{Client, Config as ClientConfig, Error as ApiError};use reqwest::header::AUTHORIZATION;use shoji::{traits::Registry as RegistryTrait, types::model::Model};+pub use types::Response;-use crate::{- api::{FetchModelsRequest, FetchedModels},- registry::RegistryError,-};--pub const MIRAI_API_SCHEME: &str = "https";-pub const MIRAI_API_HOST: &str = "sdk.trymirai.com";+use crate::registry::RegistryError;pub struct Registry {config: Config,@@ -28,11 +32,8 @@ impl Registry {headers.insert(AUTHORIZATION.to_string(), format!("Bearer {}", api_key));}- let client_config = ClientConfig::new(- format!("{MIRAI_API_SCHEME}://{MIRAI_API_HOST}/api/v1"),- Duration::from_secs(10),- headers,- );+ let client_config =+ ClientConfig::new("https://sdk.trymirai.com/api/v1".to_string(), Duration::from_secs(10), headers);let client = Client::new(client_config).map_err(|error| RegistryError::UnableToCreate {message: error.to_string(),})?;@@ -52,24 +53,65 @@ impl RegistryTrait for Registry {fn models(&self) -> Pin<Box<dyn Future<Output = Result<Vec<Model>, RegistryError>> + Send + '_>> {Box::pin(async {- self.fetch_models().await.map_err(|error| RegistryError::UnableToGetModels {- message: error.to_string(),- })+ match self.fetch_models().await {+ Ok(models) => {+ if let Err(error) = self.save_registry(&models) {+ tracing::warn!(?error, "failed to save Mirai registry");+ }+ Ok(models)+ },+ Err(error) => {+ let transient = matches!(error, ApiError::Timeout | ApiError::Network(_))+ || matches!(error, ApiError::Http { code, .. } if code >= 500);+ if transient && let Ok(models) = self.load_registry() {+ tracing::warn!(?error, "serving cached Mirai registry after fetch failure");+ return Ok(models);+ }+ Err(RegistryError::UnableToGetModels {+ message: error.to_string(),+ })+ },+ }})}}impl Registry {async fn fetch_models(&self) -> Result<Vec<Model>, ApiError> {- let response: FetchedModels = self+ let response: Response = self.client- .response(&FetchModelsRequest::new(- self.config.device.clone(),- self.config.backends.clone(),- self.config.include_traces,- std::env::var("UZU_REGISTRY_SHOW_ALL").is_ok(),- ))+ .response(&Endpoint::FetchModels {+ device: self.config.device.clone(),+ backends: self.config.backends.clone(),+ include_traces: self.config.include_traces,+ show_all: std::env::var("UZU_REGISTRY_SHOW_ALL").is_ok(),+ }).await?;- response.models().map_err(ApiError::Decode)+ response.models().ok_or_else(|| ApiError::Decode("response contained no models".to_string()))+ }++ fn registry_path(&self) -> PathBuf {+ self.config.cache_path.join("registry.json")+ }++ fn save_registry(+ &self,+ models: &[Model],+ ) -> Result<(), RegistryError> {+ let contents = serde_json::to_vec_pretty(models).map_err(|error| RegistryError::UnableToGetModels {+ message: format!("Unable to serialize registry: {}", error),+ })?;+ write(self.registry_path(), contents).map_err(|error| RegistryError::UnableToGetModels {+ message: format!("Unable to write registry: {}", error),+ })+ }++ fn load_registry(&self) -> Result<Vec<Model>, RegistryError> {+ let contents = read_to_string(self.registry_path()).map_err(|error| RegistryError::UnableToGetModels {+ message: format!("Unable to read registry: {}", error),+ })?;+ serde_json::from_str(&contents).map_err(|error| RegistryError::UnableToGetModels {+ message: format!("Unable to parse registry: {}", error),+ })}}diff --git a/crates/legacy/uzu/src/registry/mirai/types.rs b/crates/legacy/uzu/src/registry/mirai/types.rsnew file mode 100644index 000000000..3e5d6483c--- /dev/null+++ b/crates/legacy/uzu/src/registry/mirai/types.rs@@ -0,0 +1,225 @@+use hanashi::chat::EncodingConfig;+use serde::{Deserialize, Deserializer, Serialize};+use shoji::types::{+ basic::{Metadata, Value},+ model::{+ Model, ModelAccessibility, ModelBackend, ModelFamily, ModelProperties, ModelQuantization, ModelRegistry,+ ModelSpecialization, ModelVendor,+ },+};++fn get_metadata(+ metadatas: &[Metadata],+ identifier: String,+) -> Option<Metadata> {+ metadatas.iter().find(|metadata| metadata.identifier == identifier).cloned()+}++#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub struct Registry {+ pub id: String,+ pub metadata_id: String,+}++impl Registry {+ fn to(+ &self,+ metadatas: &[Metadata],+ ) -> Option<ModelRegistry> {+ let metadata = get_metadata(metadatas, self.metadata_id.clone())?;+ Some(ModelRegistry {+ identifier: self.id.clone(),+ metadata,+ })+ }+}++#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub struct Backend {+ pub id: String,+ pub version: String,+ pub metadata_id: String,+}++impl Backend {+ fn to(+ &self,+ metadatas: &[Metadata],+ ) -> Option<ModelBackend> {+ let metadata = get_metadata(metadatas, self.metadata_id.clone())?;+ Some(ModelBackend {+ identifier: self.id.clone(),+ version: self.version.clone(),+ metadata,+ })+ }+}++#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub struct Vendor {+ pub id: String,+ pub metadata_id: String,+}++impl Vendor {+ fn to(+ &self,+ metadatas: &[Metadata],+ ) -> Option<ModelVendor> {+ let metadata = get_metadata(metadatas, self.metadata_id.clone())?;+ Some(ModelVendor {+ identifier: self.id.clone(),+ metadata,+ })+ }+}++#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub struct Family {+ pub id: String,+ pub vendor: Vendor,+ pub metadata_id: String,+}++impl Family {+ fn to(+ &self,+ metadatas: &[Metadata],+ ) -> Option<ModelFamily> {+ let vendor = self.vendor.to(metadatas)?;+ let metadata = get_metadata(metadatas, self.metadata_id.clone())?;+ Some(ModelFamily {+ identifier: self.id.clone(),+ vendor,+ metadata,+ })+ }+}++#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub struct Properties {+ pub id: String,+ pub size: i64,+ pub version: Option<String>,+ pub metadata_id: String,+}++impl Properties {+ fn to(+ &self,+ metadatas: &[Metadata],+ ) -> Option<ModelProperties> {+ let metadata = get_metadata(metadatas, self.metadata_id.clone())?;+ Some(ModelProperties {+ identifier: self.id.clone(),+ size: self.size,+ version: self.version.clone(),+ metadata,+ })+ }+}++#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub struct Quantization {+ pub id: String,+ pub method: String,+ pub bits_per_weight: u32,+ pub vendor: Vendor,+ pub metadata_id: String,+}++impl Quantization {+ fn to(+ &self,+ metadatas: &[Metadata],+ ) -> Option<ModelQuantization> {+ let vendor = self.vendor.to(metadatas)?;+ let metadata = get_metadata(metadatas, self.metadata_id.clone())?;+ Some(ModelQuantization {+ identifier: self.id.clone(),+ method: self.method.clone(),+ bits_per_weight: self.bits_per_weight,+ vendor,+ metadata,+ })+ }+}+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub struct ResponseModel {+ pub id: String,+ pub registry: Registry,+ pub backends: Vec<Backend>,+ pub family: Option<Family>,+ pub properties: Option<Properties>,+ pub quantization: Option<Quantization>,+ pub specializations: Vec<ModelSpecialization>,+ pub accessibility: ModelAccessibility,+ #[serde(rename = "encodings", default, deserialize_with = "deserialize_encoding")]+ pub encoding: Option<Value>,+}++fn deserialize_encoding<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Option<Value>, D::Error> {+ let encodings = Vec::<Value>::deserialize(deserializer)?;+ Ok(EncodingConfig::select(&encodings))+}++impl ResponseModel {+ fn to(+ &self,+ metadatas: &[Metadata],+ ) -> Option<Model> {+ let registry = self.registry.to(metadatas)?;+ let backends = self.backends.iter().flat_map(|backend| backend.to(metadatas)).collect::<Vec<_>>();+ if backends.len() != self.backends.len() {+ return None;+ }+ let family = match &self.family {+ Some(family) => Some(family.to(metadatas)?),+ None => None,+ };+ let properties = match &self.properties {+ Some(properties) => Some(properties.to(metadatas)?),+ None => None,+ };+ let quantization = match &self.quantization {+ Some(quantization) => Some(quantization.to(metadatas)?),+ None => None,+ };+ Some(Model {+ identifier: self.id.clone(),+ registry,+ backends,+ family,+ properties,+ quantization,+ specializations: self.specializations.clone(),+ accessibility: self.accessibility.clone(),+ encoding: self.encoding.clone(),+ })+ }+}++#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub struct Response {+ models: Vec<ResponseModel>,+ metadatas: Vec<Metadata>,+}++impl Response {+ pub fn models(&self) -> Option<Vec<Model>> {+ let models =+ self.models.iter().flat_map(|response_model| response_model.to(&self.metadatas)).collect::<Vec<_>>();+ if models.len() != self.models.len() {+ return None;+ }+ Some(models)+ }+}diff --git a/crates/legacy/uzu/src/registry/openai/mod.rs b/crates/legacy/uzu/src/registry/openai/mod.rsindex 18e151497..dbd74293f 100644--- a/crates/legacy/uzu/src/registry/openai/mod.rs+++ b/crates/legacy/uzu/src/registry/openai/mod.rs@@ -5,12 +5,23 @@ use std::{future::Future, pin::Pin};use async_openai::{Client, config::OpenAIConfig};pub use config::Config;use fancy_regex::Regex;+use serde::Deserialize;use shoji::{traits::Registry as RegistryTrait,types::model::{Model as ShojiModel, ModelAccessibility, ModelSpecialization},};-use crate::{api::OpenAIModelsResponse, registry::RegistryError};+#[derive(Debug, Deserialize)]+struct Model {+ id: String,+}++#[derive(Debug, Deserialize)]+struct ListModelsResponse {+ data: Vec<Model>,+}++use crate::registry::RegistryError;pub struct Registry {config: Config,@@ -63,7 +74,7 @@ impl RegistryTrait for Registry {fn models(&self) -> Pin<Box<dyn Future<Output = Result<Vec<ShojiModel>, RegistryError>> + Send + '_>> {Box::pin(async {- let response: OpenAIModelsResponse =+ let response: ListModelsResponse =self.client.models().list_byot().await.map_err(|error| RegistryError::UnableToGetModels {message: error.to_string(),})?;diff --git a/crates/legacy/uzu/src/storage/config.rs b/crates/legacy/uzu/src/storage/config.rsindex 7fc2ece0c..450ef69c9 100644--- a/crates/legacy/uzu/src/storage/config.rs+++ b/crates/legacy/uzu/src/storage/config.rs@@ -4,6 +4,7 @@ use download_manager::FileDownloadManagerType;use serde::{Deserialize, Serialize};use shoji::types::model::Model;+use super::download_contents::DownloadContents;use crate::device::Device;#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]@@ -14,6 +15,8 @@ pub struct Config {pub name: String,#[serde(default)]pub download_manager_type: FileDownloadManagerType,+ #[serde(skip)]+ pub download_contents: DownloadContents,}impl Config {@@ -27,6 +30,7 @@ impl Config {base_path,name,download_manager_type: FileDownloadManagerType::default(),+ download_contents: DownloadContents::default(),}}@@ -42,11 +46,9 @@ impl Config {&self,model: &Model,) -> Option<PathBuf> {- if !model.is_downloadable() {- return None;- }+ let reference_name = model.reference_name()?;let checkpoint_version = model.checkpoint_version()?;- Some(self.cache_models_path().join("mirai").join(model.cache_identifier()).join(checkpoint_version))+ Some(self.cache_models_path().join(reference_name).join(model.cache_identifier()).join(checkpoint_version))}pub fn log_name(&self) -> String {@@ -62,4 +64,14 @@ impl Config {..self.clone()}}++ pub fn with_download_contents(+ &self,+ download_contents: DownloadContents,+ ) -> Self {+ Self {+ download_contents,+ ..self.clone()+ }+ }}diff --git a/crates/legacy/uzu/src/storage/download_contents.rs b/crates/legacy/uzu/src/storage/download_contents.rsnew file mode 100644index 000000000..7fb0ca1fb--- /dev/null+++ b/crates/legacy/uzu/src/storage/download_contents.rs@@ -0,0 +1,33 @@+use bitflags::bitflags;++bitflags! {+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+ pub struct DownloadContents: u8 {+ const CONFIG = 0b0000_0001;+ const TOKENIZER = 0b0000_0010;+ const WEIGHTS = 0b0000_0100;+ const TRACES = 0b0000_1000;+ }+}++impl Default for DownloadContents {+ fn default() -> Self {+ Self::all()+ }+}++impl DownloadContents {+ pub fn includes_file(+ &self,+ file_name: &str,+ ) -> bool {+ let flag = match file_name {+ "config.json" => Self::CONFIG,+ "tokenizer.json" => Self::TOKENIZER,+ "traces.safetensors" => Self::TRACES,+ name if name.ends_with(".safetensors") => Self::WEIGHTS,+ _ => Self::CONFIG,+ };+ self.contains(flag)+ }+}diff --git a/crates/legacy/uzu/src/storage/error.rs b/crates/legacy/uzu/src/storage/error.rsindex 8d640119c..cd04057e2 100644--- a/crates/legacy/uzu/src/storage/error.rs+++ b/crates/legacy/uzu/src/storage/error.rs@@ -1,4 +1,4 @@-use crate::storage::types::DownloadPhase;+use crate::{registry::RegistryError, storage::types::DownloadPhase};#[bindings::export(Error)]#[derive(Debug, Clone, PartialEq, thiserror::Error)]@@ -8,10 +8,19 @@ pub enum StorageError {UnableToCreateDirectory {path: String,},+ #[error("Unable to create download manager: {message}")]+ UnableToCreateDownloadManager {+ message: String,+ },#[error("Download manager error: {message}")]DownloadManager {message: String,},+ #[error("Hash not found for file: {identifier}/{name}")]+ HashNotFound {+ identifier: String,+ name: String,+ },#[error("Invalid state transition from {from:?} to {to:?}")]InvalidStateTransition {from: DownloadPhase,@@ -25,6 +34,8 @@ pub enum StorageError {ItemNotFound {identifier: String,},+ #[error(transparent)]+ Registry(#[from] RegistryError),#[error("Unsupported item: {identifier}")]UnsupportedItem {identifier: String,diff --git a/crates/legacy/uzu/src/storage/mod.rs b/crates/legacy/uzu/src/storage/mod.rsindex eaf654401..121686e36 100644--- a/crates/legacy/uzu/src/storage/mod.rs+++ b/crates/legacy/uzu/src/storage/mod.rs@@ -1,8 +1,296 @@mod config;+mod download_contents;mod error;-mod storage;pub mod types;+use std::{+ collections::{HashMap, HashSet},+ fs::{create_dir_all, read_dir, remove_dir_all},+ sync::Arc,+};+pub use config::Config;+pub use download_contents::DownloadContents;+use download_manager::{FileCheck, FileDownloadManager, FileDownloadPhase};pub use error::StorageError;-pub use storage::Storage;+use futures_util::future::join_all;+use kiban::rt::RuntimeHandle;+use shoji::types::{+ basic::File,+ model::{Model, ModelAccessibility, ModelReference},+};+use tokio::sync::broadcast::channel as tokio_broadcast_channel;++use crate::{+ helpers::SharedAccess,+ storage::types::{DownloadPhase, DownloadState, Item, StorageDownloadEventSender, StorageDownloadEventStream},+};++pub struct Storage {+ pub config: Config,++ download_manager: SharedAccess<Arc<dyn FileDownloadManager>>,+ items: SharedAccess<HashMap<String, Item>>,+ items_broadcast_sender: StorageDownloadEventSender,+ handle: RuntimeHandle,+}++impl Storage {+ pub async fn new(+ runtime_handle: RuntimeHandle,+ config: Config,+ ) -> Result<Self, StorageError> {+ create_dir_all(config.cache_path()).map_err(|_| StorageError::UnableToCreateDirectory {+ path: config.cache_path().to_string_lossy().to_string(),+ })?;++ let download_manager = SharedAccess::new(Arc::from(+ <dyn FileDownloadManager>::new(config.download_manager_type, runtime_handle.clone()).await.map_err(+ |error| StorageError::DownloadManager {+ message: error.to_string(),+ },+ )?,+ ));++ let items = SharedAccess::new(HashMap::new());++ let (items_broadcast_sender, _) = tokio_broadcast_channel(256);++ let storage = Self {+ config,+ download_manager,+ items,+ items_broadcast_sender,+ handle: runtime_handle,+ };+ Ok(storage)+ }+}++impl Storage {+ pub async fn refresh(+ &self,+ models: Vec<Model>,+ ) -> Result<(), StorageError> {+ let models = models.into_iter().filter(|model| model.is_downloadable()).collect::<Vec<_>>();+ let actual_model_identifiers: HashSet<String> = models.iter().map(|model| model.identifier.clone()).collect();++ let download_manager = { self.download_manager.lock().await.clone() };++ let existing_file_tasks =+ download_manager.get_all_file_tasks().await.map_err(|error| StorageError::DownloadManager {+ message: error.to_string(),+ })?;+ let mut active_file_tasks = Vec::with_capacity(existing_file_tasks.len());+ for task in existing_file_tasks {+ let task_state = task.state().await;+ if matches!(task_state.phase, FileDownloadPhase::Error(_)) {+ let download_id = task.download_id();+ let _ = task.cancel().await;+ let _ = download_manager.remove_file_task(download_id).await;+ } else {+ active_file_tasks.push(task);+ }+ }++ let mut items = self.items.lock().await;+ let stale_model_identifiers: Vec<String> = items+ .keys()+ .filter(|identifier| !actual_model_identifiers.contains(identifier.as_str()))+ .cloned()+ .collect();+ for identifier in stale_model_identifiers {+ if let Some(item) = items.remove(&identifier) {+ if let Err(error) = item.detach_active_downloads().await {+ tracing::warn!(?error, identifier, "failed to detach stale model file tasks");+ }+ item.stop_listening().await;+ }+ }++ for model in models {+ let identifier = model.identifier.clone();+ if items.contains_key(&identifier) {+ continue;+ }++ let files = self.resolve_model_files(&model)?;+ let total_bytes: u64 = files.iter().map(|file| file.size as u64).sum();++ let cache_path = self.config.cache_model_path(&model).ok_or(StorageError::UnsupportedItem {+ identifier: identifier.clone(),+ })?;++ let has_files_on_disk = cache_path.exists();+ let has_active_tasks = files.iter().any(|file| {+ let file_path = cache_path.join(&file.name);+ active_file_tasks.iter().any(|task| task.destination() == file_path)+ });++ let item = if has_files_on_disk || has_active_tasks {+ let mut file_tasks = Vec::new();+ for file in &files {+ let file_path = cache_path.join(&file.name);+ let file_check = FileCheck::CRC(file.crc32c().ok_or(StorageError::HashNotFound {+ identifier: identifier.clone(),+ name: file.name.clone(),+ })?);++ let file_task = download_manager+ .file_download_task(&file.url, &file_path, file_check, Some(file.size as u64))+ .await+ .map_err(|error| StorageError::DownloadManager {+ message: error.to_string(),+ })?;++ file_tasks.push(file_task);+ }++ let item = Item::new(+ identifier.clone(),+ files.into(),+ cache_path.clone(),+ DownloadState::not_downloaded(total_bytes as i64),+ download_manager.clone(),+ file_tasks,+ self.handle.clone(),+ self.items_broadcast_sender.clone(),+ );++ item.start_listening().await;+ let _ = item.reconcile().await;+ let item_state = item.state().await;++ if matches!(item_state.phase, DownloadPhase::NotDownloaded {})+ && cache_path.exists()+ && let Ok(entries) = read_dir(&cache_path)+ {+ let has_real_files = entries.flatten().any(|entry| {+ entry+ .file_name()+ .to_str()+ .is_some_and(|name| !name.ends_with(".resume_data") && !name.starts_with('.'))+ });+ if !has_real_files {+ let _ = remove_dir_all(&cache_path);+ }+ }++ item+ } else {+ Item::new(+ identifier.clone(),+ files.into(),+ cache_path.clone(),+ DownloadState::not_downloaded(total_bytes as i64),+ download_manager.clone(),+ Vec::new(),+ self.handle.clone(),+ self.items_broadcast_sender.clone(),+ )+ };++ items.insert(identifier, item);+ }++ Ok(())+ }++ pub fn subscribe(&self) -> StorageDownloadEventStream {+ StorageDownloadEventStream::new(self.items_broadcast_sender.subscribe())+ }++ pub async fn get(+ &self,+ model_identifier: &String,+ ) -> Option<Item> {+ let items = self.items.lock().await;+ items.get(model_identifier).cloned()+ }++ pub async fn state(+ &self,+ model_identifier: &String,+ ) -> Option<DownloadState> {+ let item = self.get(model_identifier).await?;+ let state = item.state().await;+ Some(state)+ }++ pub async fn states(&self) -> HashMap<String, DownloadState> {+ let items = self.items.lock().await;+ let state_futures = items.iter().map(|(identifier, item)| {+ let identifier = identifier.clone();+ async move { (identifier, item.state().await) }+ });+ join_all(state_futures).await.into_iter().collect()+ }++ pub async fn download(+ &self,+ model_identifier: &String,+ ) -> Result<(), StorageError> {+ let item = self.get(model_identifier).await.ok_or_else(|| StorageError::ItemNotFound {+ identifier: model_identifier.clone(),+ })?;+ item.download().await+ }++ pub async fn pause(+ &self,+ model_identifier: &String,+ ) -> Result<(), StorageError> {+ let item = self.get(model_identifier).await.ok_or_else(|| StorageError::ItemNotFound {+ identifier: model_identifier.clone(),+ })?;+ item.pause().await+ }++ pub async fn delete(+ &self,+ model_identifier: &String,+ ) -> Result<(), StorageError> {+ let item = self.get(model_identifier).await.ok_or_else(|| StorageError::ItemNotFound {+ identifier: model_identifier.clone(),+ })?;+ item.cancel().await+ }+}++impl Storage {+ fn resolve_model_files(+ &self,+ model: &Model,+ ) -> Result<Vec<File>, StorageError> {+ match &model.accessibility {+ ModelAccessibility::Local {+ reference,+ ..+ } => match &reference {+ ModelReference::Mirai {+ files,+ ..+ } => Ok(files+ .iter()+ .filter(|file| self.config.download_contents.includes_file(&file.name))+ .cloned()+ .collect()),+ ModelReference::HuggingFace {+ ..+ } => Err(StorageError::UnsupportedItem {+ identifier: model.identifier.clone(),+ }),+ ModelReference::Local {+ ..+ } => Err(StorageError::UnsupportedItem {+ identifier: model.identifier.clone(),+ }),+ },+ ModelAccessibility::Remote {+ ..+ } => Err(StorageError::UnsupportedItem {+ identifier: model.identifier.clone(),+ }),+ }+ }+}diff --git a/crates/legacy/uzu/src/storage/storage.rs b/crates/legacy/uzu/src/storage/storage.rsdeleted file mode 100644index 56084d72b..000000000--- a/crates/legacy/uzu/src/storage/storage.rs+++ /dev/null@@ -1,238 +0,0 @@-use std::{- collections::{HashMap, HashSet},- fs::{create_dir_all, read_dir, remove_dir_all},- sync::Arc,-};--use download_manager::{FileDownloadManager, FileDownloadPhase};-use futures_util::future::join_all;-use kiban::rt::RuntimeHandle;-use tokio::sync::broadcast::channel as tokio_broadcast_channel;--use super::{- Config, StorageError,- types::{DownloadPhase, DownloadState, Item, StorageDownloadEventSender, StorageDownloadEventStream},-};-use crate::{helpers::SharedAccess, models::ResolvedModels};--pub struct Storage {- pub config: Config,-- download_manager: SharedAccess<Arc<dyn FileDownloadManager>>,- items: SharedAccess<HashMap<String, Item>>,- items_broadcast_sender: StorageDownloadEventSender,- handle: RuntimeHandle,-}--impl Storage {- pub async fn new(- runtime_handle: RuntimeHandle,- config: Config,- ) -> Result<Self, StorageError> {- create_dir_all(config.cache_path()).map_err(|_| StorageError::UnableToCreateDirectory {- path: config.cache_path().to_string_lossy().to_string(),- })?;-- let download_manager = SharedAccess::new(Arc::from(- <dyn FileDownloadManager>::new(config.download_manager_type, runtime_handle.clone()).await.map_err(- |error| StorageError::DownloadManager {- message: error.to_string(),- },- )?,- ));-- let (items_broadcast_sender, _) = tokio_broadcast_channel(256);- Ok(Self {- config,- download_manager,- items: SharedAccess::new(HashMap::new()),- items_broadcast_sender,- handle: runtime_handle,- })- }-}--#[cfg(test)]-#[path = "../../tests/unit/storage/mod.rs"]-mod tests;--impl Storage {- pub async fn refresh(- &self,- models: &ResolvedModels,- huggingface_api_key: Option<Arc<str>>,- ) -> Result<(), StorageError> {- let models = models- .iter()- .filter_map(|model| {- let (model, files) = model.parts();- files.map(|files| (model.clone(), Arc::new(files.to_vec())))- })- .collect::<Vec<_>>();- let actual_model_identifiers: HashSet<String> =- models.iter().map(|(model, _)| model.identifier.clone()).collect();-- let download_manager = { self.download_manager.lock().await.clone() };-- let existing_file_tasks =- download_manager.get_all_file_tasks().await.map_err(|error| StorageError::DownloadManager {- message: error.to_string(),- })?;- let mut active_file_tasks = Vec::with_capacity(existing_file_tasks.len());- for task in existing_file_tasks {- let task_state = task.state().await;- if matches!(task_state.phase, FileDownloadPhase::Error(_)) {- let download_id = task.download_id();- let _ = task.cancel().await;- let _ = download_manager.remove_file_task(download_id).await;- } else {- active_file_tasks.push(task);- }- }-- let stale_items = {- let mut items = self.items.lock().await;- items- .extract_if(|identifier, _| !actual_model_identifiers.contains(identifier.as_str()))- .map(|(_, item)| item)- .collect::<Vec<_>>()- };- for item in stale_items {- if let Err(error) = item.detach_active_downloads().await {- tracing::warn!(?error, identifier = %item.identifier, "failed to detach stale model file tasks");- }- item.stop_listening().await;- }-- for (model, files) in models {- let identifier = model.identifier.clone();- let cache_path = self.config.cache_model_path(&model).ok_or(StorageError::UnsupportedItem {- identifier: identifier.clone(),- })?;- if let Some(item) = self.get(&identifier).await {- if item.matches(&cache_path, &files) {- continue;- }- item.detach_active_downloads().await?;- item.stop_listening().await;- }-- let total_bytes = Item::total_bytes(&files)?;-- let has_files_on_disk = cache_path.exists();- let has_active_tasks = files.iter().any(|file| {- let file_path = cache_path.join(&file.file.name);- active_file_tasks.iter().any(|task| task.destination() == file_path)- });-- let item = Item::new(- identifier.clone(),- Arc::clone(&files),- cache_path.clone(),- DownloadState::not_downloaded(total_bytes),- download_manager.clone(),- self.handle.clone(),- self.items_broadcast_sender.clone(),- );-- if has_files_on_disk || has_active_tasks {- item.ensure_file_tasks(huggingface_api_key.as_ref()).await?;- item.start_listening().await;- let _ = item.reconcile().await;- let item_state = item.state().await;-- if matches!(item_state.phase, DownloadPhase::NotDownloaded {})- && cache_path.exists()- && let Ok(entries) = read_dir(&cache_path)- {- let has_real_files = entries.flatten().any(|entry| {- entry- .file_name()- .to_str()- .is_some_and(|name| !name.ends_with(".resume_data") && !name.starts_with('.'))- });- if !has_real_files {- let _ = remove_dir_all(&cache_path);- }- }- }-- self.items.lock().await.insert(identifier, item);- }-- Ok(())- }-- pub fn subscribe(&self) -> StorageDownloadEventStream {- StorageDownloadEventStream::new(self.items_broadcast_sender.subscribe())- }-- pub async fn get(- &self,- model_identifier: &str,- ) -> Option<Item> {- let items = self.items.lock().await;- items.get(model_identifier).cloned()- }-- pub async fn state(- &self,- model_identifier: &str,- ) -> Option<DownloadState> {- let item = self.get(model_identifier).await?;- let state = item.state().await;- Some(state)- }-- pub async fn states(&self) -> HashMap<String, DownloadState> {- let items = self.items.lock().await;- let state_futures = items.iter().map(|(identifier, item)| {- let identifier = identifier.clone();- async move { (identifier, item.state().await) }- });- join_all(state_futures).await.into_iter().collect()- }-- pub async fn download(- &self,- model_identifier: &str,- huggingface_api_key: Option<Arc<str>>,- ) -> Result<(), StorageError> {- let item = self.item(model_identifier).await?;- match item.state().await.phase {- DownloadPhase::Downloading {} | DownloadPhase::Downloaded {} => Ok(()),- DownloadPhase::Error {- ..- } => {- item.cancel().await?;- item.download(huggingface_api_key).await- },- _ => item.download(huggingface_api_key).await,- }- }-- pub async fn pause(- &self,- model_identifier: &str,- ) -> Result<(), StorageError> {- let item = self.item(model_identifier).await?;- item.pause().await- }-- pub async fn delete(- &self,- model_identifier: &str,- ) -> Result<(), StorageError> {- let item = self.item(model_identifier).await?;- item.cancel().await- }-- async fn item(- &self,- model_identifier: &str,- ) -> Result<Item, StorageError> {- self.get(model_identifier).await.ok_or_else(|| StorageError::ItemNotFound {- identifier: model_identifier.to_string(),- })- }-}diff --git a/crates/legacy/uzu/src/storage/types/item.rs b/crates/legacy/uzu/src/storage/types/item.rsindex ef87319fc..619ee4060 100644--- a/crates/legacy/uzu/src/storage/types/item.rs+++ b/crates/legacy/uzu/src/storage/types/item.rs@@ -1,25 +1,21 @@use std::{- fmt::{Debug, Formatter, Result as FmtResult},fs::{create_dir_all, remove_dir_all},- path::{Path, PathBuf},+ path::PathBuf,sync::Arc,};-use download_manager::{- DownloadError, FileDownloadManager, FileDownloadPhase, FileDownloadState, FileDownloadTask, HttpDownloadRequest,-};+use download_manager::{DownloadError, FileDownloadManager, FileDownloadPhase, FileDownloadState, FileDownloadTask};use futures_util::future::join_all;use kiban::rt::{RuntimeHandle, TaskJoinHandle};+use shoji::types::basic::File;use tokio::sync::{broadcast::{Sender as TokioBroadcastSender, channel as tokio_broadcast_channel},mpsc::channel as tokio_mpsc_channel,};-use tokio_stream::{StreamExt as TokioStreamExt, wrappers::BroadcastStream as TokioBroadcastStream};-use uuid::Uuid;+use tokio_stream::wrappers::BroadcastStream;use crate::{helpers::SharedAccess,- models::ResolvedFile,storage::{StorageError,types::{DownloadPhase, DownloadState, StorageDownloadEventSender, reduce_file_download_states},@@ -28,7 +24,7 @@ use crate::{pub struct Item {pub identifier: String,- files: Arc<Vec<ResolvedFile>>,+ pub files: Arc<Vec<File>>,pub cache_path: PathBuf,download_state: SharedAccess<DownloadState>,@@ -42,11 +38,11 @@ pub struct Item {listener_task: SharedAccess<Option<Box<dyn TaskJoinHandle<()>>>>,}-impl Debug for Item {+impl std::fmt::Debug for Item {fn fmt(&self,- f: &mut Formatter<'_>,- ) -> FmtResult {+ f: &mut std::fmt::Formatter<'_>,+ ) -> std::fmt::Result {f.debug_struct("Item").field("identifier", &self.identifier).field("cache_path", &self.cache_path).finish()}}@@ -70,20 +66,13 @@ impl Clone for Item {}impl Item {- pub fn matches(- &self,- cache_path: &Path,- files: &[ResolvedFile],- ) -> bool {- self.cache_path == cache_path && self.files.as_slice() == files- }-pub fn new(identifier: String,- files: Arc<Vec<ResolvedFile>>,+ files: Arc<Vec<File>>,cache_path: PathBuf,download_state: DownloadState,file_download_manager: Arc<dyn FileDownloadManager>,+ file_download_tasks: Vec<Arc<dyn FileDownloadTask>>,runtime_handle: RuntimeHandle,storage_broadcast_sender: StorageDownloadEventSender,) -> Self {@@ -95,7 +84,7 @@ impl Item {cache_path,download_state: SharedAccess::new(download_state),file_download_manager,- file_download_tasks: SharedAccess::new(Vec::new()),+ file_download_tasks: SharedAccess::new(file_download_tasks),file_download_states,runtime_handle,broadcast_sender,@@ -109,17 +98,23 @@ impl Item {}async fn get_file_download_states(&self) -> Vec<FileDownloadState> {- let files = Arc::clone(&self.files);let file_tasks_guard = self.file_download_tasks.lock().await;let num_file_tasks = file_tasks_guard.len();drop(file_tasks_guard);+ // Use cached states from broadcasts - more reliable than queryinglet cache_guard = self.file_download_states.lock().await;+ // Validate cache is properly sized before using itif !cache_guard.is_empty() && cache_guard.len() == num_file_tasks {let mut states = cache_guard.clone();- for (state, file) in states.iter_mut().zip(files.iter()) {- Self::fill_total_bytes(state, file);+ for (i, state) in states.iter_mut().enumerate() {+ if state.total_bytes == 0+ && let Some(file_info) = self.files.get(i)+ && file_info.size > 0+ {+ state.total_bytes = file_info.size as u64;+ }}return states;}@@ -136,14 +131,21 @@ impl Item {drop(cache_guard);- let file_tasks = self.file_download_tasks.lock().await.clone();+ // Fall back to querying if cache not initialized or size mismatch+ let file_tasks_guard = self.file_download_tasks.lock().await;let mut states = Vec::new();- for file_task in file_tasks {+ for file_task in file_tasks_guard.iter() {states.push(file_task.state().await);}- for (state, file) in states.iter_mut().zip(files.iter()) {- Self::fill_total_bytes(state, file);+ // Patch total_bytes from registry if missing (safety net)+ for (i, state) in states.iter_mut().enumerate() {+ if state.total_bytes == 0+ && let Some(file_info) = self.files.get(i)+ && file_info.size > 0+ {+ state.total_bytes = file_info.size as u64;+ }}states@@ -151,93 +153,146 @@ impl Item {pub async fn reduce_state(&self) -> DownloadState {let file_download_states = self.get_file_download_states().await;- reduce_file_download_states(&file_download_states)++ tracing::debug!(+ "[MODEL] reduce_state: model={}, file_states count={}",+ self.identifier,+ file_download_states.len()+ );+ for (i, state) in file_download_states.iter().enumerate() {+ tracing::debug!(+ "[MODEL] File {} - phase={:?}, downloaded={}/{} bytes",+ i,+ state.phase,+ state.downloaded_bytes,+ state.total_bytes+ );+ }++ let result = reduce_file_download_states(&file_download_states);++ tracing::debug!(+ "[MODEL] reduce_state result: phase={:?}, bytes={}/{}, progress={:.1}%",+ result.phase,+ result.downloaded_bytes,+ result.total_bytes,+ result.progress() * 100.0+ );++ result}pub async fn update_state_and_broadcast(&self,new_state: DownloadState,) {- *self.download_state.lock().await = new_state.clone();+ tracing::debug!(+ "[MODEL] Broadcasting state: id={}, phase={:?}, progress={:.1}%",+ self.identifier,+ new_state.phase,+ new_state.progress() * 100.0+ );++ let mut state_guard = self.download_state.lock().await;+ *state_guard = new_state.clone();+ drop(state_guard);++ let own_receiver_count = self.broadcast_sender.receiver_count();+ let storage_receiver_count = self.storage_broadcast_sender.receiver_count();++ if own_receiver_count == 0 && storage_receiver_count == 0 {+ tracing::debug!("[MODEL] No receivers active, skipping broadcasts (likely shutdown)");+ return;+ }++ if own_receiver_count > 0 {+ let own_result = self.broadcast_sender.send(new_state.clone());++ match own_result {+ Ok(count) => tracing::debug!("[MODEL] ✓ Own broadcast sent to {} receiver(s)", count),+ Err(_) => {+ tracing::trace!("[MODEL] Own broadcast skipped (no receivers)")+ },+ }+ } else {+ tracing::trace!("[MODEL] Skipping own broadcast (no receivers)");+ }- let _ = self.broadcast_sender.send(new_state.clone());- let _ = self.storage_broadcast_sender.send((self.identifier.clone(), new_state));+ if storage_receiver_count > 0 {+ let storage_result = self.storage_broadcast_sender.send((self.identifier.clone(), new_state));++ match storage_result {+ Ok(count) => tracing::debug!("[MODEL] ✓ Storage broadcast sent to {} receiver(s)", count),+ Err(_) => tracing::debug!("[MODEL] ⚠️ Storage broadcast failed (no receivers)"),+ }+ } else {+ tracing::trace!("[MODEL] Skipping storage broadcast (no receivers)");+ }}pub async fn file_task_by_download_id(&self,- download_id: Uuid,+ download_id: uuid::Uuid,) -> Option<Arc<dyn FileDownloadTask>> {let file_tasks_guard = self.file_download_tasks.lock().await;file_tasks_guard.iter().find(|task| task.download_id() == download_id).cloned()}pub async fn reconcile(&self) -> Result<(), StorageError> {+ // Reconciliation is now handled by FileDownloadTask+ // Just update our broadcast with current statelet calculated_state = self.reduce_state().await;self.update_state_and_broadcast(calculated_state).await;Ok(())}- pub async fn ensure_file_tasks(- &self,- huggingface_api_key: Option<&Arc<str>>,- ) -> Result<bool, StorageError> {- if !self.file_download_tasks.lock().await.is_empty() {+ async fn ensure_file_tasks(&self) -> Result<bool, StorageError> {+ let mut file_tasks_guard = self.file_download_tasks.lock().await;++ // If file tasks already exist, skip creation+ if !file_tasks_guard.is_empty() {return Ok(false);}- let mut new_file_tasks = Vec::with_capacity(self.files.len());+ // Initialize file download tasks+ let mut new_file_tasks = Vec::new();for file_info in self.files.iter() {- let file_path = self.cache_path.join(&file_info.file.name);- let expected_bytes = u64::try_from(file_info.file.size).map_err(|_| StorageError::IO {- message: format!("invalid file size for {}/{}", self.identifier, file_info.file.name),- })?;- let request = if file_info.requires_authentication {- let api_key = huggingface_api_key.ok_or_else(|| StorageError::DownloadManager {- message: "Hugging Face authentication is required".to_string(),- })?;- HttpDownloadRequest::with_bearer_token(file_info.file.url.clone(), api_key)- } else {- HttpDownloadRequest::get(file_info.file.url.clone())- };+ let file_path = self.cache_path.join(&file_info.name);+ let file_check =+ download_manager::FileCheck::CRC(file_info.crc32c().ok_or(StorageError::HashNotFound {+ identifier: self.identifier.clone(),+ name: file_info.name.clone(),+ })?);let file_task = self.file_download_manager- .file_download_task(request, &file_path, file_info.check.clone(), Some(expected_bytes))+ .file_download_task(&file_info.url, &file_path, file_check, Some(file_info.size as u64)).await.map_err(|error| StorageError::DownloadManager {message: error.to_string(),})?;+ // Start listening to global broadcastfile_task.start_listening((*self.file_download_manager.global_broadcast_sender()).clone()).await;new_file_tasks.push(file_task);}- let mut file_tasks = self.file_download_tasks.lock().await;- if file_tasks.is_empty() {- *file_tasks = new_file_tasks;- Ok(true)- } else {- Ok(false)- }+ *file_tasks_guard = new_file_tasks;+ Ok(true)}- pub async fn download(- &self,- huggingface_api_key: Option<Arc<str>>,- ) -> Result<(), StorageError> {- let tasks_were_created = self.ensure_file_tasks(huggingface_api_key.as_ref()).await?;+ pub async fn download(&self) -> Result<(), StorageError> {+ tracing::debug!("[MODEL] download() called for model: {}", self.identifier);++ let tasks_were_created = self.ensure_file_tasks().await?;if tasks_were_created {+ tracing::debug!("[MODEL] Restarting listener for model: {} (file tasks created)", self.identifier);self.stop_listening().await;self.start_listening().await;}let current_state = self.reduce_state().await;- if matches!(current_state.phase, DownloadPhase::Downloaded {}) {- self.update_state_and_broadcast(current_state).await;- return Ok(());- }if !Self::can_transition_to_downloading(¤t_state.phase) {return Err(StorageError::InvalidStateTransition {from: current_state.phase.clone(),@@ -245,6 +300,7 @@ impl Item {});}+ tracing::debug!("[MODEL] Calling ensure_downloading");self.ensure_downloading().await?;self.stop_listening().await;@@ -253,10 +309,20 @@ impl Item {let downloading_state = self.reduce_state().await;self.update_state_and_broadcast(downloading_state).await;+ tracing::debug!("[MODEL] download() completed for model: {}", self.identifier);Ok(())}pub async fn pause(&self) -> Result<(), StorageError> {+ tracing::debug!("[MODEL] pause() called for model: {}", self.identifier);++ let tasks_were_created = self.ensure_file_tasks().await?;+ if tasks_were_created {+ tracing::debug!("[MODEL] Restarting listener for model: {} (file tasks created)", self.identifier);+ self.stop_listening().await;+ self.start_listening().await;+ }+let current_state = self.reduce_state().await;if !current_state.can_pause() {return Err(StorageError::InvalidStateTransition {@@ -265,25 +331,34 @@ impl Item {});}+ tracing::debug!("[MODEL] Calling ensure_paused");self.ensure_paused().await?;- let file_tasks = self.file_download_tasks.lock().await.clone();- let mut file_download_states = Vec::with_capacity(file_tasks.len());- for file_task in file_tasks {+ let file_tasks_guard = self.file_download_tasks.lock().await;+ let mut file_download_states = Vec::with_capacity(file_tasks_guard.len());+ for file_task in file_tasks_guard.iter() {file_download_states.push(file_task.state().await);}+ drop(file_tasks_guard);*self.file_download_states.lock().await = file_download_states;let paused_state = self.reduce_state().await;self.update_state_and_broadcast(paused_state).await;+ tracing::debug!("[MODEL] pause() completed for model: {}", self.identifier);Ok(())}pub async fn cancel(&self) -> Result<(), StorageError> {- let files = Arc::clone(&self.files);- for file_info in files.iter() {- let destination = self.cache_path.join(&file_info.file.name);+ let tasks_were_created = self.ensure_file_tasks().await?;+ if tasks_were_created {+ tracing::debug!("[MODEL] Restarting listener for model: {} (file tasks created)", self.identifier);+ self.stop_listening().await;+ self.start_listening().await;+ }++ for file_info in self.files.iter() {+ let destination = self.cache_path.join(&file_info.name);if let Some(owner) = self.file_download_manager.destination_foreign_lock(&destination).await {tracing::info!(?destination, %owner, "refusing to cancel: destination is locked by another manager");return Err(StorageError::InvalidStateTransition {@@ -311,13 +386,14 @@ impl Item {let _ = remove_dir_all(&self.cache_path);}- let not_downloaded_state = DownloadState::not_downloaded(Self::total_bytes(&files)?);+ let total_bytes: u64 = self.files.iter().map(|f| f.size as u64).sum();+ let not_downloaded_state = DownloadState::not_downloaded(total_bytes as i64);self.update_state_and_broadcast(not_downloaded_state).await;Ok(())}- pub async fn progress(&self) -> Result<TokioBroadcastStream<DownloadState>, StorageError> {- Ok(TokioBroadcastStream::new(self.broadcast_sender.subscribe()))+ pub async fn progress(&self) -> Result<BroadcastStream<DownloadState>, StorageError> {+ Ok(BroadcastStream::new(self.broadcast_sender.subscribe()))}pub async fn detach_active_downloads(&self) -> Result<(), StorageError> {@@ -336,7 +412,7 @@ impl Item {let download_id = file_task.download_id();file_task.cancel().await?;manager.remove_file_task(download_id).await?;- Ok::<(), DownloadError>(())+ Ok::<(), download_manager::DownloadError>(())}});let first_error = join_all(cancel_futures).await.into_iter().find_map(Result::err);@@ -352,33 +428,50 @@ impl Item {/// Handle file task state update/// Called by ModelStorage listener when a file task broadcasts a state changepub async fn handle_file_task_update(&self) {+ tracing::debug!("[MODEL] handle_file_task_update: Computing new state for model: {}", self.identifier);+let new_state = self.reduce_state().await;++ tracing::debug!(+ "[MODEL] handle_file_task_update: New state computed: phase={:?}, bytes={}/{}, progress={:.1}%",+ new_state.phase,+ new_state.downloaded_bytes,+ new_state.total_bytes,+ new_state.progress() * 100.0+ );+self.update_state_and_broadcast(new_state).await;}async fn ensure_downloading(&self) -> Result<(), StorageError> {+ tracing::debug!("[MODEL] ensure_downloading: model={}", self.identifier);+create_dir_all(&self.cache_path).map_err(|error| StorageError::IO {message: error.to_string(),})?;- let file_tasks = self.file_download_tasks.lock().await.clone();- let download_futures = file_tasks.iter().map(|file_task| {+ let file_tasks_guard = self.file_download_tasks.lock().await;+ let download_futures = file_tasks_guard.iter().map(|file_task| {let file_task = file_task.clone();async move { file_task.download().await }});let first_error = join_all(download_futures).await.into_iter().find_map(Result::err);+ drop(file_tasks_guard);+if let Some(error) = first_error {return Err(StorageError::DownloadManager {message: error.to_string(),});}+ tracing::debug!("[MODEL] ensure_downloading: All file downloads initiated");Ok(())}async fn ensure_paused(&self) -> Result<(), StorageError> {- let file_tasks = self.file_download_tasks.lock().await.clone();- let pause_futures = file_tasks.iter().map(|file_task| {+ tracing::debug!("[MODEL] ensure_paused: model={}", self.identifier);+ let file_tasks_guard = self.file_download_tasks.lock().await;+ let pause_futures = file_tasks_guard.iter().map(|file_task| {let file_task = file_task.clone();async move {if matches!(file_task.state().await.phase, FileDownloadPhase::Downloading) {@@ -397,50 +490,89 @@ impl Item {}});let first_error = join_all(pause_futures).await.into_iter().find_map(Result::err);+ drop(file_tasks_guard);+if let Some(error) = first_error {return Err(StorageError::DownloadManager {message: error.to_string(),});}+ tracing::debug!("[MODEL] ensure_paused: All file tasks paused");Ok(())}/// Start listening to file task broadcastspub async fn start_listening(&self) {- if self.listener_task.lock().await.is_some() {+ // Check if already listening+ let mut listener_guard = self.listener_task.lock().await;+ if listener_guard.is_some() {+ tracing::debug!("[MODEL] start_listening: Already listening for model: {}", self.identifier);return;}- let file_tasks = self.file_download_tasks.lock().await.clone();- let files = Arc::clone(&self.files);- let num_files = file_tasks.len();+ // Collect all file task broadcast streams and initialize cache+ let file_tasks_guard = self.file_download_tasks.lock().await;+ let num_files = file_tasks_guard.len();let mut streams = Vec::new();let mut initial_states = Vec::new();- for (idx, file_task) in file_tasks.iter().enumerate() {+ for (idx, file_task) in file_tasks_guard.iter().enumerate() {let sender = file_task.broadcast_sender();- let stream = TokioBroadcastStream::new(sender.subscribe());+ let receiver_count_before = sender.receiver_count();+ let stream = BroadcastStream::new(sender.subscribe());+ tracing::info!(+ "[MODEL] Subscribed to file task {}: model={}, receivers_before={}, receivers_after={}",+ idx,+ self.identifier,+ receiver_count_before,+ sender.receiver_count()+ );streams.push((idx, stream));let mut state = file_task.state().await;- if let Some(file) = files.get(idx) {- Self::fill_total_bytes(&mut state, file);+ if state.total_bytes == 0+ && let Some(file_info) = self.files.get(idx)+ && file_info.size > 0+ {+ state.total_bytes = file_info.size as u64;}initial_states.push(state);}+ drop(file_tasks_guard);+if streams.is_empty() {+ tracing::debug!("[MODEL] start_listening: No file tasks yet for model: {}", self.identifier);return;}+ // Pre-allocate cache with correct size BEFORE starting listener+ // This ensures all updates will have valid indices{let mut cache_guard = self.file_download_states.lock().await;*cache_guard = initial_states;debug_assert_eq!(cache_guard.len(), num_files, "Cache size must match number of file tasks");++ tracing::debug!(+ "[MODEL] Cache initialized with {} entries for model: {}",+ cache_guard.len(),+ self.identifier+ );}+ tracing::debug!(+ "[MODEL] start_listening: Starting listener for model: {} ({} streams)",+ self.identifier,+ streams.len()+ );+let model = self.clone();+ tracing::debug!("[MODEL] Spawning listener task for model: {}", self.identifier);let handle = self.runtime_handle.spawn(async move {+ tracing::debug!("[MODEL] Listener task started for model: {}", model.identifier);+ use tokio_stream::StreamExt as TokioStreamExt;++ // Fan-in: per-stream forwarders into a single bounded channellet num_streams = streams.len();let (tx, mut rx) = tokio_mpsc_channel::<(usize, FileDownloadState)>(1024);let mut forwarder_handles = Vec::with_capacity(num_streams);@@ -449,8 +581,14 @@ impl Item {let tx = tx.clone();let forwarder_handle = model.runtime_handle.spawn(async move {while let Some(item) = stream.next().await {- if let Ok(state) = item {- let _ = tx.send((idx, state)).await;+ match item {+ Ok(state) => {+ // Forward latest state; drop if aggregator has gone away+ let _ = tx.send((idx, state)).await;+ },+ Err(_e) => {+ // Lag on a single stream isn't fatal+ },}}});@@ -459,24 +597,31 @@ impl Item {let _forwarder_handles = ListenerForwarderHandles {handles: forwarder_handles,};- drop(tx);+ drop(tx); // close when all forwarders end+ // Aggregator: coalesce bursts, keep only latest state per tasklet mut pending: Vec<Option<FileDownloadState>> = vec![None; num_streams];while let Some((idx, state)) = rx.recv().await {pending[idx] = Some(state);+ // Drain quickly to coalesce into latest-onlywhile let Ok((i, s)) = rx.try_recv() {pending[i] = Some(s);}+ // Apply all latest states atomically, then notify once{let mut cache_guard = model.file_download_states.lock().await;for (i, slot) in pending.iter_mut().enumerate() {if let Some(mut s) = slot.take() {if i < cache_guard.len() {- if let Some(file) = files.get(i) {- Self::fill_total_bytes(&mut s, file);+ // Patch total bytes if 0 in incoming update+ if s.total_bytes == 0+ && let Some(file_info) = model.files.get(i)+ && file_info.size > 0+ {+ s.total_bytes = file_info.size as u64;}cache_guard[i] = s;} else {@@ -493,44 +638,23 @@ impl Item {model.handle_file_task_update().await;}++ tracing::debug!("[MODEL] All streams ended for model: {}", model.identifier);++ tracing::debug!("[MODEL] Listener task ended for model: {}", model.identifier);});- let mut listener = self.listener_task.lock().await;- if listener.is_none() {- *listener = Some(handle);- } else {- handle.abort();- }+ *listener_guard = Some(handle);}/// Stop listening to file task broadcastspub async fn stop_listening(&self) {- let handle = self.listener_task.lock().await.take();- if let Some(handle) = handle {+ let mut listener_guard = self.listener_task.lock().await;+ if let Some(handle) = listener_guard.take() {handle.abort_and_join().await;}}- pub fn total_bytes(files: &[ResolvedFile]) -> Result<i64, StorageError> {- files.iter().try_fold(0_i64, |total, file| {- total.checked_add(file.file.size).ok_or_else(|| StorageError::IO {- message: "model download size overflow".to_string(),- })- })- }-- fn fill_total_bytes(- state: &mut FileDownloadState,- file: &ResolvedFile,- ) {- if state.total_bytes == 0- && let Ok(size) = u64::try_from(file.file.size)- && size > 0- {- state.total_bytes = size;- }- }-fn can_transition_to_downloading(from: &DownloadPhase) -> bool {matches!(from,diff --git a/crates/legacy/uzu/tests/common/mod.rs b/crates/legacy/uzu/tests/common/mod.rsnew file mode 100644index 000000000..62e95933c--- /dev/null+++ b/crates/legacy/uzu/tests/common/mod.rs@@ -0,0 +1,2 @@+pub mod test_storage;+pub mod tracing_setup;diff --git a/crates/legacy/uzu/tests/unit/common/test_storage.rs b/crates/legacy/uzu/tests/common/test_storage.rssimilarity index 56%rename from crates/legacy/uzu/tests/unit/common/test_storage.rsrename to crates/legacy/uzu/tests/common/test_storage.rsindex 88185cc5d..56a958be3 100644--- a/crates/legacy/uzu/tests/unit/common/test_storage.rs+++ b/crates/legacy/uzu/tests/common/test_storage.rs@@ -1,19 +1,32 @@+#![allow(dead_code)]++use std::path::PathBuf;+use download_manager::FileDownloadManagerType;use kiban::rt::RuntimeHandle;use shoji::types::model::Model;--use crate::{+use uzu::{device::Device,- models::{ModelsResolver, ResolvedModels},+ registry::FixedRegistry,storage::{Config, Storage},};pub struct TestStorage {+ pub config: Config,+ pub registry: FixedRegistry,pub storage: Storage,+ pub base_path: PathBuf,_temp_dir_guard: tempfile::TempDir,}impl TestStorage {+ pub async fn with_models(+ tokio_handle: RuntimeHandle,+ models: Vec<Model>,+ ) -> Result<Self, Box<dyn std::error::Error>> {+ Self::with_models_and_manager(tokio_handle, models, FileDownloadManagerType::default()).await+ }+pub async fn with_models_and_manager(tokio_handle: RuntimeHandle,models: Vec<Model>,@@ -21,16 +34,17 @@ impl TestStorage {) -> Result<Self, Box<dyn std::error::Error>> {let temp_dir_guard = tempfile::tempdir()?;let base_path = temp_dir_guard.path().to_path_buf();+ let registry = FixedRegistry::new("test_registry".to_string(), models.clone());let device = Device::new()?;let config = Config::new(device, Some(base_path.clone()), "test_storage".to_string()).with_download_manager_type(download_manager_type);- let storage = Storage::new(tokio_handle, config).await?;- let resolved = ModelsResolver::new(None, base_path.join("resolved-models.json"))?- .resolve(models, &ResolvedModels::default())- .await?;- storage.refresh(&resolved, None).await?;+ let storage = Storage::new(tokio_handle, config.clone()).await?;+ storage.refresh(models).await?;Ok(Self {+ config,+ registry,storage,+ base_path,_temp_dir_guard: temp_dir_guard,})}diff --git a/crates/legacy/uzu/tests/common/tracing_setup.rs b/crates/legacy/uzu/tests/common/tracing_setup.rsnew file mode 100644index 000000000..8b73a0249--- /dev/null+++ b/crates/legacy/uzu/tests/common/tracing_setup.rs@@ -0,0 +1,30 @@+#![allow(dead_code)]++use std::sync::Once;++use tracing_subscriber::{EnvFilter, fmt, layer::SubscriberExt, util::SubscriberInitExt};++static INIT: Once = Once::new();++pub fn init_test_tracing() {+ INIT.call_once(|| {+ let timestamp = chrono::Utc::now().format("%Y%m%d_%H%M%S");+ let log_dir = std::env::temp_dir().join("uzu-storage-test-logs");+ std::fs::create_dir_all(&log_dir).ok();++ let file_appender = tracing_appender::rolling::never(&log_dir, format!("storage_{}.log", timestamp));++ let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);++ // Keep guard alive for entire test run+ std::mem::forget(guard);++ tracing_subscriber::registry()+ .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("debug")))+ .with(fmt::layer().with_writer(non_blocking).with_ansi(false))+ .with(fmt::layer().with_test_writer())+ .init();++ tracing::info!("Test logging initialized, writing to test_logs/");+ });+}diff --git a/crates/legacy/uzu/tests/integration/engine/mod.rs b/crates/legacy/uzu/tests/integration/engine/mod.rsdeleted file mode 100644index 3ce4c1d4e..000000000--- a/crates/legacy/uzu/tests/integration/engine/mod.rs+++ /dev/null@@ -1 +0,0 @@-mod live_test;diff --git a/crates/legacy/uzu/tests/integration/test_entry.rs b/crates/legacy/uzu/tests/integration/test_entry.rsdeleted file mode 100644index 863383918..000000000--- a/crates/legacy/uzu/tests/integration/test_entry.rs+++ /dev/null@@ -1 +0,0 @@-mod engine;diff --git a/crates/legacy/uzu/tests/integration/engine/live_test.rs b/crates/legacy/uzu/tests/test_engine.rssimilarity index 100%rename from crates/legacy/uzu/tests/integration/engine/live_test.rsrename to crates/legacy/uzu/tests/test_engine.rsdiff --git a/crates/legacy/uzu/tests/integration/tool/calls.rs b/crates/legacy/uzu/tests/tool/calls.rssimilarity index 100%rename from crates/legacy/uzu/tests/integration/tool/calls.rsrename to crates/legacy/uzu/tests/tool/calls.rsdiff --git a/crates/legacy/uzu/tests/integration/tool/macro_closure.rs b/crates/legacy/uzu/tests/tool/macro_closure.rssimilarity index 100%rename from crates/legacy/uzu/tests/integration/tool/macro_closure.rsrename to crates/legacy/uzu/tests/tool/macro_closure.rsdiff --git a/crates/legacy/uzu/tests/integration/tool/macro_function.rs b/crates/legacy/uzu/tests/tool/macro_function.rssimilarity index 100%rename from crates/legacy/uzu/tests/integration/tool/macro_function.rsrename to crates/legacy/uzu/tests/tool/macro_function.rsdiff --git a/crates/legacy/uzu/tests/integration/tool/mod.rs b/crates/legacy/uzu/tests/tool/mod.rssimilarity index 100%rename from crates/legacy/uzu/tests/integration/tool/mod.rsrename to crates/legacy/uzu/tests/tool/mod.rsdiff --git a/crates/legacy/uzu/tests/unit/api/request_test.rs b/crates/legacy/uzu/tests/unit/api/request_test.rsdeleted file mode 100644index 2d97c5f02..000000000--- a/crates/legacy/uzu/tests/unit/api/request_test.rs+++ /dev/null@@ -1,37 +0,0 @@-use serde_json::{json, to_value};--use super::*;-use crate::{device::Device, registry::mirai::Backend};--#[test]-fn fetch_models_request_serializes_the_api_payload() {- let request = FetchModelsRequest::new(- Device {- os_name: Some("macOS".to_string()),- cpu_name: Some("Apple".to_string()),- memory_total: 32,- home_path: "/tmp".to_string(),- },- vec![Backend {- identifier: "uzu".to_string(),- version: "1".to_string(),- }],- false,- true,- );-- assert_eq!(- to_value(request).expect("request should serialize"),- json!({- "device": {- "os_name": "macOS",- "cpu_name": "Apple",- "memory_total": 32,- "home_path": "/tmp"- },- "backends": [{ "identifier": "uzu", "version": "1" }],- "include_traces": false,- "show_all": true- })- );-}diff --git a/crates/legacy/uzu/tests/unit/api/response_test.rs b/crates/legacy/uzu/tests/unit/api/response_test.rsdeleted file mode 100644index f5fa2a1ad..000000000--- a/crates/legacy/uzu/tests/unit/api/response_test.rs+++ /dev/null@@ -1,99 +0,0 @@-use serde_json::{from_value, json};--use super::*;--#[test]-fn fetched_models_converts_one_complete_response() {- let response: FetchedModels = from_value(json!({- "metadatas": [- { "id": "registry-meta", "name": "Mirai", "description": null, "icons": [] },- { "id": "backend-meta", "name": "Uzu", "description": null, "icons": [] }- ],- "models": [{- "id": "model",- "registry": { "id": "mirai", "metadata_id": "registry-meta" },- "backends": [{ "id": "uzu", "version": "1", "metadata_id": "backend-meta" }],- "family": null,- "properties": null,- "quantization": null,- "specializations": [],- "accessibility": {- "type": "local",- "reference": {- "type": "mirai",- "toolchain_version": "1",- "repository": null,- "source_repository": null,- "files": []- }- },- "encodings": []- }]- }))- .expect("response should deserialize");-- let models = response.models().expect("response should resolve metadata references");- assert_eq!(models.len(), 1);- assert_eq!(models[0].identifier, "model");- assert!(matches!(- &models[0].accessibility,- ModelAccessibility::OnDevice {- source: ModelSource::Registry {- toolchain_version,- repository: None,- source_repository: None,- files,- },- } if toolchain_version == "1" && files.is_empty()- ));-- let filesystem: FetchedAccessibility = from_value(json!({- "type": "local",- "reference": { "type": "local", "path": "/models/local" }- }))- .expect("filesystem response should deserialize");- assert_eq!(- ModelAccessibility::from(&filesystem),- ModelAccessibility::OnDevice {- source: ModelSource::Filesystem {- path: "/models/local".to_string(),- },- }- );-}--#[test]-fn hugging_face_access_is_conservative() {- for (gated, requires_authentication) in [(json!(false), false), (json!("manual"), true), (json!(null), true)] {- let response: HuggingFaceModelResponse = from_value(json!({- "sha": "",- "private": false,- "gated": gated,- "siblings": []- }))- .expect("response should deserialize");-- assert_eq!(response.requires_authentication(), requires_authentication);- }-}--#[test]-fn fetched_models_rejects_missing_metadata() {- let response: FetchedModels = from_value(json!({- "metadatas": [],- "models": [{- "id": "model",- "registry": { "id": "mirai", "metadata_id": "missing" },- "backends": [],- "family": null,- "properties": null,- "quantization": null,- "specializations": [],- "accessibility": { "type": "remote", "repository": null },- "encodings": []- }]- }))- .expect("response should deserialize");-- assert!(response.models().is_err());-}diff --git a/crates/legacy/uzu/tests/unit/common/mod.rs b/crates/legacy/uzu/tests/unit/common/mod.rsdeleted file mode 100644index a97611d83..000000000--- a/crates/legacy/uzu/tests/unit/common/mod.rs+++ /dev/null@@ -1 +0,0 @@-pub mod test_storage;diff --git a/crates/legacy/uzu/tests/unit/engine/bridge/helpers_test.rs b/crates/legacy/uzu/tests/unit/engine/bridge/helpers_test.rsdeleted file mode 100644index b99a27cbe..000000000--- a/crates/legacy/uzu/tests/unit/engine/bridge/helpers_test.rs+++ /dev/null@@ -1,31 +0,0 @@-use tokenizers::{Tokenizer, decoders::fuse::Fuse, models::wordlevel::WordLevel};--use super::grammar_trigger_token_sequence_for_prompt;--fn tokenizer() -> Tokenizer {- let model = WordLevel::builder()- .vocab(- [("<unk>".to_string(), 0), ("</think>".to_string(), 1), ("\n\n".to_string(), 2), ("next".to_string(), 3)]- .into_iter()- .collect(),- )- .build()- .unwrap();- let mut tokenizer = Tokenizer::new(model);- tokenizer.with_decoder(Some(Fuse::new()));- tokenizer-}--#[test]-fn grammar_engages_immediately_when_prompt_closes_reasoning() {- let tokenizer = tokenizer();-- assert_eq!(grammar_trigger_token_sequence_for_prompt(Some(&[1]), &[1, 2], &tokenizer), None);-}--#[test]-fn grammar_waits_for_trigger_when_only_history_contains_it() {- let tokenizer = tokenizer();-- assert_eq!(grammar_trigger_token_sequence_for_prompt(Some(&[1]), &[1, 2, 3], &tokenizer), Some(vec![1]));-}diff --git a/crates/legacy/uzu/tests/unit/engine/cache_test.rs b/crates/legacy/uzu/tests/unit/engine/cache_test.rsdeleted file mode 100644index 1416c7ed2..000000000--- a/crates/legacy/uzu/tests/unit/engine/cache_test.rs+++ /dev/null@@ -1,92 +0,0 @@-use std::{error::Error, future::Future, pin::Pin};--use ::download_manager::FileDownloadManagerType;-use kiban::rt::RuntimeHandle;-use nagare::telemetry::Telemetry;-use shoji::{- traits::Registry as RegistryTrait,- types::model::{Model, ModelAccessibility},-};-use tempfile::tempdir;-use tokio::fs::read as tokio_read;--use super::*;-use crate::{- models::{ModelCatalog, ModelsResolver, ResolvedModel, ResolvedModels},- registry::FixedRegistry,-};--struct FailingRegistry(&'static str);--impl RegistryTrait for FailingRegistry {- type Error = RegistryError;-- fn indentifier(&self) -> String {- self.0.to_string()- }-- fn models(&self) -> Pin<Box<dyn Future<Output = Result<Vec<Model>, RegistryError>> + Send + '_>> {- Box::pin(async {- Err(RegistryError::UnableToGetModels {- message: "unavailable".to_string(),- })- })- }-}--#[tokio::test]-async fn failed_refresh_preserves_snapshot_and_registry_changes_roll_back() -> Result<(), Box<dyn Error>> {- let directory = tempdir()?;- let cache_path = directory.path().join("resolved-models.json");- let resolver = ModelsResolver::new(None, cache_path.clone())?;- let model = Model::external(- "cached-model".to_string(),- "cached-registry".to_string(),- "Cached Registry".to_string(),- "backend".to_string(),- "Backend".to_string(),- "1".to_string(),- Vec::new(),- ModelAccessibility::Remote {- repository: None,- },- None,- );- let previous = ResolvedModels::new(vec![ResolvedModel::passthrough(model)]);- resolver.save_cache(&previous).await?;- let cached = tokio_read(&cache_path).await?;- let catalog = ModelCatalog::new(None, cache_path.clone()).await?;-- let mut registry = MergedRegistry::new(Vec::new());- registry.add(Box::new(FailingRegistry("failing")))?;- let runtime_handle = RuntimeHandle::try_current()?;- let storage_config =- StorageConfig::new(Device::new()?, Some(directory.path().to_path_buf()), "cache-test".to_string())- .with_download_manager_type(FileDownloadManagerType::Universal);- let engine = Engine {- settings: SharedAccess::new(None),- registry: SharedAccess::new(registry),- catalog,- catalog_refresh_lock: SharedAccess::new(()),- storage: SharedAccess::new(Storage::new(runtime_handle, storage_config).await?),- backends: SharedAccess::new(HashMap::new()),- callback: SharedAccess::new(None),- telemetry: SharedAccess::new(Telemetry::disabled()),- huggingface_api_key: None,- };-- engine.handle_initial_catalog_refresh().await?;- let registries = engine.registry.lock().await.indentifier();-- assert!(engine.add_registry(Box::new(FixedRegistry::new("working".to_string(), Vec::new()))).await.is_err());- assert_eq!(engine.registry.lock().await.indentifier(), registries);- engine.registry.lock().await.add(Box::new(FailingRegistry("other")))?;- let registries_with_other = engine.registry.lock().await.indentifier();- assert!(engine.remove_registry("failing".to_string()).await.is_err());-- let identifiers = engine.models().await?.into_iter().map(|model| model.identifier).collect::<Vec<_>>();- assert_eq!(identifiers, ["cached-model"]);- assert_eq!(engine.registry.lock().await.indentifier(), registries_with_other);- assert_eq!(tokio_read(cache_path).await?, cached);- Ok(())-}diff --git a/crates/legacy/uzu/tests/unit/mod.rs b/crates/legacy/uzu/tests/unit/mod.rsnew file mode 100644index 000000000..80aaac2ba--- /dev/null+++ b/crates/legacy/uzu/tests/unit/mod.rs@@ -0,0 +1 @@+mod storage;diff --git a/crates/legacy/uzu/tests/unit/models/resolver_test.rs b/crates/legacy/uzu/tests/unit/models/resolver_test.rsdeleted file mode 100644index 5c5394e27..000000000--- a/crates/legacy/uzu/tests/unit/models/resolver_test.rs+++ /dev/null@@ -1,112 +0,0 @@-use std::{error::Error, path::PathBuf};--use serde_json::{Value, from_value, json};-use shoji::types::{- basic::{File, Hash, HashMethod, Repository},- model::{Model, ModelAccessibility, ModelSource},-};--use super::*;-use crate::api::HuggingFaceFileResponse;--const REVISION: &str = "5ed3b433df2e390ace215873283927856573d9e6";--fn repository() -> Repository {- Repository {- identifier: "trymirai/model".to_string(),- commit_hash: Some(REVISION.to_string()),- paths: None,- }-}--fn git_file(- name: &str,- blob_id: Option<&str>,-) -> HuggingFaceFileResponse {- hugging_face_file(json!({ "rfilename": name, "size": 6, "blobId": blob_id }))-}--fn hugging_face_file(value: Value) -> HuggingFaceFileResponse {- from_value(value).expect("Hugging Face file should deserialize")-}--#[test]-fn hugging_face_files_use_git_and_lfs_digests() -> Result<(), Box<dyn Error>> {- let resolver = ModelsResolver::new(None, PathBuf::new())?;- let files = resolver.resolve_files(- &repository(),- REVISION,- vec![- git_file("config.json", Some("ce013625030ba8dba906f756967f9e9ca394464a")),- hugging_face_file(json!({- "rfilename": "model.safetensors",- "size": null,- "blobId": null,- "lfs": {- "sha256": "5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03",- "size": 6- }- })),- ],- false,- )?;-- assert!(matches!(files[0].check, FileCheck::GitBlobSha1(_)));- assert!(matches!(files[1].check, FileCheck::Sha256(_)));- assert!(files.iter().all(|file| file.file.url.contains(REVISION)));- Ok(())-}--#[test]-fn invalid_hugging_face_file_metadata_is_rejected() -> Result<(), Box<dyn Error>> {- let resolver = ModelsResolver::new(None, PathBuf::new())?;-- let missing_digest = resolver.resolve_files(&repository(), REVISION, vec![git_file("config.json", None)], false);- let unsafe_path = resolver.resolve_files(- &repository(),- REVISION,- vec![git_file("../config.json", Some("ce013625030ba8dba906f756967f9e9ca394464a"))],- false,- );-- assert!(missing_digest.is_err());- assert!(unsafe_path.is_err());- Ok(())-}--#[tokio::test]-async fn unpinned_mirai_files_keep_crc_verification() -> Result<(), Box<dyn Error>> {- let resolver = ModelsResolver::new(None, PathBuf::new())?;- let model = Model::external(- "model".to_string(),- "registry".to_string(),- "Registry".to_string(),- "backend".to_string(),- "Backend".to_string(),- "1".to_string(),- Vec::new(),- ModelAccessibility::OnDevice {- source: ModelSource::Registry {- toolchain_version: "1".to_string(),- repository: None,- source_repository: None,- files: vec![File {- url: String::new(),- name: "model".to_string(),- size: 1,- hashes: vec![Hash {- method: HashMethod::CRC32C,- value: "AAAAAA==".to_string(),- }],- }],- },- },- None,- );-- let resolved = resolver.resolve(vec![model], &ResolvedModels::default()).await?;- let (_, files) = resolved.iter().next().expect("model should be resolved").parts();-- assert!(matches!(files.expect("model should be downloadable")[0].check, FileCheck::CRC(_)));- Ok(())-}diff --git a/crates/legacy/uzu/tests/unit/storage/lifecycle_mock_registry_model_test.rs b/crates/legacy/uzu/tests/unit/storage/lifecycle_mock_registry_model_test.rsindex 5c10b1697..b3f9d7fc3 100644--- a/crates/legacy/uzu/tests/unit/storage/lifecycle_mock_registry_model_test.rs+++ b/crates/legacy/uzu/tests/unit/storage/lifecycle_mock_registry_model_test.rs@@ -7,14 +7,14 @@ use mock_registry::{Behavior, MockRegistry};use rstest::rstest;use tokio::time::{Duration, timeout};use tokio_stream::{StreamExt, wrappers::BroadcastStream};--use super::common::test_storage::TestStorage;-use crate::{+use uzu::{engine::Downloader,helpers::SharedAccess,storage::types::{DownloadPhase, DownloadState, Item},};+use crate::common::{test_storage::TestStorage, tracing_setup::init_test_tracing};+#[rstest]#[case::universal(FileDownloadManagerType::Universal)]#[cfg_attr(target_vendor = "apple", case::apple(FileDownloadManagerType::Apple))]@@ -22,8 +22,11 @@ use crate::{async fn test_storage_mock_registry_model_download_lifecycle(#[case] download_manager_type: FileDownloadManagerType) -> Result<(), Box<dyn std::error::Error>> {+ init_test_tracing();+ tracing::info!("starting storage mock registry lifecycle test");let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;let model = registry.models.first().ok_or_else(|| std::io::Error::other("mock registry must include a model"))?;+ tracing::info!(model_identifier = %model.identifier, "loaded mock registry model");let test_storage =TestStorage::with_models_and_manager(RuntimeHandle::current(), vec![model.clone()], download_manager_type).await?;@@ -38,7 +41,8 @@ async fn test_storage_mock_registry_model_download_lifecycle(let mut progress = item.progress().await?;let total_bytes = item.state().await.total_bytes;- item.download(None).await?;+ tracing::info!(total_bytes, "starting first download");+ item.download().await?;let pre_pause_state = wait_for_item_state(&item, &mut progress, "first download reaches 25%", |state| {has_reached_fraction(state, 1, 4)})@@ -49,34 +53,48 @@ async fn test_storage_mock_registry_model_download_lifecycle(pre_pause_state.phase);+ tracing::info!("pausing storage item");item.pause().await?;item.reconcile().await?;- wait_for_item_state(&item, &mut progress, "pause transitions to Paused", |state| {+ let paused_state = wait_for_item_state(&item, &mut progress, "pause transitions to Paused", |state| {matches!(state.phase, DownloadPhase::Paused {})}).await;- item.download(None).await?;- wait_for_item_state(&item, &mut progress, "resumed download reaches 50%", |state| {+ tracing::info!(?paused_state, "observed paused state");++ tracing::info!("resuming storage item");+ item.download().await?;+ let post_resume_state = wait_for_item_state(&item, &mut progress, "resumed download reaches 50%", |state| {has_reached_fraction(state, 1, 2) || matches!(state.phase, DownloadPhase::Downloaded {})}).await;+ assert!(+ matches!(post_resume_state.phase, DownloadPhase::Downloading {} | DownloadPhase::Downloaded {}),+ "resumed model must transition through Downloading and reach Downloaded; got {:?}",+ post_resume_state.phase+ );+ tracing::info!("cancelling storage item");item.cancel().await?;let cancelled_state = item.state().await;+ tracing::info!(?cancelled_state, "observed cancelled state");assert!(matches!(cancelled_state.phase, DownloadPhase::NotDownloaded {}));assert_eq!(cancelled_state.downloaded_bytes, 0);- item.download(None).await?;+ tracing::info!("starting final download");+ item.download().await?;let final_state = wait_for_item_state(&item, &mut progress, "final download completes", |state| {matches!(state.phase, DownloadPhase::Downloaded {})}).await;+ tracing::info!(?final_state, "observed final downloaded state");+assert_eq!(final_state.downloaded_bytes, total_bytes);assert_eq!(final_state.total_bytes, total_bytes);for served_file in registry.files {let destination = item.cache_path.join(&served_file.file.name);assert_eq!(tokio::fs::read(&destination).await?, served_file.bytes.to_vec());- assert!(destination.with_added_extension("integrity").is_file());+ assert!(std::path::PathBuf::from(format!("{}.crc", destination.display())).is_file());}Ok(())@@ -89,6 +107,7 @@ async fn test_storage_mock_registry_model_download_lifecycle(async fn test_storage_cancel_does_not_delete_locked_files(#[case] download_manager_type: FileDownloadManagerType) -> Result<(), Box<dyn std::error::Error>> {+ init_test_tracing();let registry = MockRegistry::start().await?;let model = registry.models.first().ok_or_else(|| std::io::Error::other("mock registry must include a model"))?;let test_storage =@@ -127,18 +146,52 @@ async fn test_storage_cancel_does_not_delete_locked_files(#[case::universal(FileDownloadManagerType::Universal)]#[cfg_attr(target_vendor = "apple", case::apple(FileDownloadManagerType::Apple))]#[tokio::test(flavor = "multi_thread")]-async fn test_downloader_pause_updates_public_state(+async fn test_downloader_progress_stream_closes_after_pause(#[case] download_manager_type: FileDownloadManagerType) -> Result<(), Box<dyn std::error::Error>> {+ init_test_tracing();let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;let model = registry.models.first().ok_or_else(|| std::io::Error::other("mock registry must include a model"))?;let test_storage =TestStorage::with_models_and_manager(RuntimeHandle::current(), vec![model.clone()], download_manager_type).await?;- let downloader = Downloader::new(model.identifier.clone(), SharedAccess::new(test_storage.storage), None);+ let downloader = Downloader::new(model.identifier.clone(), SharedAccess::new(test_storage.storage));downloader.resume().await?;let progress = downloader.progress().await?;+ timeout(Duration::from_secs(30), async {+ loop {+ let state = downloader.state().await.expect("model state must exist");+ if matches!(state.phase, DownloadPhase::Downloading {}) {+ break;+ }+ tokio::time::sleep(Duration::from_millis(20)).await;+ }+ })+ .await?;++ downloader.pause().await?;+ timeout(Duration::from_secs(10), async { while progress.next().await.is_some() {} }).await?;++ Ok(())+}++#[rstest]+#[case::universal(FileDownloadManagerType::Universal)]+#[cfg_attr(target_vendor = "apple", case::apple(FileDownloadManagerType::Apple))]+#[tokio::test(flavor = "multi_thread")]+async fn test_downloader_pause_updates_public_state(+ #[case] download_manager_type: FileDownloadManagerType+) -> Result<(), Box<dyn std::error::Error>> {+ init_test_tracing();+ let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;+ let model = registry.models.first().ok_or_else(|| std::io::Error::other("mock registry must include a model"))?;+ let test_storage =+ TestStorage::with_models_and_manager(RuntimeHandle::current(), vec![model.clone()], download_manager_type)+ .await?;+ let downloader = Downloader::new(model.identifier.clone(), SharedAccess::new(test_storage.storage));++ downloader.resume().await?;wait_for_downloader_state(&downloader, "downloader reaches Downloading", |state| {matches!(state.phase, DownloadPhase::Downloading {})})@@ -152,7 +205,6 @@ async fn test_downloader_pause_updates_public_state(assert!(!paused_state.is_in_progress());assert!(!paused_state.can_pause());- timeout(Duration::from_secs(10), async { while progress.next().await.is_some() {} }).await?;Ok(())}@@ -163,12 +215,13 @@ async fn test_downloader_pause_updates_public_state(async fn test_downloader_progress_after_resuming_paused_model_reaches_downloaded(#[case] download_manager_type: FileDownloadManagerType) -> Result<(), Box<dyn std::error::Error>> {+ init_test_tracing();let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;let model = registry.models.first().ok_or_else(|| std::io::Error::other("mock registry must include a model"))?;let test_storage =TestStorage::with_models_and_manager(RuntimeHandle::current(), vec![model.clone()], download_manager_type).await?;- let downloader = Downloader::new(model.identifier.clone(), SharedAccess::new(test_storage.storage), None);+ let downloader = Downloader::new(model.identifier.clone(), SharedAccess::new(test_storage.storage));downloader.resume().await?;wait_for_downloader_state(&downloader, "downloader reaches Downloading before pause", |state| {@@ -193,6 +246,30 @@ async fn test_downloader_progress_after_resuming_paused_model_reaches_downloaded}}).await?;+ Ok(())+}++#[rstest]+#[case::universal(FileDownloadManagerType::Universal)]+#[cfg_attr(target_vendor = "apple", case::apple(FileDownloadManagerType::Apple))]+#[tokio::test(flavor = "multi_thread")]+async fn test_downloader_progress_for_downloaded_model_is_empty(+ #[case] download_manager_type: FileDownloadManagerType+) -> Result<(), Box<dyn std::error::Error>> {+ init_test_tracing();+ let registry = MockRegistry::start().await?;+ let model = registry.models.first().ok_or_else(|| std::io::Error::other("mock registry must include a model"))?;+ let test_storage =+ TestStorage::with_models_and_manager(RuntimeHandle::current(), vec![model.clone()], download_manager_type)+ .await?;+ let downloader = Downloader::new(model.identifier.clone(), SharedAccess::new(test_storage.storage));++ downloader.resume().await?;+ wait_for_downloader_state(&downloader, "downloader reaches Downloaded", |state| {+ matches!(state.phase, DownloadPhase::Downloaded {})+ })+ .await;+let downloaded_progress = downloader.progress().await?;assert!(downloaded_progress.next().await.is_none());Ok(())@@ -205,13 +282,16 @@ async fn wait_for_item_state(mut is_expected_state: impl FnMut(&DownloadState) -> bool,) -> DownloadState {timeout(Duration::from_secs(30), async {+ tracing::info!(label, "waiting for storage download state");let current_state = item.state().await;+ tracing::debug!(label, ?current_state, "observed initial storage download state");if is_expected_state(¤t_state) {return current_state;}while let Some(result) = progress.next().await {let state = result.expect("storage progress stream must not lag");+ tracing::debug!(label, ?state, "observed storage progress state");if is_expected_state(&state) {return state;}diff --git a/crates/legacy/uzu/tests/unit/storage/mod.rs b/crates/legacy/uzu/tests/unit/storage/mod.rsindex 1bd467379..3ae7e42c9 100644--- a/crates/legacy/uzu/tests/unit/storage/mod.rs+++ b/crates/legacy/uzu/tests/unit/storage/mod.rs@@ -1,3 +1 @@-#[path = "../common/mod.rs"]-mod common;mod lifecycle_mock_registry_model_test;diff --git a/crates/legacy/uzu/tests/unit/test_entry.rs b/crates/legacy/uzu/tests/unit/test_entry.rsnew file mode 100644index 000000000..c0d36c57c--- /dev/null+++ b/crates/legacy/uzu/tests/unit/test_entry.rs@@ -0,0 +1,4 @@+#[path = "../common/mod.rs"]+mod common;++include!("mod.rs");