Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Revert "hf downloading support" (#799)

uuuvn2d355aa90959
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 027fa96db..636254475 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -81,7 +81,7 @@ jobs:
tests: 'kernels'
- host: 'linux'
target: 'wasm32-unknown-unknown'
- crates: ['kiban', 'download-manager']
+ crates: ['kiban']
env:
CARGO_ENCODED_RUSTFLAGS: "-Dwarnings"
diff --git a/Cargo.lock b/Cargo.lock
index 4a053599a..f409446a1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1750,13 +1750,10 @@ version = "0.5.23"
dependencies = [
"async-trait",
"base64 0.22.1",
- "bitflags 2.13.0",
"block2",
"chrono",
"crc32c",
"futures-util",
- "hex",
- "http 1.4.2",
"kiban",
"libc",
"mock-registry",
@@ -1767,8 +1764,6 @@ dependencies = [
"rstest",
"serde",
"serde_json",
- "sha1",
- "sha2",
"tempfile",
"thiserror 2.0.18",
"tokio",
@@ -7537,6 +7532,7 @@ dependencies = [
"backend-remote",
"bindings",
"bindings-types",
+ "bitflags 2.13.0",
"chrono",
"confy",
"dirs",
diff --git a/Cargo.toml b/Cargo.toml
index 1b9479ca7..df5e7a80f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -152,9 +152,6 @@ crc32c = "0.6.8"
base64 = "0.22.1"
bitflags = "2.11.1"
chrono = { version = "0.4.42", features = ["serde"] }
-hex = "0.4.3"
-sha1 = "0.10.6"
-sha2 = "0.10.9"
jsonschema = "0.46.0"
# Please make sure to use compatible versions of the cargo-swift / uniffi pair.
# See: https://github.com/antoniusnaumann/cargo-swift?tab=readme-ov-file#installing-for-a-different-uniffi-version
diff --git a/apps/playground/Sources/Views/ModelListView.swift b/apps/playground/Sources/Views/ModelListView.swift
index e83e55d72..437b626db 100644
--- a/apps/playground/Sources/Views/ModelListView.swift
+++ b/apps/playground/Sources/Views/ModelListView.swift
@@ -130,7 +130,7 @@ struct ModelListView: View {
guard
let selectedIdentifier,
let model = group.modelByIdentifier[selectedIdentifier],
- model.isOnDevice(),
+ model.isLocal(),
let state = engineWrapper.downloadStates[model.identifier]
else { return }
@@ -347,7 +347,7 @@ struct ModelListView: View {
private var selectedModelState: DownloadState? {
guard let selectedIdentifier else { return nil }
guard let model = models()
- .filter({ $0.isOnDevice() })
+ .filter({ $0.isLocal() })
.first(where: { $0.identifier == selectedIdentifier }) else {
return nil
}
@@ -358,7 +358,7 @@ struct ModelListView: View {
private func select(identifier: String) {
selectedIdentifier = identifier
if let model = models()
- .filter({ $0.isOnDevice() })
+ .filter({ $0.isLocal() })
.first(where: { $0.identifier == selectedIdentifier }),
let state = engineWrapper.downloadStates[model.identifier] {
selectedModelSection = section(for: state)
@@ -488,7 +488,7 @@ struct ModelListView: View {
}
}
let cloud = all.filter { $0.isRemote() }.sorted { $0.name() < $1.name() }
- let local = all.filter { $0.isOnDevice() }
+ let local = all.filter { $0.isLocal() }
let modelByIdentifier = Dictionary(uniqueKeysWithValues: all.map { ($0.identifier, $0) })
var bySection: [ModelSection: [String]] = [:]
diff --git a/crates/legacy/cli/src/bench/runner.rs b/crates/legacy/cli/src/bench/runner.rs
index e1d4aef9e..f46695698 100644
--- a/crates/legacy/cli/src/bench/runner.rs
+++ b/crates/legacy/cli/src/bench/runner.rs
@@ -4,7 +4,7 @@ use std::{
};
use anyhow::{Context, Result};
-use shoji::types::model::{ModelFamily, ModelSource};
+use shoji::types::model::{ModelFamily, ModelReference};
use sysinfo::System;
use uzu::{
engine::{Engine, EngineConfig},
@@ -166,9 +166,9 @@ impl BenchRunner {
.to_lowercase();
let all_models = engine.models().await?;
for model in all_models {
- if let ModelAccessibility::OnDevice {
- source:
- ModelSource::Registry {
+ if let ModelAccessibility::Local {
+ reference:
+ ModelReference::Mirai {
toolchain_version: _toolchain_version,
repository,
source_repository,
diff --git a/crates/legacy/cli/src/common/thinking.rs b/crates/legacy/cli/src/common/thinking.rs
index 864f7481a..b4520c10a 100644
--- a/crates/legacy/cli/src/common/thinking.rs
+++ b/crates/legacy/cli/src/common/thinking.rs
@@ -204,7 +204,7 @@ impl ThinkingSupport {
mod tests {
use shoji::types::{
basic::Value,
- model::{ModelAccessibility, ModelSource, ModelSpecialization},
+ model::{ModelAccessibility, ModelReference, ModelSpecialization},
};
use super::*;
@@ -236,8 +236,8 @@ mod tests {
"uzu".to_string(),
"0.0".to_string(),
vec![ModelSpecialization::Chat {}],
- ModelAccessibility::OnDevice {
- source: ModelSource::Filesystem {
+ ModelAccessibility::Local {
+ reference: ModelReference::Local {
path: "/nonexistent".to_string(),
},
},
diff --git a/crates/legacy/cli/src/interactive/components/application.rs b/crates/legacy/cli/src/interactive/components/application.rs
index 1b7656e23..3a93f0bb2 100644
--- a/crates/legacy/cli/src/interactive/components/application.rs
+++ b/crates/legacy/cli/src/interactive/components/application.rs
@@ -162,7 +162,7 @@ pub fn Application(
match engine.model(identifier.clone()).await {
Ok(Some(model)) => {
let model_exists = model.is_downloadable()
- || !model.is_on_device()
+ || !model.is_local()
|| matches!(
engine.model_path(&model).await,
Some(path) if std::path::Path::new(&path).exists()
diff --git a/crates/legacy/cli/src/interactive/model.rs b/crates/legacy/cli/src/interactive/model.rs
index 5a3ef2dd2..cd124bb39 100644
--- a/crates/legacy/cli/src/interactive/model.rs
+++ b/crates/legacy/cli/src/interactive/model.rs
@@ -1,4 +1,4 @@
-use shoji::types::model::{Model, ModelAccessibility, ModelSource};
+use shoji::types::model::{Model, ModelAccessibility, ModelReference};
use sysinfo::System;
use uzu::engine::{Engine, EngineError};
@@ -116,8 +116,8 @@ fn model_shorthand_matches(
}
fn checkpoint_size_bytes(model: &Model) -> Option<u64> {
- if let ModelAccessibility::OnDevice {
- source: ModelSource::Registry {
+ if let ModelAccessibility::Local {
+ reference: ModelReference::Mirai {
files,
..
},
diff --git a/crates/legacy/cli/unit/interactive/model_test.rs b/crates/legacy/cli/unit/interactive/model_test.rs
index 844dddb65..960a9824d 100644
--- a/crates/legacy/cli/unit/interactive/model_test.rs
+++ b/crates/legacy/cli/unit/interactive/model_test.rs
@@ -124,8 +124,8 @@ fn checkpoint(
metadata: metadata(quantization_method),
}),
specializations: vec![],
- accessibility: ModelAccessibility::OnDevice {
- source: ModelSource::Registry {
+ accessibility: ModelAccessibility::Local {
+ reference: ModelReference::Mirai {
toolchain_version: "test".to_string(),
repository: None,
source_repository: None,
diff --git a/crates/legacy/download-manager/Cargo.toml b/crates/legacy/download-manager/Cargo.toml
index be3e58d76..12d78a82b 100644
--- a/crates/legacy/download-manager/Cargo.toml
+++ b/crates/legacy/download-manager/Cargo.toml
@@ -11,20 +11,15 @@ license.workspace = true
[dependencies]
kiban.workspace = true
futures-util.workspace = true
-http.workspace = true
serde.workspace = true
serde_json.workspace = true
async-trait.workspace = true
thiserror.workspace = true
crc32c.workspace = true
base64.workspace = true
-bitflags.workspace = true
tracing.workspace = true
chrono.workspace = true
reqwest.workspace = true
-hex.workspace = true
-sha1.workspace = true
-sha2.workspace = true
tokio = { workspace = true, features = ["macros", "sync"] }
[target.'cfg(target_vendor = "apple")'.dependencies]
diff --git a/crates/legacy/download-manager/docs/STATE_REDUCTION_DECISION_MATRIX.md b/crates/legacy/download-manager/docs/STATE_REDUCTION_DECISION_MATRIX.md
index b129a406c..ec256fab7 100644
--- a/crates/legacy/download-manager/docs/STATE_REDUCTION_DECISION_MATRIX.md
+++ b/crates/legacy/download-manager/docs/STATE_REDUCTION_DECISION_MATRIX.md
@@ -33,7 +33,7 @@ Determines if a file on disk is valid, invalid, or missing based on CRC validati
**CRC Caching Optimization:**
-- Valid files get an `.integrity` receipt to avoid recalculation on next launch
+- Valid files get a `.crc` cache file to avoid recalculation on next launch
- Makes subsequent launches ~100x faster for large files
- Cache is invalidated when file changes
@@ -219,7 +219,7 @@ if task.state() == Completed && checked_file_state == Valid {
```rust
if file_size < expected_bytes {
// Partial download - preserve file
- remove_integrity_receipt();
+ remove_crc_cache();
return NotDownloaded;
} else {
// Corrupted full-size file - delete
@@ -337,7 +337,7 @@ Initial: file=Exists(100MB), crc=Missing, resume=Missing, task=None
↓
Checked: Invalid (file incomplete, fails CRC)
Reconcile: Detect: 100MB < 1.1GB (partial)
-Action: PRESERVE file, remove integrity receipt
+Action: PRESERVE file, remove CRC cache
Internal: NotDownloaded
Display: NotDownloaded (0 / 1.1GB)
↓ Future download can resume from partial file if URLSession supports it
diff --git a/crates/legacy/download-manager/examples/cross_platform_download.rs b/crates/legacy/download-manager/examples/cross_platform_download.rs
index e3be5e496..0f8b81dcb 100644
--- a/crates/legacy/download-manager/examples/cross_platform_download.rs
+++ b/crates/legacy/download-manager/examples/cross_platform_download.rs
@@ -20,7 +20,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
}
println!("Creating download task...");
- let task = manager.file_download_task((&url).into(), &destination, FileCheck::None, None).await?;
+ let task = manager.file_download_task(&url, &destination, FileCheck::None, None).await?;
println!("Starting download...");
let mut progress_stream = task.progress().await?;
diff --git a/crates/legacy/download-manager/examples/wasm/src/lib.rs b/crates/legacy/download-manager/examples/wasm/src/lib.rs
index 479d0c9ab..474369046 100644
--- a/crates/legacy/download-manager/examples/wasm/src/lib.rs
+++ b/crates/legacy/download-manager/examples/wasm/src/lib.rs
@@ -74,7 +74,7 @@ async fn download_internal(
fs::asyn::remove_file(&file_path).await?;
}
- let task = manager.file_download_task((&url).into(), &file_path, FileCheck::None, None).await?;
+ let task = manager.file_download_task(&url, &file_path, FileCheck::None, None).await?;
let mut progress_stream = task.progress().await?;
task.download().await?;
diff --git a/crates/legacy/download-manager/src/backends/apple/active_task.rs b/crates/legacy/download-manager/src/backends/apple/active_task.rs
index 9cbccc565..764147991 100644
--- a/crates/legacy/download-manager/src/backends/apple/active_task.rs
+++ b/crates/legacy/download-manager/src/backends/apple/active_task.rs
@@ -5,7 +5,7 @@ use std::{
use objc2::rc::Retained;
use objc2_foundation::NSURLSessionDownloadTask;
-use tokio::{fs::write as tokio_write, sync::oneshot::channel as tokio_oneshot_channel};
+use tokio::sync::oneshot::channel as tokio_oneshot_channel;
use crate::{
DownloadId,
@@ -21,7 +21,6 @@ pub struct AppleActiveTask {
task: Retained<NSURLSessionDownloadTask>,
event_registry: AppleEventRegistry,
sink_key: AppleSinkKey,
- authenticated: bool,
}
impl AppleActiveTask {
@@ -29,14 +28,12 @@ impl AppleActiveTask {
task: Retained<NSURLSessionDownloadTask>,
event_registry: AppleEventRegistry,
download_id: DownloadId,
- authenticated: bool,
) -> Self {
let sink_key = (download_id, task.task_identifier());
Self {
task,
event_registry,
sink_key,
- authenticated,
}
}
@@ -62,12 +59,7 @@ impl ActiveTask for AppleActiveTask {
self,
destination: &Path,
) -> Result<PathBuf, <Self::Backend as DownloadBackend>::Error> {
- let resume_artifact_path = destination.with_added_extension("resume_data");
- if self.authenticated {
- drop(self);
- tokio_write(&resume_artifact_path, []).await.map_err(|error| AppleBackendError::Io(error.to_string()))?;
- return Ok(resume_artifact_path);
- }
+ let resume_artifact_path = destination.with_extension("resume_data");
let (resume_data_sender, resume_data_receiver) = tokio_oneshot_channel::<Box<[u8]>>();
let pending_resume_data_sender = Arc::new(Mutex::new(Some(resume_data_sender)));
self.unregister_event_sink();
@@ -91,7 +83,7 @@ impl ActiveTask for AppleActiveTask {
}
let resume_data_bytes =
resume_data_receiver.await.map_err(|error| AppleBackendError::ResumeData(error.to_string()))?;
- tokio_write(&resume_artifact_path, resume_data_bytes)
+ tokio::fs::write(&resume_artifact_path, resume_data_bytes)
.await
.map_err(|error| AppleBackendError::Io(error.to_string()))?;
Ok(resume_artifact_path)
@@ -101,7 +93,8 @@ impl ActiveTask for AppleActiveTask {
self,
_destination: &Path,
) -> CancelOutcome {
- drop(self);
+ self.unregister_event_sink();
+ self.task.cancel();
CancelOutcome::BestEffort
}
}
diff --git a/crates/legacy/download-manager/src/backends/apple/backend.rs b/crates/legacy/download-manager/src/backends/apple/backend.rs
index d8b3b3c3f..63d9db66b 100644
--- a/crates/legacy/download-manager/src/backends/apple/backend.rs
+++ b/crates/legacy/download-manager/src/backends/apple/backend.rs
@@ -12,7 +12,7 @@ use crate::{
common::{self, InitialTaskAttachment},
},
lock_manager::DestinationLockLease,
- traits::{ActiveDownloadGeneration, BackendEventSender, Capabilities, DownloadBackend, DownloadConfig},
+ traits::{ActiveDownloadGeneration, BackendEventSender, DownloadBackend, DownloadConfig},
};
#[derive(Clone, Debug, Default)]
@@ -22,7 +22,6 @@ impl DownloadBackend for AppleBackend {
type Context = AppleBackendContext;
type ActiveTask = AppleActiveTask;
type Error = AppleBackendError;
- const CAPABILITIES: Capabilities = Capabilities::CACHES_REDIRECTED_URL_IN_RESUME_DATA;
async fn read_resume_progress(part_path: &Path) -> Option<u64> {
resume_data_parser::read_resume_progress(part_path)
@@ -69,12 +68,7 @@ impl common::Backend for AppleBackend {
task.resume();
}
Ok(InitialTaskAttachment::Downloading {
- active_task: AppleActiveTask::new(
- task,
- context.event_registry(),
- config.download_id,
- config.request.is_authenticated(),
- ),
+ active_task: AppleActiveTask::new(task, context.event_registry(), config.download_id),
initial_downloaded_bytes,
total_bytes,
})
diff --git a/crates/legacy/download-manager/src/backends/apple/context.rs b/crates/legacy/download-manager/src/backends/apple/context.rs
index 383b92b04..4d59921ff 100644
--- a/crates/legacy/download-manager/src/backends/apple/context.rs
+++ b/crates/legacy/download-manager/src/backends/apple/context.rs
@@ -1,23 +1,19 @@
use std::{
collections::HashMap,
- fmt::{Debug, Formatter, Result as FmtResult},
path::Path,
sync::{Arc, Mutex},
};
-use http::header::{ACCEPT_ENCODING, AUTHORIZATION};
use kiban::rt::RuntimeHandle;
-use objc2::rc::Retained;
+use objc2::{rc::Retained, runtime::ProtocolObject};
use objc2_foundation::{
- NSBundle, NSData, NSMutableURLRequest, NSString, NSURL, NSURLSession, NSURLSessionConfiguration,
+ NSBundle, NSData, NSString, NSURL, NSURLSession, NSURLSessionConfiguration, NSURLSessionDelegate,
NSURLSessionDownloadTask, NSURLSessionTaskState,
};
-use tokio::{fs::read as tokio_read, sync::oneshot::channel as tokio_oneshot_channel};
+use tokio::sync::oneshot::channel as tokio_oneshot_channel;
-#[cfg(test)]
-use crate::DownloadId;
use crate::{
- DownloadInfo, HttpDownloadRequest,
+ DownloadInfo, FileCheck,
backends::apple::{
AppleActiveTask, AppleBackend, AppleBackendError, AppleEventRegistry, AppleEventSink, AppleGetTasksHandler,
AppleSessionDelegate, task_ext::AppleDownloadTaskExt,
@@ -28,15 +24,17 @@ use crate::{
pub struct AppleBackendContext {
session: Retained<NSURLSession>,
+ _delegate: Retained<AppleSessionDelegate>,
+ _delegate_protocol_object: Retained<ProtocolObject<dyn NSURLSessionDelegate>>,
event_registry: AppleEventRegistry,
runtime_handle: RuntimeHandle,
}
-impl Debug for AppleBackendContext {
+impl std::fmt::Debug for AppleBackendContext {
fn fmt(
&self,
- formatter: &mut Formatter<'_>,
- ) -> FmtResult {
+ formatter: &mut std::fmt::Formatter<'_>,
+ ) -> std::fmt::Result {
formatter.debug_struct("AppleBackendContext").finish_non_exhaustive()
}
}
@@ -45,17 +43,19 @@ impl AppleBackendContext {
pub fn new(runtime_handle: RuntimeHandle) -> Self {
let event_registry = Arc::new(Mutex::new(HashMap::new()));
let delegate = AppleSessionDelegate::new(Arc::clone(&event_registry));
- let delegate = AppleSessionDelegate::protocol_object(delegate);
+ let delegate_protocol_object = AppleSessionDelegate::protocol_object(delegate.clone());
let session = unsafe {
NSURLSession::sessionWithConfiguration_delegate_delegateQueue(
&automatic_session_configuration(),
- Some(&delegate),
+ Some(&delegate_protocol_object),
None,
)
};
Self {
session,
+ _delegate: delegate,
+ _delegate_protocol_object: delegate_protocol_object,
event_registry,
runtime_handle,
}
@@ -65,7 +65,7 @@ impl AppleBackendContext {
&self,
config: &DownloadConfig,
) -> Result<Option<Retained<NSURLSessionDownloadTask>>, AppleBackendError> {
- select_matching_download_task(self.download_tasks().await?, config)
+ self.find_download_task(config).await
}
pub(crate) async fn has_download_task_to_claim(
@@ -78,11 +78,23 @@ impl AppleBackendContext {
.any(|task| task.download_id() == Some(config.download_id) && is_live_task_state(task.state())))
}
+ #[cfg(test)]
+ #[allow(dead_code)]
+ pub(crate) fn event_sink_count_for_download(
+ &self,
+ download_id: crate::DownloadId,
+ ) -> usize {
+ self.event_registry
+ .lock()
+ .map(|registry| registry.keys().filter(|(id, _)| *id == download_id).count())
+ .unwrap_or(0)
+ }
+
#[cfg(test)]
#[allow(dead_code)]
pub(crate) fn event_sink_task_identifiers_for_download(
&self,
- download_id: DownloadId,
+ download_id: crate::DownloadId,
) -> Vec<u64> {
self.event_registry
.lock()
@@ -95,6 +107,14 @@ impl AppleBackendContext {
.unwrap_or_default()
}
+ async fn find_download_task(
+ &self,
+ config: &DownloadConfig,
+ ) -> Result<Option<Retained<NSURLSessionDownloadTask>>, AppleBackendError> {
+ let download_tasks = self.download_tasks().await?;
+ Ok(select_matching_download_task(download_tasks, config))
+ }
+
async fn download_tasks(&self) -> Result<Box<[Retained<NSURLSessionDownloadTask>]>, AppleBackendError> {
let (download_tasks_sender, download_tasks_receiver) = tokio_oneshot_channel();
let pending_download_tasks_sender = Arc::new(Mutex::new(Some(download_tasks_sender)));
@@ -139,9 +159,7 @@ impl AppleBackendContext {
fn select_matching_download_task(
download_tasks: Box<[Retained<NSURLSessionDownloadTask>]>,
config: &DownloadConfig,
-) -> Result<Option<Retained<NSURLSessionDownloadTask>>, AppleBackendError> {
- let authorization = authorization_header(&config.request)?;
- let authorization_field = NSString::from_str(AUTHORIZATION.as_str());
+) -> Option<Retained<NSURLSessionDownloadTask>> {
let mut live_match = None;
for task in download_tasks {
if task.download_id() != Some(config.download_id) {
@@ -152,14 +170,9 @@ fn select_matching_download_task(
continue;
}
let info = task.download_info();
- let source_url_matches = info.as_ref().map(|info| info.source_url == config.request.url).unwrap_or(false);
- let file_check_matches = info.as_ref().is_some_and(|info| info.resolved_file_check() == config.file_check);
- let authorization_matches = task
- .originalRequest()
- .and_then(|request| request.valueForHTTPHeaderField(&authorization_field))
- .map(|value| value.to_string())
- == authorization;
- if !(source_url_matches && file_check_matches && authorization_matches) {
+ let source_url_matches = info.as_ref().map(|info| info.source_url == config.source_url).unwrap_or(false);
+ let crc_matches = info.as_ref().map(|info| info.crc32c == config.file_check.expected_crc()).unwrap_or(false);
+ if !(source_url_matches && crc_matches) {
task.cancel();
continue;
}
@@ -169,7 +182,7 @@ fn select_matching_download_task(
task.cancel();
}
}
- Ok(live_match)
+ live_match
}
fn is_live_task_state(state: NSURLSessionTaskState) -> bool {
@@ -204,16 +217,11 @@ impl BackendContext for AppleBackendContext {
backend_event_sender: BackendEventSender,
_destination_lease: &DestinationLockLease,
) -> Result<AppleActiveTask, AppleBackendError> {
- let request = apple_request(&config)?;
- let task = self.session.downloadTaskWithRequest(&request);
+ let ns_url = NSURL::URLWithString(&NSString::from_str(&config.source_url)).ok_or(AppleBackendError::BadUrl)?;
+ let task = self.session.downloadTaskWithURL(&ns_url);
self.prepare_task(&task, Arc::clone(&config), generation, backend_event_sender);
task.resume();
- Ok(AppleActiveTask::new(
- task,
- Arc::clone(&self.event_registry),
- config.download_id,
- config.request.is_authenticated(),
- ))
+ Ok(AppleActiveTask::new(task, Arc::clone(&self.event_registry), config.download_id))
}
async fn resume(
@@ -222,23 +230,21 @@ impl BackendContext for AppleBackendContext {
generation: ActiveDownloadGeneration,
resume_artifact_path: &Path,
backend_event_sender: BackendEventSender,
- destination_lease: &DestinationLockLease,
+ _destination_lease: &DestinationLockLease,
) -> Result<AppleActiveTask, AppleBackendError> {
- if config.request.is_authenticated() {
- return self.download(config, generation, backend_event_sender, destination_lease).await;
- }
let resume_data =
- tokio_read(resume_artifact_path).await.map_err(|error| AppleBackendError::Io(error.to_string()))?;
+ tokio::fs::read(resume_artifact_path).await.map_err(|error| AppleBackendError::Io(error.to_string()))?;
let task = if resume_data.is_empty() {
- let request = apple_request(&config)?;
- self.session.downloadTaskWithRequest(&request)
+ let ns_url =
+ NSURL::URLWithString(&NSString::from_str(&config.source_url)).ok_or(AppleBackendError::BadUrl)?;
+ self.session.downloadTaskWithURL(&ns_url)
} else {
let ns_data = NSData::with_bytes(&resume_data);
self.session.downloadTaskWithResumeData(&ns_data)
};
self.prepare_task(&task, Arc::clone(&config), generation, backend_event_sender);
task.resume();
- Ok(AppleActiveTask::new(task, Arc::clone(&self.event_registry), config.download_id, false))
+ Ok(AppleActiveTask::new(task, Arc::clone(&self.event_registry), config.download_id))
}
}
@@ -250,11 +256,16 @@ impl AppleBackendContext {
generation: ActiveDownloadGeneration,
backend_event_sender: BackendEventSender,
) {
- let download_info = DownloadInfo::new(
- config.request.url.clone(),
- config.destination.to_string_lossy().to_string(),
- config.file_check.clone(),
- );
+ let download_info = match &config.file_check {
+ FileCheck::CRC(crc) => DownloadInfo::with_crc(
+ config.source_url.clone(),
+ config.destination.to_string_lossy().to_string(),
+ crc.clone(),
+ ),
+ FileCheck::None => {
+ DownloadInfo::new(config.source_url.clone(), config.destination.to_string_lossy().to_string())
+ },
+ };
task.set_download_info(&download_info);
if let Ok(mut registry) = self.event_registry.lock() {
registry.insert(
@@ -262,7 +273,6 @@ impl AppleBackendContext {
AppleEventSink {
generation,
destination: config.destination.clone(),
- expected_bytes: config.expected_bytes,
backend_event_sender,
runtime_handle: self.runtime_handle.clone(),
},
@@ -270,26 +280,3 @@ impl AppleBackendContext {
}
}
}
-
-fn apple_request(config: &DownloadConfig) -> Result<Retained<NSMutableURLRequest>, AppleBackendError> {
- let url = NSURL::URLWithString(&NSString::from_str(&config.request.url)).ok_or(AppleBackendError::BadUrl)?;
- let request = NSMutableURLRequest::requestWithURL(&url);
- if let Some(authorization) = authorization_header(&config.request)? {
- request.setValue_forHTTPHeaderField(
- Some(&NSString::from_str(&authorization)),
- &NSString::from_str(AUTHORIZATION.as_str()),
- );
- }
- request.setValue_forHTTPHeaderField(
- Some(&NSString::from_str("identity")),
- &NSString::from_str(ACCEPT_ENCODING.as_str()),
- );
- Ok(request)
-}
-
-fn authorization_header(request: &HttpDownloadRequest) -> Result<Option<String>, AppleBackendError> {
- request
- .bearer_token()
- .map(|token| token.map(|token| format!("Bearer {token}")))
- .map_err(|error| AppleBackendError::Authentication(error.to_string()))
-}
diff --git a/crates/legacy/download-manager/src/backends/apple/delegate.rs b/crates/legacy/download-manager/src/backends/apple/delegate.rs
index 223f8bf4b..2f272c8e7 100644
--- a/crates/legacy/download-manager/src/backends/apple/delegate.rs
+++ b/crates/legacy/download-manager/src/backends/apple/delegate.rs
@@ -1,14 +1,9 @@
use std::{
collections::HashMap,
- fs::{copy, create_dir_all, remove_file, rename},
- io::Error as IoError,
path::PathBuf,
- ptr::{from_ref, null_mut},
sync::{Arc, Mutex},
};
-use block2::DynBlock;
-use http::{StatusCode, uri::Scheme};
use kiban::rt::RuntimeHandle;
use objc2::{
ClassType, DefinedClass, define_class, msg_send,
@@ -16,11 +11,9 @@ use objc2::{
runtime::ProtocolObject,
};
use objc2_foundation::{
- NSError, NSHTTPURLResponse, NSMutableCopying, NSObject, NSObjectProtocol, NSString, NSURL, NSURLRequest,
- NSURLSession, NSURLSessionDelegate, NSURLSessionDownloadDelegate, NSURLSessionDownloadTask, NSURLSessionTask,
- NSURLSessionTaskDelegate,
+ NSError, NSObject, NSObjectProtocol, NSURL, NSURLSession, NSURLSessionDelegate, NSURLSessionDownloadDelegate,
+ NSURLSessionDownloadTask, NSURLSessionTask, NSURLSessionTaskDelegate,
};
-use reqwest::Url;
use crate::{
DownloadId,
@@ -33,22 +26,10 @@ use crate::{
pub struct AppleEventSink {
pub generation: ActiveDownloadGeneration,
pub destination: PathBuf,
- pub expected_bytes: Option<u64>,
pub backend_event_sender: BackendEventSender,
pub runtime_handle: RuntimeHandle,
}
-impl AppleEventSink {
- fn send_terminal(
- self,
- event: BackendEvent,
- ) {
- self.runtime_handle.clone().spawn(async move {
- let _ = self.backend_event_sender.send_terminal(event).await;
- });
- }
-}
-
pub type AppleSinkKey = (DownloadId, u64);
pub type AppleEventRegistry = Arc<Mutex<HashMap<AppleSinkKey, AppleEventSink>>>;
@@ -65,48 +46,17 @@ define_class!(
unsafe impl NSObjectProtocol for AppleSessionDelegate {}
- unsafe impl NSURLSessionDelegate for AppleSessionDelegate {}
-
- unsafe impl NSURLSessionTaskDelegate for AppleSessionDelegate {
- #[unsafe(method(URLSession:task:willPerformHTTPRedirection:newRequest:completionHandler:))]
- unsafe fn will_perform_http_redirection(
+ unsafe impl NSURLSessionDelegate for AppleSessionDelegate {
+ #[unsafe(method(URLSession:didBecomeInvalidWithError:))]
+ fn did_become_invalid_with_error(
&self,
_session: &NSURLSession,
- _task: &NSURLSessionTask,
- response: &NSHTTPURLResponse,
- request: &NSURLRequest,
- completion_handler: &DynBlock<dyn Fn(*mut NSURLRequest)>,
+ _error: Option<&NSError>,
) {
- let current_url =
- response.URL().and_then(|url| url.absoluteString()).and_then(|url| Url::parse(&url.to_string()).ok());
- let next_url =
- request.URL().and_then(|url| url.absoluteString()).and_then(|url| Url::parse(&url.to_string()).ok());
- let redirected_request = match (current_url, next_url) {
- (Some(current), Some(next)) => {
- match (Scheme::try_from(current.scheme()), Scheme::try_from(next.scheme())) {
- (Ok(current_scheme), Ok(next_scheme))
- if (current_scheme == Scheme::HTTP
- && (next_scheme == Scheme::HTTP || next_scheme == Scheme::HTTPS))
- || (current_scheme == Scheme::HTTPS && next_scheme == Scheme::HTTPS) =>
- {
- if current.origin() == next.origin() {
- from_ref(request).cast_mut()
- } else {
- let redirected = request.mutableCopy();
- redirected.setValue_forHTTPHeaderField(None, &NSString::from_str("Authorization"));
- let pointer: *const _ = &*redirected;
- completion_handler.call((pointer.cast::<NSURLRequest>().cast_mut(),));
- return;
- }
- },
- _ => null_mut(),
- }
- },
- _ => null_mut(),
- };
- completion_handler.call((redirected_request,));
}
+ }
+ unsafe impl NSURLSessionTaskDelegate for AppleSessionDelegate {
#[unsafe(method(URLSession:task:didCompleteWithError:))]
fn did_complete_with_error(
&self,
@@ -130,8 +80,9 @@ define_class!(
return;
};
let message = error.localizedDescription().to_string();
- let event = BackendEvent::error(sink.generation, message);
- sink.send_terminal(event);
+ sink.runtime_handle.clone().spawn(async move {
+ let _ = sink.backend_event_sender.send_terminal(BackendEvent::error(sink.generation, message)).await;
+ });
}
}
@@ -152,42 +103,27 @@ define_class!(
else {
return;
};
- let status_code = download_task
- .response()
- .and_then(|response| response.downcast_ref::<NSHTTPURLResponse>().map(|response| response.statusCode()))
- .and_then(|status_code| u16::try_from(status_code).ok())
- .and_then(|status_code| StatusCode::from_u16(status_code).ok());
- if !status_code.is_some_and(|status_code| status_code.is_success()) {
- let message = status_code.map_or_else(
- || "download response was not HTTP".to_string(),
- |status_code| format!("download failed with HTTP status {}", status_code.as_u16()),
- );
- let event = BackendEvent::error(sink.generation, message);
- sink.send_terminal(event);
- return;
- }
let Some(temporary_path) = location.path().map(|path| PathBuf::from(path.to_string())) else {
- let event =
- BackendEvent::error(sink.generation, "download temporary location was unavailable".to_string());
- sink.send_terminal(event);
return;
};
// Move synchronously: NSURLSession deletes the temp file as soon as this method returns.
if let Some(parent) = sink.destination.parent() {
- let _ = create_dir_all(parent);
+ let _ = std::fs::create_dir_all(parent);
}
- let move_result = rename(&temporary_path, &sink.destination).or_else(|_| {
- copy(&temporary_path, &sink.destination)?;
- let _ = remove_file(&temporary_path);
- Ok::<(), IoError>(())
+ let move_result = std::fs::rename(&temporary_path, &sink.destination).or_else(|_| {
+ std::fs::copy(&temporary_path, &sink.destination)?;
+ let _ = std::fs::remove_file(&temporary_path);
+ Ok::<(), std::io::Error>(())
});
let terminal_event = match move_result {
Ok(()) => BackendEvent::completed(sink.generation),
Err(error) => BackendEvent::error(sink.generation, format!("move into destination failed: {error}")),
};
- sink.send_terminal(terminal_event);
+ sink.runtime_handle.clone().spawn(async move {
+ let _ = sink.backend_event_sender.send_terminal(terminal_event).await;
+ });
}
#[unsafe(method(URLSession:downloadTask:didWriteData:totalBytesWritten:totalBytesExpectedToWrite:))]
@@ -208,22 +144,8 @@ define_class!(
else {
return;
};
- let downloaded_bytes = u64::try_from(cumulative_bytes_written).unwrap_or_default();
- let total_bytes = u64::try_from(total_expected_bytes_to_write).ok().filter(|total_bytes| *total_bytes > 0);
- if let Some(expected_bytes) = sink.expected_bytes
- && downloaded_bytes > expected_bytes
- {
- download_task.cancel();
- if let Ok(mut registry) = Self::ivars(self).event_registry.lock() {
- registry.remove(&key);
- }
- let event = BackendEvent::error(
- sink.generation,
- format!("response exceeded the registry size of {expected_bytes} bytes"),
- );
- sink.send_terminal(event);
- return;
- }
+ let downloaded_bytes = cumulative_bytes_written.max(0) as u64;
+ let total_bytes = (total_expected_bytes_to_write > 0).then_some(total_expected_bytes_to_write as u64);
sink.runtime_handle.clone().spawn(async move {
sink.backend_event_sender.send_progress(sink.generation, downloaded_bytes, total_bytes).await;
diff --git a/crates/legacy/download-manager/src/backends/apple/error.rs b/crates/legacy/download-manager/src/backends/apple/error.rs
index fcd52d16b..3efbc188f 100644
--- a/crates/legacy/download-manager/src/backends/apple/error.rs
+++ b/crates/legacy/download-manager/src/backends/apple/error.rs
@@ -8,6 +8,4 @@ pub enum AppleBackendError {
Io(String),
#[error("task enumeration error: {0}")]
TaskEnumeration(String),
- #[error("authentication error: {0}")]
- Authentication(String),
}
diff --git a/crates/legacy/download-manager/src/backends/apple/task_ext.rs b/crates/legacy/download-manager/src/backends/apple/task_ext.rs
index f5c71c1d7..2b50fca55 100644
--- a/crates/legacy/download-manager/src/backends/apple/task_ext.rs
+++ b/crates/legacy/download-manager/src/backends/apple/task_ext.rs
@@ -45,12 +45,12 @@ impl AppleDownloadTaskExt for NSURLSessionDownloadTask {
fn count_of_bytes_expected_to_receive(&self) -> u64 {
let bytes_expected: i64 = unsafe { msg_send![self, countOfBytesExpectedToReceive] };
- u64::try_from(bytes_expected).unwrap_or_default()
+ bytes_expected.max(0) as u64
}
fn count_of_bytes_received(&self) -> u64 {
let bytes_received: i64 = unsafe { msg_send![self, countOfBytesReceived] };
- u64::try_from(bytes_received).unwrap_or_default()
+ bytes_received.max(0) as u64
}
fn task_identifier(&self) -> u64 {
diff --git a/crates/legacy/download-manager/src/backends/common/action_executor.rs b/crates/legacy/download-manager/src/backends/common/action_executor.rs
index f24b144d9..19f836b01 100644
--- a/crates/legacy/download-manager/src/backends/common/action_executor.rs
+++ b/crates/legacy/download-manager/src/backends/common/action_executor.rs
@@ -1,10 +1,10 @@
-use std::{io::ErrorKind, path::Path};
+use std::path::Path;
use kiban::fs;
use crate::{
DownloadError,
- integrity::save_integrity_cache,
+ crc_utils::save_crc_file,
lock_manager::DestinationLockLease,
reducer::{Action, ActionPlan},
};
@@ -18,7 +18,7 @@ pub async fn apply_actions(
Action::DeleteFile {
path,
}
- | Action::DeleteIntegrityCache {
+ | Action::DeleteCrcCache {
path,
}
| Action::DeleteResumeArtifact {
@@ -26,11 +26,11 @@ pub async fn apply_actions(
} => {
remove_file_if_present(path).await?;
},
- Action::SaveIntegrityCache {
+ Action::SaveCrcCache {
destination,
- file_check,
+ crc,
} => {
- save_integrity_cache(destination, file_check).await?;
+ save_crc_file(destination, crc).await?;
},
}
}
@@ -41,7 +41,7 @@ pub async fn apply_actions(
async fn remove_file_if_present(path: &Path) -> Result<(), DownloadError> {
match fs::asyn::remove_file(path).await {
Ok(()) => Ok(()),
- Err(error) if error.kind() == ErrorKind::NotFound => Ok(()),
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(DownloadError::from(error)),
}
}
diff --git a/crates/legacy/download-manager/src/backends/common/manager.rs b/crates/legacy/download-manager/src/backends/common/manager.rs
index bcc918ad5..423a9d668 100644
--- a/crates/legacy/download-manager/src/backends/common/manager.rs
+++ b/crates/legacy/download-manager/src/backends/common/manager.rs
@@ -1,12 +1,10 @@
-use std::{io::ErrorKind, path::Path, sync::Arc};
+use std::sync::Arc;
use kiban::rt::RuntimeHandle;
use tokio_stream::wrappers::BroadcastStream as TokioBroadcastStream;
-use uuid::Uuid;
use crate::{
- DownloadError, DownloadEvent, DownloadId, FileCheck, FileDownloadManager, FileDownloadTask, HttpDownloadRequest,
- LockFileState, SharedDownloadEventSender,
+ DownloadError, DownloadEvent, FileCheck, FileDownloadManager, FileDownloadTask, LockFileState,
backends::common::{Backend, DownloadManagerState, Startup},
check_lock_file, compute_download_id,
download_log_event::{DownloadLogEvent, log},
@@ -47,7 +45,7 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {
self.state.subscribe_to_all_downloads()
}
- fn global_broadcast_sender(&self) -> SharedDownloadEventSender {
+ fn global_broadcast_sender(&self) -> crate::SharedDownloadEventSender {
self.state.global_broadcast_sender()
}
@@ -57,7 +55,7 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {
async fn remove_file_task(
&self,
- download_id: DownloadId,
+ download_id: crate::DownloadId,
) -> Result<(), DownloadError> {
let construction_lock = self.state.construction_lock(download_id).await;
let _construction_guard = construction_lock.lock().await;
@@ -74,8 +72,8 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {
async fn file_download_task(
&self,
- request: HttpDownloadRequest,
- destination_path: &Path,
+ source_url: &str,
+ destination_path: &std::path::Path,
file_check: FileCheck,
expected_bytes: Option<u64>,
) -> Result<Arc<dyn FileDownloadTask>, DownloadError> {
@@ -84,7 +82,7 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {
&& !cached_task.is_stopped()
{
let task = cached_task.public();
- ensure_cached_task_matches(&cached_task, &task, &request, &file_check, expected_bytes)?;
+ ensure_cached_task_matches(&task, source_url, &file_check, expected_bytes)?;
return Ok(task);
}
@@ -94,7 +92,7 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {
let cached_task_result = match self.state.get_task(download_id).await {
Some(cached_task) if !cached_task.is_stopped() => {
let task = cached_task.public();
- ensure_cached_task_matches(&cached_task, &task, &request, &file_check, expected_bytes)?;
+ ensure_cached_task_matches(&task, source_url, &file_check, expected_bytes)?;
Some(Ok(task))
},
Some(_) => {
@@ -108,7 +106,7 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {
} else {
let startup = observe_startup::<B>(
download_id,
- request.clone(),
+ source_url,
destination_path,
file_check.clone(),
expected_bytes,
@@ -120,7 +118,7 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {
startup,
self.context.as_ref(),
download_id,
- request.clone(),
+ source_url,
destination_path,
file_check,
expected_bytes,
@@ -152,10 +150,7 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {
let public_task: Arc<dyn FileDownloadTask> = task.clone();
let managed_task = task;
self.state
- .insert_task(
- download_id,
- CachedFileDownloadTask::new(Arc::clone(&public_task), managed_task, request),
- )
+ .insert_task(download_id, CachedFileDownloadTask::new(Arc::clone(&public_task), managed_task))
.await;
Ok(public_task)
}
@@ -169,7 +164,7 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {
async fn destination_foreign_lock(
&self,
- destination_path: &Path,
+ destination_path: &std::path::Path,
) -> Option<String> {
let lock_path = lock_path_for_destination(destination_path);
match check_lock_file(&lock_path, &self.state.manager_id, self.state.instance_id, kiban::process::id()).await {
@@ -180,17 +175,17 @@ impl<B: Backend> FileDownloadManager for DownloadManager<B> {
}
pub(crate) async fn observe_startup<B: Backend>(
- download_id: DownloadId,
- request: HttpDownloadRequest,
- destination_path: &Path,
+ download_id: crate::DownloadId,
+ source_url: &str,
+ destination_path: &std::path::Path,
file_check: FileCheck,
expected_bytes: Option<u64>,
manager_id: &str,
- manager_instance_id: Uuid,
+ manager_instance_id: uuid::Uuid,
) -> Result<Startup, DownloadError> {
Startup::observe::<B>(
download_id,
- request,
+ source_url,
destination_path,
file_check,
expected_bytes,
@@ -203,13 +198,13 @@ pub(crate) async fn observe_startup<B: Backend>(
pub(crate) async fn prepare_startup<B: Backend>(
startup: Startup,
context: &B::Context,
- download_id: DownloadId,
- request: HttpDownloadRequest,
- destination_path: &Path,
+ download_id: crate::DownloadId,
+ source_url: &str,
+ destination_path: &std::path::Path,
file_check: FileCheck,
expected_bytes: Option<u64>,
manager_id: &str,
- manager_instance_id: Uuid,
+ manager_instance_id: uuid::Uuid,
) -> Result<(Startup, Option<DestinationLockLease>), DownloadError> {
if !startup_requires_destination_lease::<B>(&startup, context).await? {
return Ok((startup, None));
@@ -218,10 +213,10 @@ pub(crate) async fn prepare_startup<B: Backend>(
let lease =
match DestinationLockLease::acquire_for_destination(destination_path, manager_id, manager_instance_id).await {
Ok(lease) => lease,
- Err(error) if error.kind() == ErrorKind::AlreadyExists => {
+ Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
let startup = observe_startup::<B>(
download_id,
- request,
+ source_url,
destination_path,
file_check,
expected_bytes,
@@ -238,7 +233,7 @@ pub(crate) async fn prepare_startup<B: Backend>(
};
let startup = match observe_startup::<B>(
download_id,
- request,
+ source_url,
destination_path,
file_check,
expected_bytes,
@@ -289,29 +284,30 @@ fn startup_can_attach_initial_task<B: Backend>(startup: &Startup) -> bool {
}
fn ensure_cached_task_matches(
- cached: &CachedFileDownloadTask,
- task: &Arc<dyn FileDownloadTask>,
- request: &HttpDownloadRequest,
+ cached: &Arc<dyn FileDownloadTask>,
+ source_url: &str,
file_check: &FileCheck,
expected_bytes: Option<u64>,
) -> Result<(), DownloadError> {
- if !cached.request_matches(request) {
+ if cached.source_url() != source_url {
return Err(DownloadError::ConflictingConfig(format!(
- "{} already requested with a different HTTP request",
- task.destination().display(),
+ "{} already requested with source_url {:?}; cannot reuse for {:?}",
+ cached.destination().display(),
+ cached.source_url(),
+ source_url,
)));
}
- if task.file_check() != file_check {
+ if cached.file_check() != file_check {
return Err(DownloadError::ConflictingConfig(format!(
"{} already requested with a different file_check",
- task.destination().display(),
+ cached.destination().display(),
)));
}
- if task.expected_bytes() != expected_bytes {
+ if cached.expected_bytes() != expected_bytes {
return Err(DownloadError::ConflictingConfig(format!(
"{} already requested with expected_bytes {:?}; got {:?}",
- task.destination().display(),
- task.expected_bytes(),
+ cached.destination().display(),
+ cached.expected_bytes(),
expected_bytes,
)));
}
diff --git a/crates/legacy/download-manager/src/backends/common/startup.rs b/crates/legacy/download-manager/src/backends/common/startup.rs
index 9aca42d10..f8629828e 100644
--- a/crates/legacy/download-manager/src/backends/common/startup.rs
+++ b/crates/legacy/download-manager/src/backends/common/startup.rs
@@ -1,14 +1,13 @@
use std::{path::Path, sync::Arc};
use kiban::fs;
-use uuid::Uuid;
use crate::{
- DownloadError, DownloadId, FileCheck, FileState, HttpDownloadRequest, LockFileState,
+ DownloadError, DownloadId, FileCheck, FileState, LockFileState,
backends::common::{Backend, action_executor::apply_actions},
check_lock_file,
+ crc_utils::crc_path_for_file,
file_download_task_actor::{ProgressCounters, PublicProjection},
- integrity::integrity_receipt_path,
lock_manager::{DestinationLockLease, lock_path_for_destination},
reducer::{ActionPlan, DiskObservation, InitialLifecycleState, LockObservation, decide, validate},
traits::DownloadConfig,
@@ -27,16 +26,19 @@ pub struct Startup {
impl Startup {
pub async fn observe<B: Backend>(
download_id: DownloadId,
- request: HttpDownloadRequest,
+ source_url: &str,
destination_path: &Path,
file_check: FileCheck,
expected_bytes: Option<u64>,
manager_id: &str,
- manager_instance_id: Uuid,
+ manager_instance_id: uuid::Uuid,
) -> Result<Self, DownloadError> {
- request.validate()?;
- let resume_artifact_path = destination_path.with_added_extension(B::RESUME_ARTIFACT_EXTENSION);
- let integrity_path = integrity_receipt_path(destination_path);
+ let resume_artifact_path = destination_path.with_extension(B::RESUME_ARTIFACT_EXTENSION);
+ let expected_crc = match &file_check {
+ FileCheck::CRC(crc) => Some(crc.clone()),
+ FileCheck::None => None,
+ };
+ let crc_path = crc_path_for_file(destination_path);
let resume_state = file_state(&resume_artifact_path).await;
let resume_size = match resume_state {
FileState::Exists => B::read_resume_progress(&resume_artifact_path).await,
@@ -44,14 +46,14 @@ impl Startup {
};
let observation = DiskObservation {
destination_state: file_state(destination_path).await,
- integrity_state: file_state(&integrity_path).await,
+ crc_state: file_state(&crc_path).await,
resume_state,
destination_size: fs::asyn::file_length(destination_path).await.ok(),
resume_size,
- file_check: file_check.clone(),
+ expected_crc,
expected_bytes,
destination_path: destination_path.to_path_buf(),
- integrity_path,
+ crc_path: Some(crc_path),
resume_artifact_path: Some(resume_artifact_path),
};
let lock_state = check_lock_file(
@@ -64,11 +66,11 @@ impl Startup {
let lock_observation = LockObservation {
state: lock_state.clone(),
};
- let validation = validate(&observation).await?;
+ let validation = validate(&observation).await;
let decision = decide(&observation, &lock_observation, &validation);
let config = Arc::new(DownloadConfig {
download_id,
- request,
+ source_url: source_url.to_string(),
destination: destination_path.to_path_buf(),
file_check,
expected_bytes,
diff --git a/crates/legacy/download-manager/src/backends/universal/context.rs b/crates/legacy/download-manager/src/backends/universal/context.rs
index ef996a79c..361b62e38 100644
--- a/crates/legacy/download-manager/src/backends/universal/context.rs
+++ b/crates/legacy/download-manager/src/backends/universal/context.rs
@@ -5,18 +5,13 @@ use std::{
};
use futures_util::StreamExt;
-use http::StatusCode;
-#[cfg(not(target_family = "wasm"))]
-use http::uri::Scheme;
use kiban::{fs, fs::PartFile, rt::RuntimeHandle, time::Instant};
use reqwest::{
- Client,
- header::{CONTENT_RANGE, RANGE},
+ StatusCode,
+ header::{CONTENT_LENGTH, CONTENT_RANGE, RANGE},
};
-#[cfg(not(target_family = "wasm"))]
-use reqwest::{header::ACCEPT_ENCODING, redirect::Policy};
use tokio::sync::{
- oneshot::{Sender as TokioOneshotSender, channel as tokio_oneshot_channel},
+ oneshot::channel as tokio_oneshot_channel,
watch::{Receiver as TokioWatchReceiver, channel as tokio_watch_channel},
};
@@ -56,7 +51,7 @@ impl BackendContext for UniversalBackendContext {
backend_event_sender: BackendEventSender,
_destination_lease: &DestinationLockLease,
) -> Result<UniversalActiveTask, UniversalBackendError> {
- let resume_artifact_path = config.destination.with_added_extension("part");
+ let resume_artifact_path = config.destination.with_extension("part");
self.start(config, generation, resume_artifact_path, backend_event_sender).await
}
@@ -117,7 +112,7 @@ async fn download_streaming(
retry_count: u16,
progress_interval: Duration,
pause_receiver: TokioWatchReceiver<bool>,
- completion_sender: TokioOneshotSender<()>,
+ completion_sender: tokio::sync::oneshot::Sender<()>,
) {
let mut pause_receiver = pause_receiver;
let result = download_streaming_with_retries(
@@ -152,26 +147,9 @@ async fn download_streaming_with_retries(
progress_interval: Duration,
pause_receiver: &mut TokioWatchReceiver<bool>,
) -> Result<DownloadStreamCompletion, String> {
- let client = Client::builder();
- #[cfg(not(target_family = "wasm"))]
- let client = client.redirect(Policy::custom(|attempt| {
- let current_scheme = attempt.previous().last().and_then(|url| Scheme::try_from(url.scheme()).ok());
- let next_scheme = Scheme::try_from(attempt.url().scheme()).ok();
- let downgrade = matches!(
- (current_scheme, next_scheme),
- (Some(current), Some(next)) if current == Scheme::HTTPS && next != Scheme::HTTPS
- );
- if downgrade || attempt.previous().len() >= 10 {
- attempt.error("unsafe redirect")
- } else {
- attempt.follow()
- }
- }));
- let client = client.build().map_err(|error| error.to_string())?;
let mut attempt = 0_u16;
loop {
match download_once(
- &client,
Arc::clone(&config),
generation,
resume_artifact_path,
@@ -193,7 +171,6 @@ async fn download_streaming_with_retries(
}
async fn download_once(
- client: &Client,
config: Arc<DownloadConfig>,
generation: ActiveDownloadGeneration,
resume_artifact_path: &Path,
@@ -201,33 +178,33 @@ async fn download_once(
progress_interval: Duration,
pause_receiver: &mut TokioWatchReceiver<bool>,
) -> Result<DownloadStreamCompletion, String> {
+ let client = reqwest::Client::new();
let resume_from_bytes = fs::asyn::file_length(resume_artifact_path).await.unwrap_or(0);
- let request = config.request.apply(client.get(&config.request.url)).map_err(|error| error.to_string())?;
- #[cfg(not(target_family = "wasm"))]
- let request = request.header(ACCEPT_ENCODING, "identity");
- let mut request = request;
+ let mut request = client.get(&config.source_url);
if resume_from_bytes > 0 {
request = request.header(RANGE, format!("bytes={resume_from_bytes}-"));
}
let response = tokio::select! {
_ = wait_for_pause(pause_receiver) => return Ok(DownloadStreamCompletion::Paused),
- response = request.send() => response.map_err(|error| error.without_url().to_string())?,
+ response = request.send() => response.map_err(|error| error.to_string())?,
};
let status = response.status();
let content_range = response.headers().get(CONTENT_RANGE).cloned();
let resume_from_bytes = if resume_from_bytes > 0 {
match status {
StatusCode::PARTIAL_CONTENT => {
- let validation = content_range
+ let header = content_range
.as_ref()
- .ok_or_else(|| "server returned 206 without Content-Range header".to_string())
- .and_then(|header| {
- header.to_str().map_err(|error| format!("non-utf8 Content-Range header: {error}"))
- })
- .and_then(|header| validate_content_range(header, resume_from_bytes, config.expected_bytes));
- if let Err(message) = validation {
- return Err(discard_invalid_resume(resume_artifact_path, message).await);
+ .ok_or_else(|| "server returned 206 without Content-Range header".to_string())?;
+ let header_value =
+ header.to_str().map_err(|error| format!("non-utf8 Content-Range header: {error}"))?;
+ let advertised_start = parse_content_range_start(header_value)
+ .ok_or_else(|| format!("server returned malformed Content-Range header: {header_value}"))?;
+ if advertised_start != resume_from_bytes {
+ return Err(format!(
+ "server returned bytes starting at {advertised_start} but client requested {resume_from_bytes}"
+ ));
}
resume_from_bytes
},
@@ -250,8 +227,12 @@ async fn download_once(
} else {
resume_from_bytes
};
- let response = response.error_for_status().map_err(|error| error.without_url().to_string())?;
- let remaining_bytes = response.content_length();
+ let response = response.error_for_status().map_err(|error| error.to_string())?;
+ let remaining_bytes = response
+ .headers()
+ .get(CONTENT_LENGTH)
+ .and_then(|value| value.to_str().ok())
+ .and_then(|value| value.parse::<u64>().ok());
let total_bytes = remaining_bytes
.map(|remaining_bytes| remaining_bytes.saturating_add(resume_from_bytes))
.or(config.expected_bytes);
@@ -275,17 +256,9 @@ async fn download_once(
let Some(chunk) = chunk else {
break;
};
- let chunk = chunk.map_err(|error| error.without_url().to_string())?;
- let chunk_len = u64::try_from(chunk.len()).map_err(|_| "response chunk length overflow".to_string())?;
- let next_downloaded_bytes =
- downloaded_bytes.checked_add(chunk_len).ok_or_else(|| "downloaded byte count overflow".to_string())?;
- if let Some(expected_bytes) = config.expected_bytes
- && next_downloaded_bytes > expected_bytes
- {
- return Err(format!("response exceeded the registry size of {expected_bytes} bytes"));
- }
+ let chunk = chunk.map_err(|error| error.to_string())?;
file.write_all(&chunk).await.map_err(|error| error.to_string())?;
- downloaded_bytes = next_downloaded_bytes;
+ downloaded_bytes = downloaded_bytes.saturating_add(chunk.len() as u64);
if last_progress_emit.elapsed() >= progress_interval {
backend_event_sender.send_progress(generation, downloaded_bytes, total_bytes).await;
@@ -308,34 +281,11 @@ async fn wait_for_pause(pause_receiver: &mut TokioWatchReceiver<bool>) {
let _ = pause_receiver.changed().await;
}
-async fn discard_invalid_resume(
- resume_artifact_path: &Path,
- message: impl Into<String>,
-) -> String {
- let _ = fs::asyn::remove_file(resume_artifact_path).await;
- message.into()
-}
-
-fn validate_content_range(
- header: &str,
- requested_start: u64,
- expected_total: Option<u64>,
-) -> Result<(), String> {
- let value = header.strip_prefix("bytes ").ok_or_else(|| format!("malformed Content-Range: {header}"))?;
- let (range, _) = value.split_once('/').ok_or_else(|| format!("malformed Content-Range: {header}"))?;
- let (start, _) = range.split_once('-').ok_or_else(|| format!("malformed Content-Range: {header}"))?;
- let start = start.parse::<u64>().map_err(|_| format!("malformed Content-Range: {header}"))?;
- if start != requested_start {
- return Err(format!("server returned bytes starting at {start} but client requested {requested_start}"));
- }
- if let Some(expected) = expected_total {
- let total = parse_content_range_total(header)
- .ok_or_else(|| format!("server returned Content-Range without a total: {header}"))?;
- if total != expected {
- return Err(format!("server advertised {total} bytes but registry declared {expected}"));
- }
- }
- Ok(())
+fn parse_content_range_start(header_value: &str) -> Option<u64> {
+ let value = header_value.strip_prefix("bytes ")?.trim_start();
+ let (range, _) = value.split_once('/')?;
+ let (start, _) = range.split_once('-')?;
+ start.parse::<u64>().ok()
}
fn parse_content_range_total(header_value: &str) -> Option<u64> {
diff --git a/crates/legacy/download-manager/src/crc_utils.rs b/crates/legacy/download-manager/src/crc_utils.rs
new file mode 100644
index 000000000..12583efbd
--- /dev/null
+++ b/crates/legacy/download-manager/src/crc_utils.rs
@@ -0,0 +1,111 @@
+use std::{
+ io::{Error as IoError, ErrorKind},
+ path::{Path, PathBuf},
+};
+
+use base64::Engine;
+use kiban::{fs, time::SystemTime};
+use serde::{Deserialize, Serialize};
+
+const CRC_CACHE_VERSION: u8 = 1;
+const CRC_READ_CHUNK_SIZE: u64 = 8 * 1024 * 1024;
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)]
+struct CrcCacheReceipt {
+ version: u8,
+ crc: String,
+ file_size: u64,
+ modified_unix_seconds: u64,
+ modified_nanos: u32,
+}
+
+pub async fn calculate_and_verify_crc(
+ file_path: &Path,
+ expected_crc32c_base64: &str,
+) -> Result<bool, IoError> {
+ let Some(expected_crc) = decode_crc32c(expected_crc32c_base64) else {
+ return Ok(false);
+ };
+
+ let file_len = kiban::fs::asyn::file_length(file_path).await?;
+ let mut offset = 0;
+ let mut actual_crc = 0;
+ while offset < file_len {
+ let end = offset.saturating_add(CRC_READ_CHUNK_SIZE).min(file_len);
+ let chunk = kiban::fs::asyn::read_range(file_path, offset..end).await?;
+ if chunk.len() != (end - offset) as usize {
+ return Err(IoError::new(ErrorKind::UnexpectedEof, "file changed during CRC verification"));
+ }
+ actual_crc = crc32c::crc32c_append(actual_crc, &chunk);
+ offset = end;
+ }
+
+ Ok(actual_crc == expected_crc)
+}
+
+pub async fn crc_cache_matches(
+ file_path: &Path,
+ expected_crc32c_base64: &str,
+) -> bool {
+ if decode_crc32c(expected_crc32c_base64).is_none() {
+ return false;
+ }
+
+ let Ok(cache_contents) = kiban::fs::asyn::read_to_string(crc_path_for_file(file_path)).await else {
+ return false;
+ };
+ let Ok(cached_receipt) = serde_json::from_str::<CrcCacheReceipt>(&cache_contents) else {
+ return false;
+ };
+ let Some(current_receipt) = CrcCacheReceipt::from_file(file_path, expected_crc32c_base64).await else {
+ return false;
+ };
+
+ cached_receipt == current_receipt
+}
+
+pub async fn save_crc_file(
+ file_path: &Path,
+ crc_value: &str,
+) -> Result<(), IoError> {
+ let Some(receipt) = CrcCacheReceipt::from_file(file_path, crc_value).await else {
+ return Ok(());
+ };
+ let receipt_json = serde_json::to_vec(&receipt).map_err(IoError::other)?;
+ fs::asyn::write(crc_path_for_file(file_path), receipt_json).await
+}
+
+pub fn crc_path_for_file(file_path: &Path) -> PathBuf {
+ PathBuf::from(format!("{}.crc", file_path.display()))
+}
+
+impl CrcCacheReceipt {
+ async fn from_file(
+ file_path: &Path,
+ crc: &str,
+ ) -> Option<Self> {
+ if !fs::asyn::is_file(file_path).await {
+ return None;
+ }
+
+ let file_size = fs::asyn::file_length(file_path).await.ok()?;
+ let modified = fs::asyn::file_modified(file_path).await.ok()?.duration_since(SystemTime::UNIX_EPOCH).ok()?;
+ Some(Self {
+ version: CRC_CACHE_VERSION,
+ crc: crc.to_string(),
+ file_size,
+ modified_unix_seconds: modified.as_secs(),
+ modified_nanos: modified.subsec_nanos(),
+ })
+ }
+}
+
+fn decode_crc32c(expected_crc32c_base64: &str) -> Option<u32> {
+ let expected_bytes = base64::engine::general_purpose::STANDARD.decode(expected_crc32c_base64).ok()?;
+
+ if expected_bytes.len() != 4 {
+ return None;
+ }
+
+ Some(u32::from_be_bytes([expected_bytes[0], expected_bytes[1], expected_bytes[2], expected_bytes[3]]))
+}
diff --git a/crates/legacy/download-manager/src/download_error.rs b/crates/legacy/download-manager/src/download_error.rs
index 792fe68a8..169ce3a23 100644
--- a/crates/legacy/download-manager/src/download_error.rs
+++ b/crates/legacy/download-manager/src/download_error.rs
@@ -1,7 +1,3 @@
-use std::io::Error as IoError;
-
-use serde_json::Error as JsonError;
-
#[derive(thiserror::Error, Clone, Debug, PartialEq, Eq)]
pub enum DownloadError {
#[error("io error: {0}")]
@@ -16,10 +12,6 @@ pub enum DownloadError {
ResumeUnsupported,
#[error("bad url")]
BadUrl,
- #[error("authenticated downloads require HTTPS")]
- InsecureAuthenticatedRequest,
- #[error("download authentication is no longer available")]
- AuthenticationUnavailable,
#[error("missing download info for task")]
MissingDownloadInfo,
#[error("resume data read failed")]
@@ -48,25 +40,16 @@ pub enum DownloadError {
ChannelClosed,
#[error("backend error: {0}")]
Backend(String),
- #[error("invalid expected {algorithm} digest")]
- InvalidDigest {
- algorithm: &'static str,
- },
- #[error("integrity verification I/O failed for {path}: {message}")]
- IntegrityIo {
- path: String,
- message: String,
- },
}
-impl From<IoError> for DownloadError {
- fn from(error: IoError) -> Self {
+impl From<std::io::Error> for DownloadError {
+ fn from(error: std::io::Error) -> Self {
Self::Io(error.to_string())
}
}
-impl From<JsonError> for DownloadError {
- fn from(error: JsonError) -> Self {
+impl From<serde_json::Error> for DownloadError {
+ fn from(error: serde_json::Error) -> Self {
Self::SerdeJson(error.to_string())
}
}
diff --git a/crates/legacy/download-manager/src/download_info.rs b/crates/legacy/download-manager/src/download_info.rs
index 93270a472..6dfa71d44 100644
--- a/crates/legacy/download-manager/src/download_info.rs
+++ b/crates/legacy/download-manager/src/download_info.rs
@@ -1,7 +1,4 @@
use serde::{Deserialize, Serialize};
-use serde_json::{Error as JsonError, from_str, to_string};
-
-use crate::FileCheck;
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)]
pub struct DownloadInfo {
@@ -9,34 +6,37 @@ pub struct DownloadInfo {
pub destination_path: String,
#[serde(default)]
pub crc32c: Option<String>,
- #[serde(default, skip_serializing_if = "Option::is_none")]
- pub file_check: Option<FileCheck>,
}
impl DownloadInfo {
pub fn new<U: Into<String>, P: Into<String>>(
source_url: U,
destination_path: P,
- file_check: FileCheck,
) -> Self {
- let crc32c = file_check.expected_crc();
Self {
source_url: source_url.into(),
destination_path: destination_path.into(),
- crc32c,
- file_check: Some(file_check),
+ crc32c: None,
}
}
- pub fn resolved_file_check(&self) -> FileCheck {
- self.file_check.clone().or_else(|| self.crc32c.clone().map(FileCheck::CRC)).unwrap_or(FileCheck::None)
+ pub fn with_crc<U: Into<String>, P: Into<String>, C: Into<String>>(
+ source_url: U,
+ destination_path: P,
+ crc32c: C,
+ ) -> Self {
+ Self {
+ source_url: source_url.into(),
+ destination_path: destination_path.into(),
+ crc32c: Some(crc32c.into()),
+ }
}
- pub fn to_json(&self) -> Result<String, JsonError> {
- to_string(self)
+ pub fn to_json(&self) -> Result<String, serde_json::Error> {
+ serde_json::to_string(self)
}
- pub fn from_json(json: &str) -> Result<Self, JsonError> {
- from_str(json)
+ pub fn from_json(json: &str) -> Result<Self, serde_json::Error> {
+ serde_json::from_str(json)
}
}
diff --git a/crates/legacy/download-manager/src/file_check.rs b/crates/legacy/download-manager/src/file_check.rs
index 1c13e8ccc..21857f77a 100644
--- a/crates/legacy/download-manager/src/file_check.rs
+++ b/crates/legacy/download-manager/src/file_check.rs
@@ -3,8 +3,6 @@ use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)]
pub enum FileCheck {
CRC(String),
- Sha256(String),
- GitBlobSha1(String),
None,
}
@@ -12,16 +10,7 @@ impl FileCheck {
pub fn expected_crc(&self) -> Option<String> {
match self {
Self::CRC(crc) => Some(crc.clone()),
- Self::Sha256(_) | Self::GitBlobSha1(_) | Self::None => None,
- }
- }
-
- pub fn verification_failure_message(&self) -> &'static str {
- match self {
- Self::CRC(_) => "CRC verification failed",
- Self::Sha256(_) => "SHA-256 verification failed",
- Self::GitBlobSha1(_) => "Git blob SHA-1 verification failed",
- Self::None => "integrity verification failed",
+ Self::None => None,
}
}
}
diff --git a/crates/legacy/download-manager/src/file_download_manager.rs b/crates/legacy/download-manager/src/file_download_manager.rs
index eed7c1d96..db5554aa4 100644
--- a/crates/legacy/download-manager/src/file_download_manager.rs
+++ b/crates/legacy/download-manager/src/file_download_manager.rs
@@ -7,7 +7,7 @@ use tokio_stream::wrappers::BroadcastStream as TokioBroadcastStream;
#[cfg(target_vendor = "apple")]
use crate::backends::apple::AppleDownloadManager;
use crate::{
- DownloadError, DownloadId, FileCheck, FileDownloadEvent, FileDownloadTask, HttpDownloadRequest,
+ DownloadError, DownloadId, FileCheck, FileDownloadEvent, FileDownloadTask,
backends::universal::UniversalDownloadManager,
};
@@ -48,7 +48,7 @@ pub trait FileDownloadManager: Send + Sync + 'static {
async fn file_download_task(
&self,
- request: HttpDownloadRequest,
+ source_url: &str,
destination_path: &Path,
file_check: FileCheck,
expected_bytes: Option<u64>,
diff --git a/crates/legacy/download-manager/src/file_download_task.rs b/crates/legacy/download-manager/src/file_download_task.rs
index b8591d45b..b699588f9 100644
--- a/crates/legacy/download-manager/src/file_download_task.rs
+++ b/crates/legacy/download-manager/src/file_download_task.rs
@@ -3,7 +3,7 @@ use std::{fmt::Debug, path::Path, sync::Arc};
use tokio::sync::broadcast::Sender as TokioBroadcastSender;
use tokio_stream::wrappers::BroadcastStream as TokioBroadcastStream;
-use crate::{DownloadError, DownloadEventSender, DownloadId, FileCheck, FileDownloadState, HttpDownloadRequest};
+use crate::{DownloadError, DownloadEventSender, DownloadId, FileCheck, FileDownloadState};
#[cfg_attr(not(target_family = "wasm"), async_trait::async_trait)]
#[cfg_attr(target_family = "wasm", async_trait::async_trait(?Send))]
@@ -41,19 +41,16 @@ pub(crate) trait ManagedFileDownloadTask: FileDownloadTask {
pub(crate) struct CachedFileDownloadTask {
public: Arc<dyn FileDownloadTask>,
managed: Arc<dyn ManagedFileDownloadTask>,
- request: HttpDownloadRequest,
}
impl CachedFileDownloadTask {
pub(crate) fn new(
public: Arc<dyn FileDownloadTask>,
managed: Arc<dyn ManagedFileDownloadTask>,
- request: HttpDownloadRequest,
) -> Self {
Self {
public,
managed,
- request,
}
}
@@ -68,11 +65,4 @@ impl CachedFileDownloadTask {
pub(crate) fn is_stopped(&self) -> bool {
self.managed.is_stopped()
}
-
- pub fn request_matches(
- &self,
- request: &HttpDownloadRequest,
- ) -> bool {
- self.request == *request
- }
}
diff --git a/crates/legacy/download-manager/src/file_download_task_actor/actor_state.rs b/crates/legacy/download-manager/src/file_download_task_actor/actor_state.rs
index 46de5c55f..f685eba71 100644
--- a/crates/legacy/download-manager/src/file_download_task_actor/actor_state.rs
+++ b/crates/legacy/download-manager/src/file_download_task_actor/actor_state.rs
@@ -15,7 +15,6 @@ pub(crate) enum DownloadActorState<B: DownloadBackend> {
active_task: B::ActiveTask,
generation: ActiveDownloadGeneration,
destination_lease: DestinationLockLease,
- resumed: bool,
},
Downloaded,
}
diff --git a/crates/legacy/download-manager/src/file_download_task_actor/download_task_actor.rs b/crates/legacy/download-manager/src/file_download_task_actor/download_task_actor.rs
index 97eba1e23..f83822e0c 100644
--- a/crates/legacy/download-manager/src/file_download_task_actor/download_task_actor.rs
+++ b/crates/legacy/download-manager/src/file_download_task_actor/download_task_actor.rs
@@ -15,18 +15,18 @@ use tokio::sync::{
};
use crate::{
- DownloadError, FileDownloadState, LockFileState, check_lock_file,
+ DownloadError, FileCheck, FileDownloadState, LockFileState, check_lock_file,
+ crc_utils::{calculate_and_verify_crc, crc_path_for_file, save_crc_file},
download_log_event::{DownloadLogEvent, log},
file_download_task_actor::{
BackendEvent, DownloadActorState, PendingProgressSlot, ProgressCounters, PublicProjection, TaskCommand,
TerminalOutcome, project_runtime_public_state,
},
- integrity::{integrity_receipt_path, save_integrity_cache, verify_file_integrity},
lock_manager::{DestinationLockLease, lock_path_for_destination},
release_lock_if_owned,
traits::{
ActiveDownloadGeneration, ActiveDownloadGenerationCounter, ActiveTask, BackendContext, BackendEventSender,
- Capabilities, DownloadBackend, DownloadConfig,
+ DownloadBackend, DownloadConfig,
},
};
@@ -347,7 +347,6 @@ impl<B: DownloadBackend> DownloadTaskActor<B> {
let from_state = current_state.name();
if let DownloadActorState::Downloading {
destination_lease,
- resumed,
..
} = current_state
{
@@ -366,14 +365,11 @@ impl<B: DownloadBackend> DownloadTaskActor<B> {
Err(message) => {
remove_file(&self.config.destination).await;
remove_resume_artifact(&self.config.destination).await;
- remove_file(&integrity_receipt_path(&self.config.destination)).await;
+ remove_file(&crc_path_for_file(&self.config.destination)).await;
self.progress_counters = ProgressCounters::default();
+ self.projection = PublicProjection::StickyError(message.clone());
release_destination_lease(destination_lease).await;
self.finish_transition(from_state, DownloadActorState::NotDownloaded);
- if self.restart_after_stale_resume(resumed).await {
- return;
- }
- self.projection = PublicProjection::StickyError(message.clone());
self.pending_terminal_outcome = Some(TerminalOutcome::Error(message));
},
}
@@ -401,19 +397,15 @@ impl<B: DownloadBackend> DownloadTaskActor<B> {
if let DownloadActorState::Downloading {
active_task,
destination_lease,
- resumed,
..
} = current_state
{
let _ = active_task.cancel(&self.config.destination).await;
remove_resume_artifact(&self.config.destination).await;
+ self.projection = PublicProjection::StickyError(message.clone());
self.progress_counters = ProgressCounters::default();
release_destination_lease(destination_lease).await;
self.finish_transition(from_state, DownloadActorState::NotDownloaded);
- if self.restart_after_stale_resume(resumed).await {
- return;
- }
- self.projection = PublicProjection::StickyError(message.clone());
self.pending_terminal_outcome = Some(TerminalOutcome::Error(message));
}
}
@@ -437,25 +429,6 @@ impl<B: DownloadBackend> DownloadTaskActor<B> {
}
}
- async fn restart_after_stale_resume(
- &mut self,
- resumed: bool,
- ) -> bool {
- if !resumed
- || self.config.request.is_authenticated()
- || !B::CAPABILITIES.contains(Capabilities::CACHES_REDIRECTED_URL_IN_RESUME_DATA)
- {
- return false;
- }
- self.projection = PublicProjection::None;
- if let Err(error) = self.start_fresh_download().await {
- let message = error.to_string();
- self.projection = PublicProjection::StickyError(message.clone());
- self.pending_terminal_outcome = Some(TerminalOutcome::Error(message));
- }
- true
- }
-
async fn start_fresh_download(&mut self) -> Result<(), DownloadError> {
let lease = self.acquire_destination_lease().await?;
let generation = self.generation_counter.allocate_next();
@@ -480,7 +453,6 @@ impl<B: DownloadBackend> DownloadTaskActor<B> {
active_task,
generation,
destination_lease: lease,
- resumed: false,
});
Ok(())
}
@@ -524,7 +496,6 @@ impl<B: DownloadBackend> DownloadTaskActor<B> {
active_task,
generation,
destination_lease: lease,
- resumed: true,
});
Ok(())
}
@@ -632,8 +603,8 @@ async fn remove_file(path: &Path) {
}
async fn remove_resume_artifact(destination: &Path) {
- remove_file(&destination.with_added_extension("part")).await;
- remove_file(&destination.with_added_extension("resume_data")).await;
+ remove_file(&destination.with_extension("part")).await;
+ remove_file(&destination.with_extension("resume_data")).await;
}
async fn validate_completed_file(config: &DownloadConfig) -> Result<u64, String> {
@@ -668,12 +639,20 @@ async fn validate_completed_file(config: &DownloadConfig) -> Result<u64, String>
let total_bytes = config.expected_bytes.unwrap_or(actual_bytes);
- match verify_file_integrity(&config.destination, &config.file_check).await {
- Ok(true) => {
- let _ = save_integrity_cache(&config.destination, &config.file_check).await;
- Ok(total_bytes)
+ match &config.file_check {
+ FileCheck::None => Ok(total_bytes),
+ FileCheck::CRC(expected_crc) => {
+ let crc_result = calculate_and_verify_crc(&config.destination, expected_crc).await;
+ match crc_result {
+ Ok(true) => {
+ let destination = config.destination.clone();
+ let expected = expected_crc.clone();
+ let _ = save_crc_file(&destination, &expected).await;
+ Ok(total_bytes)
+ },
+ Ok(false) => Err("CRC verification failed".to_string()),
+ Err(error) => Err(format!("CRC verification error: {error}")),
+ }
},
- Ok(false) => Err(config.file_check.verification_failure_message().to_string()),
- Err(error) => Err(format!("integrity verification error: {error}")),
}
}
diff --git a/crates/legacy/download-manager/src/file_download_task_actor/generic_file_download_task.rs b/crates/legacy/download-manager/src/file_download_task_actor/generic_file_download_task.rs
index 06dee5dc1..44b53aafb 100644
--- a/crates/legacy/download-manager/src/file_download_task_actor/generic_file_download_task.rs
+++ b/crates/legacy/download-manager/src/file_download_task_actor/generic_file_download_task.rs
@@ -13,7 +13,7 @@ use tokio::sync::{
Mutex as TokioMutex,
broadcast::{Sender as TokioBroadcastSender, channel as tokio_broadcast_channel},
mpsc::{Sender as TokioMpscSender, channel as tokio_mpsc_channel},
- oneshot::{Sender as TokioOneshotSender, channel as tokio_oneshot_channel},
+ oneshot::channel as tokio_oneshot_channel,
watch::{Receiver as TokioWatchReceiver, channel as tokio_watch_channel},
};
use tokio_stream::{StreamExt as TokioStreamExt, wrappers::BroadcastStream as TokioBroadcastStream};
@@ -117,7 +117,6 @@ impl<B: DownloadBackend> GenericFileDownloadTask<B> {
active_task,
generation: attachment_generation,
destination_lease,
- resumed: false,
},
ProgressCounters {
downloaded_bytes: initial_downloaded_bytes,
@@ -167,7 +166,7 @@ impl<B: DownloadBackend> GenericFileDownloadTask<B> {
async fn send_command(
&self,
- command_builder: impl FnOnce(TokioOneshotSender<Result<(), DownloadError>>) -> TaskCommand,
+ command_builder: impl FnOnce(tokio::sync::oneshot::Sender<Result<(), DownloadError>>) -> TaskCommand,
) -> Result<(), DownloadError> {
if self.is_stopped.load(Ordering::SeqCst) {
return Err(DownloadError::TaskStopped);
@@ -199,7 +198,7 @@ impl<B: DownloadBackend> fmt::Debug for GenericFileDownloadTask<B> {
formatter
.debug_struct("GenericFileDownloadTask")
.field("download_id", &self.config.download_id)
- .field("source_url", &self.config.request.url)
+ .field("source_url", &self.config.source_url)
.field("destination", &self.config.destination)
.finish()
}
@@ -207,13 +206,13 @@ impl<B: DownloadBackend> fmt::Debug for GenericFileDownloadTask<B> {
#[cfg_attr(not(target_family = "wasm"), async_trait::async_trait)]
#[cfg_attr(target_family = "wasm", async_trait::async_trait(?Send))]
-impl<B: DownloadBackend> FileDownloadTask for GenericFileDownloadTask<B> {
+impl<B: DownloadBackend> crate::FileDownloadTask for GenericFileDownloadTask<B> {
fn download_id(&self) -> DownloadId {
self.config.download_id
}
fn source_url(&self) -> &str {
- &self.config.request.url
+ &self.config.source_url
}
fn destination(&self) -> &Path {
diff --git a/crates/legacy/download-manager/src/http_download_request.rs b/crates/legacy/download-manager/src/http_download_request.rs
deleted file mode 100644
index d24be885c..000000000
--- a/crates/legacy/download-manager/src/http_download_request.rs
+++ /dev/null
@@ -1,120 +0,0 @@
-use std::{
- fmt,
- net::IpAddr,
- sync::{Arc, Weak},
-};
-
-use http::uri::Scheme;
-use reqwest::{RequestBuilder, Url};
-
-use crate::DownloadError;
-
-#[derive(Clone)]
-pub struct HttpDownloadRequest {
- pub url: String,
- bearer_token: Option<Weak<str>>,
-}
-
-impl HttpDownloadRequest {
- pub fn get(url: impl Into<String>) -> Self {
- Self {
- url: url.into(),
- bearer_token: None,
- }
- }
-
- pub fn with_bearer_token(
- url: impl Into<String>,
- bearer_token: &Arc<str>,
- ) -> Self {
- Self {
- url: url.into(),
- bearer_token: Some(Arc::downgrade(bearer_token)),
- }
- }
-
- pub fn is_authenticated(&self) -> bool {
- self.bearer_token.is_some()
- }
-
- pub fn bearer_token(&self) -> Result<Option<Arc<str>>, DownloadError> {
- self.bearer_token
- .as_ref()
- .map(|token| token.upgrade().ok_or(DownloadError::AuthenticationUnavailable))
- .transpose()
- }
-
- pub fn apply(
- &self,
- request: RequestBuilder,
- ) -> Result<RequestBuilder, DownloadError> {
- Ok(match self.bearer_token()? {
- Some(token) => request.bearer_auth(token.as_ref()),
- None => request,
- })
- }
-
- pub fn validate(&self) -> Result<(), DownloadError> {
- let url = Url::parse(&self.url).map_err(|_| DownloadError::BadUrl)?;
- let scheme = Scheme::try_from(url.scheme()).map_err(|_| DownloadError::BadUrl)?;
- if self.is_authenticated() && scheme != Scheme::HTTPS && !is_loopback(&url) {
- return Err(DownloadError::InsecureAuthenticatedRequest);
- }
- Ok(())
- }
-}
-
-impl fmt::Debug for HttpDownloadRequest {
- fn fmt(
- &self,
- formatter: &mut fmt::Formatter<'_>,
- ) -> fmt::Result {
- let end = self.url.find(['?', '#']).unwrap_or(self.url.len());
- formatter
- .debug_struct("HttpDownloadRequest")
- .field("url", &&self.url[..end])
- .field("authenticated", &self.is_authenticated())
- .finish()
- }
-}
-
-impl PartialEq for HttpDownloadRequest {
- fn eq(
- &self,
- other: &Self,
- ) -> bool {
- self.url == other.url
- && match (&self.bearer_token, &other.bearer_token) {
- (Some(left), Some(right)) => Weak::ptr_eq(left, right),
- (None, None) => true,
- _ => false,
- }
- }
-}
-
-impl Eq for HttpDownloadRequest {}
-
-impl From<String> for HttpDownloadRequest {
- fn from(url: String) -> Self {
- Self::get(url)
- }
-}
-
-impl From<&str> for HttpDownloadRequest {
- fn from(url: &str) -> Self {
- Self::get(url)
- }
-}
-
-impl From<&String> for HttpDownloadRequest {
- fn from(url: &String) -> Self {
- Self::get(url)
- }
-}
-
-fn is_loopback(url: &Url) -> bool {
- let Some(host) = url.host_str() else {
- return false;
- };
- host.eq_ignore_ascii_case("localhost") || host.parse::<IpAddr>().is_ok_and(|address| address.is_loopback())
-}
diff --git a/crates/legacy/download-manager/src/integrity.rs b/crates/legacy/download-manager/src/integrity.rs
deleted file mode 100644
index d5f613e7f..000000000
--- a/crates/legacy/download-manager/src/integrity.rs
+++ /dev/null
@@ -1,239 +0,0 @@
-use std::{
- io::{Error as IoError, ErrorKind},
- path::{Path, PathBuf},
-};
-
-use base64::{Engine, engine::general_purpose::STANDARD};
-use crc32c::crc32c_append;
-use hex::decode_to_slice;
-use kiban::{fs, time::SystemTime};
-use serde::{Deserialize, Serialize};
-use serde_json::{from_str, to_vec};
-use sha1::Sha1;
-use sha2::{Digest, Sha256};
-
-use crate::FileCheck;
-
-const INTEGRITY_RECEIPT_VERSION: u8 = 1;
-const READ_CHUNK_SIZE: u64 = 8 * 1024 * 1024;
-
-#[derive(Debug, thiserror::Error)]
-pub enum VerificationError {
- #[error("invalid expected {algorithm} digest")]
- InvalidExpectedDigest {
- algorithm: &'static str,
- },
- #[error(transparent)]
- Io(#[from] IoError),
-}
-
-#[derive(Serialize, Deserialize, PartialEq, Eq)]
-struct IntegrityReceipt {
- version: u8,
- file_check: FileCheck,
- #[serde(flatten)]
- fingerprint: FileFingerprint,
-}
-
-#[derive(Serialize, Deserialize, PartialEq, Eq)]
-struct FileFingerprint {
- file_size: u64,
- modified_unix_seconds: u64,
- modified_nanos: u32,
-}
-
-enum StreamingVerifier {
- Crc32c {
- expected: u32,
- actual: u32,
- },
- Sha256 {
- expected: [u8; 32],
- actual: Sha256,
- },
- GitBlobSha1 {
- expected: [u8; 20],
- actual: Sha1,
- },
-}
-
-pub async fn verify_file_integrity(
- file_path: &Path,
- file_check: &FileCheck,
-) -> Result<bool, VerificationError> {
- if matches!(file_check, FileCheck::None) {
- return Ok(true);
- }
-
- let file_len = fs::asyn::file_length(file_path).await?;
- let mut verifier = StreamingVerifier::new(file_check, file_len)?;
- let mut offset = 0;
- while offset < file_len {
- let end = offset.saturating_add(READ_CHUNK_SIZE).min(file_len);
- let chunk = fs::asyn::read_range(file_path, offset..end).await?;
- let expected_len = usize::try_from(end - offset)
- .map_err(|_| IoError::new(ErrorKind::InvalidData, "integrity chunk size overflow"))?;
- if chunk.len() != expected_len {
- return Err(IoError::new(ErrorKind::UnexpectedEof, "file changed during integrity verification").into());
- }
- verifier.update(&chunk);
- offset = end;
- }
- if fs::asyn::file_length(file_path).await? != file_len {
- return Err(IoError::new(ErrorKind::UnexpectedEof, "file changed during integrity verification").into());
- }
-
- Ok(verifier.matches_expected())
-}
-
-pub async fn integrity_cache_matches(
- file_path: &Path,
- file_check: &FileCheck,
-) -> bool {
- if !expected_digest_is_well_formed(file_check) {
- return false;
- }
- let Ok(cache_contents) = fs::asyn::read_to_string(integrity_receipt_path(file_path)).await else {
- return false;
- };
- let Ok(fingerprint) = FileFingerprint::from_file(file_path).await else {
- return false;
- };
-
- from_str::<IntegrityReceipt>(&cache_contents).is_ok_and(|cached| {
- cached
- == IntegrityReceipt {
- version: INTEGRITY_RECEIPT_VERSION,
- file_check: file_check.clone(),
- fingerprint,
- }
- })
-}
-
-pub async fn save_integrity_cache(
- file_path: &Path,
- file_check: &FileCheck,
-) -> Result<(), IoError> {
- if matches!(file_check, FileCheck::None) {
- return Ok(());
- }
- let fingerprint = FileFingerprint::from_file(file_path).await?;
- let receipt = to_vec(&IntegrityReceipt {
- version: INTEGRITY_RECEIPT_VERSION,
- file_check: file_check.clone(),
- fingerprint,
- })
- .map_err(IoError::other)?;
- fs::asyn::write(integrity_receipt_path(file_path), receipt).await
-}
-
-pub fn integrity_receipt_path(file_path: &Path) -> PathBuf {
- file_path.with_added_extension("integrity")
-}
-
-impl StreamingVerifier {
- fn new(
- file_check: &FileCheck,
- file_len: u64,
- ) -> Result<Self, VerificationError> {
- match file_check {
- FileCheck::CRC(expected) => Ok(Self::Crc32c {
- expected: decode_crc32c(expected).ok_or(VerificationError::InvalidExpectedDigest {
- algorithm: "crc32c",
- })?,
- actual: 0,
- }),
- FileCheck::Sha256(expected) => Ok(Self::Sha256 {
- expected: decode_hex(expected, "sha256")?,
- actual: Sha256::new(),
- }),
- FileCheck::GitBlobSha1(expected) => {
- let mut actual = Sha1::new();
- actual.update(format!("blob {file_len}\0").as_bytes());
- Ok(Self::GitBlobSha1 {
- expected: decode_hex(expected, "git_blob_sha1")?,
- actual,
- })
- },
- FileCheck::None => unreachable!("handled before constructing verifier"),
- }
- }
-
- fn update(
- &mut self,
- chunk: &[u8],
- ) {
- match self {
- Self::Crc32c {
- actual,
- ..
- } => *actual = crc32c_append(*actual, chunk),
- Self::Sha256 {
- actual,
- ..
- } => actual.update(chunk),
- Self::GitBlobSha1 {
- actual,
- ..
- } => actual.update(chunk),
- }
- }
-
- fn matches_expected(self) -> bool {
- match self {
- Self::Crc32c {
- expected,
- actual,
- } => actual == expected,
- Self::Sha256 {
- expected,
- actual,
- } => actual.finalize().as_slice() == expected,
- Self::GitBlobSha1 {
- expected,
- actual,
- } => actual.finalize().as_slice() == expected,
- }
- }
-}
-
-impl FileFingerprint {
- async fn from_file(file_path: &Path) -> Result<Self, IoError> {
- let file_size = fs::asyn::file_length(file_path).await?;
- let modified = fs::asyn::file_modified(file_path)
- .await?
- .duration_since(SystemTime::UNIX_EPOCH)
- .map_err(|error| IoError::new(ErrorKind::InvalidData, error))?;
- Ok(Self {
- file_size,
- modified_unix_seconds: modified.as_secs(),
- modified_nanos: modified.subsec_nanos(),
- })
- }
-}
-
-fn expected_digest_is_well_formed(file_check: &FileCheck) -> bool {
- match file_check {
- FileCheck::CRC(expected) => decode_crc32c(expected).is_some(),
- FileCheck::Sha256(expected) => decode_hex::<32>(expected, "sha256").is_ok(),
- FileCheck::GitBlobSha1(expected) => decode_hex::<20>(expected, "git_blob_sha1").is_ok(),
- FileCheck::None => false,
- }
-}
-
-fn decode_crc32c(expected_crc32c_base64: &str) -> Option<u32> {
- let expected_bytes = STANDARD.decode(expected_crc32c_base64).ok()?;
- let expected_bytes: [u8; 4] = expected_bytes.try_into().ok()?;
- Some(u32::from_be_bytes(expected_bytes))
-}
-
-fn decode_hex<const LENGTH: usize>(
- encoded: &str,
- algorithm: &'static str,
-) -> Result<[u8; LENGTH], VerificationError> {
- let mut decoded = [0; LENGTH];
- decode_to_slice(encoded, &mut decoded).map_err(|_| VerificationError::InvalidExpectedDigest {
- algorithm,
- })?;
- Ok(decoded)
-}
diff --git a/crates/legacy/download-manager/src/lib.rs b/crates/legacy/download-manager/src/lib.rs
index a42d474e0..844189b23 100644
--- a/crates/legacy/download-manager/src/lib.rs
+++ b/crates/legacy/download-manager/src/lib.rs
@@ -1,8 +1,5 @@
-use std::path::Path;
-
-use uuid::Uuid;
-
mod checked_file_state;
+mod crc_utils;
mod download_error;
mod download_info;
mod download_log_event;
@@ -14,8 +11,6 @@ mod file_download_phase;
mod file_download_state;
mod file_download_task;
mod file_state;
-mod http_download_request;
-mod integrity;
mod lock_file_info;
mod lock_file_state;
mod lock_manager;
@@ -38,12 +33,11 @@ pub use file_download_phase::FileDownloadPhase;
pub use file_download_state::FileDownloadState;
pub use file_download_task::FileDownloadTask;
pub use file_state::FileState;
-pub use http_download_request::HttpDownloadRequest;
pub use lock_file_info::LockFileInfo;
pub use lock_file_state::LockFileState;
pub use lock_manager::{acquire_lock, check_lock_file, release_lock_if_owned, try_acquire_lock};
-pub type DownloadId = Uuid;
-pub fn compute_download_id(destination_path: &Path) -> DownloadId {
- Uuid::new_v5(&Uuid::NAMESPACE_URL, destination_path.display().to_string().as_bytes())
+pub type DownloadId = uuid::Uuid;
+pub fn compute_download_id(destination_path: &std::path::Path) -> DownloadId {
+ uuid::Uuid::new_v5(&uuid::Uuid::NAMESPACE_URL, destination_path.display().to_string().as_bytes())
}
diff --git a/crates/legacy/download-manager/src/reducer/action.rs b/crates/legacy/download-manager/src/reducer/action.rs
index a9cc606af..e2421db93 100644
--- a/crates/legacy/download-manager/src/reducer/action.rs
+++ b/crates/legacy/download-manager/src/reducer/action.rs
@@ -2,21 +2,19 @@ use std::path::PathBuf;
use serde::{Deserialize, Serialize};
-use crate::FileCheck;
-
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)]
pub enum Action {
DeleteFile {
path: PathBuf,
},
- DeleteIntegrityCache {
+ DeleteCrcCache {
path: PathBuf,
},
DeleteResumeArtifact {
path: PathBuf,
},
- SaveIntegrityCache {
+ SaveCrcCache {
destination: PathBuf,
- file_check: FileCheck,
+ crc: String,
},
}
diff --git a/crates/legacy/download-manager/src/reducer/decision.rs b/crates/legacy/download-manager/src/reducer/decision.rs
index b28b77439..a6acae665 100644
--- a/crates/legacy/download-manager/src/reducer/decision.rs
+++ b/crates/legacy/download-manager/src/reducer/decision.rs
@@ -31,7 +31,7 @@ pub fn decide(
part_path: observation
.resume_artifact_path
.clone()
- .unwrap_or_else(|| observation.destination_path.with_added_extension("part")),
+ .unwrap_or_else(|| observation.destination_path.with_extension("part")),
}
},
CheckedFileState::Invalid | CheckedFileState::Missing => InitialLifecycleState::NotDownloaded,
@@ -96,9 +96,11 @@ fn decide_actions(
path: observation.destination_path.clone(),
});
- if observation.integrity_state == FileState::Exists {
- actions.push(Action::DeleteIntegrityCache {
- path: observation.integrity_path.clone(),
+ if observation.crc_state == FileState::Exists
+ && let Some(path) = observation.crc_path.clone()
+ {
+ actions.push(Action::DeleteCrcCache {
+ path,
});
}
}
diff --git a/crates/legacy/download-manager/src/reducer/disk_observation.rs b/crates/legacy/download-manager/src/reducer/disk_observation.rs
index fbe454fdf..ac75f6693 100644
--- a/crates/legacy/download-manager/src/reducer/disk_observation.rs
+++ b/crates/legacy/download-manager/src/reducer/disk_observation.rs
@@ -2,18 +2,18 @@ use std::path::PathBuf;
use serde::{Deserialize, Serialize};
-use crate::{FileCheck, FileState};
+use crate::FileState;
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)]
pub struct DiskObservation {
pub destination_state: FileState,
- pub integrity_state: FileState,
+ pub crc_state: FileState,
pub resume_state: FileState,
pub destination_size: Option<u64>,
pub resume_size: Option<u64>,
- pub file_check: FileCheck,
+ pub expected_crc: Option<String>,
pub expected_bytes: Option<u64>,
pub destination_path: PathBuf,
- pub integrity_path: PathBuf,
+ pub crc_path: Option<PathBuf>,
pub resume_artifact_path: Option<PathBuf>,
}
diff --git a/crates/legacy/download-manager/src/reducer/validation_outcome.rs b/crates/legacy/download-manager/src/reducer/validation_outcome.rs
index 9bc0c18b3..fba6749ab 100644
--- a/crates/legacy/download-manager/src/reducer/validation_outcome.rs
+++ b/crates/legacy/download-manager/src/reducer/validation_outcome.rs
@@ -1,8 +1,6 @@
-use std::path::Path;
-
use crate::{
- CheckedFileState, DownloadError, FileCheck, FileState,
- integrity::{VerificationError, integrity_cache_matches, verify_file_integrity},
+ CheckedFileState, FileState,
+ crc_utils::{calculate_and_verify_crc, crc_cache_matches},
reducer::{Action, ActionPlan, DiskObservation},
};
@@ -12,67 +10,56 @@ pub struct ValidationOutcome {
pub action_plan: ActionPlan,
}
-pub async fn validate(observation: &DiskObservation) -> Result<ValidationOutcome, DownloadError> {
+pub async fn validate(observation: &DiskObservation) -> ValidationOutcome {
let length_matches = match (observation.destination_size, observation.expected_bytes) {
(Some(actual), Some(expected)) => actual == expected,
_ => true,
};
- let (checked, mut actions) = match (observation.destination_state, &observation.file_check) {
+ let (checked, mut actions) = match (observation.destination_state, observation.expected_crc.as_deref()) {
(FileState::Missing, _) => (CheckedFileState::Missing, Vec::new()),
(FileState::Exists, _) if !length_matches => (CheckedFileState::Invalid, Vec::new()),
- (FileState::Exists, FileCheck::None) => (CheckedFileState::Valid, Vec::new()),
- (FileState::Exists, file_check) => validate_integrity_with_cache(observation, file_check).await?,
+ (FileState::Exists, None) => (CheckedFileState::Valid, Vec::new()),
+ (FileState::Exists, Some(expected_crc)) => validate_crc_with_cache(observation, expected_crc).await,
};
- if checked == CheckedFileState::Missing && observation.integrity_state == FileState::Exists {
- actions.push(Action::DeleteIntegrityCache {
- path: observation.integrity_path.clone(),
+ if checked == CheckedFileState::Missing
+ && observation.crc_state == FileState::Exists
+ && let Some(path) = observation.crc_path.clone()
+ {
+ actions.push(Action::DeleteCrcCache {
+ path,
});
}
- Ok(ValidationOutcome {
+ let action_plan = ActionPlan::from_ordered_actions(actions);
+
+ ValidationOutcome {
checked,
- action_plan: ActionPlan::from_ordered_actions(actions),
- })
+ action_plan,
+ }
}
-async fn validate_integrity_with_cache(
+async fn validate_crc_with_cache(
observation: &DiskObservation,
- file_check: &FileCheck,
-) -> Result<(CheckedFileState, Vec<Action>), DownloadError> {
- if observation.integrity_state == FileState::Exists
- && integrity_cache_matches(&observation.destination_path, file_check).await
+ expected_crc: &str,
+) -> (CheckedFileState, Vec<Action>) {
+ if observation.crc_state == FileState::Exists
+ && crc_cache_matches(&observation.destination_path, expected_crc).await
{
- return Ok((CheckedFileState::Valid, Vec::new()));
+ return (CheckedFileState::Valid, Vec::new());
}
- match verify_file_integrity(&observation.destination_path, file_check).await {
- Ok(true) => Ok((
+ let crc_check = calculate_and_verify_crc(&observation.destination_path, expected_crc).await.ok();
+
+ match crc_check {
+ Some(true) => (
CheckedFileState::Valid,
- vec![Action::SaveIntegrityCache {
+ vec![Action::SaveCrcCache {
destination: observation.destination_path.clone(),
- file_check: file_check.clone(),
+ crc: expected_crc.to_string(),
}],
- )),
- Ok(false) => Ok((CheckedFileState::Invalid, Vec::new())),
- Err(error) => Err(map_verification_error(&observation.destination_path, error)),
- }
-}
-
-fn map_verification_error(
- path: &Path,
- error: VerificationError,
-) -> DownloadError {
- match error {
- VerificationError::InvalidExpectedDigest {
- algorithm,
- } => DownloadError::InvalidDigest {
- algorithm,
- },
- VerificationError::Io(error) => DownloadError::IntegrityIo {
- path: path.display().to_string(),
- message: error.to_string(),
- },
+ ),
+ _ => (CheckedFileState::Invalid, Vec::new()),
}
}
diff --git a/crates/legacy/download-manager/src/traits/capabilities.rs b/crates/legacy/download-manager/src/traits/capabilities.rs
deleted file mode 100644
index 857cf2fd4..000000000
--- a/crates/legacy/download-manager/src/traits/capabilities.rs
+++ /dev/null
@@ -1,10 +0,0 @@
-use bitflags::bitflags;
-
-bitflags! {
- #[repr(transparent)]
- #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
- pub struct Capabilities: u32 {
- /// Resume data can retain a redirected CDN URL that may expire before reuse.
- const CACHES_REDIRECTED_URL_IN_RESUME_DATA = 1 << 0;
- }
-}
diff --git a/crates/legacy/download-manager/src/traits/download_backend.rs b/crates/legacy/download-manager/src/traits/download_backend.rs
index face76fd8..35de4e90c 100644
--- a/crates/legacy/download-manager/src/traits/download_backend.rs
+++ b/crates/legacy/download-manager/src/traits/download_backend.rs
@@ -2,13 +2,12 @@ use std::{fmt::Debug, future::Future, path::Path};
use kiban::{fs, maybe::MaybeSend};
-use crate::traits::{ActiveTask, BackendContext, Capabilities};
+use crate::traits::{ActiveTask, BackendContext};
pub trait DownloadBackend: Debug + Clone + Send + Sync + 'static {
type Context: BackendContext<Backend = Self>;
type ActiveTask: ActiveTask<Backend = Self>;
type Error: std::error::Error + Send + Sync + 'static;
- const CAPABILITIES: Capabilities = Capabilities::empty();
// Default = file size (correct for `.part`-style artifacts). Apple must override:
// `.resume_data` is a small metadata blob, not the downloaded bytes.
diff --git a/crates/legacy/download-manager/src/traits/download_config.rs b/crates/legacy/download-manager/src/traits/download_config.rs
index 311707fd9..041d50a7e 100644
--- a/crates/legacy/download-manager/src/traits/download_config.rs
+++ b/crates/legacy/download-manager/src/traits/download_config.rs
@@ -1,13 +1,14 @@
use std::path::PathBuf;
+use serde::{Deserialize, Serialize};
use uuid::Uuid;
-use crate::{DownloadId, FileCheck, HttpDownloadRequest};
+use crate::{DownloadId, FileCheck};
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)]
pub struct DownloadConfig {
pub download_id: DownloadId,
- pub request: HttpDownloadRequest,
+ pub source_url: String,
pub destination: PathBuf,
pub file_check: FileCheck,
pub expected_bytes: Option<u64>,
diff --git a/crates/legacy/download-manager/src/traits/mod.rs b/crates/legacy/download-manager/src/traits/mod.rs
index 67e2f6414..007ea3a41 100644
--- a/crates/legacy/download-manager/src/traits/mod.rs
+++ b/crates/legacy/download-manager/src/traits/mod.rs
@@ -4,7 +4,6 @@ mod active_task;
mod backend_context;
mod backend_event_sender;
mod cancel_outcome;
-mod capabilities;
mod download_backend;
mod download_config;
@@ -14,6 +13,5 @@ pub use active_task::ActiveTask;
pub use backend_context::BackendContext;
pub use backend_event_sender::BackendEventSender;
pub use cancel_outcome::CancelOutcome;
-pub use capabilities::Capabilities;
pub use download_backend::DownloadBackend;
pub use download_config::DownloadConfig;
diff --git a/crates/legacy/download-manager/tests/unit/backends/apple/test_context.rs b/crates/legacy/download-manager/tests/unit/backends/apple/test_context.rs
index 03dfa8194..47ecbc516 100644
--- a/crates/legacy/download-manager/tests/unit/backends/apple/test_context.rs
+++ b/crates/legacy/download-manager/tests/unit/backends/apple/test_context.rs
@@ -1,40 +1,21 @@
-use std::{
- error::Error,
- io::Error as IoError,
- path::PathBuf,
- sync::{
- Arc,
- atomic::{AtomicUsize, Ordering},
- },
- time::Duration,
-};
+use std::sync::Arc;
use kiban::rt::RuntimeHandle;
use mock_registry::{Behavior, MockRegistry};
-use tempfile::tempdir;
-use tokio::{
- fs::{read as tokio_read, write as tokio_write},
- sync::{
- Mutex as TokioMutex,
- mpsc::{Receiver as TokioMpscReceiver, channel as tokio_mpsc_channel},
- watch::channel as tokio_watch_channel,
- },
- time::{sleep as tokio_sleep, timeout as tokio_timeout},
+use tokio::sync::{
+ Mutex as TokioMutex,
+ mpsc::{Receiver as TokioMpscReceiver, channel as tokio_mpsc_channel},
+ watch::channel as tokio_watch_channel,
};
use uuid::Uuid;
-use wiremock::{
- Mock, MockServer, Request, ResponseTemplate,
- matchers::{header, method, path},
-};
use crate::{
- FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase, HttpDownloadRequest,
+ FileCheck,
backends::apple::AppleBackendContext,
- common::wait_for_phase,
compute_download_id,
file_download_task_actor::{BackendEvent, PendingProgressSlot},
lock_manager::DestinationLockLease,
- traits::{ActiveDownloadGeneration, ActiveTask, BackendContext, BackendEventSender, DownloadConfig},
+ traits::{ActiveDownloadGeneration, BackendContext, BackendEventSender, DownloadConfig},
};
fn backend_event_sender() -> (BackendEventSender, TokioMpscReceiver<BackendEvent>) {
@@ -47,102 +28,48 @@ fn backend_event_sender() -> (BackendEventSender, TokioMpscReceiver<BackendEvent
)
}
-fn download_config(
- request: impl Into<HttpDownloadRequest>,
- destination: PathBuf,
- expected_bytes: u64,
-) -> Arc<DownloadConfig> {
- Arc::new(DownloadConfig {
+#[tokio::test(flavor = "multi_thread")]
+async fn test_apple_resume_empty_resume_data_starts_fresh_download() -> Result<(), Box<dyn std::error::Error>> {
+ let registry = MockRegistry::start().await?;
+ let served_file = registry.file("config.json")?;
+ let temporary_directory = tempfile::tempdir()?;
+ let destination = temporary_directory.path().join(&served_file.file.name);
+ let resume_artifact_path = destination.with_extension("resume_data");
+ tokio::fs::write(&resume_artifact_path, b"").await?;
+
+ let context = AppleBackendContext::new(RuntimeHandle::current());
+ let config = Arc::new(DownloadConfig {
download_id: compute_download_id(&destination),
- request: request.into(),
- destination,
+ source_url: served_file.file.url.clone(),
+ destination: destination.clone(),
file_check: FileCheck::None,
- expected_bytes: Some(expected_bytes),
+ expected_bytes: Some(served_file.file.size as u64),
manager_id: "test-manager".to_string(),
manager_instance_id: Uuid::new_v4(),
- })
-}
-
-async fn destination_lease(config: &DownloadConfig) -> Result<DestinationLockLease, IoError> {
- DestinationLockLease::acquire_for_destination(&config.destination, &config.manager_id, config.manager_instance_id)
- .await
-}
-
-#[tokio::test(flavor = "multi_thread")]
-async fn apple_authenticated_redirect_strips_authorization_cross_origin() -> Result<(), Box<dyn Error>> {
- let bytes = b"private redirected file";
- let target = MockServer::start().await;
- Mock::given(method("GET"))
- .and(path("/cdn/model.bin"))
- .and(header("accept-encoding", "identity"))
- .respond_with(ResponseTemplate::new(200).set_body_bytes(bytes.as_slice()))
- .expect(1)
- .mount(&target)
- .await;
- let origin = MockServer::start().await;
- Mock::given(method("GET"))
- .and(path("/model.bin"))
- .and(header("authorization", "Bearer test-token"))
- .and(header("accept-encoding", "identity"))
- .respond_with(ResponseTemplate::new(302).insert_header("Location", format!("{}/cdn/model.bin", target.uri())))
- .expect(1)
- .mount(&origin)
- .await;
- let directory = tempdir()?;
- let destination = directory.path().join("model.bin");
- let token: Arc<str> = Arc::from("test-token");
- let config = download_config(
- HttpDownloadRequest::with_bearer_token(format!("{}/model.bin", origin.uri()), &token),
- destination.clone(),
- bytes.len() as u64,
- );
- let lease = destination_lease(&config).await?;
- let generation = ActiveDownloadGeneration::new(0);
- let (sender, mut receiver) = backend_event_sender();
- let context = AppleBackendContext::new(RuntimeHandle::current());
-
- let active_task = context.download(Arc::clone(&config), generation, sender, &lease).await?;
- let event = tokio_timeout(Duration::from_secs(5), receiver.recv())
- .await?
- .ok_or_else(|| IoError::other("backend event channel closed"))?;
-
- assert_eq!(event, BackendEvent::completed(generation));
- let target_requests = target.received_requests().await.expect("requests should be available");
- assert_eq!(target_requests.len(), 1);
- assert!(target_requests[0].headers.get("authorization").is_none());
- assert_eq!(tokio_read(destination).await?, bytes);
- drop(active_task);
- lease.release().await?;
- Ok(())
-}
-
-#[tokio::test(flavor = "multi_thread")]
-async fn apple_http_error_body_is_not_installed() -> Result<(), Box<dyn Error>> {
- let error_body = b"denied";
- let server = MockServer::start().await;
- Mock::given(method("GET"))
- .and(path("/model.bin"))
- .respond_with(ResponseTemplate::new(404).set_body_bytes(error_body.as_slice()))
- .expect(1)
- .mount(&server)
- .await;
-
- let temporary_directory = tempdir()?;
- let destination = temporary_directory.path().join("model.bin");
- let context = AppleBackendContext::new(RuntimeHandle::current());
- let config = download_config(format!("{}/model.bin", server.uri()), destination.clone(), error_body.len() as u64);
- let destination_lease = destination_lease(&config).await?;
+ });
+ let destination_lease = DestinationLockLease::acquire_for_destination(
+ &config.destination,
+ &config.manager_id,
+ config.manager_instance_id,
+ )
+ .await?;
let generation = ActiveDownloadGeneration::new(0);
let (backend_event_sender, mut backend_event_receiver) = backend_event_sender();
- let active_task =
- context.download(Arc::clone(&config), generation, backend_event_sender, &destination_lease).await?;
- let event = tokio_timeout(Duration::from_secs(5), backend_event_receiver.recv())
+ let active_task = context
+ .resume(Arc::clone(&config), generation, &resume_artifact_path, backend_event_sender, &destination_lease)
+ .await?;
+ let event = tokio::time::timeout(std::time::Duration::from_secs(5), backend_event_receiver.recv())
.await?
- .ok_or_else(|| IoError::other("backend event channel closed"))?;
+ .ok_or_else(|| {
+ std::io::Error::new(
+ std::io::ErrorKind::UnexpectedEof,
+ "backend event channel closed before empty resume-data fallback completed",
+ )
+ })?;
- assert_eq!(event, BackendEvent::error(generation, "download failed with HTTP status 404".to_string()));
- assert!(!destination.exists());
+ assert_eq!(event, BackendEvent::completed(generation));
+ assert_eq!(tokio::fs::read(&destination).await?, served_file.bytes.to_vec());
drop(active_task);
destination_lease.release().await?;
@@ -150,138 +77,75 @@ async fn apple_http_error_body_is_not_installed() -> Result<(), Box<dyn Error>>
}
#[tokio::test(flavor = "multi_thread")]
-async fn apple_download_can_pause_and_resume() -> Result<(), Box<dyn Error>> {
+async fn dropping_apple_active_task_unregisters_event_sink() -> Result<(), Box<dyn std::error::Error>> {
let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;
- let served_file = registry.file("tokenizer.json")?;
- let temporary_directory = tempdir()?;
+ let served_file = registry.file("config.json")?;
+ let temporary_directory = tempfile::tempdir()?;
let destination = temporary_directory.path().join(&served_file.file.name);
let context = AppleBackendContext::new(RuntimeHandle::current());
- let config = download_config(served_file.file.url.clone(), destination.clone(), served_file.file.size as u64);
- let destination_lease = destination_lease(&config).await?;
- let initial_generation = ActiveDownloadGeneration::new(0);
- let (initial_event_sender, _initial_event_receiver) = backend_event_sender();
- let active_task =
- context.download(Arc::clone(&config), initial_generation, initial_event_sender, &destination_lease).await?;
-
- tokio_sleep(Duration::from_millis(100)).await;
- let resume_artifact_path = active_task.pause(&destination).await?;
- assert!(!tokio_read(&resume_artifact_path).await?.is_empty());
+ let download_id = compute_download_id(&destination);
+ let config = Arc::new(DownloadConfig {
+ download_id,
+ source_url: served_file.file.url.clone(),
+ destination,
+ file_check: FileCheck::None,
+ expected_bytes: Some(served_file.file.size as u64),
+ manager_id: "test-manager".to_string(),
+ manager_instance_id: Uuid::new_v4(),
+ });
+ let destination_lease = DestinationLockLease::acquire_for_destination(
+ &config.destination,
+ &config.manager_id,
+ config.manager_instance_id,
+ )
+ .await?;
- let resumed_generation = ActiveDownloadGeneration::new(1);
- let (resumed_event_sender, mut resumed_event_receiver) = backend_event_sender();
- let resumed_task = context
- .resume(
- Arc::clone(&config),
- resumed_generation,
- &resume_artifact_path,
- resumed_event_sender,
- &destination_lease,
- )
+ let (backend_event_sender, _backend_event_receiver) = backend_event_sender();
+ let active_task = context
+ .download(Arc::clone(&config), ActiveDownloadGeneration::new(0), backend_event_sender, &destination_lease)
.await?;
- let event = tokio_timeout(Duration::from_secs(10), resumed_event_receiver.recv())
- .await?
- .ok_or_else(|| IoError::other("backend event channel closed before resumed download completed"))?;
-
- assert_eq!(event, BackendEvent::completed(resumed_generation));
- assert_eq!(tokio_read(&destination).await?, served_file.bytes.to_vec());
+ assert_eq!(
+ context.event_sink_count_for_download(download_id),
+ 1,
+ "precondition: exactly one event sink should be registered for download_id after starting download",
+ );
- drop(resumed_task);
+ drop(active_task);
destination_lease.release().await?;
+
+ assert_eq!(
+ context.event_sink_count_for_download(download_id),
+ 0,
+ "dropping AppleActiveTask must unregister its event sink",
+ );
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
-async fn apple_failed_resume_restarts_from_source() -> Result<(), Box<dyn Error>> {
+async fn registry_distinguishes_generations_for_same_download_id() -> Result<(), Box<dyn std::error::Error>> {
let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;
- let served_file = registry.file("tokenizer.json")?;
- let temporary_directory = tempdir()?;
+ let served_file = registry.file("config.json")?;
+ let temporary_directory = tempfile::tempdir()?;
let destination = temporary_directory.path().join(&served_file.file.name);
- let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Apple, RuntimeHandle::current()).await?;
- let task = manager
- .file_download_task(
- (&served_file.file.url).into(),
- &destination,
- FileCheck::None,
- Some(served_file.file.size as u64),
- )
- .await?;
-
- task.download().await?;
- tokio_sleep(Duration::from_millis(100)).await;
- task.pause().await?;
- tokio_write(destination.with_added_extension("resume_data"), b"invalid resume data").await?;
-
- let mut progress = task.progress().await?;
- task.download().await?;
- let state = wait_for_phase(&task, &mut progress, |phase| matches!(phase, FileDownloadPhase::Downloaded)).await;
-
- assert_eq!(state.downloaded_bytes, served_file.file.size as u64);
- assert_eq!(tokio_read(destination).await?, served_file.bytes.to_vec());
- Ok(())
-}
-
-#[tokio::test(flavor = "multi_thread")]
-async fn apple_authenticated_pause_restarts_once_without_resume_data() -> Result<(), Box<dyn Error>> {
- let bytes = b"private model";
- let attempts = Arc::new(AtomicUsize::new(0));
- let server = MockServer::start().await;
- Mock::given(method("GET"))
- .and(path("/model.bin"))
- .and(header("authorization", "Bearer test-token"))
- .respond_with({
- let attempts = Arc::clone(&attempts);
- move |_: &Request| {
- if attempts.fetch_add(1, Ordering::Relaxed) == 0 {
- ResponseTemplate::new(200).set_body_bytes(bytes.as_slice()).set_delay(Duration::from_secs(1))
- } else {
- ResponseTemplate::new(500)
- }
- }
- })
- .mount(&server)
- .await;
- let directory = tempdir()?;
- let destination = directory.path().join("model.bin");
- let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Apple, RuntimeHandle::current()).await?;
- let token: Arc<str> = Arc::from("test-token");
- let request = HttpDownloadRequest::with_bearer_token(format!("{}/model.bin", server.uri()), &token);
- let task = manager.file_download_task(request, &destination, FileCheck::None, Some(bytes.len() as u64)).await?;
-
- task.download().await?;
- tokio_timeout(Duration::from_secs(5), async {
- while attempts.load(Ordering::Relaxed) == 0 {
- tokio_sleep(Duration::from_millis(10)).await;
- }
- })
- .await?;
- task.pause().await?;
-
- assert!(tokio_read(destination.with_added_extension("resume_data")).await?.is_empty());
- let mut progress = task.progress().await?;
- task.download().await?;
- wait_for_phase(&task, &mut progress, |phase| matches!(phase, FileDownloadPhase::Error(_))).await;
-
- assert_eq!(attempts.load(Ordering::Relaxed), 2, "authenticated fresh resume must not be retried");
- assert!(!destination.exists());
- Ok(())
-}
-
-#[tokio::test(flavor = "multi_thread")]
-async fn registry_distinguishes_generations_for_same_download_id() -> Result<(), Box<dyn Error>> {
- let bytes = b"model";
- let server = MockServer::start().await;
- Mock::given(method("GET"))
- .and(path("/model.bin"))
- .respond_with(ResponseTemplate::new(200).set_body_bytes(bytes.as_slice()).set_delay(Duration::from_secs(5)))
- .mount(&server)
- .await;
- let temporary_directory = tempdir()?;
- let destination = temporary_directory.path().join("model.bin");
let download_id = compute_download_id(&destination);
+
let context = AppleBackendContext::new(RuntimeHandle::current());
- let config = download_config(format!("{}/model.bin", server.uri()), destination, bytes.len() as u64);
- let destination_lease = destination_lease(&config).await?;
+ let config = Arc::new(DownloadConfig {
+ download_id,
+ source_url: served_file.file.url.clone(),
+ destination,
+ file_check: FileCheck::None,
+ expected_bytes: Some(served_file.file.size as u64),
+ manager_id: "test-manager".to_string(),
+ manager_instance_id: Uuid::new_v4(),
+ });
+ let destination_lease = DestinationLockLease::acquire_for_destination(
+ &config.destination,
+ &config.manager_id,
+ config.manager_instance_id,
+ )
+ .await?;
let (backend_event_sender_first, _backend_event_receiver_first) = backend_event_sender();
let first_task = context
@@ -308,28 +172,35 @@ async fn registry_distinguishes_generations_for_same_download_id() -> Result<(),
assert_ne!(both_keys[0], both_keys[1], "the two generations must have distinct task identifiers");
drop(first_task);
- assert_eq!(context.event_sink_task_identifiers_for_download(download_id).len(), 1);
drop(second_task);
- assert!(context.event_sink_task_identifiers_for_download(download_id).is_empty());
destination_lease.release().await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
-async fn claim_cancels_task_with_different_credentials() -> Result<(), Box<dyn Error>> {
+async fn claim_cancels_mismatched_task() -> Result<(), Box<dyn std::error::Error>> {
let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;
let served_file = registry.file("tokenizer.json")?;
- let temporary_directory = tempdir()?;
+ let temporary_directory = tempfile::tempdir()?;
let destination = temporary_directory.path().join(&served_file.file.name);
+ let download_id = compute_download_id(&destination);
let context = AppleBackendContext::new(RuntimeHandle::current());
- let original_token: Arc<str> = Arc::from("original-token");
- let original_config = download_config(
- HttpDownloadRequest::with_bearer_token(served_file.file.url.clone(), &original_token),
- destination,
- served_file.file.size as u64,
- );
- let destination_lease = destination_lease(&original_config).await?;
+ let original_config = Arc::new(DownloadConfig {
+ download_id,
+ source_url: served_file.file.url.clone(),
+ destination: destination.clone(),
+ file_check: FileCheck::None,
+ expected_bytes: Some(served_file.file.size as u64),
+ manager_id: "test-manager".to_string(),
+ manager_instance_id: Uuid::new_v4(),
+ });
+ let destination_lease = DestinationLockLease::acquire_for_destination(
+ &original_config.destination,
+ &original_config.manager_id,
+ original_config.manager_instance_id,
+ )
+ .await?;
let (backend_event_sender, _backend_event_receiver) = backend_event_sender();
let original_task = context
@@ -346,9 +217,8 @@ async fn claim_cancels_task_with_different_credentials() -> Result<(), Box<dyn E
"precondition: original task should be visible before the mismatched claim",
);
- let replacement_token: Arc<str> = Arc::from("replacement-token");
let mismatched_config = DownloadConfig {
- request: HttpDownloadRequest::with_bearer_token(served_file.file.url.clone(), &replacement_token),
+ source_url: "http://example.invalid/different-url".to_string(),
..(*original_config).clone()
};
assert!(
@@ -363,7 +233,7 @@ async fn claim_cancels_task_with_different_credentials() -> Result<(), Box<dyn E
cancelled = true;
break;
}
- tokio_sleep(Duration::from_millis(20)).await;
+ tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
assert!(
diff --git a/crates/legacy/download-manager/tests/unit/backends/common/test_actor_task_drop.rs b/crates/legacy/download-manager/tests/unit/backends/common/test_actor_task_drop.rs
index 1115497ca..ef2deb6a3 100644
--- a/crates/legacy/download-manager/tests/unit/backends/common/test_actor_task_drop.rs
+++ b/crates/legacy/download-manager/tests/unit/backends/common/test_actor_task_drop.rs
@@ -1,10 +1,8 @@
-use std::{error::Error, sync::Arc, time::Duration};
+use std::{path::PathBuf, sync::Arc};
use download_manager::{DownloadError, FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase};
use kiban::rt::RuntimeHandle;
use rstest::rstest;
-use tempfile::tempdir;
-use tokio::{fs::write as tokio_write, time::sleep as tokio_sleep};
use crate::common::{Behavior, MockRegistry, wait_for_phase};
@@ -15,15 +13,15 @@ use crate::common::{Behavior, MockRegistry, wait_for_phase};
async fn remove_paused_task_deletes_resume_artifact(
#[case] download_manager_type: FileDownloadManagerType,
#[case] resume_artifact_extension: &str,
-) -> Result<(), Box<dyn Error>> {
- let temporary_directory = tempdir()?;
+) -> Result<(), Box<dyn std::error::Error>> {
+ let temporary_directory = tempfile::tempdir()?;
let destination = temporary_directory.path().join("model.bin");
- let resume_artifact = destination.with_added_extension(resume_artifact_extension);
- tokio_write(&resume_artifact, b"partial").await?;
+ let resume_artifact = destination.with_extension(resume_artifact_extension);
+ tokio::fs::write(&resume_artifact, b"partial").await?;
let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await?;
let task = manager
- .file_download_task("http://example.invalid/model.bin".into(), &destination, FileCheck::None, Some(100))
+ .file_download_task("http://example.invalid/model.bin", &destination, FileCheck::None, Some(100))
.await?;
assert!(matches!(task.state().await.phase, FileDownloadPhase::Paused));
@@ -42,17 +40,17 @@ async fn remove_paused_task_deletes_resume_artifact(
async fn dropping_active_download_cancels_backend_before_releasing_lock(
#[case] download_manager_type: FileDownloadManagerType,
#[case] resume_artifact_extension: &str,
-) -> Result<(), Box<dyn Error>> {
+) -> Result<(), Box<dyn std::error::Error>> {
let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;
let tokenizer = registry.file("tokenizer.json")?;
- let temp_dir = tempdir()?;
+ let temp_dir = tempfile::tempdir()?;
let destination = temp_dir.path().join(&tokenizer.file.name);
- let lock_path = destination.with_added_extension("lock");
- let resume_artifact = destination.with_added_extension(resume_artifact_extension);
+ let lock_path = PathBuf::from(format!("{}.lock", destination.display()));
+ let resume_artifact = destination.with_extension(resume_artifact_extension);
let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();
let task = manager
.file_download_task(
- (&tokenizer.file.url).into(),
+ &tokenizer.file.url,
&destination,
FileCheck::CRC(tokenizer.crc32c()?),
Some(tokenizer.file.size as u64),
@@ -69,7 +67,7 @@ async fn dropping_active_download_cancels_backend_before_releasing_lock(
assert!(matches!(task.download().await, Err(DownloadError::TaskStopped)));
let replacement_task = manager
.file_download_task(
- (&tokenizer.file.url).into(),
+ &tokenizer.file.url,
&destination,
FileCheck::CRC(tokenizer.crc32c()?),
Some(tokenizer.file.size as u64),
@@ -85,13 +83,13 @@ async fn dropping_active_download_cancels_backend_before_releasing_lock(
lock_released = true;
break;
}
- tokio_sleep(Duration::from_millis(20)).await;
+ tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
assert!(lock_released);
assert!(!resume_artifact.exists());
let destination_size_at_release = destination.metadata().map(|m| m.len()).unwrap_or(0);
- tokio_sleep(Duration::from_millis(200)).await;
+ tokio::time::sleep(std::time::Duration::from_millis(200)).await;
let destination_size_after = destination.metadata().map(|m| m.len()).unwrap_or(0);
assert_eq!(destination_size_at_release, destination_size_after);
Ok(())
diff --git a/crates/legacy/download-manager/tests/unit/backends/common/test_corrupt_body.rs b/crates/legacy/download-manager/tests/unit/backends/common/test_corrupt_body.rs
index 0f86fde99..a036eb5ab 100644
--- a/crates/legacy/download-manager/tests/unit/backends/common/test_corrupt_body.rs
+++ b/crates/legacy/download-manager/tests/unit/backends/common/test_corrupt_body.rs
@@ -18,7 +18,7 @@ async fn test_corrupt_body_fails_crc(
let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();
let task = manager
.file_download_task(
- (&tokenizer.file.url).into(),
+ &tokenizer.file.url,
&destination,
FileCheck::CRC(tokenizer.crc32c()?),
Some(tokenizer.file.size as u64),
@@ -48,7 +48,7 @@ async fn test_corrupt_body_cancel_resets_error_state(
let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await?;
let task = manager
.file_download_task(
- (&tokenizer.file.url).into(),
+ &tokenizer.file.url,
&destination,
FileCheck::CRC(tokenizer.crc32c()?),
Some(tokenizer.file.size as u64),
diff --git a/crates/legacy/download-manager/tests/unit/backends/common/test_lifecycle.rs b/crates/legacy/download-manager/tests/unit/backends/common/test_lifecycle.rs
index f834351bc..8edeff37c 100644
--- a/crates/legacy/download-manager/tests/unit/backends/common/test_lifecycle.rs
+++ b/crates/legacy/download-manager/tests/unit/backends/common/test_lifecycle.rs
@@ -1,10 +1,6 @@
use download_manager::{FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase};
-use hex::encode as hex_encode;
use kiban::rt::RuntimeHandle;
use rstest::rstest;
-use sha2::{Digest, Sha256};
-use tempfile::tempdir;
-use tokio::fs::read as tokio_read;
use crate::common::{MockRegistry, wait_for_phase};
@@ -17,13 +13,13 @@ async fn test_download_fresh_completes(
) -> Result<(), Box<dyn std::error::Error>> {
let registry = MockRegistry::start().await?;
let tokenizer = registry.file("tokenizer.json")?;
- let temp_dir = tempdir().unwrap();
+ let temp_dir = tempfile::tempdir().unwrap();
let destination = temp_dir.path().join(&tokenizer.file.name);
let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();
let task = manager
.file_download_task(
- (&tokenizer.file.url).into(),
+ &tokenizer.file.url,
&destination,
FileCheck::CRC(tokenizer.crc32c()?),
Some(tokenizer.file.size as u64),
@@ -37,32 +33,7 @@ async fn test_download_fresh_completes(
assert_eq!(state.downloaded_bytes, tokenizer.file.size as u64);
assert_eq!(state.total_bytes, tokenizer.file.size as u64);
- assert_eq!(tokio_read(&destination).await.unwrap(), tokenizer.bytes.to_vec());
- assert!(destination.with_added_extension("integrity").is_file());
- Ok(())
-}
-
-#[tokio::test(flavor = "multi_thread")]
-async fn test_sha256_download_completes() -> Result<(), Box<dyn std::error::Error>> {
- let registry = MockRegistry::start().await?;
- let tokenizer = registry.file("tokenizer.json")?;
- let temporary_directory = tempdir()?;
- let destination = temporary_directory.path().join("tokenizer.json");
- let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Universal, RuntimeHandle::current()).await?;
- let check = FileCheck::Sha256(hex_encode(Sha256::digest(&tokenizer.bytes)));
- let task = manager
- .file_download_task(
- (&tokenizer.file.url).into(),
- &destination,
- check,
- Some(tokenizer.file.size as u64),
- )
- .await?;
- let mut progress = task.progress().await?;
-
- task.download().await?;
- wait_for_phase(&task, &mut progress, |phase| matches!(phase, FileDownloadPhase::Downloaded)).await;
-
- assert_eq!(tokio_read(destination).await?, tokenizer.bytes.as_ref());
+ assert_eq!(tokio::fs::read(&destination).await.unwrap(), tokenizer.bytes.to_vec());
+ assert!(std::path::PathBuf::from(format!("{}.crc", destination.display())).is_file());
Ok(())
}
diff --git a/crates/legacy/download-manager/tests/unit/backends/common/test_manager_startup.rs b/crates/legacy/download-manager/tests/unit/backends/common/test_manager_startup.rs
index e5beb4d5d..ae2c29040 100644
--- a/crates/legacy/download-manager/tests/unit/backends/common/test_manager_startup.rs
+++ b/crates/legacy/download-manager/tests/unit/backends/common/test_manager_startup.rs
@@ -1,27 +1,21 @@
-use std::{error::Error, path::Path, process::id, sync::Arc};
-
use chrono::Utc;
-use download_manager::{
- DownloadError, FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase, FileDownloadTask,
-};
+use download_manager::{FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase};
use kiban::rt::RuntimeHandle;
-use serde_json::{json, to_vec};
use tempfile::tempdir;
-use tokio::fs::{create_dir as tokio_create_dir, write as tokio_write};
use crate::common::MockRegistry;
#[tokio::test(flavor = "multi_thread")]
-async fn test_manager_startup_valid_existing_file_is_downloaded() -> Result<(), Box<dyn Error>> {
+async fn test_manager_startup_valid_existing_file_is_downloaded() -> Result<(), Box<dyn std::error::Error>> {
let registry = MockRegistry::start().await?;
let served_file = registry.file("config.json")?;
let expected_crc = served_file.crc32c()?;
let temporary_directory = tempdir()?;
let destination = temporary_directory.path().join(&served_file.file.name);
- let integrity_path = destination.with_added_extension("integrity");
- let resume_artifact = destination.with_added_extension("part");
- tokio_write(&destination, served_file.bytes.as_ref()).await?;
- tokio_write(&resume_artifact, b"partial").await?;
+ let crc_path = std::path::PathBuf::from(format!("{}.crc", destination.display()));
+ let resume_artifact = destination.with_extension("part");
+ tokio::fs::write(&destination, served_file.bytes.as_ref()).await?;
+ tokio::fs::write(&resume_artifact, b"partial").await?;
let task = manager_task(
&served_file.file.url,
@@ -33,26 +27,57 @@ async fn test_manager_startup_valid_existing_file_is_downloaded() -> Result<(),
assert_eq!(task.state().await.phase, FileDownloadPhase::Downloaded);
assert!(!resume_artifact.exists());
- assert!(integrity_path.is_file());
+ let crc_receipt: serde_json::Value = serde_json::from_str(&tokio::fs::read_to_string(crc_path).await?)?;
+ assert_eq!(crc_receipt["version"].as_u64(), Some(1));
+ assert_eq!(crc_receipt["crc"].as_str(), Some(expected_crc.as_str()));
+ assert_eq!(crc_receipt["file_size"].as_u64(), Some(served_file.file.size as u64));
+ assert!(crc_receipt["modified_unix_seconds"].as_u64().is_some());
+ assert!(crc_receipt["modified_nanos"].as_u64().is_some());
+ Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn test_manager_startup_rejects_legacy_crc_cache_after_file_changes() -> Result<(), Box<dyn std::error::Error>> {
+ let registry = MockRegistry::start().await?;
+ let served_file = registry.file("config.json")?;
+ let expected_crc = served_file.crc32c()?;
+ let temporary_directory = tempdir()?;
+ let destination = temporary_directory.path().join(&served_file.file.name);
+ let crc_path = std::path::PathBuf::from(format!("{}.crc", destination.display()));
+ let mut stale_destination_bytes = served_file.bytes.to_vec();
+ stale_destination_bytes[0] = stale_destination_bytes[0].wrapping_add(1);
+ tokio::fs::write(&destination, stale_destination_bytes).await?;
+ tokio::fs::write(&crc_path, &expected_crc).await?;
+
+ let task = manager_task(
+ &served_file.file.url,
+ &destination,
+ FileCheck::CRC(expected_crc),
+ Some(served_file.file.size as u64),
+ )
+ .await?;
+
+ assert_eq!(task.state().await.phase, FileDownloadPhase::NotDownloaded);
+ assert!(!destination.exists());
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
-async fn test_manager_startup_revalidates_stale_integrity_receipt() -> Result<(), Box<dyn Error>> {
+async fn test_manager_startup_revalidates_stale_metadata_crc_cache() -> Result<(), Box<dyn std::error::Error>> {
let registry = MockRegistry::start().await?;
let served_file = registry.file("config.json")?;
let expected_crc = served_file.crc32c()?;
let temporary_directory = tempdir()?;
let destination = temporary_directory.path().join(&served_file.file.name);
- let integrity_path = destination.with_added_extension("integrity");
+ let crc_path = std::path::PathBuf::from(format!("{}.crc", destination.display()));
let mut stale_destination_bytes = served_file.bytes.to_vec();
stale_destination_bytes[0] = stale_destination_bytes[0].wrapping_add(1);
- tokio_write(&destination, stale_destination_bytes).await?;
- tokio_write(
- &integrity_path,
- to_vec(&json!({
+ tokio::fs::write(&destination, stale_destination_bytes).await?;
+ tokio::fs::write(
+ &crc_path,
+ serde_json::to_vec(&serde_json::json!({
"version": 1,
- "file_check": { "CRC": expected_crc.clone() },
+ "crc": expected_crc.clone(),
"file_size": served_file.file.size,
"modified_unix_seconds": 0,
"modified_nanos": 0,
@@ -74,12 +99,12 @@ async fn test_manager_startup_revalidates_stale_integrity_receipt() -> Result<()
}
#[tokio::test(flavor = "multi_thread")]
-async fn test_manager_startup_directory_destination_is_not_downloaded() -> Result<(), Box<dyn Error>> {
+async fn test_manager_startup_directory_destination_is_not_downloaded() -> Result<(), Box<dyn std::error::Error>> {
let registry = MockRegistry::start().await?;
let served_file = registry.file("config.json")?;
let temporary_directory = tempdir()?;
let destination = temporary_directory.path().join(&served_file.file.name);
- tokio_create_dir(&destination).await?;
+ tokio::fs::create_dir(&destination).await?;
let task = manager_task(&served_file.file.url, &destination, FileCheck::None, None).await?;
@@ -88,23 +113,23 @@ async fn test_manager_startup_directory_destination_is_not_downloaded() -> Resul
}
#[tokio::test(flavor = "multi_thread")]
-async fn test_manager_startup_foreign_lock_preserves_files() -> Result<(), Box<dyn Error>> {
+async fn test_manager_startup_foreign_lock_preserves_files() -> Result<(), Box<dyn std::error::Error>> {
let registry = MockRegistry::start().await?;
let served_file = registry.file("config.json")?;
let temporary_directory = tempdir()?;
let destination = temporary_directory.path().join(&served_file.file.name);
- let integrity_path = destination.with_added_extension("integrity");
- let resume_artifact = destination.with_added_extension("part");
- let lock_path = destination.with_added_extension("lock");
- tokio_write(&destination, b"corrupt").await?;
- tokio_write(&integrity_path, served_file.crc32c()?).await?;
- tokio_write(&resume_artifact, b"partial").await?;
- tokio_write(
+ let crc_path = std::path::PathBuf::from(format!("{}.crc", destination.display()));
+ let resume_artifact = destination.with_extension("part");
+ let lock_path = std::path::PathBuf::from(format!("{}.lock", destination.display()));
+ tokio::fs::write(&destination, b"corrupt").await?;
+ tokio::fs::write(&crc_path, served_file.crc32c()?).await?;
+ tokio::fs::write(&resume_artifact, b"partial").await?;
+ tokio::fs::write(
&lock_path,
- to_vec(&json!({
+ serde_json::to_vec(&serde_json::json!({
"manager_id": "foreign-manager",
"acquired_at": Utc::now(),
- "process_id": id(),
+ "process_id": std::process::id(),
}))?,
)
.await?;
@@ -119,17 +144,17 @@ async fn test_manager_startup_foreign_lock_preserves_files() -> Result<(), Box<d
assert!(matches!(task.state().await.phase, FileDownloadPhase::LockedByOther(_)));
assert!(destination.exists());
- assert!(integrity_path.exists());
+ assert!(crc_path.exists());
assert!(resume_artifact.exists());
Ok(())
}
async fn manager_task(
source_url: &str,
- destination: &Path,
+ destination: &std::path::Path,
file_check: FileCheck,
expected_bytes: Option<u64>,
-) -> Result<Arc<dyn FileDownloadTask>, DownloadError> {
+) -> Result<std::sync::Arc<dyn download_manager::FileDownloadTask>, download_manager::DownloadError> {
let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Universal, RuntimeHandle::current()).await?;
- manager.file_download_task(source_url.into(), destination, file_check, expected_bytes).await
+ manager.file_download_task(source_url, destination, file_check, expected_bytes).await
}
diff --git a/crates/legacy/download-manager/tests/unit/backends/common/test_state_and_lock_regression.rs b/crates/legacy/download-manager/tests/unit/backends/common/test_state_and_lock_regression.rs
index c4a0d23be..d5d486054 100644
--- a/crates/legacy/download-manager/tests/unit/backends/common/test_state_and_lock_regression.rs
+++ b/crates/legacy/download-manager/tests/unit/backends/common/test_state_and_lock_regression.rs
@@ -1,9 +1,8 @@
-use std::{error::Error, sync::Arc};
+use std::sync::Arc;
use download_manager::{DownloadError, FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase};
use kiban::rt::RuntimeHandle;
use rstest::rstest;
-use tempfile::tempdir;
use crate::common::{Behavior, MockRegistry, wait_for_phase};
@@ -13,22 +12,22 @@ use crate::common::{Behavior, MockRegistry, wait_for_phase};
#[tokio::test(flavor = "multi_thread")]
async fn test_concurrent_task_creation_returns_same_task(
#[case] download_manager_type: FileDownloadManagerType
-) -> Result<(), Box<dyn Error>> {
+) -> Result<(), Box<dyn std::error::Error>> {
let registry = MockRegistry::start().await?;
let tokenizer = registry.file("tokenizer.json")?;
- let temp_dir = tempdir().unwrap();
+ let temp_dir = tempfile::tempdir().unwrap();
let destination = temp_dir.path().join(&tokenizer.file.name);
let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();
let (first, second) = tokio::join!(
manager.file_download_task(
- (&tokenizer.file.url).into(),
+ &tokenizer.file.url,
&destination,
FileCheck::CRC(tokenizer.crc32c()?),
Some(tokenizer.file.size as u64),
),
manager.file_download_task(
- (&tokenizer.file.url).into(),
+ &tokenizer.file.url,
&destination,
FileCheck::CRC(tokenizer.crc32c()?),
Some(tokenizer.file.size as u64),
@@ -41,41 +40,73 @@ async fn test_concurrent_task_creation_returns_same_task(
Ok(())
}
+#[rstest]
+#[case::universal(FileDownloadManagerType::Universal)]
+#[cfg_attr(target_vendor = "apple", case::apple(FileDownloadManagerType::Apple))]
#[tokio::test(flavor = "multi_thread")]
-async fn conflicting_task_configuration_is_rejected() -> Result<(), Box<dyn Error>> {
+async fn different_urls_targeting_same_destination_are_rejected(
+ #[case] download_manager_type: FileDownloadManagerType
+) -> Result<(), Box<dyn std::error::Error>> {
let registry = MockRegistry::start().await?;
let tokenizer = registry.file("tokenizer.json")?;
- let temp_dir = tempdir().unwrap();
+ let temp_dir = tempfile::tempdir().unwrap();
let destination = temp_dir.path().join(&tokenizer.file.name);
- let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Universal, RuntimeHandle::current()).await?;
+ let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();
- manager
+ let _first = manager
.file_download_task(
- (&tokenizer.file.url).into(),
+ &tokenizer.file.url,
&destination,
FileCheck::CRC(tokenizer.crc32c()?),
Some(tokenizer.file.size as u64),
)
- .await?;
- let different_url = manager
+ .await
+ .unwrap();
+ let second = manager
.file_download_task(
- "http://example.invalid/different-url".into(),
+ "http://example.invalid/different-url",
&destination,
FileCheck::None,
Some(tokenizer.file.size as u64),
)
.await;
- let different_size = manager
+
+ assert!(matches!(second, Err(DownloadError::ConflictingConfig(_))));
+ Ok(())
+}
+
+#[rstest]
+#[case::universal(FileDownloadManagerType::Universal)]
+#[cfg_attr(target_vendor = "apple", case::apple(FileDownloadManagerType::Apple))]
+#[tokio::test(flavor = "multi_thread")]
+async fn different_expected_bytes_targeting_same_destination_are_rejected(
+ #[case] download_manager_type: FileDownloadManagerType
+) -> Result<(), Box<dyn std::error::Error>> {
+ let registry = MockRegistry::start().await?;
+ let tokenizer = registry.file("tokenizer.json")?;
+ let temp_dir = tempfile::tempdir().unwrap();
+ let destination = temp_dir.path().join(&tokenizer.file.name);
+ let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();
+
+ let _first = manager
+ .file_download_task(
+ &tokenizer.file.url,
+ &destination,
+ FileCheck::CRC(tokenizer.crc32c()?),
+ Some(tokenizer.file.size as u64),
+ )
+ .await
+ .unwrap();
+ let second = manager
.file_download_task(
- (&tokenizer.file.url).into(),
+ &tokenizer.file.url,
&destination,
FileCheck::CRC(tokenizer.crc32c()?),
Some(tokenizer.file.size as u64 + 1),
)
.await;
- assert!(matches!(different_url, Err(DownloadError::ConflictingConfig(_))));
- assert!(matches!(different_size, Err(DownloadError::ConflictingConfig(_))));
+ assert!(matches!(second, Err(DownloadError::ConflictingConfig(_))));
Ok(())
}
@@ -85,16 +116,16 @@ async fn conflicting_task_configuration_is_rejected() -> Result<(), Box<dyn Erro
#[tokio::test(flavor = "multi_thread")]
async fn separate_managers_in_same_process_cannot_share_destination_lock(
#[case] download_manager_type: FileDownloadManagerType
-) -> Result<(), Box<dyn Error>> {
+) -> Result<(), Box<dyn std::error::Error>> {
let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;
let tokenizer = registry.file("tokenizer.json")?;
- let temp_dir = tempdir().unwrap();
+ let temp_dir = tempfile::tempdir().unwrap();
let destination = temp_dir.path().join(&tokenizer.file.name);
let manager_a = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();
let task_a = manager_a
.file_download_task(
- (&tokenizer.file.url).into(),
+ &tokenizer.file.url,
&destination,
FileCheck::CRC(tokenizer.crc32c()?),
Some(tokenizer.file.size as u64),
@@ -108,7 +139,7 @@ async fn separate_managers_in_same_process_cannot_share_destination_lock(
let manager_b = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();
let task_b = manager_b
.file_download_task(
- (&tokenizer.file.url).into(),
+ &tokenizer.file.url,
&destination,
FileCheck::CRC(tokenizer.crc32c()?),
Some(tokenizer.file.size as u64),
diff --git a/crates/legacy/download-manager/tests/unit/backends/common/test_truncated_body.rs b/crates/legacy/download-manager/tests/unit/backends/common/test_truncated_body.rs
index 8c370e59a..a33df50cf 100644
--- a/crates/legacy/download-manager/tests/unit/backends/common/test_truncated_body.rs
+++ b/crates/legacy/download-manager/tests/unit/backends/common/test_truncated_body.rs
@@ -1,7 +1,6 @@
use download_manager::{FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase};
use kiban::rt::RuntimeHandle;
use rstest::rstest;
-use tempfile::tempdir;
use crate::common::{Behavior, MockRegistry, wait_for_phase};
@@ -14,17 +13,12 @@ async fn truncated_body_without_crc_fails_length_check(
) -> Result<(), Box<dyn std::error::Error>> {
let registry = MockRegistry::start_with(Behavior::TRUNCATE_BODY).await?;
let served_file = registry.file("config.json")?;
- let temp_dir = tempdir().unwrap();
+ let temp_dir = tempfile::tempdir().unwrap();
let destination = temp_dir.path().join(&served_file.file.name);
let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await.unwrap();
let task = manager
- .file_download_task(
- (&served_file.file.url).into(),
- &destination,
- FileCheck::None,
- Some(served_file.file.size as u64),
- )
+ .file_download_task(&served_file.file.url, &destination, FileCheck::None, Some(served_file.file.size as u64))
.await
.unwrap();
let mut progress = task.progress().await.unwrap();
@@ -42,27 +36,3 @@ async fn truncated_body_without_crc_fails_length_check(
);
Ok(())
}
-
-#[rstest]
-#[case::universal(FileDownloadManagerType::Universal)]
-#[cfg_attr(target_vendor = "apple", case::apple(FileDownloadManagerType::Apple))]
-#[tokio::test(flavor = "multi_thread")]
-async fn oversized_body_is_rejected_before_install(
- #[case] download_manager_type: FileDownloadManagerType
-) -> Result<(), Box<dyn std::error::Error>> {
- let registry = MockRegistry::start().await?;
- let tokenizer = registry.file("tokenizer.json")?;
- let temporary_directory = tempdir()?;
- let destination = temporary_directory.path().join(&tokenizer.file.name);
- let manager = <dyn FileDownloadManager>::new(download_manager_type, RuntimeHandle::current()).await?;
- let task = manager
- .file_download_task((&tokenizer.file.url).into(), &destination, FileCheck::None, Some(1))
- .await?;
- let mut progress = task.progress().await?;
-
- task.download().await?;
- wait_for_phase(&task, &mut progress, |phase| matches!(phase, FileDownloadPhase::Error(_))).await;
-
- assert!(!destination.exists());
- Ok(())
-}
diff --git a/crates/legacy/download-manager/tests/unit/backends/universal/mod.rs b/crates/legacy/download-manager/tests/unit/backends/universal/mod.rs
index af7de0b39..522185cd3 100644
--- a/crates/legacy/download-manager/tests/unit/backends/universal/mod.rs
+++ b/crates/legacy/download-manager/tests/unit/backends/universal/mod.rs
@@ -1,3 +1,2 @@
mod test_resume;
mod test_resume_content_range_validation;
-mod test_size_limit;
diff --git a/crates/legacy/download-manager/tests/unit/backends/universal/test_resume.rs b/crates/legacy/download-manager/tests/unit/backends/universal/test_resume.rs
index 1bbe04b72..6a47c2f51 100644
--- a/crates/legacy/download-manager/tests/unit/backends/universal/test_resume.rs
+++ b/crates/legacy/download-manager/tests/unit/backends/universal/test_resume.rs
@@ -1,39 +1,33 @@
-use std::{error::Error, sync::Arc, time::Duration};
+use std::time::Duration;
-use download_manager::{FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase, HttpDownloadRequest};
+use download_manager::{FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase};
use kiban::rt::RuntimeHandle;
use tempfile::tempdir;
-use tokio::{
- fs::{read as tokio_read, write as tokio_write},
- time::timeout as tokio_timeout,
-};
+use tokio::time::timeout;
use wiremock::{
Mock, MockServer, ResponseTemplate,
- matchers::{header, method, path},
+ matchers::{method, path},
};
#[tokio::test(flavor = "multi_thread")]
-async fn resume_sends_range_and_restarts_when_ignored() -> Result<(), Box<dyn Error>> {
+async fn test_universal_resume_restarts_when_server_ignores_range() -> Result<(), Box<dyn std::error::Error>> {
let bytes = b"complete file body";
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/model.bin"))
- .and(header("accept-encoding", "identity"))
- .and(header("range", "bytes=7-"))
.respond_with(ResponseTemplate::new(200).set_body_bytes(bytes.as_slice()))
- .expect(1)
.mount(&server)
.await;
let temporary_directory = tempdir().unwrap();
let destination = temporary_directory.path().join("model.bin");
- let part_path = destination.with_added_extension("part");
- tokio_write(&part_path, b"partial").await?;
+ let part_path = destination.with_extension("part");
+ tokio::fs::write(&part_path, b"partial").await?;
let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Universal, RuntimeHandle::current()).await?;
let task = manager
.file_download_task(
- (&format!("{}/model.bin", server.uri())).into(),
+ &format!("{}/model.bin", server.uri()),
&destination,
FileCheck::None,
Some(bytes.len() as u64),
@@ -42,109 +36,9 @@ async fn resume_sends_range_and_restarts_when_ignored() -> Result<(), Box<dyn Er
assert_eq!(task.state().await.phase, FileDownloadPhase::Paused);
task.download().await?;
- tokio_timeout(Duration::from_secs(10), task.wait()).await?;
+ timeout(Duration::from_secs(10), task.wait()).await?;
assert_eq!(task.state().await.phase, FileDownloadPhase::Downloaded);
- assert_eq!(tokio_read(&destination).await?, bytes);
- Ok(())
-}
-
-#[tokio::test(flavor = "multi_thread")]
-async fn authenticated_redirect_strips_authorization_cross_origin() -> Result<(), Box<dyn Error>> {
- let bytes = b"private redirected file";
- let target = MockServer::start().await;
- Mock::given(method("GET"))
- .and(path("/cdn/model.bin"))
- .and(header("accept-encoding", "identity"))
- .respond_with(ResponseTemplate::new(200).set_body_bytes(bytes.as_slice()))
- .expect(1)
- .mount(&target)
- .await;
- let origin = MockServer::start().await;
- Mock::given(method("GET"))
- .and(path("/model.bin"))
- .and(header("authorization", "Bearer test-token"))
- .and(header("accept-encoding", "identity"))
- .respond_with(ResponseTemplate::new(302).insert_header("Location", format!("{}/cdn/model.bin", target.uri())))
- .expect(1)
- .mount(&origin)
- .await;
- let directory = tempdir()?;
- let destination = directory.path().join("model.bin");
- let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Universal, RuntimeHandle::current()).await?;
- let token: Arc<str> = Arc::from("test-token");
- let request = HttpDownloadRequest::with_bearer_token(format!("{}/model.bin", origin.uri()), &token);
-
- let task = manager.file_download_task(request, &destination, FileCheck::None, Some(bytes.len() as u64)).await?;
- task.download().await?;
- tokio_timeout(Duration::from_secs(10), task.wait()).await?;
-
- let target_requests = target.received_requests().await.expect("requests should be available");
- assert_eq!(target_requests.len(), 1);
- assert!(target_requests[0].headers.get("authorization").is_none());
- assert_eq!(tokio_read(destination).await?, bytes);
- Ok(())
-}
-
-#[tokio::test(flavor = "multi_thread")]
-async fn authenticated_resume_sends_bearer_and_range() -> Result<(), Box<dyn Error>> {
- let bytes = b"private file body";
- let resume_from = 7;
- let server = MockServer::start().await;
- Mock::given(method("GET"))
- .and(path("/model.bin"))
- .and(header("authorization", "Bearer test-token"))
- .and(header("range", "bytes=7-"))
- .respond_with(
- ResponseTemplate::new(206)
- .insert_header(
- "Content-Range",
- format!("bytes {resume_from}-{}/{}", bytes.len() - 1, bytes.len()),
- )
- .set_body_bytes(&bytes[resume_from..]),
- )
- .expect(1)
- .mount(&server)
- .await;
- let directory = tempdir()?;
- let destination = directory.path().join("model.bin");
- tokio_write(destination.with_added_extension("part"), &bytes[..resume_from]).await?;
- let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Universal, RuntimeHandle::current()).await?;
- let token: Arc<str> = Arc::from("test-token");
- let request = HttpDownloadRequest::with_bearer_token(format!("{}/model.bin", server.uri()), &token);
- let task = manager.file_download_task(request, &destination, FileCheck::None, Some(bytes.len() as u64)).await?;
-
- task.download().await?;
- tokio_timeout(Duration::from_secs(10), task.wait()).await?;
-
- assert_eq!(task.state().await.phase, FileDownloadPhase::Downloaded);
- assert_eq!(tokio_read(destination).await?, bytes);
- Ok(())
-}
-
-#[tokio::test(flavor = "multi_thread")]
-async fn signed_redirect_url_is_redacted_from_errors() -> Result<(), Box<dyn Error>> {
- let server = MockServer::start().await;
- Mock::given(method("GET")).and(path("/model.bin")).respond_with(ResponseTemplate::new(500)).mount(&server).await;
- let directory = tempdir()?;
- let destination = directory.path().join("model.bin");
- let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Universal, RuntimeHandle::current()).await?;
- let task = manager
- .file_download_task(
- (&format!("{}/model.bin?X-Amz-Signature=super-secret", server.uri())).into(),
- &destination,
- FileCheck::None,
- Some(1),
- )
- .await?;
-
- task.download().await?;
- tokio_timeout(Duration::from_secs(10), task.wait()).await?;
-
- let FileDownloadPhase::Error(message) = task.state().await.phase else {
- panic!("request should fail");
- };
- assert!(!message.contains("super-secret"));
- assert!(!message.contains("X-Amz-Signature"));
+ assert_eq!(tokio::fs::read(&destination).await?, bytes);
Ok(())
}
diff --git a/crates/legacy/download-manager/tests/unit/backends/universal/test_resume_content_range_validation.rs b/crates/legacy/download-manager/tests/unit/backends/universal/test_resume_content_range_validation.rs
index c244558a1..b5de0614f 100644
--- a/crates/legacy/download-manager/tests/unit/backends/universal/test_resume_content_range_validation.rs
+++ b/crates/legacy/download-manager/tests/unit/backends/universal/test_resume_content_range_validation.rs
@@ -1,23 +1,16 @@
-use std::{error::Error, time::Duration};
+use std::time::Duration;
use download_manager::{FileCheck, FileDownloadManager, FileDownloadManagerType, FileDownloadPhase};
use kiban::rt::RuntimeHandle;
-use rstest::rstest;
use tempfile::tempdir;
-use tokio::{
- fs::{read as tokio_read, write as tokio_write},
- time::timeout as tokio_timeout,
-};
+use tokio::time::timeout;
use wiremock::{
Mock, MockServer, ResponseTemplate,
- matchers::{header, method, path},
+ matchers::{method, path},
};
-#[rstest]
-#[case::offset("bytes 0-4/10")]
-#[case::total("bytes 5-9/11")]
#[tokio::test(flavor = "multi_thread")]
-async fn invalid_content_range_restarts_from_zero(#[case] content_range: &str) -> Result<(), Box<dyn Error>> {
+async fn resume_rejects_206_when_content_range_offset_mismatches_request() -> Result<(), Box<dyn std::error::Error>> {
let full_bytes: &[u8] = b"abcdefghij";
let partial_bytes: &[u8] = b"abcde";
let total = full_bytes.len();
@@ -25,44 +18,39 @@ async fn invalid_content_range_restarts_from_zero(#[case] content_range: &str) -
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/model.bin"))
- .and(header("range", "bytes=5-"))
.respond_with(
ResponseTemplate::new(206)
- .set_body_bytes(&full_bytes[partial_bytes.len()..])
- .insert_header("Content-Range", content_range),
+ .set_body_bytes(partial_bytes)
+ .insert_header("Content-Range", format!("bytes 0-{}/{}", total - 1, total)),
)
- .with_priority(1)
- .expect(1)
- .mount(&server)
- .await;
- Mock::given(method("GET"))
- .and(path("/model.bin"))
- .respond_with(ResponseTemplate::new(200).set_body_bytes(full_bytes))
- .with_priority(2)
- .expect(1)
.mount(&server)
.await;
let temporary_directory = tempdir().unwrap();
let destination = temporary_directory.path().join("model.bin");
- let part_path = destination.with_added_extension("part");
- tokio_write(&part_path, partial_bytes).await?;
+ let part_path = destination.with_extension("part");
+ tokio::fs::write(&part_path, partial_bytes).await?;
let manager = <dyn FileDownloadManager>::new(FileDownloadManagerType::Universal, RuntimeHandle::current()).await?;
let task = manager
- .file_download_task(
- (&format!("{}/model.bin", server.uri())).into(),
- &destination,
- FileCheck::None,
- Some(total as u64),
- )
+ .file_download_task(&format!("{}/model.bin", server.uri()), &destination, FileCheck::None, Some(total as u64))
.await?;
assert_eq!(task.state().await.phase, FileDownloadPhase::Paused);
task.download().await?;
- tokio_timeout(Duration::from_secs(10), task.wait()).await?;
+ timeout(Duration::from_secs(10), task.wait()).await?;
- assert_eq!(task.state().await.phase, FileDownloadPhase::Downloaded);
- assert_eq!(tokio_read(destination).await?, full_bytes);
- Ok(())
+ let final_phase = task.state().await.phase;
+ match &final_phase {
+ FileDownloadPhase::Error(_) => Ok(()),
+ FileDownloadPhase::Downloaded => {
+ assert_eq!(
+ tokio::fs::read(&destination).await?,
+ full_bytes,
+ "phase reached Downloaded but file content was corrupted by accepting a misaligned Content-Range",
+ );
+ Ok(())
+ },
+ other => panic!("unexpected final phase: {:?}", other),
+ }
}
diff --git a/crates/legacy/download-manager/tests/unit/backends/universal/test_size_limit.rs b/crates/legacy/download-manager/tests/unit/backends/universal/test_size_limit.rs
deleted file mode 100644
index 252fcd6b9..000000000
--- a/crates/legacy/download-manager/tests/unit/backends/universal/test_size_limit.rs
+++ /dev/null
@@ -1,77 +0,0 @@
-use std::{error::Error, sync::Arc, time::Duration};
-
-use kiban::{fs, rt::RuntimeHandle};
-use tempfile::tempdir;
-use tokio::{
- sync::{
- Mutex as TokioMutex,
- mpsc::{Receiver as TokioMpscReceiver, channel as tokio_mpsc_channel},
- watch::channel as tokio_watch_channel,
- },
- time::timeout as tokio_timeout,
-};
-use uuid::Uuid;
-use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method};
-
-use crate::{
- FileCheck,
- backends::universal::UniversalBackendContext,
- compute_download_id,
- file_download_task_actor::{BackendEvent, PendingProgressSlot},
- lock_manager::DestinationLockLease,
- traits::{ActiveDownloadGeneration, BackendContext, BackendEventSender, DownloadConfig},
-};
-
-fn backend_event_sender() -> (BackendEventSender, TokioMpscReceiver<BackendEvent>) {
- let (sender, receiver) = tokio_mpsc_channel(8);
- let pending_progress = Arc::new(TokioMutex::new(PendingProgressSlot::default()));
- let (progress_waker, _) = tokio_watch_channel(());
- (BackendEventSender::new(Uuid::new_v4(), sender, pending_progress, progress_waker), receiver)
-}
-
-#[tokio::test(flavor = "multi_thread")]
-async fn response_body_cannot_grow_part_beyond_expected_size() -> Result<(), Box<dyn Error>> {
- let server = MockServer::start().await;
- Mock::given(method("GET"))
- .respond_with(ResponseTemplate::new(200).set_body_bytes(vec![0; 256 * 1024]))
- .expect(1)
- .mount(&server)
- .await;
- let directory = tempdir()?;
- let destination = directory.path().join("model.bin");
- let config = Arc::new(DownloadConfig {
- download_id: compute_download_id(&destination),
- request: server.uri().into(),
- destination: destination.clone(),
- file_check: FileCheck::None,
- expected_bytes: Some(1),
- manager_id: "test-manager".to_string(),
- manager_instance_id: Uuid::new_v4(),
- });
- let destination_lease = DestinationLockLease::acquire_for_destination(
- &destination,
- &config.manager_id,
- config.manager_instance_id,
- )
- .await?;
- let mut context = UniversalBackendContext::new(RuntimeHandle::current());
- context.retries = 0;
- let (sender, mut receiver) = backend_event_sender();
-
- let active_task = context
- .download(
- Arc::clone(&config),
- ActiveDownloadGeneration::new(0),
- sender,
- &destination_lease,
- )
- .await?;
- let event = tokio_timeout(Duration::from_secs(5), receiver.recv()).await?;
-
- assert!(matches!(event, Some(BackendEvent::Error { .. })));
- assert!(fs::asyn::file_length(destination.with_added_extension("part")).await.unwrap_or(0) <= 1);
- assert!(!destination.exists());
- drop(active_task);
- destination_lease.release().await?;
- Ok(())
-}
diff --git a/crates/legacy/download-manager/tests/unit/mod.rs b/crates/legacy/download-manager/tests/unit/mod.rs
index 5712649e9..0637eea96 100644
--- a/crates/legacy/download-manager/tests/unit/mod.rs
+++ b/crates/legacy/download-manager/tests/unit/mod.rs
@@ -1,4 +1,2 @@
mod backends;
-mod test_http_download_request;
-mod test_integrity;
mod test_lock_manager;
diff --git a/crates/legacy/download-manager/tests/unit/test_http_download_request.rs b/crates/legacy/download-manager/tests/unit/test_http_download_request.rs
deleted file mode 100644
index e93c1d311..000000000
--- a/crates/legacy/download-manager/tests/unit/test_http_download_request.rs
+++ /dev/null
@@ -1,45 +0,0 @@
-use std::sync::Arc;
-
-use crate::{DownloadError, HttpDownloadRequest};
-
-#[test]
-fn request_debug_redacts_bearer_token() {
- let token: Arc<str> = Arc::from("secret-token");
- let request = HttpDownloadRequest::with_bearer_token("https://example.com/model?signature=signed-secret", &token);
- let debug = format!("{request:?}");
-
- assert!(debug.contains("authenticated: true"));
- assert!(!debug.contains("secret-token"));
- assert!(!debug.contains("signed-secret"));
-}
-
-#[test]
-fn authenticated_request_requires_https_except_for_loopback() {
- let token: Arc<str> = Arc::from("token");
- let insecure = HttpDownloadRequest::with_bearer_token("http://example.com/model", &token);
- let loopback = HttpDownloadRequest::with_bearer_token("http://127.0.0.1/model", &token);
-
- assert_eq!(insecure.validate(), Err(DownloadError::InsecureAuthenticatedRequest));
- assert_eq!(loopback.validate(), Ok(()));
-}
-
-#[test]
-fn request_does_not_keep_credentials_alive() {
- let token: Arc<str> = Arc::from("token");
- let request = HttpDownloadRequest::with_bearer_token("https://example.com/model", &token);
- drop(token);
-
- assert_eq!(request.bearer_token(), Err(DownloadError::AuthenticationUnavailable));
-}
-
-#[test]
-fn authenticated_requests_match_only_the_same_credential_owner() {
- let first_token: Arc<str> = Arc::from("token");
- let second_token: Arc<str> = Arc::from("token");
- let first = HttpDownloadRequest::with_bearer_token("https://example.com/model", &first_token);
- let first_clone = HttpDownloadRequest::with_bearer_token("https://example.com/model", &first_token);
- let second = HttpDownloadRequest::with_bearer_token("https://example.com/model", &second_token);
-
- assert_eq!(first, first_clone);
- assert_ne!(first, second);
-}
diff --git a/crates/legacy/download-manager/tests/unit/test_integrity.rs b/crates/legacy/download-manager/tests/unit/test_integrity.rs
deleted file mode 100644
index 145bbe706..000000000
--- a/crates/legacy/download-manager/tests/unit/test_integrity.rs
+++ /dev/null
@@ -1,87 +0,0 @@
-use std::error::Error;
-
-use base64::{Engine, engine::general_purpose::STANDARD};
-use crc32c::crc32c;
-use tempfile::tempdir;
-use tokio::fs::write as tokio_write;
-
-use crate::{
- DownloadInfo, FileCheck,
- integrity::{VerificationError, integrity_cache_matches, save_integrity_cache, verify_file_integrity},
-};
-
-const HELLO: &[u8] = b"hello\n";
-const HELLO_SHA256: &str = "5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03";
-const HELLO_GIT_BLOB_SHA1: &str = "ce013625030ba8dba906f756967f9e9ca394464a";
-
-#[tokio::test]
-async fn sha256_detects_same_size_corruption() -> Result<(), Box<dyn Error>> {
- let directory = tempdir()?;
- let path = directory.path().join("model.bin");
- let check = FileCheck::Sha256(HELLO_SHA256.to_string());
- tokio_write(&path, HELLO).await?;
- assert!(verify_file_integrity(&path, &check).await?);
-
- tokio_write(&path, b"jello\n").await?;
- assert!(!verify_file_integrity(&path, &check).await?);
- Ok(())
-}
-
-#[tokio::test]
-async fn git_blob_sha1_includes_the_blob_header() -> Result<(), Box<dyn Error>> {
- let directory = tempdir()?;
- let path = directory.path().join("config.json");
- let check = FileCheck::GitBlobSha1(HELLO_GIT_BLOB_SHA1.to_string());
- tokio_write(&path, HELLO).await?;
-
- assert!(verify_file_integrity(&path, &check).await?);
- Ok(())
-}
-
-#[tokio::test]
-async fn digest_cache_is_invalidated_when_file_changes() -> Result<(), Box<dyn Error>> {
- let directory = tempdir()?;
- let path = directory.path().join("weights.bin");
- let check = FileCheck::Sha256(HELLO_SHA256.to_string());
- tokio_write(&path, HELLO).await?;
- save_integrity_cache(&path, &check).await?;
- assert!(integrity_cache_matches(&path, &check).await);
-
- tokio_write(&path, b"jello\n").await?;
- assert!(!integrity_cache_matches(&path, &check).await);
- Ok(())
-}
-
-#[tokio::test]
-async fn crc_and_invalid_digest_behavior_remain_explicit() -> Result<(), Box<dyn Error>> {
- let directory = tempdir()?;
- let path = directory.path().join("tokenizer.json");
- tokio_write(&path, HELLO).await?;
- let crc = STANDARD.encode(crc32c(HELLO).to_be_bytes());
- assert!(verify_file_integrity(&path, &FileCheck::CRC(crc)).await?);
-
- let error = verify_file_integrity(&path, &FileCheck::Sha256("invalid".to_string()))
- .await
- .expect_err("invalid digest must not be treated as corrupt content");
- assert!(matches!(
- error,
- VerificationError::InvalidExpectedDigest {
- algorithm: "sha256"
- }
- ));
- Ok(())
-}
-
-#[test]
-fn apple_task_info_preserves_new_and_legacy_checks() -> Result<(), Box<dyn Error>> {
- let check = FileCheck::Sha256(HELLO_SHA256.to_string());
- let info = DownloadInfo::new("https://example.com/model", "/tmp/model", check.clone());
- let restored = DownloadInfo::from_json(&info.to_json()?)?;
- assert_eq!(restored.resolved_file_check(), check);
-
- let legacy = DownloadInfo::from_json(
- r#"{"source_url":"https://example.com/model","destination_path":"/tmp/model","crc32c":"AAAAAA=="}"#,
- )?;
- assert_eq!(legacy.resolved_file_check(), FileCheck::CRC("AAAAAA==".to_string()));
- Ok(())
-}
diff --git a/crates/legacy/mock-registry/src/model.rs b/crates/legacy/mock-registry/src/model.rs
index a07146481..8b59bdca0 100644
--- a/crates/legacy/mock-registry/src/model.rs
+++ b/crates/legacy/mock-registry/src/model.rs
@@ -1,6 +1,6 @@
use shoji::types::{
basic::Metadata,
- model::{Model, ModelAccessibility, ModelBackend, ModelRegistry, ModelSource},
+ model::{Model, ModelAccessibility, ModelBackend, ModelReference, ModelRegistry},
};
use crate::ServedFile;
@@ -36,8 +36,8 @@ pub fn mock_model(
properties: None,
quantization: None,
specializations: vec![],
- accessibility: ModelAccessibility::OnDevice {
- source: ModelSource::Registry {
+ accessibility: ModelAccessibility::Local {
+ reference: ModelReference::Mirai {
toolchain_version: "1.0".to_string(),
repository: None,
source_repository: None,
diff --git a/crates/legacy/shoji/src/types/basic/file.rs b/crates/legacy/shoji/src/types/basic/file.rs
index 7ab0e454c..c6f5ca40c 100644
--- a/crates/legacy/shoji/src/types/basic/file.rs
+++ b/crates/legacy/shoji/src/types/basic/file.rs
@@ -18,4 +18,9 @@ impl File {
pub fn crc32c(&self) -> Option<String> {
self.hashes.iter().find(|hash| hash.method == HashMethod::CRC32C).map(|hash| hash.value.clone())
}
+
+ #[bindings::export(Method(Getter))]
+ pub fn md5(&self) -> Option<String> {
+ self.hashes.iter().find(|hash| hash.method == HashMethod::MD5).map(|hash| hash.value.clone())
+ }
}
diff --git a/crates/legacy/shoji/src/types/basic/hash.rs b/crates/legacy/shoji/src/types/basic/hash.rs
index 3c6581b94..7a8032e83 100644
--- a/crates/legacy/shoji/src/types/basic/hash.rs
+++ b/crates/legacy/shoji/src/types/basic/hash.rs
@@ -6,6 +6,8 @@ use serde::{Deserialize, Serialize};
pub enum HashMethod {
#[serde(rename = "crc32c")]
CRC32C,
+ #[serde(rename = "md5")]
+ MD5,
}
#[bindings::export(Structure(Class))]
diff --git a/crates/legacy/shoji/src/types/model/accessibility.rs b/crates/legacy/shoji/src/types/model/accessibility.rs
index 85392e835..82d01870d 100644
--- a/crates/legacy/shoji/src/types/model/accessibility.rs
+++ b/crates/legacy/shoji/src/types/model/accessibility.rs
@@ -1,13 +1,13 @@
use serde::{Deserialize, Serialize};
-use crate::types::{basic::Repository, model::ModelSource};
+use crate::types::{basic::Repository, model::ModelReference};
#[bindings::export(Enumeration)]
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ModelAccessibility {
- OnDevice {
- source: ModelSource,
+ Local {
+ reference: ModelReference,
},
Remote {
repository: Option<Repository>,
diff --git a/crates/legacy/shoji/src/types/model/mod.rs b/crates/legacy/shoji/src/types/model/mod.rs
index 69fdf1941..a30a19526 100644
--- a/crates/legacy/shoji/src/types/model/mod.rs
+++ b/crates/legacy/shoji/src/types/model/mod.rs
@@ -4,8 +4,8 @@ mod family;
mod model;
mod properties;
mod quantization;
+mod reference;
mod registry;
-mod source;
mod specialization;
mod vendor;
@@ -15,7 +15,7 @@ pub use family::ModelFamily;
pub use model::Model;
pub use properties::ModelProperties;
pub use quantization::ModelQuantization;
+pub use reference::ModelReference;
pub use registry::ModelRegistry;
-pub use source::ModelSource;
pub use specialization::ModelSpecialization;
pub use vendor::ModelVendor;
diff --git a/crates/legacy/shoji/src/types/model/model.rs b/crates/legacy/shoji/src/types/model/model.rs
index a3ca910a8..31fa14c3a 100644
--- a/crates/legacy/shoji/src/types/model/model.rs
+++ b/crates/legacy/shoji/src/types/model/model.rs
@@ -3,8 +3,8 @@ use serde::{Deserialize, Serialize};
use crate::types::{
basic::{Metadata, Value},
model::{
- ModelAccessibility, ModelBackend, ModelFamily, ModelProperties, ModelQuantization, ModelRegistry, ModelSource,
- ModelSpecialization,
+ ModelAccessibility, ModelBackend, ModelFamily, ModelProperties, ModelQuantization, ModelReference,
+ ModelRegistry, ModelSpecialization,
},
};
@@ -43,8 +43,8 @@ impl Model {
}
#[bindings::export(Method(Getter))]
- pub fn is_on_device(&self) -> bool {
- matches!(self.accessibility, ModelAccessibility::OnDevice { .. })
+ pub fn is_local(&self) -> bool {
+ matches!(self.accessibility, ModelAccessibility::Local { .. })
}
#[bindings::export(Method(Getter))]
@@ -56,8 +56,8 @@ impl Model {
pub fn is_downloadable(&self) -> bool {
matches!(
self.accessibility,
- ModelAccessibility::OnDevice {
- source: ModelSource::Registry { .. }
+ ModelAccessibility::Local {
+ reference: ModelReference::Mirai { .. } | ModelReference::HuggingFace { .. }
}
)
}
@@ -75,11 +75,11 @@ impl Model {
#[bindings::export(Method(Getter))]
pub fn repo_ids(&self) -> Vec<String> {
match &self.accessibility {
- ModelAccessibility::OnDevice {
- source,
+ ModelAccessibility::Local {
+ reference,
..
- } => match source {
- ModelSource::Registry {
+ } => match reference {
+ ModelReference::Mirai {
repository,
source_repository,
..
@@ -93,7 +93,10 @@ impl Model {
}
result
},
- ModelSource::Filesystem {
+ ModelReference::HuggingFace {
+ repository,
+ } => vec![repository.identifier.clone()],
+ ModelReference::Local {
..
} => vec![],
},
@@ -111,16 +114,19 @@ impl Model {
}
#[bindings::export(Method(Getter))]
- pub fn filesystem_path(&self) -> Option<String> {
+ pub fn local_external_path(&self) -> Option<String> {
match &self.accessibility {
- ModelAccessibility::OnDevice {
- source,
+ ModelAccessibility::Local {
+ reference,
..
- } => match source {
- ModelSource::Registry {
+ } => match reference {
+ ModelReference::Mirai {
+ ..
+ } => None,
+ ModelReference::HuggingFace {
..
} => None,
- ModelSource::Filesystem {
+ ModelReference::Local {
path,
} => Some(path.clone()),
},
@@ -133,10 +139,10 @@ impl Model {
#[bindings::export(Method(Getter))]
pub fn reference_name(&self) -> Option<String> {
match &self.accessibility {
- ModelAccessibility::OnDevice {
- source,
+ ModelAccessibility::Local {
+ reference,
..
- } => Some(source.name()),
+ } => Some(reference.name()),
ModelAccessibility::Remote {
..
} => None,
@@ -146,19 +152,18 @@ impl Model {
#[bindings::export(Method(Getter))]
pub fn checkpoint_version(&self) -> Option<String> {
match &self.accessibility {
- ModelAccessibility::OnDevice {
- source,
+ ModelAccessibility::Local {
+ reference,
..
- } => match source {
- ModelSource::Registry {
+ } => match reference {
+ ModelReference::Mirai {
toolchain_version,
- repository,
..
- } => repository
- .as_ref()
- .and_then(|repository| repository.commit_hash.clone())
- .or_else(|| Some(toolchain_version.clone())),
- ModelSource::Filesystem {
+ } => Some(toolchain_version.clone()),
+ ModelReference::HuggingFace {
+ repository,
+ } => repository.commit_hash.clone(),
+ ModelReference::Local {
..
} => None,
},
diff --git a/crates/legacy/shoji/src/types/model/source.rs b/crates/legacy/shoji/src/types/model/reference.rs
similarity index 58%
rename from crates/legacy/shoji/src/types/model/source.rs
rename to crates/legacy/shoji/src/types/model/reference.rs
index dc16e1325..019e93a8d 100644
--- a/crates/legacy/shoji/src/types/model/source.rs
+++ b/crates/legacy/shoji/src/types/model/reference.rs
@@ -5,28 +5,33 @@ use crate::types::basic::{File, Repository};
#[bindings::export(Enumeration)]
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
-pub enum ModelSource {
- #[serde(rename = "managed")]
- Registry {
+pub enum ModelReference {
+ Mirai {
toolchain_version: String,
repository: Option<Repository>,
source_repository: Option<Repository>,
files: Vec<File>,
},
- Filesystem {
+ HuggingFace {
+ repository: Repository,
+ },
+ Local {
path: String,
},
}
-impl ModelSource {
+impl ModelReference {
pub fn name(&self) -> String {
match self {
- ModelSource::Registry {
+ ModelReference::Mirai {
+ ..
+ } => "mirai".to_string(),
+ ModelReference::HuggingFace {
..
- } => "registry".to_string(),
- ModelSource::Filesystem {
+ } => "huggingface".to_string(),
+ ModelReference::Local {
..
- } => "filesystem".to_string(),
+ } => "local".to_string(),
}
}
}
diff --git a/crates/legacy/uzu/Cargo.toml b/crates/legacy/uzu/Cargo.toml
index aae7731de..3317e73d2 100644
--- a/crates/legacy/uzu/Cargo.toml
+++ b/crates/legacy/uzu/Cargo.toml
@@ -63,6 +63,7 @@ uzu-engine.workspace = true
kiban.workspace = true
shoji.workspace = true
anyhow.workspace = true
+bitflags.workspace = true
uniffi = { workspace = true, optional = true }
napi = { workspace = true, optional = true }
napi-derive = { workspace = true, optional = true }
@@ -133,11 +134,11 @@ rstest.workspace = true
[[test]]
name = "tool"
-path = "tests/integration/tool/mod.rs"
+path = "tests/tool/mod.rs"
[[test]]
-name = "integration"
-path = "tests/integration/test_entry.rs"
+name = "unit"
+path = "tests/unit/test_entry.rs"
[build-dependencies]
napi-build = { workspace = true, optional = true }
diff --git a/crates/legacy/uzu/bindings/python/tests/test_engine.py b/crates/legacy/uzu/bindings/python/tests/test_engine.py
index cf7ed5d7f..b765195ee 100644
--- a/crates/legacy/uzu/bindings/python/tests/test_engine.py
+++ b/crates/legacy/uzu/bindings/python/tests/test_engine.py
@@ -1,6 +1,6 @@
import asyncio
-from uzu import ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfig, ReasoningEffort, SamplingMethod
+from uzu import ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfig
def test_chat_reply_produces_text() -> None:
@@ -16,17 +16,15 @@ def test_chat_reply_produces_text() -> None:
session = await engine.chat(model, ChatConfig.create())
messages = [
- ChatMessage.system()
- .with_text("You are a helpful assistant")
- .with_reasoning_effort(ReasoningEffort.Disabled),
+ ChatMessage.system().with_text("You are a helpful assistant"),
ChatMessage.user().with_text("Hi"),
]
- config = ChatReplyConfig.create().with_token_limit(64).with_sampling_method(SamplingMethod.Greedy())
- replies = await session.reply(messages, config)
+ replies = await session.reply(messages, ChatReplyConfig.create())
assert replies, "Reply has no messages"
message = replies[-1].message
+ assert message.reasoning is not None
assert message.text is not None
asyncio.run(run())
diff --git a/crates/legacy/uzu/bindings/python/uzu/__init__.pyi b/crates/legacy/uzu/bindings/python/uzu/__init__.pyi
index fca414766..95c6d64c3 100644
--- a/crates/legacy/uzu/bindings/python/uzu/__init__.pyi
+++ b/crates/legacy/uzu/bindings/python/uzu/__init__.pyi
@@ -61,8 +61,8 @@ __all__ = [
"ModelFamily",
"ModelProperties",
"ModelQuantization",
+ "ModelReference",
"ModelRegistry",
- "ModelSource",
"ModelSpecialization",
"ModelVendor",
"PcmBatch",
@@ -696,7 +696,7 @@ class Engine:
def remove_registry(self, registry_identifier: builtins.str) -> collections.abc.Awaitable[None]: ...
def remove_backend(self, identifier: builtins.str) -> collections.abc.Awaitable[None]: ...
def models(self) -> collections.abc.Awaitable[builtins.list[Model]]: ...
- def models_on_device(self) -> collections.abc.Awaitable[builtins.list[Model]]: ...
+ def models_local(self) -> collections.abc.Awaitable[builtins.list[Model]]: ...
def models_remote(self) -> collections.abc.Awaitable[builtins.list[Model]]: ...
def models_downloadable(self) -> collections.abc.Awaitable[builtins.list[Model]]: ...
def models_for_chat(self) -> collections.abc.Awaitable[builtins.list[Model]]: ...
@@ -798,6 +798,8 @@ class File:
def hashes(self) -> builtins.list[Hash]: ...
@property
def crc32c(self) -> typing.Optional[builtins.str]: ...
+ @property
+ def md5(self) -> typing.Optional[builtins.str]: ...
def __new__(cls, url: builtins.str, name: builtins.str, size: builtins.int, hashes: typing.Sequence[Hash]) -> File: ...
class Grammar:
@@ -885,7 +887,7 @@ class Model:
@property
def name(self) -> builtins.str: ...
@property
- def is_on_device(self) -> builtins.bool: ...
+ def is_local(self) -> builtins.bool: ...
@property
def is_remote(self) -> builtins.bool: ...
@property
@@ -897,7 +899,7 @@ class Model:
@property
def repo_ids(self) -> builtins.list[builtins.str]: ...
@property
- def filesystem_path(self) -> typing.Optional[builtins.str]: ...
+ def local_external_path(self) -> typing.Optional[builtins.str]: ...
@property
def reference_name(self) -> typing.Optional[builtins.str]: ...
@property
@@ -918,11 +920,11 @@ class Model:
class ModelAccessibility:
@typing.final
- class OnDevice(ModelAccessibility):
- __match_args__ = ("source",)
+ class Local(ModelAccessibility):
+ __match_args__ = ("reference",)
@property
- def source(self) -> ModelSource: ...
- def __new__(cls, source: ModelSource) -> ModelAccessibility.OnDevice: ...
+ def reference(self) -> ModelReference: ...
+ def __new__(cls, reference: ModelReference) -> ModelAccessibility.Local: ...
@typing.final
class Remote(ModelAccessibility):
@@ -987,19 +989,9 @@ class ModelQuantization:
def name(self) -> builtins.str: ...
def __new__(cls, identifier: builtins.str, method: builtins.str, bits_per_weight: builtins.int, vendor: ModelVendor, metadata: Metadata) -> ModelQuantization: ...
-class ModelRegistry:
- @property
- def identifier(self) -> builtins.str: ...
- @property
- def metadata(self) -> Metadata: ...
- @property
- def name(self) -> builtins.str: ...
- def __new__(cls, identifier: builtins.str, metadata: Metadata) -> ModelRegistry: ...
-
-class ModelSource:
+class ModelReference:
@typing.final
- class Registry(ModelSource):
+ class Mirai(ModelReference):
__match_args__ = ("toolchain_version", "repository", "source_repository", "files",)
@property
def toolchain_version(self) -> builtins.str: ...
@@ -1009,17 +1001,34 @@ class ModelSource:
def source_repository(self) -> typing.Optional[Repository]: ...
@property
def files(self) -> builtins.list[File]: ...
- def __new__(cls, toolchain_version: builtins.str, repository: typing.Optional[Repository], source_repository: typing.Optional[Repository], files: typing.Sequence[File]) -> ModelSource.Registry: ...
+ def __new__(cls, toolchain_version: builtins.str, repository: typing.Optional[Repository], source_repository: typing.Optional[Repository], files: typing.Sequence[File]) -> ModelReference.Mirai: ...
@typing.final
- class Filesystem(ModelSource):
+ class HuggingFace(ModelReference):
+ __match_args__ = ("repository",)
+ @property
+ def repository(self) -> Repository: ...
+ def __new__(cls, repository: Repository) -> ModelReference.HuggingFace: ...
+
+ @typing.final
+ class Local(ModelReference):
__match_args__ = ("path",)
@property
def path(self) -> builtins.str: ...
- def __new__(cls, path: builtins.str) -> ModelSource.Filesystem: ...
+ def __new__(cls, path: builtins.str) -> ModelReference.Local: ...
...
+class ModelRegistry:
+ @property
+ def identifier(self) -> builtins.str: ...
+ @property
+ def metadata(self) -> Metadata: ...
+ @property
+ def name(self) -> builtins.str: ...
+ def __new__(cls, identifier: builtins.str, metadata: Metadata) -> ModelRegistry: ...
+
class ModelSpecialization:
@typing.final
class Chat(ModelSpecialization):
@@ -1352,6 +1361,7 @@ class DownloadManagerType(enum.Enum):
@typing.final
class HashMethod(enum.Enum):
CRC32C = ...
+ MD5 = ...
@typing.final
class ImageFormat(enum.Enum):
diff --git a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
index 762a75aeb..e095452fc 100644
--- a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
+++ b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
@@ -1741,6 +1741,14 @@ public func crc32c() -> String? {
})
}
+public func md5() -> String? {
+ return try! FfiConverterOptionString.lift(try! rustCall() {
+ uniffi_shoji_fn_method_file_md5(
+ FfiConverterTypeFile_lower(self),$0
+ )
+})
+}
+
}
@@ -2019,14 +2027,6 @@ public func checkpointVersion() -> String? {
})
}
-public func filesystemPath() -> String? {
- return try! FfiConverterOptionString.lift(try! rustCall() {
- uniffi_shoji_fn_method_model_filesystem_path(
- FfiConverterTypeModel_lower(self),$0
- )
-})
-}
-
public func isChatCapable() -> Bool {
return try! FfiConverterBool.lift(try! rustCall() {
uniffi_shoji_fn_method_model_is_chat_capable(
@@ -2051,9 +2051,9 @@ public func isDownloadable() -> Bool {
})
}
-public func isOnDevice() -> Bool {
+public func isLocal() -> Bool {
return try! FfiConverterBool.lift(try! rustCall() {
- uniffi_shoji_fn_method_model_is_on_device(
+ uniffi_shoji_fn_method_model_is_local(
FfiConverterTypeModel_lower(self),$0
)
})
@@ -2099,6 +2099,14 @@ public func isTranslationCapable() -> Bool {
})
}
+public func localExternalPath() -> String? {
+ return try! FfiConverterOptionString.lift(try! rustCall() {
+ uniffi_shoji_fn_method_model_local_external_path(
+ FfiConverterTypeModel_lower(self),$0
+ )
+})
+}
+
public func name() -> String {
return try! FfiConverterString.lift(try! rustCall() {
uniffi_shoji_fn_method_model_name(
@@ -4192,6 +4200,7 @@ public func FfiConverterTypeGrammar_lower(_ value: Grammar) -> RustBuffer {
public enum HashMethod: Equatable, Hashable, Codable {
case crc32c
+ case md5
@@ -4215,6 +4224,8 @@ public struct FfiConverterTypeHashMethod: FfiConverterRustBuffer {
case 1: return .crc32c
+ case 2: return .md5
+
default: throw UniffiInternalError.unexpectedEnumCase
}
}
@@ -4226,6 +4237,10 @@ public struct FfiConverterTypeHashMethod: FfiConverterRustBuffer {
case .crc32c:
writeInt(&buf, Int32(1))
+
+ case .md5:
+ writeInt(&buf, Int32(2))
+
}
}
}
@@ -4392,7 +4407,7 @@ public func FfiConverterTypeImageTheme_lower(_ value: ImageTheme) -> RustBuffer
public enum ModelAccessibility: Equatable, Hashable, Codable {
- case onDevice(source: ModelSource
+ case local(reference: ModelReference
)
case remote(repository: Repository?
)
@@ -4417,7 +4432,7 @@ public struct FfiConverterTypeModelAccessibility: FfiConverterRustBuffer {
let variant: Int32 = try readInt(&buf)
switch variant {
- case 1: return .onDevice(source: try FfiConverterTypeModelSource.read(from: &buf)
+ case 1: return .local(reference: try FfiConverterTypeModelReference.read(from: &buf)
)
case 2: return .remote(repository: try FfiConverterOptionTypeRepository.read(from: &buf)
@@ -4431,9 +4446,9 @@ public struct FfiConverterTypeModelAccessibility: FfiConverterRustBuffer {
switch value {
- case let .onDevice(source):
+ case let .local(reference):
writeInt(&buf, Int32(1))
- FfiConverterTypeModelSource.write(source, into: &buf)
+ FfiConverterTypeModelReference.write(reference, into: &buf)
case let .remote(repository):
@@ -4463,11 +4478,13 @@ public func FfiConverterTypeModelAccessibility_lower(_ value: ModelAccessibility
// Note that we don't yet support `indirect` for enums.
// See https://github.com/mozilla/uniffi-rs/issues/396 for further discussion.
-public enum ModelSource: Equatable, Hashable, Codable {
+public enum ModelReference: Equatable, Hashable, Codable {
- case registry(toolchainVersion: String, repository: Repository?, sourceRepository: Repository?, files: [File]
+ case mirai(toolchainVersion: String, repository: Repository?, sourceRepository: Repository?, files: [File]
+ )
+ case huggingFace(repository: Repository
)
- case filesystem(path: String
+ case local(path: String
)
@@ -4477,34 +4494,37 @@ public enum ModelSource: Equatable, Hashable, Codable {
}
#if compiler(>=6)
-extension ModelSource: Sendable {}
+extension ModelReference: Sendable {}
#endif
#if swift(>=5.8)
@_documentation(visibility: private)
#endif
-public struct FfiConverterTypeModelSource: FfiConverterRustBuffer {
- typealias SwiftType = ModelSource
+public struct FfiConverterTypeModelReference: FfiConverterRustBuffer {
+ typealias SwiftType = ModelReference
- public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ModelSource {
+ public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ModelReference {
let variant: Int32 = try readInt(&buf)
switch variant {
- case 1: return .registry(toolchainVersion: try FfiConverterString.read(from: &buf), repository: try FfiConverterOptionTypeRepository.read(from: &buf), sourceRepository: try FfiConverterOptionTypeRepository.read(from: &buf), files: try FfiConverterSequenceTypeFile.read(from: &buf)
+ case 1: return .mirai(toolchainVersion: try FfiConverterString.read(from: &buf), repository: try FfiConverterOptionTypeRepository.read(from: &buf), sourceRepository: try FfiConverterOptionTypeRepository.read(from: &buf), files: try FfiConverterSequenceTypeFile.read(from: &buf)
+ )
+
+ case 2: return .huggingFace(repository: try FfiConverterTypeRepository.read(from: &buf)
)
- case 2: return .filesystem(path: try FfiConverterString.read(from: &buf)
+ case 3: return .local(path: try FfiConverterString.read(from: &buf)
)
default: throw UniffiInternalError.unexpectedEnumCase
}
}
- public static func write(_ value: ModelSource, into buf: inout [UInt8]) {
+ public static func write(_ value: ModelReference, into buf: inout [UInt8]) {
switch value {
- case let .registry(toolchainVersion,repository,sourceRepository,files):
+ case let .mirai(toolchainVersion,repository,sourceRepository,files):
writeInt(&buf, Int32(1))
FfiConverterString.write(toolchainVersion, into: &buf)
FfiConverterOptionTypeRepository.write(repository, into: &buf)
@@ -4512,8 +4532,13 @@ public struct FfiConverterTypeModelSource: FfiConverterRustBuffer {
FfiConverterSequenceTypeFile.write(files, into: &buf)
- case let .filesystem(path):
+ case let .huggingFace(repository):
writeInt(&buf, Int32(2))
+ FfiConverterTypeRepository.write(repository, into: &buf)
+
+
+ case let .local(path):
+ writeInt(&buf, Int32(3))
FfiConverterString.write(path, into: &buf)
}
@@ -4524,15 +4549,15 @@ public struct FfiConverterTypeModelSource: FfiConverterRustBuffer {
#if swift(>=5.8)
@_documentation(visibility: private)
#endif
-public func FfiConverterTypeModelSource_lift(_ buf: RustBuffer) throws -> ModelSource {
- return try FfiConverterTypeModelSource.lift(buf)
+public func FfiConverterTypeModelReference_lift(_ buf: RustBuffer) throws -> ModelReference {
+ return try FfiConverterTypeModelReference.lift(buf)
}
#if swift(>=5.8)
@_documentation(visibility: private)
#endif
-public func FfiConverterTypeModelSource_lower(_ value: ModelSource) -> RustBuffer {
- return FfiConverterTypeModelSource.lower(value)
+public func FfiConverterTypeModelReference_lower(_ value: ModelReference) -> RustBuffer {
+ return FfiConverterTypeModelReference.lower(value)
}
diff --git a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swift b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swift
index 472aae101..fea4d4fbe 100644
--- a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swift
+++ b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swift
@@ -902,7 +902,7 @@ public protocol EngineProtocol: AnyObject, Sendable {
func modelsForTranslation() async throws -> [Model]
- func modelsOnDevice() async throws -> [Model]
+ func modelsLocal() async throws -> [Model]
func modelsRemote() async throws -> [Model]
@@ -1390,11 +1390,11 @@ open func modelsForTranslation()async throws -> [Model] {
)
}
-open func modelsOnDevice()async throws -> [Model] {
+open func modelsLocal()async throws -> [Model] {
return
try await uniffiRustCallAsync(
rustFutureFunc: {
- uniffi_uzu_fn_method_engine_models_on_device(
+ uniffi_uzu_fn_method_engine_models_local(
self.uniffiCloneHandle()
)
@@ -3114,14 +3114,20 @@ public enum StorageError: Swift.Error, Equatable, Hashable, Codable, Foundation.
case UnableToCreateDirectory(path: String
)
+ case UnableToCreateDownloadManager(message: String
+ )
case DownloadManager(message: String
)
+ case HashNotFound(identifier: String, name: String
+ )
case InvalidStateTransition(from: DownloadPhase, to: DownloadPhase
)
case Io(message: String
)
case ItemNotFound(identifier: String
)
+ case Registry(RegistryError
+ )
case UnsupportedItem(identifier: String
)
@@ -3156,20 +3162,30 @@ public struct FfiConverterTypeStorageError: FfiConverterRustBuffer {
case 1: return .UnableToCreateDirectory(
path: try FfiConverterString.read(from: &buf)
)
- case 2: return .DownloadManager(
+ case 2: return .UnableToCreateDownloadManager(
message: try FfiConverterString.read(from: &buf)
)
- case 3: return .InvalidStateTransition(
+ case 3: return .DownloadManager(
+ message: try FfiConverterString.read(from: &buf)
+ )
+ case 4: return .HashNotFound(
+ identifier: try FfiConverterString.read(from: &buf),
+ name: try FfiConverterString.read(from: &buf)
+ )
+ case 5: return .InvalidStateTransition(
from: try FfiConverterTypeDownloadPhase.read(from: &buf),
to: try FfiConverterTypeDownloadPhase.read(from: &buf)
)
- case 4: return .Io(
+ case 6: return .Io(
message: try FfiConverterString.read(from: &buf)
)
- case 5: return .ItemNotFound(
+ case 7: return .ItemNotFound(
identifier: try FfiConverterString.read(from: &buf)
)
- case 6: return .UnsupportedItem(
+ case 8: return .Registry(
+ try FfiConverterTypeRegistryError.read(from: &buf)
+ )
+ case 9: return .UnsupportedItem(
identifier: try FfiConverterString.read(from: &buf)
)
@@ -3189,29 +3205,45 @@ public struct FfiConverterTypeStorageError: FfiConverterRustBuffer {
FfiConverterString.write(path, into: &buf)
- case let .DownloadManager(message):
+ case let .UnableToCreateDownloadManager(message):
writeInt(&buf, Int32(2))
FfiConverterString.write(message, into: &buf)
- case let .InvalidStateTransition(from,to):
+ case let .DownloadManager(message):
writeInt(&buf, Int32(3))
+ FfiConverterString.write(message, into: &buf)
+
+
+ case let .HashNotFound(identifier,name):
+ writeInt(&buf, Int32(4))
+ FfiConverterString.write(identifier, into: &buf)
+ FfiConverterString.write(name, into: &buf)
+
+
+ case let .InvalidStateTransition(from,to):
+ writeInt(&buf, Int32(5))
FfiConverterTypeDownloadPhase.write(from, into: &buf)
FfiConverterTypeDownloadPhase.write(to, into: &buf)
case let .Io(message):
- writeInt(&buf, Int32(4))
+ writeInt(&buf, Int32(6))
FfiConverterString.write(message, into: &buf)
case let .ItemNotFound(identifier):
- writeInt(&buf, Int32(5))
+ writeInt(&buf, Int32(7))
FfiConverterString.write(identifier, into: &buf)
+ case let .Registry(v1):
+ writeInt(&buf, Int32(8))
+ FfiConverterTypeRegistryError.write(v1, into: &buf)
+
+
case let .UnsupportedItem(identifier):
- writeInt(&buf, Int32(6))
+ writeInt(&buf, Int32(9))
FfiConverterString.write(identifier, into: &buf)
}
@@ -3767,7 +3799,7 @@ private let initializationResult: InitializationResult = {
if (uniffi_uzu_checksum_method_engine_models_for_translation() != 56970) {
return InitializationResult.apiChecksumMismatch
}
- if (uniffi_uzu_checksum_method_engine_models_on_device() != 11803) {
+ if (uniffi_uzu_checksum_method_engine_models_local() != 2150) {
return InitializationResult.apiChecksumMismatch
}
if (uniffi_uzu_checksum_method_engine_models_remote() != 36115) {
diff --git a/crates/legacy/uzu/bindings/swift/Tests/UzuTests/EngineTests.swift b/crates/legacy/uzu/bindings/swift/Tests/UzuTests/EngineTests.swift
index c4aee2336..895d53626 100644
--- a/crates/legacy/uzu/bindings/swift/Tests/UzuTests/EngineTests.swift
+++ b/crates/legacy/uzu/bindings/swift/Tests/UzuTests/EngineTests.swift
@@ -20,20 +20,14 @@ final class EngineTests: XCTestCase {
let session = try await engine.chat(model: model, config: .create())
let messages = [
- ChatMessage.system()
- .withText(text: "You are a helpful assistant")
- .withReasoningEffort(reasoningEffort: .disabled),
+ ChatMessage.system().withText(text: "You are a helpful assistant"),
ChatMessage.user().withText(text: "Hi"),
]
- let reply = try await session.reply(
- input: messages,
- config: .create()
- .withTokenLimit(tokenLimit: 64)
- .withSamplingMethod(samplingMethod: .greedy)
- )
+ let reply = try await session.reply(input: messages, config: .create())
let message = try XCTUnwrap(reply.last?.message, "Reply has no messages")
+ XCTAssertNotNil(message.reasoning())
XCTAssertNotNil(message.text())
}
}
diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts
index a662c9d67..96d38b508 100644
--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts
+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts
@@ -435,6 +435,7 @@ export declare class File {
hashes: Array<Hash>
constructor(url: string, name: string, size: number, hashes: Array<Hash>)
get crc32C(): string | null
+ get md5(): string | null
}
export declare class GrammarJsonAny {
@@ -491,13 +492,13 @@ export declare class Model {
encoding?: any
constructor(identifier: string, registry: ModelRegistry, backends: Array<ModelBackend>, family?: ModelFamily, properties?: ModelProperties, quantization?: ModelQuantization, specializations: Array<ModelSpecialization>, accessibility: ModelAccessibility, encoding?: any)
get name(): string
- get isOnDevice(): boolean
+ get isLocal(): boolean
get isRemote(): boolean
get isDownloadable(): boolean
get isQuantized(): boolean
get cacheIdentifier(): string
get repoIds(): Array<string>
- get filesystemPath(): string | null
+ get localExternalPath(): string | null
get referenceName(): string | null
get checkpointVersion(): string | null
@@ -509,9 +510,9 @@ export declare class Model {
get isSpeculationCapable(): boolean
}
-export declare class ModelAccessibilityOnDevice {
- source: ModelSource
- constructor(source: ModelSource)
+export declare class ModelAccessibilityLocal {
+ reference: ModelReference
+ constructor(reference: ModelReference)
}
export declare class ModelAccessibilityRemote {
@@ -554,19 +555,17 @@ export declare class ModelQuantization {
get name(): string
}
-export declare class ModelRegistry {
- identifier: string
- metadata: Metadata
- constructor(identifier: string, metadata: Metadata)
- get name(): string
+export declare class ModelReferenceHuggingFace {
+ repository: Repository
+ constructor(repository: Repository)
}
-export declare class ModelSourceFilesystem {
+export declare class ModelReferenceLocal {
path: string
constructor(path: string)
}
-export declare class ModelSourceRegistry {
+export declare class ModelReferenceMirai {
toolchainVersion: string
repository?: Repository
sourceRepository?: Repository
@@ -574,6 +573,13 @@ export declare class ModelSourceRegistry {
constructor(toolchainVersion: string, repository?: Repository, sourceRepository?: Repository, files: Array<File>)
}
+export declare class ModelRegistry {
+ identifier: string
+ metadata: Metadata
+ constructor(identifier: string, metadata: Metadata)
+ get name(): string
+}
+
export declare class ModelSpecializationChat {
constructor()
@@ -794,7 +800,8 @@ export type Grammar =
GrammarJsonAny | GrammarJsonSchema | GrammarRegex
export declare const enum HashMethod {
- CRC32C = 'CRC32C'
+ CRC32C = 'CRC32C',
+ MD5 = 'MD5'
}
export declare const enum ImageFormat {
@@ -809,10 +816,10 @@ export declare const enum ImageTheme {
}
export type ModelAccessibility =
- ModelAccessibilityOnDevice | ModelAccessibilityRemote
+ ModelAccessibilityLocal | ModelAccessibilityRemote
-export type ModelSource =
- ModelSourceRegistry | ModelSourceFilesystem
+export type ModelReference =
+ ModelReferenceMirai | ModelReferenceHuggingFace | ModelReferenceLocal
export type ModelSpecialization =
ModelSpecializationChat | ModelSpecializationClassification | ModelSpecializationTextToSpeech | ModelSpecializationTranslation | ModelSpecializationSpeculation
@@ -933,7 +940,7 @@ export declare class Engine {
removeRegistry(registryIdentifier: string): Promise<void>
removeBackend(identifier: string): Promise<void>
get models(): Promise<Array<Model>>
- get modelsOnDevice(): Promise<Array<Model>>
+ get modelsLocal(): Promise<Array<Model>>
get modelsRemote(): Promise<Array<Model>>
get modelsDownloadable(): Promise<Array<Model>>
get modelsForChat(): Promise<Array<Model>>
@@ -1057,8 +1064,11 @@ export type SettingsError =
export type StorageError =
| { type: 'UnableToCreateDirectory', path: string }
+ | { type: 'UnableToCreateDownloadManager', message: string }
| { type: 'DownloadManager', message: string }
+ | { type: 'HashNotFound', identifier: string, name: string }
| { type: 'InvalidStateTransition', from: DownloadPhase, to: DownloadPhase }
| { type: 'IO', message: string }
| { type: 'ItemNotFound', identifier: string }
+ | { type: 'Registry', field0: RegistryError }
| { type: 'UnsupportedItem', identifier: string }
diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts
index a662c9d67..96d38b508 100644
--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts
+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts
@@ -435,6 +435,7 @@ export declare class File {
hashes: Array<Hash>
constructor(url: string, name: string, size: number, hashes: Array<Hash>)
get crc32C(): string | null
+ get md5(): string | null
}
export declare class GrammarJsonAny {
@@ -491,13 +492,13 @@ export declare class Model {
encoding?: any
constructor(identifier: string, registry: ModelRegistry, backends: Array<ModelBackend>, family?: ModelFamily, properties?: ModelProperties, quantization?: ModelQuantization, specializations: Array<ModelSpecialization>, accessibility: ModelAccessibility, encoding?: any)
get name(): string
- get isOnDevice(): boolean
+ get isLocal(): boolean
get isRemote(): boolean
get isDownloadable(): boolean
get isQuantized(): boolean
get cacheIdentifier(): string
get repoIds(): Array<string>
- get filesystemPath(): string | null
+ get localExternalPath(): string | null
get referenceName(): string | null
get checkpointVersion(): string | null
@@ -509,9 +510,9 @@ export declare class Model {
get isSpeculationCapable(): boolean
}
-export declare class ModelAccessibilityOnDevice {
- source: ModelSource
- constructor(source: ModelSource)
+export declare class ModelAccessibilityLocal {
+ reference: ModelReference
+ constructor(reference: ModelReference)
}
export declare class ModelAccessibilityRemote {
@@ -554,19 +555,17 @@ export declare class ModelQuantization {
get name(): string
}
-export declare class ModelRegistry {
- identifier: string
- metadata: Metadata
- constructor(identifier: string, metadata: Metadata)
- get name(): string
+export declare class ModelReferenceHuggingFace {
+ repository: Repository
+ constructor(repository: Repository)
}
-export declare class ModelSourceFilesystem {
+export declare class ModelReferenceLocal {
path: string
constructor(path: string)
}
-export declare class ModelSourceRegistry {
+export declare class ModelReferenceMirai {
toolchainVersion: string
repository?: Repository
sourceRepository?: Repository
@@ -574,6 +573,13 @@ export declare class ModelSourceRegistry {
constructor(toolchainVersion: string, repository?: Repository, sourceRepository?: Repository, files: Array<File>)
}
+export declare class ModelRegistry {
+ identifier: string
+ metadata: Metadata
+ constructor(identifier: string, metadata: Metadata)
+ get name(): string
+}
+
export declare class ModelSpecializationChat {
constructor()
@@ -794,7 +800,8 @@ export type Grammar =
GrammarJsonAny | GrammarJsonSchema | GrammarRegex
export declare const enum HashMethod {
- CRC32C = 'CRC32C'
+ CRC32C = 'CRC32C',
+ MD5 = 'MD5'
}
export declare const enum ImageFormat {
@@ -809,10 +816,10 @@ export declare const enum ImageTheme {
}
export type ModelAccessibility =
- ModelAccessibilityOnDevice | ModelAccessibilityRemote
+ ModelAccessibilityLocal | ModelAccessibilityRemote
-export type ModelSource =
- ModelSourceRegistry | ModelSourceFilesystem
+export type ModelReference =
+ ModelReferenceMirai | ModelReferenceHuggingFace | ModelReferenceLocal
export type ModelSpecialization =
ModelSpecializationChat | ModelSpecializationClassification | ModelSpecializationTextToSpeech | ModelSpecializationTranslation | ModelSpecializationSpeculation
@@ -933,7 +940,7 @@ export declare class Engine {
removeRegistry(registryIdentifier: string): Promise<void>
removeBackend(identifier: string): Promise<void>
get models(): Promise<Array<Model>>
- get modelsOnDevice(): Promise<Array<Model>>
+ get modelsLocal(): Promise<Array<Model>>
get modelsRemote(): Promise<Array<Model>>
get modelsDownloadable(): Promise<Array<Model>>
get modelsForChat(): Promise<Array<Model>>
@@ -1057,8 +1064,11 @@ export type SettingsError =
export type StorageError =
| { type: 'UnableToCreateDirectory', path: string }
+ | { type: 'UnableToCreateDownloadManager', message: string }
| { type: 'DownloadManager', message: string }
+ | { type: 'HashNotFound', identifier: string, name: string }
| { type: 'InvalidStateTransition', from: DownloadPhase, to: DownloadPhase }
| { type: 'IO', message: string }
| { type: 'ItemNotFound', identifier: string }
+ | { type: 'Registry', field0: RegistryError }
| { type: 'UnsupportedItem', identifier: string }
diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.js b/crates/legacy/uzu/bindings/typescript/src/napi/index.js
index 1e4ed47fa..a919f8a45 100644
--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.js
+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.js
@@ -573,15 +573,16 @@ module.exports.Hash = nativeBinding.Hash
module.exports.Image = nativeBinding.Image
module.exports.Metadata = nativeBinding.Metadata
module.exports.Model = nativeBinding.Model
-module.exports.ModelAccessibilityOnDevice = nativeBinding.ModelAccessibilityOnDevice
+module.exports.ModelAccessibilityLocal = nativeBinding.ModelAccessibilityLocal
module.exports.ModelAccessibilityRemote = nativeBinding.ModelAccessibilityRemote
module.exports.ModelBackend = nativeBinding.ModelBackend
module.exports.ModelFamily = nativeBinding.ModelFamily
module.exports.ModelProperties = nativeBinding.ModelProperties
module.exports.ModelQuantization = nativeBinding.ModelQuantization
+module.exports.ModelReferenceHuggingFace = nativeBinding.ModelReferenceHuggingFace
+module.exports.ModelReferenceLocal = nativeBinding.ModelReferenceLocal
+module.exports.ModelReferenceMirai = nativeBinding.ModelReferenceMirai
module.exports.ModelRegistry = nativeBinding.ModelRegistry
-module.exports.ModelSourceFilesystem = nativeBinding.ModelSourceFilesystem
-module.exports.ModelSourceRegistry = nativeBinding.ModelSourceRegistry
module.exports.ModelSpecializationChat = nativeBinding.ModelSpecializationChat
module.exports.ModelSpecializationClassification = nativeBinding.ModelSpecializationClassification
module.exports.ModelSpecializationSpeculation = nativeBinding.ModelSpecializationSpeculation
diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs b/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs
index 07d34a3ce..624762937 100644
--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs
+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs
@@ -70,15 +70,16 @@ export const Hash = cjs.Hash;
export const Image = cjs.Image;
export const Metadata = cjs.Metadata;
export const Model = cjs.Model;
-export const ModelAccessibilityOnDevice = cjs.ModelAccessibilityOnDevice;
+export const ModelAccessibilityLocal = cjs.ModelAccessibilityLocal;
export const ModelAccessibilityRemote = cjs.ModelAccessibilityRemote;
export const ModelBackend = cjs.ModelBackend;
export const ModelFamily = cjs.ModelFamily;
export const ModelProperties = cjs.ModelProperties;
export const ModelQuantization = cjs.ModelQuantization;
+export const ModelReferenceHuggingFace = cjs.ModelReferenceHuggingFace;
+export const ModelReferenceLocal = cjs.ModelReferenceLocal;
+export const ModelReferenceMirai = cjs.ModelReferenceMirai;
export const ModelRegistry = cjs.ModelRegistry;
-export const ModelSourceFilesystem = cjs.ModelSourceFilesystem;
-export const ModelSourceRegistry = cjs.ModelSourceRegistry;
export const ModelSpecializationChat = cjs.ModelSpecializationChat;
export const ModelSpecializationClassification = cjs.ModelSpecializationClassification;
export const ModelSpecializationSpeculation = cjs.ModelSpecializationSpeculation;
diff --git a/crates/legacy/uzu/bindings/typescript/tests/engine.test.ts b/crates/legacy/uzu/bindings/typescript/tests/engine.test.ts
index 898c69854..d03bd7e05 100644
--- a/crates/legacy/uzu/bindings/typescript/tests/engine.test.ts
+++ b/crates/legacy/uzu/bindings/typescript/tests/engine.test.ts
@@ -1,12 +1,4 @@
-import {
- ChatConfig,
- ChatMessage,
- ChatReplyConfig,
- Engine,
- EngineConfig,
- ReasoningEffort,
- SamplingMethodGreedy,
-} from '@trymirai/uzu';
+import { ChatConfig, ChatMessage, ChatReplyConfig, Engine, EngineConfig } from '@trymirai/uzu';
jest.setTimeout(600_000);
@@ -23,18 +15,14 @@ test('chat reply produces text', async () => {
const session = await engine.chat(model!, ChatConfig.create());
const messages = [
- ChatMessage.system()
- .withText('You are a helpful assistant')
- .withReasoningEffort('Disabled' as ReasoningEffort),
+ ChatMessage.system().withText('You are a helpful assistant'),
ChatMessage.user().withText('Hi'),
];
- const reply = await session.reply(
- messages,
- ChatReplyConfig.create().withTokenLimit(64).withSamplingMethod(new SamplingMethodGreedy()),
- );
+ const reply = await session.reply(messages, ChatReplyConfig.create());
const message = reply[reply.length - 1]?.message;
expect(message).toBeDefined();
+ expect(message!.reasoning).not.toBeNull();
expect(message!.text).not.toBeNull();
});
diff --git a/crates/legacy/uzu/src/api/mod.rs b/crates/legacy/uzu/src/api/mod.rs
deleted file mode 100644
index 7dbc13c43..000000000
--- a/crates/legacy/uzu/src/api/mod.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-mod request;
-mod response;
-
-pub use request::FetchModelsRequest;
-pub use response::{FetchedModels, HuggingFaceFileResponse, HuggingFaceModelResponse, OpenAIModelsResponse};
diff --git a/crates/legacy/uzu/src/api/request.rs b/crates/legacy/uzu/src/api/request.rs
deleted file mode 100644
index d67697f09..000000000
--- a/crates/legacy/uzu/src/api/request.rs
+++ /dev/null
@@ -1,59 +0,0 @@
-use indexmap::IndexMap;
-use nagare::api::{Config, Endpoint, Payload};
-use reqwest::{Method, header::CONTENT_TYPE};
-use serde::Serialize;
-use serde_json::json;
-
-use crate::{device::Device, registry::mirai::Backend};
-
-#[derive(Serialize)]
-pub struct FetchModelsRequest {
- device: Device,
- backends: Vec<Backend>,
- include_traces: bool,
- show_all: bool,
-}
-
-impl FetchModelsRequest {
- pub fn new(
- device: Device,
- backends: Vec<Backend>,
- include_traces: bool,
- show_all: bool,
- ) -> Self {
- Self {
- device,
- backends,
- include_traces,
- show_all,
- }
- }
-}
-
-impl Endpoint for FetchModelsRequest {
- fn method(&self) -> Method {
- Method::POST
- }
-
- fn path(&self) -> String {
- "fetch/models".to_string()
- }
-
- fn headers(&self) -> IndexMap<String, String> {
- IndexMap::from([(CONTENT_TYPE.to_string(), "application/json".to_string())])
- }
-
- fn payload(
- &self,
- _: &Config,
- ) -> Payload {
- Payload {
- query: None,
- body: Some(json!(self)),
- }
- }
-}
-
-#[cfg(test)]
-#[path = "../../tests/unit/api/request_test.rs"]
-mod tests;
diff --git a/crates/legacy/uzu/src/api/response.rs b/crates/legacy/uzu/src/api/response.rs
deleted file mode 100644
index a620dedb2..000000000
--- a/crates/legacy/uzu/src/api/response.rs
+++ /dev/null
@@ -1,344 +0,0 @@
-use hanashi::chat::EncodingConfig;
-use serde::{Deserialize, Deserializer};
-use shoji::types::{
- basic::{File, Metadata, Repository, Value},
- model::{
- Model, ModelAccessibility, ModelBackend, ModelFamily, ModelProperties, ModelQuantization, ModelRegistry,
- ModelSource, ModelSpecialization, ModelVendor,
- },
-};
-
-fn get_metadata(
- metadatas: &[Metadata],
- identifier: &str,
-) -> Result<Metadata, String> {
- metadatas
- .iter()
- .find(|metadata| metadata.identifier == identifier)
- .cloned()
- .ok_or_else(|| format!("missing metadata {identifier}"))
-}
-
-#[derive(Deserialize)]
-#[serde(rename_all = "snake_case")]
-struct Registry {
- id: String,
- metadata_id: String,
-}
-
-impl Registry {
- fn to(
- &self,
- metadatas: &[Metadata],
- ) -> Result<ModelRegistry, String> {
- let metadata = get_metadata(metadatas, &self.metadata_id)?;
- Ok(ModelRegistry {
- identifier: self.id.clone(),
- metadata,
- })
- }
-}
-
-#[derive(Deserialize)]
-#[serde(rename_all = "snake_case")]
-struct Backend {
- id: String,
- version: String,
- metadata_id: String,
-}
-
-impl Backend {
- fn to(
- &self,
- metadatas: &[Metadata],
- ) -> Result<ModelBackend, String> {
- let metadata = get_metadata(metadatas, &self.metadata_id)?;
- Ok(ModelBackend {
- identifier: self.id.clone(),
- version: self.version.clone(),
- metadata,
- })
- }
-}
-
-#[derive(Deserialize)]
-#[serde(rename_all = "snake_case")]
-struct Vendor {
- id: String,
- metadata_id: String,
-}
-
-impl Vendor {
- fn to(
- &self,
- metadatas: &[Metadata],
- ) -> Result<ModelVendor, String> {
- let metadata = get_metadata(metadatas, &self.metadata_id)?;
- Ok(ModelVendor {
- identifier: self.id.clone(),
- metadata,
- })
- }
-}
-
-#[derive(Deserialize)]
-#[serde(rename_all = "snake_case")]
-struct Family {
- id: String,
- vendor: Vendor,
- metadata_id: String,
-}
-
-impl Family {
- fn to(
- &self,
- metadatas: &[Metadata],
- ) -> Result<ModelFamily, String> {
- let vendor = self.vendor.to(metadatas)?;
- let metadata = get_metadata(metadatas, &self.metadata_id)?;
- Ok(ModelFamily {
- identifier: self.id.clone(),
- vendor,
- metadata,
- })
- }
-}
-
-#[derive(Deserialize)]
-#[serde(rename_all = "snake_case")]
-struct Properties {
- id: String,
- size: i64,
- version: Option<String>,
- metadata_id: String,
-}
-
-impl Properties {
- fn to(
- &self,
- metadatas: &[Metadata],
- ) -> Result<ModelProperties, String> {
- let metadata = get_metadata(metadatas, &self.metadata_id)?;
- Ok(ModelProperties {
- identifier: self.id.clone(),
- size: self.size,
- version: self.version.clone(),
- metadata,
- })
- }
-}
-
-#[derive(Deserialize)]
-#[serde(rename_all = "snake_case")]
-struct Quantization {
- id: String,
- method: String,
- bits_per_weight: u32,
- vendor: Vendor,
- metadata_id: String,
-}
-
-impl Quantization {
- fn to(
- &self,
- metadatas: &[Metadata],
- ) -> Result<ModelQuantization, String> {
- let vendor = self.vendor.to(metadatas)?;
- let metadata = get_metadata(metadatas, &self.metadata_id)?;
- Ok(ModelQuantization {
- identifier: self.id.clone(),
- method: self.method.clone(),
- bits_per_weight: self.bits_per_weight,
- vendor,
- metadata,
- })
- }
-}
-
-#[derive(Deserialize)]
-#[serde(tag = "type", rename_all = "snake_case")]
-enum FetchedAccessibility {
- Local {
- reference: FetchedReference,
- },
- Remote {
- repository: Option<Repository>,
- },
-}
-
-impl From<&FetchedAccessibility> for ModelAccessibility {
- fn from(accessibility: &FetchedAccessibility) -> Self {
- match accessibility {
- FetchedAccessibility::Local {
- reference,
- } => Self::OnDevice {
- source: reference.into(),
- },
- FetchedAccessibility::Remote {
- repository,
- } => Self::Remote {
- repository: repository.clone(),
- },
- }
- }
-}
-
-#[derive(Deserialize)]
-#[serde(tag = "type", rename_all = "snake_case")]
-enum FetchedReference {
- Mirai {
- toolchain_version: String,
- repository: Option<Repository>,
- source_repository: Option<Repository>,
- files: Vec<File>,
- },
- Local {
- path: String,
- },
-}
-
-impl From<&FetchedReference> for ModelSource {
- fn from(reference: &FetchedReference) -> Self {
- match reference {
- FetchedReference::Mirai {
- toolchain_version,
- repository,
- source_repository,
- files,
- } => Self::Registry {
- toolchain_version: toolchain_version.clone(),
- repository: repository.clone(),
- source_repository: source_repository.clone(),
- files: files.clone(),
- },
- FetchedReference::Local {
- path,
- } => Self::Filesystem {
- path: path.clone(),
- },
- }
- }
-}
-
-#[derive(Deserialize)]
-#[serde(rename_all = "snake_case")]
-struct FetchedModel {
- id: String,
- registry: Registry,
- backends: Vec<Backend>,
- family: Option<Family>,
- properties: Option<Properties>,
- quantization: Option<Quantization>,
- specializations: Vec<ModelSpecialization>,
- accessibility: FetchedAccessibility,
- #[serde(rename = "encodings", default, deserialize_with = "deserialize_encoding")]
- encoding: Option<Value>,
-}
-
-fn deserialize_encoding<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Option<Value>, D::Error> {
- let encodings = Vec::<Value>::deserialize(deserializer)?;
- Ok(EncodingConfig::select(&encodings))
-}
-
-impl FetchedModel {
- fn to(
- &self,
- metadatas: &[Metadata],
- ) -> Result<Model, String> {
- let registry = self.registry.to(metadatas)?;
- let backends = self.backends.iter().map(|backend| backend.to(metadatas)).collect::<Result<Vec<_>, _>>()?;
- let family = self.family.as_ref().map(|family| family.to(metadatas)).transpose()?;
- let properties = self.properties.as_ref().map(|properties| properties.to(metadatas)).transpose()?;
- let quantization = self.quantization.as_ref().map(|quantization| quantization.to(metadatas)).transpose()?;
- Ok(Model {
- identifier: self.id.clone(),
- registry,
- backends,
- family,
- properties,
- quantization,
- specializations: self.specializations.clone(),
- accessibility: (&self.accessibility).into(),
- encoding: self.encoding.clone(),
- })
- }
-}
-
-#[derive(Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub struct FetchedModels {
- models: Vec<FetchedModel>,
- metadatas: Vec<Metadata>,
-}
-
-impl FetchedModels {
- pub fn models(&self) -> Result<Vec<Model>, String> {
- self.models.iter().map(|model| model.to(&self.metadatas)).collect()
- }
-}
-
-#[derive(Deserialize)]
-pub struct HuggingFaceModelResponse {
- pub sha: String,
- pub private: Option<bool>,
- pub gated: Option<HuggingFaceGatedResponse>,
- pub siblings: Vec<HuggingFaceFileResponse>,
-}
-
-impl HuggingFaceModelResponse {
- pub fn requires_authentication(&self) -> bool {
- if self.private != Some(false) {
- return true;
- }
- match &self.gated {
- Some(HuggingFaceGatedResponse::Boolean(false)) => false,
- Some(HuggingFaceGatedResponse::Approval(approval)) => {
- matches!(approval, HuggingFaceGatedApproval::Auto | HuggingFaceGatedApproval::Manual)
- },
- Some(HuggingFaceGatedResponse::Boolean(true)) | None => true,
- }
- }
-}
-
-#[derive(Deserialize)]
-#[serde(untagged)]
-pub enum HuggingFaceGatedResponse {
- Boolean(bool),
- Approval(HuggingFaceGatedApproval),
-}
-
-#[derive(Deserialize)]
-#[serde(rename_all = "lowercase")]
-pub enum HuggingFaceGatedApproval {
- Auto,
- Manual,
-}
-
-#[derive(Deserialize)]
-pub struct HuggingFaceFileResponse {
- pub rfilename: String,
- pub size: Option<u64>,
- #[serde(rename = "blobId")]
- pub blob_id: Option<String>,
- pub lfs: Option<HuggingFaceLfsResponse>,
-}
-
-#[derive(Deserialize)]
-pub struct HuggingFaceLfsResponse {
- pub sha256: Option<String>,
- pub size: Option<u64>,
-}
-
-#[derive(Deserialize)]
-pub struct OpenAIModelsResponse {
- pub data: Vec<OpenAIModelResponse>,
-}
-
-#[derive(Deserialize)]
-pub struct OpenAIModelResponse {
- pub id: String,
-}
-
-#[cfg(test)]
-#[path = "../../tests/unit/api/response_test.rs"]
-mod tests;
diff --git a/crates/legacy/uzu/src/engine/bridge/helpers.rs b/crates/legacy/uzu/src/engine/bridge/helpers.rs
index 6e612f036..1208b2b82 100644
--- a/crates/legacy/uzu/src/engine/bridge/helpers.rs
+++ b/crates/legacy/uzu/src/engine/bridge/helpers.rs
@@ -133,5 +133,41 @@ pub fn get_sampling_method<B: Backend>(
}
#[cfg(all(test, feature = "capability-grammar"))]
-#[path = "../../../tests/unit/engine/bridge/helpers_test.rs"]
-mod tests;
+mod tests {
+ use tokenizers::{Tokenizer, decoders::fuse::Fuse, models::wordlevel::WordLevel};
+
+ use super::grammar_trigger_token_sequence_for_prompt;
+
+ fn tokenizer() -> Tokenizer {
+ let model = WordLevel::builder()
+ .vocab(
+ [
+ ("<unk>".to_string(), 0),
+ ("</think>".to_string(), 1),
+ ("\n\n".to_string(), 2),
+ ("next".to_string(), 3),
+ ]
+ .into_iter()
+ .collect(),
+ )
+ .build()
+ .unwrap();
+ let mut tokenizer = Tokenizer::new(model);
+ tokenizer.with_decoder(Some(Fuse::new()));
+ tokenizer
+ }
+
+ #[test]
+ fn grammar_engages_immediately_when_prompt_closes_reasoning() {
+ let tokenizer = tokenizer();
+
+ assert_eq!(grammar_trigger_token_sequence_for_prompt(Some(&[1]), &[1, 2], &tokenizer), None);
+ }
+
+ #[test]
+ fn grammar_waits_for_trigger_when_only_history_contains_it() {
+ let tokenizer = tokenizer();
+
+ assert_eq!(grammar_trigger_token_sequence_for_prompt(Some(&[1]), &[1, 2, 3], &tokenizer), Some(vec![1]));
+ }
+}
diff --git a/crates/legacy/uzu/src/engine/config.rs b/crates/legacy/uzu/src/engine/config.rs
index 6875a9120..4c63aec0e 100644
--- a/crates/legacy/uzu/src/engine/config.rs
+++ b/crates/legacy/uzu/src/engine/config.rs
@@ -19,7 +19,7 @@ pub const KEY_BASETEN_API_KEY: &str = "BASETEN_API_KEY";
pub const KEY_OPENROUTER_API_KEY: &str = "OPENROUTER_API_KEY";
#[bindings::export(Structure(Class))]
-#[derive(Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub struct EngineConfig {
pub application_identifier: Option<String>,
diff --git a/crates/legacy/uzu/src/engine/downloader.rs b/crates/legacy/uzu/src/engine/downloader.rs
index e9bcf0d5f..feb6c2214 100644
--- a/crates/legacy/uzu/src/engine/downloader.rs
+++ b/crates/legacy/uzu/src/engine/downloader.rs
@@ -1,8 +1,5 @@
-use std::{sync::Weak, time::Duration};
-
-use kiban::time::sleep as kiban_sleep;
use serde::{Deserialize, Serialize};
-use tokio_stream::{StreamExt as TokioStreamExt, wrappers::BroadcastStream as TokioBroadcastStream};
+use tokio_stream::{StreamExt, wrappers::BroadcastStream};
use crate::{
engine::EngineError,
@@ -18,25 +15,22 @@ use crate::{
pub struct Downloader {
identifier: String,
storage: SharedAccess<Storage>,
- huggingface_api_key: Option<Weak<str>>,
}
impl Downloader {
pub fn new(
identifier: String,
storage: SharedAccess<Storage>,
- huggingface_api_key: Option<Weak<str>>,
) -> Self {
Self {
identifier,
storage,
- huggingface_api_key,
}
}
async fn wait_for_resume_to_be_observable(
&self,
- mut stream: TokioBroadcastStream<(String, DownloadState)>,
+ mut stream: BroadcastStream<(String, DownloadState)>,
) -> Result<(), EngineError> {
if self.state().await.is_some_and(|state| Self::is_resume_observable_phase(&state.phase)) {
return Ok(());
@@ -84,22 +78,25 @@ impl Downloader {
let state = self.state().await.ok_or(StorageError::ItemNotFound {
identifier: self.identifier.clone(),
})?;
- let reset = match state.phase {
- DownloadPhase::Downloading {} | DownloadPhase::Downloaded {} => return Ok(()),
+
+ match state.phase {
+ DownloadPhase::Downloading {} | DownloadPhase::Downloaded {} => Ok(()),
+ DownloadPhase::NotDownloaded {} | DownloadPhase::Paused {} | DownloadPhase::Locked {} => {
+ let stream = self.storage.lock().await.subscribe();
+ self.storage.lock().await.download(&self.identifier).await?;
+ self.wait_for_resume_to_be_observable(stream).await
+ },
DownloadPhase::Error {
..
- } => true,
- DownloadPhase::NotDownloaded {} | DownloadPhase::Paused {} | DownloadPhase::Locked {} => false,
- };
- let stream = self.storage.lock().await.subscribe();
- let storage = self.storage.lock().await;
- if reset {
- storage.delete(&self.identifier).await?;
+ } => {
+ let stream = self.storage.lock().await.subscribe();
+ let storage = self.storage.lock().await;
+ storage.delete(&self.identifier).await?;
+ storage.download(&self.identifier).await?;
+ drop(storage);
+ self.wait_for_resume_to_be_observable(stream).await
+ },
}
- let huggingface_api_key = self.huggingface_api_key.as_ref().and_then(Weak::upgrade);
- storage.download(&self.identifier, huggingface_api_key).await?;
- drop(storage);
- self.wait_for_resume_to_be_observable(stream).await
}
#[bindings::export(Method)]
@@ -129,14 +126,14 @@ impl Downloader {
#[derive(Clone)]
pub struct DownloaderStream {
identifier: String,
- stream: SharedAccess<Option<TokioBroadcastStream<(String, DownloadState)>>>,
+ stream: SharedAccess<Option<BroadcastStream<(String, DownloadState)>>>,
storage: Option<SharedAccess<Storage>>,
}
impl DownloaderStream {
pub(crate) fn new(
identifier: String,
- stream: TokioBroadcastStream<(String, DownloadState)>,
+ stream: BroadcastStream<(String, DownloadState)>,
storage: SharedAccess<Storage>,
) -> Self {
Self {
@@ -170,7 +167,7 @@ impl DownloaderStream {
Some(_) => {},
None => return true,
}
- kiban_sleep(Duration::from_millis(20)).await;
+ tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
true
}
diff --git a/crates/legacy/uzu/src/engine/mod.rs b/crates/legacy/uzu/src/engine/mod.rs
index 0d3a362c6..ea53ea698 100644
--- a/crates/legacy/uzu/src/engine/mod.rs
+++ b/crates/legacy/uzu/src/engine/mod.rs
@@ -14,7 +14,7 @@ pub use download_manager::DownloadManagerType;
pub use downloader::{Downloader, DownloaderStream, DownloaderStreamUpdate};
pub use error::EngineError;
use indexmap::{IndexMap, IndexSet};
-use kiban::rt::{RuntimeHandle as KibanRuntimeHandle, spawn as kiban_spawn};
+use kiban::rt::RuntimeHandle;
use nagare::{
api::Config as ClientConfig,
chat::{ChatInstance, ChatSession},
@@ -29,21 +29,17 @@ use shoji::{
session::chat::ChatConfig,
},
};
-use tokio_stream::{StreamExt as TokioStreamExt, wrappers::BroadcastStream as TokioBroadcastStream};
+use tokio_stream::{StreamExt, wrappers::BroadcastStream};
use crate::{
device::Device,
engine::bridge::UzuLlmBackend,
helpers::{SharedAccess, is_endpoint_reachable},
logs,
- models::ModelCatalog,
registry::{
CachedRegistry, MergedRegistry, RegistryError,
local::{Config as LocalRegistryConfig, Registry as LocalRegistry},
- mirai::{
- Backend as MiraiBackend, Config as MiraiRegistryConfig, MIRAI_API_HOST, MIRAI_API_SCHEME,
- Registry as MiraiRegistry,
- },
+ mirai::{Backend as MiraiBackend, Config as MiraiRegistryConfig, Registry as MiraiRegistry},
openai::{Config as OpenAIConfig, Registry as OpenAIRegistry},
},
settings::Settings,
@@ -58,18 +54,15 @@ use crate::{
pub struct Engine {
settings: SharedAccess<Option<Settings>>,
registry: SharedAccess<MergedRegistry>,
- catalog: ModelCatalog,
- catalog_refresh_lock: SharedAccess<()>,
storage: SharedAccess<Storage>,
backends: SharedAccess<HashMap<String, Arc<dyn Backend>>>,
callback: SharedAccess<Option<Arc<EngineCallback>>>,
telemetry: SharedAccess<Telemetry>,
- huggingface_api_key: Option<Arc<str>>,
}
impl Engine {
pub async fn new(config: EngineConfig) -> Result<Self, EngineError> {
- let runtime_handle = KibanRuntimeHandle::try_current().map_err(|error| EngineError::TokioError {
+ let runtime_handle = RuntimeHandle::try_current().map_err(|error| EngineError::TokioError {
message: error.to_string(),
})?;
@@ -87,7 +80,7 @@ impl Engine {
let telemetry = SharedAccess::new({
let client_config = ClientConfig::new(
- format!("{MIRAI_API_SCHEME}://{MIRAI_API_HOST}/api/v2"),
+ "https://sdk.trymirai.com/api/v2".to_string(),
Duration::from_secs(10),
IndexMap::new(),
);
@@ -110,21 +103,15 @@ impl Engine {
let storage_cache_path = storage_config.cache_path();
logs::start(storage_config.cache_path(), &storage_config.log_name(), false);
- let huggingface_api_key = config.huggingface_api_key.clone().map(Arc::<str>::from);
- let catalog =
- ModelCatalog::new(huggingface_api_key.clone(), storage_cache_path.join("resolved-models.json")).await?;
let storage = SharedAccess::new(Storage::new(runtime_handle, storage_config).await?);
let engine = Self {
settings: SharedAccess::new(settings),
storage,
registry,
- catalog,
- catalog_refresh_lock: SharedAccess::new(()),
backends: SharedAccess::new(HashMap::new()),
callback: SharedAccess::new(None),
telemetry,
- huggingface_api_key,
};
engine.spawn_storage_listener().await;
@@ -141,11 +128,12 @@ impl Engine {
version: uzu_backend_version.clone(),
}],
include_traces: false,
+ cache_path: storage_cache_path,
};
let mirai_registry = Box::new(MiraiRegistry::new(mirai_registry_config)?);
engine.add_backend(Arc::new(uzu_backend) as Arc<dyn Backend>).await;
- engine.registry.lock().await.add(Box::new(CachedRegistry::new(mirai_registry)))?;
+ engine.add_registry(mirai_registry).await?;
if let Some(lalamo_path) = config.lalamo_path {
let lalamo_registry = LocalRegistry::new(LocalRegistryConfig::lalamo(
@@ -153,7 +141,7 @@ impl Engine {
uzu_backend_version.clone(),
lalamo_path,
))?;
- engine.registry.lock().await.add(Box::new(CachedRegistry::new(Box::new(lalamo_registry))))?;
+ engine.add_registry(Box::new(lalamo_registry)).await?;
}
if let Some(local_path) = config.local_path {
let local_registry = LocalRegistry::new(LocalRegistryConfig::local(
@@ -161,7 +149,7 @@ impl Engine {
uzu_backend_version.clone(),
local_path,
))?;
- engine.registry.lock().await.add(Box::new(CachedRegistry::new(Box::new(local_registry))))?;
+ engine.add_registry(Box::new(local_registry)).await?;
}
}
@@ -199,19 +187,14 @@ impl Engine {
for config in openai_configs {
let registry = OpenAIRegistry::new(config.clone())?;
let backend = OpenAIBackend::new(config.into()).map_err(|_| EngineError::UnableToCreateBackend {})?;
- engine.registry.lock().await.add(Box::new(CachedRegistry::new(Box::new(registry))))?;
+ engine.add_registry(Box::new(registry)).await?;
engine.add_backend(Arc::new(backend) as Arc<dyn Backend>).await;
}
- engine.handle_initial_catalog_refresh().await?;
Ok(engine)
}
}
-#[cfg(test)]
-#[path = "../../tests/unit/engine/cache_test.rs"]
-mod tests;
-
#[bindings::export(Implementation)]
impl Engine {
#[bindings::export(Method(Factory))]
@@ -237,15 +220,8 @@ impl Engine {
&self,
registry: Box<dyn Registry<Error = RegistryError>>,
) -> Result<(), EngineError> {
- let identifier = registry.indentifier();
- let refresh = self.catalog_refresh_lock.lock().await;
self.registry.lock().await.add(Box::new(CachedRegistry::new(registry)))?;
- if let Err(error) = self.refresh_catalog().await {
- self.registry.lock().await.remove(&identifier);
- return Err(error);
- }
- drop(refresh);
- self.notify_callback().await;
+ self.handle_registry_refresh().await?;
Ok(())
}
@@ -264,16 +240,8 @@ impl Engine {
&self,
registry_identifier: String,
) -> Result<(), EngineError> {
- let refresh = self.catalog_refresh_lock.lock().await;
- let removed = self.registry.lock().await.remove(&registry_identifier);
- if let Err(error) = self.refresh_catalog().await {
- if let Some((index, registry)) = removed {
- self.registry.lock().await.restore(index, registry);
- }
- return Err(error);
- }
- drop(refresh);
- self.notify_callback().await;
+ self.registry.lock().await.remove(&registry_identifier)?;
+ self.handle_registry_refresh().await?;
Ok(())
}
@@ -290,12 +258,12 @@ impl Engine {
impl Engine {
#[bindings::export(Method(Getter))]
pub async fn models(&self) -> Result<Vec<Model>, EngineError> {
- Ok(self.catalog.models().await)
+ self.registry.lock().await.models().await.map_err(EngineError::from)
}
#[bindings::export(Method(Getter))]
- pub async fn models_on_device(&self) -> Result<Vec<Model>, EngineError> {
- Ok(self.models().await?.into_iter().filter(|model| model.is_on_device()).collect())
+ pub async fn models_local(&self) -> Result<Vec<Model>, EngineError> {
+ Ok(self.models().await?.into_iter().filter(|model| model.is_local()).collect())
}
#[bindings::export(Method(Getter))]
@@ -422,7 +390,7 @@ impl Engine {
&self,
identifier: String,
) -> Result<Option<Model>, EngineError> {
- Ok(self.models().await?.into_iter().find(|model| model.identifier == identifier))
+ self.registry.lock().await.model_by_identifier(&identifier).await.map_err(EngineError::from)
}
#[bindings::export(Method)]
@@ -430,7 +398,7 @@ impl Engine {
&self,
repo_id: String,
) -> Result<Option<Model>, EngineError> {
- Ok(self.models().await?.into_iter().find(|model| model.repo_ids().contains(&repo_id)))
+ self.registry.lock().await.model_by_repo_id(&repo_id).await.map_err(EngineError::from)
}
#[bindings::export(Method)]
@@ -455,11 +423,11 @@ impl Engine {
&self,
model: &Model,
) -> Option<String> {
- if !model.is_on_device() {
+ if !model.is_local() {
return None;
}
- if let Some(filesystem_path) = model.filesystem_path() {
- return Some(filesystem_path);
+ if let Some(local_external_path) = model.local_external_path() {
+ return Some(local_external_path);
}
let storage = self.storage.lock().await;
let state = storage.state(&model.identifier).await?;
@@ -482,11 +450,7 @@ impl Engine {
&self,
model: &Model,
) -> Downloader {
- Downloader::new(
- model.identifier.clone(),
- self.storage.clone(),
- self.huggingface_api_key.as_ref().map(Arc::downgrade),
- )
+ Downloader::new(model.identifier.clone(), self.storage.clone())
}
#[bindings::export(Method)]
@@ -603,45 +567,24 @@ impl Engine {
}
impl Engine {
- pub async fn storage_subscribe(&self) -> TokioBroadcastStream<(String, DownloadState)> {
+ pub async fn storage_subscribe(&self) -> BroadcastStream<(String, DownloadState)> {
self.storage.lock().await.subscribe()
}
- async fn refresh_catalog(&self) -> Result<(), EngineError> {
+ async fn handle_registry_refresh(&self) -> Result<(), EngineError> {
let models = self.registry.lock().await.models().await?;
- let resolved = self.catalog.resolve(models).await?;
- self.storage.lock().await.refresh(&resolved, self.huggingface_api_key.clone()).await?;
- self.catalog.commit(resolved).await;
- Ok(())
- }
-
- async fn handle_initial_catalog_refresh(&self) -> Result<(), EngineError> {
- let _refresh = self.catalog_refresh_lock.lock().await;
- match self.refresh_catalog().await {
- Ok(()) => Ok(()),
- Err(error) => {
- let previous = self.catalog.snapshot().await;
- if previous.is_empty() {
- return Err(error);
- }
- tracing::warn!(?error, "serving previous resolved models after initial refresh failure");
- self.storage.lock().await.refresh(&previous, self.huggingface_api_key.clone()).await?;
- Ok(())
- },
- }
- }
-
- async fn notify_callback(&self) {
+ self.storage.lock().await.refresh(models).await?;
if let Some(callback) = self.callback.lock().await.as_ref().cloned() {
callback.on_event();
- }
+ };
+ Ok(())
}
async fn spawn_storage_listener(&self) {
let mut stream = self.storage_subscribe().await;
let callback = self.callback.clone();
let telemetry = self.telemetry.lock().await.clone();
- kiban_spawn(async move {
+ tokio::spawn(async move {
let mut last_phase: HashMap<String, DownloadPhase> = HashMap::new();
while let Some(update) = stream.next().await {
let Ok((id, state)) = update else {
diff --git a/crates/legacy/uzu/src/lib.rs b/crates/legacy/uzu/src/lib.rs
index 41226d96a..6db09af94 100644
--- a/crates/legacy/uzu/src/lib.rs
+++ b/crates/legacy/uzu/src/lib.rs
@@ -31,8 +31,6 @@ pyo3_stub_gen::define_stub_info_gatherer!(pyo3_bindings_annotations);
#[cfg(feature = "bindings-pyo3")]
pyo3_stub_gen::reexport_module_members!("uzu" from "uzu._tool"; "UzuToolFunction", "uzu_tool_function");
-#[cfg(not(target_family = "wasm"))]
-mod api;
#[cfg(not(target_family = "wasm"))]
pub mod device;
#[cfg(not(target_family = "wasm"))]
@@ -42,8 +40,6 @@ pub mod helpers;
#[cfg(not(target_family = "wasm"))]
pub mod logs;
#[cfg(not(target_family = "wasm"))]
-mod models;
-#[cfg(not(target_family = "wasm"))]
pub mod player;
#[cfg(not(target_family = "wasm"))]
pub mod registry;
diff --git a/crates/legacy/uzu/src/models/catalog.rs b/crates/legacy/uzu/src/models/catalog.rs
deleted file mode 100644
index a4f80c0da..000000000
--- a/crates/legacy/uzu/src/models/catalog.rs
+++ /dev/null
@@ -1,59 +0,0 @@
-use std::{path::PathBuf, sync::Arc};
-
-use shoji::types::model::Model;
-
-use super::{ModelsResolver, ResolvedModels};
-use crate::{helpers::SharedAccess, registry::RegistryError};
-
-#[derive(Clone)]
-pub struct ModelCatalog {
- resolver: ModelsResolver,
- published: SharedAccess<ResolvedModels>,
-}
-
-impl ModelCatalog {
- pub async fn new(
- api_key: Option<Arc<str>>,
- cache_path: PathBuf,
- ) -> Result<Self, RegistryError> {
- let resolver = ModelsResolver::new(api_key, cache_path)?;
- let published = match resolver.load_cache().await {
- Ok(Some(models)) => models,
- Ok(None) => ResolvedModels::default(),
- Err(error) => {
- tracing::warn!(?error, "failed to load resolved models cache");
- ResolvedModels::default()
- },
- };
- Ok(Self {
- resolver,
- published: SharedAccess::new(published),
- })
- }
-
- pub async fn models(&self) -> Vec<Model> {
- self.published.lock().await.models()
- }
-
- pub async fn snapshot(&self) -> ResolvedModels {
- self.published.lock().await.clone()
- }
-
- pub async fn resolve(
- &self,
- models: Vec<Model>,
- ) -> Result<ResolvedModels, RegistryError> {
- let previous = self.snapshot().await;
- self.resolver.resolve(models, &previous).await
- }
-
- pub async fn commit(
- &self,
- models: ResolvedModels,
- ) {
- if let Err(error) = self.resolver.save_cache(&models).await {
- tracing::warn!(?error, "failed to save resolved models cache");
- }
- *self.published.lock().await = models;
- }
-}
diff --git a/crates/legacy/uzu/src/models/mod.rs b/crates/legacy/uzu/src/models/mod.rs
deleted file mode 100644
index 631aa0401..000000000
--- a/crates/legacy/uzu/src/models/mod.rs
+++ /dev/null
@@ -1,7 +0,0 @@
-mod catalog;
-mod resolved;
-mod resolver;
-
-pub use catalog::ModelCatalog;
-pub use resolved::{ResolvedFile, ResolvedModel, ResolvedModels};
-pub use resolver::ModelsResolver;
diff --git a/crates/legacy/uzu/src/models/resolved.rs b/crates/legacy/uzu/src/models/resolved.rs
deleted file mode 100644
index d181e239a..000000000
--- a/crates/legacy/uzu/src/models/resolved.rs
+++ /dev/null
@@ -1,104 +0,0 @@
-use download_manager::FileCheck;
-use serde::{Deserialize, Serialize};
-use shoji::types::{basic::File, model::Model};
-
-const CACHE_VERSION: u8 = 2;
-
-#[derive(Clone, Serialize, Deserialize)]
-pub struct ResolvedModels {
- version: u8,
- models: Vec<ResolvedModel>,
- #[serde(skip)]
- reusable_hugging_face_files: bool,
-}
-
-impl Default for ResolvedModels {
- fn default() -> Self {
- Self {
- version: CACHE_VERSION,
- models: Vec::new(),
- reusable_hugging_face_files: false,
- }
- }
-}
-
-#[derive(Clone, Serialize, Deserialize)]
-pub struct ResolvedModel {
- model: Model,
- files: Option<Vec<ResolvedFile>>,
-}
-
-#[derive(Clone, PartialEq, Serialize, Deserialize)]
-pub struct ResolvedFile {
- pub file: File,
- pub check: FileCheck,
- pub requires_authentication: bool,
-}
-
-impl ResolvedModels {
- pub fn new(models: Vec<ResolvedModel>) -> Self {
- Self {
- version: CACHE_VERSION,
- models,
- reusable_hugging_face_files: true,
- }
- }
-
- pub fn models(&self) -> Vec<Model> {
- self.models.iter().map(|resolved| resolved.model.clone()).collect()
- }
-
- pub fn iter(&self) -> impl Iterator<Item = &ResolvedModel> {
- self.models.iter()
- }
-
- pub fn is_empty(&self) -> bool {
- self.models.is_empty()
- }
-
- pub fn validate_cache(mut self) -> Option<Self> {
- if self.version != CACHE_VERSION {
- return None;
- }
- self.reusable_hugging_face_files = false;
- Some(self)
- }
-
- pub fn reusable_hugging_face_files(
- &self,
- model: &Model,
- ) -> Option<Vec<ResolvedFile>> {
- if !self.reusable_hugging_face_files {
- return None;
- }
- self.models
- .iter()
- .find(|resolved| {
- resolved.model.identifier == model.identifier && resolved.model.accessibility == model.accessibility
- })
- .and_then(|resolved| resolved.files.clone())
- }
-}
-
-impl ResolvedModel {
- pub fn downloadable(
- model: Model,
- files: Vec<ResolvedFile>,
- ) -> Self {
- Self {
- model,
- files: Some(files),
- }
- }
-
- pub fn passthrough(model: Model) -> Self {
- Self {
- model,
- files: None,
- }
- }
-
- pub fn parts(&self) -> (&Model, Option<&[ResolvedFile]>) {
- (&self.model, self.files.as_deref())
- }
-}
diff --git a/crates/legacy/uzu/src/models/resolver.rs b/crates/legacy/uzu/src/models/resolver.rs
deleted file mode 100644
index 36d304802..000000000
--- a/crates/legacy/uzu/src/models/resolver.rs
+++ /dev/null
@@ -1,276 +0,0 @@
-use std::{collections::HashSet, io::ErrorKind, path::PathBuf, sync::Arc, time::Duration};
-
-use download_manager::FileCheck;
-use futures_util::{StreamExt, TryStreamExt, stream};
-use kiban::fs::asyn::{
- create_dir_all as kiban_create_dir_all, read as kiban_read, rename as kiban_rename, write as kiban_write,
-};
-use reqwest::{Client, Url};
-use serde_json::{from_slice, to_vec};
-use shoji::types::{
- basic::{File, Repository},
- model::{Model, ModelAccessibility, ModelSource},
-};
-use uuid::Uuid;
-
-use super::{ResolvedFile, ResolvedModel, ResolvedModels};
-use crate::{
- api::{HuggingFaceFileResponse, HuggingFaceModelResponse},
- registry::RegistryError,
-};
-
-const HUGGING_FACE_URL: &str = "https://huggingface.co";
-const MAX_CONCURRENT_REQUESTS: usize = 8;
-const SIDECAR_EXTENSIONS: [&str; 4] = ["part", "resume_data", "lock", "integrity"];
-
-#[derive(Clone)]
-pub struct ModelsResolver {
- client: Client,
- endpoint: Url,
- api_key: Option<Arc<str>>,
- cache_path: PathBuf,
-}
-
-impl ModelsResolver {
- pub fn new(
- api_key: Option<Arc<str>>,
- cache_path: PathBuf,
- ) -> Result<Self, RegistryError> {
- let client =
- Client::builder().https_only(true).timeout(Duration::from_secs(30)).build().map_err(resolution_error)?;
- let endpoint = Url::parse(HUGGING_FACE_URL).map_err(resolution_error)?;
- Ok(Self {
- client,
- endpoint,
- api_key,
- cache_path,
- })
- }
-
- pub async fn load_cache(&self) -> Result<Option<ResolvedModels>, RegistryError> {
- let contents = match kiban_read(&self.cache_path).await {
- Ok(contents) => contents,
- Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None),
- Err(error) => return Err(resolution_error(error)),
- };
- let models = from_slice::<ResolvedModels>(&contents).map_err(resolution_error)?;
- Ok(models.validate_cache())
- }
-
- pub async fn save_cache(
- &self,
- models: &ResolvedModels,
- ) -> Result<(), RegistryError> {
- let parent = self.cache_path.parent().ok_or_else(|| resolution_error("invalid resolved models cache path"))?;
- kiban_create_dir_all(parent).await.map_err(resolution_error)?;
- let temporary = self.cache_path.with_added_extension(format!("{}.tmp", Uuid::new_v4()));
- kiban_write(&temporary, to_vec(models).map_err(resolution_error)?).await.map_err(resolution_error)?;
- kiban_rename(temporary, &self.cache_path).await.map_err(resolution_error)
- }
-
- pub async fn resolve(
- &self,
- models: Vec<Model>,
- previous: &ResolvedModels,
- ) -> Result<ResolvedModels, RegistryError> {
- let futures = models.into_iter().map(|model| self.resolve_model(model, previous));
- let resolved = stream::iter(futures).buffered(MAX_CONCURRENT_REQUESTS).try_collect::<Vec<_>>().await?;
- Ok(ResolvedModels::new(resolved))
- }
-
- async fn resolve_model(
- &self,
- model: Model,
- previous: &ResolvedModels,
- ) -> Result<ResolvedModel, RegistryError> {
- let ModelAccessibility::OnDevice {
- source:
- ModelSource::Registry {
- repository,
- files,
- ..
- },
- } = &model.accessibility
- else {
- return Ok(ResolvedModel::passthrough(model));
- };
- let Some((repository, revision)) = repository
- .as_ref()
- .and_then(|repository| repository.commit_hash.as_deref().map(|revision| (repository, revision)))
- else {
- let files = files
- .iter()
- .cloned()
- .map(|file| {
- let check = file.crc32c().map(FileCheck::CRC).ok_or_else(|| {
- resolution_error(format!("missing CRC32C for {}/{}", model.identifier, file.name))
- })?;
- Ok(ResolvedFile {
- file,
- check,
- requires_authentication: false,
- })
- })
- .collect::<Result<Vec<_>, RegistryError>>()?;
- return Ok(ResolvedModel::downloadable(model, files));
- };
- if let Some(files) = previous.reusable_hugging_face_files(&model) {
- return Ok(ResolvedModel::downloadable(model, files));
- }
- let files = self.resolve_hugging_face(repository, revision).await?;
- Ok(ResolvedModel::downloadable(model, files))
- }
-
- async fn resolve_hugging_face(
- &self,
- repository: &Repository,
- revision: &str,
- ) -> Result<Vec<ResolvedFile>, RegistryError> {
- if !is_lower_hex(revision, 40) {
- return Err(resolution_error("invalid commit"));
- }
- let mut metadata_url = self.endpoint.clone();
- metadata_url
- .path_segments_mut()
- .map_err(|_| resolution_error("invalid Hugging Face endpoint"))?
- .clear()
- .extend(["api", "models"])
- .extend(repository.identifier.split('/'))
- .extend(["revision", revision]);
- metadata_url.query_pairs_mut().append_pair("blobs", "true");
- let mut request = self.client.get(metadata_url);
- if let Some(api_key) = &self.api_key {
- request = request.bearer_auth(api_key.as_ref());
- }
- let response = request.send().await.map_err(resolution_error)?;
- if !response.status().is_success() {
- return Err(resolution_error(format!("Hugging Face returned HTTP {}", response.status().as_u16())));
- }
- let response: HuggingFaceModelResponse = response.json().await.map_err(resolution_error)?;
- if response.sha != revision {
- return Err(resolution_error(format!(
- "Hugging Face returned revision {} instead of {revision}",
- response.sha
- )));
- }
- let requires_authentication = response.requires_authentication();
- if requires_authentication && self.api_key.is_none() {
- return Err(resolution_error("Hugging Face authentication is required"));
- }
- self.resolve_files(repository, revision, response.siblings, requires_authentication)
- }
-
- fn resolve_files(
- &self,
- repository: &Repository,
- revision: &str,
- siblings: Vec<HuggingFaceFileResponse>,
- requires_authentication: bool,
- ) -> Result<Vec<ResolvedFile>, RegistryError> {
- let mut resolve_base = self.endpoint.clone();
- resolve_base
- .path_segments_mut()
- .map_err(|_| resolution_error("invalid Hugging Face endpoint"))?
- .clear()
- .extend(repository.identifier.split('/'))
- .extend(["resolve", revision]);
- let mut names = HashSet::with_capacity(siblings.len());
- let mut files = Vec::with_capacity(siblings.len());
- for sibling in siblings
- .iter()
- .filter(|sibling| repository.paths.as_ref().is_none_or(|paths| paths.contains(&sibling.rfilename)))
- {
- validate_path(&sibling.rfilename)?;
- let normalized_name = PathBuf::from(sibling.rfilename.to_lowercase());
- if !names.insert(normalized_name) {
- return Err(invalid_file(&sibling.rfilename));
- }
- let (size, check) = file_size_and_check(sibling)?;
- let mut url = resolve_base.clone();
- url.path_segments_mut()
- .map_err(|_| resolution_error("invalid Hugging Face endpoint"))?
- .extend(sibling.rfilename.split('/'));
- files.push(ResolvedFile {
- file: File {
- url: url.into(),
- name: sibling.rfilename.clone(),
- size: i64::try_from(size).map_err(|_| invalid_file("file size overflow"))?,
- hashes: Vec::new(),
- },
- check,
- requires_authentication,
- });
- }
- let sidecars = names
- .iter()
- .flat_map(|name| SIDECAR_EXTENSIONS.iter().map(|extension| name.with_added_extension(extension)))
- .collect::<HashSet<_>>();
- let collides = names.iter().any(|name| {
- name.ancestors().skip(1).any(|parent| names.contains(parent))
- || name.ancestors().any(|path| sidecars.contains(path))
- });
- if files.is_empty() || collides {
- return Err(invalid_file("empty or colliding manifest"));
- }
- Ok(files)
- }
-}
-
-fn file_size_and_check(file: &HuggingFaceFileResponse) -> Result<(u64, FileCheck), RegistryError> {
- if let Some(lfs) = &file.lfs {
- if let (Some(size), Some(lfs_size)) = (file.size, lfs.size)
- && size != lfs_size
- {
- return Err(resolution_error(format!("Hugging Face size mismatch for {}", file.rfilename)));
- }
- let size = lfs
- .size
- .or(file.size)
- .ok_or_else(|| resolution_error(format!("Hugging Face is missing size for {}", file.rfilename)))?;
- let digest = lfs
- .sha256
- .as_deref()
- .filter(|digest| is_lower_hex(digest, 64))
- .ok_or_else(|| resolution_error(format!("missing or invalid SHA-256 for {}", file.rfilename)))?;
- Ok((size, FileCheck::Sha256(digest.to_string())))
- } else {
- let size = file
- .size
- .ok_or_else(|| resolution_error(format!("Hugging Face is missing size for {}", file.rfilename)))?;
- let digest = file
- .blob_id
- .as_deref()
- .filter(|digest| is_lower_hex(digest, 40))
- .ok_or_else(|| resolution_error(format!("missing or invalid Git blob SHA-1 for {}", file.rfilename)))?;
- Ok((size, FileCheck::GitBlobSha1(digest.to_string())))
- }
-}
-
-fn validate_path(path: &str) -> Result<(), RegistryError> {
- if path.contains(['\\', ':']) || path.split('/').any(|segment| segment.is_empty() || matches!(segment, "." | ".."))
- {
- return Err(invalid_file(path));
- }
- Ok(())
-}
-
-fn is_lower_hex(
- value: &str,
- length: usize,
-) -> bool {
- value.len() == length && value.bytes().all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f'))
-}
-
-fn invalid_file(path: &str) -> RegistryError {
- resolution_error(format!("invalid file: {path}"))
-}
-
-fn resolution_error(message: impl ToString) -> RegistryError {
- RegistryError::UnableToGetModels {
- message: message.to_string(),
- }
-}
-
-#[cfg(test)]
-#[path = "../../tests/unit/models/resolver_test.rs"]
-mod tests;
diff --git a/crates/legacy/uzu/src/registry/local/config.rs b/crates/legacy/uzu/src/registry/local/config.rs
index 6b6ae2291..b6c189a6a 100644
--- a/crates/legacy/uzu/src/registry/local/config.rs
+++ b/crates/legacy/uzu/src/registry/local/config.rs
@@ -77,8 +77,8 @@ impl Config {
}
fn resolve_model(mut model: Model) -> Result<Model, RegistryError> {
- let model_path = model.filesystem_path().ok_or_else(|| RegistryError::UnableToGetModels {
- message: format!("Filesystem model {} has no path", model.identifier),
+ let model_path = model.local_external_path().ok_or_else(|| RegistryError::UnableToGetModels {
+ message: format!("Local model {} has no local path", model.identifier),
})?;
let model_path = PathBuf::from(model_path);
let specialization = Self::resolve_model_specialization(&model_path)?;
diff --git a/crates/legacy/uzu/src/registry/local/mod.rs b/crates/legacy/uzu/src/registry/local/mod.rs
index 331d87adc..15c0842c2 100644
--- a/crates/legacy/uzu/src/registry/local/mod.rs
+++ b/crates/legacy/uzu/src/registry/local/mod.rs
@@ -8,7 +8,7 @@ use shoji::{
traits::Registry as RegistryTrait,
types::{
basic::Value,
- model::{Model, ModelAccessibility, ModelSource, ModelSpecialization},
+ model::{Model, ModelAccessibility, ModelReference, ModelSpecialization},
},
};
@@ -98,8 +98,8 @@ impl Registry {
self.config.backend_identifier.clone(),
self.config.backend_version.clone(),
vec![ModelSpecialization::Chat {}],
- ModelAccessibility::OnDevice {
- source: ModelSource::Filesystem {
+ ModelAccessibility::Local {
+ reference: ModelReference::Local {
path: path.to_string_lossy().to_string(),
},
},
diff --git a/crates/legacy/uzu/src/registry/merged.rs b/crates/legacy/uzu/src/registry/merged.rs
index 32622ad70..57aa502ac 100644
--- a/crates/legacy/uzu/src/registry/merged.rs
+++ b/crates/legacy/uzu/src/registry/merged.rs
@@ -19,7 +19,7 @@ impl MergedRegistry {
&mut self,
registry: Box<dyn Registry<Error = RegistryError>>,
) -> Result<(), RegistryError> {
- if self.registries.iter().any(|current| current.indentifier() == registry.indentifier()) {
+ if self.registries.iter().any(|current_registry| current_registry.indentifier() == registry.indentifier()) {
return Err(RegistryError::UnableToAddRegistry {
identifier: registry.indentifier(),
});
@@ -31,17 +31,9 @@ impl MergedRegistry {
pub fn remove(
&mut self,
identifier: &str,
- ) -> Option<(usize, Box<dyn Registry<Error = RegistryError>>)> {
- let index = self.registries.iter().position(|registry| registry.indentifier() == identifier)?;
- Some((index, self.registries.remove(index)))
- }
-
- pub fn restore(
- &mut self,
- index: usize,
- registry: Box<dyn Registry<Error = RegistryError>>,
- ) {
- self.registries.insert(index, registry);
+ ) -> Result<(), RegistryError> {
+ self.registries.retain(|registry| registry.indentifier() != identifier);
+ Ok(())
}
}
@@ -57,8 +49,13 @@ impl Registry for MergedRegistry {
let results = futures::future::join_all(self.registries.iter().map(|registry| registry.models())).await;
let mut models = Vec::new();
- for result in results {
- models.extend(result?);
+ for (registry, result) in self.registries.iter().zip(results) {
+ match result {
+ Ok(registry_models) => models.extend(registry_models),
+ Err(error) => {
+ tracing::warn!(?error, registry = %registry.indentifier(), "skipping registry that failed to list models");
+ },
+ }
}
Ok(models)
})
diff --git a/crates/legacy/uzu/src/registry/mirai/api.rs b/crates/legacy/uzu/src/registry/mirai/api.rs
new file mode 100644
index 000000000..64a907019
--- /dev/null
+++ b/crates/legacy/uzu/src/registry/mirai/api.rs
@@ -0,0 +1,59 @@
+use indexmap::IndexMap;
+use nagare::api::{Config, Endpoint as EndpointTrait, Payload};
+use reqwest::{Method, header::CONTENT_TYPE};
+use serde_json::json;
+
+use crate::{device::Device, registry::mirai::config::Backend};
+
+pub enum Endpoint {
+ FetchModels {
+ device: Device,
+ backends: Vec<Backend>,
+ include_traces: bool,
+ show_all: bool,
+ },
+}
+
+impl EndpointTrait for Endpoint {
+ fn method(&self) -> Method {
+ match self {
+ Endpoint::FetchModels {
+ ..
+ } => Method::POST,
+ }
+ }
+
+ fn path(&self) -> String {
+ match self {
+ Endpoint::FetchModels {
+ ..
+ } => "fetch/models".to_string(),
+ }
+ }
+
+ fn headers(&self) -> IndexMap<String, String> {
+ IndexMap::from([(CONTENT_TYPE.to_string(), "application/json".to_string())])
+ }
+
+ fn payload(
+ &self,
+ _: &Config,
+ ) -> Payload {
+ match self {
+ Endpoint::FetchModels {
+ device,
+ backends,
+ include_traces,
+ show_all,
+ } => Payload {
+ query: None,
+ body: Some(json!({
+ "device": device,
+ "backends": backends,
+ "include_traces": include_traces,
+ "show_all": show_all
+ })),
+ },
+ }
+ }
+}
diff --git a/crates/legacy/uzu/src/registry/mirai/config.rs b/crates/legacy/uzu/src/registry/mirai/config.rs
index 75c8e432e..0c264a58c 100644
--- a/crates/legacy/uzu/src/registry/mirai/config.rs
+++ b/crates/legacy/uzu/src/registry/mirai/config.rs
@@ -1,3 +1,5 @@
+use std::path::PathBuf;
+
use serde::{Deserialize, Serialize};
use crate::device::Device;
@@ -16,6 +18,7 @@ pub struct Config {
pub device: Device,
pub backends: Vec<Backend>,
pub include_traces: bool,
+ pub cache_path: PathBuf,
}
impl Config {
@@ -24,12 +27,14 @@ impl Config {
device: Device,
backends: Vec<Backend>,
include_traces: bool,
+ cache_path: PathBuf,
) -> Self {
Self {
api_key,
device,
backends,
include_traces,
+ cache_path,
}
}
}
diff --git a/crates/legacy/uzu/src/registry/mirai/mod.rs b/crates/legacy/uzu/src/registry/mirai/mod.rs
index b9a277a12..978fddac2 100644
--- a/crates/legacy/uzu/src/registry/mirai/mod.rs
+++ b/crates/legacy/uzu/src/registry/mirai/mod.rs
@@ -1,20 +1,24 @@
+mod api;
mod config;
+mod types;
-use std::{future::Future, pin::Pin, time::Duration};
+use std::{
+ fs::{read_to_string, write},
+ future::Future,
+ path::PathBuf,
+ pin::Pin,
+ time::Duration,
+};
+pub use api::Endpoint;
pub use config::{Backend, Config};
use indexmap::IndexMap;
use nagare::api::{Client, Config as ClientConfig, Error as ApiError};
use reqwest::header::AUTHORIZATION;
use shoji::{traits::Registry as RegistryTrait, types::model::Model};
+pub use types::Response;
-use crate::{
- api::{FetchModelsRequest, FetchedModels},
- registry::RegistryError,
-};
-
-pub const MIRAI_API_SCHEME: &str = "https";
-pub const MIRAI_API_HOST: &str = "sdk.trymirai.com";
+use crate::registry::RegistryError;
pub struct Registry {
config: Config,
@@ -28,11 +32,8 @@ impl Registry {
headers.insert(AUTHORIZATION.to_string(), format!("Bearer {}", api_key));
}
- let client_config = ClientConfig::new(
- format!("{MIRAI_API_SCHEME}://{MIRAI_API_HOST}/api/v1"),
- Duration::from_secs(10),
- headers,
- );
+ let client_config =
+ ClientConfig::new("https://sdk.trymirai.com/api/v1".to_string(), Duration::from_secs(10), headers);
let client = Client::new(client_config).map_err(|error| RegistryError::UnableToCreate {
message: error.to_string(),
})?;
@@ -52,24 +53,65 @@ impl RegistryTrait for Registry {
fn models(&self) -> Pin<Box<dyn Future<Output = Result<Vec<Model>, RegistryError>> + Send + '_>> {
Box::pin(async {
- self.fetch_models().await.map_err(|error| RegistryError::UnableToGetModels {
- message: error.to_string(),
- })
+ match self.fetch_models().await {
+ Ok(models) => {
+ if let Err(error) = self.save_registry(&models) {
+ tracing::warn!(?error, "failed to save Mirai registry");
+ }
+ Ok(models)
+ },
+ Err(error) => {
+ let transient = matches!(error, ApiError::Timeout | ApiError::Network(_))
+ || matches!(error, ApiError::Http { code, .. } if code >= 500);
+ if transient && let Ok(models) = self.load_registry() {
+ tracing::warn!(?error, "serving cached Mirai registry after fetch failure");
+ return Ok(models);
+ }
+ Err(RegistryError::UnableToGetModels {
+ message: error.to_string(),
+ })
+ },
+ }
})
}
}
impl Registry {
async fn fetch_models(&self) -> Result<Vec<Model>, ApiError> {
- let response: FetchedModels = self
+ let response: Response = self
.client
- .response(&FetchModelsRequest::new(
- self.config.device.clone(),
- self.config.backends.clone(),
- self.config.include_traces,
- std::env::var("UZU_REGISTRY_SHOW_ALL").is_ok(),
- ))
+ .response(&Endpoint::FetchModels {
+ device: self.config.device.clone(),
+ backends: self.config.backends.clone(),
+ include_traces: self.config.include_traces,
+ show_all: std::env::var("UZU_REGISTRY_SHOW_ALL").is_ok(),
+ })
.await?;
- response.models().map_err(ApiError::Decode)
+ response.models().ok_or_else(|| ApiError::Decode("response contained no models".to_string()))
+ }
+
+ fn registry_path(&self) -> PathBuf {
+ self.config.cache_path.join("registry.json")
+ }
+
+ fn save_registry(
+ &self,
+ models: &[Model],
+ ) -> Result<(), RegistryError> {
+ let contents = serde_json::to_vec_pretty(models).map_err(|error| RegistryError::UnableToGetModels {
+ message: format!("Unable to serialize registry: {}", error),
+ })?;
+ write(self.registry_path(), contents).map_err(|error| RegistryError::UnableToGetModels {
+ message: format!("Unable to write registry: {}", error),
+ })
+ }
+
+ fn load_registry(&self) -> Result<Vec<Model>, RegistryError> {
+ let contents = read_to_string(self.registry_path()).map_err(|error| RegistryError::UnableToGetModels {
+ message: format!("Unable to read registry: {}", error),
+ })?;
+ serde_json::from_str(&contents).map_err(|error| RegistryError::UnableToGetModels {
+ message: format!("Unable to parse registry: {}", error),
+ })
}
}
diff --git a/crates/legacy/uzu/src/registry/mirai/types.rs b/crates/legacy/uzu/src/registry/mirai/types.rs
new file mode 100644
index 000000000..3e5d6483c
--- /dev/null
+++ b/crates/legacy/uzu/src/registry/mirai/types.rs
@@ -0,0 +1,225 @@
+use hanashi::chat::EncodingConfig;
+use serde::{Deserialize, Deserializer, Serialize};
+use shoji::types::{
+ basic::{Metadata, Value},
+ model::{
+ Model, ModelAccessibility, ModelBackend, ModelFamily, ModelProperties, ModelQuantization, ModelRegistry,
+ ModelSpecialization, ModelVendor,
+ },
+};
+
+fn get_metadata(
+ metadatas: &[Metadata],
+ identifier: String,
+) -> Option<Metadata> {
+ metadatas.iter().find(|metadata| metadata.identifier == identifier).cloned()
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct Registry {
+ pub id: String,
+ pub metadata_id: String,
+}
+
+impl Registry {
+ fn to(
+ &self,
+ metadatas: &[Metadata],
+ ) -> Option<ModelRegistry> {
+ let metadata = get_metadata(metadatas, self.metadata_id.clone())?;
+ Some(ModelRegistry {
+ identifier: self.id.clone(),
+ metadata,
+ })
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct Backend {
+ pub id: String,
+ pub version: String,
+ pub metadata_id: String,
+}
+
+impl Backend {
+ fn to(
+ &self,
+ metadatas: &[Metadata],
+ ) -> Option<ModelBackend> {
+ let metadata = get_metadata(metadatas, self.metadata_id.clone())?;
+ Some(ModelBackend {
+ identifier: self.id.clone(),
+ version: self.version.clone(),
+ metadata,
+ })
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct Vendor {
+ pub id: String,
+ pub metadata_id: String,
+}
+
+impl Vendor {
+ fn to(
+ &self,
+ metadatas: &[Metadata],
+ ) -> Option<ModelVendor> {
+ let metadata = get_metadata(metadatas, self.metadata_id.clone())?;
+ Some(ModelVendor {
+ identifier: self.id.clone(),
+ metadata,
+ })
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct Family {
+ pub id: String,
+ pub vendor: Vendor,
+ pub metadata_id: String,
+}
+
+impl Family {
+ fn to(
+ &self,
+ metadatas: &[Metadata],
+ ) -> Option<ModelFamily> {
+ let vendor = self.vendor.to(metadatas)?;
+ let metadata = get_metadata(metadatas, self.metadata_id.clone())?;
+ Some(ModelFamily {
+ identifier: self.id.clone(),
+ vendor,
+ metadata,
+ })
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct Properties {
+ pub id: String,
+ pub size: i64,
+ pub version: Option<String>,
+ pub metadata_id: String,
+}
+
+impl Properties {
+ fn to(
+ &self,
+ metadatas: &[Metadata],
+ ) -> Option<ModelProperties> {
+ let metadata = get_metadata(metadatas, self.metadata_id.clone())?;
+ Some(ModelProperties {
+ identifier: self.id.clone(),
+ size: self.size,
+ version: self.version.clone(),
+ metadata,
+ })
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct Quantization {
+ pub id: String,
+ pub method: String,
+ pub bits_per_weight: u32,
+ pub vendor: Vendor,
+ pub metadata_id: String,
+}
+
+impl Quantization {
+ fn to(
+ &self,
+ metadatas: &[Metadata],
+ ) -> Option<ModelQuantization> {
+ let vendor = self.vendor.to(metadatas)?;
+ let metadata = get_metadata(metadatas, self.metadata_id.clone())?;
+ Some(ModelQuantization {
+ identifier: self.id.clone(),
+ method: self.method.clone(),
+ bits_per_weight: self.bits_per_weight,
+ vendor,
+ metadata,
+ })
+ }
+}
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct ResponseModel {
+ pub id: String,
+ pub registry: Registry,
+ pub backends: Vec<Backend>,
+ pub family: Option<Family>,
+ pub properties: Option<Properties>,
+ pub quantization: Option<Quantization>,
+ pub specializations: Vec<ModelSpecialization>,
+ pub accessibility: ModelAccessibility,
+ #[serde(rename = "encodings", default, deserialize_with = "deserialize_encoding")]
+ pub encoding: Option<Value>,
+}
+
+fn deserialize_encoding<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Option<Value>, D::Error> {
+ let encodings = Vec::<Value>::deserialize(deserializer)?;
+ Ok(EncodingConfig::select(&encodings))
+}
+
+impl ResponseModel {
+ fn to(
+ &self,
+ metadatas: &[Metadata],
+ ) -> Option<Model> {
+ let registry = self.registry.to(metadatas)?;
+ let backends = self.backends.iter().flat_map(|backend| backend.to(metadatas)).collect::<Vec<_>>();
+ if backends.len() != self.backends.len() {
+ return None;
+ }
+ let family = match &self.family {
+ Some(family) => Some(family.to(metadatas)?),
+ None => None,
+ };
+ let properties = match &self.properties {
+ Some(properties) => Some(properties.to(metadatas)?),
+ None => None,
+ };
+ let quantization = match &self.quantization {
+ Some(quantization) => Some(quantization.to(metadatas)?),
+ None => None,
+ };
+ Some(Model {
+ identifier: self.id.clone(),
+ registry,
+ backends,
+ family,
+ properties,
+ quantization,
+ specializations: self.specializations.clone(),
+ accessibility: self.accessibility.clone(),
+ encoding: self.encoding.clone(),
+ })
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct Response {
+ models: Vec<ResponseModel>,
+ metadatas: Vec<Metadata>,
+}
+
+impl Response {
+ pub fn models(&self) -> Option<Vec<Model>> {
+ let models =
+ self.models.iter().flat_map(|response_model| response_model.to(&self.metadatas)).collect::<Vec<_>>();
+ if models.len() != self.models.len() {
+ return None;
+ }
+ Some(models)
+ }
+}
diff --git a/crates/legacy/uzu/src/registry/openai/mod.rs b/crates/legacy/uzu/src/registry/openai/mod.rs
index 18e151497..dbd74293f 100644
--- a/crates/legacy/uzu/src/registry/openai/mod.rs
+++ b/crates/legacy/uzu/src/registry/openai/mod.rs
@@ -5,12 +5,23 @@ use std::{future::Future, pin::Pin};
use async_openai::{Client, config::OpenAIConfig};
pub use config::Config;
use fancy_regex::Regex;
+use serde::Deserialize;
use shoji::{
traits::Registry as RegistryTrait,
types::model::{Model as ShojiModel, ModelAccessibility, ModelSpecialization},
};
-use crate::{api::OpenAIModelsResponse, registry::RegistryError};
+#[derive(Debug, Deserialize)]
+struct Model {
+ id: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct ListModelsResponse {
+ data: Vec<Model>,
+}
+
+use crate::registry::RegistryError;
pub struct Registry {
config: Config,
@@ -63,7 +74,7 @@ impl RegistryTrait for Registry {
fn models(&self) -> Pin<Box<dyn Future<Output = Result<Vec<ShojiModel>, RegistryError>> + Send + '_>> {
Box::pin(async {
- let response: OpenAIModelsResponse =
+ let response: ListModelsResponse =
self.client.models().list_byot().await.map_err(|error| RegistryError::UnableToGetModels {
message: error.to_string(),
})?;
diff --git a/crates/legacy/uzu/src/storage/config.rs b/crates/legacy/uzu/src/storage/config.rs
index 7fc2ece0c..450ef69c9 100644
--- a/crates/legacy/uzu/src/storage/config.rs
+++ b/crates/legacy/uzu/src/storage/config.rs
@@ -4,6 +4,7 @@ use download_manager::FileDownloadManagerType;
use serde::{Deserialize, Serialize};
use shoji::types::model::Model;
+use super::download_contents::DownloadContents;
use crate::device::Device;
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -14,6 +15,8 @@ pub struct Config {
pub name: String,
#[serde(default)]
pub download_manager_type: FileDownloadManagerType,
+ #[serde(skip)]
+ pub download_contents: DownloadContents,
}
impl Config {
@@ -27,6 +30,7 @@ impl Config {
base_path,
name,
download_manager_type: FileDownloadManagerType::default(),
+ download_contents: DownloadContents::default(),
}
}
@@ -42,11 +46,9 @@ impl Config {
&self,
model: &Model,
) -> Option<PathBuf> {
- if !model.is_downloadable() {
- return None;
- }
+ let reference_name = model.reference_name()?;
let checkpoint_version = model.checkpoint_version()?;
- Some(self.cache_models_path().join("mirai").join(model.cache_identifier()).join(checkpoint_version))
+ Some(self.cache_models_path().join(reference_name).join(model.cache_identifier()).join(checkpoint_version))
}
pub fn log_name(&self) -> String {
@@ -62,4 +64,14 @@ impl Config {
..self.clone()
}
}
+
+ pub fn with_download_contents(
+ &self,
+ download_contents: DownloadContents,
+ ) -> Self {
+ Self {
+ download_contents,
+ ..self.clone()
+ }
+ }
}
diff --git a/crates/legacy/uzu/src/storage/download_contents.rs b/crates/legacy/uzu/src/storage/download_contents.rs
new file mode 100644
index 000000000..7fb0ca1fb
--- /dev/null
+++ b/crates/legacy/uzu/src/storage/download_contents.rs
@@ -0,0 +1,33 @@
+use bitflags::bitflags;
+
+bitflags! {
+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+ pub struct DownloadContents: u8 {
+ const CONFIG = 0b0000_0001;
+ const TOKENIZER = 0b0000_0010;
+ const WEIGHTS = 0b0000_0100;
+ const TRACES = 0b0000_1000;
+ }
+}
+
+impl Default for DownloadContents {
+ fn default() -> Self {
+ Self::all()
+ }
+}
+
+impl DownloadContents {
+ pub fn includes_file(
+ &self,
+ file_name: &str,
+ ) -> bool {
+ let flag = match file_name {
+ "config.json" => Self::CONFIG,
+ "tokenizer.json" => Self::TOKENIZER,
+ "traces.safetensors" => Self::TRACES,
+ name if name.ends_with(".safetensors") => Self::WEIGHTS,
+ _ => Self::CONFIG,
+ };
+ self.contains(flag)
+ }
+}
diff --git a/crates/legacy/uzu/src/storage/error.rs b/crates/legacy/uzu/src/storage/error.rs
index 8d640119c..cd04057e2 100644
--- a/crates/legacy/uzu/src/storage/error.rs
+++ b/crates/legacy/uzu/src/storage/error.rs
@@ -1,4 +1,4 @@
-use crate::storage::types::DownloadPhase;
+use crate::{registry::RegistryError, storage::types::DownloadPhase};
#[bindings::export(Error)]
#[derive(Debug, Clone, PartialEq, thiserror::Error)]
@@ -8,10 +8,19 @@ pub enum StorageError {
UnableToCreateDirectory {
path: String,
},
+ #[error("Unable to create download manager: {message}")]
+ UnableToCreateDownloadManager {
+ message: String,
+ },
#[error("Download manager error: {message}")]
DownloadManager {
message: String,
},
+ #[error("Hash not found for file: {identifier}/{name}")]
+ HashNotFound {
+ identifier: String,
+ name: String,
+ },
#[error("Invalid state transition from {from:?} to {to:?}")]
InvalidStateTransition {
from: DownloadPhase,
@@ -25,6 +34,8 @@ pub enum StorageError {
ItemNotFound {
identifier: String,
},
+ #[error(transparent)]
+ Registry(#[from] RegistryError),
#[error("Unsupported item: {identifier}")]
UnsupportedItem {
identifier: String,
diff --git a/crates/legacy/uzu/src/storage/mod.rs b/crates/legacy/uzu/src/storage/mod.rs
index eaf654401..121686e36 100644
--- a/crates/legacy/uzu/src/storage/mod.rs
+++ b/crates/legacy/uzu/src/storage/mod.rs
@@ -1,8 +1,296 @@
mod config;
+mod download_contents;
mod error;
-mod storage;
pub mod types;
+use std::{
+ collections::{HashMap, HashSet},
+ fs::{create_dir_all, read_dir, remove_dir_all},
+ sync::Arc,
+};
+
pub use config::Config;
+pub use download_contents::DownloadContents;
+use download_manager::{FileCheck, FileDownloadManager, FileDownloadPhase};
pub use error::StorageError;
-pub use storage::Storage;
+use futures_util::future::join_all;
+use kiban::rt::RuntimeHandle;
+use shoji::types::{
+ basic::File,
+ model::{Model, ModelAccessibility, ModelReference},
+};
+use tokio::sync::broadcast::channel as tokio_broadcast_channel;
+
+use crate::{
+ helpers::SharedAccess,
+ storage::types::{DownloadPhase, DownloadState, Item, StorageDownloadEventSender, StorageDownloadEventStream},
+};
+
+pub struct Storage {
+ pub config: Config,
+
+ download_manager: SharedAccess<Arc<dyn FileDownloadManager>>,
+ items: SharedAccess<HashMap<String, Item>>,
+ items_broadcast_sender: StorageDownloadEventSender,
+ handle: RuntimeHandle,
+}
+
+impl Storage {
+ pub async fn new(
+ runtime_handle: RuntimeHandle,
+ config: Config,
+ ) -> Result<Self, StorageError> {
+ create_dir_all(config.cache_path()).map_err(|_| StorageError::UnableToCreateDirectory {
+ path: config.cache_path().to_string_lossy().to_string(),
+ })?;
+
+ let download_manager = SharedAccess::new(Arc::from(
+ <dyn FileDownloadManager>::new(config.download_manager_type, runtime_handle.clone()).await.map_err(
+ |error| StorageError::DownloadManager {
+ message: error.to_string(),
+ },
+ )?,
+ ));
+
+ let items = SharedAccess::new(HashMap::new());
+
+ let (items_broadcast_sender, _) = tokio_broadcast_channel(256);
+
+ let storage = Self {
+ config,
+ download_manager,
+ items,
+ items_broadcast_sender,
+ handle: runtime_handle,
+ };
+ Ok(storage)
+ }
+}
+
+impl Storage {
+ pub async fn refresh(
+ &self,
+ models: Vec<Model>,
+ ) -> Result<(), StorageError> {
+ let models = models.into_iter().filter(|model| model.is_downloadable()).collect::<Vec<_>>();
+ let actual_model_identifiers: HashSet<String> = models.iter().map(|model| model.identifier.clone()).collect();
+
+ let download_manager = { self.download_manager.lock().await.clone() };
+
+ let existing_file_tasks =
+ download_manager.get_all_file_tasks().await.map_err(|error| StorageError::DownloadManager {
+ message: error.to_string(),
+ })?;
+ let mut active_file_tasks = Vec::with_capacity(existing_file_tasks.len());
+ for task in existing_file_tasks {
+ let task_state = task.state().await;
+ if matches!(task_state.phase, FileDownloadPhase::Error(_)) {
+ let download_id = task.download_id();
+ let _ = task.cancel().await;
+ let _ = download_manager.remove_file_task(download_id).await;
+ } else {
+ active_file_tasks.push(task);
+ }
+ }
+
+ let mut items = self.items.lock().await;
+ let stale_model_identifiers: Vec<String> = items
+ .keys()
+ .filter(|identifier| !actual_model_identifiers.contains(identifier.as_str()))
+ .cloned()
+ .collect();
+ for identifier in stale_model_identifiers {
+ if let Some(item) = items.remove(&identifier) {
+ if let Err(error) = item.detach_active_downloads().await {
+ tracing::warn!(?error, identifier, "failed to detach stale model file tasks");
+ }
+ item.stop_listening().await;
+ }
+ }
+
+ for model in models {
+ let identifier = model.identifier.clone();
+ if items.contains_key(&identifier) {
+ continue;
+ }
+
+ let files = self.resolve_model_files(&model)?;
+ let total_bytes: u64 = files.iter().map(|file| file.size as u64).sum();
+
+ let cache_path = self.config.cache_model_path(&model).ok_or(StorageError::UnsupportedItem {
+ identifier: identifier.clone(),
+ })?;
+
+ let has_files_on_disk = cache_path.exists();
+ let has_active_tasks = files.iter().any(|file| {
+ let file_path = cache_path.join(&file.name);
+ active_file_tasks.iter().any(|task| task.destination() == file_path)
+ });
+
+ let item = if has_files_on_disk || has_active_tasks {
+ let mut file_tasks = Vec::new();
+ for file in &files {
+ let file_path = cache_path.join(&file.name);
+ let file_check = FileCheck::CRC(file.crc32c().ok_or(StorageError::HashNotFound {
+ identifier: identifier.clone(),
+ name: file.name.clone(),
+ })?);
+
+ let file_task = download_manager
+ .file_download_task(&file.url, &file_path, file_check, Some(file.size as u64))
+ .await
+ .map_err(|error| StorageError::DownloadManager {
+ message: error.to_string(),
+ })?;
+
+ file_tasks.push(file_task);
+ }
+
+ let item = Item::new(
+ identifier.clone(),
+ files.into(),
+ cache_path.clone(),
+ DownloadState::not_downloaded(total_bytes as i64),
+ download_manager.clone(),
+ file_tasks,
+ self.handle.clone(),
+ self.items_broadcast_sender.clone(),
+ );
+
+ item.start_listening().await;
+ let _ = item.reconcile().await;
+ let item_state = item.state().await;
+
+ if matches!(item_state.phase, DownloadPhase::NotDownloaded {})
+ && cache_path.exists()
+ && let Ok(entries) = read_dir(&cache_path)
+ {
+ let has_real_files = entries.flatten().any(|entry| {
+ entry
+ .file_name()
+ .to_str()
+ .is_some_and(|name| !name.ends_with(".resume_data") && !name.starts_with('.'))
+ });
+ if !has_real_files {
+ let _ = remove_dir_all(&cache_path);
+ }
+ }
+
+ item
+ } else {
+ Item::new(
+ identifier.clone(),
+ files.into(),
+ cache_path.clone(),
+ DownloadState::not_downloaded(total_bytes as i64),
+ download_manager.clone(),
+ Vec::new(),
+ self.handle.clone(),
+ self.items_broadcast_sender.clone(),
+ )
+ };
+
+ items.insert(identifier, item);
+ }
+
+ Ok(())
+ }
+
+ pub fn subscribe(&self) -> StorageDownloadEventStream {
+ StorageDownloadEventStream::new(self.items_broadcast_sender.subscribe())
+ }
+
+ pub async fn get(
+ &self,
+ model_identifier: &String,
+ ) -> Option<Item> {
+ let items = self.items.lock().await;
+ items.get(model_identifier).cloned()
+ }
+
+ pub async fn state(
+ &self,
+ model_identifier: &String,
+ ) -> Option<DownloadState> {
+ let item = self.get(model_identifier).await?;
+ let state = item.state().await;
+ Some(state)
+ }
+
+ pub async fn states(&self) -> HashMap<String, DownloadState> {
+ let items = self.items.lock().await;
+ let state_futures = items.iter().map(|(identifier, item)| {
+ let identifier = identifier.clone();
+ async move { (identifier, item.state().await) }
+ });
+ join_all(state_futures).await.into_iter().collect()
+ }
+
+ pub async fn download(
+ &self,
+ model_identifier: &String,
+ ) -> Result<(), StorageError> {
+ let item = self.get(model_identifier).await.ok_or_else(|| StorageError::ItemNotFound {
+ identifier: model_identifier.clone(),
+ })?;
+ item.download().await
+ }
+
+ pub async fn pause(
+ &self,
+ model_identifier: &String,
+ ) -> Result<(), StorageError> {
+ let item = self.get(model_identifier).await.ok_or_else(|| StorageError::ItemNotFound {
+ identifier: model_identifier.clone(),
+ })?;
+ item.pause().await
+ }
+
+ pub async fn delete(
+ &self,
+ model_identifier: &String,
+ ) -> Result<(), StorageError> {
+ let item = self.get(model_identifier).await.ok_or_else(|| StorageError::ItemNotFound {
+ identifier: model_identifier.clone(),
+ })?;
+ item.cancel().await
+ }
+}
+
+impl Storage {
+ fn resolve_model_files(
+ &self,
+ model: &Model,
+ ) -> Result<Vec<File>, StorageError> {
+ match &model.accessibility {
+ ModelAccessibility::Local {
+ reference,
+ ..
+ } => match &reference {
+ ModelReference::Mirai {
+ files,
+ ..
+ } => Ok(files
+ .iter()
+ .filter(|file| self.config.download_contents.includes_file(&file.name))
+ .cloned()
+ .collect()),
+ ModelReference::HuggingFace {
+ ..
+ } => Err(StorageError::UnsupportedItem {
+ identifier: model.identifier.clone(),
+ }),
+ ModelReference::Local {
+ ..
+ } => Err(StorageError::UnsupportedItem {
+ identifier: model.identifier.clone(),
+ }),
+ },
+ ModelAccessibility::Remote {
+ ..
+ } => Err(StorageError::UnsupportedItem {
+ identifier: model.identifier.clone(),
+ }),
+ }
+ }
+}
diff --git a/crates/legacy/uzu/src/storage/storage.rs b/crates/legacy/uzu/src/storage/storage.rs
deleted file mode 100644
index 56084d72b..000000000
--- a/crates/legacy/uzu/src/storage/storage.rs
+++ /dev/null
@@ -1,238 +0,0 @@
-use std::{
- collections::{HashMap, HashSet},
- fs::{create_dir_all, read_dir, remove_dir_all},
- sync::Arc,
-};
-
-use download_manager::{FileDownloadManager, FileDownloadPhase};
-use futures_util::future::join_all;
-use kiban::rt::RuntimeHandle;
-use tokio::sync::broadcast::channel as tokio_broadcast_channel;
-
-use super::{
- Config, StorageError,
- types::{DownloadPhase, DownloadState, Item, StorageDownloadEventSender, StorageDownloadEventStream},
-};
-use crate::{helpers::SharedAccess, models::ResolvedModels};
-
-pub struct Storage {
- pub config: Config,
-
- download_manager: SharedAccess<Arc<dyn FileDownloadManager>>,
- items: SharedAccess<HashMap<String, Item>>,
- items_broadcast_sender: StorageDownloadEventSender,
- handle: RuntimeHandle,
-}
-
-impl Storage {
- pub async fn new(
- runtime_handle: RuntimeHandle,
- config: Config,
- ) -> Result<Self, StorageError> {
- create_dir_all(config.cache_path()).map_err(|_| StorageError::UnableToCreateDirectory {
- path: config.cache_path().to_string_lossy().to_string(),
- })?;
-
- let download_manager = SharedAccess::new(Arc::from(
- <dyn FileDownloadManager>::new(config.download_manager_type, runtime_handle.clone()).await.map_err(
- |error| StorageError::DownloadManager {
- message: error.to_string(),
- },
- )?,
- ));
-
- let (items_broadcast_sender, _) = tokio_broadcast_channel(256);
- Ok(Self {
- config,
- download_manager,
- items: SharedAccess::new(HashMap::new()),
- items_broadcast_sender,
- handle: runtime_handle,
- })
- }
-}
-
-#[cfg(test)]
-#[path = "../../tests/unit/storage/mod.rs"]
-mod tests;
-
-impl Storage {
- pub async fn refresh(
- &self,
- models: &ResolvedModels,
- huggingface_api_key: Option<Arc<str>>,
- ) -> Result<(), StorageError> {
- let models = models
- .iter()
- .filter_map(|model| {
- let (model, files) = model.parts();
- files.map(|files| (model.clone(), Arc::new(files.to_vec())))
- })
- .collect::<Vec<_>>();
- let actual_model_identifiers: HashSet<String> =
- models.iter().map(|(model, _)| model.identifier.clone()).collect();
-
- let download_manager = { self.download_manager.lock().await.clone() };
-
- let existing_file_tasks =
- download_manager.get_all_file_tasks().await.map_err(|error| StorageError::DownloadManager {
- message: error.to_string(),
- })?;
- let mut active_file_tasks = Vec::with_capacity(existing_file_tasks.len());
- for task in existing_file_tasks {
- let task_state = task.state().await;
- if matches!(task_state.phase, FileDownloadPhase::Error(_)) {
- let download_id = task.download_id();
- let _ = task.cancel().await;
- let _ = download_manager.remove_file_task(download_id).await;
- } else {
- active_file_tasks.push(task);
- }
- }
-
- let stale_items = {
- let mut items = self.items.lock().await;
- items
- .extract_if(|identifier, _| !actual_model_identifiers.contains(identifier.as_str()))
- .map(|(_, item)| item)
- .collect::<Vec<_>>()
- };
- for item in stale_items {
- if let Err(error) = item.detach_active_downloads().await {
- tracing::warn!(?error, identifier = %item.identifier, "failed to detach stale model file tasks");
- }
- item.stop_listening().await;
- }
-
- for (model, files) in models {
- let identifier = model.identifier.clone();
- let cache_path = self.config.cache_model_path(&model).ok_or(StorageError::UnsupportedItem {
- identifier: identifier.clone(),
- })?;
- if let Some(item) = self.get(&identifier).await {
- if item.matches(&cache_path, &files) {
- continue;
- }
- item.detach_active_downloads().await?;
- item.stop_listening().await;
- }
-
- let total_bytes = Item::total_bytes(&files)?;
-
- let has_files_on_disk = cache_path.exists();
- let has_active_tasks = files.iter().any(|file| {
- let file_path = cache_path.join(&file.file.name);
- active_file_tasks.iter().any(|task| task.destination() == file_path)
- });
-
- let item = Item::new(
- identifier.clone(),
- Arc::clone(&files),
- cache_path.clone(),
- DownloadState::not_downloaded(total_bytes),
- download_manager.clone(),
- self.handle.clone(),
- self.items_broadcast_sender.clone(),
- );
-
- if has_files_on_disk || has_active_tasks {
- item.ensure_file_tasks(huggingface_api_key.as_ref()).await?;
- item.start_listening().await;
- let _ = item.reconcile().await;
- let item_state = item.state().await;
-
- if matches!(item_state.phase, DownloadPhase::NotDownloaded {})
- && cache_path.exists()
- && let Ok(entries) = read_dir(&cache_path)
- {
- let has_real_files = entries.flatten().any(|entry| {
- entry
- .file_name()
- .to_str()
- .is_some_and(|name| !name.ends_with(".resume_data") && !name.starts_with('.'))
- });
- if !has_real_files {
- let _ = remove_dir_all(&cache_path);
- }
- }
- }
-
- self.items.lock().await.insert(identifier, item);
- }
-
- Ok(())
- }
-
- pub fn subscribe(&self) -> StorageDownloadEventStream {
- StorageDownloadEventStream::new(self.items_broadcast_sender.subscribe())
- }
-
- pub async fn get(
- &self,
- model_identifier: &str,
- ) -> Option<Item> {
- let items = self.items.lock().await;
- items.get(model_identifier).cloned()
- }
-
- pub async fn state(
- &self,
- model_identifier: &str,
- ) -> Option<DownloadState> {
- let item = self.get(model_identifier).await?;
- let state = item.state().await;
- Some(state)
- }
-
- pub async fn states(&self) -> HashMap<String, DownloadState> {
- let items = self.items.lock().await;
- let state_futures = items.iter().map(|(identifier, item)| {
- let identifier = identifier.clone();
- async move { (identifier, item.state().await) }
- });
- join_all(state_futures).await.into_iter().collect()
- }
-
- pub async fn download(
- &self,
- model_identifier: &str,
- huggingface_api_key: Option<Arc<str>>,
- ) -> Result<(), StorageError> {
- let item = self.item(model_identifier).await?;
- match item.state().await.phase {
- DownloadPhase::Downloading {} | DownloadPhase::Downloaded {} => Ok(()),
- DownloadPhase::Error {
- ..
- } => {
- item.cancel().await?;
- item.download(huggingface_api_key).await
- },
- _ => item.download(huggingface_api_key).await,
- }
- }
-
- pub async fn pause(
- &self,
- model_identifier: &str,
- ) -> Result<(), StorageError> {
- let item = self.item(model_identifier).await?;
- item.pause().await
- }
-
- pub async fn delete(
- &self,
- model_identifier: &str,
- ) -> Result<(), StorageError> {
- let item = self.item(model_identifier).await?;
- item.cancel().await
- }
-
- async fn item(
- &self,
- model_identifier: &str,
- ) -> Result<Item, StorageError> {
- self.get(model_identifier).await.ok_or_else(|| StorageError::ItemNotFound {
- identifier: model_identifier.to_string(),
- })
- }
-}
diff --git a/crates/legacy/uzu/src/storage/types/item.rs b/crates/legacy/uzu/src/storage/types/item.rs
index ef87319fc..619ee4060 100644
--- a/crates/legacy/uzu/src/storage/types/item.rs
+++ b/crates/legacy/uzu/src/storage/types/item.rs
@@ -1,25 +1,21 @@
use std::{
- fmt::{Debug, Formatter, Result as FmtResult},
fs::{create_dir_all, remove_dir_all},
- path::{Path, PathBuf},
+ path::PathBuf,
sync::Arc,
};
-use download_manager::{
- DownloadError, FileDownloadManager, FileDownloadPhase, FileDownloadState, FileDownloadTask, HttpDownloadRequest,
-};
+use download_manager::{DownloadError, FileDownloadManager, FileDownloadPhase, FileDownloadState, FileDownloadTask};
use futures_util::future::join_all;
use kiban::rt::{RuntimeHandle, TaskJoinHandle};
+use shoji::types::basic::File;
use tokio::sync::{
broadcast::{Sender as TokioBroadcastSender, channel as tokio_broadcast_channel},
mpsc::channel as tokio_mpsc_channel,
};
-use tokio_stream::{StreamExt as TokioStreamExt, wrappers::BroadcastStream as TokioBroadcastStream};
-use uuid::Uuid;
+use tokio_stream::wrappers::BroadcastStream;
use crate::{
helpers::SharedAccess,
- models::ResolvedFile,
storage::{
StorageError,
types::{DownloadPhase, DownloadState, StorageDownloadEventSender, reduce_file_download_states},
@@ -28,7 +24,7 @@ use crate::{
pub struct Item {
pub identifier: String,
- files: Arc<Vec<ResolvedFile>>,
+ pub files: Arc<Vec<File>>,
pub cache_path: PathBuf,
download_state: SharedAccess<DownloadState>,
@@ -42,11 +38,11 @@ pub struct Item {
listener_task: SharedAccess<Option<Box<dyn TaskJoinHandle<()>>>>,
}
-impl Debug for Item {
+impl std::fmt::Debug for Item {
fn fmt(
&self,
- f: &mut Formatter<'_>,
- ) -> FmtResult {
+ f: &mut std::fmt::Formatter<'_>,
+ ) -> std::fmt::Result {
f.debug_struct("Item").field("identifier", &self.identifier).field("cache_path", &self.cache_path).finish()
}
}
@@ -70,20 +66,13 @@ impl Clone for Item {
}
impl Item {
- pub fn matches(
- &self,
- cache_path: &Path,
- files: &[ResolvedFile],
- ) -> bool {
- self.cache_path == cache_path && self.files.as_slice() == files
- }
-
pub fn new(
identifier: String,
- files: Arc<Vec<ResolvedFile>>,
+ files: Arc<Vec<File>>,
cache_path: PathBuf,
download_state: DownloadState,
file_download_manager: Arc<dyn FileDownloadManager>,
+ file_download_tasks: Vec<Arc<dyn FileDownloadTask>>,
runtime_handle: RuntimeHandle,
storage_broadcast_sender: StorageDownloadEventSender,
) -> Self {
@@ -95,7 +84,7 @@ impl Item {
cache_path,
download_state: SharedAccess::new(download_state),
file_download_manager,
- file_download_tasks: SharedAccess::new(Vec::new()),
+ file_download_tasks: SharedAccess::new(file_download_tasks),
file_download_states,
runtime_handle,
broadcast_sender,
@@ -109,17 +98,23 @@ impl Item {
}
async fn get_file_download_states(&self) -> Vec<FileDownloadState> {
- let files = Arc::clone(&self.files);
let file_tasks_guard = self.file_download_tasks.lock().await;
let num_file_tasks = file_tasks_guard.len();
drop(file_tasks_guard);
+ // Use cached states from broadcasts - more reliable than querying
let cache_guard = self.file_download_states.lock().await;
+ // Validate cache is properly sized before using it
if !cache_guard.is_empty() && cache_guard.len() == num_file_tasks {
let mut states = cache_guard.clone();
- for (state, file) in states.iter_mut().zip(files.iter()) {
- Self::fill_total_bytes(state, file);
+ for (i, state) in states.iter_mut().enumerate() {
+ if state.total_bytes == 0
+ && let Some(file_info) = self.files.get(i)
+ && file_info.size > 0
+ {
+ state.total_bytes = file_info.size as u64;
+ }
}
return states;
}
@@ -136,14 +131,21 @@ impl Item {
drop(cache_guard);
- let file_tasks = self.file_download_tasks.lock().await.clone();
+ // Fall back to querying if cache not initialized or size mismatch
+ let file_tasks_guard = self.file_download_tasks.lock().await;
let mut states = Vec::new();
- for file_task in file_tasks {
+ for file_task in file_tasks_guard.iter() {
states.push(file_task.state().await);
}
- for (state, file) in states.iter_mut().zip(files.iter()) {
- Self::fill_total_bytes(state, file);
+ // Patch total_bytes from registry if missing (safety net)
+ for (i, state) in states.iter_mut().enumerate() {
+ if state.total_bytes == 0
+ && let Some(file_info) = self.files.get(i)
+ && file_info.size > 0
+ {
+ state.total_bytes = file_info.size as u64;
+ }
}
states
@@ -151,93 +153,146 @@ impl Item {
pub async fn reduce_state(&self) -> DownloadState {
let file_download_states = self.get_file_download_states().await;
- reduce_file_download_states(&file_download_states)
+
+ tracing::debug!(
+ "[MODEL] reduce_state: model={}, file_states count={}",
+ self.identifier,
+ file_download_states.len()
+ );
+ for (i, state) in file_download_states.iter().enumerate() {
+ tracing::debug!(
+ "[MODEL] File {} - phase={:?}, downloaded={}/{} bytes",
+ i,
+ state.phase,
+ state.downloaded_bytes,
+ state.total_bytes
+ );
+ }
+
+ let result = reduce_file_download_states(&file_download_states);
+
+ tracing::debug!(
+ "[MODEL] reduce_state result: phase={:?}, bytes={}/{}, progress={:.1}%",
+ result.phase,
+ result.downloaded_bytes,
+ result.total_bytes,
+ result.progress() * 100.0
+ );
+
+ result
}
pub async fn update_state_and_broadcast(
&self,
new_state: DownloadState,
) {
- *self.download_state.lock().await = new_state.clone();
+ tracing::debug!(
+ "[MODEL] Broadcasting state: id={}, phase={:?}, progress={:.1}%",
+ self.identifier,
+ new_state.phase,
+ new_state.progress() * 100.0
+ );
+
+ let mut state_guard = self.download_state.lock().await;
+ *state_guard = new_state.clone();
+ drop(state_guard);
+
+ let own_receiver_count = self.broadcast_sender.receiver_count();
+ let storage_receiver_count = self.storage_broadcast_sender.receiver_count();
+
+ if own_receiver_count == 0 && storage_receiver_count == 0 {
+ tracing::debug!("[MODEL] No receivers active, skipping broadcasts (likely shutdown)");
+ return;
+ }
+
+ if own_receiver_count > 0 {
+ let own_result = self.broadcast_sender.send(new_state.clone());
+
+ match own_result {
+ Ok(count) => tracing::debug!("[MODEL] ✓ Own broadcast sent to {} receiver(s)", count),
+ Err(_) => {
+ tracing::trace!("[MODEL] Own broadcast skipped (no receivers)")
+ },
+ }
+ } else {
+ tracing::trace!("[MODEL] Skipping own broadcast (no receivers)");
+ }
- let _ = self.broadcast_sender.send(new_state.clone());
- let _ = self.storage_broadcast_sender.send((self.identifier.clone(), new_state));
+ if storage_receiver_count > 0 {
+ let storage_result = self.storage_broadcast_sender.send((self.identifier.clone(), new_state));
+
+ match storage_result {
+ Ok(count) => tracing::debug!("[MODEL] ✓ Storage broadcast sent to {} receiver(s)", count),
+ Err(_) => tracing::debug!("[MODEL] ⚠️ Storage broadcast failed (no receivers)"),
+ }
+ } else {
+ tracing::trace!("[MODEL] Skipping storage broadcast (no receivers)");
+ }
}
pub async fn file_task_by_download_id(
&self,
- download_id: Uuid,
+ download_id: uuid::Uuid,
) -> Option<Arc<dyn FileDownloadTask>> {
let file_tasks_guard = self.file_download_tasks.lock().await;
file_tasks_guard.iter().find(|task| task.download_id() == download_id).cloned()
}
pub async fn reconcile(&self) -> Result<(), StorageError> {
+ // Reconciliation is now handled by FileDownloadTask
+ // Just update our broadcast with current state
let calculated_state = self.reduce_state().await;
self.update_state_and_broadcast(calculated_state).await;
Ok(())
}
- pub async fn ensure_file_tasks(
- &self,
- huggingface_api_key: Option<&Arc<str>>,
- ) -> Result<bool, StorageError> {
- if !self.file_download_tasks.lock().await.is_empty() {
+ async fn ensure_file_tasks(&self) -> Result<bool, StorageError> {
+ let mut file_tasks_guard = self.file_download_tasks.lock().await;
+
+ // If file tasks already exist, skip creation
+ if !file_tasks_guard.is_empty() {
return Ok(false);
}
- let mut new_file_tasks = Vec::with_capacity(self.files.len());
+ // Initialize file download tasks
+ let mut new_file_tasks = Vec::new();
for file_info in self.files.iter() {
- let file_path = self.cache_path.join(&file_info.file.name);
- let expected_bytes = u64::try_from(file_info.file.size).map_err(|_| StorageError::IO {
- message: format!("invalid file size for {}/{}", self.identifier, file_info.file.name),
- })?;
- let request = if file_info.requires_authentication {
- let api_key = huggingface_api_key.ok_or_else(|| StorageError::DownloadManager {
- message: "Hugging Face authentication is required".to_string(),
- })?;
- HttpDownloadRequest::with_bearer_token(file_info.file.url.clone(), api_key)
- } else {
- HttpDownloadRequest::get(file_info.file.url.clone())
- };
+ let file_path = self.cache_path.join(&file_info.name);
+ let file_check =
+ download_manager::FileCheck::CRC(file_info.crc32c().ok_or(StorageError::HashNotFound {
+ identifier: self.identifier.clone(),
+ name: file_info.name.clone(),
+ })?);
let file_task = self
.file_download_manager
- .file_download_task(request, &file_path, file_info.check.clone(), Some(expected_bytes))
+ .file_download_task(&file_info.url, &file_path, file_check, Some(file_info.size as u64))
.await
.map_err(|error| StorageError::DownloadManager {
message: error.to_string(),
})?;
+ // Start listening to global broadcast
file_task.start_listening((*self.file_download_manager.global_broadcast_sender()).clone()).await;
new_file_tasks.push(file_task);
}
- let mut file_tasks = self.file_download_tasks.lock().await;
- if file_tasks.is_empty() {
- *file_tasks = new_file_tasks;
- Ok(true)
- } else {
- Ok(false)
- }
+ *file_tasks_guard = new_file_tasks;
+ Ok(true)
}
- pub async fn download(
- &self,
- huggingface_api_key: Option<Arc<str>>,
- ) -> Result<(), StorageError> {
- let tasks_were_created = self.ensure_file_tasks(huggingface_api_key.as_ref()).await?;
+ pub async fn download(&self) -> Result<(), StorageError> {
+ tracing::debug!("[MODEL] download() called for model: {}", self.identifier);
+
+ let tasks_were_created = self.ensure_file_tasks().await?;
if tasks_were_created {
+ tracing::debug!("[MODEL] Restarting listener for model: {} (file tasks created)", self.identifier);
self.stop_listening().await;
self.start_listening().await;
}
let current_state = self.reduce_state().await;
- if matches!(current_state.phase, DownloadPhase::Downloaded {}) {
- self.update_state_and_broadcast(current_state).await;
- return Ok(());
- }
if !Self::can_transition_to_downloading(&current_state.phase) {
return Err(StorageError::InvalidStateTransition {
from: current_state.phase.clone(),
@@ -245,6 +300,7 @@ impl Item {
});
}
+ tracing::debug!("[MODEL] Calling ensure_downloading");
self.ensure_downloading().await?;
self.stop_listening().await;
@@ -253,10 +309,20 @@ impl Item {
let downloading_state = self.reduce_state().await;
self.update_state_and_broadcast(downloading_state).await;
+ tracing::debug!("[MODEL] download() completed for model: {}", self.identifier);
Ok(())
}
pub async fn pause(&self) -> Result<(), StorageError> {
+ tracing::debug!("[MODEL] pause() called for model: {}", self.identifier);
+
+ let tasks_were_created = self.ensure_file_tasks().await?;
+ if tasks_were_created {
+ tracing::debug!("[MODEL] Restarting listener for model: {} (file tasks created)", self.identifier);
+ self.stop_listening().await;
+ self.start_listening().await;
+ }
+
let current_state = self.reduce_state().await;
if !current_state.can_pause() {
return Err(StorageError::InvalidStateTransition {
@@ -265,25 +331,34 @@ impl Item {
});
}
+ tracing::debug!("[MODEL] Calling ensure_paused");
self.ensure_paused().await?;
- let file_tasks = self.file_download_tasks.lock().await.clone();
- let mut file_download_states = Vec::with_capacity(file_tasks.len());
- for file_task in file_tasks {
+ let file_tasks_guard = self.file_download_tasks.lock().await;
+ let mut file_download_states = Vec::with_capacity(file_tasks_guard.len());
+ for file_task in file_tasks_guard.iter() {
file_download_states.push(file_task.state().await);
}
+ drop(file_tasks_guard);
*self.file_download_states.lock().await = file_download_states;
let paused_state = self.reduce_state().await;
self.update_state_and_broadcast(paused_state).await;
+ tracing::debug!("[MODEL] pause() completed for model: {}", self.identifier);
Ok(())
}
pub async fn cancel(&self) -> Result<(), StorageError> {
- let files = Arc::clone(&self.files);
- for file_info in files.iter() {
- let destination = self.cache_path.join(&file_info.file.name);
+ let tasks_were_created = self.ensure_file_tasks().await?;
+ if tasks_were_created {
+ tracing::debug!("[MODEL] Restarting listener for model: {} (file tasks created)", self.identifier);
+ self.stop_listening().await;
+ self.start_listening().await;
+ }
+
+ for file_info in self.files.iter() {
+ let destination = self.cache_path.join(&file_info.name);
if let Some(owner) = self.file_download_manager.destination_foreign_lock(&destination).await {
tracing::info!(?destination, %owner, "refusing to cancel: destination is locked by another manager");
return Err(StorageError::InvalidStateTransition {
@@ -311,13 +386,14 @@ impl Item {
let _ = remove_dir_all(&self.cache_path);
}
- let not_downloaded_state = DownloadState::not_downloaded(Self::total_bytes(&files)?);
+ let total_bytes: u64 = self.files.iter().map(|f| f.size as u64).sum();
+ let not_downloaded_state = DownloadState::not_downloaded(total_bytes as i64);
self.update_state_and_broadcast(not_downloaded_state).await;
Ok(())
}
- pub async fn progress(&self) -> Result<TokioBroadcastStream<DownloadState>, StorageError> {
- Ok(TokioBroadcastStream::new(self.broadcast_sender.subscribe()))
+ pub async fn progress(&self) -> Result<BroadcastStream<DownloadState>, StorageError> {
+ Ok(BroadcastStream::new(self.broadcast_sender.subscribe()))
}
pub async fn detach_active_downloads(&self) -> Result<(), StorageError> {
@@ -336,7 +412,7 @@ impl Item {
let download_id = file_task.download_id();
file_task.cancel().await?;
manager.remove_file_task(download_id).await?;
- Ok::<(), DownloadError>(())
+ Ok::<(), download_manager::DownloadError>(())
}
});
let first_error = join_all(cancel_futures).await.into_iter().find_map(Result::err);
@@ -352,33 +428,50 @@ impl Item {
/// Handle file task state update
/// Called by ModelStorage listener when a file task broadcasts a state change
pub async fn handle_file_task_update(&self) {
+ tracing::debug!("[MODEL] handle_file_task_update: Computing new state for model: {}", self.identifier);
+
let new_state = self.reduce_state().await;
+
+ tracing::debug!(
+ "[MODEL] handle_file_task_update: New state computed: phase={:?}, bytes={}/{}, progress={:.1}%",
+ new_state.phase,
+ new_state.downloaded_bytes,
+ new_state.total_bytes,
+ new_state.progress() * 100.0
+ );
+
self.update_state_and_broadcast(new_state).await;
}
async fn ensure_downloading(&self) -> Result<(), StorageError> {
+ tracing::debug!("[MODEL] ensure_downloading: model={}", self.identifier);
+
create_dir_all(&self.cache_path).map_err(|error| StorageError::IO {
message: error.to_string(),
})?;
- let file_tasks = self.file_download_tasks.lock().await.clone();
- let download_futures = file_tasks.iter().map(|file_task| {
+ let file_tasks_guard = self.file_download_tasks.lock().await;
+ let download_futures = file_tasks_guard.iter().map(|file_task| {
let file_task = file_task.clone();
async move { file_task.download().await }
});
let first_error = join_all(download_futures).await.into_iter().find_map(Result::err);
+ drop(file_tasks_guard);
+
if let Some(error) = first_error {
return Err(StorageError::DownloadManager {
message: error.to_string(),
});
}
+ tracing::debug!("[MODEL] ensure_downloading: All file downloads initiated");
Ok(())
}
async fn ensure_paused(&self) -> Result<(), StorageError> {
- let file_tasks = self.file_download_tasks.lock().await.clone();
- let pause_futures = file_tasks.iter().map(|file_task| {
+ tracing::debug!("[MODEL] ensure_paused: model={}", self.identifier);
+ let file_tasks_guard = self.file_download_tasks.lock().await;
+ let pause_futures = file_tasks_guard.iter().map(|file_task| {
let file_task = file_task.clone();
async move {
if matches!(file_task.state().await.phase, FileDownloadPhase::Downloading) {
@@ -397,50 +490,89 @@ impl Item {
}
});
let first_error = join_all(pause_futures).await.into_iter().find_map(Result::err);
+ drop(file_tasks_guard);
+
if let Some(error) = first_error {
return Err(StorageError::DownloadManager {
message: error.to_string(),
});
}
+ tracing::debug!("[MODEL] ensure_paused: All file tasks paused");
Ok(())
}
/// Start listening to file task broadcasts
pub async fn start_listening(&self) {
- if self.listener_task.lock().await.is_some() {
+ // Check if already listening
+ let mut listener_guard = self.listener_task.lock().await;
+ if listener_guard.is_some() {
+ tracing::debug!("[MODEL] start_listening: Already listening for model: {}", self.identifier);
return;
}
- let file_tasks = self.file_download_tasks.lock().await.clone();
- let files = Arc::clone(&self.files);
- let num_files = file_tasks.len();
+ // Collect all file task broadcast streams and initialize cache
+ let file_tasks_guard = self.file_download_tasks.lock().await;
+ let num_files = file_tasks_guard.len();
let mut streams = Vec::new();
let mut initial_states = Vec::new();
- for (idx, file_task) in file_tasks.iter().enumerate() {
+ for (idx, file_task) in file_tasks_guard.iter().enumerate() {
let sender = file_task.broadcast_sender();
- let stream = TokioBroadcastStream::new(sender.subscribe());
+ let receiver_count_before = sender.receiver_count();
+ let stream = BroadcastStream::new(sender.subscribe());
+ tracing::info!(
+ "[MODEL] Subscribed to file task {}: model={}, receivers_before={}, receivers_after={}",
+ idx,
+ self.identifier,
+ receiver_count_before,
+ sender.receiver_count()
+ );
streams.push((idx, stream));
let mut state = file_task.state().await;
- if let Some(file) = files.get(idx) {
- Self::fill_total_bytes(&mut state, file);
+ if state.total_bytes == 0
+ && let Some(file_info) = self.files.get(idx)
+ && file_info.size > 0
+ {
+ state.total_bytes = file_info.size as u64;
}
initial_states.push(state);
}
+ drop(file_tasks_guard);
+
if streams.is_empty() {
+ tracing::debug!("[MODEL] start_listening: No file tasks yet for model: {}", self.identifier);
return;
}
+ // Pre-allocate cache with correct size BEFORE starting listener
+ // This ensures all updates will have valid indices
{
let mut cache_guard = self.file_download_states.lock().await;
*cache_guard = initial_states;
debug_assert_eq!(cache_guard.len(), num_files, "Cache size must match number of file tasks");
+
+ tracing::debug!(
+ "[MODEL] Cache initialized with {} entries for model: {}",
+ cache_guard.len(),
+ self.identifier
+ );
}
+ tracing::debug!(
+ "[MODEL] start_listening: Starting listener for model: {} ({} streams)",
+ self.identifier,
+ streams.len()
+ );
+
let model = self.clone();
+ tracing::debug!("[MODEL] Spawning listener task for model: {}", self.identifier);
let handle = self.runtime_handle.spawn(async move {
+ tracing::debug!("[MODEL] Listener task started for model: {}", model.identifier);
+ use tokio_stream::StreamExt as TokioStreamExt;
+
+ // Fan-in: per-stream forwarders into a single bounded channel
let num_streams = streams.len();
let (tx, mut rx) = tokio_mpsc_channel::<(usize, FileDownloadState)>(1024);
let mut forwarder_handles = Vec::with_capacity(num_streams);
@@ -449,8 +581,14 @@ impl Item {
let tx = tx.clone();
let forwarder_handle = model.runtime_handle.spawn(async move {
while let Some(item) = stream.next().await {
- if let Ok(state) = item {
- let _ = tx.send((idx, state)).await;
+ match item {
+ Ok(state) => {
+ // Forward latest state; drop if aggregator has gone away
+ let _ = tx.send((idx, state)).await;
+ },
+ Err(_e) => {
+ // Lag on a single stream isn't fatal
+ },
}
}
});
@@ -459,24 +597,31 @@ impl Item {
let _forwarder_handles = ListenerForwarderHandles {
handles: forwarder_handles,
};
- drop(tx);
+ drop(tx); // close when all forwarders end
+ // Aggregator: coalesce bursts, keep only latest state per task
let mut pending: Vec<Option<FileDownloadState>> = vec![None; num_streams];
while let Some((idx, state)) = rx.recv().await {
pending[idx] = Some(state);
+ // Drain quickly to coalesce into latest-only
while let Ok((i, s)) = rx.try_recv() {
pending[i] = Some(s);
}
+ // Apply all latest states atomically, then notify once
{
let mut cache_guard = model.file_download_states.lock().await;
for (i, slot) in pending.iter_mut().enumerate() {
if let Some(mut s) = slot.take() {
if i < cache_guard.len() {
- if let Some(file) = files.get(i) {
- Self::fill_total_bytes(&mut s, file);
+ // Patch total bytes if 0 in incoming update
+ if s.total_bytes == 0
+ && let Some(file_info) = model.files.get(i)
+ && file_info.size > 0
+ {
+ s.total_bytes = file_info.size as u64;
}
cache_guard[i] = s;
} else {
@@ -493,44 +638,23 @@ impl Item {
model.handle_file_task_update().await;
}
+
+ tracing::debug!("[MODEL] All streams ended for model: {}", model.identifier);
+
+ tracing::debug!("[MODEL] Listener task ended for model: {}", model.identifier);
});
- let mut listener = self.listener_task.lock().await;
- if listener.is_none() {
- *listener = Some(handle);
- } else {
- handle.abort();
- }
+ *listener_guard = Some(handle);
}
/// Stop listening to file task broadcasts
pub async fn stop_listening(&self) {
- let handle = self.listener_task.lock().await.take();
- if let Some(handle) = handle {
+ let mut listener_guard = self.listener_task.lock().await;
+ if let Some(handle) = listener_guard.take() {
handle.abort_and_join().await;
}
}
- pub fn total_bytes(files: &[ResolvedFile]) -> Result<i64, StorageError> {
- files.iter().try_fold(0_i64, |total, file| {
- total.checked_add(file.file.size).ok_or_else(|| StorageError::IO {
- message: "model download size overflow".to_string(),
- })
- })
- }
-
- fn fill_total_bytes(
- state: &mut FileDownloadState,
- file: &ResolvedFile,
- ) {
- if state.total_bytes == 0
- && let Ok(size) = u64::try_from(file.file.size)
- && size > 0
- {
- state.total_bytes = size;
- }
- }
-
fn can_transition_to_downloading(from: &DownloadPhase) -> bool {
matches!(
from,
diff --git a/crates/legacy/uzu/tests/common/mod.rs b/crates/legacy/uzu/tests/common/mod.rs
new file mode 100644
index 000000000..62e95933c
--- /dev/null
+++ b/crates/legacy/uzu/tests/common/mod.rs
@@ -0,0 +1,2 @@
+pub mod test_storage;
+pub mod tracing_setup;
diff --git a/crates/legacy/uzu/tests/unit/common/test_storage.rs b/crates/legacy/uzu/tests/common/test_storage.rs
similarity index 56%
rename from crates/legacy/uzu/tests/unit/common/test_storage.rs
rename to crates/legacy/uzu/tests/common/test_storage.rs
index 88185cc5d..56a958be3 100644
--- a/crates/legacy/uzu/tests/unit/common/test_storage.rs
+++ b/crates/legacy/uzu/tests/common/test_storage.rs
@@ -1,19 +1,32 @@
+#![allow(dead_code)]
+
+use std::path::PathBuf;
+
use download_manager::FileDownloadManagerType;
use kiban::rt::RuntimeHandle;
use shoji::types::model::Model;
-
-use crate::{
+use uzu::{
device::Device,
- models::{ModelsResolver, ResolvedModels},
+ registry::FixedRegistry,
storage::{Config, Storage},
};
pub struct TestStorage {
+ pub config: Config,
+ pub registry: FixedRegistry,
pub storage: Storage,
+ pub base_path: PathBuf,
_temp_dir_guard: tempfile::TempDir,
}
impl TestStorage {
+ pub async fn with_models(
+ tokio_handle: RuntimeHandle,
+ models: Vec<Model>,
+ ) -> Result<Self, Box<dyn std::error::Error>> {
+ Self::with_models_and_manager(tokio_handle, models, FileDownloadManagerType::default()).await
+ }
+
pub async fn with_models_and_manager(
tokio_handle: RuntimeHandle,
models: Vec<Model>,
@@ -21,16 +34,17 @@ impl TestStorage {
) -> Result<Self, Box<dyn std::error::Error>> {
let temp_dir_guard = tempfile::tempdir()?;
let base_path = temp_dir_guard.path().to_path_buf();
+ let registry = FixedRegistry::new("test_registry".to_string(), models.clone());
let device = Device::new()?;
let config = Config::new(device, Some(base_path.clone()), "test_storage".to_string())
.with_download_manager_type(download_manager_type);
- let storage = Storage::new(tokio_handle, config).await?;
- let resolved = ModelsResolver::new(None, base_path.join("resolved-models.json"))?
- .resolve(models, &ResolvedModels::default())
- .await?;
- storage.refresh(&resolved, None).await?;
+ let storage = Storage::new(tokio_handle, config.clone()).await?;
+ storage.refresh(models).await?;
Ok(Self {
+ config,
+ registry,
storage,
+ base_path,
_temp_dir_guard: temp_dir_guard,
})
}
diff --git a/crates/legacy/uzu/tests/common/tracing_setup.rs b/crates/legacy/uzu/tests/common/tracing_setup.rs
new file mode 100644
index 000000000..8b73a0249
--- /dev/null
+++ b/crates/legacy/uzu/tests/common/tracing_setup.rs
@@ -0,0 +1,30 @@
+#![allow(dead_code)]
+
+use std::sync::Once;
+
+use tracing_subscriber::{EnvFilter, fmt, layer::SubscriberExt, util::SubscriberInitExt};
+
+static INIT: Once = Once::new();
+
+pub fn init_test_tracing() {
+ INIT.call_once(|| {
+ let timestamp = chrono::Utc::now().format("%Y%m%d_%H%M%S");
+ let log_dir = std::env::temp_dir().join("uzu-storage-test-logs");
+ std::fs::create_dir_all(&log_dir).ok();
+
+ let file_appender = tracing_appender::rolling::never(&log_dir, format!("storage_{}.log", timestamp));
+
+ let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
+
+ // Keep guard alive for entire test run
+ std::mem::forget(guard);
+
+ tracing_subscriber::registry()
+ .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("debug")))
+ .with(fmt::layer().with_writer(non_blocking).with_ansi(false))
+ .with(fmt::layer().with_test_writer())
+ .init();
+
+ tracing::info!("Test logging initialized, writing to test_logs/");
+ });
+}
diff --git a/crates/legacy/uzu/tests/integration/engine/mod.rs b/crates/legacy/uzu/tests/integration/engine/mod.rs
deleted file mode 100644
index 3ce4c1d4e..000000000
--- a/crates/legacy/uzu/tests/integration/engine/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-mod live_test;
diff --git a/crates/legacy/uzu/tests/integration/test_entry.rs b/crates/legacy/uzu/tests/integration/test_entry.rs
deleted file mode 100644
index 863383918..000000000
--- a/crates/legacy/uzu/tests/integration/test_entry.rs
+++ /dev/null
@@ -1 +0,0 @@
-mod engine;
diff --git a/crates/legacy/uzu/tests/integration/engine/live_test.rs b/crates/legacy/uzu/tests/test_engine.rs
similarity index 100%
rename from crates/legacy/uzu/tests/integration/engine/live_test.rs
rename to crates/legacy/uzu/tests/test_engine.rs
diff --git a/crates/legacy/uzu/tests/integration/tool/calls.rs b/crates/legacy/uzu/tests/tool/calls.rs
similarity index 100%
rename from crates/legacy/uzu/tests/integration/tool/calls.rs
rename to crates/legacy/uzu/tests/tool/calls.rs
diff --git a/crates/legacy/uzu/tests/integration/tool/macro_closure.rs b/crates/legacy/uzu/tests/tool/macro_closure.rs
similarity index 100%
rename from crates/legacy/uzu/tests/integration/tool/macro_closure.rs
rename to crates/legacy/uzu/tests/tool/macro_closure.rs
diff --git a/crates/legacy/uzu/tests/integration/tool/macro_function.rs b/crates/legacy/uzu/tests/tool/macro_function.rs
similarity index 100%
rename from crates/legacy/uzu/tests/integration/tool/macro_function.rs
rename to crates/legacy/uzu/tests/tool/macro_function.rs
diff --git a/crates/legacy/uzu/tests/integration/tool/mod.rs b/crates/legacy/uzu/tests/tool/mod.rs
similarity index 100%
rename from crates/legacy/uzu/tests/integration/tool/mod.rs
rename to crates/legacy/uzu/tests/tool/mod.rs
diff --git a/crates/legacy/uzu/tests/unit/api/request_test.rs b/crates/legacy/uzu/tests/unit/api/request_test.rs
deleted file mode 100644
index 2d97c5f02..000000000
--- a/crates/legacy/uzu/tests/unit/api/request_test.rs
+++ /dev/null
@@ -1,37 +0,0 @@
-use serde_json::{json, to_value};
-
-use super::*;
-use crate::{device::Device, registry::mirai::Backend};
-
-#[test]
-fn fetch_models_request_serializes_the_api_payload() {
- let request = FetchModelsRequest::new(
- Device {
- os_name: Some("macOS".to_string()),
- cpu_name: Some("Apple".to_string()),
- memory_total: 32,
- home_path: "/tmp".to_string(),
- },
- vec![Backend {
- identifier: "uzu".to_string(),
- version: "1".to_string(),
- }],
- false,
- true,
- );
-
- assert_eq!(
- to_value(request).expect("request should serialize"),
- json!({
- "device": {
- "os_name": "macOS",
- "cpu_name": "Apple",
- "memory_total": 32,
- "home_path": "/tmp"
- },
- "backends": [{ "identifier": "uzu", "version": "1" }],
- "include_traces": false,
- "show_all": true
- })
- );
-}
diff --git a/crates/legacy/uzu/tests/unit/api/response_test.rs b/crates/legacy/uzu/tests/unit/api/response_test.rs
deleted file mode 100644
index f5fa2a1ad..000000000
--- a/crates/legacy/uzu/tests/unit/api/response_test.rs
+++ /dev/null
@@ -1,99 +0,0 @@
-use serde_json::{from_value, json};
-
-use super::*;
-
-#[test]
-fn fetched_models_converts_one_complete_response() {
- let response: FetchedModels = from_value(json!({
- "metadatas": [
- { "id": "registry-meta", "name": "Mirai", "description": null, "icons": [] },
- { "id": "backend-meta", "name": "Uzu", "description": null, "icons": [] }
- ],
- "models": [{
- "id": "model",
- "registry": { "id": "mirai", "metadata_id": "registry-meta" },
- "backends": [{ "id": "uzu", "version": "1", "metadata_id": "backend-meta" }],
- "family": null,
- "properties": null,
- "quantization": null,
- "specializations": [],
- "accessibility": {
- "type": "local",
- "reference": {
- "type": "mirai",
- "toolchain_version": "1",
- "repository": null,
- "source_repository": null,
- "files": []
- }
- },
- "encodings": []
- }]
- }))
- .expect("response should deserialize");
-
- let models = response.models().expect("response should resolve metadata references");
- assert_eq!(models.len(), 1);
- assert_eq!(models[0].identifier, "model");
- assert!(matches!(
- &models[0].accessibility,
- ModelAccessibility::OnDevice {
- source: ModelSource::Registry {
- toolchain_version,
- repository: None,
- source_repository: None,
- files,
- },
- } if toolchain_version == "1" && files.is_empty()
- ));
-
- let filesystem: FetchedAccessibility = from_value(json!({
- "type": "local",
- "reference": { "type": "local", "path": "/models/local" }
- }))
- .expect("filesystem response should deserialize");
- assert_eq!(
- ModelAccessibility::from(&filesystem),
- ModelAccessibility::OnDevice {
- source: ModelSource::Filesystem {
- path: "/models/local".to_string(),
- },
- }
- );
-}
-
-#[test]
-fn hugging_face_access_is_conservative() {
- for (gated, requires_authentication) in [(json!(false), false), (json!("manual"), true), (json!(null), true)] {
- let response: HuggingFaceModelResponse = from_value(json!({
- "sha": "",
- "private": false,
- "gated": gated,
- "siblings": []
- }))
- .expect("response should deserialize");
-
- assert_eq!(response.requires_authentication(), requires_authentication);
- }
-}
-
-#[test]
-fn fetched_models_rejects_missing_metadata() {
- let response: FetchedModels = from_value(json!({
- "metadatas": [],
- "models": [{
- "id": "model",
- "registry": { "id": "mirai", "metadata_id": "missing" },
- "backends": [],
- "family": null,
- "properties": null,
- "quantization": null,
- "specializations": [],
- "accessibility": { "type": "remote", "repository": null },
- "encodings": []
- }]
- }))
- .expect("response should deserialize");
-
- assert!(response.models().is_err());
-}
diff --git a/crates/legacy/uzu/tests/unit/common/mod.rs b/crates/legacy/uzu/tests/unit/common/mod.rs
deleted file mode 100644
index a97611d83..000000000
--- a/crates/legacy/uzu/tests/unit/common/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-pub mod test_storage;
diff --git a/crates/legacy/uzu/tests/unit/engine/bridge/helpers_test.rs b/crates/legacy/uzu/tests/unit/engine/bridge/helpers_test.rs
deleted file mode 100644
index b99a27cbe..000000000
--- a/crates/legacy/uzu/tests/unit/engine/bridge/helpers_test.rs
+++ /dev/null
@@ -1,31 +0,0 @@
-use tokenizers::{Tokenizer, decoders::fuse::Fuse, models::wordlevel::WordLevel};
-
-use super::grammar_trigger_token_sequence_for_prompt;
-
-fn tokenizer() -> Tokenizer {
- let model = WordLevel::builder()
- .vocab(
- [("<unk>".to_string(), 0), ("</think>".to_string(), 1), ("\n\n".to_string(), 2), ("next".to_string(), 3)]
- .into_iter()
- .collect(),
- )
- .build()
- .unwrap();
- let mut tokenizer = Tokenizer::new(model);
- tokenizer.with_decoder(Some(Fuse::new()));
- tokenizer
-}
-
-#[test]
-fn grammar_engages_immediately_when_prompt_closes_reasoning() {
- let tokenizer = tokenizer();
-
- assert_eq!(grammar_trigger_token_sequence_for_prompt(Some(&[1]), &[1, 2], &tokenizer), None);
-}
-
-#[test]
-fn grammar_waits_for_trigger_when_only_history_contains_it() {
- let tokenizer = tokenizer();
-
- assert_eq!(grammar_trigger_token_sequence_for_prompt(Some(&[1]), &[1, 2, 3], &tokenizer), Some(vec![1]));
-}
diff --git a/crates/legacy/uzu/tests/unit/engine/cache_test.rs b/crates/legacy/uzu/tests/unit/engine/cache_test.rs
deleted file mode 100644
index 1416c7ed2..000000000
--- a/crates/legacy/uzu/tests/unit/engine/cache_test.rs
+++ /dev/null
@@ -1,92 +0,0 @@
-use std::{error::Error, future::Future, pin::Pin};
-
-use ::download_manager::FileDownloadManagerType;
-use kiban::rt::RuntimeHandle;
-use nagare::telemetry::Telemetry;
-use shoji::{
- traits::Registry as RegistryTrait,
- types::model::{Model, ModelAccessibility},
-};
-use tempfile::tempdir;
-use tokio::fs::read as tokio_read;
-
-use super::*;
-use crate::{
- models::{ModelCatalog, ModelsResolver, ResolvedModel, ResolvedModels},
- registry::FixedRegistry,
-};
-
-struct FailingRegistry(&'static str);
-
-impl RegistryTrait for FailingRegistry {
- type Error = RegistryError;
-
- fn indentifier(&self) -> String {
- self.0.to_string()
- }
-
- fn models(&self) -> Pin<Box<dyn Future<Output = Result<Vec<Model>, RegistryError>> + Send + '_>> {
- Box::pin(async {
- Err(RegistryError::UnableToGetModels {
- message: "unavailable".to_string(),
- })
- })
- }
-}
-
-#[tokio::test]
-async fn failed_refresh_preserves_snapshot_and_registry_changes_roll_back() -> Result<(), Box<dyn Error>> {
- let directory = tempdir()?;
- let cache_path = directory.path().join("resolved-models.json");
- let resolver = ModelsResolver::new(None, cache_path.clone())?;
- let model = Model::external(
- "cached-model".to_string(),
- "cached-registry".to_string(),
- "Cached Registry".to_string(),
- "backend".to_string(),
- "Backend".to_string(),
- "1".to_string(),
- Vec::new(),
- ModelAccessibility::Remote {
- repository: None,
- },
- None,
- );
- let previous = ResolvedModels::new(vec![ResolvedModel::passthrough(model)]);
- resolver.save_cache(&previous).await?;
- let cached = tokio_read(&cache_path).await?;
- let catalog = ModelCatalog::new(None, cache_path.clone()).await?;
-
- let mut registry = MergedRegistry::new(Vec::new());
- registry.add(Box::new(FailingRegistry("failing")))?;
- let runtime_handle = RuntimeHandle::try_current()?;
- let storage_config =
- StorageConfig::new(Device::new()?, Some(directory.path().to_path_buf()), "cache-test".to_string())
- .with_download_manager_type(FileDownloadManagerType::Universal);
- let engine = Engine {
- settings: SharedAccess::new(None),
- registry: SharedAccess::new(registry),
- catalog,
- catalog_refresh_lock: SharedAccess::new(()),
- storage: SharedAccess::new(Storage::new(runtime_handle, storage_config).await?),
- backends: SharedAccess::new(HashMap::new()),
- callback: SharedAccess::new(None),
- telemetry: SharedAccess::new(Telemetry::disabled()),
- huggingface_api_key: None,
- };
-
- engine.handle_initial_catalog_refresh().await?;
- let registries = engine.registry.lock().await.indentifier();
-
- assert!(engine.add_registry(Box::new(FixedRegistry::new("working".to_string(), Vec::new()))).await.is_err());
- assert_eq!(engine.registry.lock().await.indentifier(), registries);
- engine.registry.lock().await.add(Box::new(FailingRegistry("other")))?;
- let registries_with_other = engine.registry.lock().await.indentifier();
- assert!(engine.remove_registry("failing".to_string()).await.is_err());
-
- let identifiers = engine.models().await?.into_iter().map(|model| model.identifier).collect::<Vec<_>>();
- assert_eq!(identifiers, ["cached-model"]);
- assert_eq!(engine.registry.lock().await.indentifier(), registries_with_other);
- assert_eq!(tokio_read(cache_path).await?, cached);
- Ok(())
-}
diff --git a/crates/legacy/uzu/tests/unit/mod.rs b/crates/legacy/uzu/tests/unit/mod.rs
new file mode 100644
index 000000000..80aaac2ba
--- /dev/null
+++ b/crates/legacy/uzu/tests/unit/mod.rs
@@ -0,0 +1 @@
+mod storage;
diff --git a/crates/legacy/uzu/tests/unit/models/resolver_test.rs b/crates/legacy/uzu/tests/unit/models/resolver_test.rs
deleted file mode 100644
index 5c5394e27..000000000
--- a/crates/legacy/uzu/tests/unit/models/resolver_test.rs
+++ /dev/null
@@ -1,112 +0,0 @@
-use std::{error::Error, path::PathBuf};
-
-use serde_json::{Value, from_value, json};
-use shoji::types::{
- basic::{File, Hash, HashMethod, Repository},
- model::{Model, ModelAccessibility, ModelSource},
-};
-
-use super::*;
-use crate::api::HuggingFaceFileResponse;
-
-const REVISION: &str = "5ed3b433df2e390ace215873283927856573d9e6";
-
-fn repository() -> Repository {
- Repository {
- identifier: "trymirai/model".to_string(),
- commit_hash: Some(REVISION.to_string()),
- paths: None,
- }
-}
-
-fn git_file(
- name: &str,
- blob_id: Option<&str>,
-) -> HuggingFaceFileResponse {
- hugging_face_file(json!({ "rfilename": name, "size": 6, "blobId": blob_id }))
-}
-
-fn hugging_face_file(value: Value) -> HuggingFaceFileResponse {
- from_value(value).expect("Hugging Face file should deserialize")
-}
-
-#[test]
-fn hugging_face_files_use_git_and_lfs_digests() -> Result<(), Box<dyn Error>> {
- let resolver = ModelsResolver::new(None, PathBuf::new())?;
- let files = resolver.resolve_files(
- &repository(),
- REVISION,
- vec![
- git_file("config.json", Some("ce013625030ba8dba906f756967f9e9ca394464a")),
- hugging_face_file(json!({
- "rfilename": "model.safetensors",
- "size": null,
- "blobId": null,
- "lfs": {
- "sha256": "5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03",
- "size": 6
- }
- })),
- ],
- false,
- )?;
-
- assert!(matches!(files[0].check, FileCheck::GitBlobSha1(_)));
- assert!(matches!(files[1].check, FileCheck::Sha256(_)));
- assert!(files.iter().all(|file| file.file.url.contains(REVISION)));
- Ok(())
-}
-
-#[test]
-fn invalid_hugging_face_file_metadata_is_rejected() -> Result<(), Box<dyn Error>> {
- let resolver = ModelsResolver::new(None, PathBuf::new())?;
-
- let missing_digest = resolver.resolve_files(&repository(), REVISION, vec![git_file("config.json", None)], false);
- let unsafe_path = resolver.resolve_files(
- &repository(),
- REVISION,
- vec![git_file("../config.json", Some("ce013625030ba8dba906f756967f9e9ca394464a"))],
- false,
- );
-
- assert!(missing_digest.is_err());
- assert!(unsafe_path.is_err());
- Ok(())
-}
-
-#[tokio::test]
-async fn unpinned_mirai_files_keep_crc_verification() -> Result<(), Box<dyn Error>> {
- let resolver = ModelsResolver::new(None, PathBuf::new())?;
- let model = Model::external(
- "model".to_string(),
- "registry".to_string(),
- "Registry".to_string(),
- "backend".to_string(),
- "Backend".to_string(),
- "1".to_string(),
- Vec::new(),
- ModelAccessibility::OnDevice {
- source: ModelSource::Registry {
- toolchain_version: "1".to_string(),
- repository: None,
- source_repository: None,
- files: vec![File {
- url: String::new(),
- name: "model".to_string(),
- size: 1,
- hashes: vec![Hash {
- method: HashMethod::CRC32C,
- value: "AAAAAA==".to_string(),
- }],
- }],
- },
- },
- None,
- );
-
- let resolved = resolver.resolve(vec![model], &ResolvedModels::default()).await?;
- let (_, files) = resolved.iter().next().expect("model should be resolved").parts();
-
- assert!(matches!(files.expect("model should be downloadable")[0].check, FileCheck::CRC(_)));
- Ok(())
-}
diff --git a/crates/legacy/uzu/tests/unit/storage/lifecycle_mock_registry_model_test.rs b/crates/legacy/uzu/tests/unit/storage/lifecycle_mock_registry_model_test.rs
index 5c10b1697..b3f9d7fc3 100644
--- a/crates/legacy/uzu/tests/unit/storage/lifecycle_mock_registry_model_test.rs
+++ b/crates/legacy/uzu/tests/unit/storage/lifecycle_mock_registry_model_test.rs
@@ -7,14 +7,14 @@ use mock_registry::{Behavior, MockRegistry};
use rstest::rstest;
use tokio::time::{Duration, timeout};
use tokio_stream::{StreamExt, wrappers::BroadcastStream};
-
-use super::common::test_storage::TestStorage;
-use crate::{
+use uzu::{
engine::Downloader,
helpers::SharedAccess,
storage::types::{DownloadPhase, DownloadState, Item},
};
+use crate::common::{test_storage::TestStorage, tracing_setup::init_test_tracing};
+
#[rstest]
#[case::universal(FileDownloadManagerType::Universal)]
#[cfg_attr(target_vendor = "apple", case::apple(FileDownloadManagerType::Apple))]
@@ -22,8 +22,11 @@ use crate::{
async fn test_storage_mock_registry_model_download_lifecycle(
#[case] download_manager_type: FileDownloadManagerType
) -> Result<(), Box<dyn std::error::Error>> {
+ init_test_tracing();
+ tracing::info!("starting storage mock registry lifecycle test");
let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;
let model = registry.models.first().ok_or_else(|| std::io::Error::other("mock registry must include a model"))?;
+ tracing::info!(model_identifier = %model.identifier, "loaded mock registry model");
let test_storage =
TestStorage::with_models_and_manager(RuntimeHandle::current(), vec![model.clone()], download_manager_type)
.await?;
@@ -38,7 +41,8 @@ async fn test_storage_mock_registry_model_download_lifecycle(
let mut progress = item.progress().await?;
let total_bytes = item.state().await.total_bytes;
- item.download(None).await?;
+ tracing::info!(total_bytes, "starting first download");
+ item.download().await?;
let pre_pause_state = wait_for_item_state(&item, &mut progress, "first download reaches 25%", |state| {
has_reached_fraction(state, 1, 4)
})
@@ -49,34 +53,48 @@ async fn test_storage_mock_registry_model_download_lifecycle(
pre_pause_state.phase
);
+ tracing::info!("pausing storage item");
item.pause().await?;
item.reconcile().await?;
- wait_for_item_state(&item, &mut progress, "pause transitions to Paused", |state| {
+ let paused_state = wait_for_item_state(&item, &mut progress, "pause transitions to Paused", |state| {
matches!(state.phase, DownloadPhase::Paused {})
})
.await;
- item.download(None).await?;
- wait_for_item_state(&item, &mut progress, "resumed download reaches 50%", |state| {
+ tracing::info!(?paused_state, "observed paused state");
+
+ tracing::info!("resuming storage item");
+ item.download().await?;
+ let post_resume_state = wait_for_item_state(&item, &mut progress, "resumed download reaches 50%", |state| {
has_reached_fraction(state, 1, 2) || matches!(state.phase, DownloadPhase::Downloaded {})
})
.await;
+ assert!(
+ matches!(post_resume_state.phase, DownloadPhase::Downloading {} | DownloadPhase::Downloaded {}),
+ "resumed model must transition through Downloading and reach Downloaded; got {:?}",
+ post_resume_state.phase
+ );
+ tracing::info!("cancelling storage item");
item.cancel().await?;
let cancelled_state = item.state().await;
+ tracing::info!(?cancelled_state, "observed cancelled state");
assert!(matches!(cancelled_state.phase, DownloadPhase::NotDownloaded {}));
assert_eq!(cancelled_state.downloaded_bytes, 0);
- item.download(None).await?;
+ tracing::info!("starting final download");
+ item.download().await?;
let final_state = wait_for_item_state(&item, &mut progress, "final download completes", |state| {
matches!(state.phase, DownloadPhase::Downloaded {})
})
.await;
+ tracing::info!(?final_state, "observed final downloaded state");
+
assert_eq!(final_state.downloaded_bytes, total_bytes);
assert_eq!(final_state.total_bytes, total_bytes);
for served_file in registry.files {
let destination = item.cache_path.join(&served_file.file.name);
assert_eq!(tokio::fs::read(&destination).await?, served_file.bytes.to_vec());
- assert!(destination.with_added_extension("integrity").is_file());
+ assert!(std::path::PathBuf::from(format!("{}.crc", destination.display())).is_file());
}
Ok(())
@@ -89,6 +107,7 @@ async fn test_storage_mock_registry_model_download_lifecycle(
async fn test_storage_cancel_does_not_delete_locked_files(
#[case] download_manager_type: FileDownloadManagerType
) -> Result<(), Box<dyn std::error::Error>> {
+ init_test_tracing();
let registry = MockRegistry::start().await?;
let model = registry.models.first().ok_or_else(|| std::io::Error::other("mock registry must include a model"))?;
let test_storage =
@@ -127,18 +146,52 @@ async fn test_storage_cancel_does_not_delete_locked_files(
#[case::universal(FileDownloadManagerType::Universal)]
#[cfg_attr(target_vendor = "apple", case::apple(FileDownloadManagerType::Apple))]
#[tokio::test(flavor = "multi_thread")]
-async fn test_downloader_pause_updates_public_state(
+async fn test_downloader_progress_stream_closes_after_pause(
#[case] download_manager_type: FileDownloadManagerType
) -> Result<(), Box<dyn std::error::Error>> {
+ init_test_tracing();
let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;
let model = registry.models.first().ok_or_else(|| std::io::Error::other("mock registry must include a model"))?;
let test_storage =
TestStorage::with_models_and_manager(RuntimeHandle::current(), vec![model.clone()], download_manager_type)
.await?;
- let downloader = Downloader::new(model.identifier.clone(), SharedAccess::new(test_storage.storage), None);
+ let downloader = Downloader::new(model.identifier.clone(), SharedAccess::new(test_storage.storage));
downloader.resume().await?;
let progress = downloader.progress().await?;
+ timeout(Duration::from_secs(30), async {
+ loop {
+ let state = downloader.state().await.expect("model state must exist");
+ if matches!(state.phase, DownloadPhase::Downloading {}) {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+ })
+ .await?;
+
+ downloader.pause().await?;
+ timeout(Duration::from_secs(10), async { while progress.next().await.is_some() {} }).await?;
+
+ Ok(())
+}
+
+#[rstest]
+#[case::universal(FileDownloadManagerType::Universal)]
+#[cfg_attr(target_vendor = "apple", case::apple(FileDownloadManagerType::Apple))]
+#[tokio::test(flavor = "multi_thread")]
+async fn test_downloader_pause_updates_public_state(
+ #[case] download_manager_type: FileDownloadManagerType
+) -> Result<(), Box<dyn std::error::Error>> {
+ init_test_tracing();
+ let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;
+ let model = registry.models.first().ok_or_else(|| std::io::Error::other("mock registry must include a model"))?;
+ let test_storage =
+ TestStorage::with_models_and_manager(RuntimeHandle::current(), vec![model.clone()], download_manager_type)
+ .await?;
+ let downloader = Downloader::new(model.identifier.clone(), SharedAccess::new(test_storage.storage));
+
+ downloader.resume().await?;
wait_for_downloader_state(&downloader, "downloader reaches Downloading", |state| {
matches!(state.phase, DownloadPhase::Downloading {})
})
@@ -152,7 +205,6 @@ async fn test_downloader_pause_updates_public_state(
assert!(!paused_state.is_in_progress());
assert!(!paused_state.can_pause());
- timeout(Duration::from_secs(10), async { while progress.next().await.is_some() {} }).await?;
Ok(())
}
@@ -163,12 +215,13 @@ async fn test_downloader_pause_updates_public_state(
async fn test_downloader_progress_after_resuming_paused_model_reaches_downloaded(
#[case] download_manager_type: FileDownloadManagerType
) -> Result<(), Box<dyn std::error::Error>> {
+ init_test_tracing();
let registry = MockRegistry::start_with(Behavior::THROTTLED).await?;
let model = registry.models.first().ok_or_else(|| std::io::Error::other("mock registry must include a model"))?;
let test_storage =
TestStorage::with_models_and_manager(RuntimeHandle::current(), vec![model.clone()], download_manager_type)
.await?;
- let downloader = Downloader::new(model.identifier.clone(), SharedAccess::new(test_storage.storage), None);
+ let downloader = Downloader::new(model.identifier.clone(), SharedAccess::new(test_storage.storage));
downloader.resume().await?;
wait_for_downloader_state(&downloader, "downloader reaches Downloading before pause", |state| {
@@ -193,6 +246,30 @@ async fn test_downloader_progress_after_resuming_paused_model_reaches_downloaded
}
})
.await?;
+ Ok(())
+}
+
+#[rstest]
+#[case::universal(FileDownloadManagerType::Universal)]
+#[cfg_attr(target_vendor = "apple", case::apple(FileDownloadManagerType::Apple))]
+#[tokio::test(flavor = "multi_thread")]
+async fn test_downloader_progress_for_downloaded_model_is_empty(
+ #[case] download_manager_type: FileDownloadManagerType
+) -> Result<(), Box<dyn std::error::Error>> {
+ init_test_tracing();
+ let registry = MockRegistry::start().await?;
+ let model = registry.models.first().ok_or_else(|| std::io::Error::other("mock registry must include a model"))?;
+ let test_storage =
+ TestStorage::with_models_and_manager(RuntimeHandle::current(), vec![model.clone()], download_manager_type)
+ .await?;
+ let downloader = Downloader::new(model.identifier.clone(), SharedAccess::new(test_storage.storage));
+
+ downloader.resume().await?;
+ wait_for_downloader_state(&downloader, "downloader reaches Downloaded", |state| {
+ matches!(state.phase, DownloadPhase::Downloaded {})
+ })
+ .await;
+
let downloaded_progress = downloader.progress().await?;
assert!(downloaded_progress.next().await.is_none());
Ok(())
@@ -205,13 +282,16 @@ async fn wait_for_item_state(
mut is_expected_state: impl FnMut(&DownloadState) -> bool,
) -> DownloadState {
timeout(Duration::from_secs(30), async {
+ tracing::info!(label, "waiting for storage download state");
let current_state = item.state().await;
+ tracing::debug!(label, ?current_state, "observed initial storage download state");
if is_expected_state(&current_state) {
return current_state;
}
while let Some(result) = progress.next().await {
let state = result.expect("storage progress stream must not lag");
+ tracing::debug!(label, ?state, "observed storage progress state");
if is_expected_state(&state) {
return state;
}
diff --git a/crates/legacy/uzu/tests/unit/storage/mod.rs b/crates/legacy/uzu/tests/unit/storage/mod.rs
index 1bd467379..3ae7e42c9 100644
--- a/crates/legacy/uzu/tests/unit/storage/mod.rs
+++ b/crates/legacy/uzu/tests/unit/storage/mod.rs
@@ -1,3 +1 @@
-#[path = "../common/mod.rs"]
-mod common;
mod lifecycle_mock_registry_model_test;
diff --git a/crates/legacy/uzu/tests/unit/test_entry.rs b/crates/legacy/uzu/tests/unit/test_entry.rs
new file mode 100644
index 000000000..c0d36c57c
--- /dev/null
+++ b/crates/legacy/uzu/tests/unit/test_entry.rs
@@ -0,0 +1,4 @@
+#[path = "../common/mod.rs"]
+mod common;
+
+include!("mod.rs");