A high-performance inference engine for AI models
clean old models
eugenec31cc3acdeae
diff --git a/crates/legacy/cli/src/interactive/components/selected_model.rs b/crates/legacy/cli/src/interactive/components/selected_model.rsindex da9b1a524..e1362a947 100644--- a/crates/legacy/cli/src/interactive/components/selected_model.rs+++ b/crates/legacy/cli/src/interactive/components/selected_model.rs@@ -183,7 +183,7 @@ pub fn SelectedModel(let is_downloaded = matches!(download_state.phase, DownloadPhase::Downloaded {});let is_downloading = download_state.is_in_progress();let percent = (download_state.progress() * 100.0).round() as u32;- let status = match download_state.phase {+ let status = match &download_state.phase {DownloadPhase::Downloading {} => format!("{percent}%"),DownloadPhase::Locked {..@@ -194,6 +194,9 @@ pub fn SelectedModel(.map(|specializations| specializations.name()).collect::<Vec<String>>().join(", "),+ DownloadPhase::Error {+ message,+ } => format!("Error: {message}"),_ => download_state.name(),};let progress_value = download_state.progress();diff --git a/crates/legacy/cli/src/interactive/non_interactive.rs b/crates/legacy/cli/src/interactive/non_interactive.rsindex 9018b9795..eda0697f8 100644--- a/crates/legacy/cli/src/interactive/non_interactive.rs+++ b/crates/legacy/cli/src/interactive/non_interactive.rs@@ -44,9 +44,12 @@ pub async fn run_non_interactive(announced = true;}}- if !engine.download_state(&model).await.is_some_and(|state| matches!(state.phase, DownloadPhase::Downloaded {}))- {- bail!("Model download did not complete: {}", model.identifier);+ match engine.download_state(&model).await.map(|state| state.phase) {+ Some(DownloadPhase::Downloaded {}) => {},+ Some(DownloadPhase::Error {+ message,+ }) => bail!("Model download failed: {}: {message}", model.identifier),+ _ => bail!("Model download did not complete: {}", model.identifier),}}diff --git a/crates/legacy/download-manager/src/locks/destination_lock.rs b/crates/legacy/download-manager/src/locks/destination_lock.rsindex 1733dc33d..ae0ec3ec9 100644--- a/crates/legacy/download-manager/src/locks/destination_lock.rs+++ b/crates/legacy/download-manager/src/locks/destination_lock.rs@@ -45,6 +45,22 @@ impl DestinationLock {}}+ pub async fn held_within(directory: &Path) -> bool {+ let mut directories = vec![directory.to_path_buf()];+ while let Some(directory) = directories.pop() {+ for path in fs::asyn::read_dir(&directory).await.unwrap_or_default() {+ if !fs::asyn::is_file(&path).await {+ directories.push(path);+ } else if path.extension().is_some_and(|extension| extension == "lock")+ && !matches!(FileLock::try_acquire(&path).await, Ok(Some(_)))+ {+ return true;+ }+ }+ }+ false+ }+pub async fn owner(destination: &Path) -> LockOwner {fs::asyn::read_to_string(Self::path_for(destination)).awaitdiff --git a/crates/legacy/kiban/src/fs/asyn.rs b/crates/legacy/kiban/src/fs/asyn.rsindex 862f74651..95b55b44a 100644--- a/crates/legacy/kiban/src/fs/asyn.rs+++ b/crates/legacy/kiban/src/fs/asyn.rs@@ -1,4 +1,8 @@-use std::{io, ops::Range, path::Path};+use std::{+ io,+ ops::Range,+ path::{Path, PathBuf},+};use crate::time::SystemTime;@@ -78,6 +82,25 @@ pub async fn read_range(}}+pub async fn read_dir(path: impl AsRef<Path>) -> Result<Vec<PathBuf>, io::Error> {+ #[cfg(target_family = "wasm")]+ {+ let path = path.as_ref();+ let names = super::asyn_opfs::dir_entries(path.to_str().unwrap()).await?;+ Ok(names.into_iter().map(|name| path.join(name)).collect())+ }++ #[cfg(not(target_family = "wasm"))]+ {+ let mut entries = tokio::fs::read_dir(path.as_ref()).await?;+ let mut paths = Vec::new();+ while let Some(entry) = entries.next_entry().await? {+ paths.push(entry.path());+ }+ Ok(paths)+ }+}+pub async fn read_to_string(path: impl AsRef<Path>) -> Result<String, io::Error> {#[cfg(target_family = "wasm")]{@@ -90,6 +113,14 @@ pub async fn read_to_string(path: impl AsRef<Path>) -> Result<String, io::Error>tokio::fs::read_to_string(path.as_ref()).await}+pub async fn remove_dir_all(path: impl AsRef<Path>) -> Result<(), io::Error> {+ #[cfg(target_family = "wasm")]+ return super::asyn_opfs::dir_remove_all(path.as_ref().to_str().unwrap()).await;++ #[cfg(not(target_family = "wasm"))]+ tokio::fs::remove_dir_all(path.as_ref()).await+}+pub async fn remove_file(path: impl AsRef<Path>) -> Result<(), io::Error> {#[cfg(target_family = "wasm")]return super::asyn_opfs::file_remove(path.as_ref().to_str().unwrap()).await;diff --git a/crates/legacy/kiban/src/fs/asyn_opfs.rs b/crates/legacy/kiban/src/fs/asyn_opfs.rsindex cf4bbc3ba..6f2d57fad 100644--- a/crates/legacy/kiban/src/fs/asyn_opfs.rs+++ b/crates/legacy/kiban/src/fs/asyn_opfs.rs@@ -20,6 +20,20 @@ pub async fn dir_create_all(path: &str) -> Result<(), io::Error> {Ok(())}+pub async fn dir_entries(path: &str) -> Result<Vec<String>, io::Error> {+ let root = get_root_dir().await?;+ let (parent, name) = resolve_parent(&root, path).await?;+ let directory = get_dir_handle(&parent, &name, false).await?;+ let entries = directory.entries().await.map_err(|err| js_value_to_io_error(&err, ErrorKind::Other))?;+ Ok(entries.into_iter().map(|(name, _)| name).collect())+}++pub async fn dir_remove_all(path: &str) -> Result<(), io::Error> {+ let root = get_root_dir().await?;+ let (parent, name) = resolve_parent(&root, path).await?;+ parent.remove_entry_with_options(&name, true).await.map_err(|err| js_value_to_io_error(&err, ErrorKind::Other))+}+pub async fn exists(path: &str) -> Result<bool, io::Error> {let root = get_root_dir().await?;let (parent, name) = match resolve_parent(&root, path).await {diff --git a/crates/legacy/shoji/src/traits/registry.rs b/crates/legacy/shoji/src/traits/registry.rsindex f5bf35ecb..2a7c29f41 100644--- a/crates/legacy/shoji/src/traits/registry.rs+++ b/crates/legacy/shoji/src/traits/registry.rs@@ -9,6 +9,10 @@ pub trait Registry: Send + Sync {fn models(&self) -> Pin<Box<dyn Future<Output = Result<Vec<Model>, Self::Error>> + Send + '_>>;+ fn listing(&self) -> Pin<Box<dyn Future<Output = Result<(Vec<Model>, bool), Self::Error>> + Send + '_>> {+ Box::pin(async { Ok((self.models().await?, true)) })+ }+fn model_by_identifier(&self,identifier: &ModelIdentifier,diff --git a/crates/legacy/uzu/src/engine/mod.rs b/crates/legacy/uzu/src/engine/mod.rsindex ada3739af..7671a668c 100644--- a/crates/legacy/uzu/src/engine/mod.rs+++ b/crates/legacy/uzu/src/engine/mod.rs@@ -579,8 +579,8 @@ impl Engine {}async fn handle_registry_refresh(&self) -> Result<(), EngineError> {- let models = self.registry.lock().await.models().await?;- self.storage.refresh(&models).await?;+ let (models, complete) = self.registry.lock().await.listing().await?;+ self.storage.refresh(&models, complete).await?;if let Some(callback) = self.callback.lock().await.as_ref().cloned() {callback.on_event();};diff --git a/crates/legacy/uzu/src/registry/cached.rs b/crates/legacy/uzu/src/registry/cached.rsindex 7120dbff3..d9ead9d92 100644--- a/crates/legacy/uzu/src/registry/cached.rs+++ b/crates/legacy/uzu/src/registry/cached.rs@@ -7,14 +7,14 @@ use crate::registry::RegistryError;pub struct CachedRegistry {registry: Box<dyn Registry<Error = RegistryError>>,- models: Mutex<Option<Vec<Model>>>,+ listing: Mutex<Option<(Vec<Model>, bool)>>,}impl CachedRegistry {pub fn new(registry: Box<dyn Registry<Error = RegistryError>>) -> Self {Self {registry,- models: Mutex::new(None),+ listing: Mutex::new(None),}}}@@ -27,14 +27,18 @@ impl Registry for CachedRegistry {}fn models(&self) -> Pin<Box<dyn Future<Output = Result<Vec<Model>, RegistryError>> + Send + '_>> {+ Box::pin(async { Ok(self.listing().await?.0) })+ }++ fn listing(&self) -> Pin<Box<dyn Future<Output = Result<(Vec<Model>, bool), RegistryError>> + Send + '_>> {Box::pin(async {- let mut cached_models = self.models.lock().await;- if let Some(cached_models) = cached_models.as_ref() {- Ok(cached_models.clone())+ let mut cached_listing = self.listing.lock().await;+ if let Some(cached_listing) = cached_listing.as_ref() {+ Ok(cached_listing.clone())} else {- let models = self.registry.models().await?;- *cached_models = Some(models.clone());- Ok(models)+ let listing = self.registry.listing().await?;+ *cached_listing = Some(listing.clone());+ Ok(listing)}})}diff --git a/crates/legacy/uzu/src/registry/merged.rs b/crates/legacy/uzu/src/registry/merged.rsindex c8b669a4a..f83d6e973 100644--- a/crates/legacy/uzu/src/registry/merged.rs+++ b/crates/legacy/uzu/src/registry/merged.rs@@ -93,19 +93,28 @@ impl Registry for MergedRegistry {}fn models(&self) -> Pin<Box<dyn Future<Output = Result<Vec<Model>, RegistryError>> + Send + '_>> {+ Box::pin(async { Ok(self.listing().await?.0) })+ }++ fn listing(&self) -> Pin<Box<dyn Future<Output = Result<(Vec<Model>, bool), RegistryError>> + Send + '_>> {Box::pin(async {- let results = futures::future::join_all(self.registries.iter().map(|registry| registry.models())).await;+ let results = futures::future::join_all(self.registries.iter().map(|registry| registry.listing())).await;let mut models = Vec::new();+ let mut complete = true;for (registry, result) in self.registries.iter().zip(results) {match result {- Ok(registry_models) => models.extend(registry_models),+ Ok((registry_models, registry_complete)) => {+ models.extend(registry_models);+ complete &= registry_complete;+ },Err(error) => {+ complete = false;tracing::warn!(?error, registry = %registry.identifier(), "skipping registry that failed to list models");},}}- Ok(models)+ Ok((models, complete))})}}diff --git a/crates/legacy/uzu/src/registry/mirai/registry.rs b/crates/legacy/uzu/src/registry/mirai/registry.rsindex d91a90d97..f00bea2ca 100644--- a/crates/legacy/uzu/src/registry/mirai/registry.rs+++ b/crates/legacy/uzu/src/registry/mirai/registry.rs@@ -74,10 +74,15 @@ impl RegistryTrait for Registry {}fn models(&self) -> Pin<Box<dyn Future<Output = Result<Vec<Model>, RegistryError>> + Send + '_>> {+ Box::pin(async { Ok(self.listing().await?.0) })+ }++ fn listing(&self) -> Pin<Box<dyn Future<Output = Result<(Vec<Model>, bool), RegistryError>> + Send + '_>> {Box::pin(async {let cached = self.load_registry();match self.fetch_models().await {Ok(models) => {+ let listed = models.len();let models = self.resolve(models, cached.unwrap_or_default()).await;let saved = serde_json::to_vec_pretty(&models).map_err(io::Error::other)@@ -85,14 +90,15 @@ impl RegistryTrait for Registry {if let Err(error) = saved {tracing::warn!(?error, "failed to save Mirai registry");}- Ok(models)+ let complete = models.len() == listed;+ Ok((models, complete))},Err(error) => {if error.is_transient()&& let Ok(models) = cached{tracing::warn!(?error, "serving cached Mirai registry after fetch failure");- return Ok(models);+ return Ok((models, false));}Err(RegistryError::UnableToGetModels {message: error.to_string(),diff --git a/crates/legacy/uzu/src/storage/storage.rs b/crates/legacy/uzu/src/storage/storage.rsindex 856ed463b..298873467 100644--- a/crates/legacy/uzu/src/storage/storage.rs+++ b/crates/legacy/uzu/src/storage/storage.rs@@ -1,6 +1,10 @@-use std::{collections::HashMap, path::PathBuf, sync::Arc};+use std::{+ collections::HashMap,+ path::{Path, PathBuf},+ sync::Arc,+};-use download_manager::{Checksum, DownloadManager, DownloadState, DownloadTask, DownloadTaskRequest};+use download_manager::{Checksum, DestinationLock, DownloadManager, DownloadState, DownloadTask, DownloadTaskRequest};use futures_util::future::join_all;use kiban::{fs, rt::RuntimeHandle};use shoji::types::{@@ -53,18 +57,13 @@ impl Storage {model: &Model,) -> Option<PathBuf> {let checkpoint_version = model.checkpoint_version()?;- Some(- Self::cache_path(&self.config)- .join("models")- .join("mirai")- .join(model.cache_identifier())- .join(checkpoint_version),- )+ Some(self.models_path().join(model.cache_identifier()).join(checkpoint_version))}pub async fn refresh(&self,models: &[Model],+ complete: bool,) -> Result<(), StorageError> {let mut requests = HashMap::new();for model in models {@@ -79,6 +78,29 @@ impl Storage {};requests.entry(model.identifier.clone()).or_insert(self.request(model, files)?);}+ let destinations: Vec<&Path> = requests.values().map(|request| request.destination.as_path()).collect();+ let is_listed = |path: &Path| destinations.iter().any(|destination| destination.starts_with(path));+ for model_path in fs::asyn::read_dir(self.models_path()).await.unwrap_or_default() {+ let old_paths = if is_listed(&model_path) {+ fs::asyn::read_dir(&model_path).await.unwrap_or_default()+ } else if complete && !destinations.is_empty() {+ vec![model_path]+ } else {+ continue;+ };+ for old_path in old_paths {+ if is_listed(&old_path)+ || fs::asyn::is_file(&old_path).await+ || DestinationLock::held_within(&old_path).await+ {+ continue;+ }+ match fs::asyn::remove_dir_all(&old_path).await {+ Ok(()) => tracing::info!(path = %old_path.display(), "removed old model"),+ Err(error) => tracing::warn!(?error, path = %old_path.display(), "failed to remove old model"),+ }+ }+ }let missing: Vec<(ModelIdentifier, DownloadTaskRequest)> = {let mut tasks = self.tasks.lock().await;tasks.retain(|identifier, (task, forwarder)| {@@ -161,6 +183,10 @@ impl Storage {})}+ fn models_path(&self) -> PathBuf {+ Self::cache_path(&self.config).join("models").join("mirai")+ }+fn request(&self,model: &Model,diff --git a/crates/legacy/uzu/tests/common/test_storage.rs b/crates/legacy/uzu/tests/common/test_storage.rsindex d5a279eb1..18fc9413d 100644--- a/crates/legacy/uzu/tests/common/test_storage.rs+++ b/crates/legacy/uzu/tests/common/test_storage.rs@@ -30,7 +30,7 @@ impl TestStorage {huggingface_api_key.map(|token| BearerToken::from(token.to_string())),);let storage = Storage::new(tokio_handle, config).await?;- storage.refresh(&models).await?;+ storage.refresh(&models, false).await?;Ok(Self {storage: Arc::new(storage),_temp_dir_guard: temp_dir_guard,diff --git a/crates/legacy/uzu/tests/storage.rs b/crates/legacy/uzu/tests/storage.rsindex 2a7a20eac..ca2e422fb 100644--- a/crates/legacy/uzu/tests/storage.rs+++ b/crates/legacy/uzu/tests/storage.rs@@ -105,11 +105,11 @@ async fn model_lifecycle(#[case] kind: DownloadManagerType) -> Result<(), Box<dy{files.truncate(1);}- storage.refresh(&[changed.clone()]).await?;+ storage.refresh(&[changed.clone()], false).await?;assert_eq!(storage.state(&identifier).await?.total_bytes, served.file.size);- storage.refresh(&[model.clone(), changed.clone()]).await?;+ storage.refresh(&[model.clone(), changed.clone()], false).await?;assert_eq!(storage.state(&identifier).await?.total_bytes, total_bytes);- storage.refresh(&[changed, model.clone()]).await?;+ storage.refresh(&[changed, model.clone()], false).await?;assert_eq!(storage.state(&identifier).await?.total_bytes, served.file.size);Ok(())}