A high-performance inference engine for AI models
Fix session lifecycle, tool-call parsing, and storage state (#1005)
Anton Spivak9ea6e47f04b0
diff --git a/crates/legacy/download-manager/src/backends/backend.rs b/crates/legacy/download-manager/src/backends/backend.rsindex 8cd568fb0..a8f5fd71e 100644--- a/crates/legacy/download-manager/src/backends/backend.rs+++ b/crates/legacy/download-manager/src/backends/backend.rs@@ -140,9 +140,16 @@ pub trait Backend: Send + Sync {async fn remove_files(&self,config: &DownloadConfig,- ) {- let _ = fs::asyn::remove_file(&config.resume_artifact_path).await;- let _ = fs::asyn::remove_file(&config.destination).await;+ ) -> Result<(), BackendError> {+ let mut result = Ok(());+ for path in [&config.resume_artifact_path, &config.destination] {+ if let Err(error) = fs::asyn::remove_file(path).await+ && error.kind() != std::io::ErrorKind::NotFound+ {+ result = result.and(Err(error.into()));+ }+ }+ result}}diff --git a/crates/legacy/download-manager/src/file_download/file_download_worker.rs b/crates/legacy/download-manager/src/file_download/file_download_worker.rsindex c93e4c5b2..c5e083add 100644--- a/crates/legacy/download-manager/src/file_download/file_download_worker.rs+++ b/crates/legacy/download-manager/src/file_download/file_download_worker.rs@@ -166,7 +166,7 @@ impl FileDownloadWorker {},None => self.lock().await?,};- self.backend.remove_files(&self.config).await;+ self.backend.remove_files(&self.config).await?;self.set(DownloadPhase::NotDownloaded {}, 0, None);Ok(())}@@ -215,7 +215,7 @@ impl FileDownloadWorker {self.set(DownloadPhase::Downloaded {}, total_bytes, Some(total_bytes));},Err(error) => {- self.backend.remove_files(&self.config).await;+ let _ = self.backend.remove_files(&self.config).await;self.set(DownloadPhase::Error {message: error.to_string(),diff --git a/crates/legacy/hanashi/src/chat/hanashi/mod.rs b/crates/legacy/hanashi/src/chat/hanashi/mod.rsindex 258674add..734ffb6fc 100644--- a/crates/legacy/hanashi/src/chat/hanashi/mod.rs+++ b/crates/legacy/hanashi/src/chat/hanashi/mod.rs@@ -35,6 +35,7 @@ pub struct HanashiEncodingImpl {tokenizer: Arc<Tokenizer>,parser: TokenStreamParser,framing_tokens: HashSet<String>,+ pending_framing_tokens: Vec<Token>,renderer: Renderer,validator: Validator,@@ -60,6 +61,7 @@ impl HanashiEncodingImpl {tokenizer,parser,framing_tokens,+ pending_framing_tokens: Vec::new(),renderer,validator,state: State::default(),@@ -83,6 +85,7 @@ impl EncodingTrait for HanashiEncodingImpl {fn reset(&mut self) -> Result<(), Self::Error> {self.parser.reset();+ self.pending_framing_tokens.clear();self.validator.reset();self.state = State::default();self.tokenizer_decode_ids = vec![];@@ -159,10 +162,11 @@ impl EncodingTrait for HanashiEncodingImpl {) -> Result<(), Self::Error> {for token_id in &token_ids {let token = self.resolve_token(*token_id, true)?;- self.push_token_to_parser(&token, false)?;+ self.push_decoded_token(&token)?;self.state.tokens.push(token);}+ self.parser.flush_extraction();self.update_messages_from_parser_state()?;Ok(())}diff --git a/crates/legacy/hanashi/src/chat/hanashi/token.rs b/crates/legacy/hanashi/src/chat/hanashi/token.rsindex 4b98a905b..ecb1ef0ff 100644--- a/crates/legacy/hanashi/src/chat/hanashi/token.rs+++ b/crates/legacy/hanashi/src/chat/hanashi/token.rs@@ -1,6 +1,8 @@use shoji::types::basic::Token;use token_stream_parser::types::Token as ParserToken;+use super::{Error, HanashiEncodingImpl};+pub trait ToParserToken {type Output;@@ -18,3 +20,59 @@ impl ToParserToken for Token {}}}++impl HanashiEncodingImpl {+ pub fn flush(&mut self) -> Result<bool, Error> {+ if self.pending_framing_tokens.is_empty() {+ return Ok(false);+ }+ for token in std::mem::take(&mut self.pending_framing_tokens) {+ self.push_token_to_parser(&token, true)?;+ }+ self.parser.flush_extraction();+ self.update_messages_from_parser_state()?;+ Ok(true)+ }++ pub(super) fn push_decoded_token(+ &mut self,+ token: &Token,+ ) -> Result<(), Error> {+ if token.is_special || token.value.is_empty() || self.framing_tokens.contains(&token.value) {+ self.flush()?;+ return self.push_token_to_parser(token, true);+ }+ loop {+ let text: String = self+ .pending_framing_tokens+ .iter()+ .map(|token| token.value.as_str())+ .chain(std::iter::once(token.value.as_str()))+ .collect();+ let marker = self+ .framing_tokens+ .iter()+ .filter(|marker| {+ !self.tokenizer.get_added_vocabulary().is_special_token(marker) && marker.starts_with(&text)+ })+ .filter_map(|marker| self.tokenizer.token_to_id(marker).map(|id| (marker.len() == text.len(), id)))+ .min_by_key(|(complete, _)| !complete);+ match marker {+ Some((true, id)) => {+ self.pending_framing_tokens.clear();+ // Normalize the parser view only; State.tokens retains the sampled IDs.+ let marker = self.resolve_token(id, false)?;+ return self.push_token_to_parser(&marker, true);+ },+ Some((false, _)) => {+ self.pending_framing_tokens.push(token.clone());+ return Ok(());+ },+ None if self.pending_framing_tokens.is_empty() => return self.push_token_to_parser(token, true),+ None => {+ self.flush()?;+ },+ }+ }+ }+}diff --git a/crates/legacy/hanashi/src/chat/mod.rs b/crates/legacy/hanashi/src/chat/mod.rsindex ebfc296b3..8ffdbd2db 100644--- a/crates/legacy/hanashi/src/chat/mod.rs+++ b/crates/legacy/hanashi/src/chat/mod.rs@@ -48,6 +48,13 @@ impl Encoding {}}+ pub fn flush(&mut self) -> Result<bool, Error> {+ match self {+ Self::Hanashi(inner) => inner.flush().map_err(Into::into),+ Self::Harmony(_) => Ok(false),+ }+ }+pub fn tokenize(&self,text: &str,diff --git a/crates/legacy/nagare/src/chat/token.rs b/crates/legacy/nagare/src/chat/token.rsindex 9e3e06e6a..47d4f458c 100644--- a/crates/legacy/nagare/src/chat/token.rs+++ b/crates/legacy/nagare/src/chat/token.rs@@ -144,6 +144,7 @@ impl Session {}}let time_prefill_start = Instant::now();+ let cancel_token = cancel_token.child_token();let stream = instance.stream(&self.input_tokens, self.state.as_mut(), config.clone(), cancel_token.clone());let output_tokens_start = self.encoding.state().tokens.len();@@ -176,22 +177,34 @@ impl Session {return None;}- match inner.next().await {- Some(event) => {- state.metrics = inner.metrics();- state.memory_usage = instance.peak_memory_usage();- let output = Self::build_output(event, &mut state);- let terminated = terminated || matches!(&output, Ok(out) if out.finish_reason.is_some());- Some((output, (inner, state, terminated, false)))- },- None => {- if !terminated && state.cancel_token.is_cancelled() {- let output = Self::render_output(&mut state, Some(ChatReplyFinishReason::Cancelled));- Some((output, (inner, state, true, true)))- } else {- None- }+ while let Some(event) = inner.next().await {+ if terminated && event.is_ok() {+ continue;+ }+ state.metrics = inner.metrics();+ state.memory_usage = instance.peak_memory_usage();+ let output = Self::build_output(event, &mut state);+ let terminated = terminated || matches!(&output, Ok(out) if out.finish_reason.is_some());+ if terminated {+ state.cancel_token.cancel();+ }+ return Some((output, (inner, state, terminated, false)));+ }++ match state.encoding.flush() {+ Err(error) => Some((+ Err(ChatSessionError::Backend {+ message: error.to_string(),+ }),+ (inner, state, true, true),+ )),+ Ok(flushed) if !terminated && (flushed || state.cancel_token.is_cancelled()) => {+ let finish_reason =+ state.cancel_token.is_cancelled().then_some(ChatReplyFinishReason::Cancelled);+ let output = Self::render_output(&mut state, finish_reason);+ Some((output, (inner, state, true, true)))},+ _ => None,}},)@@ -298,6 +311,11 @@ impl Session {finish_reason: Option<ChatReplyFinishReason>,) -> Result<Output, ChatSessionError> {let have_finish_reason = finish_reason.is_some();+ if have_finish_reason {+ state.encoding.flush().map_err(|error| ChatSessionError::Backend {+ message: error.to_string(),+ })?;+ }let stats = state.get_stats(have_finish_reason)?;let Some(message) = state.encoding.state().messages.last() else {return Ok(Output {diff --git a/crates/legacy/nagare/src/chat/tool_cancellation_tests.rs b/crates/legacy/nagare/src/chat/tool_cancellation_tests.rsindex 12e208319..5aaad744c 100644--- a/crates/legacy/nagare/src/chat/tool_cancellation_tests.rs+++ b/crates/legacy/nagare/src/chat/tool_cancellation_tests.rs@@ -75,6 +75,17 @@ impl BackendInstance for FunctionGemma {}impl TokenInstance for FunctionGemma {+ fn with_config(+ &self,+ _config: ChatConfig,+ ) -> Box<dyn TokenInstance> {+ Box::new(Self {+ tokenizer: self.tokenizer.clone(),+ cached_tokens: StdMutex::new(Vec::new()),+ prompts: StdMutex::new(Vec::new()),+ })+ }+fn tokenizer(&self) -> Arc<Tokenizer> {self.tokenizer.clone()}diff --git a/crates/legacy/shoji/src/traits/backend/chat_token.rs b/crates/legacy/shoji/src/traits/backend/chat_token.rsindex 7b8632bf7..40961794c 100644--- a/crates/legacy/shoji/src/traits/backend/chat_token.rs+++ b/crates/legacy/shoji/src/traits/backend/chat_token.rs@@ -45,6 +45,11 @@ pub trait Instance:StreamMetrics = StreamMetrics,>{+ fn with_config(+ &self,+ config: ChatConfig,+ ) -> Box<dyn Instance>;+fn tokenizer(&self) -> Arc<Tokenizer>;fn max_context_length(&self) -> Option<usize>;diff --git a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swift b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swiftindex a5b98eed2..d2faad574 100644--- a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swift+++ b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/uzu.swift@@ -2905,6 +2905,7 @@ public enum StorageError: Swift.Error, Equatable, Hashable, Codable, Foundation.)case DownloadManager(message: String)+ case Lockedcase ModelNotFound(identifier: String)case UnsupportedModel(identifier: String@@ -2944,10 +2945,11 @@ public struct FfiConverterTypeStorageError: FfiConverterRustBuffer {case 2: return .DownloadManager(message: try FfiConverterString.read(from: &buf))- case 3: return .ModelNotFound(+ case 3: return .Locked+ case 4: return .ModelNotFound(identifier: try FfiConverterString.read(from: &buf))- case 4: return .UnsupportedModel(+ case 5: return .UnsupportedModel(identifier: try FfiConverterString.read(from: &buf))@@ -2972,13 +2974,17 @@ public struct FfiConverterTypeStorageError: FfiConverterRustBuffer {FfiConverterString.write(message, into: &buf)- case let .ModelNotFound(identifier):+ case .Locked:writeInt(&buf, Int32(3))+++ case let .ModelNotFound(identifier):+ writeInt(&buf, Int32(4))FfiConverterString.write(identifier, into: &buf)case let .UnsupportedModel(identifier):- writeInt(&buf, Int32(4))+ writeInt(&buf, Int32(5))FfiConverterString.write(identifier, into: &buf)}diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mtsindex 02569a527..5b8cc1e6b 100644--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts@@ -1076,5 +1076,6 @@ export type SettingsError =export type StorageError =| { type: 'UnableToCreateDirectory'; path: string }| { type: 'DownloadManager'; message: string }+ | { type: 'Locked' }| { type: 'ModelNotFound'; identifier: ModelIdentifier }| { type: 'UnsupportedModel'; identifier: ModelIdentifier }diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.tsindex 02569a527..5b8cc1e6b 100644--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts@@ -1076,5 +1076,6 @@ export type SettingsError =export type StorageError =| { type: 'UnableToCreateDirectory'; path: string }| { type: 'DownloadManager'; message: string }+ | { type: 'Locked' }| { type: 'ModelNotFound'; identifier: ModelIdentifier }| { type: 'UnsupportedModel'; identifier: ModelIdentifier }diff --git a/crates/legacy/uzu/src/device/os.rs b/crates/legacy/uzu/src/device/os.rsindex 2ffba6db8..948c30f9b 100644--- a/crates/legacy/uzu/src/device/os.rs+++ b/crates/legacy/uzu/src/device/os.rs@@ -2,7 +2,7 @@ use std::path::PathBuf;#[allow(unreachable_code)]pub fn home_path() -> Option<PathBuf> {- #[cfg(target_os = "ios")]+ #[cfg(all(target_os = "ios", not(target_abi = "macabi")))]{use objc2::rc::autoreleasepool;use objc2_foundation::{NSFileManager, NSSearchPathDirectory, NSSearchPathDomainMask};diff --git a/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs b/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rsindex 7285b0fb0..6beb189e0 100644--- a/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs+++ b/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs@@ -40,9 +40,10 @@ use crate::engine::bridge::{helpers::{error_stream, get_max_context_length, get_sampling_method},};+#[derive(Clone)]pub struct UzuChatTokenBackendInstance<B: Backend> {engine: Arc<Engine<B>>,- model: LanguageModel<B>,+ model: Arc<LanguageModel<B>>,config: ChatConfig,stop_token_ids: Vec<i32>,max_context_length: Option<u32>,@@ -76,7 +77,7 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {Ok(Self {engine,- model,+ model: Arc::new(model),config,stop_token_ids,max_context_length,@@ -95,7 +96,6 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {fn state(&self) -> Pin<Box<dyn Future<Output = Result<Box<dyn State>, BackendError>> + Send + '_>> {Box::pin(async move {- let max_context_length = get_max_context_length(&self.model, self.config.context_length.clone());let sampling_seed = match &self.config.sampling_seed {SamplingSeed::Default {} => rand::random(),SamplingSeed::Custom {@@ -103,7 +103,7 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {} => *seed as u64,};self.model- .create_empty_state(max_context_length, sampling_seed)+ .create_empty_state(self.max_context_length, sampling_seed).map_err(|err| BackendError::from(err.to_string())).map(|state| Box::new(UzuChatTokenBackendInstanceState::new(state)) as Box<dyn State>)})@@ -170,30 +170,41 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {let metrics = stream.metrics().clone();let inner = stream::unfold(- (stream, 0, cancel_token.child_token()),- move |(mut stream, mut tokens_generated, cancel_token)| async move {- if cancel_token.is_cancelled() {+ (stream, 0, cancel_token.child_token(), false),+ move |(mut stream, mut tokens_generated, cancel_token, finished)| async move {+ if finished {return None;}- let output = if token_limit.is_some_and(|token_limit| tokens_generated >= token_limit) {- cancel_token.cancel();- Ok(TokenStreamOutput::LimitReached)+ let cancelled = cancel_token.is_cancelled();+ let output = if cancelled || token_limit.is_some_and(|token_limit| tokens_generated >= token_limit) {+ match stream.close().await {+ Ok(()) if cancelled => return None,+ Ok(()) => Ok(TokenStreamOutput::LimitReached),+ Err(error) => Err(BackendError::from(error.to_string())),+ }} else {- match stream.generate().await {+ match stream.generate().await.map_err(|error| BackendError::from(error.to_string())) {Ok(Some(token)) => {tokens_generated += 1;Ok(TokenStreamOutput::Token(token))},- Ok(None) => return None,+ Ok(None) => match stream.close().await {+ Ok(()) => return None,+ Err(error) => Err(BackendError::from(error.to_string())),+ },Err(error) => {- cancel_token.cancel();- Err(BackendError::from(error.to_string()))+ let _ = stream.close().await;+ Err(error)},}};+ let finished = !matches!(&output, Ok(TokenStreamOutput::Token(_)));+ if output.is_err() {+ cancel_token.cancel();+ }let metrics = stream.metrics().clone();- Some(((output, metrics), (stream, tokens_generated, cancel_token)))+ Some(((output, metrics), (stream, tokens_generated, cancel_token, finished)))},);@@ -209,6 +220,17 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {}impl<B: Backend> ChatTokenBackendInstance for UzuChatTokenBackendInstance<B> {+ fn with_config(+ &self,+ config: ChatConfig,+ ) -> Box<dyn ChatTokenBackendInstance> {+ Box::new(Self {+ max_context_length: get_max_context_length(&self.model, config.context_length.clone()),+ config,+ ..self.clone()+ })+ }+fn tokenizer(&self) -> Arc<Tokenizer> {self.model.tokenizer().clone()}diff --git a/crates/legacy/uzu/src/engine/mod.rs b/crates/legacy/uzu/src/engine/mod.rsindex 0e8638f32..b42bab1e8 100644--- a/crates/legacy/uzu/src/engine/mod.rs+++ b/crates/legacy/uzu/src/engine/mod.rs@@ -712,6 +712,13 @@ impl Engine {Ok(self.storage.ready_state(&model.identifier).await?)}+ pub async fn refresh_download_state(+ &self,+ model: &Model,+ ) -> Result<DownloadState, EngineError> {+ Ok(self.storage.refresh_state(&model.identifier).await?)+ }+pub fn catalog_is_refreshing(&self) -> bool {self.catalog_refreshes.load(Ordering::SeqCst) != 0}diff --git a/crates/legacy/uzu/src/storage/error.rs b/crates/legacy/uzu/src/storage/error.rsindex 8efcc0c2f..28e0e9056 100644--- a/crates/legacy/uzu/src/storage/error.rs+++ b/crates/legacy/uzu/src/storage/error.rs@@ -1,4 +1,4 @@-use download_manager::DownloadError;+use download_manager::{BackendError, DownloadError, FileDownloadError, LockError};use shoji::types::model::ModelIdentifier;#[bindings::export(Error)]@@ -13,6 +13,8 @@ pub enum StorageError {DownloadManager {message: String,},+ #[error("Model is locked by another manager")]+ Locked {},#[error("Model not found: {identifier}")]ModelNotFound {identifier: ModelIdentifier,@@ -25,8 +27,22 @@ pub enum StorageError {impl From<DownloadError> for StorageError {fn from(error: DownloadError) -> Self {- Self::DownloadManager {- message: error.to_string(),+ match error {+ DownloadError::Lock(LockError::LockedByOther {+ ..+ })+ | DownloadError::Backend(BackendError::Lock(LockError::LockedByOther {+ ..+ }))+ | DownloadError::FileDownload(FileDownloadError::Lock(LockError::LockedByOther {+ ..+ }))+ | DownloadError::FileDownload(FileDownloadError::Backend(BackendError::Lock(LockError::LockedByOther {+ ..+ }))) => Self::Locked {},+ error => Self::DownloadManager {+ message: error.to_string(),+ },}}}diff --git a/crates/legacy/uzu/src/storage/storage.rs b/crates/legacy/uzu/src/storage/storage.rsindex a7c442157..374f6017c 100644--- a/crates/legacy/uzu/src/storage/storage.rs+++ b/crates/legacy/uzu/src/storage/storage.rs@@ -164,6 +164,34 @@ impl Storage {Ok(self.model(identifier).await?.state())}+ pub async fn refresh_state(+ &self,+ identifier: &ModelIdentifier,+ ) -> Result<DownloadState, StorageError> {+ let _refresh = self.refresh_lock.lock().await;+ {+ let mut tasks = self.tasks.lock().await;+ let task = tasks.get(identifier).ok_or_else(|| StorageError::ModelNotFound {+ identifier: identifier.clone(),+ })?;+ if !task.state().is_in_progress() {+ if let Ok(native) = task.ready().await {+ native.pause().await?;+ }+ let replacement = Arc::new(ModelTask::new(+ identifier.clone(),+ task.request.clone(),+ task.state().total_bytes,+ Arc::clone(&self.download_manager),+ self.events.clone(),+ Some(Arc::clone(task)),+ ));+ tasks.insert(identifier.clone(), replacement);+ }+ }+ self.ready_state(identifier).await+ }+pub async fn states(&self) -> HashMap<ModelIdentifier, DownloadState> {self.tasks.lock().await.iter().map(|(identifier, task)| (identifier.clone(), task.state())).collect()}diff --git a/crates/legacy/uzu/tests/storage.rs b/crates/legacy/uzu/tests/storage.rsindex b67cccf57..ca8e99d22 100644--- a/crates/legacy/uzu/tests/storage.rs+++ b/crates/legacy/uzu/tests/storage.rs@@ -14,7 +14,7 @@ use uuid::Uuid;use uzu::{engine::Downloader,registry::mirai::HUGGING_FACE_URL,- storage::{DownloadManagerType, DownloadPhase, DownloadState, Storage},+ storage::{DownloadManagerType, DownloadPhase, DownloadState, Storage, StorageError},types::{basic::{File, Repository},model::{Model, ModelAccessibility, ModelIdentifier, ModelSource},@@ -90,7 +90,7 @@ async fn model_lifecycle(#[case] kind: DownloadManagerType) -> Result<(), Box<dy).await?;let refused = storage.delete(&identifier).await.expect_err("delete must be refused while locked");- assert!(refused.to_string().contains("foreign-manager"), "unexpected error: {refused}");+ assert!(matches!(refused, StorageError::Locked {}), "unexpected error: {refused}");assert!(destination.exists());let mut changed = model.clone();diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rsindex 365e82c22..93dfef6e0 100644--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs@@ -906,6 +906,8 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {fn drop(&mut self) {- block_on(self.close()).unwrap();+ if !matches!(self.decoding_state, DecodingState::Closed) {+ let _ = block_on(self.close());+ }}}