A high-performance inference engine for AI models
add session run autoreleaspool
eugened0d7032555c9
diff --git a/crates/uzu/src/session/session.rs b/crates/uzu/src/session/session.rsindex 9d9388781..9da0294b5 100644--- a/crates/uzu/src/session/session.rs+++ b/crates/uzu/src/session/session.rs@@ -157,59 +157,65 @@ impl Session {whereF: Fn(Output) -> bool,{- let context_mode = self- .generator- .as_ref()- .ok_or(Error::GeneratorNotLoaded)?- .decoding_config- .context_mode- .clone();-- match &context_mode {- ContextMode::None => {- let generator =- self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;- generator.reset_state();- },- ContextMode::Static {- input,- } => {- if self.static_context.is_none() {- let mut prefill_config = config.clone();- prefill_config.tokens_limit = 0;- let (_out, run_context) =- self.extend(input.clone(), None, prefill_config)?;- self.static_context = Some(run_context);- }- let tmp = self.static_context.take();- if let Some(ref run_context) = tmp {- self.reconfigure_generator(Some(run_context))?;- }- self.static_context = tmp;- },- ContextMode::Dynamic => {},- }+ autoreleasepool(|_| {+ let context_mode = self+ .generator+ .as_ref()+ .ok_or(Error::GeneratorNotLoaded)?+ .decoding_config+ .context_mode+ .clone();++ match &context_mode {+ ContextMode::None => {+ let generator = self+ .generator+ .as_mut()+ .ok_or(Error::GeneratorNotLoaded)?;+ generator.reset_state();+ },+ ContextMode::Static {+ input,+ } => {+ if self.static_context.is_none() {+ let mut prefill_config = config.clone();+ prefill_config.tokens_limit = 0;+ let (_out, run_context) =+ self.extend(input.clone(), None, prefill_config)?;+ self.static_context = Some(run_context);+ }+ let tmp = self.static_context.take();+ if let Some(ref run_context) = tmp {+ self.reconfigure_generator(Some(run_context))?;+ }+ self.static_context = tmp;+ },+ ContextMode::Dynamic => {},+ }- let output = self.run_internal(input, config, progress)?;-- match context_mode {- ContextMode::None- | ContextMode::Static {- ..- } => {- let generator =- self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;- generator.reset_state();- },- ContextMode::Dynamic => {},- }- Ok(output)+ let output = self.run_internal(input, config, progress)?;++ match context_mode {+ ContextMode::None+ | ContextMode::Static {+ ..+ } => {+ let generator = self+ .generator+ .as_mut()+ .ok_or(Error::GeneratorNotLoaded)?;+ generator.reset_state();+ },+ ContextMode::Dynamic => {},+ }+ Ok(output)+ })}pub fn reset(&mut self) -> Result<(), Error> {let generator =self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;- generator.reset_state();+ autoreleasepool(|_| generator.reset_state());Ok(())}}diff --git a/crates/uzu/src/utils/storage.rs b/crates/uzu/src/utils/storage.rsindex c6059cea7..85a575764 100644--- a/crates/uzu/src/utils/storage.rs+++ b/crates/uzu/src/utils/storage.rs@@ -1,7 +1,6 @@use std::path::PathBuf;-use objc2::rc::Retained;-use objc2_foundation::{NSHomeDirectory, NSString};+use objc2_foundation::NSHomeDirectory;/// `NSSearchPathDirectory` values (Darwin) so they can be passed directly to/// `NSFileManager::URLsForDirectory:inDomains:` via an `as u64` cast.@@ -102,11 +101,7 @@ pub fn storage_path() -> PathBuf {// `~/Library/Containers/<bundle-id>/Data`. Persist model files inside// the app-scoped caches directory to comply with sandbox rules:// `~/Library/Containers/<bundle-id>/Data/Library/Caches/…`.- let home: String = unsafe {- let ns_str: Retained<NSString> = NSHomeDirectory();- ns_str.to_string()- };-+ let home = NSHomeDirectory().to_string();PathBuf::from(&home).join("Library").join("Caches")};