Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

add session run autoreleaspool

eugened0d7032555c9
diff --git a/crates/uzu/src/session/session.rs b/crates/uzu/src/session/session.rs
index 9d9388781..9da0294b5 100644
--- a/crates/uzu/src/session/session.rs
+++ b/crates/uzu/src/session/session.rs
@@ -157,59 +157,65 @@ impl Session {
where
F: Fn(Output) -> bool,
{
- let context_mode = self
- .generator
- .as_ref()
- .ok_or(Error::GeneratorNotLoaded)?
- .decoding_config
- .context_mode
- .clone();
-
- match &context_mode {
- ContextMode::None => {
- let generator =
- self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;
- generator.reset_state();
- },
- ContextMode::Static {
- input,
- } => {
- if self.static_context.is_none() {
- let mut prefill_config = config.clone();
- prefill_config.tokens_limit = 0;
- let (_out, run_context) =
- self.extend(input.clone(), None, prefill_config)?;
- self.static_context = Some(run_context);
- }
- let tmp = self.static_context.take();
- if let Some(ref run_context) = tmp {
- self.reconfigure_generator(Some(run_context))?;
- }
- self.static_context = tmp;
- },
- ContextMode::Dynamic => {},
- }
+ autoreleasepool(|_| {
+ let context_mode = self
+ .generator
+ .as_ref()
+ .ok_or(Error::GeneratorNotLoaded)?
+ .decoding_config
+ .context_mode
+ .clone();
+
+ match &context_mode {
+ ContextMode::None => {
+ let generator = self
+ .generator
+ .as_mut()
+ .ok_or(Error::GeneratorNotLoaded)?;
+ generator.reset_state();
+ },
+ ContextMode::Static {
+ input,
+ } => {
+ if self.static_context.is_none() {
+ let mut prefill_config = config.clone();
+ prefill_config.tokens_limit = 0;
+ let (_out, run_context) =
+ self.extend(input.clone(), None, prefill_config)?;
+ self.static_context = Some(run_context);
+ }
+ let tmp = self.static_context.take();
+ if let Some(ref run_context) = tmp {
+ self.reconfigure_generator(Some(run_context))?;
+ }
+ self.static_context = tmp;
+ },
+ ContextMode::Dynamic => {},
+ }
- let output = self.run_internal(input, config, progress)?;
-
- match context_mode {
- ContextMode::None
- | ContextMode::Static {
- ..
- } => {
- let generator =
- self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;
- generator.reset_state();
- },
- ContextMode::Dynamic => {},
- }
- Ok(output)
+ let output = self.run_internal(input, config, progress)?;
+
+ match context_mode {
+ ContextMode::None
+ | ContextMode::Static {
+ ..
+ } => {
+ let generator = self
+ .generator
+ .as_mut()
+ .ok_or(Error::GeneratorNotLoaded)?;
+ generator.reset_state();
+ },
+ ContextMode::Dynamic => {},
+ }
+ Ok(output)
+ })
}
pub fn reset(&mut self) -> Result<(), Error> {
let generator =
self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;
- generator.reset_state();
+ autoreleasepool(|_| generator.reset_state());
Ok(())
}
}
diff --git a/crates/uzu/src/utils/storage.rs b/crates/uzu/src/utils/storage.rs
index c6059cea7..85a575764 100644
--- a/crates/uzu/src/utils/storage.rs
+++ b/crates/uzu/src/utils/storage.rs
@@ -1,7 +1,6 @@
use std::path::PathBuf;
-use objc2::rc::Retained;
-use objc2_foundation::{NSHomeDirectory, NSString};
+use objc2_foundation::NSHomeDirectory;
/// `NSSearchPathDirectory` values (Darwin) so they can be passed directly to
/// `NSFileManager::URLsForDirectory:inDomains:` via an `as u64` cast.
@@ -102,11 +101,7 @@ pub fn storage_path() -> PathBuf {
// `~/Library/Containers/<bundle-id>/Data`. Persist model files inside
// the app-scoped caches directory to comply with sandbox rules:
// `~/Library/Containers/<bundle-id>/Data/Library/Caches/…`.
- let home: String = unsafe {
- let ns_str: Retained<NSString> = NSHomeDirectory();
- ns_str.to_string()
- };
-
+ let home = NSHomeDirectory().to_string();
PathBuf::from(&home).join("Library").join("Caches")
};