Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Hotfix memory usage (#794)

uuuvnc7d29276383d
diff --git a/crates/uzu-engine/src/engine/language_model/stream/stream.rs b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
index 85622c5d6..6a99d30f9 100644
--- a/crates/uzu-engine/src/engine/language_model/stream/stream.rs
+++ b/crates/uzu-engine/src/engine/language_model/stream/stream.rs
@@ -75,6 +75,16 @@ impl<B: Backend> ForwardPassChaining<B> {
},
}
}
+
+ fn drop_output_norm(&mut self) {
+ match self {
+ Self::Constant {
+ output_norm,
+ ..
+ } => drop(output_norm.take()),
+ Self::InFlight(pending) => drop(pending.output_norm.take()),
+ }
+ }
}
struct DecodingStatePending<B: Backend> {
@@ -570,8 +580,10 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
&self.model_state.prng,
self.allocation_pool.clone(),
)?;
+ prev_output.drop_output_norm();
(trie, None, false)
} else {
+ prev_output.drop_output_norm();
let (token, chain_copy) = match &prev_output {
ForwardPassChaining::Constant {
token,
@@ -583,12 +595,15 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
};
let input_flat_trie = input_trie.linearize();
- let mut encoder = if let Some(encoder) = encoder {
- encoder
- } else {
+ if let Some(accept_encoder) = encoder.take() {
+ pending.push(accept_encoder.end_encoding().submit());
+ }
+
+ self.allocation_pool = Arc::new(self.model.engine.context.create_allocation_pool(false));
+
+ let mut encoder =
Encoder::<B>::new_with_pool_name(&self.model.engine.context, self.allocation_pool.clone(), Some("decode"))
- .map_err(LanguageModelStreamError::Backend)?
- };
+ .map_err(LanguageModelStreamError::Backend)?;
let token_ids = if let Some(chain_copy) = chain_copy {
let mut token_ids =