A high-performance inference engine for AI models
Session reset added
Aleksei Savin5121d90b2965
diff --git a/crates/uzu/src/session/session.rs b/crates/uzu/src/session/session.rsindex ad88dd710..16d1043fd 100644--- a/crates/uzu/src/session/session.rs+++ b/crates/uzu/src/session/session.rs@@ -148,6 +148,15 @@ impl Session {Ok(output)}+ pub fn reset(&mut self) -> Result<(), Error> {+ let generator =+ self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;+ generator.reset_state();+ Ok(())+ }+}++impl Session {fn extend(&mut self,input: Input,@@ -164,6 +173,44 @@ impl Session {Ok((output, new_context))}+ fn reconfigure_generator(+ &mut self,+ context: Option<&Context>,+ ) -> Result<(), Error> {+ let generator =+ self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;+ generator.reset_state();+ if let Some(ctx) = context {+ let mut generator_cache = generator.context.kv_cache.borrow_mut();+ for (ctx_layer, gen_layer) in+ ctx.kv_cache.data.iter().zip(generator_cache.data.iter_mut())+ {+ let copy_rows = ctx_layer.prefix_segment_length();+ if copy_rows > 0 {+ gen_layer.keys.borrow_mut().copy_slice(+ &ctx_layer.keys.borrow(),+ 1,+ 0..copy_rows,+ 0,+ );+ gen_layer.values.borrow_mut().copy_slice(+ &ctx_layer.values.borrow(),+ 1,+ 0..copy_rows,+ 0,+ );+ }+ gen_layer.state = ctx_layer.state.clone();+ gen_layer.prefix_token_positions =+ ctx_layer.prefix_token_positions.clone();+ }+ drop(generator_cache);++ generator.tokens = ctx.tokens.clone();+ }+ Ok(())+ }+fn run_internal<F>(&mut self,input: Input,@@ -357,44 +404,6 @@ impl Session {.clone_with_duration(run_start.elapsed().as_secs_f64()))}- fn reconfigure_generator(- &mut self,- context: Option<&Context>,- ) -> Result<(), Error> {- let generator =- self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;- generator.reset_state();- if let Some(ctx) = context {- let mut generator_cache = generator.context.kv_cache.borrow_mut();- for (ctx_layer, gen_layer) in- ctx.kv_cache.data.iter().zip(generator_cache.data.iter_mut())- {- let copy_rows = ctx_layer.prefix_segment_length();- if copy_rows > 0 {- gen_layer.keys.borrow_mut().copy_slice(- &ctx_layer.keys.borrow(),- 1,- 0..copy_rows,- 0,- );- gen_layer.values.borrow_mut().copy_slice(- &ctx_layer.values.borrow(),- 1,- 0..copy_rows,- 0,- );- }- gen_layer.state = ctx_layer.state.clone();- gen_layer.prefix_token_positions =- ctx_layer.prefix_token_positions.clone();- }- drop(generator_cache);-- generator.tokens = ctx.tokens.clone();- }- Ok(())- }-fn build_context_from_generator(&self) -> Result<Context, Error> {let generator =self.generator.as_ref().ok_or(Error::GeneratorNotLoaded)?;