Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Session reset added

Aleksei Savin5121d90b2965
diff --git a/crates/uzu/src/session/session.rs b/crates/uzu/src/session/session.rs
index ad88dd710..16d1043fd 100644
--- a/crates/uzu/src/session/session.rs
+++ b/crates/uzu/src/session/session.rs
@@ -148,6 +148,15 @@ impl Session {
Ok(output)
}
+ pub fn reset(&mut self) -> Result<(), Error> {
+ let generator =
+ self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;
+ generator.reset_state();
+ Ok(())
+ }
+}
+
+impl Session {
fn extend(
&mut self,
input: Input,
@@ -164,6 +173,44 @@ impl Session {
Ok((output, new_context))
}
+ fn reconfigure_generator(
+ &mut self,
+ context: Option<&Context>,
+ ) -> Result<(), Error> {
+ let generator =
+ self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;
+ generator.reset_state();
+ if let Some(ctx) = context {
+ let mut generator_cache = generator.context.kv_cache.borrow_mut();
+ for (ctx_layer, gen_layer) in
+ ctx.kv_cache.data.iter().zip(generator_cache.data.iter_mut())
+ {
+ let copy_rows = ctx_layer.prefix_segment_length();
+ if copy_rows > 0 {
+ gen_layer.keys.borrow_mut().copy_slice(
+ &ctx_layer.keys.borrow(),
+ 1,
+ 0..copy_rows,
+ 0,
+ );
+ gen_layer.values.borrow_mut().copy_slice(
+ &ctx_layer.values.borrow(),
+ 1,
+ 0..copy_rows,
+ 0,
+ );
+ }
+ gen_layer.state = ctx_layer.state.clone();
+ gen_layer.prefix_token_positions =
+ ctx_layer.prefix_token_positions.clone();
+ }
+ drop(generator_cache);
+
+ generator.tokens = ctx.tokens.clone();
+ }
+ Ok(())
+ }
+
fn run_internal<F>(
&mut self,
input: Input,
@@ -357,44 +404,6 @@ impl Session {
.clone_with_duration(run_start.elapsed().as_secs_f64()))
}
- fn reconfigure_generator(
- &mut self,
- context: Option<&Context>,
- ) -> Result<(), Error> {
- let generator =
- self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;
- generator.reset_state();
- if let Some(ctx) = context {
- let mut generator_cache = generator.context.kv_cache.borrow_mut();
- for (ctx_layer, gen_layer) in
- ctx.kv_cache.data.iter().zip(generator_cache.data.iter_mut())
- {
- let copy_rows = ctx_layer.prefix_segment_length();
- if copy_rows > 0 {
- gen_layer.keys.borrow_mut().copy_slice(
- &ctx_layer.keys.borrow(),
- 1,
- 0..copy_rows,
- 0,
- );
- gen_layer.values.borrow_mut().copy_slice(
- &ctx_layer.values.borrow(),
- 1,
- 0..copy_rows,
- 0,
- );
- }
- gen_layer.state = ctx_layer.state.clone();
- gen_layer.prefix_token_positions =
- ctx_layer.prefix_token_positions.clone();
- }
- drop(generator_cache);
-
- generator.tokens = ctx.tokens.clone();
- }
- Ok(())
- }
-
fn build_context_from_generator(&self) -> Result<Context, Error> {
let generator =
self.generator.as_ref().ok_or(Error::GeneratorNotLoaded)?;