Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

kv cache clear fix

LuckyIYI91e1ef9cd310
diff --git a/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs b/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs
index 959e6d206..2a16a8381 100644
--- a/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs
@@ -368,6 +368,7 @@ impl KVCache {
prefix_len,
} => {
*prefix_len = 0;
+ layer.prefix_token_positions.clear();
},
KVCacheLayerState::Windowed {
ring_offset,
@@ -376,9 +377,9 @@ impl KVCache {
} => {
*ring_offset = 0;
*ring_length = 0;
+ layer.prefix_token_positions.fill(INVALID_POSITION);
},
}
- layer.prefix_token_positions.fill(INVALID_POSITION);
}
}
diff --git a/crates/uzu/tests/text_session_test.rs b/crates/uzu/tests/text_session_test.rs
index 268c948c9..6fafb2aaa 100644
--- a/crates/uzu/tests/text_session_test.rs
+++ b/crates/uzu/tests/text_session_test.rs
@@ -80,9 +80,6 @@ fn run_scenario(
system_prompt: Option<String>,
user_prompts: Vec<String>,
) {
- unsafe {
- std::env::set_var("UZU_SKIP_RAM_CHECK", "1");
- }
let mut session =
Session::new(build_model_path(), build_decoding_config()).unwrap();