Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

max instead of sum for total context length

LuckyIYIb5bcbc3f282d
diff --git a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs b/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs
index b418aa226..ff63d40f0 100644
--- a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs
@@ -93,7 +93,7 @@ impl CacheLayers {
max_prefix_length: usize,
max_suffix_length: usize,
) -> Self {
- let total_context_length = max_prefix_length + max_suffix_length;
+ let total_context_length = max_prefix_length.max(max_suffix_length);
let kv_shapes: Vec<[usize; 3]> = model_shape
.kv_cache_layer_shapes(max_prefix_length, max_suffix_length)
.collect();