A high-performance inference engine for AI models
max instead of sum for total context length
LuckyIYIb5bcbc3f282d
diff --git a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs b/crates/uzu/src/backends/metal/forward_pass/cache_layers.rsindex b418aa226..ff63d40f0 100644--- a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs+++ b/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs@@ -93,7 +93,7 @@ impl CacheLayers {max_prefix_length: usize,max_suffix_length: usize,) -> Self {- let total_context_length = max_prefix_length + max_suffix_length;+ let total_context_length = max_prefix_length.max(max_suffix_length);let kv_shapes: Vec<[usize; 3]> = model_shape.kv_cache_layer_shapes(max_prefix_length, max_suffix_length).collect();