A high-performance inference engine for AI models
added function constants and changed address space
LuckyIYIea165b72c9be
diff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metalindex b149770e7..b35251ede 100644--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal+++ b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal@@ -12,11 +12,11 @@ template <typename T>void updateKVCache(device T* inPlaceKeys,device T* inPlaceValues,- device int2* swaps,- constant int& swapCount,- constant int& numHeads,- constant int& maxSequenceLength,- constant int& headDim,+ const constant int2* swaps,+ const constant int& swapCount,+ const constant int& numHeads,+ const constant int& maxSequenceLength,+ const constant int& headDim,const uint2 position) {for (int i = 0; i < swapCount; ++i) {@@ -35,11 +35,11 @@ void updateKVCache(#define outerArguments(T) \(device T * inPlaceKeys [[buffer(0)]], \device T * inPlaceValues [[buffer(1)]], \- device int2 * swaps [[buffer(2)]], \- constant int& swapCount [[buffer(3)]], \- constant int& numHeads [[buffer(4)]], \- constant int& maxSequenceLength [[buffer(5)]], \- constant int& headDim [[buffer(6)]], \+ const constant int2 * swaps [[buffer(2)]], \+ const constant int& swapCount [[buffer(3)]], \+ const constant int& numHeads [[buffer(4)]], \+ const constant int& maxSequenceLength [[buffer(5)]], \+ const constant int& headDim [[buffer(6)]], \const uint2 position [[thread_position_in_grid]])#define innerArguments \diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rsindex 4ea13a2ba..d2798e147 100644--- a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs@@ -2,7 +2,8 @@ use std::mem::size_of;use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef,- ComputePipelineState as MTLComputePipelineState, MTLSize,+ ComputePipelineState as MTLComputePipelineState, FunctionConstantValues,+ MTLDataType, MTLSize,};use crate::backends::metal::{KernelDataType, MTLContext, MTLError};@@ -17,10 +18,22 @@ fn fn_suffix(dt: KernelDataType) -> &'static str {dt.function_name_suffix()}+fn make_function_constants(has_bias: bool) -> FunctionConstantValues {+ let function_constants = FunctionConstantValues::new();+ function_constants.set_constant_value_at_index(+ &has_bias as *const bool as *const std::ffi::c_void,+ MTLDataType::Bool,+ 0,+ );+ function_constants+}+pub struct ShortConvKernel {pack_pipeline: MTLComputePipelineState,- prefill_pipeline: MTLComputePipelineState,- decode_pipeline: MTLComputePipelineState,+ prefill_pipeline_no_bias: MTLComputePipelineState,+ prefill_pipeline_with_bias: MTLComputePipelineState,+ decode_pipeline_no_bias: MTLComputePipelineState,+ decode_pipeline_with_bias: MTLComputePipelineState,}pub struct ShortConvPackArguments<'a> {@@ -76,17 +89,58 @@ impl ShortConvKernel {let pack_pipeline = context.compute_pipeline_state(&pack_name, None).map_err(ShortConvKernelError::MetalError)?;- let prefill_pipeline = context- .compute_pipeline_state(&prefill_name, None)- .map_err(ShortConvKernelError::MetalError)?;- let decode_pipeline = context- .compute_pipeline_state(&decode_name, None)- .map_err(ShortConvKernelError::MetalError)?;++ let prefill_pipeline_no_bias = {+ let function_constants = make_function_constants(false);+ let cache_key = format!("{}_has_bias_0", prefill_name);+ context+ .compute_pipeline_state_cached(+ &cache_key,+ &prefill_name,+ Some(&function_constants),+ )+ .map_err(ShortConvKernelError::MetalError)?+ };+ let prefill_pipeline_with_bias = {+ let function_constants = make_function_constants(true);+ let cache_key = format!("{}_has_bias_1", prefill_name);+ context+ .compute_pipeline_state_cached(+ &cache_key,+ &prefill_name,+ Some(&function_constants),+ )+ .map_err(ShortConvKernelError::MetalError)?+ };+ let decode_pipeline_no_bias = {+ let function_constants = make_function_constants(false);+ let cache_key = format!("{}_has_bias_0", decode_name);+ context+ .compute_pipeline_state_cached(+ &cache_key,+ &decode_name,+ Some(&function_constants),+ )+ .map_err(ShortConvKernelError::MetalError)?+ };+ let decode_pipeline_with_bias = {+ let function_constants = make_function_constants(true);+ let cache_key = format!("{}_has_bias_1", decode_name);+ context+ .compute_pipeline_state_cached(+ &cache_key,+ &decode_name,+ Some(&function_constants),+ )+ .map_err(ShortConvKernelError::MetalError)?+ };Ok(Self {pack_pipeline,- prefill_pipeline,- decode_pipeline,+ prefill_pipeline_no_bias,+ prefill_pipeline_with_bias,+ decode_pipeline_no_bias,+ decode_pipeline_with_bias,})}@@ -153,11 +207,19 @@ impl ShortConvKernel {let tap_count = (args.kernel_size - 1).max(0);let work_len = args.suffix_len + tap_count as usize;- compute_encoder.set_compute_pipeline_state(&self.prefill_pipeline);+ let has_bias = args.b.is_some();+ let pipeline = if has_bias {+ &self.prefill_pipeline_with_bias+ } else {+ &self.prefill_pipeline_no_bias+ };+ compute_encoder.set_compute_pipeline_state(pipeline);compute_encoder.set_buffer(0, Some(args.padded), 0);compute_encoder.set_buffer(1, Some(args.in_proj), 0);compute_encoder.set_buffer(2, Some(args.w), 0);- compute_encoder.set_buffer(3, args.b.map(|v| &**v), 0);+ if has_bias {+ compute_encoder.set_buffer(3, args.b.map(|v| &**v), 0);+ }compute_encoder.set_buffer(4, Some(args.out), 0);compute_encoder.set_buffer(5, Some(args.state_out), 0);compute_encoder.set_bytes(@@ -211,10 +273,18 @@ impl ShortConvKernel {return Ok(());}- compute_encoder.set_compute_pipeline_state(&self.decode_pipeline);+ let has_bias = args.b.is_some();+ let pipeline = if has_bias {+ &self.decode_pipeline_with_bias+ } else {+ &self.decode_pipeline_no_bias+ };+ compute_encoder.set_compute_pipeline_state(pipeline);compute_encoder.set_buffer(0, Some(args.in_proj), 0);compute_encoder.set_buffer(1, Some(args.w), 0);- compute_encoder.set_buffer(2, args.b.map(|v| &**v), 0);+ if has_bias {+ compute_encoder.set_buffer(2, args.b.map(|v| &**v), 0);+ }compute_encoder.set_buffer(3, Some(args.state), 0);compute_encoder.set_buffer(4, Some(args.out), 0);compute_encoder.set_buffer(5, Some(args.next_state), 0);diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metalindex 12819218f..d184f9262 100644--- a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal+++ b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal@@ -3,6 +3,8 @@using namespace metal;+constant bool has_bias [[function_constant(0)]];+template <typename T>[[kernel, max_total_threads_per_threadgroup(32)]]void short_conv_pack_kernel(@@ -51,7 +53,7 @@ void short_conv_prefill_kernel(device const T* padded [[buffer(0)]],device const T* in_proj [[buffer(1)]],device const T* w [[buffer(2)]],- device const T* b [[buffer(3)]],+ device const T* b [[buffer(3), function_constant(has_bias)]],device T* out [[buffer(4)]],device T* state_out [[buffer(5)]],constant const size_t& suffix_len [[buffer(6)]],@@ -72,11 +74,13 @@ void short_conv_prefill_kernel(}const device T* w_row = w + size_t(channel_idx) * size_t(kernel_size);- const bool has_bias = b != nullptr;// Threads [0..suffix_len-1]: Compute outputsif (token_idx < suffix_len) {- float acc = has_bias ? float(b[channel_idx]) : 0.0f;+ float acc = 0.0f;+ if (has_bias) {+ acc = float(b[channel_idx]);+ }// Convolve using padded bufferfor (int tap = 0; tap < kernel_size; ++tap) {@@ -117,7 +121,7 @@ template <typename T>void short_conv_decode_kernel(device const T* in_proj [[buffer(0)]],device const T* w [[buffer(1)]],- device const T* b [[buffer(2)]],+ device const T* b [[buffer(2), function_constant(has_bias)]],device const T* state [[buffer(3)]],device T* out [[buffer(4)]],device T* next_state [[buffer(5)]],@@ -138,7 +142,6 @@ void short_conv_decode_kernel(const int tap_count = max(kernel_size - 1, 0);const size_t state_offset = size_t(channel_idx) * state_stride;const device T* w_row = w + size_t(channel_idx) * size_t(kernel_size);- const bool has_bias = b != nullptr;size_t in_proj_idx = size_t(token_idx) * in_proj_stride + size_t(channel_idx);float pre_conv_gate = float(in_proj[in_proj_idx]);@@ -147,7 +150,10 @@ void short_conv_decode_kernel(float x = x_in * pre_conv_gate;- float acc = has_bias ? float(b[channel_idx]) : 0.0f;+ float acc = 0.0f;+ if (has_bias) {+ acc = float(b[channel_idx]);+ }for (int tap = 0; tap < tap_count; ++tap) {float sample = float(state[state_offset + size_t(tap)]);@@ -192,7 +198,7 @@ void short_conv_decode_kernel(device const type* padded [[buffer(0)]], \device const type* in_proj [[buffer(1)]], \device const type* w [[buffer(2)]], \- device const type* b [[buffer(3)]], \+ device const type* b [[buffer(3), function_constant(has_bias)]], \device type* out [[buffer(4)]], \device type* state_out [[buffer(5)]], \constant const size_t& suffix_len [[buffer(6)]], \@@ -208,7 +214,7 @@ void short_conv_decode_kernel(kernel void short_conv_decode_kernel<type>( \device const type* x [[buffer(0)]], \device const type* w [[buffer(1)]], \- device const type* b [[buffer(2)]], \+ device const type* b [[buffer(2), function_constant(has_bias)]], \device const type* state [[buffer(3)]], \device type* out [[buffer(4)]], \device type* next_state [[buffer(5)]], \diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metalindex f0d8c0646..2e1bf340a 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal@@ -4,6 +4,7 @@using namespace metal;constant int activation_type [[function_constant(0)]];+constant bool has_bias [[function_constant(1)]];constant int ACTIVATION_IDENTITY = 0;constant int ACTIVATION_SILU = 1;@@ -75,7 +76,7 @@ template <typename T>void conv1d_scan_kernel(device const T* padded [[buffer(0)]], // (prefix+suffix, channels)device const T* w [[buffer(1)]], // (channels, kernel)- device const T* b [[buffer(2)]], // optional (channels)+ device const T* b [[buffer(2), function_constant(has_bias)]], // optional (channels)device T* x_out [[buffer(3)]], // (suffix, inner_dim)device T* b_out [[buffer(4)]], // (suffix, proj_dim)device T* c_out [[buffer(5)]], // (suffix, proj_dim)@@ -106,10 +107,12 @@ void conv1d_scan_kernel(const size_t state_offset = size_t(channel_idx) * state_stride;const size_t weight_offset = size_t(channel_idx) * size_t(kernel_value);const device T* w_row = w + weight_offset;- const bool has_bias = b != nullptr;if (token_idx < suffix_len) {- float acc = has_bias ? float(b[channel_idx]) : 0.0f;+ float acc = 0.0f;+ if (has_bias) {+ acc = float(b[channel_idx]);+ }for (int tap = 0; tap < kernel_value; ++tap) {const size_t padded_idx = size_t(token_idx) + size_t(tap);float sample = 0.0f;@@ -148,7 +151,7 @@ template <typename T>void conv1d_decode_kernel(device const T* x [[buffer(0)]],device const T* w [[buffer(1)]],- device const T* b [[buffer(2)]],+ device const T* b [[buffer(2), function_constant(has_bias)]],device const T* state [[buffer(3)]],device T* x_out [[buffer(4)]],device T* b_out [[buffer(5)]],@@ -177,9 +180,11 @@ void conv1d_decode_kernel(const size_t x_idx = size_t(token_idx) * row_stride + size_t(channel_idx);const size_t state_offset = size_t(channel_idx) * state_stride;const device T* w_row = w + size_t(channel_idx) * size_t(kernel_value);- const bool has_bias = b != nullptr;- float acc = has_bias ? float(b[channel_idx]) : 0.0f;+ float acc = 0.0f;+ if (has_bias) {+ acc = float(b[channel_idx]);+ }const int state_taps = max(kernel_value - 1, 0);for (int tap = 0; tap < state_taps; ++tap) {const size_t state_index = state_offset + size_t(tap);@@ -238,7 +243,7 @@ void conv1d_decode_kernel(kernel void conv1d_scan_kernel<type>( \device const type* padded [[buffer(0)]], \device const type* w [[buffer(1)]], \- device const type* b [[buffer(2)]], \+ device const type* b [[buffer(2), function_constant(has_bias)]], \device type* x_out [[buffer(3)]], \device type* b_out [[buffer(4)]], \device type* c_out [[buffer(5)]], \@@ -266,7 +271,7 @@ instantiate_conv1d_scan_kernel(half, half);kernel void conv1d_decode_kernel<type>( \device const type* x [[buffer(0)]], \device const type* w [[buffer(1)]], \- device const type* b [[buffer(2)]], \+ device const type* b [[buffer(2), function_constant(has_bias)]], \device const type* state [[buffer(3)]], \device type* x_out [[buffer(4)]], \device type* b_out [[buffer(5)]], \diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rsindex 031af0d15..edb43c7db 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs@@ -28,7 +28,10 @@ fn activation_to_int(activation: &Activation) -> i32 {}}-fn make_function_constants(activation: &Activation) -> FunctionConstantValues {+fn make_function_constants(+ activation: &Activation,+ has_bias: bool,+) -> FunctionConstantValues {let function_constants = FunctionConstantValues::new();let activation_type = activation_to_int(activation);@@ -38,13 +41,21 @@ fn make_function_constants(activation: &Activation) -> FunctionConstantValues {0,);+ function_constants.set_constant_value_at_index(+ &has_bias as *const bool as *const std::ffi::c_void,+ MTLDataType::Bool,+ 1,+ );+function_constants}pub struct Conv1dScanKernel {- pipeline: MTLComputePipelineState,+ pipeline_no_bias: MTLComputePipelineState,+ pipeline_with_bias: MTLComputePipelineState,pack_pipeline: MTLComputePipelineState,- decode_pipeline: MTLComputePipelineState,+ decode_pipeline_no_bias: MTLComputePipelineState,+ decode_pipeline_with_bias: MTLComputePipelineState,}pub struct Conv1dScanArguments<'a> {@@ -103,31 +114,62 @@ impl Conv1dScanKernel {format!("conv1d_pack_prefix_kernel_{}", fn_suffix(data_type));let decode_name =format!("conv1d_decode_kernel_{}", fn_suffix(data_type));- let function_constants = make_function_constants(activation);let activation_id = activation_to_int(activation);- let cache_key = format!("{}_act_{}", fn_name, activation_id);- let pipeline = context- .compute_pipeline_state_cached(- &cache_key,- &fn_name,- Some(&function_constants),- )- .map_err(SSMKernelError::MetalError)?;+ let pipeline_no_bias = {+ let function_constants = make_function_constants(activation, false);+ let cache_key = format!("{}_act_{}_bias_0", fn_name, activation_id);+ context+ .compute_pipeline_state_cached(+ &cache_key,+ &fn_name,+ Some(&function_constants),+ )+ .map_err(SSMKernelError::MetalError)?+ };+ let pipeline_with_bias = {+ let function_constants = make_function_constants(activation, true);+ let cache_key = format!("{}_act_{}_bias_1", fn_name, activation_id);+ context+ .compute_pipeline_state_cached(+ &cache_key,+ &fn_name,+ Some(&function_constants),+ )+ .map_err(SSMKernelError::MetalError)?+ };let pack_pipeline = context.compute_pipeline_state(&pack_name, None).map_err(SSMKernelError::MetalError)?;- let decode_cache_key = format!("{}_act_{}", decode_name, activation_id);- let decode_pipeline = context- .compute_pipeline_state_cached(- &decode_cache_key,- &decode_name,- Some(&function_constants),- )- .map_err(SSMKernelError::MetalError)?;+ let decode_pipeline_no_bias = {+ let function_constants = make_function_constants(activation, false);+ let cache_key =+ format!("{}_act_{}_bias_0", decode_name, activation_id);+ context+ .compute_pipeline_state_cached(+ &cache_key,+ &decode_name,+ Some(&function_constants),+ )+ .map_err(SSMKernelError::MetalError)?+ };+ let decode_pipeline_with_bias = {+ let function_constants = make_function_constants(activation, true);+ let cache_key =+ format!("{}_act_{}_bias_1", decode_name, activation_id);+ context+ .compute_pipeline_state_cached(+ &cache_key,+ &decode_name,+ Some(&function_constants),+ )+ .map_err(SSMKernelError::MetalError)?+ };Ok(Self {- pipeline,+ pipeline_no_bias,+ pipeline_with_bias,pack_pipeline,- decode_pipeline,+ decode_pipeline_no_bias,+ decode_pipeline_with_bias,})}@@ -190,13 +232,17 @@ impl Conv1dScanKernel {return Ok(());}- compute_encoder.set_compute_pipeline_state(&self.decode_pipeline);+ let has_bias = args.b.is_some();+ let pipeline = if has_bias {+ &self.decode_pipeline_with_bias+ } else {+ &self.decode_pipeline_no_bias+ };+ compute_encoder.set_compute_pipeline_state(pipeline);compute_encoder.set_buffer(0, Some(args.x), 0);compute_encoder.set_buffer(1, Some(args.w), 0);- if let Some(bias) = args.b {- compute_encoder.set_buffer(2, Some(bias), 0);- } else {- compute_encoder.set_buffer(2, None, 0);+ if has_bias {+ compute_encoder.set_buffer(2, args.b.map(|v| &**v), 0);}compute_encoder.set_buffer(3, Some(args.state), 0);compute_encoder.set_buffer(4, Some(args.x_out), 0);@@ -263,14 +309,18 @@ impl Conv1dScanKernel {return Ok(());}- compute_encoder.set_compute_pipeline_state(&self.pipeline);+ let has_bias = args.b.is_some();+ let pipeline = if has_bias {+ &self.pipeline_with_bias+ } else {+ &self.pipeline_no_bias+ };+ compute_encoder.set_compute_pipeline_state(pipeline);compute_encoder.set_buffer(0, Some(args.padded), 0);compute_encoder.set_buffer(1, Some(args.w), 0);- if let Some(bias) = args.b {- compute_encoder.set_buffer(2, Some(bias), 0);- } else {- compute_encoder.set_buffer(2, None, 0);+ if has_bias {+ compute_encoder.set_buffer(2, args.b.map(|v| &**v), 0);}compute_encoder.set_buffer(3, Some(args.x_out), 0);compute_encoder.set_buffer(4, Some(args.b_out), 0);diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rsindex 88e4f7b3e..fc3192d7a 100644--- a/crates/uzu/tests/attention_kernel_test.rs+++ b/crates/uzu/tests/attention_kernel_test.rs@@ -683,20 +683,20 @@ fn test_single_pass_attention_basic() {};let is_causal = false; // Non-causal attention for this test- let variant = kernel.choose_variant(seq_len, head_dim, is_causal);+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);println!("Using kernel variant: {:?}", variant);println!("Supports single-pass for head_dim={}: {}",head_dim,- kernel.supports_single_pass(head_dim, is_causal)+ kernel.supports_single_pass(head_dim, is_causal, false));println!("Supports two-pass for head_dim={}: {}",head_dim,- kernel.supports_two_pass(head_dim, is_causal)+ kernel.supports_two_pass(head_dim, is_causal, false));- if !kernel.supports_single_pass(head_dim, false) {+ if !kernel.supports_single_pass(head_dim, false, false) {panic!("Single-pass kernel not supported for head_dim={}", head_dim);}@@ -952,7 +952,7 @@ fn test_single_pass_attention_with_mask() {},};- if !kernel.supports_single_pass(head_dim, false) {+ if !kernel.supports_single_pass(head_dim, false, true) {panic!("Single-pass kernel not supported for head_dim={}", head_dim);}@@ -1033,7 +1033,7 @@ fn test_single_pass_attention_with_sinks() {},};- if !kernel.supports_single_pass(head_dim, false) {+ if !kernel.supports_single_pass(head_dim, false, false) {panic!("Single-pass kernel not supported for head_dim={}", head_dim);}@@ -1111,7 +1111,7 @@ fn test_single_pass_attention_with_sinks_long_sequence() {},};- if !kernel.supports_single_pass(head_dim, false) {+ if !kernel.supports_single_pass(head_dim, false, false) {panic!("Single-pass kernel not supported for head_dim={}", head_dim);}@@ -1189,10 +1189,10 @@ fn test_single_pass_attention_gqa() {};let is_causal = false; // Non-causal attention for this test- let variant = kernel.choose_variant(seq_len, head_dim, is_causal);+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);println!("Using kernel variant for GQA: {:?}", variant);- if !kernel.supports_single_pass(head_dim, false) {+ if !kernel.supports_single_pass(head_dim, false, false) {panic!("Single-pass kernel not supported for head_dim={}", head_dim);}@@ -1339,11 +1339,11 @@ fn test_two_pass_attention() {let scale = 1.0 / (head_dim as f32).sqrt();let is_causal = false; // Non-causal attention for this test- if !kernel.supports_two_pass(head_dim, is_causal) {+ if !kernel.supports_two_pass(head_dim, is_causal, false) {panic!("Two-pass kernel not supported for head_dim={}", head_dim);}- let variant = kernel.choose_variant(seq_len, head_dim, is_causal);+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);if !matches!(variant, AttentionKernelVariant::TwoPass) {panic!("Two-pass not selected for seq_len={}. Got {:?}",@@ -1409,11 +1409,11 @@ fn test_two_pass_attention_gqa() {let scale = 1.0 / (head_dim as f32).sqrt();let is_causal = false; // Non-causal attention for this test- if !kernel.supports_two_pass(head_dim, is_causal) {+ if !kernel.supports_two_pass(head_dim, is_causal, false) {panic!("Two-pass kernel not supported for head_dim={}", head_dim);}- let variant = kernel.choose_variant(seq_len, head_dim, is_causal);+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);if !matches!(variant, AttentionKernelVariant::TwoPass) {panic!("Two-pass not selected for GQA seq_len={}. Got {:?}",@@ -1484,7 +1484,7 @@ fn perf_two_pass_attention() {let scale = 1.0 / (head_dim as f32).sqrt();let is_causal = false;- if !kernel.supports_two_pass(head_dim, is_causal) {+ if !kernel.supports_two_pass(head_dim, is_causal, false) {println!("Skipping two-pass perf test: not supported for head_dim={}",head_dim@@ -1492,7 +1492,7 @@ fn perf_two_pass_attention() {return;}- let variant = kernel.choose_variant(seq_len, head_dim, is_causal);+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);if !matches!(variant, AttentionKernelVariant::TwoPass) {println!("Skipping two-pass perf test: variant {:?} selected instead",diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rsindex abfc37aae..a10c41db3 100644--- a/crates/uzu/tests/kv_cache_state_test.rs+++ b/crates/uzu/tests/kv_cache_state_test.rs@@ -60,8 +60,16 @@ fn make_test_layer(};let shape = [1, total_len.max(1), 1];- let keys = std::cell::RefCell::new(context.array(&shape, DataType::F32));- let values = std::cell::RefCell::new(context.array(&shape, DataType::F32));+ let keys = std::cell::RefCell::new(context.array(+ &shape,+ DataType::F32,+ "kv_cache_keys".to_string(),+ ));+ let values = std::cell::RefCell::new(context.array(+ &shape,+ DataType::F32,+ "kv_cache_values".to_string(),+ ));let prefix_token_positions = match &state {KVCacheLayerState::Full {