Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

added function constants and changed address space

LuckyIYIea165b72c9be
diff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
index b149770e7..b35251ede 100644
--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
@@ -12,11 +12,11 @@ template <typename T>
void updateKVCache(
device T* inPlaceKeys,
device T* inPlaceValues,
- device int2* swaps,
- constant int& swapCount,
- constant int& numHeads,
- constant int& maxSequenceLength,
- constant int& headDim,
+ const constant int2* swaps,
+ const constant int& swapCount,
+ const constant int& numHeads,
+ const constant int& maxSequenceLength,
+ const constant int& headDim,
const uint2 position
) {
for (int i = 0; i < swapCount; ++i) {
@@ -35,11 +35,11 @@ void updateKVCache(
#define outerArguments(T) \
(device T * inPlaceKeys [[buffer(0)]], \
device T * inPlaceValues [[buffer(1)]], \
- device int2 * swaps [[buffer(2)]], \
- constant int& swapCount [[buffer(3)]], \
- constant int& numHeads [[buffer(4)]], \
- constant int& maxSequenceLength [[buffer(5)]], \
- constant int& headDim [[buffer(6)]], \
+ const constant int2 * swaps [[buffer(2)]], \
+ const constant int& swapCount [[buffer(3)]], \
+ const constant int& numHeads [[buffer(4)]], \
+ const constant int& maxSequenceLength [[buffer(5)]], \
+ const constant int& headDim [[buffer(6)]], \
const uint2 position [[thread_position_in_grid]])
#define innerArguments \
diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
index 4ea13a2ba..d2798e147 100644
--- a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
@@ -2,7 +2,8 @@ use std::mem::size_of;
use metal::{
Buffer as MTLBuffer, ComputeCommandEncoderRef,
- ComputePipelineState as MTLComputePipelineState, MTLSize,
+ ComputePipelineState as MTLComputePipelineState, FunctionConstantValues,
+ MTLDataType, MTLSize,
};
use crate::backends::metal::{KernelDataType, MTLContext, MTLError};
@@ -17,10 +18,22 @@ fn fn_suffix(dt: KernelDataType) -> &'static str {
dt.function_name_suffix()
}
+fn make_function_constants(has_bias: bool) -> FunctionConstantValues {
+ let function_constants = FunctionConstantValues::new();
+ function_constants.set_constant_value_at_index(
+ &has_bias as *const bool as *const std::ffi::c_void,
+ MTLDataType::Bool,
+ 0,
+ );
+ function_constants
+}
+
pub struct ShortConvKernel {
pack_pipeline: MTLComputePipelineState,
- prefill_pipeline: MTLComputePipelineState,
- decode_pipeline: MTLComputePipelineState,
+ prefill_pipeline_no_bias: MTLComputePipelineState,
+ prefill_pipeline_with_bias: MTLComputePipelineState,
+ decode_pipeline_no_bias: MTLComputePipelineState,
+ decode_pipeline_with_bias: MTLComputePipelineState,
}
pub struct ShortConvPackArguments<'a> {
@@ -76,17 +89,58 @@ impl ShortConvKernel {
let pack_pipeline = context
.compute_pipeline_state(&pack_name, None)
.map_err(ShortConvKernelError::MetalError)?;
- let prefill_pipeline = context
- .compute_pipeline_state(&prefill_name, None)
- .map_err(ShortConvKernelError::MetalError)?;
- let decode_pipeline = context
- .compute_pipeline_state(&decode_name, None)
- .map_err(ShortConvKernelError::MetalError)?;
+
+ let prefill_pipeline_no_bias = {
+ let function_constants = make_function_constants(false);
+ let cache_key = format!("{}_has_bias_0", prefill_name);
+ context
+ .compute_pipeline_state_cached(
+ &cache_key,
+ &prefill_name,
+ Some(&function_constants),
+ )
+ .map_err(ShortConvKernelError::MetalError)?
+ };
+ let prefill_pipeline_with_bias = {
+ let function_constants = make_function_constants(true);
+ let cache_key = format!("{}_has_bias_1", prefill_name);
+ context
+ .compute_pipeline_state_cached(
+ &cache_key,
+ &prefill_name,
+ Some(&function_constants),
+ )
+ .map_err(ShortConvKernelError::MetalError)?
+ };
+ let decode_pipeline_no_bias = {
+ let function_constants = make_function_constants(false);
+ let cache_key = format!("{}_has_bias_0", decode_name);
+ context
+ .compute_pipeline_state_cached(
+ &cache_key,
+ &decode_name,
+ Some(&function_constants),
+ )
+ .map_err(ShortConvKernelError::MetalError)?
+ };
+ let decode_pipeline_with_bias = {
+ let function_constants = make_function_constants(true);
+ let cache_key = format!("{}_has_bias_1", decode_name);
+ context
+ .compute_pipeline_state_cached(
+ &cache_key,
+ &decode_name,
+ Some(&function_constants),
+ )
+ .map_err(ShortConvKernelError::MetalError)?
+ };
Ok(Self {
pack_pipeline,
- prefill_pipeline,
- decode_pipeline,
+ prefill_pipeline_no_bias,
+ prefill_pipeline_with_bias,
+ decode_pipeline_no_bias,
+ decode_pipeline_with_bias,
})
}
@@ -153,11 +207,19 @@ impl ShortConvKernel {
let tap_count = (args.kernel_size - 1).max(0);
let work_len = args.suffix_len + tap_count as usize;
- compute_encoder.set_compute_pipeline_state(&self.prefill_pipeline);
+ let has_bias = args.b.is_some();
+ let pipeline = if has_bias {
+ &self.prefill_pipeline_with_bias
+ } else {
+ &self.prefill_pipeline_no_bias
+ };
+ compute_encoder.set_compute_pipeline_state(pipeline);
compute_encoder.set_buffer(0, Some(args.padded), 0);
compute_encoder.set_buffer(1, Some(args.in_proj), 0);
compute_encoder.set_buffer(2, Some(args.w), 0);
- compute_encoder.set_buffer(3, args.b.map(|v| &**v), 0);
+ if has_bias {
+ compute_encoder.set_buffer(3, args.b.map(|v| &**v), 0);
+ }
compute_encoder.set_buffer(4, Some(args.out), 0);
compute_encoder.set_buffer(5, Some(args.state_out), 0);
compute_encoder.set_bytes(
@@ -211,10 +273,18 @@ impl ShortConvKernel {
return Ok(());
}
- compute_encoder.set_compute_pipeline_state(&self.decode_pipeline);
+ let has_bias = args.b.is_some();
+ let pipeline = if has_bias {
+ &self.decode_pipeline_with_bias
+ } else {
+ &self.decode_pipeline_no_bias
+ };
+ compute_encoder.set_compute_pipeline_state(pipeline);
compute_encoder.set_buffer(0, Some(args.in_proj), 0);
compute_encoder.set_buffer(1, Some(args.w), 0);
- compute_encoder.set_buffer(2, args.b.map(|v| &**v), 0);
+ if has_bias {
+ compute_encoder.set_buffer(2, args.b.map(|v| &**v), 0);
+ }
compute_encoder.set_buffer(3, Some(args.state), 0);
compute_encoder.set_buffer(4, Some(args.out), 0);
compute_encoder.set_buffer(5, Some(args.next_state), 0);
diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
index 12819218f..d184f9262 100644
--- a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
+++ b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
@@ -3,6 +3,8 @@
using namespace metal;
+constant bool has_bias [[function_constant(0)]];
+
template <typename T>
[[kernel, max_total_threads_per_threadgroup(32)]]
void short_conv_pack_kernel(
@@ -51,7 +53,7 @@ void short_conv_prefill_kernel(
device const T* padded [[buffer(0)]],
device const T* in_proj [[buffer(1)]],
device const T* w [[buffer(2)]],
- device const T* b [[buffer(3)]],
+ device const T* b [[buffer(3), function_constant(has_bias)]],
device T* out [[buffer(4)]],
device T* state_out [[buffer(5)]],
constant const size_t& suffix_len [[buffer(6)]],
@@ -72,11 +74,13 @@ void short_conv_prefill_kernel(
}
const device T* w_row = w + size_t(channel_idx) * size_t(kernel_size);
- const bool has_bias = b != nullptr;
// Threads [0..suffix_len-1]: Compute outputs
if (token_idx < suffix_len) {
- float acc = has_bias ? float(b[channel_idx]) : 0.0f;
+ float acc = 0.0f;
+ if (has_bias) {
+ acc = float(b[channel_idx]);
+ }
// Convolve using padded buffer
for (int tap = 0; tap < kernel_size; ++tap) {
@@ -117,7 +121,7 @@ template <typename T>
void short_conv_decode_kernel(
device const T* in_proj [[buffer(0)]],
device const T* w [[buffer(1)]],
- device const T* b [[buffer(2)]],
+ device const T* b [[buffer(2), function_constant(has_bias)]],
device const T* state [[buffer(3)]],
device T* out [[buffer(4)]],
device T* next_state [[buffer(5)]],
@@ -138,7 +142,6 @@ void short_conv_decode_kernel(
const int tap_count = max(kernel_size - 1, 0);
const size_t state_offset = size_t(channel_idx) * state_stride;
const device T* w_row = w + size_t(channel_idx) * size_t(kernel_size);
- const bool has_bias = b != nullptr;
size_t in_proj_idx = size_t(token_idx) * in_proj_stride + size_t(channel_idx);
float pre_conv_gate = float(in_proj[in_proj_idx]);
@@ -147,7 +150,10 @@ void short_conv_decode_kernel(
float x = x_in * pre_conv_gate;
- float acc = has_bias ? float(b[channel_idx]) : 0.0f;
+ float acc = 0.0f;
+ if (has_bias) {
+ acc = float(b[channel_idx]);
+ }
for (int tap = 0; tap < tap_count; ++tap) {
float sample = float(state[state_offset + size_t(tap)]);
@@ -192,7 +198,7 @@ void short_conv_decode_kernel(
device const type* padded [[buffer(0)]], \
device const type* in_proj [[buffer(1)]], \
device const type* w [[buffer(2)]], \
- device const type* b [[buffer(3)]], \
+ device const type* b [[buffer(3), function_constant(has_bias)]], \
device type* out [[buffer(4)]], \
device type* state_out [[buffer(5)]], \
constant const size_t& suffix_len [[buffer(6)]], \
@@ -208,7 +214,7 @@ void short_conv_decode_kernel(
kernel void short_conv_decode_kernel<type>( \
device const type* x [[buffer(0)]], \
device const type* w [[buffer(1)]], \
- device const type* b [[buffer(2)]], \
+ device const type* b [[buffer(2), function_constant(has_bias)]], \
device const type* state [[buffer(3)]], \
device type* out [[buffer(4)]], \
device type* next_state [[buffer(5)]], \
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal
index f0d8c0646..2e1bf340a 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal
@@ -4,6 +4,7 @@
using namespace metal;
constant int activation_type [[function_constant(0)]];
+constant bool has_bias [[function_constant(1)]];
constant int ACTIVATION_IDENTITY = 0;
constant int ACTIVATION_SILU = 1;
@@ -75,7 +76,7 @@ template <typename T>
void conv1d_scan_kernel(
device const T* padded [[buffer(0)]], // (prefix+suffix, channels)
device const T* w [[buffer(1)]], // (channels, kernel)
- device const T* b [[buffer(2)]], // optional (channels)
+ device const T* b [[buffer(2), function_constant(has_bias)]], // optional (channels)
device T* x_out [[buffer(3)]], // (suffix, inner_dim)
device T* b_out [[buffer(4)]], // (suffix, proj_dim)
device T* c_out [[buffer(5)]], // (suffix, proj_dim)
@@ -106,10 +107,12 @@ void conv1d_scan_kernel(
const size_t state_offset = size_t(channel_idx) * state_stride;
const size_t weight_offset = size_t(channel_idx) * size_t(kernel_value);
const device T* w_row = w + weight_offset;
- const bool has_bias = b != nullptr;
if (token_idx < suffix_len) {
- float acc = has_bias ? float(b[channel_idx]) : 0.0f;
+ float acc = 0.0f;
+ if (has_bias) {
+ acc = float(b[channel_idx]);
+ }
for (int tap = 0; tap < kernel_value; ++tap) {
const size_t padded_idx = size_t(token_idx) + size_t(tap);
float sample = 0.0f;
@@ -148,7 +151,7 @@ template <typename T>
void conv1d_decode_kernel(
device const T* x [[buffer(0)]],
device const T* w [[buffer(1)]],
- device const T* b [[buffer(2)]],
+ device const T* b [[buffer(2), function_constant(has_bias)]],
device const T* state [[buffer(3)]],
device T* x_out [[buffer(4)]],
device T* b_out [[buffer(5)]],
@@ -177,9 +180,11 @@ void conv1d_decode_kernel(
const size_t x_idx = size_t(token_idx) * row_stride + size_t(channel_idx);
const size_t state_offset = size_t(channel_idx) * state_stride;
const device T* w_row = w + size_t(channel_idx) * size_t(kernel_value);
- const bool has_bias = b != nullptr;
- float acc = has_bias ? float(b[channel_idx]) : 0.0f;
+ float acc = 0.0f;
+ if (has_bias) {
+ acc = float(b[channel_idx]);
+ }
const int state_taps = max(kernel_value - 1, 0);
for (int tap = 0; tap < state_taps; ++tap) {
const size_t state_index = state_offset + size_t(tap);
@@ -238,7 +243,7 @@ void conv1d_decode_kernel(
kernel void conv1d_scan_kernel<type>( \
device const type* padded [[buffer(0)]], \
device const type* w [[buffer(1)]], \
- device const type* b [[buffer(2)]], \
+ device const type* b [[buffer(2), function_constant(has_bias)]], \
device type* x_out [[buffer(3)]], \
device type* b_out [[buffer(4)]], \
device type* c_out [[buffer(5)]], \
@@ -266,7 +271,7 @@ instantiate_conv1d_scan_kernel(half, half);
kernel void conv1d_decode_kernel<type>( \
device const type* x [[buffer(0)]], \
device const type* w [[buffer(1)]], \
- device const type* b [[buffer(2)]], \
+ device const type* b [[buffer(2), function_constant(has_bias)]], \
device const type* state [[buffer(3)]], \
device type* x_out [[buffer(4)]], \
device type* b_out [[buffer(5)]], \
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs
index 031af0d15..edb43c7db 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs
+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs
@@ -28,7 +28,10 @@ fn activation_to_int(activation: &Activation) -> i32 {
}
}
-fn make_function_constants(activation: &Activation) -> FunctionConstantValues {
+fn make_function_constants(
+ activation: &Activation,
+ has_bias: bool,
+) -> FunctionConstantValues {
let function_constants = FunctionConstantValues::new();
let activation_type = activation_to_int(activation);
@@ -38,13 +41,21 @@ fn make_function_constants(activation: &Activation) -> FunctionConstantValues {
0,
);
+ function_constants.set_constant_value_at_index(
+ &has_bias as *const bool as *const std::ffi::c_void,
+ MTLDataType::Bool,
+ 1,
+ );
+
function_constants
}
pub struct Conv1dScanKernel {
- pipeline: MTLComputePipelineState,
+ pipeline_no_bias: MTLComputePipelineState,
+ pipeline_with_bias: MTLComputePipelineState,
pack_pipeline: MTLComputePipelineState,
- decode_pipeline: MTLComputePipelineState,
+ decode_pipeline_no_bias: MTLComputePipelineState,
+ decode_pipeline_with_bias: MTLComputePipelineState,
}
pub struct Conv1dScanArguments<'a> {
@@ -103,31 +114,62 @@ impl Conv1dScanKernel {
format!("conv1d_pack_prefix_kernel_{}", fn_suffix(data_type));
let decode_name =
format!("conv1d_decode_kernel_{}", fn_suffix(data_type));
- let function_constants = make_function_constants(activation);
let activation_id = activation_to_int(activation);
- let cache_key = format!("{}_act_{}", fn_name, activation_id);
- let pipeline = context
- .compute_pipeline_state_cached(
- &cache_key,
- &fn_name,
- Some(&function_constants),
- )
- .map_err(SSMKernelError::MetalError)?;
+ let pipeline_no_bias = {
+ let function_constants = make_function_constants(activation, false);
+ let cache_key = format!("{}_act_{}_bias_0", fn_name, activation_id);
+ context
+ .compute_pipeline_state_cached(
+ &cache_key,
+ &fn_name,
+ Some(&function_constants),
+ )
+ .map_err(SSMKernelError::MetalError)?
+ };
+ let pipeline_with_bias = {
+ let function_constants = make_function_constants(activation, true);
+ let cache_key = format!("{}_act_{}_bias_1", fn_name, activation_id);
+ context
+ .compute_pipeline_state_cached(
+ &cache_key,
+ &fn_name,
+ Some(&function_constants),
+ )
+ .map_err(SSMKernelError::MetalError)?
+ };
let pack_pipeline = context
.compute_pipeline_state(&pack_name, None)
.map_err(SSMKernelError::MetalError)?;
- let decode_cache_key = format!("{}_act_{}", decode_name, activation_id);
- let decode_pipeline = context
- .compute_pipeline_state_cached(
- &decode_cache_key,
- &decode_name,
- Some(&function_constants),
- )
- .map_err(SSMKernelError::MetalError)?;
+ let decode_pipeline_no_bias = {
+ let function_constants = make_function_constants(activation, false);
+ let cache_key =
+ format!("{}_act_{}_bias_0", decode_name, activation_id);
+ context
+ .compute_pipeline_state_cached(
+ &cache_key,
+ &decode_name,
+ Some(&function_constants),
+ )
+ .map_err(SSMKernelError::MetalError)?
+ };
+ let decode_pipeline_with_bias = {
+ let function_constants = make_function_constants(activation, true);
+ let cache_key =
+ format!("{}_act_{}_bias_1", decode_name, activation_id);
+ context
+ .compute_pipeline_state_cached(
+ &cache_key,
+ &decode_name,
+ Some(&function_constants),
+ )
+ .map_err(SSMKernelError::MetalError)?
+ };
Ok(Self {
- pipeline,
+ pipeline_no_bias,
+ pipeline_with_bias,
pack_pipeline,
- decode_pipeline,
+ decode_pipeline_no_bias,
+ decode_pipeline_with_bias,
})
}
@@ -190,13 +232,17 @@ impl Conv1dScanKernel {
return Ok(());
}
- compute_encoder.set_compute_pipeline_state(&self.decode_pipeline);
+ let has_bias = args.b.is_some();
+ let pipeline = if has_bias {
+ &self.decode_pipeline_with_bias
+ } else {
+ &self.decode_pipeline_no_bias
+ };
+ compute_encoder.set_compute_pipeline_state(pipeline);
compute_encoder.set_buffer(0, Some(args.x), 0);
compute_encoder.set_buffer(1, Some(args.w), 0);
- if let Some(bias) = args.b {
- compute_encoder.set_buffer(2, Some(bias), 0);
- } else {
- compute_encoder.set_buffer(2, None, 0);
+ if has_bias {
+ compute_encoder.set_buffer(2, args.b.map(|v| &**v), 0);
}
compute_encoder.set_buffer(3, Some(args.state), 0);
compute_encoder.set_buffer(4, Some(args.x_out), 0);
@@ -263,14 +309,18 @@ impl Conv1dScanKernel {
return Ok(());
}
- compute_encoder.set_compute_pipeline_state(&self.pipeline);
+ let has_bias = args.b.is_some();
+ let pipeline = if has_bias {
+ &self.pipeline_with_bias
+ } else {
+ &self.pipeline_no_bias
+ };
+ compute_encoder.set_compute_pipeline_state(pipeline);
compute_encoder.set_buffer(0, Some(args.padded), 0);
compute_encoder.set_buffer(1, Some(args.w), 0);
- if let Some(bias) = args.b {
- compute_encoder.set_buffer(2, Some(bias), 0);
- } else {
- compute_encoder.set_buffer(2, None, 0);
+ if has_bias {
+ compute_encoder.set_buffer(2, args.b.map(|v| &**v), 0);
}
compute_encoder.set_buffer(3, Some(args.x_out), 0);
compute_encoder.set_buffer(4, Some(args.b_out), 0);
diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rs
index 88e4f7b3e..fc3192d7a 100644
--- a/crates/uzu/tests/attention_kernel_test.rs
+++ b/crates/uzu/tests/attention_kernel_test.rs
@@ -683,20 +683,20 @@ fn test_single_pass_attention_basic() {
};
let is_causal = false; // Non-causal attention for this test
- let variant = kernel.choose_variant(seq_len, head_dim, is_causal);
+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);
println!("Using kernel variant: {:?}", variant);
println!(
"Supports single-pass for head_dim={}: {}",
head_dim,
- kernel.supports_single_pass(head_dim, is_causal)
+ kernel.supports_single_pass(head_dim, is_causal, false)
);
println!(
"Supports two-pass for head_dim={}: {}",
head_dim,
- kernel.supports_two_pass(head_dim, is_causal)
+ kernel.supports_two_pass(head_dim, is_causal, false)
);
- if !kernel.supports_single_pass(head_dim, false) {
+ if !kernel.supports_single_pass(head_dim, false, false) {
panic!("Single-pass kernel not supported for head_dim={}", head_dim);
}
@@ -952,7 +952,7 @@ fn test_single_pass_attention_with_mask() {
},
};
- if !kernel.supports_single_pass(head_dim, false) {
+ if !kernel.supports_single_pass(head_dim, false, true) {
panic!("Single-pass kernel not supported for head_dim={}", head_dim);
}
@@ -1033,7 +1033,7 @@ fn test_single_pass_attention_with_sinks() {
},
};
- if !kernel.supports_single_pass(head_dim, false) {
+ if !kernel.supports_single_pass(head_dim, false, false) {
panic!("Single-pass kernel not supported for head_dim={}", head_dim);
}
@@ -1111,7 +1111,7 @@ fn test_single_pass_attention_with_sinks_long_sequence() {
},
};
- if !kernel.supports_single_pass(head_dim, false) {
+ if !kernel.supports_single_pass(head_dim, false, false) {
panic!("Single-pass kernel not supported for head_dim={}", head_dim);
}
@@ -1189,10 +1189,10 @@ fn test_single_pass_attention_gqa() {
};
let is_causal = false; // Non-causal attention for this test
- let variant = kernel.choose_variant(seq_len, head_dim, is_causal);
+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);
println!("Using kernel variant for GQA: {:?}", variant);
- if !kernel.supports_single_pass(head_dim, false) {
+ if !kernel.supports_single_pass(head_dim, false, false) {
panic!("Single-pass kernel not supported for head_dim={}", head_dim);
}
@@ -1339,11 +1339,11 @@ fn test_two_pass_attention() {
let scale = 1.0 / (head_dim as f32).sqrt();
let is_causal = false; // Non-causal attention for this test
- if !kernel.supports_two_pass(head_dim, is_causal) {
+ if !kernel.supports_two_pass(head_dim, is_causal, false) {
panic!("Two-pass kernel not supported for head_dim={}", head_dim);
}
- let variant = kernel.choose_variant(seq_len, head_dim, is_causal);
+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);
if !matches!(variant, AttentionKernelVariant::TwoPass) {
panic!(
"Two-pass not selected for seq_len={}. Got {:?}",
@@ -1409,11 +1409,11 @@ fn test_two_pass_attention_gqa() {
let scale = 1.0 / (head_dim as f32).sqrt();
let is_causal = false; // Non-causal attention for this test
- if !kernel.supports_two_pass(head_dim, is_causal) {
+ if !kernel.supports_two_pass(head_dim, is_causal, false) {
panic!("Two-pass kernel not supported for head_dim={}", head_dim);
}
- let variant = kernel.choose_variant(seq_len, head_dim, is_causal);
+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);
if !matches!(variant, AttentionKernelVariant::TwoPass) {
panic!(
"Two-pass not selected for GQA seq_len={}. Got {:?}",
@@ -1484,7 +1484,7 @@ fn perf_two_pass_attention() {
let scale = 1.0 / (head_dim as f32).sqrt();
let is_causal = false;
- if !kernel.supports_two_pass(head_dim, is_causal) {
+ if !kernel.supports_two_pass(head_dim, is_causal, false) {
println!(
"Skipping two-pass perf test: not supported for head_dim={}",
head_dim
@@ -1492,7 +1492,7 @@ fn perf_two_pass_attention() {
return;
}
- let variant = kernel.choose_variant(seq_len, head_dim, is_causal);
+ let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);
if !matches!(variant, AttentionKernelVariant::TwoPass) {
println!(
"Skipping two-pass perf test: variant {:?} selected instead",
diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rs
index abfc37aae..a10c41db3 100644
--- a/crates/uzu/tests/kv_cache_state_test.rs
+++ b/crates/uzu/tests/kv_cache_state_test.rs
@@ -60,8 +60,16 @@ fn make_test_layer(
};
let shape = [1, total_len.max(1), 1];
- let keys = std::cell::RefCell::new(context.array(&shape, DataType::F32));
- let values = std::cell::RefCell::new(context.array(&shape, DataType::F32));
+ let keys = std::cell::RefCell::new(context.array(
+ &shape,
+ DataType::F32,
+ "kv_cache_keys".to_string(),
+ ));
+ let values = std::cell::RefCell::new(context.array(
+ &shape,
+ DataType::F32,
+ "kv_cache_values".to_string(),
+ ));
let prefix_token_positions = match &state {
KVCacheLayerState::Full {