Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

speed-up-attention

Eugene Bokhand6cba68ee4d1
diff --git a/crates/uzu/build.rs b/crates/uzu/build.rs
index 67a591726..ad44b1284 100644
--- a/crates/uzu/build.rs
+++ b/crates/uzu/build.rs
@@ -53,6 +53,36 @@ fn generate_metal_bindings() {
bindings
.write_to_file(&bindings_path)
.expect("Failed to write shared_types.rs");
+
+ // ---------------------------------------------------------------------
+ // Gemm attention shared types
+ // ---------------------------------------------------------------------
+ let attention_dir =
+ manifest_dir.join("src/backends/metal/kernel/attention");
+ let attn_types_header = attention_dir.join("gemm_types.h");
+
+ println!("cargo:rerun-if-changed={}", attn_types_header.display());
+
+ let attn_bindings = bindgen::Builder::default()
+ .header(attn_types_header.to_string_lossy())
+ // Generate for C mode (no Metal-specific stuff)
+ .clang_arg("-x")
+ .clang_arg("c")
+ // Derive common traits
+ .derive_default(true)
+ .derive_copy(true)
+ // Only generate types we care about
+ .allowlist_type("AttnParams")
+ .allowlist_type("AttnMaskParams")
+ // Use core types
+ .use_core()
+ .generate()
+ .expect("Failed to generate bindings from gemm_types.h");
+
+ let attn_bindings_path = attention_dir.join("gemm_types.rs");
+ attn_bindings
+ .write_to_file(&attn_bindings_path)
+ .expect("Failed to write gemm_types.rs");
}
fn compile_metal_shaders() {
diff --git a/crates/uzu/src/backends/metal/encodable_block/attention.rs b/crates/uzu/src/backends/metal/encodable_block/attention.rs
index 5bf1e1273..8fe9dcec5 100644
--- a/crates/uzu/src/backends/metal/encodable_block/attention.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/attention.rs
@@ -7,12 +7,28 @@ use crate::backends::metal::{
KernelDataType, MTLContext,
forward_pass::{ArrayId, ForwardPassState, HashMapId},
kernel::attention::{
- AttentionError, AttentionKernel, AttentionKernelVariant,
- AttentionSinglePassArguments, AttentionTwoPassArguments,
- KVCacheUpdateArguments,
+ AttentionError, AttentionGemmArguments, AttentionKernel,
+ AttentionKernelVariant, AttentionSinglePassArguments,
+ AttentionTwoPassArguments, KVCacheUpdateArguments,
},
};
+fn env_gemm_attention_enabled() -> bool {
+ static VALUE: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
+ *VALUE.get_or_init(|| {
+ let raw = std::env::var("UZU_USE_GEMM_ATTENTION").ok();
+ let Some(raw) = raw else {
+ return true;
+ };
+ let v = raw.trim().to_ascii_uppercase();
+ match v.as_str() {
+ "1" | "YES" | "TRUE" | "ON" => true,
+ "0" | "NO" | "FALSE" | "OFF" => false,
+ _ => true,
+ }
+ })
+}
+
pub struct Attention {
kernel: AttentionKernel,
layer_index: usize,
@@ -134,11 +150,40 @@ impl EncodableBlock for Attention {
let scale =
self.attention_scale.unwrap_or(1.0f32 / (head_dim as f32).sqrt());
- let variant = self.kernel.choose_variant(
- sequence_length,
- head_dim,
- self.is_causal,
- );
+ let gemm_enabled = env_gemm_attention_enabled();
+ let use_gemm = gemm_enabled
+ && suffix_length > 8
+ && matches!(head_dim, 64 | 128 | 256);
+ if !gemm_enabled {
+ static PRINT_ONCE: std::sync::Once = std::sync::Once::new();
+ PRINT_ONCE.call_once(|| {
+ eprintln!(
+ "[uzu] Gemm attention disabled via UZU_USE_GEMM_ATTENTION"
+ );
+ });
+ }
+ if use_gemm {
+ static PRINT_ONCE: std::sync::Once = std::sync::Once::new();
+ PRINT_ONCE.call_once(|| {
+ eprintln!(
+ "[uzu] Using Gemm attention (suffix_len={}, seq_len={}, head_dim={}, gqa_factor={}, causal={})",
+ suffix_length,
+ sequence_length,
+ head_dim,
+ gqa_factor,
+ self.is_causal
+ );
+ });
+ }
+ let variant = if use_gemm {
+ AttentionKernelVariant::SinglePass
+ } else {
+ self.kernel.choose_variant(
+ sequence_length,
+ head_dim,
+ self.is_causal,
+ )
+ };
let rotated_queries_binding = state.arrays(&[ArrayId::RotatedQueries]);
let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);
@@ -279,7 +324,32 @@ impl EncodableBlock for Attention {
match variant {
AttentionKernelVariant::SinglePass => {
- if let Err(e) = self.kernel.encode_single_pass(
+ if use_gemm {
+ let mtl = &**state.mtl_context();
+ if let Err(e) = self.kernel.encode_gemm(
+ mtl,
+ compute_encoder,
+ AttentionGemmArguments {
+ queries_buffer: &queries_buffer,
+ keys_buffer: &key_cache_buffer,
+ values_buffer: &value_cache_buffer,
+ output_buffer: &attention_output_buffer,
+ mask_buffer: Some(&attention_bias_buffer),
+ sinks_buffer: sinks_buffer.as_ref(),
+ num_heads,
+ num_groups,
+ suffix_length,
+ sequence_length,
+ segment_prefix_length,
+ max_sequence_length,
+ head_dim,
+ is_causal: self.is_causal,
+ scale,
+ },
+ ) {
+ eprintln!("Failed to encode gemm attention: {:?}", e);
+ }
+ } else if let Err(e) = self.kernel.encode_single_pass(
compute_encoder,
AttentionSinglePassArguments {
queries_buffer: &queries_buffer,
diff --git a/crates/uzu/src/backends/metal/encodable_block/decoder.rs b/crates/uzu/src/backends/metal/encodable_block/decoder.rs
index cb9a3398a..349f0713b 100644
--- a/crates/uzu/src/backends/metal/encodable_block/decoder.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/decoder.rs
@@ -155,6 +155,7 @@ impl Decoder {
ArrayId::Main,
&decoder_weight_loader.subtree("output_norm").unwrap(),
)
+ .map(RMSNorm::with_sampling_range)
.expect("Failed to create output RMS norm kernel"),
);
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
index 0b0ce7f34..12833c077 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
@@ -111,15 +111,22 @@ impl EncodableBlock for FullPrecisionEmbeddingReadout {
_parameters: &EncodingParameters,
) {
let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
- let batch_size = state.active_suffix_length();
+ let batch_size = state.sampling_length();
+ if batch_size == 0 {
+ return;
+ }
+ let sampling_start = state.sampling_start();
let mut input_array_mut = arrays[0].borrow_mut();
let mut output_array_mut = arrays[1].borrow_mut();
+ let elem_size = input_array_mut.data_type().size_in_bytes();
let input_buffer = unsafe { input_array_mut.mtl_buffer() };
let output_buffer = unsafe { output_array_mut.mtl_buffer() };
+ let a_offset = (sampling_start * self.model_dim * elem_size) as u64;
let args = MatmulArguments {
a: input_buffer,
+ a_offset,
b: &self.weights_buffer,
c: None,
d: output_buffer,
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
index b96674207..09b5ed693 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
@@ -240,15 +240,22 @@ impl EncodableBlock for QuantizedEmbeddingReadout {
_parameters: &EncodingParameters,
) {
let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
- let batch_size = state.active_suffix_length();
+ let batch_size = state.sampling_length();
+ if batch_size == 0 {
+ return;
+ }
+ let sampling_start = state.sampling_start();
let mut input_array_mut = arrays[0].borrow_mut();
let mut output_array_mut = arrays[1].borrow_mut();
+ let elem_size = input_array_mut.data_type().size_in_bytes();
let input_buffer = unsafe { input_array_mut.mtl_buffer() };
let output_buffer = unsafe { output_array_mut.mtl_buffer() };
+ let a_offset = (sampling_start * self.model_dim * elem_size) as u64;
let args = QuantizedMatmulArguments {
a_buffer: input_buffer,
+ a_offset,
b_buffer: &self.weights_buffer,
scales_buffer: &self.scales_buffer,
zero_points_or_biases_buffer: &self.biases_buffer,
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
index a39340dd7..5d77b2ba3 100644
--- a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
@@ -122,6 +122,7 @@ impl EncodableBlock for FullPrecisionLinear {
let args = MatmulArguments {
a: input_buffer,
+ a_offset: 0,
b: &self.weights_buffer,
c: None,
d: output_buffer,
@@ -176,6 +177,7 @@ impl EncodableBlock for FullPrecisionLinear {
let args = MatmulArguments {
a: input_buffer,
+ a_offset: 0,
b: &self.weights_buffer,
c: None,
d: output_buffer,
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs
index 1027bab77..44c65b358 100644
--- a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs
@@ -241,6 +241,7 @@ impl EncodableBlock for QuantizedLinear {
let args = QuantizedMatmulArguments {
a_buffer: input_buffer,
+ a_offset: 0,
b_buffer: &self.weights_buffer,
scales_buffer: &self.scales_buffer,
zero_points_or_biases_buffer: &self.zero_points_or_biases_buffer,
@@ -299,6 +300,7 @@ impl EncodableBlock for QuantizedLinear {
let args = QuantizedMatmulArguments {
a_buffer: input_buffer,
+ a_offset: 0,
b_buffer: &self.weights_buffer,
scales_buffer: &self.scales_buffer,
zero_points_or_biases_buffer: &self.zero_points_or_biases_buffer,
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs
index f7f73f22e..e24c95130 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs
@@ -24,6 +24,7 @@ pub struct RMSNorm {
input_array_id: ArrayId,
output_array_id: ArrayId,
scales_buffer: MTLBuffer,
+ use_sampling_range: bool,
}
impl RMSNorm {
@@ -84,8 +85,16 @@ impl RMSNorm {
input_array_id,
output_array_id,
scales_buffer,
+ use_sampling_range: false,
})
}
+
+ /// When enabled, this RMSNorm only runs on `state.sampling_start()..+state.sampling_length()`.
+ /// This is useful for the final output norm before readout/sampling in prefill.
+ pub fn with_sampling_range(mut self) -> Self {
+ self.use_sampling_range = true;
+ self
+ }
}
impl EncodableBlock for RMSNorm {
@@ -126,18 +135,39 @@ impl EncodableBlock for RMSNorm {
let mut input_array = input_binding[0].borrow_mut();
let mut output_array = output_binding[0].borrow_mut();
+ let suffix_length = input_shape[0];
+ let input_elem_size = input_array.data_type().size_in_bytes();
+ let output_elem_size = output_array.data_type().size_in_bytes();
let input_buffer = unsafe { input_array.mtl_buffer() };
let output_buffer = unsafe { output_array.mtl_buffer() };
- let batch_size = input_shape[0] as i32;
+ let (batch_start, batch_len) = if self.use_sampling_range {
+ (state.sampling_start(), state.sampling_length())
+ } else {
+ (0, state.active_suffix_length())
+ };
+ let batch_len = batch_len.min(suffix_length.saturating_sub(batch_start));
+ if batch_len == 0 {
+ return;
+ }
+
+ let row_size_in_bytes = input_shape[1] * input_elem_size;
+ let input_offset = (batch_start * row_size_in_bytes) as u64;
+
+ let output_row_size_in_bytes = input_shape[1] * output_elem_size;
+ let output_offset = (batch_start * output_row_size_in_bytes) as u64;
+
+ let batch_size = batch_len as i32;
let model_dim = input_shape[1] as i32;
if let Err(e) = self.kernel.encode(
compute_encoder,
RMSNormArguments {
input_buffer: &input_buffer,
+ input_offset,
scales_buffer: &self.scales_buffer,
output_buffer: &output_buffer,
+ output_offset,
batch_size,
model_dim,
epsilon: self.config.epsilon,
diff --git a/crates/uzu/src/backends/metal/encodable_block/sampling.rs b/crates/uzu/src/backends/metal/encodable_block/sampling.rs
index 952efc38c..26c4b023f 100644
--- a/crates/uzu/src/backends/metal/encodable_block/sampling.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/sampling.rs
@@ -92,7 +92,11 @@ impl EncodableBlock for Sampling {
use crate::Array;
logits.shape()
};
- let batch_size = state.active_suffix_length();
+ let batch_size = state.sampling_length();
+ if batch_size == 0 {
+ return;
+ }
+ let sampling_start = state.sampling_start();
let vocab_size = logits_shape[1];
let seeds_binding = state.arrays(&[ArrayId::TokenSeeds]);
@@ -102,16 +106,28 @@ impl EncodableBlock for Sampling {
state.sampling_output().unwrap().borrow_mut();
let sampling_method = state.sampling_method().unwrap();
- let seeds_offset = seeds.buffer_offset();
-
- let bitmask_buffer = state
- .token_bitmask()
- .map(|cell| unsafe { cell.borrow_mut().mtl_buffer().clone() });
+ let seeds_offset =
+ seeds.buffer_offset() + sampling_start * std::mem::size_of::<u64>();
+
+ let (bitmask_buffer, bitmask_offset) =
+ state.token_bitmask().map_or((None, 0usize), |cell| {
+ let mut bitmask = cell.borrow_mut();
+ let bitmask_row_len = {
+ use crate::Array;
+ bitmask.shape()[1]
+ };
+ let bitmask_offset = bitmask.buffer_offset()
+ + sampling_start
+ * bitmask_row_len
+ * std::mem::size_of::<u32>();
+ (Some(unsafe { bitmask.mtl_buffer().clone() }), bitmask_offset)
+ });
if let Err(e) = self.kernel.encode_with_encoder(
unsafe { &logits.mtl_buffer() },
unsafe { Some(&seeds.mtl_buffer()) },
seeds_offset,
bitmask_buffer.as_ref(),
+ bitmask_offset,
unsafe { &output_buffer_ref.mtl_buffer() },
sampling_method,
batch_size,
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
index 4c8ac5964..7888e5452 100644
--- a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
@@ -106,6 +106,8 @@ impl ForwardPassState {
token_bitmask: Option<&[u32]>,
token_seeds: &[u64],
active_suffix_length: usize,
+ sampling_start: usize,
+ sampling_length: usize,
is_prefilling: bool,
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
skip_token_ids_copy: bool,
@@ -254,6 +256,8 @@ impl ForwardPassState {
#[cfg(feature = "tracing")]
traces,
active_suffix_length,
+ sampling_start,
+ sampling_length,
is_prefilling,
},
);
@@ -951,6 +955,26 @@ impl ForwardPassState {
}
}
+ /// Start index (within the suffix batch) for which we need logits/sampling.
+ pub fn sampling_start(&self) -> usize {
+ match &self.mode {
+ ForwardPassMode::LanguageModelGenerator(state) => {
+ state.sampling_start
+ },
+ ForwardPassMode::Classifier(_) => 0,
+ }
+ }
+
+ /// Number of batch items for which we need logits/sampling.
+ pub fn sampling_length(&self) -> usize {
+ match &self.mode {
+ ForwardPassMode::LanguageModelGenerator(state) => {
+ state.sampling_length
+ },
+ ForwardPassMode::Classifier(_) => self.common_aux.suffix_length,
+ }
+ }
+
pub fn is_prefilling(&self) -> bool {
match &self.mode {
ForwardPassMode::LanguageModelGenerator(state) => {
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mode.rs b/crates/uzu/src/backends/metal/forward_pass/state/mode.rs
index f5d9bbcb5..1826f1189 100644
--- a/crates/uzu/src/backends/metal/forward_pass/state/mode.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/state/mode.rs
@@ -21,6 +21,8 @@ pub struct LanguageModelGeneratorModeState {
#[cfg(feature = "tracing")]
pub traces: Rc<RefCell<ActivationTrace>>,
pub active_suffix_length: usize,
+ pub sampling_start: usize,
+ pub sampling_length: usize,
pub is_prefilling: bool,
}
diff --git a/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal b/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal
new file mode 100644
index 000000000..2190ff680
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal
@@ -0,0 +1,486 @@
+#include <metal_stdlib>
+
+#include "../matmul/loader.h"
+#include "../matmul/mma.h"
+
+#include "gemm_types.h"
+
+using namespace metal;
+using namespace uzu::matmul;
+using namespace uzu::attention;
+
+///////////////////////////////////////////////////////////////////////////////
+// Function constants for compile-time specialization
+///////////////////////////////////////////////////////////////////////////////
+
+// Alignment flags (match Gemm convention / our matmul indices)
+constant bool align_Q [[function_constant(200)]];
+constant bool align_K [[function_constant(201)]];
+
+// Feature flags
+constant bool has_mask [[function_constant(300)]];
+constant bool do_causal [[function_constant(301)]];
+constant bool has_sinks [[function_constant(302)]];
+
+///////////////////////////////////////////////////////////////////////////////
+// Helpers
+///////////////////////////////////////////////////////////////////////////////
+
+template <typename T>
+struct TransformScale {
+ T scale;
+ METAL_FUNC TransformScale(T scale_) : scale(scale_) {}
+
+ METAL_FUNC T apply(T x) const { return scale * x; }
+};
+
+template <typename T>
+METAL_FUNC T row_reduce_max(T v) {
+ // BaseMMAFrag::get_coord mapping groups lanes for a row as:
+ // {lane, lane^1, lane^8, (lane^1)^8}. Reduce in two steps.
+ v = metal::max(v, simd_shuffle_xor(v, 1));
+ v = metal::max(v, simd_shuffle_xor(v, 8));
+ return v;
+}
+
+template <typename T>
+METAL_FUNC T row_reduce_sum(T v) {
+ v += simd_shuffle_xor(v, 1);
+ v += simd_shuffle_xor(v, 8);
+ return v;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Gemm attention kernel
+///////////////////////////////////////////////////////////////////////////////
+
+// clang-format off
+template <
+ typename T,
+ int BQ,
+ int BK,
+ int BD,
+ int WM,
+ int WN,
+ typename AccumType = float>
+[[kernel, max_total_threads_per_threadgroup(WM * WN * 32)]] void attention_gemm(
+ const device T* Q [[buffer(0)]],
+ const device T* K [[buffer(1)]],
+ const device T* V [[buffer(2)]],
+ device T* O [[buffer(3)]],
+ const constant AttnParams* params [[buffer(4)]],
+ const constant AttnMaskParams* mask_params [[buffer(5), function_constant(has_mask)]],
+ const device T* mask [[buffer(6), function_constant(has_mask)]],
+ const device float* sinks [[buffer(7), function_constant(has_sinks)]],
+ uint simd_lane_id [[thread_index_in_simdgroup]],
+ uint simd_group_id [[simdgroup_index_in_threadgroup]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint3 lid [[thread_position_in_threadgroup]]) { // clang-format on
+
+ // Pacifying compiler
+ (void)lid;
+
+ // -------------------------------------------------------------------------
+ // Pointer setup (all strides are in elements)
+ // tid.x: query tile index (BQ rows)
+ // tid.y: query head index
+ // tid.z: batch index (currently 1 in uzu, but kept for completeness)
+ const int64_t batch_idx = int64_t(tid.z);
+ const int64_t head_idx = int64_t(tid.y);
+ const int64_t q_tile_idx = int64_t(tid.x);
+
+ Q += batch_idx * params->q_strides[0] + head_idx * params->q_strides[1] +
+ q_tile_idx * int64_t(BQ) * params->q_strides[2];
+
+ const int kv_head_idx = int(tid.y) / params->gqa_factor;
+ K += batch_idx * params->k_strides[0] +
+ int64_t(kv_head_idx) * params->k_strides[1];
+ V += batch_idx * params->v_strides[0] +
+ int64_t(kv_head_idx) * params->v_strides[1];
+
+ O += batch_idx * params->o_strides[0] + head_idx * params->o_strides[1] +
+ q_tile_idx * int64_t(BQ) * params->o_strides[2];
+
+ if (has_mask) {
+ mask += batch_idx * mask_params->m_strides[0] +
+ head_idx * mask_params->m_strides[1];
+ }
+
+ // -------------------------------------------------------------------------
+ // Threadgroup memory
+ constexpr short padQ = 16 / sizeof(T);
+ constexpr short padK = 16 / sizeof(T);
+ constexpr short padV = 16 / sizeof(T);
+
+ constexpr short LDQ_tgp = BD + padQ;
+ constexpr short LDK_tgp = BD + padK; // K stored as [BK, BD] row-major
+ constexpr short LDV_tgp = BD + padV;
+
+ constexpr short tgp_mem_k = BK * (BD + padK);
+ constexpr short tgp_mem_v = BK * (BD + padV);
+ constexpr short tgp_mem_s = tgp_mem_k > tgp_mem_v ? tgp_mem_k : tgp_mem_v;
+
+ threadgroup T Q_smem[BQ * (BD + padQ)];
+ threadgroup T KV_smem[tgp_mem_s];
+
+ threadgroup T* Qs = Q_smem;
+ threadgroup T* Ks = KV_smem;
+ threadgroup T* Vs = KV_smem;
+
+ // -------------------------------------------------------------------------
+ // Block loaders
+ using QBlockLoader = BlockLoader<
+ T,
+ /*BROWS=*/BQ,
+ /*BCOLS=*/BD,
+ /*dst_ld=*/LDQ_tgp,
+ /*reduction_dim=*/1,
+ /*tgp_size=*/WM * WN * 32>;
+
+ using KBlockLoader = BlockLoader<
+ T,
+ /*BROWS=*/BK,
+ /*BCOLS=*/BD,
+ /*dst_ld=*/LDK_tgp,
+ /*reduction_dim=*/0,
+ /*tgp_size=*/WM * WN * 32>;
+
+ using VBlockLoader = BlockLoader<
+ T,
+ /*BROWS=*/BK,
+ /*BCOLS=*/BD,
+ /*dst_ld=*/LDV_tgp,
+ /*reduction_dim=*/0,
+ /*tgp_size=*/WM * WN * 32>;
+
+ const int q_src_ld = int(params->q_strides[2]);
+ const int k_src_ld = int(params->k_strides[2]);
+ const int v_src_ld = int(params->v_strides[2]);
+
+ thread QBlockLoader loader_q(Q, q_src_ld, Qs, simd_group_id, simd_lane_id);
+ thread KBlockLoader loader_k(K, k_src_ld, Ks, simd_group_id, simd_lane_id);
+ thread VBlockLoader loader_v(V, v_src_ld, Vs, simd_group_id, simd_lane_id);
+
+ TransformScale<T> ts(static_cast<T>(params->scale * M_LOG2E_F));
+
+ // -------------------------------------------------------------------------
+ // MMA tiles
+ constexpr short kFragSize = 8;
+ using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;
+
+ constexpr int kNWarps = WM * WN;
+ static_assert(
+ BQ >= (kNWarps * kFragSize) && BQ % (kNWarps * kFragSize) == 0,
+ "Each simdgroup must host at least 1 simdgroup matrix along Q sequence.");
+
+ // Q seq frags per warp (we keep TQ == 1 for the 32-row block layout)
+ constexpr int TQ = BQ / (kNWarps * kFragSize);
+ // KV sequence frags
+ constexpr int TK = BK / kFragSize;
+ // HeadDim frags
+ constexpr int TD = BD / kFragSize;
+
+ static_assert(TQ == 1, "Expected TQ == 1");
+
+ MMATile<AccumType, TQ, 1, MMAFrag_acc_t> Qtile;
+ MMATile<AccumType, 1, TK, MMAFrag_acc_t> Ktile; // represents K^T slice
+ MMATile<AccumType, TQ, TK, MMAFrag_acc_t> Stile;
+ MMATile<AccumType, 1, 1, MMAFrag_acc_t> Vtile;
+ MMATile<AccumType, TQ, TD, MMAFrag_acc_t> Otile;
+
+ Otile.clear();
+
+ // -------------------------------------------------------------------------
+ // Lane coordinates and pointer offsets
+ const short2 simd_coord = MMAFrag_acc_t::get_coord(simd_lane_id);
+ const short sm = simd_coord.y;
+ const short sn = simd_coord.x;
+
+ const short tm = kFragSize * TQ * short(simd_group_id);
+
+ // Qs is row-major [BQ, BD]
+ const short Qs_offset = (tm + sm) * LDQ_tgp + sn;
+ constexpr short Qs_tile_stride = kFragSize;
+
+ // Ks is row-major [BK, BD] but we load K^T by swapping strides:
+ // B_str_k = 1, B_str_n = LDK_tgp => offset = sn * LDK_tgp + sm
+ const short Ks_offset = sn * LDK_tgp + sm;
+ constexpr short Ks_tile_stride = kFragSize; // advance along head-dim (contiguous)
+
+ // Vs is row-major [BK, BD]
+ const short Vs_offset = sm * LDV_tgp + sn;
+
+ // -------------------------------------------------------------------------
+ // Load Q block once (and apply scaling)
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (!align_Q && int(tid.x) == params->nq_aligned) {
+ loader_q.load_safe(short2(BD, params->q_rem));
+ } else {
+ loader_q.load_unsafe();
+ }
+ loader_q.apply_inplace_op(ts);
+
+ // -------------------------------------------------------------------------
+ // Streaming softmax state for this row (shared across lanes in a row)
+ const AccumType neg_inf = static_cast<AccumType>(-1e9f) * M_LOG2E_F;
+ AccumType max_score = -INFINITY;
+ AccumType sum_score = AccumType(0);
+
+ if (has_sinks) {
+ max_score = M_LOG2E_F * static_cast<AccumType>(sinks[tid.y]);
+ sum_score = AccumType(1);
+ }
+
+ // Determine K block loop limit (causal can early-stop)
+ int kb_lim = params->nk;
+ if (do_causal) {
+ const int q_max = (int(tid.x) + 1) * BQ + params->q_off;
+ kb_lim = (q_max + BK - 1) / BK;
+ kb_lim = min(params->nk, kb_lim);
+ }
+
+ const int q_rel = int(tid.x) * BQ + int(tm) + int(sm); // [0, q_len)
+ const int q_abs = q_rel + params->q_off; // [0, k_len)
+
+ // Loop over KV blocks
+ for (int kb = 0; kb < kb_lim; kb++) {
+ // Load K block
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (!align_K && kb == params->nk_aligned) {
+ loader_k.load_safe(short2(BD, params->k_rem));
+ } else {
+ loader_k.load_unsafe();
+ }
+
+ // Compute S = Q @ K^T for this block
+ Stile.clear();
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ UZU_PRAGMA_UNROLL
+ for (short dd = 0; dd < TD; dd++) {
+ simdgroup_barrier(mem_flags::mem_none);
+
+ Qtile.template load<T, 1, 1, LDQ_tgp, 1>(&Qs[Qs_offset + dd * Qs_tile_stride]);
+ Ktile.template load<T, 1, 1, 1, LDK_tgp>(&Ks[Ks_offset + dd * Ks_tile_stride]);
+
+ simdgroup_barrier(mem_flags::mem_none);
+ tile_matmad(Stile, Qtile, Ktile, Stile);
+ }
+
+ // Mask out tail keys for the last (unaligned) K block
+ if (!align_K && kb == params->nk_aligned) {
+ const int k_rem = params->k_rem;
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < TK; j++) {
+ thread auto& frag = Stile.frag_at(0, j);
+ const int col0 = int(sn) + int(j) * kFragSize;
+ if (col0 >= k_rem) {
+ frag[0] = neg_inf;
+ }
+ if ((col0 + 1) >= k_rem) {
+ frag[1] = neg_inf;
+ }
+ }
+ }
+
+ // Causal mask (only needed for the last few blocks near the diagonal)
+ if (do_causal) {
+ const int tail_blocks = (BQ + BK - 1) / BK + int(!align_K);
+ const int tail_start = kb_lim - tail_blocks;
+ if (kb >= tail_start) {
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < TK; j++) {
+ thread auto& frag = Stile.frag_at(0, j);
+ const int col_base = kb * BK + int(sn) + int(j) * kFragSize;
+ if (q_abs < col_base) {
+ frag[0] = neg_inf;
+ }
+ if (q_abs < (col_base + 1)) {
+ frag[1] = neg_inf;
+ }
+ }
+ }
+ }
+
+ // Add external mask (additive bias in natural-log domain; convert to log2)
+ if (has_mask && q_rel < params->q_len) {
+ const int64_t row_stride = mask_params->m_strides[2];
+ const int64_t row_base = int64_t(q_rel) * row_stride;
+
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < TK; j++) {
+ thread auto& frag = Stile.frag_at(0, j);
+ const int col_base = kb * BK + int(sn) + int(j) * kFragSize;
+
+ const int k0 = col_base;
+ const int k1 = col_base + 1;
+
+ if (k0 < params->k_len) {
+ AccumType mv =
+ static_cast<AccumType>(mask[row_base + int64_t(k0)]);
+ mv = metal::max(mv, static_cast<AccumType>(-1e9f));
+ frag[0] += M_LOG2E_F * mv;
+ }
+ if (k1 < params->k_len) {
+ AccumType mv =
+ static_cast<AccumType>(mask[row_base + int64_t(k1)]);
+ mv = metal::max(mv, static_cast<AccumType>(-1e9f));
+ frag[1] += M_LOG2E_F * mv;
+ }
+ }
+ }
+
+ // Load V block (overwriting K in shared memory)
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (!align_K && kb == params->nk_aligned) {
+ loader_v.load_safe(short2(BD, params->k_rem));
+ } else {
+ loader_v.load_unsafe();
+ }
+
+ // -----------------------------------------------------------------------
+ // Streaming softmax update for this block
+
+ // Row max for this block
+ AccumType block_max_local = -INFINITY;
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < TK; j++) {
+ const thread auto& frag = Stile.frag_at(0, j);
+ block_max_local = metal::max(block_max_local, frag[0]);
+ block_max_local = metal::max(block_max_local, frag[1]);
+ }
+ const AccumType block_max = row_reduce_max(block_max_local);
+
+ const AccumType new_max = metal::max(max_score, block_max);
+ const AccumType factor = fast::exp2(max_score - new_max);
+ max_score = new_max;
+
+ // Rescale running sum
+ sum_score *= factor;
+
+ // exp2(S - new_max) and row sum
+ AccumType block_sum_local = AccumType(0);
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < TK; j++) {
+ thread auto& frag = Stile.frag_at(0, j);
+ frag[0] = fast::exp2(frag[0] - new_max);
+ frag[1] = fast::exp2(frag[1] - new_max);
+ block_sum_local += frag[0] + frag[1];
+ }
+ const AccumType block_sum = row_reduce_sum(block_sum_local);
+ sum_score += block_sum;
+
+ // Rescale output accumulator
+ UZU_PRAGMA_UNROLL
+ for (short id = 0; id < TD; id++) {
+ thread auto& frag = Otile.frag_at(0, id);
+ frag[0] *= factor;
+ frag[1] *= factor;
+ }
+
+ // Accumulate output: Otile += Stile * Vblock
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ UZU_PRAGMA_UNROLL
+ for (short id = 0; id < TD; id++) {
+ UZU_PRAGMA_UNROLL
+ for (short ik = 0; ik < TK; ik++) {
+ if constexpr (BD == 128) {
+ simdgroup_barrier(mem_flags::mem_none);
+ }
+
+ const short kk = ik * kFragSize;
+ const short dd = id * kFragSize;
+
+ Vtile.template load<T, 1, 1, LDV_tgp, 1>(&Vs[Vs_offset + kk * LDV_tgp + dd]);
+
+ if constexpr (BD == 128) {
+ simdgroup_barrier(mem_flags::mem_none);
+ }
+
+ MMAFrag_acc_t::mma(
+ Otile.frag_at(0, id),
+ Stile.frag_at(0, ik),
+ Vtile.frag_at(0, 0),
+ Otile.frag_at(0, id));
+ }
+ }
+
+ // Prepare for next iteration
+ loader_k.next();
+ loader_v.next();
+ }
+
+ // -------------------------------------------------------------------------
+ // Normalize output by sum_score (avoid div-by-zero for masked-out rows)
+ const AccumType inv_sum = AccumType(1) / sum_score;
+ UZU_PRAGMA_UNROLL
+ for (short id = 0; id < TD; id++) {
+ thread auto& frag = Otile.frag_at(0, id);
+ frag[0] *= inv_sum;
+ frag[1] *= inv_sum;
+ }
+
+ threadgroup_barrier(mem_flags::mem_none);
+
+ // Store results (O is row-major with row-stride params->o_strides[2])
+ O += int64_t(tm + sm) * params->o_strides[2] + int64_t(sn);
+
+ if (!align_Q && int(tid.x) == params->nq_aligned) {
+ const short2 dst_tile_dims = short2(BD - sn, params->q_rem - (tm + sm));
+
+ if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0) {
+ return;
+ }
+
+ Otile.template store_safe<T, 1, 1>(
+ O, int(params->o_strides[2]), dst_tile_dims);
+ } else {
+ Otile.template store<T, 1, 1>(O, int(params->o_strides[2]));
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Kernel instantiations (BQ=32, BK=32, WM=2, WN=2)
+///////////////////////////////////////////////////////////////////////////////
+
+#define instantiate_attention_gemm(type_name, element_type, head_dim_value) \
+ template [[host_name("attention_gemm_" #type_name "_" #head_dim_value)]] \
+ [[kernel]] void attention_gemm< \
+ element_type, \
+ 32, \
+ 32, \
+ head_dim_value, \
+ 2, \
+ 2, \
+ float>( \
+ const device element_type* Q [[buffer(0)]], \
+ const device element_type* K [[buffer(1)]], \
+ const device element_type* V [[buffer(2)]], \
+ device element_type* O [[buffer(3)]], \
+ const constant AttnParams* params [[buffer(4)]], \
+ const constant AttnMaskParams* mask_params \
+ [[buffer(5), function_constant(has_mask)]], \
+ const device element_type* mask [[buffer(6), function_constant(has_mask)]], \
+ const device float* sinks [[buffer(7), function_constant(has_sinks)]], \
+ uint simd_lane_id [[thread_index_in_simdgroup]], \
+ uint simd_group_id [[simdgroup_index_in_threadgroup]], \
+ uint3 tid [[threadgroup_position_in_grid]], \
+ uint3 lid [[thread_position_in_threadgroup]]);
+
+instantiate_attention_gemm(f16, half, 64)
+instantiate_attention_gemm(f16, half, 128)
+instantiate_attention_gemm(f16, half, 256)
+
+instantiate_attention_gemm(bf16, bfloat, 64)
+instantiate_attention_gemm(bf16, bfloat, 128)
+instantiate_attention_gemm(bf16, bfloat, 256)
+
+instantiate_attention_gemm(f32, float, 64)
+instantiate_attention_gemm(f32, float, 128)
+instantiate_attention_gemm(f32, float, 256)
+
+#undef instantiate_attention_gemm
+
+
diff --git a/crates/uzu/src/backends/metal/kernel/attention/gemm_types.h b/crates/uzu/src/backends/metal/kernel/attention/gemm_types.h
new file mode 100644
index 000000000..2b634d5d3
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/attention/gemm_types.h
@@ -0,0 +1,79 @@
+// Shared types for Gemm attention kernel - used by both Metal shaders and Rust
+// (via bindgen). This header must be C-compatible for bindgen to parse it.
+//
+// NOTE: Only wrap namespaces for Metal builds. For C/bindgen builds, keep
+// everything in the global namespace.
+//
+// SPDX-License-Identifier: MIT
+
+#pragma once
+
+// Metal and C have different type systems
+// __METAL_VERSION__ is defined when compiling with the Metal shader compiler
+#ifdef __METAL_VERSION__
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu {
+namespace attention {
+
+#else
+#include <stdint.h>
+#endif
+
+///////////////////////////////////////////////////////////////////////////////
+// Gemm Attention Parameters
+///////////////////////////////////////////////////////////////////////////////
+
+/// Parameters for the Gemm attention kernel.
+///
+/// All strides are in **elements**, not bytes.
+/// Q/K/V/O are treated as 3D tensors with strides for:
+/// - [0] batch
+/// - [1] head (or kv-head for K/V)
+/// - [2] sequence (row stride for [seq, head_dim])
+struct AttnParams {
+ int64_t q_strides[3];
+ int64_t k_strides[3];
+ int64_t v_strides[3];
+ int64_t o_strides[3];
+
+ int gqa_factor;
+ float scale;
+
+ // Query length (suffix length) and key length (prefix + suffix)
+ int q_len;
+ int k_len;
+
+ // Absolute offset of the first query token in the full key sequence.
+ // For LLM decode/prefill this is the segment prefix length.
+ int q_off;
+
+ // Query tiling metadata (BQ tiles)
+ int nq_aligned; // q_len / BQ
+ int q_rem; // q_len % BQ
+
+ // Key tiling metadata (BK tiles)
+ int nk; // ceil(k_len / BK)
+ int nk_aligned; // k_len / BK
+ int k_rem; // k_len % BK
+};
+
+/// Parameters describing the additive attention mask.
+///
+/// Mask is treated as a 2D matrix [qL, kL] with:
+/// - per-batch base offset (M_strides[0])
+/// - per-head base offset (M_strides[1])
+/// - row stride in elements (M_strides[2])
+/// Column stride is assumed to be 1.
+struct AttnMaskParams {
+ int64_t m_strides[3];
+};
+
+// Close Metal namespace
+#ifdef __METAL_VERSION__
+} // namespace attention
+} // namespace uzu
+#endif
+
+
diff --git a/crates/uzu/src/backends/metal/kernel/attention/gemm_types.rs b/crates/uzu/src/backends/metal/kernel/attention/gemm_types.rs
new file mode 100644
index 000000000..49ac92d5a
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/attention/gemm_types.rs
@@ -0,0 +1,69 @@
+/* automatically generated by rust-bindgen 0.72.1 */
+
+#[doc = " Parameters for the Gemm attention kernel.\n\n All strides are in **elements**, not bytes.\n Q/K/V/O are treated as 3D tensors with strides for:\n - [0] batch\n - [1] head (or kv-head for K/V)\n - [2] sequence (row stride for [seq, head_dim])"]
+#[repr(C)]
+#[derive(Debug, Default, Copy, Clone)]
+pub struct AttnParams {
+ pub q_strides: [i64; 3usize],
+ pub k_strides: [i64; 3usize],
+ pub v_strides: [i64; 3usize],
+ pub o_strides: [i64; 3usize],
+ pub gqa_factor: ::core::ffi::c_int,
+ pub scale: f32,
+ pub q_len: ::core::ffi::c_int,
+ pub k_len: ::core::ffi::c_int,
+ pub q_off: ::core::ffi::c_int,
+ pub nq_aligned: ::core::ffi::c_int,
+ pub q_rem: ::core::ffi::c_int,
+ pub nk: ::core::ffi::c_int,
+ pub nk_aligned: ::core::ffi::c_int,
+ pub k_rem: ::core::ffi::c_int,
+}
+#[allow(clippy::unnecessary_operation, clippy::identity_op)]
+const _: () = {
+ ["Size of AttnParams"][::core::mem::size_of::<AttnParams>() - 136usize];
+ ["Alignment of AttnParams"][::core::mem::align_of::<AttnParams>() - 8usize];
+ ["Offset of field: AttnParams::q_strides"]
+ [::core::mem::offset_of!(AttnParams, q_strides) - 0usize];
+ ["Offset of field: AttnParams::k_strides"]
+ [::core::mem::offset_of!(AttnParams, k_strides) - 24usize];
+ ["Offset of field: AttnParams::v_strides"]
+ [::core::mem::offset_of!(AttnParams, v_strides) - 48usize];
+ ["Offset of field: AttnParams::o_strides"]
+ [::core::mem::offset_of!(AttnParams, o_strides) - 72usize];
+ ["Offset of field: AttnParams::gqa_factor"]
+ [::core::mem::offset_of!(AttnParams, gqa_factor) - 96usize];
+ ["Offset of field: AttnParams::scale"]
+ [::core::mem::offset_of!(AttnParams, scale) - 100usize];
+ ["Offset of field: AttnParams::q_len"]
+ [::core::mem::offset_of!(AttnParams, q_len) - 104usize];
+ ["Offset of field: AttnParams::k_len"]
+ [::core::mem::offset_of!(AttnParams, k_len) - 108usize];
+ ["Offset of field: AttnParams::q_off"]
+ [::core::mem::offset_of!(AttnParams, q_off) - 112usize];
+ ["Offset of field: AttnParams::nq_aligned"]
+ [::core::mem::offset_of!(AttnParams, nq_aligned) - 116usize];
+ ["Offset of field: AttnParams::q_rem"]
+ [::core::mem::offset_of!(AttnParams, q_rem) - 120usize];
+ ["Offset of field: AttnParams::nk"]
+ [::core::mem::offset_of!(AttnParams, nk) - 124usize];
+ ["Offset of field: AttnParams::nk_aligned"]
+ [::core::mem::offset_of!(AttnParams, nk_aligned) - 128usize];
+ ["Offset of field: AttnParams::k_rem"]
+ [::core::mem::offset_of!(AttnParams, k_rem) - 132usize];
+};
+#[doc = " Parameters describing the additive attention mask.\n\n Mask is treated as a 2D matrix [qL, kL] with:\n - per-batch base offset (M_strides[0])\n - per-head base offset (M_strides[1])\n - row stride in elements (M_strides[2])\n Column stride is assumed to be 1."]
+#[repr(C)]
+#[derive(Debug, Default, Copy, Clone)]
+pub struct AttnMaskParams {
+ pub m_strides: [i64; 3usize],
+}
+#[allow(clippy::unnecessary_operation, clippy::identity_op)]
+const _: () = {
+ ["Size of AttnMaskParams"]
+ [::core::mem::size_of::<AttnMaskParams>() - 24usize];
+ ["Alignment of AttnMaskParams"]
+ [::core::mem::align_of::<AttnMaskParams>() - 8usize];
+ ["Offset of field: AttnMaskParams::m_strides"]
+ [::core::mem::offset_of!(AttnMaskParams, m_strides) - 0usize];
+};
diff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
index 1366ca20d..ffec0bb84 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
@@ -9,6 +9,9 @@ use thiserror::Error;
use crate::backends::metal::{KernelDataType, MTLContext, MTLError};
+mod gemm_types;
+use gemm_types::{AttnMaskParams, AttnParams};
+
#[derive(Debug, Clone, Copy)]
pub enum AttentionKernelVariant {
SinglePass,
@@ -25,6 +28,7 @@ pub struct AttentionKernelPipelines {
}
pub struct AttentionKernel {
+ data_type: KernelDataType,
pipelines: AttentionKernelPipelines,
}
@@ -100,6 +104,24 @@ pub struct KVCacheUpdateArguments<'a> {
pub max_sequence_length: usize,
}
+pub struct AttentionGemmArguments<'a> {
+ pub queries_buffer: &'a MTLBuffer, // buffer(0)
+ pub keys_buffer: &'a MTLBuffer, // buffer(1)
+ pub values_buffer: &'a MTLBuffer, // buffer(2)
+ pub output_buffer: &'a MTLBuffer, // buffer(3)
+ pub mask_buffer: Option<&'a MTLBuffer>, // buffer(6)
+ pub sinks_buffer: Option<&'a MTLBuffer>, // buffer(7)
+ pub num_heads: usize,
+ pub num_groups: usize,
+ pub suffix_length: usize, // qL
+ pub sequence_length: usize, // kL (prefix + suffix)
+ pub segment_prefix_length: usize, // qL_off
+ pub max_sequence_length: usize, // stride for K/V cache
+ pub head_dim: usize,
+ pub is_causal: bool,
+ pub scale: f32,
+}
+
fn make_function_constants(
has_sinks_value: bool,
is_causal_value: bool,
@@ -220,6 +242,7 @@ impl AttentionKernel {
.ok();
Ok(Self {
+ data_type,
pipelines: AttentionKernelPipelines {
single_pass,
two_pass_1,
@@ -554,4 +577,163 @@ impl AttentionKernel {
);
Ok(())
}
+
+ pub fn encode_gemm(
+ &self,
+ context: &MTLContext,
+ compute_encoder: &ComputeCommandEncoderRef,
+ args: AttentionGemmArguments,
+ ) -> Result<(), AttentionError> {
+ const BQ: usize = 32;
+ const BK: usize = 32;
+ const WM: u64 = 2;
+ const WN: u64 = 2;
+
+ if !matches!(args.head_dim, 64 | 128 | 256) {
+ return Err(AttentionError::UnsupportedHeadDim(args.head_dim));
+ }
+
+ let align_q = (args.suffix_length % BQ) == 0;
+ let align_k = (args.sequence_length % BK) == 0;
+ let has_mask = args.mask_buffer.is_some();
+ let has_sinks = args.sinks_buffer.is_some();
+
+ let fcv = FunctionConstantValues::new();
+ fcv.set_constant_value_at_index(
+ &align_q as *const bool as *const _,
+ MTLDataType::Bool,
+ 200,
+ );
+ fcv.set_constant_value_at_index(
+ &align_k as *const bool as *const _,
+ MTLDataType::Bool,
+ 201,
+ );
+ fcv.set_constant_value_at_index(
+ &has_mask as *const bool as *const _,
+ MTLDataType::Bool,
+ 300,
+ );
+ fcv.set_constant_value_at_index(
+ &args.is_causal as *const bool as *const _,
+ MTLDataType::Bool,
+ 301,
+ );
+ fcv.set_constant_value_at_index(
+ &has_sinks as *const bool as *const _,
+ MTLDataType::Bool,
+ 302,
+ );
+
+ // Kernel name matches gemm_attention.metal instantiations:
+ // attention_gemm_{f16|bf16|f32}_{head_dim}
+ let type_name = match self.data_type {
+ KernelDataType::Float16 => "f16",
+ KernelDataType::BFloat16 => "bf16",
+ KernelDataType::Float32 => "f32",
+ };
+
+ let function_name =
+ format!("attention_gemm_{}_{}", type_name, args.head_dim);
+
+ let cache_key = format!(
+ "{}_aq{}_ak{}_m{}_c{}_s{}",
+ function_name,
+ align_q as u8,
+ align_k as u8,
+ has_mask as u8,
+ args.is_causal as u8,
+ has_sinks as u8
+ );
+
+ let (pipeline, _) = context
+ .compute_pipeline_state_with_reflection_cached(
+ &cache_key,
+ &function_name,
+ Some(&fcv),
+ )
+ .map_err(AttentionError::MetalError)?;
+
+ compute_encoder.set_compute_pipeline_state(&pipeline);
+
+ // Buffers
+ compute_encoder.set_buffer(0, Some(args.queries_buffer), 0);
+ compute_encoder.set_buffer(1, Some(args.keys_buffer), 0);
+ compute_encoder.set_buffer(2, Some(args.values_buffer), 0);
+ compute_encoder.set_buffer(3, Some(args.output_buffer), 0);
+
+ // Params (all strides in elements)
+ let q_head_stride = (args.suffix_length * args.head_dim) as i64;
+ let q_seq_stride = args.head_dim as i64;
+
+ let kv_head_stride = (args.max_sequence_length * args.head_dim) as i64;
+ let kv_seq_stride = args.head_dim as i64;
+
+ let o_head_stride = args.head_dim as i64;
+ let o_seq_stride = (args.num_heads * args.head_dim) as i64;
+
+ let nq = (args.suffix_length + BQ - 1) / BQ;
+ let nk = (args.sequence_length + BK - 1) / BK;
+ let nq_aligned = args.suffix_length / BQ;
+ let nk_aligned = args.sequence_length / BK;
+
+ let params = AttnParams {
+ q_strides: [0, q_head_stride, q_seq_stride],
+ k_strides: [0, kv_head_stride, kv_seq_stride],
+ v_strides: [0, kv_head_stride, kv_seq_stride],
+ o_strides: [0, o_head_stride, o_seq_stride],
+ gqa_factor: (args.num_heads / args.num_groups) as i32,
+ scale: args.scale,
+ q_len: args.suffix_length as i32,
+ k_len: args.sequence_length as i32,
+ q_off: args.segment_prefix_length as i32,
+ nq_aligned: nq_aligned as i32,
+ q_rem: (args.suffix_length - nq_aligned * BQ) as i32,
+ nk: nk as i32,
+ nk_aligned: nk_aligned as i32,
+ k_rem: (args.sequence_length - nk_aligned * BK) as i32,
+ };
+
+ compute_encoder.set_bytes(
+ 4,
+ size_of::<AttnParams>() as u64,
+ &params as *const AttnParams as *const _,
+ );
+
+ if let Some(mask_buffer) = args.mask_buffer {
+ let mask_params = AttnMaskParams {
+ // We use a shared bias matrix for all heads/batches.
+ m_strides: [0, 0, args.sequence_length as i64],
+ };
+ compute_encoder.set_bytes(
+ 5,
+ size_of::<AttnMaskParams>() as u64,
+ &mask_params as *const AttnMaskParams as *const _,
+ );
+ compute_encoder.set_buffer(6, Some(mask_buffer), 0);
+ }
+
+ if let Some(sinks_buffer) = args.sinks_buffer {
+ compute_encoder.set_buffer(7, Some(sinks_buffer), 0);
+ }
+
+ // Dispatch
+ let threadgroups_per_grid = MTLSize {
+ width: nq as u64,
+ height: args.num_heads as u64,
+ depth: 1,
+ };
+ let threads_per_threadgroup = MTLSize {
+ width: 32,
+ height: WM,
+ depth: WN,
+ };
+
+ compute_encoder.dispatch_thread_groups(
+ threadgroups_per_grid,
+ threads_per_threadgroup,
+ );
+
+ Ok(())
+ }
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/arguments.rs b/crates/uzu/src/backends/metal/kernel/matmul/arguments.rs
index 72fbea9ef..56dc4e2f6 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/arguments.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/arguments.rs
@@ -3,6 +3,8 @@ use metal::Buffer as MTLBuffer;
#[derive(Debug, Clone)]
pub struct MatmulArguments<'a> {
pub a: &'a MTLBuffer,
+ /// Byte offset into `a` (used for slicing the batch dimension).
+ pub a_offset: u64,
pub b: &'a MTLBuffer,
pub c: Option<&'a MTLBuffer>,
pub d: &'a MTLBuffer,
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
index df810d944..cdaf8c0bd 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
@@ -220,25 +220,19 @@ impl GemvKernel {
let pipeline = self.get_pipeline(context, configuration)?;
encoder.set_compute_pipeline_state(pipeline);
- encoder.set_buffer(
- 0,
- Some(if matrix_is_rhs {
- arguments.b
- } else {
- arguments.a
- }),
- 0,
- );
+ let (buf0, off0) = if matrix_is_rhs {
+ (arguments.b, 0)
+ } else {
+ (arguments.a, arguments.a_offset)
+ };
+ encoder.set_buffer(0, Some(buf0), off0);
- encoder.set_buffer(
- 1,
- Some(if matrix_is_rhs {
- arguments.a
- } else {
- arguments.b
- }),
- 0,
- );
+ let (buf1, off1) = if matrix_is_rhs {
+ (arguments.a, arguments.a_offset)
+ } else {
+ (arguments.b, 0)
+ };
+ encoder.set_buffer(1, Some(buf1), off1);
if configuration.do_axpby {
if let Some(bias) = bias_buffer {
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
index 4eafd1413..929e6cd50 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
@@ -239,7 +239,7 @@ impl MatmulKernel {
DeviceClass::Unknown(_) => 'g',
};
- // Default steel matmul heuristics (non-NAX), matching observed
+ // Default gemm matmul heuristics (non-NAX), matching observed
// choices on M2 Max: prefer 64x64x16 wm2/wn2 for large shapes and
// fall back to 64x32x32 when transpose_b=true and not N-aligned.
match device_class_code {
@@ -401,7 +401,7 @@ impl MatmulKernel {
enc.set_compute_pipeline_state(ps);
// Set buffers
- enc.set_buffer(0, Some(args.a), 0);
+ enc.set_buffer(0, Some(args.a), args.a_offset);
enc.set_buffer(1, Some(args.b), 0);
if use_out_source {
if let Some(c_buf) = args.c {
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/splitk.rs b/crates/uzu/src/backends/metal/kernel/matmul/splitk.rs
index b55e4e6b0..8cc7e7bb2 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/splitk.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/splitk.rs
@@ -264,7 +264,7 @@ impl SplitKGemm {
self.get_partial_pipeline(mtl, tile_config, mn_aligned, k_aligned)?;
enc.set_compute_pipeline_state(partial_ps);
- enc.set_buffer(0, Some(args.a), 0);
+ enc.set_buffer(0, Some(args.a), args.a_offset);
enc.set_buffer(1, Some(args.b), 0);
enc.set_buffer(2, Some(&accumulator_buffer), 0);
enc.set_bytes(
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
index 824227fd8..f468b20de 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
@@ -43,6 +43,8 @@ pub struct QuantizedMatmulKernel {
#[derive(Debug, Clone, Copy)]
pub struct QuantizedMatmulArguments<'a> {
pub a_buffer: &'a MTLBuffer,
+ /// Byte offset into `a_buffer` (used for slicing the batch dimension).
+ pub a_offset: u64,
pub b_buffer: &'a MTLBuffer,
pub scales_buffer: &'a MTLBuffer,
pub zero_points_or_biases_buffer: &'a MTLBuffer,
@@ -203,7 +205,7 @@ impl QuantizedMatmulKernel {
encoder.set_buffer(0, Some(args.b_buffer), 0);
encoder.set_buffer(1, Some(args.scales_buffer), 0);
encoder.set_buffer(2, Some(args.zero_points_or_biases_buffer), 0);
- encoder.set_buffer(3, Some(args.a_buffer), 0);
+ encoder.set_buffer(3, Some(args.a_buffer), args.a_offset);
encoder.set_buffer(4, Some(args.output_buffer), 0);
let k: i32 = args.input_dim;
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
index 39b476540..961af1179 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
@@ -41,8 +41,10 @@ pub struct RMSNormKernel {
#[derive(Debug)]
pub struct RMSNormArguments<'a> {
pub input_buffer: &'a MTLBuffer,
+ pub input_offset: u64,
pub scales_buffer: &'a MTLBuffer,
pub output_buffer: &'a MTLBuffer,
+ pub output_offset: u64,
pub batch_size: i32,
pub model_dim: i32,
pub epsilon: f32,
@@ -207,9 +209,10 @@ impl RMSNormKernel {
compute_encoder.set_compute_pipeline_state(&self.pipeline);
// Set buffers
- compute_encoder.set_buffer(0, Some(args.input_buffer), 0);
+ compute_encoder.set_buffer(0, Some(args.input_buffer), args.input_offset);
compute_encoder.set_buffer(1, Some(args.scales_buffer), 0);
- compute_encoder.set_buffer(2, Some(args.output_buffer), 0);
+ compute_encoder
+ .set_buffer(2, Some(args.output_buffer), args.output_offset);
// Set parameters
compute_encoder.set_bytes(
diff --git a/crates/uzu/src/backends/metal/kernel/rope/mod.rs b/crates/uzu/src/backends/metal/kernel/rope/mod.rs
index 9b05d911c..d188af441 100644
--- a/crates/uzu/src/backends/metal/kernel/rope/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/rope/mod.rs
@@ -111,9 +111,9 @@ impl RopeKernel {
);
let threads_per_threadgroup = MTLSize {
- width: 32,
+ width: 1,
height: 1,
- depth: 1,
+ depth: 32,
};
let total_threads = MTLSize {
width: num_heads as u64,
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
index b5dcf636d..8014fc1f0 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
@@ -268,6 +268,7 @@ impl SamplingKernel {
seeds_buffer: Option<&MTLBuffer>,
seeds_offset: usize,
bitmask_buffer: Option<&MTLBuffer>,
+ bitmask_offset: usize,
sampled_tokens_buffer: &MTLBuffer,
sampling_method: SamplingMethod,
batch_size: usize,
@@ -280,6 +281,7 @@ impl SamplingKernel {
seeds_buffer,
seeds_offset,
bitmask_buffer,
+ bitmask_offset,
sampled_tokens_buffer,
sampling_method,
batch_size,
@@ -296,6 +298,7 @@ impl SamplingKernel {
seeds_buffer: Option<&MTLBuffer>,
seeds_offset: usize,
bitmask_buffer: Option<&MTLBuffer>,
+ bitmask_offset: usize,
sampled_tokens_buffer: &MTLBuffer,
sampling_method: SamplingMethod,
batch_size: usize,
@@ -321,6 +324,7 @@ impl SamplingKernel {
self.encode_bitmask(
last_logits_buffer,
bitmask_buffer,
+ bitmask_offset,
&self.bitmask_applied_logits,
batch_size as u32,
vocab_size as u32,
@@ -428,6 +432,7 @@ impl SamplingKernel {
&self,
logits_buffer: &MTLBuffer,
bitmask_buffer: &MTLBuffer,
+ bitmask_offset: usize,
processed_logits_buffer: &MTLBuffer,
batch_size: u32,
vocab_size: u32,
@@ -436,7 +441,11 @@ impl SamplingKernel {
compute_encoder.set_compute_pipeline_state(&self.bitmask_pipeline);
compute_encoder.set_buffer(0, Some(logits_buffer), 0);
- compute_encoder.set_buffer(1, Some(bitmask_buffer), 0);
+ compute_encoder.set_buffer(
+ 1,
+ Some(bitmask_buffer),
+ bitmask_offset as u64,
+ );
compute_encoder.set_buffer(2, Some(processed_logits_buffer), 0);
compute_encoder.set_bytes(
3,
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index f65947fd3..e604040b1 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -101,6 +101,8 @@ impl LanguageModelGenerator {
let flat_trie = suffix_root.linearize();
let active_suffix_length = flat_trie.len() - 1;
+ let has_grammar = compiled_grammar.is_some();
+
let token_ids = tokens
.iter()
.copied()
@@ -141,7 +143,7 @@ impl LanguageModelGenerator {
(
step_token_ids,
step_token_positions,
- step_token_bitmask,
+ step_token_bitmasks,
step_token_seeds,
),
) in
@@ -154,25 +156,6 @@ impl LanguageModelGenerator {
let step_token_ids = step_token_ids.collect::<Box<[u64]>>();
let step_token_positions =
step_token_positions.collect::<Box<[usize]>>();
- let step_token_bitmask = step_token_bitmask
- .map(|mask| match mask {
- Some(mask) => Either::Left(
- mask.iter()
- .copied()
- .take(single_token_bitmask_size)
- .chain(repeat_n(
- 0u32,
- single_token_bitmask_size
- .saturating_sub(mask.len()),
- )),
- ),
- None => Either::Right(repeat_n(
- u32::MAX,
- single_token_bitmask_size,
- )),
- })
- .flatten()
- .collect::<Box<[u32]>>();
let step_token_seeds = step_token_seeds.collect::<Box<[u64]>>();
let active_suffix_length = step_token_positions
@@ -183,6 +166,53 @@ impl LanguageModelGenerator {
let should_sample_after_step =
sample_suffix && is_last_prefill_step;
+ // If we sample on the last prefill step, we only need logits/sampling
+ // for tokens that are beyond the prompt prefix (i.e. starting at the
+ // suffix-root token, which is the last prompt token).
+ let (sampling_start, sampling_length) = if should_sample_after_step
+ {
+ let suffix_root_index_in_step =
+ (tokens_length - 1).saturating_sub(tokens_start_index);
+ let sampling_length = active_suffix_length
+ .saturating_sub(suffix_root_index_in_step);
+ debug_assert!(
+ sampling_length > 0,
+ "Expected at least one token to sample on the last prefill step"
+ );
+ (suffix_root_index_in_step, sampling_length)
+ } else {
+ (0, 0)
+ };
+
+ let step_token_bitmask: Option<Box<[u32]>> =
+ if has_grammar && sampling_length > 0 {
+ Some(
+ step_token_bitmasks
+ .map(|mask| match mask {
+ Some(mask) => Either::Left(
+ mask.iter()
+ .copied()
+ .take(single_token_bitmask_size)
+ .chain(repeat_n(
+ 0u32,
+ single_token_bitmask_size
+ .saturating_sub(mask.len()),
+ )),
+ ),
+ None => Either::Right(repeat_n(
+ u32::MAX,
+ single_token_bitmask_size,
+ )),
+ })
+ .flatten()
+ .collect::<Box<[u32]>>(),
+ )
+ } else {
+ // Drain the chunk iterator to keep the other chunked iterators aligned.
+ let _ = step_token_bitmasks.count();
+ None
+ };
+
let should_capture =
self.gpu_capture.should_capture_prefill(step == 0);
@@ -199,10 +229,12 @@ impl LanguageModelGenerator {
let task = LanguageModelGeneratorRunTask {
token_ids: &step_token_ids,
token_positions: &step_token_positions,
- token_bitmask: Some(&step_token_bitmask),
+ token_bitmask: step_token_bitmask.as_deref(),
token_seeds: &step_token_seeds,
expected_number_of_new_tokens: prefill_step_size,
active_suffix_length,
+ sampling_start,
+ sampling_length,
is_prefilling: !should_sample_after_step,
};
@@ -273,10 +305,8 @@ impl LanguageModelGenerator {
let last_suffix_start = prefill_step_size * (prefill_steps - 1);
let suffix_root_index = (tokens_length - last_suffix_start) - 1;
- let (accepted_tokens, accepted_token_indices) = flat_trie.accept(
- &sampled_tokens[suffix_root_index..],
- compiled_grammar.as_deref_mut(),
- );
+ let (accepted_tokens, accepted_token_indices) =
+ flat_trie.accept(&sampled_tokens, compiled_grammar.as_deref_mut());
self.update_cache_layers(
&accepted_token_indices
@@ -321,27 +351,32 @@ impl LanguageModelGenerator {
.chain(repeat_n(0, suffix_length - active_suffix_length))
.collect::<Box<[u64]>>();
- let single_token_bitmask_size =
- self.context.model_shape.bitmask_shape(1)[1];
- let token_bitmask = flat_trie
- .token_masks()
- .chain(repeat_n(None, suffix_length - active_suffix_length))
- .map(|mask| match mask {
- Some(mask) => Either::Left(
- mask.iter().copied().take(single_token_bitmask_size).chain(
- repeat_n(
- 0u32,
- single_token_bitmask_size
- .saturating_sub(mask.len()),
+ let token_bitmask: Option<Box<[u32]>> =
+ compiled_grammar.is_some().then(|| {
+ let single_token_bitmask_size =
+ self.context.model_shape.bitmask_shape(1)[1];
+ flat_trie
+ .token_masks()
+ .chain(repeat_n(None, suffix_length - active_suffix_length))
+ .map(|mask| match mask {
+ Some(mask) => Either::Left(
+ mask.iter()
+ .copied()
+ .take(single_token_bitmask_size)
+ .chain(repeat_n(
+ 0u32,
+ single_token_bitmask_size
+ .saturating_sub(mask.len()),
+ )),
),
- ),
- ),
- None => {
- Either::Right(repeat_n(u32::MAX, single_token_bitmask_size))
- },
- })
- .flatten()
- .collect::<Box<[u32]>>();
+ None => Either::Right(repeat_n(
+ u32::MAX,
+ single_token_bitmask_size,
+ )),
+ })
+ .flatten()
+ .collect::<Box<[u32]>>()
+ });
let start_position = self.tokens.len() - 1;
let token_positions = flat_trie
@@ -361,10 +396,12 @@ impl LanguageModelGenerator {
let task = LanguageModelGeneratorRunTask {
token_ids: &token_ids,
token_positions: &token_positions,
- token_bitmask: Some(&token_bitmask),
+ token_bitmask: token_bitmask.as_deref(),
token_seeds: &token_seeds,
expected_number_of_new_tokens: 1,
active_suffix_length,
+ sampling_start: 0,
+ sampling_length: active_suffix_length,
is_prefilling: false,
};
@@ -466,6 +503,8 @@ impl LanguageModelGenerator {
token_seeds: &[0], // Ignored, using async buffer
expected_number_of_new_tokens: 1,
active_suffix_length: 1,
+ sampling_start: 0,
+ sampling_length: 1,
is_prefilling: false,
};
@@ -489,6 +528,8 @@ impl LanguageModelGenerator {
task.token_bitmask,
&task.token_seeds,
task.active_suffix_length,
+ /*sampling_start=*/ 0,
+ /*sampling_length=*/ task.active_suffix_length,
task.is_prefilling,
None,
skip_token_ids_copy,
@@ -651,6 +692,8 @@ impl LanguageModelGenerator {
token_seeds: &token_seeds,
expected_number_of_new_tokens: suffix_length,
active_suffix_length: suffix_length,
+ sampling_start: 0,
+ sampling_length: suffix_length,
is_prefilling: false,
};
@@ -736,7 +779,6 @@ impl LanguageModelGenerator {
root_command_buffer.wait_until_completed();
let run_time = run_start.elapsed().as_secs_f64();
-
if should_capture {
self.gpu_capture.stop_capture("decode");
}
@@ -756,7 +798,7 @@ impl LanguageModelGenerator {
let output_view = output_buffer
.as_view::<u32>()
.map_err(|_| Error::SamplingFailed)?;
- let batch_size = state.active_suffix_length();
+ let batch_size = state.sampling_length();
let mut result = Vec::with_capacity(batch_size);
for i in 0..batch_size {
diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rs
index 2ce090ee7..f6cc9efbe 100644
--- a/crates/uzu/src/language_model/tasks.rs
+++ b/crates/uzu/src/language_model/tasks.rs
@@ -37,6 +37,17 @@ pub struct LanguageModelGeneratorRunTask<'a> {
pub token_seeds: &'a [u64],
pub expected_number_of_new_tokens: usize,
pub active_suffix_length: usize,
+ /// Start index (within this task's suffix batch) from which we need logits
+ /// and sampling results.
+ ///
+ /// - For normal decode/generate: 0
+ /// - For prefill+sample: index of the "suffix root" token inside the last prefill step
+ pub sampling_start: usize,
+ /// Number of batch items to produce logits + sampling results for.
+ ///
+ /// Typically `active_suffix_length - sampling_start`, but stored explicitly
+ /// so it can be validated and used consistently by downstream encoders.
+ pub sampling_length: usize,
pub is_prefilling: bool,
}
@@ -49,6 +60,8 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {
token_seeds: self.token_seeds,
expected_number_of_new_tokens: self.expected_number_of_new_tokens,
active_suffix_length: self.active_suffix_length,
+ sampling_start: self.sampling_start,
+ sampling_length: self.sampling_length,
is_prefilling: self.is_prefilling,
}
}
@@ -58,8 +71,13 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {
tokens_count: usize,
) -> String {
format!(
- "tokens:{}_suffix:{}",
- tokens_count, self.expected_number_of_new_tokens
+ "tokens:{}_suffix:{}_active:{}_prefill:{}_sample_start:{}_sample_len:{}",
+ tokens_count,
+ self.expected_number_of_new_tokens,
+ self.active_suffix_length,
+ self.is_prefilling,
+ self.sampling_start,
+ self.sampling_length
)
}
@@ -80,6 +98,8 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {
self.token_bitmask,
self.token_seeds,
self.active_suffix_length,
+ self.sampling_start,
+ self.sampling_length,
self.is_prefilling,
external_bias_fn,
false,
diff --git a/crates/uzu/src/session/parameter/prefill_step_size.rs b/crates/uzu/src/session/parameter/prefill_step_size.rs
index 795a527b7..ef861b38d 100644
--- a/crates/uzu/src/session/parameter/prefill_step_size.rs
+++ b/crates/uzu/src/session/parameter/prefill_step_size.rs
@@ -2,6 +2,19 @@ use crate::{
config::LanguageModelConfig, session::parameter::ConfigResolvableValue,
};
+fn env_prefill_step_size_default_override() -> Option<usize> {
+ static OVERRIDE: std::sync::OnceLock<Option<usize>> =
+ std::sync::OnceLock::new();
+ *OVERRIDE.get_or_init(|| {
+ let raw = std::env::var("UZU_PREFILL_STEP_SIZE").ok()?;
+ let trimmed = raw.trim();
+ if trimmed.is_empty() {
+ return None;
+ }
+ trimmed.parse::<usize>().ok()
+ })
+}
+
#[derive(Debug, Clone, Copy)]
pub enum PrefillStepSize {
Default,
@@ -20,7 +33,7 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {
&self,
config: &LanguageModelConfig,
) -> usize {
- let default_limit: usize = 1024;
+ let default_limit: usize = 2048;
let model_context_length =
config.model_config.transformer_config.context_length;
@@ -44,7 +57,9 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {
let proposed_value = match self {
PrefillStepSize::Default => {
- if cfg!(target_os = "ios") {
+ if let Some(v) = env_prefill_step_size_default_override() {
+ v
+ } else if cfg!(target_os = "ios") {
64
} else {
256
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rs
index f0d480209..4b2b83d3e 100644
--- a/crates/uzu/src/tracer/trace_validator.rs
+++ b/crates/uzu/src/tracer/trace_validator.rs
@@ -286,6 +286,8 @@ impl TraceValidator {
None,
&token_seeds,
token_ids.len(),
+ /*sampling_start=*/ 0,
+ /*sampling_length=*/ token_ids.len(),
false,
None,
false,
diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rs
index 35d85d6d1..e241ab646 100644
--- a/crates/uzu/tests/attention_kernel_test.rs
+++ b/crates/uzu/tests/attention_kernel_test.rs
@@ -9,8 +9,8 @@ use ndarray::{Array2, Array3, Array4, s};
use uzu::backends::metal::{
KernelDataType, MTLContext,
kernel::attention::{
- AttentionKernel, AttentionKernelVariant, AttentionSinglePassArguments,
- AttentionTwoPassArguments,
+ AttentionGemmArguments, AttentionKernel, AttentionKernelVariant,
+ AttentionSinglePassArguments, AttentionTwoPassArguments,
},
metal_extensions::command_buffer_extensions::CommandBufferTimingAccess,
};
@@ -297,6 +297,26 @@ fn create_mask_buffer(
)
}
+fn create_mask_2d_buffer(
+ mask: &Array3<f32>,
+ context: &MTLContext,
+) -> metal::Buffer {
+ let (_batch_size, seq_len, _) = mask.dim();
+
+ let mut mask_data = vec![0.0f32; seq_len * seq_len];
+ for i in 0..seq_len {
+ for j in 0..seq_len {
+ mask_data[i * seq_len + j] = mask[[0, i, j]];
+ }
+ }
+
+ context.device.new_buffer_with_data(
+ mask_data.as_ptr() as *const _,
+ (mask_data.len() * size_of::<f32>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ )
+}
+
fn create_sinks_buffer(
sinks: &[f32],
context: &MTLContext,
@@ -404,6 +424,151 @@ fn run_single_pass_attention(
Ok(kernel_output)
}
+fn run_single_pass_attention_with_is_causal(
+ kernel: &AttentionKernel,
+ context: &MTLContext,
+ queries: &Array4<f32>,
+ keys: &Array4<f32>,
+ values: &Array4<f32>,
+ mask: Option<&Array3<f32>>,
+ sinks: Option<&[f32]>,
+ scale: f32,
+ is_causal: bool,
+) -> Result<Array4<f32>, Box<dyn std::error::Error>> {
+ let (batch_size, num_heads, seq_len, head_dim) = queries.dim();
+ let (_batch_size, num_kv_heads, _seq_len, _head_dim) = keys.dim();
+
+ let query_buffer = create_query_buffer(queries, context);
+ let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);
+ let value_cache_buffer =
+ create_value_cache_buffer(values, seq_len, context);
+
+ let mask_buffer = mask.map(|m| create_mask_buffer(m, num_heads, context));
+ let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));
+
+ let output_buffer = context.device.new_buffer(
+ (num_heads * seq_len * head_dim * size_of::<f32>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let command_buffer = context.command_queue.new_command_buffer();
+ let compute_encoder = command_buffer.new_compute_command_encoder();
+
+ let args = AttentionSinglePassArguments {
+ queries_buffer: &query_buffer,
+ keys_buffer: &key_cache_buffer,
+ values_buffer: &value_cache_buffer,
+ output_buffer: &output_buffer,
+ gqa_factor: (num_heads / num_kv_heads) as i32,
+ sequence_length: seq_len as i32,
+ k_head_stride: (seq_len * head_dim) as i32,
+ k_seq_stride: head_dim as i32,
+ v_head_stride: (seq_len * head_dim) as i32,
+ v_seq_stride: head_dim as i32,
+ scale,
+ mask_buffer: mask_buffer.as_ref(),
+ mask_kv_seq_stride: 1,
+ mask_q_seq_stride: seq_len as i32,
+ mask_head_stride: 0,
+ sinks_buffer: sinks_buffer.as_ref(),
+ num_heads,
+ suffix_length: seq_len,
+ head_dim,
+ is_causal,
+ };
+
+ kernel.encode_single_pass(&compute_encoder, args)?;
+
+ compute_encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
+
+ let output_ptr = output_buffer.contents() as *const f32;
+ let output_slice = unsafe {
+ std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim)
+ };
+
+ let kernel_output = convert_kernel_output(
+ output_slice,
+ batch_size,
+ num_heads,
+ seq_len,
+ head_dim,
+ );
+
+ Ok(kernel_output)
+}
+
+fn run_gemm_attention(
+ kernel: &AttentionKernel,
+ context: &MTLContext,
+ queries: &Array4<f32>,
+ keys: &Array4<f32>,
+ values: &Array4<f32>,
+ mask: Option<&Array3<f32>>,
+ sinks: Option<&[f32]>,
+ scale: f32,
+ is_causal: bool,
+) -> Result<Array4<f32>, Box<dyn std::error::Error>> {
+ let (batch_size, num_heads, seq_len, head_dim) = queries.dim();
+ let (_batch_size, num_kv_heads, _seq_len, _head_dim) = keys.dim();
+
+ let query_buffer = create_query_buffer(queries, context);
+ let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);
+ let value_cache_buffer =
+ create_value_cache_buffer(values, seq_len, context);
+
+ let mask_buffer = mask.map(|m| create_mask_2d_buffer(m, context));
+ let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));
+
+ let output_buffer = context.device.new_buffer(
+ (num_heads * seq_len * head_dim * size_of::<f32>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let command_buffer = context.command_queue.new_command_buffer();
+ let compute_encoder = command_buffer.new_compute_command_encoder();
+
+ let args = AttentionGemmArguments {
+ queries_buffer: &query_buffer,
+ keys_buffer: &key_cache_buffer,
+ values_buffer: &value_cache_buffer,
+ output_buffer: &output_buffer,
+ mask_buffer: mask_buffer.as_ref(),
+ sinks_buffer: sinks_buffer.as_ref(),
+ num_heads,
+ num_groups: num_kv_heads,
+ suffix_length: seq_len,
+ sequence_length: seq_len,
+ segment_prefix_length: 0,
+ max_sequence_length: seq_len,
+ head_dim,
+ is_causal,
+ scale,
+ };
+
+ kernel.encode_gemm(context, &compute_encoder, args)?;
+
+ compute_encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
+
+ let output_ptr = output_buffer.contents() as *const f32;
+ let output_slice = unsafe {
+ std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim)
+ };
+
+ let kernel_output = convert_kernel_output(
+ output_slice,
+ batch_size,
+ num_heads,
+ seq_len,
+ head_dim,
+ );
+
+ Ok(kernel_output)
+}
+
fn compare_results(
kernel_output: &Array4<f32>,
reference_output: &Array4<f32>,
@@ -555,6 +720,154 @@ fn test_single_pass_attention_basic() {
}
}
+#[test]
+fn test_gemm_attention_basic() {
+ let device = Device::system_default().expect("No Metal device found");
+ let command_queue = device.new_command_queue();
+ let context = match MTLContext::new(device, command_queue) {
+ Ok(ctx) => ctx,
+ Err(e) => {
+ panic!("Failed to create MTLContext: {:?}", e);
+ },
+ };
+
+ let batch_size = 1;
+ let num_heads = 4;
+ let num_kv_heads = 4;
+ let seq_len = 32;
+ let head_dim = 64;
+ let scale = 1.0 / (head_dim as f32).sqrt();
+
+ let (queries, keys, values, _mask) = create_test_data(
+ batch_size,
+ num_heads,
+ num_kv_heads,
+ seq_len,
+ head_dim,
+ 123,
+ );
+
+ let kernel = AttentionKernel::new(&context, KernelDataType::Float32)
+ .expect("Failed to create AttentionKernel");
+
+ let reference_output =
+ reference_attention(&queries, &keys, &values, None, None, scale);
+
+ let kernel_output = run_gemm_attention(
+ &kernel, &context, &queries, &keys, &values, None, None, scale,
+ /*is_causal=*/ false,
+ )
+ .expect("run gemm attention");
+
+ let tolerance = 1e-1;
+ if let Err(e) = compare_results(
+ &kernel_output,
+ &reference_output,
+ tolerance,
+ "Gemm attention",
+ ) {
+ panic!("{}", e);
+ }
+}
+
+#[test]
+fn test_matrix_attention_matches_vector_and_cpu_seq256() {
+ let device = Device::system_default().expect("No Metal device found");
+ let command_queue = device.new_command_queue();
+ let context = match MTLContext::new(device, command_queue) {
+ Ok(ctx) => ctx,
+ Err(e) => {
+ panic!("Failed to create MTLContext: {:?}", e);
+ },
+ };
+
+ let batch_size = 1;
+ let num_heads = 8;
+ let num_kv_heads = 2;
+ let seq_len = 256;
+ let head_dim = 64;
+ let scale = 1.0 / (head_dim as f32).sqrt();
+ let is_causal = true;
+
+ let (queries, keys, values, mask) = create_test_data(
+ batch_size,
+ num_heads,
+ num_kv_heads,
+ seq_len,
+ head_dim,
+ 2026,
+ );
+
+ let kernel = AttentionKernel::new(&context, KernelDataType::Float32)
+ .expect("Failed to create AttentionKernel");
+
+ // CPU reference (uses the same additive mask)
+ let reference_output =
+ reference_attention(&queries, &keys, &values, Some(&mask), None, scale);
+
+ // vector attention path (single-pass)
+ let vector_output = run_single_pass_attention_with_is_causal(
+ &kernel,
+ &context,
+ &queries,
+ &keys,
+ &values,
+ Some(&mask),
+ None,
+ scale,
+ is_causal,
+ )
+ .expect("run vector attention");
+
+ // matrix attention path (gemm)
+ let matrix_output = run_gemm_attention(
+ &kernel,
+ &context,
+ &queries,
+ &keys,
+ &values,
+ Some(&mask),
+ None,
+ scale,
+ is_causal,
+ )
+ .expect("run matrix attention");
+
+ // Compare both to CPU
+ let tol_cpu = 5e-2;
+ compare_results(
+ &vector_output,
+ &reference_output,
+ tol_cpu,
+ "vector single-pass attention vs CPU",
+ )
+ .unwrap();
+ compare_results(
+ &matrix_output,
+ &reference_output,
+ tol_cpu,
+ "matrix attention vs CPU",
+ )
+ .unwrap();
+
+ let max_diff_vector_matrix = vector_output
+ .iter()
+ .zip(matrix_output.iter())
+ .map(|(a, b)| (a - b).abs())
+ .fold(0.0f32, f32::max);
+ let tol_vector_matrix = 5e-2;
+ println!(
+ "Max absolute difference (vector single-pass attention vs matrix attention): {}",
+ max_diff_vector_matrix
+ );
+ assert!(
+ max_diff_vector_matrix <= tol_vector_matrix,
+ "vector single-pass attention and matrix attention differ too much: max_diff = {} (tol={})",
+ max_diff_vector_matrix,
+ tol_vector_matrix
+ );
+}
+
#[test]
fn test_single_pass_attention_with_mask() {
let device = Device::system_default().expect("No Metal device found");
diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rs
index cb2dd970e..397b05072 100644
--- a/crates/uzu/tests/matmul_correctness_test.rs
+++ b/crates/uzu/tests/matmul_correctness_test.rs
@@ -64,6 +64,7 @@ fn run_metal_matmul(
&enc,
MatmulArguments {
a: &a_buf,
+ a_offset: 0,
b: &b_buf,
c: None,
d: &d_buf,
diff --git a/crates/uzu/tests/quant_matmul_test.rs b/crates/uzu/tests/quant_matmul_test.rs
index d9c8687ce..528045382 100644
--- a/crates/uzu/tests/quant_matmul_test.rs
+++ b/crates/uzu/tests/quant_matmul_test.rs
@@ -538,6 +538,7 @@ fn execute_quantized_matmul(
for _ in 0..3 {
let args = QuantizedMatmulArguments {
a_buffer: &x_buf,
+ a_offset: 0,
b_buffer: &w_buf,
scales_buffer: &s_buf,
zero_points_or_biases_buffer: &b_buf,
@@ -560,6 +561,7 @@ fn execute_quantized_matmul(
for _ in 0..iterations {
let args = QuantizedMatmulArguments {
a_buffer: &x_buf,
+ a_offset: 0,
b_buffer: &w_buf,
scales_buffer: &s_buf,
zero_points_or_biases_buffer: &b_buf,
diff --git a/crates/uzu/tests/text_session_test.rs b/crates/uzu/tests/text_session_test.rs
index f531da86e..43f0776a8 100644
--- a/crates/uzu/tests/text_session_test.rs
+++ b/crates/uzu/tests/text_session_test.rs
@@ -18,8 +18,10 @@ fn build_decoding_config() -> DecodingConfig {
#[test]
fn test_text_session_base() {
- let text = String::from("Tell about London");
- run(text, build_decoding_config(), 128);
+ let text = String::from("
+London is one of the world’s most historically rich, culturally diverse, and economically influential cities. Located along the River Thames in southeastern England, it serves as the capital of both the United Kingdom and England. The city is known for its deep historical layers, ranging from Roman ruins and medieval fortresses to cutting-edge skyscrapers and modern cultural institutions.\n\n## 🌆 What London Is\nLondon is a global metropolis composed of 32 boroughs and the City of London, the latter being its historic financial core. It is home to over 9 million residents and attracts tens of millions of visitors each year. The city's character is shaped by its extraordinary diversity—people from hundreds of nationalities and cultures live, work, and study here. More than 300 languages are spoken, making London one of the most linguistically diverse cities on Earth.\n\n## 🏛️ History in Depth\n- **Roman Foundation (1st century CE):** The Romans founded Londinium around 47–50 CE. The city grew rapidly because of its strategic riverside location and became a major commercial center.\n- **Medieval Era:** After the fall of Rome, London persisted as a trading hub. In the medieval period it grew around institutions like Westminster Abbey and the Tower of London, becoming a political and economic heartland.\n- **Tudor & Stuart London:** The 16th and 17th centuries saw major cultural and political developments such as the reign of Henry VIII, the rise of Shakespeare’s theatre, the Great Plague (1665), and the Great Fire of London (1666), which reshaped the city.\n- **18th–19th Century Expansion:** During the Industrial Revolution and the height of the British Empire, London became the world’s largest city and a global center for finance, trade, and innovation.\n- **20th–21st Century:** London endured extensive bombings during World War II, leading to major rebuilding. In recent decades it has reinvented itself as a global powerhouse for finance, technology, media, arts, and higher education.\n\n## 🗽 Famous Places and Landmarks\nLondon’s landmarks span more than 1,000 years of architecture and history:\n- **Big Ben & the Houses of Parliament:** Iconic symbols of British democracy and Gothic Revival architecture.\n- **The Tower of London:** A UNESCO World Heritage Site built in the 11th century; home to the Crown Jewels.\n- **Tower Bridge:** A Victorian engineering marvel combining bascule and suspension bridge elements.\n- **Buckingham Palace:** The monarch’s official residence, famous for the Changing of the Guard ceremony.\n- **Westminster Abbey:** A historic church where kings and queens have been crowned since 1066.\n- **The British Museum:** Houses millions of artifacts from around the world, including the Rosetta Stone.\n- **The London Eye:** A giant observation wheel offering panoramic city views.\n- **St Paul's Cathedral:** An architectural masterpiece by Sir Christopher Wren, known for its dome.\n- **Trafalgar Square:** A major public square dominated by Nelson’s Column.\n- **The Shard:** One of Europe's tallest buildings, offering a modern contrast to London’s older skyline.\n\n## 🎭 Culture, Arts & Lifestyle\nLondon is a cultural superpower:\n- **Museums & Galleries:** The National Gallery, Tate Modern, Victoria & Albert Museum, Science Museum, Natural History Museum, and many others—many free to the public.\n- **Performing Arts:** The West End theatre district is internationally renowned; London also has major opera houses, symphony orchestras, dance companies, and fringe theatres.\n- **Music:** From classical concerts at the Royal Albert Hall to modern arenas like The O2, London has shaped genres from punk to grime.\n- **Food:** London's culinary scene is global—Michelin-starred restaurants, historic pubs, international street food markets, and diverse immigrant cuisines.\n- **Education:** Home to world-leading institutions like University College London (UCL), Imperial College, King’s College London, and the London School of Economics.\n\n## 🚇 Transport & Infrastructure\nLondon’s transport network is one of the most extensive on the planet:\n- **The London Underground (The Tube):** Opened in 1863, it is the world’s oldest metro system, serving 270 stations across 11 lines.\n- **Buses:** The red double-decker bus remains one of the city’s most recognizable symbols.\n- **Trains & Overground:** Extensive rail links connect Greater London and surrounding regions.\n- **River Thames Services:** High-speed and commuter boats offer scenic travel routes.\n- **Cycling:** London has expanded cycle paths and a popular bike-sharing scheme.\n\n## 🌳 Green Spaces & Nature\nDespite its density, London is considered one of the greenest major cities:\n- **Hyde Park:** Famous for its lakes, open lawns, and Speakers’ Corner.\n- **Regent’s Park:** Contains the London Zoo and manicured gardens.\n- **Richmond Park:** A vast royal park known for free-roaming deer.\n- **Hampstead Heath:** Offers woodlands, ponds, and views of the city skyline from Parliament Hill.\n- **Kew Gardens:** A world-famous botanical garden and UNESCO World Heritage Site.\n\n## 🌦️ Climate & Weather\nLondon’s weather is mild but unpredictable:\n- Winters are cool and damp, rarely extremely cold.\n- Summers are warm, with occasional heatwaves.\n- Rainfall is spread throughout the year, often in light drizzle.\n- The city's maritime climate means conditions can change quickly.\n\n## 🧭 Modern London Today\nLondon today is a global hub for:\n- Finance and banking (the City & Canary Wharf)\n- Technology and innovation (Silicon Roundabout, King’s Cross tech district)\n- Media, film, publishing, and fashion\n- Tourism and cultural exchange\n- International diplomacy and global business\n\nIts combination of history, diversity, and constant reinvention makes it one of the most unique cities in the world.\n\nIf you want, I can make it even more detailed—focusing on history, architecture, neighborhoods, economic data, travel guidance, or anything else! Modern London continues to evolve rapidly. New architecture rises beside medieval churches, technological innovation reshapes work and communication, and social change constantly challenges established norms. Yet the city’s identity remains rooted in contrast: old and new, local and global, tradition and reinvention.
+");
+ run(text, build_decoding_config(), 1280);
}
#[test]