A high-performance inference engine for AI models
Async decoding for transformers
LuckyIYI9f1f07ea186b
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/mod.rs b/crates/uzu/src/backends/metal/encodable_block/layer/mod.rsindex 15a5156fc..5aea3c675 100644--- a/crates/uzu/src/backends/metal/encodable_block/layer/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/layer/mod.rs@@ -5,4 +5,3 @@ mod mixer;pub use executables::LayerExecutables;pub(crate) use mixer::MixerExecutables;-diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rsindex b277e2f69..7497d8f95 100644--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs@@ -20,13 +20,13 @@ mod layer;mod linear;mod mamba_mixer;mod mlp;-mod short_conv_mixer;mod moe_block;mod normalization;mod pooling;mod prediction_head;mod rope;mod sampling;+mod short_conv_mixer;mod tensor_add_swap;mod tensor_copy;pub mod transformer_layer;@@ -43,7 +43,6 @@ pub use layer::LayerExecutables;pub use linear::QuantizedLinear;pub(crate) use mamba_mixer::MambaMixer;pub use mlp::MlpBlock;-pub(crate) use short_conv_mixer::ShortConvMixer;pub use moe_block::{MoeBlock, SharedMoeWeights};pub use normalization::{LayerNorm, Normalization, NormalizationError, QKNorm, RMSNorm,@@ -52,6 +51,7 @@ pub use pooling::Pooling;pub use prediction_head::ClassifierPredictionHead;pub use rope::Rope;pub use sampling::Sampling;+pub(crate) use short_conv_mixer::ShortConvMixer;pub use tensor_add_swap::TensorAddSwap;pub use tensor_copy::TensorCopy;diff --git a/crates/uzu/src/backends/metal/encodable_block/rope.rs b/crates/uzu/src/backends/metal/encodable_block/rope.rsindex e81874c6a..94b8731da 100644--- a/crates/uzu/src/backends/metal/encodable_block/rope.rs+++ b/crates/uzu/src/backends/metal/encodable_block/rope.rs@@ -85,6 +85,8 @@ impl EncodableBlock for Rope {command_buffer.root_command_buffer().to_owned();let compute_encoder = mtl_command_buffer.new_compute_command_encoder();+ let token_positions_offset = token_positions.buffer_offset();+self.kernel.encode(&compute_encoder,RopeKernelArguments {@@ -94,6 +96,7 @@ impl EncodableBlock for Rope {token_positions_buffer: unsafe {&token_positions.mtl_buffer()},+ token_positions_offset,rotated_queries_buffer: unsafe {&rotated_queries.mtl_buffer()},diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rsindex 5df658b94..ff919af81 100644--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs@@ -10,7 +10,8 @@ use crate::{compilation_parameters::CompilationConfig,forward_pass::{ArrayId, ForwardPassState},kernel::short_conv::{- ShortConvDecodeArguments, ShortConvKernel, ShortConvPrefillArguments,+ ShortConvDecodeArguments, ShortConvKernel,+ ShortConvPrefillArguments,},},config::{DecoderLayerType, ShortConvConfig},@@ -37,10 +38,7 @@ fn resolve_subtree<'tree>(return subtree;}}- panic!(- "Could not find any of {:?} in parameter tree",- candidates- );+ panic!("Could not find any of {:?} in parameter tree", candidates);}impl ShortConvMixer {@@ -64,8 +62,10 @@ impl ShortConvMixer {let mixer_tree = resolve_subtree(decoder_layer_loader, &["mixer"]);let conv_tree = resolve_subtree(&mixer_tree, &["conv"]);- let data_type: DataType =- short_conv_config.in_projection_config.activation_precision().into();+ let data_type: DataType = short_conv_config+ .in_projection_config+ .activation_precision()+ .into();let kernel_data_type: KernelDataType = data_type.into();let in_projection = transformer_layer::linear_block(@@ -99,8 +99,9 @@ impl ShortConvMixer {None};- let short_conv_kernel = ShortConvKernel::new(mtl_context, kernel_data_type)- .expect("Failed to create short conv kernel");+ let short_conv_kernel =+ ShortConvKernel::new(mtl_context, kernel_data_type)+ .expect("Failed to create short conv kernel");Self {layer_index,@@ -173,7 +174,8 @@ impl ShortConvMixer {let kernel_size = self.config.kernel_size;let state_stride = kernel_size.saturating_sub(1);- let mtl_command_buffer = command_buffer.root_command_buffer().to_owned();+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();let compute = mtl_command_buffer.new_compute_command_encoder();self.short_conv_kernel@@ -227,7 +229,8 @@ impl ShortConvMixer {let kernel_size = self.config.kernel_size;let state_stride = kernel_size.saturating_sub(1);- let mtl_command_buffer = command_buffer.root_command_buffer().to_owned();+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();let compute = mtl_command_buffer.new_compute_command_encoder();self.short_conv_kernel@@ -263,4 +266,3 @@ impl EncodableBlock for ShortConvMixer {self.encode_pipeline(state, command_buffer, parameters);}}-diff --git a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs b/crates/uzu/src/backends/metal/forward_pass/cache_layers.rsindex a7b4cff7b..b418aa226 100644--- a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs+++ b/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs@@ -4,7 +4,10 @@ use metal::CommandBuffer as MTLCommandBuffer;use super::{super::{MTLContext, MetalArray},- kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState},+ kv_cache_layer::{+ AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState,+ KVSlice,+ },model_shape::ModelShape,short_conv_layer::ShortConvLayer,ssm_layer::SSMLayer,@@ -21,6 +24,13 @@ pub enum CacheLayer {ShortConv(ShortConvLayer),}+#[derive(Clone)]+pub enum CacheLayerSlice {+ Transformer(KVSlice),+ StateSpace,+ ShortConv,+}+impl CacheLayer {pub fn as_transformer(&self) -> Option<&KVCacheLayer> {match self {@@ -71,6 +81,11 @@ pub struct CacheLayers {pub data: Box<[CacheLayer]>,}+#[derive(Clone)]+pub struct CacheLayersSlice {+ pub layers: Vec<CacheLayerSlice>,+}+impl CacheLayers {pub fn new(context: &MTLContext,@@ -277,6 +292,69 @@ impl CacheLayers {}}+ pub fn attention_bias_updates_after_acceptance(+ &self,+ accepted_len: usize,+ ) -> Vec<AttentionBiasUpdate> {+ self.data+ .iter()+ .filter_map(|layer| match layer {+ CacheLayer::Transformer(kv) => {+ kv.attention_bias_update_after_acceptance(accepted_len)+ },+ _ => None,+ })+ .collect()+ }++ pub fn slice(+ &self,+ context: &MTLContext,+ range: std::ops::Range<usize>,+ ) -> Option<CacheLayersSlice> {+ let mut layers = Vec::with_capacity(self.data.len());+ for layer in self.data.iter() {+ match layer {+ CacheLayer::Transformer(kv) => {+ let Some(slice) = kv.slice(context, range.clone()) else {+ return None;+ };+ layers.push(CacheLayerSlice::Transformer(slice));+ },+ CacheLayer::StateSpace(_) => {+ layers.push(CacheLayerSlice::StateSpace)+ },+ CacheLayer::ShortConv(_) => {+ layers.push(CacheLayerSlice::ShortConv)+ },+ }+ }++ Some(CacheLayersSlice {+ layers,+ })+ }++ pub fn apply_slice(+ &mut self,+ slice: &CacheLayersSlice,+ range: Option<std::ops::Range<usize>>,+ ) {+ for (layer, snapshot) in self.data.iter_mut().zip(slice.layers.iter()) {+ match (layer, snapshot) {+ (+ CacheLayer::Transformer(kv),+ CacheLayerSlice::Transformer(s),+ ) => {+ kv.apply_slice(s, range.clone());+ },+ (CacheLayer::StateSpace(_), CacheLayerSlice::StateSpace) => {},+ (CacheLayer::ShortConv(_), CacheLayerSlice::ShortConv) => {},+ _ => {},+ }+ }+ }+pub fn clone(&self,context: &MTLContext,diff --git a/crates/uzu/src/backends/metal/forward_pass/kv_cache_layer.rs b/crates/uzu/src/backends/metal/forward_pass/kv_cache_layer.rsindex 558d865b8..5ac083771 100644--- a/crates/uzu/src/backends/metal/forward_pass/kv_cache_layer.rs+++ b/crates/uzu/src/backends/metal/forward_pass/kv_cache_layer.rs@@ -11,6 +11,31 @@ use crate::{pub type ArrayCell = RefCell<MetalArray>;+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub struct AttentionBiasUpdate {+ pub key: Option<usize>,+ pub unmask_col: i32,+ pub mask_col: i32,+}++#[derive(Clone)]+pub enum KVSlice {+ Full {+ base_prefix_len: usize,+ base_positions_len: usize,+ positions: Vec<usize>,+ },+ Window {+ window_length: usize,+ base_ring_offset: usize,+ base_ring_length: usize,+ slots: Vec<usize>,+ positions: Vec<usize>, // per slot+ keys: MetalArray, // [num_groups, slots.len(), head_dim]+ values: MetalArray, // [num_groups, slots.len(), head_dim]+ },+}+#[derive(Clone, Debug)]pub enum KVCacheLayerState {Full {@@ -292,4 +317,221 @@ impl KVCacheLayer {},}}++ pub fn attention_bias_update_after_acceptance(+ &self,+ accepted_len: usize,+ ) -> Option<AttentionBiasUpdate> {+ if accepted_len != 1 {+ return None;+ }++ match self.state {+ KVCacheLayerState::Full {+ ..+ } => None,+ KVCacheLayerState::Windowed {+ ring_offset,+ ring_length,+ window_length,+ } => {+ let newest_slot =+ (ring_offset + window_length - 1) % window_length;+ let unmask_col = (ring_length > 0)+ .then_some(newest_slot as i32)+ .unwrap_or(-1);+ let mask_col = (ring_length == window_length)+ .then_some(ring_offset as i32)+ .unwrap_or(-1);++ Some(AttentionBiasUpdate {+ key: Some(window_length),+ unmask_col,+ mask_col,+ })+ },+ }+ }++ pub fn slice(+ &self,+ context: &MTLContext,+ range: std::ops::Range<usize>,+ ) -> Option<KVSlice> {+ match self.state {+ KVCacheLayerState::Full {+ prefix_len,+ } => Some(KVSlice::Full {+ base_prefix_len: prefix_len,+ base_positions_len: self.prefix_token_positions.len(),+ positions: self.prefix_token_positions.clone(),+ }),+ KVCacheLayerState::Windowed {+ ring_offset,+ ring_length,+ window_length,+ } => {+ let len = range.end.saturating_sub(range.start);+ if len == 0 || len > window_length {+ return None;+ }+ let keys = self.keys.borrow();+ let values = self.values.borrow();+ let shape = keys.shape();+ let num_groups = shape[0];+ let head_dim = shape[2];+ let dtype = keys.data_type();++ let slice_shape = [num_groups, len, head_dim];+ let mut slice_keys = context.array(&slice_shape, dtype);+ let mut slice_values = context.array(&slice_shape, dtype);++ let slots: Vec<usize> = (range.start..range.end)+ .enumerate()+ .map(|(offset, _)| {+ let x = ring_length + offset;+ if x < window_length {+ (ring_offset + x) % window_length+ } else {+ (ring_offset + (x - window_length)) % window_length+ }+ })+ .collect();++ let positions: Vec<usize> = slots+ .iter()+ .map(|&s| self.prefix_token_positions[s])+ .collect();++ for (i, &slot) in slots.iter().enumerate() {+ slice_keys.copy_slice(&keys, 1, slot..slot + 1, i);+ slice_values.copy_slice(&values, 1, slot..slot + 1, i);+ }++ Some(KVSlice::Window {+ window_length,+ base_ring_offset: ring_offset,+ base_ring_length: ring_length,+ slots,+ positions,+ keys: slice_keys,+ values: slice_values,+ })+ },+ }+ }++ pub fn apply_slice(+ &mut self,+ slice: &KVSlice,+ range: Option<std::ops::Range<usize>>,+ ) {+ match (slice, &mut self.state) {+ (+ KVSlice::Full {+ base_prefix_len,+ base_positions_len,+ positions,+ },+ KVCacheLayerState::Full {+ prefix_len,+ },+ ) => match range {+ None => {+ *prefix_len = *base_prefix_len;+ self.prefix_token_positions.clone_from(positions);+ self.prefix_token_positions.truncate(*base_positions_len);+ },+ Some(r) => {+ let accepted = r.start;+ *prefix_len = base_prefix_len.saturating_add(accepted);+ let keep_positions =+ base_positions_len.saturating_add(accepted);+ self.prefix_token_positions.truncate(keep_positions);+ },+ },+ (+ KVSlice::Window {+ window_length,+ base_ring_offset,+ base_ring_length,+ slots,+ positions,+ keys,+ values,+ },+ KVCacheLayerState::Windowed {+ ring_offset,+ ring_length,+ window_length: w_len,+ },+ ) => {+ *w_len = *window_length;+ match range {+ None => {+ *ring_offset = *base_ring_offset;+ *ring_length = *base_ring_length;++ for (i, &slot) in slots.iter().enumerate() {+ self.prefix_token_positions[slot] = positions[i];+ }++ let mut dst_keys = self.keys.borrow_mut();+ let mut dst_values = self.values.borrow_mut();+ for (i, &slot) in slots.iter().enumerate() {+ dst_keys.copy_slice(keys, 1, i..i + 1, slot);+ dst_values.copy_slice(values, 1, i..i + 1, slot);+ }+ },+ Some(r) => {+ let len = r.end.saturating_sub(r.start);+ if len == 0 {+ return;+ }++ let accepted = r.start;+ let base_len = *base_ring_length;+ let w = *window_length;++ let (new_offset, new_len) = if base_len < w {+ let after = base_len.saturating_add(accepted);+ if after <= w {+ (*base_ring_offset, after)+ } else {+ let overflow = after - w;+ ((base_ring_offset + overflow) % w, w)+ }+ } else {+ ((base_ring_offset + accepted) % w, w)+ };++ *ring_offset = new_offset;+ *ring_length = new_len;++ let mut dst_keys = self.keys.borrow_mut();+ let mut dst_values = self.values.borrow_mut();++ for (i, &slot) in slots[r.clone()].iter().enumerate() {+ let src_i = r.start + i;+ self.prefix_token_positions[slot] =+ positions[src_i];+ dst_keys.copy_slice(+ keys,+ 1,+ src_i..src_i + 1,+ slot,+ );+ dst_values.copy_slice(+ values,+ 1,+ src_i..src_i + 1,+ slot,+ );+ }+ },+ }+ },+ _ => {},+ }+ }}diff --git a/crates/uzu/src/backends/metal/forward_pass/mod.rs b/crates/uzu/src/backends/metal/forward_pass/mod.rsindex 4be2ae19b..37b24c7d7 100644--- a/crates/uzu/src/backends/metal/forward_pass/mod.rs+++ b/crates/uzu/src/backends/metal/forward_pass/mod.rs@@ -14,7 +14,9 @@ pub mod traces;pub use cache_layers::{CacheLayer, CacheLayers};pub use encoder_resolver::EncoderResolver;pub use io_arrays::IOArrays;-pub use kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState};+pub use kv_cache_layer::{+ AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState,+};pub use model_shape::ModelShape;pub use mpsgraph_block::MPSGraphBlock;pub use scratch_buffers::ScratchBuffers;diff --git a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs b/crates/uzu/src/backends/metal/forward_pass/model_shape.rsindex c6ede5ae2..b537b81dd 100644--- a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs+++ b/crates/uzu/src/backends/metal/forward_pass/model_shape.rs@@ -489,9 +489,9 @@ impl ModelShape {let inner_dim = num_heads * head_dim;Some(conv_dim + inner_dim + num_heads)},- DecoderLayerType::ShortConv { .. } => {- Some(self.model_dim * 3)- },+ DecoderLayerType::ShortConv {+ ..+ } => Some(self.model_dim * 3),_ => None,}).max()diff --git a/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs b/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rsindex b9fd49008..f285c3f6e 100644--- a/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs+++ b/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs@@ -5,8 +5,7 @@ use mpsgraph::{use objc2::rc::Retained;use super::{- EncodableBlock, EncodingParameters, ForwardPassState,- io_arrays::IOArrays,+ EncodableBlock, EncodingParameters, ForwardPassState, io_arrays::IOArrays,};pub struct MPSGraphBlock {diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs b/crates/uzu/src/backends/metal/forward_pass/state/mod.rsindex 2468fd782..03b98b33a 100644--- a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs+++ b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs@@ -109,6 +109,7 @@ impl ForwardPassState {is_prefilling: bool,external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,skip_token_ids_copy: bool,+ skip_attention_bias_fill: bool,async_positions: Option<(&metal::Buffer, usize)>,async_seeds: Option<(&metal::Buffer, usize)>,) -> Self {@@ -220,6 +221,7 @@ impl ForwardPassState {act_dtype,token_positions,external_bias_fn,+ skip_attention_bias_fill,);// Common aux buffers@@ -277,6 +279,7 @@ impl ForwardPassState {act_dtype: DataType,token_positions: &[usize],external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,+ skip_fill: bool,) -> HashMap<Option<usize>, ArrayCell> {let cache_ref = cache_layers.borrow();let mut attention_bias_map: HashMap<Option<usize>, MetalArray> =@@ -302,13 +305,16 @@ impl ForwardPassState {// Use cache_layers' fill_attention_bias which properly handles// both causal masking and sliding window constraints- cache_layers.borrow().fill_attention_bias(- &mut attention_bias_map,- token_positions,- suffix_length,- context,- external_bias_fn,- );+ // Skip fill for async decode passes after the first one (bias already set)+ if !skip_fill {+ cache_layers.borrow().fill_attention_bias(+ &mut attention_bias_map,+ token_positions,+ suffix_length,+ context,+ external_bias_fn,+ );+ }attention_bias_map.into_iter()diff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/mod.rs b/crates/uzu/src/backends/metal/kernel/kv_cache_update/mod.rsindex d9252733c..3bd31ae28 100644--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/kv_cache_update/mod.rs@@ -116,10 +116,16 @@ impl KVCacheUpdate {return Err(KVCacheUpdateError::MaxSequenceLengthExceeded);}- let indices_ptr = self.indices_buffer.contents() as *mut Swap;- unsafe {- for (i, swap) in swaps.iter().enumerate() {- *indices_ptr.add(i) = *swap;+ // For small swap counts, use set_bytes to avoid shared buffer race in async mode.+ // Metal's set_bytes limit is 4KB; 32 swaps = 256 bytes, well under limit.+ let use_inline_bytes = swaps.len() <= 32;++ if !use_inline_bytes {+ let indices_ptr = self.indices_buffer.contents() as *mut Swap;+ unsafe {+ for (i, swap) in swaps.iter().enumerate() {+ *indices_ptr.add(i) = *swap;+ }}}@@ -134,7 +140,15 @@ impl KVCacheUpdate {// Set buffers and parameterscompute_encoder.set_buffer(0, Some(&layer_data.key_buffer), 0);compute_encoder.set_buffer(1, Some(&layer_data.value_buffer), 0);- compute_encoder.set_buffer(2, Some(&self.indices_buffer), 0);+ if use_inline_bytes {+ compute_encoder.set_bytes(+ 2,+ (swaps.len() * size_of::<Swap>()) as u64,+ swaps.as_ptr() as *const std::ffi::c_void,+ );+ } else {+ compute_encoder.set_buffer(2, Some(&self.indices_buffer), 0);+ }compute_encoder.set_bytes(3,size_of::<i32>() as u64,@@ -194,78 +208,6 @@ pub fn create_swaps_direct(swaps}-/// Original implementation using cycles - kept for reference-/// This function generates the minimum set of swaps needed to move elements-/// from source positions to destination positions.-pub fn create_swaps(- source_indices: &[usize],- destination_indices: &[usize],-) -> Vec<Swap> {- // The two arrays must be of the same length.- if source_indices.len() != destination_indices.len() {- return Vec::new();- }-- // Build mappings between source and destination indices- let mut forward_mapping = std::collections::HashMap::new();- let mut reverse_mapping = std::collections::HashMap::new();-- for (&s, &d) in source_indices.iter().zip(destination_indices.iter()) {- if s != d {- forward_mapping.insert(s, d);- reverse_mapping.insert(d, s);- }- }-- let mut swaps = Vec::new();- let mut visited = std::collections::HashSet::new();-- // Process keys in sorted order- let all_keys: std::collections::HashSet<_> =- forward_mapping.keys().chain(reverse_mapping.keys()).copied().collect();-- let all_keys_vec: Vec<usize> = all_keys.into_iter().collect();- for &s in &all_keys_vec {- if visited.contains(&s) {- continue;- }-- visited.insert(s);- let mut cycle = vec![s];- let mut iterator = s;-- while let Some(&next) = forward_mapping.get(&iterator) {- iterator = next;- if visited.contains(&iterator) {- break;- }- visited.insert(iterator);- cycle.push(iterator);- }-- // Reverse direction- iterator = s;- while let Some(&next) = reverse_mapping.get(&iterator) {- iterator = next;- if visited.contains(&iterator) {- break;- }- visited.insert(iterator);- cycle.push(iterator);- }-- let first = cycle[0];- for &next in cycle.iter().skip(1) {- swaps.push(Swap {- source: first as i32,- destination: next as i32,- });- }- }-- swaps-}-#[cfg(test)]mod tests {use super::*;diff --git a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metalnew file mode 100644index 000000000..babdccb16--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal@@ -0,0 +1,32 @@+#include <metal_stdlib>+using namespace metal;++template <typename T>+kernel void update_attention_mask(+ device T* mask [[buffer(0)]],+ constant int& unmask_col [[buffer(1)]],+ constant int& mask_col [[buffer(2)]],+ uint tid [[thread_position_in_grid]]+) {+ if (tid == 0) {+ if (unmask_col >= 0) {+ mask[unmask_col] = T(0);+ }+ if (mask_col >= 0) {+ mask[mask_col] = -numeric_limits<T>::infinity();+ }+ }+}++#define instantiate_mask_update(name, type) \+ template [[host_name("update_attention_mask_" #name)]] \+ kernel void update_attention_mask<type>( \+ device type* mask [[buffer(0)]], \+ constant int& unmask_col [[buffer(1)]], \+ constant int& mask_col [[buffer(2)]], \+ uint tid [[thread_position_in_grid]] \+ );++instantiate_mask_update(float, float)+instantiate_mask_update(half, half)+instantiate_mask_update(bfloat, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/mask_update/mod.rs b/crates/uzu/src/backends/metal/kernel/mask_update/mod.rsnew file mode 100644index 000000000..9889bb2a1--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/mask_update/mod.rs@@ -0,0 +1,50 @@+use metal::{+ Buffer as MTLBuffer, ComputeCommandEncoderRef,+ ComputePipelineState as MTLComputePipelineState, MTLSize,+};++use crate::backends::metal::{MTLContext, kernel::KernelDataType};++pub struct MaskUpdateKernel {+ pipeline: MTLComputePipelineState,+}++impl MaskUpdateKernel {+ pub fn new(+ context: &MTLContext,+ data_type: KernelDataType,+ ) -> Result<Self, crate::backends::metal::error::MTLError> {+ let function_name = format!(+ "update_attention_mask_{}",+ data_type.function_name_suffix()+ );+ let (pipeline, _) = context+ .compute_pipeline_state_with_reflection(&function_name, None)?;++ Ok(Self {+ pipeline,+ })+ }++ pub fn encode(+ &self,+ encoder: &ComputeCommandEncoderRef,+ mask_buffer: &MTLBuffer,+ unmask_col: i32,+ mask_col: i32,+ ) {+ encoder.set_compute_pipeline_state(&self.pipeline);+ encoder.set_buffer(0, Some(mask_buffer), 0);+ encoder.set_bytes(+ 1,+ std::mem::size_of::<i32>() as u64,+ &unmask_col as *const i32 as *const _,+ );+ encoder.set_bytes(+ 2,+ std::mem::size_of::<i32>() as u64,+ &mask_col as *const i32 as *const _,+ );+ encoder.dispatch_threads(MTLSize::new(1, 1, 1), MTLSize::new(1, 1, 1));+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp/mod.rsindex 400354529..aefaff465 100644--- a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs@@ -53,7 +53,9 @@ impl MlpGateActMulEncodable {}impl MlpGateActMulKernel {- fn kernel_name_for_type(data_type: DataType) -> Result<&'static str, MTLError> {+ fn kernel_name_for_type(+ data_type: DataType+ ) -> Result<&'static str, MTLError> {match data_type {DataType::F16 => Ok("mlp_activation_mul_f16"),DataType::F32 => Ok("mlp_activation_mul_f32"),@@ -65,15 +67,22 @@ impl MlpGateActMulKernel {}}- pub fn new(context: &MTLContext, data_type: DataType) -> Result<Self, MTLError> {+ pub fn new(+ context: &MTLContext,+ data_type: DataType,+ ) -> Result<Self, MTLError> {let fn_name = Self::kernel_name_for_type(data_type)?;let pipeline = context.compute_pipeline_state(fn_name, None)?;- Ok(Self { pipeline })+ Ok(Self {+ pipeline,+ })}fn act_code(act: &Activation) -> u16 {match act {- Activation::SiLU { .. } => 0,+ Activation::SiLU {+ ..+ } => 0,Activation::Gelu => 1,Activation::Identity => {panic!("Identity activation is not supported for MLP kernels")diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex 294c76591..bcf40230c 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -3,6 +3,7 @@ mod data_type;pub mod embedding;pub mod kv_cache_update;pub mod layer_norm;+pub mod mask_update;pub mod media_kernels;pub mod mlp;pub mod moe;@@ -27,6 +28,7 @@ pub use attention::{pub use data_type::KernelDataType;pub use kv_cache_update::KVCacheUpdate;pub use layer_norm::{LayerNormArguments, LayerNormError, LayerNormKernel};+pub use mask_update::MaskUpdateKernel;pub use moe::{MoeBlockBasesArguments, MoeCountsOffsetsFusedArguments,MoeCountsOffsetsFusedError, MoeCountsOffsetsFusedKernel,diff --git a/crates/uzu/src/backends/metal/kernel/rope/mod.rs b/crates/uzu/src/backends/metal/kernel/rope/mod.rsindex 5ece64af7..9b05d911c 100644--- a/crates/uzu/src/backends/metal/kernel/rope/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/rope/mod.rs@@ -26,6 +26,7 @@ pub struct RopeKernelArguments<'a> {pub cosines_buffer: &'a MTLBuffer, // buffer(1)pub sines_buffer: &'a MTLBuffer, // buffer(2)pub token_positions_buffer: &'a MTLBuffer, // buffer(3)+ pub token_positions_offset: usize, // byte offset into token_positions_bufferpub rotated_queries_buffer: &'a MTLBuffer, // buffer(4)pub rotated_keys_buffer: &'a MTLBuffer, // buffer(5)pub head_dim: usize,@@ -68,7 +69,11 @@ impl RopeKernel {compute_encoder.set_buffer(0, Some(args.qkv_buffer), 0);compute_encoder.set_buffer(1, Some(args.cosines_buffer), 0);compute_encoder.set_buffer(2, Some(args.sines_buffer), 0);- compute_encoder.set_buffer(3, Some(args.token_positions_buffer), 0);+ compute_encoder.set_buffer(+ 3,+ Some(args.token_positions_buffer),+ args.token_positions_offset as u64,+ );compute_encoder.set_buffer(4, Some(args.rotated_queries_buffer), 0);compute_encoder.set_buffer(5, Some(args.rotated_keys_buffer), 0);diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rsindex 5b2cc91f1..02de18eba 100644--- a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs@@ -189,4 +189,3 @@ impl ShortConvKernel {Ok(())}}-diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rsindex 7145e7278..1dae0f2c3 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs@@ -5,7 +5,9 @@ use metal::{ComputePipelineState as MTLComputePipelineState,};-use super::{KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch};+use super::{+ KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch,+};use crate::backends::metal::error::MTLError;#[derive(Debug)]diff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rsindex dd67c59ad..743ff1d23 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs@@ -5,7 +5,9 @@ use metal::{ComputePipelineState as MTLComputePipelineState,};-use super::{KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch};+use super::{+ KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch,+};use crate::backends::metal::error::MTLError;pub struct TensorCopyKernel {diff --git a/crates/uzu/src/backends/metal/language_model_generator_context.rs b/crates/uzu/src/backends/metal/language_model_generator_context.rsindex 97b1bc1a6..3b85afdc4 100644--- a/crates/uzu/src/backends/metal/language_model_generator_context.rs+++ b/crates/uzu/src/backends/metal/language_model_generator_context.rs@@ -16,11 +16,11 @@ use super::{compilation_parameters::CompilationConfig,encodable_block::Sampling,forward_pass::{ScratchBuffers, SharedBuffers},- kernel::TokenCopyKernel,+ kernel::{MaskUpdateKernel, TokenCopyKernel},};use crate::{- DataType, DecoderConfig,- config::{LanguageModelConfig, ModelMetadata},+ DataType,+ config::{DecoderConfig, LanguageModelConfig, ModelMetadata},language_model::rng::DerivableSeed,parameters::ParameterLoader,session::{@@ -147,6 +147,8 @@ pub struct LanguageModelGeneratorContext {/// Kernel for copying sampled tokens in async pipelinepub token_copy: TokenCopyKernel,+ /// Kernel for updating attention mask between async passes+ pub mask_update: Option<MaskUpdateKernel>,/// Pre-allocated buffers for async generationpub async_buffers: AsyncBuffers,}@@ -263,6 +265,16 @@ impl LanguageModelGeneratorContext {let token_copy = TokenCopyKernel::new(&mtl_context).map_err(|_| Error::UnableToCreateMetalContext)?;+ // Create mask update kernel if model has attention layers+ let mask_update = if decoder_config.has_attention_layers() {+ Some(+ MaskUpdateKernel::new(&mtl_context, kernel_data_type)+ .map_err(|_| Error::UnableToCreateMetalContext)?,+ )+ } else {+ None+ };+let async_batch_size =decoding_config.async_batch_size.resolve(model_path);let async_buffers = AsyncBuffers::new(@@ -288,6 +300,7 @@ impl LanguageModelGeneratorContext {gpu_sampler,next_seed,token_copy,+ mask_update,async_buffers,};diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex 5a0af9763..20ef6409d 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -16,7 +16,8 @@ use super::{use crate::{Array,backends::metal::forward_pass::{- EncodableBlock, EncodingParameters, ForwardPassState, INVALID_POSITION,+ AttentionBiasUpdate, EncodableBlock, EncodingParameters,+ ForwardPassState, INVALID_POSITION,},session::{config::DecodingConfig,@@ -408,6 +409,19 @@ impl LanguageModelGenerator {tokens_to_generate: usize,) {let prefill_count = self.tokens.len();++ // Initialize attention bias buffers to zero for async mode+ // This ensures unwritten columns (beyond what pass 0 fills) are 0 (attend)+ // rather than garbage (which could be -inf and mask incorrectly)+ for (_, mask_buffer) in+ &self.context.scratch_buffers.attention_window_size_to_bias+ {+ unsafe {+ let ptr = mask_buffer.contents() as *mut u8;+ std::ptr::write_bytes(ptr, 0, mask_buffer.length() as usize);+ }+ }+self.context.async_buffers.prepare_positions(prefill_count, tokens_to_generate);@@ -451,13 +465,29 @@ impl LanguageModelGenerator {let last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;- let token_ids: [u64; 1] = [last_token];- let token_positions: [usize; 1] = [0]; // Ignored, using async buffer- let token_seeds: [u64; 1] = [0]; // Ignored, using async buffer+ let token_position = unsafe {+ let ptr = async_positions_buffer.contents() as *const u32;+ *ptr.add(pass_idx) as usize+ };++ let task = LanguageModelGeneratorRunTask {+ token_ids: &[last_token],+ token_positions: &[token_position],+ token_bitmask: None,+ token_seeds: &[0], // Ignored, using async buffer+ expected_number_of_new_tokens: 1,+ active_suffix_length: 1,+ is_prefilling: false,+ };let async_positions = Some((&async_positions_buffer, pass_idx));let async_seeds = Some((&async_seeds_buffer, pass_idx));+ let skip_attention_bias_fill =+ pass_idx > 0 && self.context.decoder_config.has_attention_layers();++ let skip_token_ids_copy = pass_idx > 0;+let mut state = ForwardPassState::new_llm(self.context.mtl_context.clone(),&self.context.decoder_config,@@ -465,19 +495,20 @@ impl LanguageModelGenerator {&self.context.scratch_buffers,self.context.cache_layers.clone(),self.context.shared_buffers.clone(),- &token_ids,- &token_positions,- None, // token_bitmask - ignored, using async buffer- &token_seeds,- 1, // active_suffix_length- false, // is_prefilling- None, // external_bias_fn- is_continuation,+ &task.token_ids,+ &task.token_positions,+ task.token_bitmask,+ &task.token_seeds,+ task.active_suffix_length,+ task.is_prefilling,+ None,+ skip_token_ids_copy,+ skip_attention_bias_fill,async_positions,async_seeds,);- if let Some(method) = state.sampling_method_mut() {- *method = Some(sampling_method);+ if let Some(sm) = state.sampling_method_mut() {+ *sm = Some(sampling_method);}self.context.reset_command_buffer();@@ -490,7 +521,6 @@ impl LanguageModelGenerator {.encode_wait_for_event(&async_event, current_counter);}- // Encode forward passself.context.executables.encode(&mut state,&self.context.command_buffer,@@ -527,6 +557,49 @@ impl LanguageModelGenerator {);encoder.end_encoding();+ // Scatter + register for all transformer layers+ {+ let cb = root_command_buffer.to_owned();+ self.context.cache_layers.borrow_mut().update_after_acceptance(+ &[0],+ None,+ &cb,+ &self.context.kv_cache_update,+ );+ self.context+ .cache_layers+ .borrow_mut()+ .register_accepted_tokens(&[token_position]);+ }++ let encoder = root_command_buffer.new_compute_command_encoder();++ if let Some(mask_update) = &self.context.mask_update {+ let updates: Vec<AttentionBiasUpdate> = self+ .context+ .cache_layers+ .borrow()+ .attention_bias_updates_after_acceptance(1);+ for (window_size, mask_buffer) in+ &self.context.scratch_buffers.attention_window_size_to_bias+ {+ if let Some(update) =+ updates.iter().find(|u| &u.key == window_size)+ {+ if update.unmask_col >= 0 || update.mask_col >= 0 {+ mask_update.encode(+ encoder,+ mask_buffer,+ update.unmask_col,+ update.mask_col,+ );+ }+ }+ }+ }++ encoder.end_encoding();+// Signal event for next passlet next_counter = current_counter + 1;root_command_buffer.encode_signal_event(&async_event, next_counter);diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rsindex 0be74166e..2ce090ee7 100644--- a/crates/uzu/src/language_model/tasks.rs+++ b/crates/uzu/src/language_model/tasks.rs@@ -83,6 +83,7 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {self.is_prefilling,external_bias_fn,false,+ false,None,None,)diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rsindex 4de7b809f..672c68560 100644--- a/crates/uzu/src/session/chat_session.rs+++ b/crates/uzu/src/session/chat_session.rs@@ -268,8 +268,11 @@ impl ChatSession {F: Fn(Output) -> bool,{let run_start = Instant::now();- let text =- self.input_processor.process(&input, config.enable_thinking)?;+ let text = self.input_processor.process(+ &input,+ config.enable_thinking,+ config.tokens_limit > 0,+ )?;let tokens: Vec<u64> = self.tokenizer.encode(text.as_str(), false)@@ -296,7 +299,7 @@ impl ChatSession {}let prefix_offset = language_model_generator.tokens.len();- let prefix_len_before = prefix_offset.saturating_sub(1);+ let prefix_len_before = prefix_offset;let eos_tokens: Vec<u64> = language_model_config.generation_config@@ -393,7 +396,6 @@ impl ChatSession {let can_use_async =language_model_generator.decoding_config.generate_suffix_length()== 1- && !language_model_generator.has_attention_layers()&& compiled_grammar.is_none();let generate_output = if can_use_async {@@ -539,6 +541,15 @@ impl ChatSession {while in_flight > 0 || next_to_submit < tokens_to_generate {let batch_end =std::cmp::min(next_to_submit + batch_size, tokens_to_generate);+ let batch_submitted = batch_end - next_to_submit;+ let accepted_before_batch = results.len();++ let cache_slice = llm+ .context+ .cache_layers+ .borrow()+ .slice(&llm.context.mtl_context, 0..batch_submitted);+for idx in next_to_submit..batch_end {let batch_sender = sender.clone();let last_callback_nanos = last_nanos.clone();@@ -556,7 +567,6 @@ impl ChatSession {})?;in_flight += 1;}- let batch_submitted = batch_end - next_to_submit;next_to_submit = batch_end;for _ in 0..batch_submitted {@@ -604,6 +614,16 @@ impl ChatSession {let _ = receiver.recv();in_flight -= 1;}+ if let Some(slice) = cache_slice {+ let accepted_in_batch =+ results.len().saturating_sub(accepted_before_batch);+ if accepted_in_batch < batch_submitted {+ llm.context.cache_layers.borrow_mut().apply_slice(+ &slice,+ Some(accepted_in_batch..batch_submitted),+ );+ }+ }return Self::build_output(model_metadata,tokenizer,@@ -749,14 +769,12 @@ impl ChatSession {fn decode_generated_tokens(tokenizer: &Tokenizer,- language_model_generator: &LanguageModelGenerator,- run_context: &RunContext,+ prefill_tokens: &[u64],+ generate_results: &[GenerateResult],) -> Result<String, Error> {- let start_idx =- run_context.prefix_len_before + run_context.input_tokens_len;- let generated_tokens: Vec<u32> = language_model_generator.tokens- [start_idx..]+ let generated_tokens: Vec<u32> = prefill_tokens.iter()+ .chain(generate_results.iter().flat_map(|r| r.tokens.iter())).map(|&v| v as u32).collect();tokenizer@@ -776,8 +794,8 @@ impl ChatSession {) -> Result<Output, Error> {let text = Self::decode_generated_tokens(tokenizer,- language_model_generator,- run_context,+ &run_context.prefill_result.tokens,+ generate_results,)?;let parsed = output_parser.parse(text);let start_idx =diff --git a/crates/uzu/src/session/classification_session.rs b/crates/uzu/src/session/classification_session.rsindex cae8fb761..8db0c4063 100644--- a/crates/uzu/src/session/classification_session.rs+++ b/crates/uzu/src/session/classification_session.rs@@ -64,7 +64,7 @@ impl ClassificationSession {input: Input,) -> Result<ClassificationOutput, Error> {let preprocessing_start = std::time::Instant::now();- let text = self.input_processor.process(&input, false)?;+ let text = self.input_processor.process(&input, false, false)?;let tokens: Vec<u64> = self.tokenizer.encode(text.as_str(), false)diff --git a/crates/uzu/src/session/helpers/input_processor.rs b/crates/uzu/src/session/helpers/input_processor.rsindex cb79abf0d..8a3a27c94 100644--- a/crates/uzu/src/session/helpers/input_processor.rs+++ b/crates/uzu/src/session/helpers/input_processor.rs@@ -16,6 +16,7 @@ pub trait InputProcessor: Send + Sync {&self,input: &Input,enable_thinking: bool,+ add_generation_prompt: bool,) -> Result<String, Error>;}@@ -79,6 +80,7 @@ impl InputProcessor for InputProcessorDefault {&self,input: &Input,enable_thinking: bool,+ add_generation_prompt: bool,) -> Result<String, Error> {let messages = input.get_messages();for message in &messages {@@ -110,7 +112,7 @@ impl InputProcessor for InputProcessorDefault {let result = template.render(context!(messages => messages,- add_generation_prompt => true,+ add_generation_prompt => add_generation_prompt,bos_token => bos_token,enable_thinking => enable_thinking))diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rsindex 27d689ee2..04ac184d6 100644--- a/crates/uzu/src/tracer/trace_validator.rs+++ b/crates/uzu/src/tracer/trace_validator.rs@@ -289,6 +289,7 @@ impl TraceValidator {false,None,false,+ false,None,None,);diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rsindex 3414d3569..d1c122344 100644--- a/crates/uzu/tests/kv_cache_state_test.rs+++ b/crates/uzu/tests/kv_cache_state_test.rs@@ -433,3 +433,157 @@ fn kv_cache_state_and_mask_scenarios() {run_scenario(&context, scenario);}}++#[test]+fn kv_cache_slice_apply_contiguous_window() {+ let Some(context) = create_test_context() else {+ return;+ };++ let mut layer = make_test_layer(+ &context,+ KVCacheLayerState::Windowed {+ ring_offset: 1,+ ring_length: 3,+ window_length: 4,+ },+ 4,+ 0,+ );+ layer.prefix_token_positions = vec![10, 11, 12, INVALID_POSITION];+ let (initial_keys, initial_values) = fill_arrays(&mut layer);++ let slice = layer.slice(&context, 0..2).expect("slice should exist");+ // Mutate the captured slots.+ {+ layer.prefix_token_positions[0] = 999;+ layer.prefix_token_positions[1] = 998;+ let mut keys = layer.keys.borrow_mut();+ let mut values = layer.values.borrow_mut();+ keys.as_slice_mut::<f32>().unwrap()[0] = -1.0;+ keys.as_slice_mut::<f32>().unwrap()[1] = -2.0;+ values.as_slice_mut::<f32>().unwrap()[0] = -3.0;+ values.as_slice_mut::<f32>().unwrap()[1] = -4.0;+ }++ layer.apply_slice(&slice, None);++ assert_eq!(+ layer.prefix_token_positions,+ vec![10, 11, 12, INVALID_POSITION],+ "positions restored for contiguous slice"+ );++ let keys_after = layer.keys.borrow().as_slice::<f32>().unwrap().to_vec();+ let values_after =+ layer.values.borrow().as_slice::<f32>().unwrap().to_vec();+ assert_eq!(+ keys_after[0..4],+ initial_keys[0..4],+ "keys restored for contiguous slice"+ );+ assert_eq!(+ values_after[0..4],+ initial_values[0..4],+ "values restored for contiguous slice"+ );+}++#[test]+fn kv_cache_slice_apply_wrap_window() {+ let Some(context) = create_test_context() else {+ return;+ };++ let mut layer = make_test_layer(+ &context,+ KVCacheLayerState::Windowed {+ ring_offset: 3,+ ring_length: 3,+ window_length: 4,+ },+ 4,+ 0,+ );+ layer.prefix_token_positions = vec![30, 31, 32, 33];+ let (initial_keys, initial_values) = fill_arrays(&mut layer);++ let slice = layer.slice(&context, 2..4).expect("slice should exist");+ // Captured slots are expected to wrap; mutate them.+ {+ layer.prefix_token_positions[2] = 777;+ layer.prefix_token_positions[3] = 778;+ let mut keys = layer.keys.borrow_mut();+ let mut values = layer.values.borrow_mut();+ keys.as_slice_mut::<f32>().unwrap()[2] = -11.0;+ keys.as_slice_mut::<f32>().unwrap()[3] = -12.0;+ values.as_slice_mut::<f32>().unwrap()[2] = -13.0;+ values.as_slice_mut::<f32>().unwrap()[3] = -14.0;+ }++ layer.apply_slice(&slice, None);++ assert_eq!(+ layer.prefix_token_positions,+ vec![30, 31, 32, 33],+ "positions restored for wrapped slice"+ );++ let keys_after = layer.keys.borrow().as_slice::<f32>().unwrap().to_vec();+ let values_after =+ layer.values.borrow().as_slice::<f32>().unwrap().to_vec();+ assert_eq!(+ keys_after[0..4],+ initial_keys[0..4],+ "keys restored for wrapped slice"+ );+ assert_eq!(+ values_after[0..4],+ initial_values[0..4],+ "values restored for wrapped slice"+ );+}++#[test]+fn kv_cache_slice_apply_full_restores_metadata() {+ let Some(context) = create_test_context() else {+ return;+ };++ let mut layer = make_test_layer(+ &context,+ KVCacheLayerState::Full {+ prefix_len: 3,+ },+ 6,+ 0,+ );+ layer.prefix_token_positions = vec![1, 2, 3];++ let slice = layer.slice(&context, 0..2).expect("full slice exists");++ // Mutate metadata.+ if let KVCacheLayerState::Full {+ prefix_len,+ } = &mut layer.state+ {+ *prefix_len = 1;+ }+ layer.prefix_token_positions = vec![9, 9, 9, 9];++ layer.apply_slice(&slice, None);++ assert_eq!(+ layer.prefix_token_positions,+ vec![1, 2, 3],+ "full slice restores positions"+ );+ if let KVCacheLayerState::Full {+ prefix_len,+ } = &layer.state+ {+ assert_eq!(*prefix_len, 3, "full slice restores prefix_len");+ } else {+ panic!("expected full state");+ }+}diff --git a/crates/uzu/tests/moe/moe_experts_test.rs b/crates/uzu/tests/moe/moe_experts_test.rsindex ef1bb14af..f15cdf774 100644--- a/crates/uzu/tests/moe/moe_experts_test.rs+++ b/crates/uzu/tests/moe/moe_experts_test.rs@@ -38,7 +38,14 @@ struct MoeTestData {}impl MoeTestData {- fn generate(rng: &mut StdRng, t: usize, k: usize, d_model: usize, d_ff: usize, e: usize) -> Self {+ fn generate(+ rng: &mut StdRng,+ t: usize,+ k: usize,+ d_model: usize,+ d_ff: usize,+ e: usize,+ ) -> Self {let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();@@ -49,7 +56,8 @@ impl MoeTestData {let topk_probs: Vec<bf16> = (0..t).flat_map(|_| {- let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();+ let raw: Vec<f32> =+ (0..k).map(|_| rng.random_range(0.1..1.0)).collect();let sum: f32 = raw.iter().sum();raw.iter().map(|p| bf16::from_f32(p / sum)).collect::<Vec<_>>()})@@ -68,7 +76,15 @@ impl MoeTestData {.map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();- Self { x, topk_ids, topk_probs, w13, w2, up_biases, down_biases }+ Self {+ x,+ topk_ids,+ topk_probs,+ w13,+ w2,+ up_biases,+ down_biases,+ }}}@@ -124,7 +140,12 @@ fn scatter_by_expert(row_expert_map[bucket_row] = topk_ids[orig_row] as u32;}- ScatterResult { x_perm, offsets, row_expert_map, perm_idx }+ ScatterResult {+ x_perm,+ offsets,+ row_expert_map,+ perm_idx,+ }}/// Gather + finalize: reorder from bucketed order and apply weighted sum@@ -240,11 +261,13 @@ fn cpu_moe_reference(}} else {// SwiGLU or GEGLU - need gate projection too- let mut acc_gate = f32::from(up_biases[bias_base + d_ff + h]);+ let mut acc_gate =+ f32::from(up_biases[bias_base + d_ff + h]);for d in 0..d_model {let x_val = f32::from(x[x_offset + d]);- let w_val =- f32::from(w13_all[w13_base + (d_ff + h) * d_model + d]);+ let w_val = f32::from(+ w13_all[w13_base + (d_ff + h) * d_model + d],+ );acc_gate += x_val * w_val;}@@ -321,9 +344,8 @@ fn test_two_pass_decode_correctness() {.collect();// Generate routing: topk_ids [T*K], topk_probs [T*K]- let topk_ids: Vec<i32> = (0..t * k)- .map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32)- .collect();+ let topk_ids: Vec<i32> =+ (0..t * k).map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32).collect();let topk_probs: Vec<bf16> = (0..t).flat_map(|_| {let raw: Vec<f32> =@@ -531,21 +553,39 @@ fn test_two_pass_decode_multi_token() {let gating_code = 2u32;let silu_alpha = 1.0f32;- eprintln!("[2-pass decode multi-token] T={}, K={}, d_model={}, d_ff={}, E={}", t, k, d_model, d_ff, e);+ eprintln!(+ "[2-pass decode multi-token] T={}, K={}, d_model={}, d_ff={}, E={}",+ t, k, d_model, d_ff, e+ );let data = MoeTestData::generate(&mut rng, t, k, d_model, d_ff, e);let scatter = scatter_by_expert(&data.x, &data.topk_ids, t, k, d_model, e);let y_expected = cpu_moe_reference(- &data.x, &data.topk_ids, &data.topk_probs, &data.w13, &data.w2,- &data.up_biases, &data.down_biases, t, d_model, d_ff, k, gating_code,- silu_alpha, f32::NEG_INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::INFINITY,+ &data.x,+ &data.topk_ids,+ &data.topk_probs,+ &data.w13,+ &data.w2,+ &data.up_biases,+ &data.down_biases,+ t,+ d_model,+ d_ff,+ k,+ gating_code,+ silu_alpha,+ f32::NEG_INFINITY,+ f32::INFINITY,+ f32::NEG_INFINITY,+ f32::INFINITY,);// GPU bufferslet x_perm_buf = alloc_buffer_with_data(&ctx, &scatter.x_perm);let offsets_buf = alloc_buffer_with_data(&ctx, &scatter.offsets);- let row_expert_map_buf = alloc_buffer_with_data(&ctx, &scatter.row_expert_map);+ let row_expert_map_buf =+ alloc_buffer_with_data(&ctx, &scatter.row_expert_map);let w13_buf = alloc_buffer_with_data(&ctx, &data.w13);let w2_buf = alloc_buffer_with_data(&ctx, &data.w2);let up_biases_buf = alloc_buffer_with_data(&ctx, &data.up_biases);@@ -561,28 +601,59 @@ fn test_two_pass_decode_multi_token() {let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);- let experts_kernel = MoeExpertsTwoPassDecodeKernel::new(&ctx).expect("kernel");+ let experts_kernel =+ MoeExpertsTwoPassDecodeKernel::new(&ctx).expect("kernel");let cb = ctx.command_queue.new_command_buffer();- experts_kernel.encode(&cb, MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf, expert_offsets: &offsets_buf,- row_expert_map: &row_expert_map_buf, hidden_buffer: &hidden_buf,- output_buffer: &y_partial_buf, w13_all: &w13_buf, w2_all: &w2_buf,- up_biases: &up_biases_buf, down_biases: &down_biases_buf,- tile_counts: &tile_counts_buf, tile_offsets: &tile_offsets_buf,- tile_map: &tile_map_buf, total_tiles: &total_tiles_buf,- dispatch_args: &dispatch_args_buf, total_rows: sum_k, d_model, d_ff, e,- num_tiles_k: ((d_ff + 63) / 64) as u32, gating_code,- gate_clip_min: f32::NEG_INFINITY, gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY, up_clip_max: f32::INFINITY,- silu_alpha, data_type: KernelDataType::BFloat16,- }).expect("encode");+ experts_kernel+ .encode(+ &cb,+ MoeExpertsTwoPassArguments {+ x_perm_buffer: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &row_expert_map_buf,+ hidden_buffer: &hidden_buf,+ output_buffer: &y_partial_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &tile_counts_buf,+ tile_offsets: &tile_offsets_buf,+ tile_map: &tile_map_buf,+ total_tiles: &total_tiles_buf,+ dispatch_args: &dispatch_args_buf,+ total_rows: sum_k,+ d_model,+ d_ff,+ e,+ num_tiles_k: ((d_ff + 63) / 64) as u32,+ gating_code,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ silu_alpha,+ data_type: KernelDataType::BFloat16,+ },+ )+ .expect("encode");cb.commit();cb.wait_until_completed();let y_partial_gpu = unsafe {- std::slice::from_raw_parts(y_partial_buf.contents() as *const bf16, sum_k * d_model)+ std::slice::from_raw_parts(+ y_partial_buf.contents() as *const bf16,+ sum_k * d_model,+ )};- let y_gpu = gather_and_finalize(y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);+ let y_gpu = gather_and_finalize(+ y_partial_gpu,+ &data.topk_probs,+ &scatter.perm_idx,+ t,+ k,+ d_model,+ );assert_bf16_close(&y_gpu, &y_expected, 0.02, "2-pass decode multi-token");eprintln!("[2-pass decode multi-token] ✓ PASSED");@@ -602,21 +673,39 @@ fn test_two_pass_prefill_correctness() {let gating_code = 2u32;let silu_alpha = 1.0f32;- eprintln!("[2-pass prefill] T={}, K={}, d_model={}, d_ff={}, E={}", t, k, d_model, d_ff, e);+ eprintln!(+ "[2-pass prefill] T={}, K={}, d_model={}, d_ff={}, E={}",+ t, k, d_model, d_ff, e+ );let data = MoeTestData::generate(&mut rng, t, k, d_model, d_ff, e);let scatter = scatter_by_expert(&data.x, &data.topk_ids, t, k, d_model, e);let y_expected = cpu_moe_reference(- &data.x, &data.topk_ids, &data.topk_probs, &data.w13, &data.w2,- &data.up_biases, &data.down_biases, t, d_model, d_ff, k, gating_code,- silu_alpha, f32::NEG_INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::INFINITY,+ &data.x,+ &data.topk_ids,+ &data.topk_probs,+ &data.w13,+ &data.w2,+ &data.up_biases,+ &data.down_biases,+ t,+ d_model,+ d_ff,+ k,+ gating_code,+ silu_alpha,+ f32::NEG_INFINITY,+ f32::INFINITY,+ f32::NEG_INFINITY,+ f32::INFINITY,);// GPU bufferslet x_perm_buf = alloc_buffer_with_data(&ctx, &scatter.x_perm);let offsets_buf = alloc_buffer_with_data(&ctx, &scatter.offsets);- let row_expert_map_buf = alloc_buffer_with_data(&ctx, &scatter.row_expert_map);+ let row_expert_map_buf =+ alloc_buffer_with_data(&ctx, &scatter.row_expert_map);let w13_buf = alloc_buffer_with_data(&ctx, &data.w13);let w2_buf = alloc_buffer_with_data(&ctx, &data.w2);let up_biases_buf = alloc_buffer_with_data(&ctx, &data.up_biases);@@ -632,28 +721,59 @@ fn test_two_pass_prefill_correctness() {let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);- let experts_kernel = MoeExpertsTwoPassPrefillKernel::new(&ctx).expect("kernel");+ let experts_kernel =+ MoeExpertsTwoPassPrefillKernel::new(&ctx).expect("kernel");let cb = ctx.command_queue.new_command_buffer();- experts_kernel.encode(&cb, MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf, expert_offsets: &offsets_buf,- row_expert_map: &row_expert_map_buf, hidden_buffer: &hidden_buf,- output_buffer: &y_partial_buf, w13_all: &w13_buf, w2_all: &w2_buf,- up_biases: &up_biases_buf, down_biases: &down_biases_buf,- tile_counts: &tile_counts_buf, tile_offsets: &tile_offsets_buf,- tile_map: &tile_map_buf, total_tiles: &total_tiles_buf,- dispatch_args: &dispatch_args_buf, total_rows: sum_k, d_model, d_ff, e,- num_tiles_k: ((d_ff + 63) / 64) as u32, gating_code,- gate_clip_min: f32::NEG_INFINITY, gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY, up_clip_max: f32::INFINITY,- silu_alpha, data_type: KernelDataType::BFloat16,- }).expect("encode");+ experts_kernel+ .encode(+ &cb,+ MoeExpertsTwoPassArguments {+ x_perm_buffer: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &row_expert_map_buf,+ hidden_buffer: &hidden_buf,+ output_buffer: &y_partial_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &tile_counts_buf,+ tile_offsets: &tile_offsets_buf,+ tile_map: &tile_map_buf,+ total_tiles: &total_tiles_buf,+ dispatch_args: &dispatch_args_buf,+ total_rows: sum_k,+ d_model,+ d_ff,+ e,+ num_tiles_k: ((d_ff + 63) / 64) as u32,+ gating_code,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ silu_alpha,+ data_type: KernelDataType::BFloat16,+ },+ )+ .expect("encode");cb.commit();cb.wait_until_completed();let y_partial_gpu = unsafe {- std::slice::from_raw_parts(y_partial_buf.contents() as *const bf16, sum_k * d_model)+ std::slice::from_raw_parts(+ y_partial_buf.contents() as *const bf16,+ sum_k * d_model,+ )};- let y_gpu = gather_and_finalize(y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);+ let y_gpu = gather_and_finalize(+ y_partial_gpu,+ &data.topk_probs,+ &scatter.perm_idx,+ t,+ k,+ d_model,+ );assert_bf16_close(&y_gpu, &y_expected, 0.02, "2-pass prefill");eprintln!("[2-pass prefill] ✓ PASSED");