Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Async decoding for transformers

LuckyIYI9f1f07ea186b
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/mod.rs b/crates/uzu/src/backends/metal/encodable_block/layer/mod.rs
index 15a5156fc..5aea3c675 100644
--- a/crates/uzu/src/backends/metal/encodable_block/layer/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/layer/mod.rs
@@ -5,4 +5,3 @@ mod mixer;
pub use executables::LayerExecutables;
pub(crate) use mixer::MixerExecutables;
-
diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rs
index b277e2f69..7497d8f95 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs
@@ -20,13 +20,13 @@ mod layer;
mod linear;
mod mamba_mixer;
mod mlp;
-mod short_conv_mixer;
mod moe_block;
mod normalization;
mod pooling;
mod prediction_head;
mod rope;
mod sampling;
+mod short_conv_mixer;
mod tensor_add_swap;
mod tensor_copy;
pub mod transformer_layer;
@@ -43,7 +43,6 @@ pub use layer::LayerExecutables;
pub use linear::QuantizedLinear;
pub(crate) use mamba_mixer::MambaMixer;
pub use mlp::MlpBlock;
-pub(crate) use short_conv_mixer::ShortConvMixer;
pub use moe_block::{MoeBlock, SharedMoeWeights};
pub use normalization::{
LayerNorm, Normalization, NormalizationError, QKNorm, RMSNorm,
@@ -52,6 +51,7 @@ pub use pooling::Pooling;
pub use prediction_head::ClassifierPredictionHead;
pub use rope::Rope;
pub use sampling::Sampling;
+pub(crate) use short_conv_mixer::ShortConvMixer;
pub use tensor_add_swap::TensorAddSwap;
pub use tensor_copy::TensorCopy;
diff --git a/crates/uzu/src/backends/metal/encodable_block/rope.rs b/crates/uzu/src/backends/metal/encodable_block/rope.rs
index e81874c6a..94b8731da 100644
--- a/crates/uzu/src/backends/metal/encodable_block/rope.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/rope.rs
@@ -85,6 +85,8 @@ impl EncodableBlock for Rope {
command_buffer.root_command_buffer().to_owned();
let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
+ let token_positions_offset = token_positions.buffer_offset();
+
self.kernel.encode(
&compute_encoder,
RopeKernelArguments {
@@ -94,6 +96,7 @@ impl EncodableBlock for Rope {
token_positions_buffer: unsafe {
&token_positions.mtl_buffer()
},
+ token_positions_offset,
rotated_queries_buffer: unsafe {
&rotated_queries.mtl_buffer()
},
diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
index 5df658b94..ff919af81 100644
--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
@@ -10,7 +10,8 @@ use crate::{
compilation_parameters::CompilationConfig,
forward_pass::{ArrayId, ForwardPassState},
kernel::short_conv::{
- ShortConvDecodeArguments, ShortConvKernel, ShortConvPrefillArguments,
+ ShortConvDecodeArguments, ShortConvKernel,
+ ShortConvPrefillArguments,
},
},
config::{DecoderLayerType, ShortConvConfig},
@@ -37,10 +38,7 @@ fn resolve_subtree<'tree>(
return subtree;
}
}
- panic!(
- "Could not find any of {:?} in parameter tree",
- candidates
- );
+ panic!("Could not find any of {:?} in parameter tree", candidates);
}
impl ShortConvMixer {
@@ -64,8 +62,10 @@ impl ShortConvMixer {
let mixer_tree = resolve_subtree(decoder_layer_loader, &["mixer"]);
let conv_tree = resolve_subtree(&mixer_tree, &["conv"]);
- let data_type: DataType =
- short_conv_config.in_projection_config.activation_precision().into();
+ let data_type: DataType = short_conv_config
+ .in_projection_config
+ .activation_precision()
+ .into();
let kernel_data_type: KernelDataType = data_type.into();
let in_projection = transformer_layer::linear_block(
@@ -99,8 +99,9 @@ impl ShortConvMixer {
None
};
- let short_conv_kernel = ShortConvKernel::new(mtl_context, kernel_data_type)
- .expect("Failed to create short conv kernel");
+ let short_conv_kernel =
+ ShortConvKernel::new(mtl_context, kernel_data_type)
+ .expect("Failed to create short conv kernel");
Self {
layer_index,
@@ -173,7 +174,8 @@ impl ShortConvMixer {
let kernel_size = self.config.kernel_size;
let state_stride = kernel_size.saturating_sub(1);
- let mtl_command_buffer = command_buffer.root_command_buffer().to_owned();
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
let compute = mtl_command_buffer.new_compute_command_encoder();
self.short_conv_kernel
@@ -227,7 +229,8 @@ impl ShortConvMixer {
let kernel_size = self.config.kernel_size;
let state_stride = kernel_size.saturating_sub(1);
- let mtl_command_buffer = command_buffer.root_command_buffer().to_owned();
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
let compute = mtl_command_buffer.new_compute_command_encoder();
self.short_conv_kernel
@@ -263,4 +266,3 @@ impl EncodableBlock for ShortConvMixer {
self.encode_pipeline(state, command_buffer, parameters);
}
}
-
diff --git a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs b/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs
index a7b4cff7b..b418aa226 100644
--- a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs
@@ -4,7 +4,10 @@ use metal::CommandBuffer as MTLCommandBuffer;
use super::{
super::{MTLContext, MetalArray},
- kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState},
+ kv_cache_layer::{
+ AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState,
+ KVSlice,
+ },
model_shape::ModelShape,
short_conv_layer::ShortConvLayer,
ssm_layer::SSMLayer,
@@ -21,6 +24,13 @@ pub enum CacheLayer {
ShortConv(ShortConvLayer),
}
+#[derive(Clone)]
+pub enum CacheLayerSlice {
+ Transformer(KVSlice),
+ StateSpace,
+ ShortConv,
+}
+
impl CacheLayer {
pub fn as_transformer(&self) -> Option<&KVCacheLayer> {
match self {
@@ -71,6 +81,11 @@ pub struct CacheLayers {
pub data: Box<[CacheLayer]>,
}
+#[derive(Clone)]
+pub struct CacheLayersSlice {
+ pub layers: Vec<CacheLayerSlice>,
+}
+
impl CacheLayers {
pub fn new(
context: &MTLContext,
@@ -277,6 +292,69 @@ impl CacheLayers {
}
}
+ pub fn attention_bias_updates_after_acceptance(
+ &self,
+ accepted_len: usize,
+ ) -> Vec<AttentionBiasUpdate> {
+ self.data
+ .iter()
+ .filter_map(|layer| match layer {
+ CacheLayer::Transformer(kv) => {
+ kv.attention_bias_update_after_acceptance(accepted_len)
+ },
+ _ => None,
+ })
+ .collect()
+ }
+
+ pub fn slice(
+ &self,
+ context: &MTLContext,
+ range: std::ops::Range<usize>,
+ ) -> Option<CacheLayersSlice> {
+ let mut layers = Vec::with_capacity(self.data.len());
+ for layer in self.data.iter() {
+ match layer {
+ CacheLayer::Transformer(kv) => {
+ let Some(slice) = kv.slice(context, range.clone()) else {
+ return None;
+ };
+ layers.push(CacheLayerSlice::Transformer(slice));
+ },
+ CacheLayer::StateSpace(_) => {
+ layers.push(CacheLayerSlice::StateSpace)
+ },
+ CacheLayer::ShortConv(_) => {
+ layers.push(CacheLayerSlice::ShortConv)
+ },
+ }
+ }
+
+ Some(CacheLayersSlice {
+ layers,
+ })
+ }
+
+ pub fn apply_slice(
+ &mut self,
+ slice: &CacheLayersSlice,
+ range: Option<std::ops::Range<usize>>,
+ ) {
+ for (layer, snapshot) in self.data.iter_mut().zip(slice.layers.iter()) {
+ match (layer, snapshot) {
+ (
+ CacheLayer::Transformer(kv),
+ CacheLayerSlice::Transformer(s),
+ ) => {
+ kv.apply_slice(s, range.clone());
+ },
+ (CacheLayer::StateSpace(_), CacheLayerSlice::StateSpace) => {},
+ (CacheLayer::ShortConv(_), CacheLayerSlice::ShortConv) => {},
+ _ => {},
+ }
+ }
+ }
+
pub fn clone(
&self,
context: &MTLContext,
diff --git a/crates/uzu/src/backends/metal/forward_pass/kv_cache_layer.rs b/crates/uzu/src/backends/metal/forward_pass/kv_cache_layer.rs
index 558d865b8..5ac083771 100644
--- a/crates/uzu/src/backends/metal/forward_pass/kv_cache_layer.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/kv_cache_layer.rs
@@ -11,6 +11,31 @@ use crate::{
pub type ArrayCell = RefCell<MetalArray>;
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct AttentionBiasUpdate {
+ pub key: Option<usize>,
+ pub unmask_col: i32,
+ pub mask_col: i32,
+}
+
+#[derive(Clone)]
+pub enum KVSlice {
+ Full {
+ base_prefix_len: usize,
+ base_positions_len: usize,
+ positions: Vec<usize>,
+ },
+ Window {
+ window_length: usize,
+ base_ring_offset: usize,
+ base_ring_length: usize,
+ slots: Vec<usize>,
+ positions: Vec<usize>, // per slot
+ keys: MetalArray, // [num_groups, slots.len(), head_dim]
+ values: MetalArray, // [num_groups, slots.len(), head_dim]
+ },
+}
+
#[derive(Clone, Debug)]
pub enum KVCacheLayerState {
Full {
@@ -292,4 +317,221 @@ impl KVCacheLayer {
},
}
}
+
+ pub fn attention_bias_update_after_acceptance(
+ &self,
+ accepted_len: usize,
+ ) -> Option<AttentionBiasUpdate> {
+ if accepted_len != 1 {
+ return None;
+ }
+
+ match self.state {
+ KVCacheLayerState::Full {
+ ..
+ } => None,
+ KVCacheLayerState::Windowed {
+ ring_offset,
+ ring_length,
+ window_length,
+ } => {
+ let newest_slot =
+ (ring_offset + window_length - 1) % window_length;
+ let unmask_col = (ring_length > 0)
+ .then_some(newest_slot as i32)
+ .unwrap_or(-1);
+ let mask_col = (ring_length == window_length)
+ .then_some(ring_offset as i32)
+ .unwrap_or(-1);
+
+ Some(AttentionBiasUpdate {
+ key: Some(window_length),
+ unmask_col,
+ mask_col,
+ })
+ },
+ }
+ }
+
+ pub fn slice(
+ &self,
+ context: &MTLContext,
+ range: std::ops::Range<usize>,
+ ) -> Option<KVSlice> {
+ match self.state {
+ KVCacheLayerState::Full {
+ prefix_len,
+ } => Some(KVSlice::Full {
+ base_prefix_len: prefix_len,
+ base_positions_len: self.prefix_token_positions.len(),
+ positions: self.prefix_token_positions.clone(),
+ }),
+ KVCacheLayerState::Windowed {
+ ring_offset,
+ ring_length,
+ window_length,
+ } => {
+ let len = range.end.saturating_sub(range.start);
+ if len == 0 || len > window_length {
+ return None;
+ }
+ let keys = self.keys.borrow();
+ let values = self.values.borrow();
+ let shape = keys.shape();
+ let num_groups = shape[0];
+ let head_dim = shape[2];
+ let dtype = keys.data_type();
+
+ let slice_shape = [num_groups, len, head_dim];
+ let mut slice_keys = context.array(&slice_shape, dtype);
+ let mut slice_values = context.array(&slice_shape, dtype);
+
+ let slots: Vec<usize> = (range.start..range.end)
+ .enumerate()
+ .map(|(offset, _)| {
+ let x = ring_length + offset;
+ if x < window_length {
+ (ring_offset + x) % window_length
+ } else {
+ (ring_offset + (x - window_length)) % window_length
+ }
+ })
+ .collect();
+
+ let positions: Vec<usize> = slots
+ .iter()
+ .map(|&s| self.prefix_token_positions[s])
+ .collect();
+
+ for (i, &slot) in slots.iter().enumerate() {
+ slice_keys.copy_slice(&keys, 1, slot..slot + 1, i);
+ slice_values.copy_slice(&values, 1, slot..slot + 1, i);
+ }
+
+ Some(KVSlice::Window {
+ window_length,
+ base_ring_offset: ring_offset,
+ base_ring_length: ring_length,
+ slots,
+ positions,
+ keys: slice_keys,
+ values: slice_values,
+ })
+ },
+ }
+ }
+
+ pub fn apply_slice(
+ &mut self,
+ slice: &KVSlice,
+ range: Option<std::ops::Range<usize>>,
+ ) {
+ match (slice, &mut self.state) {
+ (
+ KVSlice::Full {
+ base_prefix_len,
+ base_positions_len,
+ positions,
+ },
+ KVCacheLayerState::Full {
+ prefix_len,
+ },
+ ) => match range {
+ None => {
+ *prefix_len = *base_prefix_len;
+ self.prefix_token_positions.clone_from(positions);
+ self.prefix_token_positions.truncate(*base_positions_len);
+ },
+ Some(r) => {
+ let accepted = r.start;
+ *prefix_len = base_prefix_len.saturating_add(accepted);
+ let keep_positions =
+ base_positions_len.saturating_add(accepted);
+ self.prefix_token_positions.truncate(keep_positions);
+ },
+ },
+ (
+ KVSlice::Window {
+ window_length,
+ base_ring_offset,
+ base_ring_length,
+ slots,
+ positions,
+ keys,
+ values,
+ },
+ KVCacheLayerState::Windowed {
+ ring_offset,
+ ring_length,
+ window_length: w_len,
+ },
+ ) => {
+ *w_len = *window_length;
+ match range {
+ None => {
+ *ring_offset = *base_ring_offset;
+ *ring_length = *base_ring_length;
+
+ for (i, &slot) in slots.iter().enumerate() {
+ self.prefix_token_positions[slot] = positions[i];
+ }
+
+ let mut dst_keys = self.keys.borrow_mut();
+ let mut dst_values = self.values.borrow_mut();
+ for (i, &slot) in slots.iter().enumerate() {
+ dst_keys.copy_slice(keys, 1, i..i + 1, slot);
+ dst_values.copy_slice(values, 1, i..i + 1, slot);
+ }
+ },
+ Some(r) => {
+ let len = r.end.saturating_sub(r.start);
+ if len == 0 {
+ return;
+ }
+
+ let accepted = r.start;
+ let base_len = *base_ring_length;
+ let w = *window_length;
+
+ let (new_offset, new_len) = if base_len < w {
+ let after = base_len.saturating_add(accepted);
+ if after <= w {
+ (*base_ring_offset, after)
+ } else {
+ let overflow = after - w;
+ ((base_ring_offset + overflow) % w, w)
+ }
+ } else {
+ ((base_ring_offset + accepted) % w, w)
+ };
+
+ *ring_offset = new_offset;
+ *ring_length = new_len;
+
+ let mut dst_keys = self.keys.borrow_mut();
+ let mut dst_values = self.values.borrow_mut();
+
+ for (i, &slot) in slots[r.clone()].iter().enumerate() {
+ let src_i = r.start + i;
+ self.prefix_token_positions[slot] =
+ positions[src_i];
+ dst_keys.copy_slice(
+ keys,
+ 1,
+ src_i..src_i + 1,
+ slot,
+ );
+ dst_values.copy_slice(
+ values,
+ 1,
+ src_i..src_i + 1,
+ slot,
+ );
+ }
+ },
+ }
+ },
+ _ => {},
+ }
+ }
}
diff --git a/crates/uzu/src/backends/metal/forward_pass/mod.rs b/crates/uzu/src/backends/metal/forward_pass/mod.rs
index 4be2ae19b..37b24c7d7 100644
--- a/crates/uzu/src/backends/metal/forward_pass/mod.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/mod.rs
@@ -14,7 +14,9 @@ pub mod traces;
pub use cache_layers::{CacheLayer, CacheLayers};
pub use encoder_resolver::EncoderResolver;
pub use io_arrays::IOArrays;
-pub use kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState};
+pub use kv_cache_layer::{
+ AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState,
+};
pub use model_shape::ModelShape;
pub use mpsgraph_block::MPSGraphBlock;
pub use scratch_buffers::ScratchBuffers;
diff --git a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs b/crates/uzu/src/backends/metal/forward_pass/model_shape.rs
index c6ede5ae2..b537b81dd 100644
--- a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/model_shape.rs
@@ -489,9 +489,9 @@ impl ModelShape {
let inner_dim = num_heads * head_dim;
Some(conv_dim + inner_dim + num_heads)
},
- DecoderLayerType::ShortConv { .. } => {
- Some(self.model_dim * 3)
- },
+ DecoderLayerType::ShortConv {
+ ..
+ } => Some(self.model_dim * 3),
_ => None,
})
.max()
diff --git a/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs b/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs
index b9fd49008..f285c3f6e 100644
--- a/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs
@@ -5,8 +5,7 @@ use mpsgraph::{
use objc2::rc::Retained;
use super::{
- EncodableBlock, EncodingParameters, ForwardPassState,
- io_arrays::IOArrays,
+ EncodableBlock, EncodingParameters, ForwardPassState, io_arrays::IOArrays,
};
pub struct MPSGraphBlock {
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
index 2468fd782..03b98b33a 100644
--- a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
@@ -109,6 +109,7 @@ impl ForwardPassState {
is_prefilling: bool,
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
skip_token_ids_copy: bool,
+ skip_attention_bias_fill: bool,
async_positions: Option<(&metal::Buffer, usize)>,
async_seeds: Option<(&metal::Buffer, usize)>,
) -> Self {
@@ -220,6 +221,7 @@ impl ForwardPassState {
act_dtype,
token_positions,
external_bias_fn,
+ skip_attention_bias_fill,
);
// Common aux buffers
@@ -277,6 +279,7 @@ impl ForwardPassState {
act_dtype: DataType,
token_positions: &[usize],
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
+ skip_fill: bool,
) -> HashMap<Option<usize>, ArrayCell> {
let cache_ref = cache_layers.borrow();
let mut attention_bias_map: HashMap<Option<usize>, MetalArray> =
@@ -302,13 +305,16 @@ impl ForwardPassState {
// Use cache_layers' fill_attention_bias which properly handles
// both causal masking and sliding window constraints
- cache_layers.borrow().fill_attention_bias(
- &mut attention_bias_map,
- token_positions,
- suffix_length,
- context,
- external_bias_fn,
- );
+ // Skip fill for async decode passes after the first one (bias already set)
+ if !skip_fill {
+ cache_layers.borrow().fill_attention_bias(
+ &mut attention_bias_map,
+ token_positions,
+ suffix_length,
+ context,
+ external_bias_fn,
+ );
+ }
attention_bias_map
.into_iter()
diff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/mod.rs b/crates/uzu/src/backends/metal/kernel/kv_cache_update/mod.rs
index d9252733c..3bd31ae28 100644
--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/kv_cache_update/mod.rs
@@ -116,10 +116,16 @@ impl KVCacheUpdate {
return Err(KVCacheUpdateError::MaxSequenceLengthExceeded);
}
- let indices_ptr = self.indices_buffer.contents() as *mut Swap;
- unsafe {
- for (i, swap) in swaps.iter().enumerate() {
- *indices_ptr.add(i) = *swap;
+ // For small swap counts, use set_bytes to avoid shared buffer race in async mode.
+ // Metal's set_bytes limit is 4KB; 32 swaps = 256 bytes, well under limit.
+ let use_inline_bytes = swaps.len() <= 32;
+
+ if !use_inline_bytes {
+ let indices_ptr = self.indices_buffer.contents() as *mut Swap;
+ unsafe {
+ for (i, swap) in swaps.iter().enumerate() {
+ *indices_ptr.add(i) = *swap;
+ }
}
}
@@ -134,7 +140,15 @@ impl KVCacheUpdate {
// Set buffers and parameters
compute_encoder.set_buffer(0, Some(&layer_data.key_buffer), 0);
compute_encoder.set_buffer(1, Some(&layer_data.value_buffer), 0);
- compute_encoder.set_buffer(2, Some(&self.indices_buffer), 0);
+ if use_inline_bytes {
+ compute_encoder.set_bytes(
+ 2,
+ (swaps.len() * size_of::<Swap>()) as u64,
+ swaps.as_ptr() as *const std::ffi::c_void,
+ );
+ } else {
+ compute_encoder.set_buffer(2, Some(&self.indices_buffer), 0);
+ }
compute_encoder.set_bytes(
3,
size_of::<i32>() as u64,
@@ -194,78 +208,6 @@ pub fn create_swaps_direct(
swaps
}
-/// Original implementation using cycles - kept for reference
-/// This function generates the minimum set of swaps needed to move elements
-/// from source positions to destination positions.
-pub fn create_swaps(
- source_indices: &[usize],
- destination_indices: &[usize],
-) -> Vec<Swap> {
- // The two arrays must be of the same length.
- if source_indices.len() != destination_indices.len() {
- return Vec::new();
- }
-
- // Build mappings between source and destination indices
- let mut forward_mapping = std::collections::HashMap::new();
- let mut reverse_mapping = std::collections::HashMap::new();
-
- for (&s, &d) in source_indices.iter().zip(destination_indices.iter()) {
- if s != d {
- forward_mapping.insert(s, d);
- reverse_mapping.insert(d, s);
- }
- }
-
- let mut swaps = Vec::new();
- let mut visited = std::collections::HashSet::new();
-
- // Process keys in sorted order
- let all_keys: std::collections::HashSet<_> =
- forward_mapping.keys().chain(reverse_mapping.keys()).copied().collect();
-
- let all_keys_vec: Vec<usize> = all_keys.into_iter().collect();
- for &s in &all_keys_vec {
- if visited.contains(&s) {
- continue;
- }
-
- visited.insert(s);
- let mut cycle = vec![s];
- let mut iterator = s;
-
- while let Some(&next) = forward_mapping.get(&iterator) {
- iterator = next;
- if visited.contains(&iterator) {
- break;
- }
- visited.insert(iterator);
- cycle.push(iterator);
- }
-
- // Reverse direction
- iterator = s;
- while let Some(&next) = reverse_mapping.get(&iterator) {
- iterator = next;
- if visited.contains(&iterator) {
- break;
- }
- visited.insert(iterator);
- cycle.push(iterator);
- }
-
- let first = cycle[0];
- for &next in cycle.iter().skip(1) {
- swaps.push(Swap {
- source: first as i32,
- destination: next as i32,
- });
- }
- }
-
- swaps
-}
-
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal
new file mode 100644
index 000000000..babdccb16
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal
@@ -0,0 +1,32 @@
+#include <metal_stdlib>
+using namespace metal;
+
+template <typename T>
+kernel void update_attention_mask(
+ device T* mask [[buffer(0)]],
+ constant int& unmask_col [[buffer(1)]],
+ constant int& mask_col [[buffer(2)]],
+ uint tid [[thread_position_in_grid]]
+) {
+ if (tid == 0) {
+ if (unmask_col >= 0) {
+ mask[unmask_col] = T(0);
+ }
+ if (mask_col >= 0) {
+ mask[mask_col] = -numeric_limits<T>::infinity();
+ }
+ }
+}
+
+#define instantiate_mask_update(name, type) \
+ template [[host_name("update_attention_mask_" #name)]] \
+ kernel void update_attention_mask<type>( \
+ device type* mask [[buffer(0)]], \
+ constant int& unmask_col [[buffer(1)]], \
+ constant int& mask_col [[buffer(2)]], \
+ uint tid [[thread_position_in_grid]] \
+ );
+
+instantiate_mask_update(float, float)
+instantiate_mask_update(half, half)
+instantiate_mask_update(bfloat, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/mask_update/mod.rs b/crates/uzu/src/backends/metal/kernel/mask_update/mod.rs
new file mode 100644
index 000000000..9889bb2a1
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/mask_update/mod.rs
@@ -0,0 +1,50 @@
+use metal::{
+ Buffer as MTLBuffer, ComputeCommandEncoderRef,
+ ComputePipelineState as MTLComputePipelineState, MTLSize,
+};
+
+use crate::backends::metal::{MTLContext, kernel::KernelDataType};
+
+pub struct MaskUpdateKernel {
+ pipeline: MTLComputePipelineState,
+}
+
+impl MaskUpdateKernel {
+ pub fn new(
+ context: &MTLContext,
+ data_type: KernelDataType,
+ ) -> Result<Self, crate::backends::metal::error::MTLError> {
+ let function_name = format!(
+ "update_attention_mask_{}",
+ data_type.function_name_suffix()
+ );
+ let (pipeline, _) = context
+ .compute_pipeline_state_with_reflection(&function_name, None)?;
+
+ Ok(Self {
+ pipeline,
+ })
+ }
+
+ pub fn encode(
+ &self,
+ encoder: &ComputeCommandEncoderRef,
+ mask_buffer: &MTLBuffer,
+ unmask_col: i32,
+ mask_col: i32,
+ ) {
+ encoder.set_compute_pipeline_state(&self.pipeline);
+ encoder.set_buffer(0, Some(mask_buffer), 0);
+ encoder.set_bytes(
+ 1,
+ std::mem::size_of::<i32>() as u64,
+ &unmask_col as *const i32 as *const _,
+ );
+ encoder.set_bytes(
+ 2,
+ std::mem::size_of::<i32>() as u64,
+ &mask_col as *const i32 as *const _,
+ );
+ encoder.dispatch_threads(MTLSize::new(1, 1, 1), MTLSize::new(1, 1, 1));
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
index 400354529..aefaff465 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
@@ -53,7 +53,9 @@ impl MlpGateActMulEncodable {
}
impl MlpGateActMulKernel {
- fn kernel_name_for_type(data_type: DataType) -> Result<&'static str, MTLError> {
+ fn kernel_name_for_type(
+ data_type: DataType
+ ) -> Result<&'static str, MTLError> {
match data_type {
DataType::F16 => Ok("mlp_activation_mul_f16"),
DataType::F32 => Ok("mlp_activation_mul_f32"),
@@ -65,15 +67,22 @@ impl MlpGateActMulKernel {
}
}
- pub fn new(context: &MTLContext, data_type: DataType) -> Result<Self, MTLError> {
+ pub fn new(
+ context: &MTLContext,
+ data_type: DataType,
+ ) -> Result<Self, MTLError> {
let fn_name = Self::kernel_name_for_type(data_type)?;
let pipeline = context.compute_pipeline_state(fn_name, None)?;
- Ok(Self { pipeline })
+ Ok(Self {
+ pipeline,
+ })
}
fn act_code(act: &Activation) -> u16 {
match act {
- Activation::SiLU { .. } => 0,
+ Activation::SiLU {
+ ..
+ } => 0,
Activation::Gelu => 1,
Activation::Identity => {
panic!("Identity activation is not supported for MLP kernels")
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 294c76591..bcf40230c 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -3,6 +3,7 @@ mod data_type;
pub mod embedding;
pub mod kv_cache_update;
pub mod layer_norm;
+pub mod mask_update;
pub mod media_kernels;
pub mod mlp;
pub mod moe;
@@ -27,6 +28,7 @@ pub use attention::{
pub use data_type::KernelDataType;
pub use kv_cache_update::KVCacheUpdate;
pub use layer_norm::{LayerNormArguments, LayerNormError, LayerNormKernel};
+pub use mask_update::MaskUpdateKernel;
pub use moe::{
MoeBlockBasesArguments, MoeCountsOffsetsFusedArguments,
MoeCountsOffsetsFusedError, MoeCountsOffsetsFusedKernel,
diff --git a/crates/uzu/src/backends/metal/kernel/rope/mod.rs b/crates/uzu/src/backends/metal/kernel/rope/mod.rs
index 5ece64af7..9b05d911c 100644
--- a/crates/uzu/src/backends/metal/kernel/rope/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/rope/mod.rs
@@ -26,6 +26,7 @@ pub struct RopeKernelArguments<'a> {
pub cosines_buffer: &'a MTLBuffer, // buffer(1)
pub sines_buffer: &'a MTLBuffer, // buffer(2)
pub token_positions_buffer: &'a MTLBuffer, // buffer(3)
+ pub token_positions_offset: usize, // byte offset into token_positions_buffer
pub rotated_queries_buffer: &'a MTLBuffer, // buffer(4)
pub rotated_keys_buffer: &'a MTLBuffer, // buffer(5)
pub head_dim: usize,
@@ -68,7 +69,11 @@ impl RopeKernel {
compute_encoder.set_buffer(0, Some(args.qkv_buffer), 0);
compute_encoder.set_buffer(1, Some(args.cosines_buffer), 0);
compute_encoder.set_buffer(2, Some(args.sines_buffer), 0);
- compute_encoder.set_buffer(3, Some(args.token_positions_buffer), 0);
+ compute_encoder.set_buffer(
+ 3,
+ Some(args.token_positions_buffer),
+ args.token_positions_offset as u64,
+ );
compute_encoder.set_buffer(4, Some(args.rotated_queries_buffer), 0);
compute_encoder.set_buffer(5, Some(args.rotated_keys_buffer), 0);
diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
index 5b2cc91f1..02de18eba 100644
--- a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
@@ -189,4 +189,3 @@ impl ShortConvKernel {
Ok(())
}
}
-
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs
index 7145e7278..1dae0f2c3 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs
@@ -5,7 +5,9 @@ use metal::{
ComputePipelineState as MTLComputePipelineState,
};
-use super::{KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch};
+use super::{
+ KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch,
+};
use crate::backends::metal::error::MTLError;
#[derive(Debug)]
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs
index dd67c59ad..743ff1d23 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs
@@ -5,7 +5,9 @@ use metal::{
ComputePipelineState as MTLComputePipelineState,
};
-use super::{KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch};
+use super::{
+ KernelDataType, MTLContext, metal_extensions::ComputeEncoderDispatch,
+};
use crate::backends::metal::error::MTLError;
pub struct TensorCopyKernel {
diff --git a/crates/uzu/src/backends/metal/language_model_generator_context.rs b/crates/uzu/src/backends/metal/language_model_generator_context.rs
index 97b1bc1a6..3b85afdc4 100644
--- a/crates/uzu/src/backends/metal/language_model_generator_context.rs
+++ b/crates/uzu/src/backends/metal/language_model_generator_context.rs
@@ -16,11 +16,11 @@ use super::{
compilation_parameters::CompilationConfig,
encodable_block::Sampling,
forward_pass::{ScratchBuffers, SharedBuffers},
- kernel::TokenCopyKernel,
+ kernel::{MaskUpdateKernel, TokenCopyKernel},
};
use crate::{
- DataType, DecoderConfig,
- config::{LanguageModelConfig, ModelMetadata},
+ DataType,
+ config::{DecoderConfig, LanguageModelConfig, ModelMetadata},
language_model::rng::DerivableSeed,
parameters::ParameterLoader,
session::{
@@ -147,6 +147,8 @@ pub struct LanguageModelGeneratorContext {
/// Kernel for copying sampled tokens in async pipeline
pub token_copy: TokenCopyKernel,
+ /// Kernel for updating attention mask between async passes
+ pub mask_update: Option<MaskUpdateKernel>,
/// Pre-allocated buffers for async generation
pub async_buffers: AsyncBuffers,
}
@@ -263,6 +265,16 @@ impl LanguageModelGeneratorContext {
let token_copy = TokenCopyKernel::new(&mtl_context)
.map_err(|_| Error::UnableToCreateMetalContext)?;
+ // Create mask update kernel if model has attention layers
+ let mask_update = if decoder_config.has_attention_layers() {
+ Some(
+ MaskUpdateKernel::new(&mtl_context, kernel_data_type)
+ .map_err(|_| Error::UnableToCreateMetalContext)?,
+ )
+ } else {
+ None
+ };
+
let async_batch_size =
decoding_config.async_batch_size.resolve(model_path);
let async_buffers = AsyncBuffers::new(
@@ -288,6 +300,7 @@ impl LanguageModelGeneratorContext {
gpu_sampler,
next_seed,
token_copy,
+ mask_update,
async_buffers,
};
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index 5a0af9763..20ef6409d 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -16,7 +16,8 @@ use super::{
use crate::{
Array,
backends::metal::forward_pass::{
- EncodableBlock, EncodingParameters, ForwardPassState, INVALID_POSITION,
+ AttentionBiasUpdate, EncodableBlock, EncodingParameters,
+ ForwardPassState, INVALID_POSITION,
},
session::{
config::DecodingConfig,
@@ -408,6 +409,19 @@ impl LanguageModelGenerator {
tokens_to_generate: usize,
) {
let prefill_count = self.tokens.len();
+
+ // Initialize attention bias buffers to zero for async mode
+ // This ensures unwritten columns (beyond what pass 0 fills) are 0 (attend)
+ // rather than garbage (which could be -inf and mask incorrectly)
+ for (_, mask_buffer) in
+ &self.context.scratch_buffers.attention_window_size_to_bias
+ {
+ unsafe {
+ let ptr = mask_buffer.contents() as *mut u8;
+ std::ptr::write_bytes(ptr, 0, mask_buffer.length() as usize);
+ }
+ }
+
self.context
.async_buffers
.prepare_positions(prefill_count, tokens_to_generate);
@@ -451,13 +465,29 @@ impl LanguageModelGenerator {
let last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;
- let token_ids: [u64; 1] = [last_token];
- let token_positions: [usize; 1] = [0]; // Ignored, using async buffer
- let token_seeds: [u64; 1] = [0]; // Ignored, using async buffer
+ let token_position = unsafe {
+ let ptr = async_positions_buffer.contents() as *const u32;
+ *ptr.add(pass_idx) as usize
+ };
+
+ let task = LanguageModelGeneratorRunTask {
+ token_ids: &[last_token],
+ token_positions: &[token_position],
+ token_bitmask: None,
+ token_seeds: &[0], // Ignored, using async buffer
+ expected_number_of_new_tokens: 1,
+ active_suffix_length: 1,
+ is_prefilling: false,
+ };
let async_positions = Some((&async_positions_buffer, pass_idx));
let async_seeds = Some((&async_seeds_buffer, pass_idx));
+ let skip_attention_bias_fill =
+ pass_idx > 0 && self.context.decoder_config.has_attention_layers();
+
+ let skip_token_ids_copy = pass_idx > 0;
+
let mut state = ForwardPassState::new_llm(
self.context.mtl_context.clone(),
&self.context.decoder_config,
@@ -465,19 +495,20 @@ impl LanguageModelGenerator {
&self.context.scratch_buffers,
self.context.cache_layers.clone(),
self.context.shared_buffers.clone(),
- &token_ids,
- &token_positions,
- None, // token_bitmask - ignored, using async buffer
- &token_seeds,
- 1, // active_suffix_length
- false, // is_prefilling
- None, // external_bias_fn
- is_continuation,
+ &task.token_ids,
+ &task.token_positions,
+ task.token_bitmask,
+ &task.token_seeds,
+ task.active_suffix_length,
+ task.is_prefilling,
+ None,
+ skip_token_ids_copy,
+ skip_attention_bias_fill,
async_positions,
async_seeds,
);
- if let Some(method) = state.sampling_method_mut() {
- *method = Some(sampling_method);
+ if let Some(sm) = state.sampling_method_mut() {
+ *sm = Some(sampling_method);
}
self.context.reset_command_buffer();
@@ -490,7 +521,6 @@ impl LanguageModelGenerator {
.encode_wait_for_event(&async_event, current_counter);
}
- // Encode forward pass
self.context.executables.encode(
&mut state,
&self.context.command_buffer,
@@ -527,6 +557,49 @@ impl LanguageModelGenerator {
);
encoder.end_encoding();
+ // Scatter + register for all transformer layers
+ {
+ let cb = root_command_buffer.to_owned();
+ self.context.cache_layers.borrow_mut().update_after_acceptance(
+ &[0],
+ None,
+ &cb,
+ &self.context.kv_cache_update,
+ );
+ self.context
+ .cache_layers
+ .borrow_mut()
+ .register_accepted_tokens(&[token_position]);
+ }
+
+ let encoder = root_command_buffer.new_compute_command_encoder();
+
+ if let Some(mask_update) = &self.context.mask_update {
+ let updates: Vec<AttentionBiasUpdate> = self
+ .context
+ .cache_layers
+ .borrow()
+ .attention_bias_updates_after_acceptance(1);
+ for (window_size, mask_buffer) in
+ &self.context.scratch_buffers.attention_window_size_to_bias
+ {
+ if let Some(update) =
+ updates.iter().find(|u| &u.key == window_size)
+ {
+ if update.unmask_col >= 0 || update.mask_col >= 0 {
+ mask_update.encode(
+ encoder,
+ mask_buffer,
+ update.unmask_col,
+ update.mask_col,
+ );
+ }
+ }
+ }
+ }
+
+ encoder.end_encoding();
+
// Signal event for next pass
let next_counter = current_counter + 1;
root_command_buffer.encode_signal_event(&async_event, next_counter);
diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rs
index 0be74166e..2ce090ee7 100644
--- a/crates/uzu/src/language_model/tasks.rs
+++ b/crates/uzu/src/language_model/tasks.rs
@@ -83,6 +83,7 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {
self.is_prefilling,
external_bias_fn,
false,
+ false,
None,
None,
)
diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rs
index 4de7b809f..672c68560 100644
--- a/crates/uzu/src/session/chat_session.rs
+++ b/crates/uzu/src/session/chat_session.rs
@@ -268,8 +268,11 @@ impl ChatSession {
F: Fn(Output) -> bool,
{
let run_start = Instant::now();
- let text =
- self.input_processor.process(&input, config.enable_thinking)?;
+ let text = self.input_processor.process(
+ &input,
+ config.enable_thinking,
+ config.tokens_limit > 0,
+ )?;
let tokens: Vec<u64> = self
.tokenizer
.encode(text.as_str(), false)
@@ -296,7 +299,7 @@ impl ChatSession {
}
let prefix_offset = language_model_generator.tokens.len();
- let prefix_len_before = prefix_offset.saturating_sub(1);
+ let prefix_len_before = prefix_offset;
let eos_tokens: Vec<u64> = language_model_config
.generation_config
@@ -393,7 +396,6 @@ impl ChatSession {
let can_use_async =
language_model_generator.decoding_config.generate_suffix_length()
== 1
- && !language_model_generator.has_attention_layers()
&& compiled_grammar.is_none();
let generate_output = if can_use_async {
@@ -539,6 +541,15 @@ impl ChatSession {
while in_flight > 0 || next_to_submit < tokens_to_generate {
let batch_end =
std::cmp::min(next_to_submit + batch_size, tokens_to_generate);
+ let batch_submitted = batch_end - next_to_submit;
+ let accepted_before_batch = results.len();
+
+ let cache_slice = llm
+ .context
+ .cache_layers
+ .borrow()
+ .slice(&llm.context.mtl_context, 0..batch_submitted);
+
for idx in next_to_submit..batch_end {
let batch_sender = sender.clone();
let last_callback_nanos = last_nanos.clone();
@@ -556,7 +567,6 @@ impl ChatSession {
})?;
in_flight += 1;
}
- let batch_submitted = batch_end - next_to_submit;
next_to_submit = batch_end;
for _ in 0..batch_submitted {
@@ -604,6 +614,16 @@ impl ChatSession {
let _ = receiver.recv();
in_flight -= 1;
}
+ if let Some(slice) = cache_slice {
+ let accepted_in_batch =
+ results.len().saturating_sub(accepted_before_batch);
+ if accepted_in_batch < batch_submitted {
+ llm.context.cache_layers.borrow_mut().apply_slice(
+ &slice,
+ Some(accepted_in_batch..batch_submitted),
+ );
+ }
+ }
return Self::build_output(
model_metadata,
tokenizer,
@@ -749,14 +769,12 @@ impl ChatSession {
fn decode_generated_tokens(
tokenizer: &Tokenizer,
- language_model_generator: &LanguageModelGenerator,
- run_context: &RunContext,
+ prefill_tokens: &[u64],
+ generate_results: &[GenerateResult],
) -> Result<String, Error> {
- let start_idx =
- run_context.prefix_len_before + run_context.input_tokens_len;
- let generated_tokens: Vec<u32> = language_model_generator.tokens
- [start_idx..]
+ let generated_tokens: Vec<u32> = prefill_tokens
.iter()
+ .chain(generate_results.iter().flat_map(|r| r.tokens.iter()))
.map(|&v| v as u32)
.collect();
tokenizer
@@ -776,8 +794,8 @@ impl ChatSession {
) -> Result<Output, Error> {
let text = Self::decode_generated_tokens(
tokenizer,
- language_model_generator,
- run_context,
+ &run_context.prefill_result.tokens,
+ generate_results,
)?;
let parsed = output_parser.parse(text);
let start_idx =
diff --git a/crates/uzu/src/session/classification_session.rs b/crates/uzu/src/session/classification_session.rs
index cae8fb761..8db0c4063 100644
--- a/crates/uzu/src/session/classification_session.rs
+++ b/crates/uzu/src/session/classification_session.rs
@@ -64,7 +64,7 @@ impl ClassificationSession {
input: Input,
) -> Result<ClassificationOutput, Error> {
let preprocessing_start = std::time::Instant::now();
- let text = self.input_processor.process(&input, false)?;
+ let text = self.input_processor.process(&input, false, false)?;
let tokens: Vec<u64> = self
.tokenizer
.encode(text.as_str(), false)
diff --git a/crates/uzu/src/session/helpers/input_processor.rs b/crates/uzu/src/session/helpers/input_processor.rs
index cb79abf0d..8a3a27c94 100644
--- a/crates/uzu/src/session/helpers/input_processor.rs
+++ b/crates/uzu/src/session/helpers/input_processor.rs
@@ -16,6 +16,7 @@ pub trait InputProcessor: Send + Sync {
&self,
input: &Input,
enable_thinking: bool,
+ add_generation_prompt: bool,
) -> Result<String, Error>;
}
@@ -79,6 +80,7 @@ impl InputProcessor for InputProcessorDefault {
&self,
input: &Input,
enable_thinking: bool,
+ add_generation_prompt: bool,
) -> Result<String, Error> {
let messages = input.get_messages();
for message in &messages {
@@ -110,7 +112,7 @@ impl InputProcessor for InputProcessorDefault {
let result = template
.render(context!(
messages => messages,
- add_generation_prompt => true,
+ add_generation_prompt => add_generation_prompt,
bos_token => bos_token,
enable_thinking => enable_thinking
))
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rs
index 27d689ee2..04ac184d6 100644
--- a/crates/uzu/src/tracer/trace_validator.rs
+++ b/crates/uzu/src/tracer/trace_validator.rs
@@ -289,6 +289,7 @@ impl TraceValidator {
false,
None,
false,
+ false,
None,
None,
);
diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rs
index 3414d3569..d1c122344 100644
--- a/crates/uzu/tests/kv_cache_state_test.rs
+++ b/crates/uzu/tests/kv_cache_state_test.rs
@@ -433,3 +433,157 @@ fn kv_cache_state_and_mask_scenarios() {
run_scenario(&context, scenario);
}
}
+
+#[test]
+fn kv_cache_slice_apply_contiguous_window() {
+ let Some(context) = create_test_context() else {
+ return;
+ };
+
+ let mut layer = make_test_layer(
+ &context,
+ KVCacheLayerState::Windowed {
+ ring_offset: 1,
+ ring_length: 3,
+ window_length: 4,
+ },
+ 4,
+ 0,
+ );
+ layer.prefix_token_positions = vec![10, 11, 12, INVALID_POSITION];
+ let (initial_keys, initial_values) = fill_arrays(&mut layer);
+
+ let slice = layer.slice(&context, 0..2).expect("slice should exist");
+ // Mutate the captured slots.
+ {
+ layer.prefix_token_positions[0] = 999;
+ layer.prefix_token_positions[1] = 998;
+ let mut keys = layer.keys.borrow_mut();
+ let mut values = layer.values.borrow_mut();
+ keys.as_slice_mut::<f32>().unwrap()[0] = -1.0;
+ keys.as_slice_mut::<f32>().unwrap()[1] = -2.0;
+ values.as_slice_mut::<f32>().unwrap()[0] = -3.0;
+ values.as_slice_mut::<f32>().unwrap()[1] = -4.0;
+ }
+
+ layer.apply_slice(&slice, None);
+
+ assert_eq!(
+ layer.prefix_token_positions,
+ vec![10, 11, 12, INVALID_POSITION],
+ "positions restored for contiguous slice"
+ );
+
+ let keys_after = layer.keys.borrow().as_slice::<f32>().unwrap().to_vec();
+ let values_after =
+ layer.values.borrow().as_slice::<f32>().unwrap().to_vec();
+ assert_eq!(
+ keys_after[0..4],
+ initial_keys[0..4],
+ "keys restored for contiguous slice"
+ );
+ assert_eq!(
+ values_after[0..4],
+ initial_values[0..4],
+ "values restored for contiguous slice"
+ );
+}
+
+#[test]
+fn kv_cache_slice_apply_wrap_window() {
+ let Some(context) = create_test_context() else {
+ return;
+ };
+
+ let mut layer = make_test_layer(
+ &context,
+ KVCacheLayerState::Windowed {
+ ring_offset: 3,
+ ring_length: 3,
+ window_length: 4,
+ },
+ 4,
+ 0,
+ );
+ layer.prefix_token_positions = vec![30, 31, 32, 33];
+ let (initial_keys, initial_values) = fill_arrays(&mut layer);
+
+ let slice = layer.slice(&context, 2..4).expect("slice should exist");
+ // Captured slots are expected to wrap; mutate them.
+ {
+ layer.prefix_token_positions[2] = 777;
+ layer.prefix_token_positions[3] = 778;
+ let mut keys = layer.keys.borrow_mut();
+ let mut values = layer.values.borrow_mut();
+ keys.as_slice_mut::<f32>().unwrap()[2] = -11.0;
+ keys.as_slice_mut::<f32>().unwrap()[3] = -12.0;
+ values.as_slice_mut::<f32>().unwrap()[2] = -13.0;
+ values.as_slice_mut::<f32>().unwrap()[3] = -14.0;
+ }
+
+ layer.apply_slice(&slice, None);
+
+ assert_eq!(
+ layer.prefix_token_positions,
+ vec![30, 31, 32, 33],
+ "positions restored for wrapped slice"
+ );
+
+ let keys_after = layer.keys.borrow().as_slice::<f32>().unwrap().to_vec();
+ let values_after =
+ layer.values.borrow().as_slice::<f32>().unwrap().to_vec();
+ assert_eq!(
+ keys_after[0..4],
+ initial_keys[0..4],
+ "keys restored for wrapped slice"
+ );
+ assert_eq!(
+ values_after[0..4],
+ initial_values[0..4],
+ "values restored for wrapped slice"
+ );
+}
+
+#[test]
+fn kv_cache_slice_apply_full_restores_metadata() {
+ let Some(context) = create_test_context() else {
+ return;
+ };
+
+ let mut layer = make_test_layer(
+ &context,
+ KVCacheLayerState::Full {
+ prefix_len: 3,
+ },
+ 6,
+ 0,
+ );
+ layer.prefix_token_positions = vec![1, 2, 3];
+
+ let slice = layer.slice(&context, 0..2).expect("full slice exists");
+
+ // Mutate metadata.
+ if let KVCacheLayerState::Full {
+ prefix_len,
+ } = &mut layer.state
+ {
+ *prefix_len = 1;
+ }
+ layer.prefix_token_positions = vec![9, 9, 9, 9];
+
+ layer.apply_slice(&slice, None);
+
+ assert_eq!(
+ layer.prefix_token_positions,
+ vec![1, 2, 3],
+ "full slice restores positions"
+ );
+ if let KVCacheLayerState::Full {
+ prefix_len,
+ } = &layer.state
+ {
+ assert_eq!(*prefix_len, 3, "full slice restores prefix_len");
+ } else {
+ panic!("expected full state");
+ }
+}
diff --git a/crates/uzu/tests/moe/moe_experts_test.rs b/crates/uzu/tests/moe/moe_experts_test.rs
index ef1bb14af..f15cdf774 100644
--- a/crates/uzu/tests/moe/moe_experts_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_test.rs
@@ -38,7 +38,14 @@ struct MoeTestData {
}
impl MoeTestData {
- fn generate(rng: &mut StdRng, t: usize, k: usize, d_model: usize, d_ff: usize, e: usize) -> Self {
+ fn generate(
+ rng: &mut StdRng,
+ t: usize,
+ k: usize,
+ d_model: usize,
+ d_ff: usize,
+ e: usize,
+ ) -> Self {
let x: Vec<bf16> = (0..t * d_model)
.map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
.collect();
@@ -49,7 +56,8 @@ impl MoeTestData {
let topk_probs: Vec<bf16> = (0..t)
.flat_map(|_| {
- let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
+ let raw: Vec<f32> =
+ (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
let sum: f32 = raw.iter().sum();
raw.iter().map(|p| bf16::from_f32(p / sum)).collect::<Vec<_>>()
})
@@ -68,7 +76,15 @@ impl MoeTestData {
.map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
.collect();
- Self { x, topk_ids, topk_probs, w13, w2, up_biases, down_biases }
+ Self {
+ x,
+ topk_ids,
+ topk_probs,
+ w13,
+ w2,
+ up_biases,
+ down_biases,
+ }
}
}
@@ -124,7 +140,12 @@ fn scatter_by_expert(
row_expert_map[bucket_row] = topk_ids[orig_row] as u32;
}
- ScatterResult { x_perm, offsets, row_expert_map, perm_idx }
+ ScatterResult {
+ x_perm,
+ offsets,
+ row_expert_map,
+ perm_idx,
+ }
}
/// Gather + finalize: reorder from bucketed order and apply weighted sum
@@ -240,11 +261,13 @@ fn cpu_moe_reference(
}
} else {
// SwiGLU or GEGLU - need gate projection too
- let mut acc_gate = f32::from(up_biases[bias_base + d_ff + h]);
+ let mut acc_gate =
+ f32::from(up_biases[bias_base + d_ff + h]);
for d in 0..d_model {
let x_val = f32::from(x[x_offset + d]);
- let w_val =
- f32::from(w13_all[w13_base + (d_ff + h) * d_model + d]);
+ let w_val = f32::from(
+ w13_all[w13_base + (d_ff + h) * d_model + d],
+ );
acc_gate += x_val * w_val;
}
@@ -321,9 +344,8 @@ fn test_two_pass_decode_correctness() {
.collect();
// Generate routing: topk_ids [T*K], topk_probs [T*K]
- let topk_ids: Vec<i32> = (0..t * k)
- .map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32)
- .collect();
+ let topk_ids: Vec<i32> =
+ (0..t * k).map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32).collect();
let topk_probs: Vec<bf16> = (0..t)
.flat_map(|_| {
let raw: Vec<f32> =
@@ -531,21 +553,39 @@ fn test_two_pass_decode_multi_token() {
let gating_code = 2u32;
let silu_alpha = 1.0f32;
- eprintln!("[2-pass decode multi-token] T={}, K={}, d_model={}, d_ff={}, E={}", t, k, d_model, d_ff, e);
+ eprintln!(
+ "[2-pass decode multi-token] T={}, K={}, d_model={}, d_ff={}, E={}",
+ t, k, d_model, d_ff, e
+ );
let data = MoeTestData::generate(&mut rng, t, k, d_model, d_ff, e);
let scatter = scatter_by_expert(&data.x, &data.topk_ids, t, k, d_model, e);
let y_expected = cpu_moe_reference(
- &data.x, &data.topk_ids, &data.topk_probs, &data.w13, &data.w2,
- &data.up_biases, &data.down_biases, t, d_model, d_ff, k, gating_code,
- silu_alpha, f32::NEG_INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::INFINITY,
+ &data.x,
+ &data.topk_ids,
+ &data.topk_probs,
+ &data.w13,
+ &data.w2,
+ &data.up_biases,
+ &data.down_biases,
+ t,
+ d_model,
+ d_ff,
+ k,
+ gating_code,
+ silu_alpha,
+ f32::NEG_INFINITY,
+ f32::INFINITY,
+ f32::NEG_INFINITY,
+ f32::INFINITY,
);
// GPU buffers
let x_perm_buf = alloc_buffer_with_data(&ctx, &scatter.x_perm);
let offsets_buf = alloc_buffer_with_data(&ctx, &scatter.offsets);
- let row_expert_map_buf = alloc_buffer_with_data(&ctx, &scatter.row_expert_map);
+ let row_expert_map_buf =
+ alloc_buffer_with_data(&ctx, &scatter.row_expert_map);
let w13_buf = alloc_buffer_with_data(&ctx, &data.w13);
let w2_buf = alloc_buffer_with_data(&ctx, &data.w2);
let up_biases_buf = alloc_buffer_with_data(&ctx, &data.up_biases);
@@ -561,28 +601,59 @@ fn test_two_pass_decode_multi_token() {
let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);
let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
- let experts_kernel = MoeExpertsTwoPassDecodeKernel::new(&ctx).expect("kernel");
+ let experts_kernel =
+ MoeExpertsTwoPassDecodeKernel::new(&ctx).expect("kernel");
let cb = ctx.command_queue.new_command_buffer();
- experts_kernel.encode(&cb, MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf, expert_offsets: &offsets_buf,
- row_expert_map: &row_expert_map_buf, hidden_buffer: &hidden_buf,
- output_buffer: &y_partial_buf, w13_all: &w13_buf, w2_all: &w2_buf,
- up_biases: &up_biases_buf, down_biases: &down_biases_buf,
- tile_counts: &tile_counts_buf, tile_offsets: &tile_offsets_buf,
- tile_map: &tile_map_buf, total_tiles: &total_tiles_buf,
- dispatch_args: &dispatch_args_buf, total_rows: sum_k, d_model, d_ff, e,
- num_tiles_k: ((d_ff + 63) / 64) as u32, gating_code,
- gate_clip_min: f32::NEG_INFINITY, gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY, up_clip_max: f32::INFINITY,
- silu_alpha, data_type: KernelDataType::BFloat16,
- }).expect("encode");
+ experts_kernel
+ .encode(
+ &cb,
+ MoeExpertsTwoPassArguments {
+ x_perm_buffer: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &row_expert_map_buf,
+ hidden_buffer: &hidden_buf,
+ output_buffer: &y_partial_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &tile_counts_buf,
+ tile_offsets: &tile_offsets_buf,
+ tile_map: &tile_map_buf,
+ total_tiles: &total_tiles_buf,
+ dispatch_args: &dispatch_args_buf,
+ total_rows: sum_k,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k: ((d_ff + 63) / 64) as u32,
+ gating_code,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ silu_alpha,
+ data_type: KernelDataType::BFloat16,
+ },
+ )
+ .expect("encode");
cb.commit();
cb.wait_until_completed();
let y_partial_gpu = unsafe {
- std::slice::from_raw_parts(y_partial_buf.contents() as *const bf16, sum_k * d_model)
+ std::slice::from_raw_parts(
+ y_partial_buf.contents() as *const bf16,
+ sum_k * d_model,
+ )
};
- let y_gpu = gather_and_finalize(y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);
+ let y_gpu = gather_and_finalize(
+ y_partial_gpu,
+ &data.topk_probs,
+ &scatter.perm_idx,
+ t,
+ k,
+ d_model,
+ );
assert_bf16_close(&y_gpu, &y_expected, 0.02, "2-pass decode multi-token");
eprintln!("[2-pass decode multi-token] ✓ PASSED");
@@ -602,21 +673,39 @@ fn test_two_pass_prefill_correctness() {
let gating_code = 2u32;
let silu_alpha = 1.0f32;
- eprintln!("[2-pass prefill] T={}, K={}, d_model={}, d_ff={}, E={}", t, k, d_model, d_ff, e);
+ eprintln!(
+ "[2-pass prefill] T={}, K={}, d_model={}, d_ff={}, E={}",
+ t, k, d_model, d_ff, e
+ );
let data = MoeTestData::generate(&mut rng, t, k, d_model, d_ff, e);
let scatter = scatter_by_expert(&data.x, &data.topk_ids, t, k, d_model, e);
let y_expected = cpu_moe_reference(
- &data.x, &data.topk_ids, &data.topk_probs, &data.w13, &data.w2,
- &data.up_biases, &data.down_biases, t, d_model, d_ff, k, gating_code,
- silu_alpha, f32::NEG_INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::INFINITY,
+ &data.x,
+ &data.topk_ids,
+ &data.topk_probs,
+ &data.w13,
+ &data.w2,
+ &data.up_biases,
+ &data.down_biases,
+ t,
+ d_model,
+ d_ff,
+ k,
+ gating_code,
+ silu_alpha,
+ f32::NEG_INFINITY,
+ f32::INFINITY,
+ f32::NEG_INFINITY,
+ f32::INFINITY,
);
// GPU buffers
let x_perm_buf = alloc_buffer_with_data(&ctx, &scatter.x_perm);
let offsets_buf = alloc_buffer_with_data(&ctx, &scatter.offsets);
- let row_expert_map_buf = alloc_buffer_with_data(&ctx, &scatter.row_expert_map);
+ let row_expert_map_buf =
+ alloc_buffer_with_data(&ctx, &scatter.row_expert_map);
let w13_buf = alloc_buffer_with_data(&ctx, &data.w13);
let w2_buf = alloc_buffer_with_data(&ctx, &data.w2);
let up_biases_buf = alloc_buffer_with_data(&ctx, &data.up_biases);
@@ -632,28 +721,59 @@ fn test_two_pass_prefill_correctness() {
let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);
let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
- let experts_kernel = MoeExpertsTwoPassPrefillKernel::new(&ctx).expect("kernel");
+ let experts_kernel =
+ MoeExpertsTwoPassPrefillKernel::new(&ctx).expect("kernel");
let cb = ctx.command_queue.new_command_buffer();
- experts_kernel.encode(&cb, MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf, expert_offsets: &offsets_buf,
- row_expert_map: &row_expert_map_buf, hidden_buffer: &hidden_buf,
- output_buffer: &y_partial_buf, w13_all: &w13_buf, w2_all: &w2_buf,
- up_biases: &up_biases_buf, down_biases: &down_biases_buf,
- tile_counts: &tile_counts_buf, tile_offsets: &tile_offsets_buf,
- tile_map: &tile_map_buf, total_tiles: &total_tiles_buf,
- dispatch_args: &dispatch_args_buf, total_rows: sum_k, d_model, d_ff, e,
- num_tiles_k: ((d_ff + 63) / 64) as u32, gating_code,
- gate_clip_min: f32::NEG_INFINITY, gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY, up_clip_max: f32::INFINITY,
- silu_alpha, data_type: KernelDataType::BFloat16,
- }).expect("encode");
+ experts_kernel
+ .encode(
+ &cb,
+ MoeExpertsTwoPassArguments {
+ x_perm_buffer: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &row_expert_map_buf,
+ hidden_buffer: &hidden_buf,
+ output_buffer: &y_partial_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &tile_counts_buf,
+ tile_offsets: &tile_offsets_buf,
+ tile_map: &tile_map_buf,
+ total_tiles: &total_tiles_buf,
+ dispatch_args: &dispatch_args_buf,
+ total_rows: sum_k,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k: ((d_ff + 63) / 64) as u32,
+ gating_code,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ silu_alpha,
+ data_type: KernelDataType::BFloat16,
+ },
+ )
+ .expect("encode");
cb.commit();
cb.wait_until_completed();
let y_partial_gpu = unsafe {
- std::slice::from_raw_parts(y_partial_buf.contents() as *const bf16, sum_k * d_model)
+ std::slice::from_raw_parts(
+ y_partial_buf.contents() as *const bf16,
+ sum_k * d_model,
+ )
};
- let y_gpu = gather_and_finalize(y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);
+ let y_gpu = gather_and_finalize(
+ y_partial_gpu,
+ &data.topk_probs,
+ &scatter.perm_idx,
+ t,
+ k,
+ d_model,
+ );
assert_bf16_close(&y_gpu, &y_expected, 0.02, "2-pass prefill");
eprintln!("[2-pass prefill] ✓ PASSED");