A high-performance inference engine for AI models
Chunked GDN improvements (#978)
CCYehe744e7cf4dc5
diff --git a/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs b/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rsindex 648c1ad58..e51798d8f 100644--- a/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs+++ b/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs@@ -26,6 +26,7 @@ pub trait DeltaNetChunkedPrefill: Sized + Send + Sync {context: &<Self::Backend as Backend>::Context,outer_data_type: DataType,head_dim: u32,+ groups_per_head: u32,) -> Result<Option<Self>, <Self::Backend as Backend>::Error>;fn should_use(@@ -48,6 +49,7 @@ impl<B: Backend<Kernels: Kernels<DeltaNetChunkedPrefill = Unsupported<B>>>> Delt_context: &B::Context,_outer_data_type: DataType,_head_dim: u32,+ _groups_per_head: u32,) -> Result<Option<Self>, B::Error> {Ok(None)}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_pack.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_pack.rsindex 9f97cdadc..fe58ff141 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_pack.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_pack.rs@@ -4,35 +4,34 @@ use uzu_engine_macros::kernel;use crate::data_type::DataTypeRepr;-#[kernel(Conv1dPack)]-#[variants(StateT, f32, bf16)]-#[variants(InputT, f32, bf16)]-pub fn conv1d_pack<StateT: DataTypeRepr + Float, InputT: DataTypeRepr + Float>(- state_in: *const StateT,- x: *const InputT,- padded: *mut StateT,- state_stride: u32,- row_stride: u32,+#[kernel(DeltaNetConvPack)]+#[variants(T, f32, bf16)]+#[variants(KERNEL_SIZE, 4)]+pub fn delta_net_conv_pack<T: DataTypeRepr + Float, const KERNEL_SIZE: u32>(+ state_in: *const f32,+ x: *const T,+ packed: *mut f32,suffix_len: u32,- num_channels: u32,+ tile_tokens: u32,+ row_stride: u32,+ conv_dim: u32,) {- let state_stride = state_stride as usize;+ let taps = KERNEL_SIZE as usize - 1;+ let tile_tokens = tile_tokens as usize;let row_stride = row_stride as usize;- let suffix_len = suffix_len as usize;- let num_channels = num_channels as usize;+ let conv_dim = conv_dim as usize;- unsafe {- for channel_idx in 0..num_channels {- for row_idx in 0..state_stride + suffix_len {- let padded_index = row_idx * row_stride + channel_idx;- if row_idx < state_stride {- let state_index = channel_idx * state_stride + row_idx;- *padded.add(padded_index) = *state_in.add(state_index);- } else {- let token = row_idx - state_stride;- let x_index = token * row_stride + channel_idx;- *padded.add(padded_index) = StateT::from((*x.add(x_index)).to_f32().unwrap()).unwrap();- }+ for tile in 0..(suffix_len as usize).div_ceil(tile_tokens) {+ for tap in 0..taps {+ for channel in 0..conv_dim {+ let value = unsafe {+ if tile == 0 {+ *state_in.add(channel * taps + tap)+ } else {+ (*x.add((tile * tile_tokens - taps + tap) * row_stride + channel)).to_f32().unwrap()+ }+ };+ unsafe { *packed.add((tile * taps + tap) * conv_dim + channel) = value };}}}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rsindex 991154674..af84400d8 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs@@ -4,69 +4,61 @@ use uzu_engine_macros::kernel;use crate::{backends::common::gpu_types::activation_type::ActivationType, data_type::DataTypeRepr};-// Multi-token causal conv1d with SiLU for DeltaNet.-// Reads from conv_padded (Conv1dPack output), writes conv'd+SiLU'd values-// back to in_proj with stride out_stride, updates conv state.+// Multi-token causal conv1d with SiLU for DeltaNet, in place on in_proj. Each token tile starts from its packed rows (see+// DeltaNetConvPack), so tiles are independent; the last tile writes the new conv state.//-// conv_padded: [state_stride + suffix_len, row_stride] — packed history + input-// conv_weight: [conv_dim, kernel_size]-// bias: [conv_dim] (optional)-// in_proj: [suffix_len, out_stride] — first conv_dim channels overwritten-// state_out: [conv_dim, state_stride]+// packed: [num_tiles, KERNEL_SIZE - 1, conv_dim]+// conv_weight: [conv_dim, KERNEL_SIZE]+// bias: [conv_dim] (optional)+// in_proj: [suffix_len, out_stride] - first conv_dim channels overwritten+// state_out: [conv_dim, KERNEL_SIZE - 1]#[kernel(DeltaNetConvScan)]#[variants(T, f32, bf16)]-pub fn delta_net_conv_scan<T: DataTypeRepr + Float>(- conv_padded: *const f32,+#[variants(KERNEL_SIZE, 4)]+pub fn delta_net_conv_scan<T: DataTypeRepr + Float, const KERNEL_SIZE: u32>(+ packed: *const f32,conv_weight: *const f32,#[optional(has_bias)] bias: Option<*const f32>,in_proj: *mut T,state_out: *mut f32,suffix_len: u32,- kernel_size: u32,- row_stride: u32,- state_stride: u32,+ tile_tokens: u32,conv_dim: u32,out_stride: u32,#[specialize] has_bias: bool,) {+ let kernel_size = KERNEL_SIZE as usize;+ let taps = kernel_size - 1;let suffix_len = suffix_len as usize;- let kernel_size = kernel_size as usize;- let row_stride = row_stride as usize;- let state_stride = state_stride as usize;+ let tile_tokens = tile_tokens as usize;let conv_dim = conv_dim as usize;let out_stride = out_stride as usize;- for token in 0..suffix_len {+ for tile in 0..suffix_len.div_ceil(tile_tokens) {for channel in 0..conv_dim {- let w_offset = channel * kernel_size;-- let mut acc = if has_bias {- unsafe { *bias.unwrap().add(channel) }- } else {- 0.0f32- };-- for tap in 0..kernel_size {- let padded_row = token + tap;- let padded_idx = padded_row * row_stride + channel;- let sample = unsafe { *conv_padded.add(padded_idx) };- acc += unsafe { *conv_weight.add(w_offset + tap) } * sample;- }-- unsafe {- *in_proj.add(token * out_stride + channel) = T::from(ActivationType::SILU.activate(acc)).unwrap();+ let mut window: Vec<f32> =+ (0..taps).map(|tap| unsafe { *packed.add((tile * taps + tap) * conv_dim + channel) }).collect();+ for token in tile * tile_tokens..((tile + 1) * tile_tokens).min(suffix_len) {+ let sample = unsafe { (*in_proj.add(token * out_stride + channel)).to_f32().unwrap() };+ let mut acc = if has_bias {+ unsafe { *bias.unwrap().add(channel) }+ } else {+ 0.0f32+ };+ for (tap, &previous) in window.iter().enumerate() {+ acc += unsafe { *conv_weight.add(channel * kernel_size + tap) } * previous;+ }+ acc += unsafe { *conv_weight.add(channel * kernel_size + taps) } * sample;+ unsafe {+ *in_proj.add(token * out_stride + channel) = T::from(ActivationType::SILU.activate(acc)).unwrap();+ }+ window.remove(0);+ window.push(sample);}- }- }-- // state_out layout: [conv_dim, state_stride] (channel-major)- for channel in 0..conv_dim {- for tap in 0..state_stride {- let padded_row = suffix_len + tap;- let padded_idx = padded_row * row_stride + channel;- let sample = unsafe { *conv_padded.add(padded_idx) };- unsafe {- *state_out.add(channel * state_stride + tap) = sample;+ if (tile + 1) * tile_tokens >= suffix_len {+ for (tap, &value) in window.iter().enumerate() {+ unsafe { *state_out.add(channel * taps + tap) = value };+ }}}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/a_diag_inv.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/a_diag_inv.metaldeleted file mode 100644index 266139d08..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/a_diag_inv.metal+++ /dev/null@@ -1,92 +0,0 @@-#include <metal_stdlib>-#include "../../common/defines.h"-#include "../../common/dsl.h"-#include "../common/tri_inv.h"--using namespace metal;--#define DIAG_BLOCK_SIZE 16u--// kk: [chunks, num_k_heads, CHUNK_SIZE, CHUNK_SIZE] K K^T from Gram.-// beta: [suffix_len, num_v_heads].-// g: [num_v_heads, suffix_len] cumulative log decay.-// a_packed: [chunks, num_v_heads, num_blocks, ceil(num_blocks/2),-// DIAG_BLOCK_SIZE, 2*DIAG_BLOCK_SIZE].-// a_inv: [chunks, num_v_heads, num_blocks, DIAG_BLOCK_SIZE, DIAG_BLOCK_SIZE].-//-// One threadgroup owns one (chunk, v-head, column-pair, row-block). It materializes-// A[row,col] = beta[row] * exp(g[row] - g[col]) * kk[row,col] for col < row,-// writes packed column-pair blocks through the diagonal pair, copies the-// diagonal block into threadgroup memory, then writes (I + A_diag)^-1 to a_inv.-template <uint CHUNK_SIZE>-VARIANTS(CHUNK_SIZE, 32, 64)-KERNEL(DeltaNetChunkedADiagInv)(- device const float* kk,- device const float* beta,- device const float* g,- device float* a_packed,- device float* a_inv,- constant const uint& num_v_heads,- constant const uint& num_k_heads,- constant const uint& suffix_len,- threadgroup float diag_a_tile[DIAG_BLOCK_SIZE * DIAG_BLOCK_SIZE],- const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),- const uint hv_idx GROUPS(num_v_heads),- const uint block_col_pair_idx GROUPS(- CHUNK_SIZE.div_ceil(DIAG_BLOCK_SIZE) * CHUNK_SIZE.div_ceil(DIAG_BLOCK_SIZE).div_ceil(2)- ),- const uint lane THREADS(METAL_SIMD_SIZE)-) {- constexpr uint num_blocks = (CHUNK_SIZE + DIAG_BLOCK_SIZE - 1) / DIAG_BLOCK_SIZE;- constexpr uint num_col_pairs = (num_blocks + 1) / 2;- const uint block_idx = block_col_pair_idx / num_col_pairs;- const uint col_pair_idx = block_col_pair_idx - block_idx * num_col_pairs;- const uint hk_idx = hv_idx / (num_v_heads / num_k_heads);- const uint token_base = chunk_idx * CHUNK_SIZE;- const uint row_base = block_idx * DIAG_BLOCK_SIZE;- const uint kk_base = (chunk_idx * num_k_heads + hk_idx) * CHUNK_SIZE * CHUNK_SIZE;- const uint diag_col_pair = block_idx / 2;- const uint diag_col_offset = (block_idx % 2) * DIAG_BLOCK_SIZE;- const device float* g_head = g + hv_idx * suffix_len;- if (col_pair_idx > diag_col_pair) {- return;- }-- METAL_PRAGMA_UNROLL- for (uint idx = lane; idx < DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE; idx += METAL_SIMD_SIZE) {- const uint local_col = idx % (2 * DIAG_BLOCK_SIZE);- const uint local_row = (idx / (2 * DIAG_BLOCK_SIZE)) % DIAG_BLOCK_SIZE;- const uint row = row_base + local_row;- const uint col = col_pair_idx * 2 * DIAG_BLOCK_SIZE + local_col;- const uint row_token = token_base + row;- const uint col_token = token_base + col;-- float value = 0.0f;- if (row < CHUNK_SIZE && col < CHUNK_SIZE && row_token < suffix_len && col_token < suffix_len && col < row) {- const float beta_row = beta[row_token * num_v_heads + hv_idx];- const float g_row = g_head[row_token];- const float g_col = g_head[col_token];- value = beta_row * fast::exp(g_row - g_col) * kk[kk_base + row * CHUNK_SIZE + col];- }-- const uint out_idx =- (((chunk_idx * num_v_heads + hv_idx) * num_blocks + block_idx) * num_col_pairs + col_pair_idx) *- (DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE) +- local_row * (2 * DIAG_BLOCK_SIZE) + local_col;- a_packed[out_idx] = value;- if (col_pair_idx == diag_col_pair && local_col >= diag_col_offset &&- local_col < diag_col_offset + DIAG_BLOCK_SIZE) {- diag_a_tile[local_row * DIAG_BLOCK_SIZE + local_col - diag_col_offset] = value;- }- }-- if (col_pair_idx != diag_col_pair) {- return;- }- threadgroup_barrier(mem_flags::mem_threadgroup);- device float* a_inv_block =- a_inv + ((chunk_idx * num_v_heads + hv_idx) * num_blocks + block_idx) * (DIAG_BLOCK_SIZE * DIAG_BLOCK_SIZE);- const uint block_tokens =- token_base + row_base < suffix_len ? min(uint(DIAG_BLOCK_SIZE), suffix_len - token_base - row_base) : 0u;- invert_lower_triangular_block<DIAG_BLOCK_SIZE>(a_inv_block, diag_a_tile, block_tokens, lane);-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/causal_inv.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/causal_inv.metaldeleted file mode 100644index efccb1b1c..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/causal_inv.metal+++ /dev/null@@ -1,102 +0,0 @@-#include <metal_stdlib>-#include "../../common/defines.h"-#include "../../common/dsl.h"-#include "../../matmul/common/fragment.h"-#include "../../matmul/common/simdgroup_fragment_ops.h"--using namespace metal;-using namespace uzu::matmul;--#define DIAG_BLOCK_SIZE 16u--// Materializes dense T = (I + A)^-1 as bf16 for output and state. This is block-// forward substitution over a_packed plus the per-block inverses from ADiagInv.-template <uint CHUNK_SIZE, uint BV>-VARIANTS(CHUNK_SIZE, 32, 64)-VARIANTS(BV, 16, 32)-KERNEL(DeltaNetChunkedCausalInv)(- device const float* a_packed,- device const float* a_inv,- device bfloat* t_out,- constant const uint& num_v_heads,- constant const uint& suffix_len,- const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),- const uint hv_idx GROUPS(num_v_heads),- const uint tile_idx GROUPS(CHUNK_SIZE.div_ceil(BV)),- const uint lane THREADS(METAL_SIMD_SIZE)-) {- using Ops = SimdgroupFragmentOps;- constexpr ushort TOKEN_FRAGMENTS = DIAG_BLOCK_SIZE / Ops::FRAGMENT_ROWS;- constexpr ushort VALUE_FRAGMENTS = BV / Ops::FRAGMENT_COLS;- using TileFragment = Fragment<float, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, Ops>;- using MatrixFragment = OperandFragment<float, TOKEN_FRAGMENTS, TOKEN_FRAGMENTS, Ops>;- using ValueFragment = OperandFragment<bfloat, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, Ops>;- using PairMatrixFragment = OperandFragment<float, TOKEN_FRAGMENTS, 2 * TOKEN_FRAGMENTS, Ops>;- using PairValueFragment = OperandFragment<bfloat, 2 * TOKEN_FRAGMENTS, VALUE_FRAGMENTS, Ops>;-- constexpr uint num_blocks = (CHUNK_SIZE + DIAG_BLOCK_SIZE - 1) / DIAG_BLOCK_SIZE;- constexpr uint num_col_pairs = (num_blocks + 1) / 2;- const uint slice_base = tile_idx * BV;-- const device float* a_blocks = a_packed + (chunk_idx * num_v_heads + hv_idx) * num_blocks * num_col_pairs *- (DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE);- const device float* inv_blocks =- a_inv + (chunk_idx * num_v_heads + hv_idx) * num_blocks * (DIAG_BLOCK_SIZE * DIAG_BLOCK_SIZE);- device bfloat* t_head = t_out + (chunk_idx * num_v_heads + hv_idx) * CHUNK_SIZE * CHUNK_SIZE + slice_base;-- for (uint block_idx = 0; block_idx < num_blocks; ++block_idx) {- const uint row_base = block_idx * DIAG_BLOCK_SIZE;-- // -- RHS = identity slice: acc[row, col] = (row_base+row == slice_base+col) -- TileFragment acc_t;- acc_t.map_coords(lane, [&](short local_row, short local_col, float) {- const uint global_row = row_base + uint(local_row);- const uint global_col = slice_base + uint(local_col);- return global_row == global_col ? 1.0f : 0.0f;- });-- // -- Forward substitution: subtract A . T_prev (T_prev in bf16, own slice) --- // a_packed stays positive for block inverse; dense solve subtracts it.- const uint num_full_pairs = block_idx / 2;- for (uint pair_idx = 0; pair_idx < num_full_pairs; ++pair_idx) {- PairMatrixFragment a_frag;- const device float* a_pair =- a_blocks + (block_idx * num_col_pairs + pair_idx) * (DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE);- a_frag.load_from(lane, fragment_source(a_pair, 2 * DIAG_BLOCK_SIZE));- a_frag.map([](float value) { return -value; });-- PairValueFragment t_prev_frag;- const device bfloat* t_prev = t_head + pair_idx * 2 * DIAG_BLOCK_SIZE * CHUNK_SIZE;- t_prev_frag.load_from(lane, fragment_source(t_prev, CHUNK_SIZE));- fragment_mma(acc_t, a_frag, t_prev_frag);- }-- for (uint prev_block_idx = num_full_pairs * 2; prev_block_idx < block_idx; ++prev_block_idx) {- MatrixFragment a_frag;- const device float* a_block =- a_blocks + (block_idx * num_col_pairs + prev_block_idx / 2) * (DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE) +- (prev_block_idx % 2) * DIAG_BLOCK_SIZE;- a_frag.load_from(lane, fragment_source(a_block, 2 * DIAG_BLOCK_SIZE));- a_frag.map([](float value) { return -value; });-- ValueFragment t_prev_frag;- const device bfloat* t_prev = t_head + prev_block_idx * DIAG_BLOCK_SIZE * CHUNK_SIZE;- t_prev_frag.load_from(lane, fragment_source(t_prev, CHUNK_SIZE));- fragment_mma(acc_t, a_frag, t_prev_frag);- }-- // -- Apply the diagonal-block inverse and store T ------------------------- MatrixFragment inv_frag;- inv_frag.load_from(- lane,- fragment_source(inv_blocks + block_idx * DIAG_BLOCK_SIZE * DIAG_BLOCK_SIZE, DIAG_BLOCK_SIZE)- );-- TileFragment t_block;- t_block.clear();- fragment_mma(t_block, inv_frag, acc_t);- t_block.store(lane, t_head + row_base * CHUNK_SIZE, CHUNK_SIZE);-- simdgroup_barrier(mem_flags::mem_device);- }-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/cumsum.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/cumsum.metaldeleted file mode 100644index 37e5277e8..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/cumsum.metal+++ /dev/null@@ -1,37 +0,0 @@-#include <metal_stdlib>-#include "../../common/defines.h"-#include "../../common/dsl.h"--using namespace metal;--template <uint CHUNK_SIZE>-VARIANTS(CHUNK_SIZE, 32, 64)-KERNEL(DeltaNetChunkedCumsum)(- device const float* log_decay,- device float* g_out,- constant const uint& num_v_heads,- constant const uint& suffix_len,- const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),- const uint hv_idx GROUPS(num_v_heads),- const uint lane THREADS(METAL_SIMD_SIZE)-) {- const uint token_begin = chunk_idx * CHUNK_SIZE;- const uint token_end = min(token_begin + CHUNK_SIZE, suffix_len);-- float carry = 0.0f;- METAL_PRAGMA_UNROLL- for (uint token_offset = 0; token_offset < CHUNK_SIZE; token_offset += METAL_SIMD_SIZE) {- const uint token = token_begin + token_offset + lane;- const bool valid = token < token_end;- const uint input_offset = token * num_v_heads + hv_idx;- const uint output_offset = hv_idx * suffix_len + token;- const float value = valid ? log_decay[input_offset] : 0.0f;- const float prefix = carry + simd_prefix_inclusive_sum(value);-- if (valid) {- g_out[output_offset] = prefix;- }-- carry += simd_sum(value);- }-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/gram.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/gram.metaldeleted file mode 100644index 2566e9499..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/gram.metal+++ /dev/null@@ -1,114 +0,0 @@-#include <metal_stdlib>-#include "../../common/defines.h"-#include "../../common/dsl.h"-#include "../common/gram.h"--using namespace metal;-using namespace uzu::matmul;--#define ROW_TILE 16u-#define COL_TILE 32u--// q_norm/k_norm: [suffix_len, key_dim], g: [num_v_heads, suffix_len].-// kk_out: [chunks, num_k_heads, CHUNK_SIZE, CHUNK_SIZE].-// qk_scaled_out: [chunks, num_v_heads, CHUNK_SIZE, CHUNK_SIZE].-// Computes K K^T per k-head and decay-scaled Q K^T per grouped v-head.-// MXU did not improve e2e vs simdgroup.-template <uint HEAD_K_DIM, uint CHUNK_SIZE>-VARIANTS(HEAD_K_DIM, 128)-VARIANTS(CHUNK_SIZE, 32, 64)-KERNEL(DeltaNetChunkedGram)(- device const float* q_norm,- device const float* k_norm,- device const float* g,- device float* kk_out,- device float* qk_scaled_out,- constant const uint& num_v_heads,- constant const uint& num_k_heads,- constant const uint& key_dim,- constant const uint& suffix_len,- const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),- const uint hk_idx GROUPS(num_k_heads),- const uint tile_idx GROUPS(CHUNK_SIZE.div_ceil(ROW_TILE) * CHUNK_SIZE.div_ceil(COL_TILE)),- const uint lane THREADS(METAL_SIMD_SIZE)-) {- using Ops = SimdgroupFragmentOps;- using InputType = float;- constexpr ushort ROW_FRAGMENTS = ROW_TILE / Ops::FRAGMENT_ROWS;- constexpr ushort COL_FRAGMENTS = COL_TILE / Ops::FRAGMENT_COLS;- using AccFragment = Fragment<float, ROW_FRAGMENTS, COL_FRAGMENTS, Ops>;- using LeftFragment = OperandFragment<InputType, ROW_FRAGMENTS, 1, Ops>;- using RightFragment = OperandFragment<InputType, 1, COL_FRAGMENTS, Ops, ReadTranspose>;-- static_assert(HEAD_K_DIM % Ops::FRAGMENT_ROWS == 0, "HEAD_K_DIM must align to fragment K");-- constexpr uint col_tiles = (CHUNK_SIZE + COL_TILE - 1) / COL_TILE;- const uint row_tile_idx = tile_idx / col_tiles;- const uint col_tile_idx = tile_idx - row_tile_idx * col_tiles;- const uint row_base = row_tile_idx * ROW_TILE;- const uint col_base = col_tile_idx * COL_TILE;- const uint chunk_token_base = chunk_idx * CHUNK_SIZE;-- const uint tile_rows = min(uint(ROW_TILE), uint(CHUNK_SIZE - row_base));- const uint tile_cols = min(uint(COL_TILE), uint(CHUNK_SIZE - col_base));- const uint row_token_base = chunk_token_base + row_base;- const uint col_token_base = chunk_token_base + col_base;- const uint valid_rows = row_token_base < suffix_len ? min(tile_rows, suffix_len - row_token_base) : 0u;- const uint valid_cols = col_token_base < suffix_len ? min(tile_cols, suffix_len - col_token_base) : 0u;- // Upper-causal tiles are zero.- if (valid_rows == 0 || valid_cols == 0 || col_base >= row_base + valid_rows) {- return;- }-- AccFragment kk_acc;- AccFragment qk_acc;- kk_acc.clear();- qk_acc.clear();-- for (uint k_block_start = 0; k_block_start < HEAD_K_DIM; k_block_start += Ops::FRAGMENT_ROWS) {- const device float* k_rows = k_norm + row_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;- const device float* q_rows = q_norm + row_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;- const device float* k_cols = k_norm + col_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;-- accumulate_dual_gram_tile<AccFragment, LeftFragment, RightFragment>(- kk_acc,- qk_acc,- k_rows,- q_rows,- k_cols,- int(key_dim),- ushort(valid_rows),- ushort(valid_cols),- Ops::FRAGMENT_ROWS,- valid_rows == tile_rows,- valid_cols == tile_cols,- ushort(lane)- );- }-- const short2 tile_dims = short2(tile_cols, tile_rows);- const uint kk_base = (chunk_idx * num_k_heads + hk_idx) * CHUNK_SIZE * CHUNK_SIZE + row_base * CHUNK_SIZE + col_base;- kk_acc.store_safe(lane, kk_out + kk_base, CHUNK_SIZE, tile_dims);-- // Store qk for each grouped v-head with decay scale and causal mask.- const uint valid_tokens = chunk_token_base < suffix_len ? min(uint(CHUNK_SIZE), suffix_len - chunk_token_base) : 0u;- const uint groups_per_head = num_v_heads / num_k_heads;- for (uint group = 0; group < groups_per_head; ++group) {- const uint hv_idx = hk_idx * groups_per_head + group;- const device float* g_head = g + hv_idx * suffix_len;- AccFragment scaled = qk_acc;- scaled.map_coords(lane, [&](short r, short c, float value) {- const uint row = row_base + uint(r);- const uint col = col_base + uint(c);- if (row >= valid_tokens || col >= valid_tokens || col > row) {- return 0.0f;- }- const float g_row = g_head[chunk_token_base + row];- const float g_col = g_head[chunk_token_base + col];- return value * fast::exp(g_row - g_col);- });- const uint dst_base =- (chunk_idx * num_v_heads + hv_idx) * CHUNK_SIZE * CHUNK_SIZE + row_base * CHUNK_SIZE + col_base;- scaled.store_safe(lane, qk_scaled_out + dst_base, CHUNK_SIZE, tile_dims);- }-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/mod.rsindex 93b523059..655a59f70 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/mod.rs@@ -1,6 +1,5 @@use super::super::{- DeltaNetChunkedADiagInvMetalKernel, DeltaNetChunkedCausalInvMetalKernel, DeltaNetChunkedCumsumMetalKernel,- DeltaNetChunkedGramMetalKernel, DeltaNetChunkedOutputAndStateMetalKernel, DeltaNetPrefillPrepMetalKernel,+ DeltaNetChunkedOutputAndStateMetalKernel, DeltaNetChunkedPrecomputeMetalKernel, DeltaNetPrefillPrepMetalKernel,};use crate::{backends::{@@ -18,17 +17,13 @@ use crate::{const MXU_MIN_T: u32 = 256;const CHUNK_SIZE: u32 = 64;-const BLOCK_SIZE: u32 = 16;const VT: u32 = 32;const INNER_DATA_TYPE: DataType = DataType::F32;pub struct MetalDeltaNetChunkedPrefill {min_t: u32,prep: DeltaNetPrefillPrepMetalKernel,- cumsum: DeltaNetChunkedCumsumMetalKernel,- gram: DeltaNetChunkedGramMetalKernel,- a_diag_inv: DeltaNetChunkedADiagInvMetalKernel,- causal_inv: DeltaNetChunkedCausalInvMetalKernel,+ precompute: DeltaNetChunkedPrecomputeMetalKernel,output_and_state: DeltaNetChunkedOutputAndStateMetalKernel,}@@ -39,6 +34,7 @@ impl DeltaNetChunkedPrefill for MetalDeltaNetChunkedPrefill {context: &MetalContext,outer_data_type: DataType,head_dim: u32,+ groups_per_head: u32,) -> Result<Option<Self>, <Metal as Backend>::Error> {if outer_data_type == DataType::F16 {return Ok(None);@@ -48,6 +44,11 @@ impl DeltaNetChunkedPrefill for MetalDeltaNetChunkedPrefill {return Ok(None);}+ // The precompute kernel assigns one grouped v-head to each of its four simdgroups.+ if groups_per_head > 4 {+ return Ok(None);+ }+let scratch_data_type = if outer_data_type == DataType::BF16 {DataType::BF16} else {@@ -64,10 +65,7 @@ impl DeltaNetChunkedPrefill for MetalDeltaNetChunkedPrefill {true,false,)?,- cumsum: DeltaNetChunkedCumsumMetalKernel::new(context, CHUNK_SIZE)?,- gram: DeltaNetChunkedGramMetalKernel::new(context, head_dim, CHUNK_SIZE)?,- a_diag_inv: DeltaNetChunkedADiagInvMetalKernel::new(context, CHUNK_SIZE)?,- causal_inv: DeltaNetChunkedCausalInvMetalKernel::new(context, CHUNK_SIZE, VT)?,+ precompute: DeltaNetChunkedPrecomputeMetalKernel::new(context, head_dim, CHUNK_SIZE)?,output_and_state: DeltaNetChunkedOutputAndStateMetalKernel::new(context,outer_data_type,@@ -94,8 +92,6 @@ impl DeltaNetChunkedPrefill for MetalDeltaNetChunkedPrefill {) -> Result<(), <Metal as Backend>::Error> {let suffix_len = args.suffix_len;let num_chunks = suffix_len.div_ceil(CHUNK_SIZE);- let num_blocks = CHUNK_SIZE.div_ceil(BLOCK_SIZE);- let num_col_pairs = num_blocks.div_ceil(2);let mut q_norm = command_buffer.allocate_scratch_for_shape(&[suffix_len, args.key_dim], INNER_DATA_TYPE)?;let mut k_norm = command_buffer.allocate_scratch_for_shape(&[suffix_len, args.key_dim], INNER_DATA_TYPE)?;@@ -103,18 +99,8 @@ impl DeltaNetChunkedPrefill for MetalDeltaNetChunkedPrefill {let mut log_decay =command_buffer.allocate_scratch_for_shape(&[suffix_len, args.num_heads], INNER_DATA_TYPE)?;let mut g = command_buffer.allocate_scratch_for_shape(&[suffix_len, args.num_heads], INNER_DATA_TYPE)?;- let mut kk = command_buffer- .allocate_scratch_for_shape(&[num_chunks, args.num_groups, CHUNK_SIZE, CHUNK_SIZE], INNER_DATA_TYPE)?;let mut qk = command_buffer.allocate_scratch_for_shape(&[num_chunks, args.num_heads, CHUNK_SIZE, CHUNK_SIZE], INNER_DATA_TYPE)?;- let mut a_packed = command_buffer.allocate_scratch_for_shape(- &[num_chunks, args.num_heads, num_blocks, num_col_pairs, BLOCK_SIZE, 2 * BLOCK_SIZE],- INNER_DATA_TYPE,- )?;- let mut a_inv = command_buffer.allocate_scratch_for_shape(- &[num_chunks, args.num_heads, num_blocks, BLOCK_SIZE, BLOCK_SIZE],- INNER_DATA_TYPE,- )?;let mut t_mat = command_buffer.allocate_scratch_for_shape(&[num_chunks, args.num_heads, CHUNK_SIZE, CHUNK_SIZE], DataType::BF16)?;@@ -134,31 +120,20 @@ impl DeltaNetChunkedPrefill for MetalDeltaNetChunkedPrefill {suffix_len,command_buffer,);- self.cumsum.encode(&log_decay, &mut g, args.num_heads, suffix_len, command_buffer);- self.gram.encode(+ self.precompute.encode(&q_norm,&k_norm,- &g,- &mut kk,+ &log_decay,+ &beta,+ &mut g,&mut qk,+ &mut t_mat,args.num_heads,args.num_groups,args.key_dim,suffix_len,command_buffer,);- self.a_diag_inv.encode(- &kk,- &beta,- &g,- &mut a_packed,- &mut a_inv,- args.num_heads,- args.num_groups,- suffix_len,- command_buffer,- );- self.causal_inv.encode(&a_packed, &a_inv, &mut t_mat, args.num_heads, suffix_len, command_buffer);self.output_and_state.encode(&q_norm,&k_norm,diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/output_and_state.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/output_and_state.metalindex dee18b6aa..36158d572 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/output_and_state.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/output_and_state.metal@@ -74,6 +74,10 @@ KERNEL(DeltaNetChunkedOutputAndState)(constexpr ushort TOKEN_FRAGMENTS = TOKEN_TILE / FR;constexpr ushort VALUE_FRAGMENTS = VT / FC;constexpr ushort KEY_FRAGMENTS = OUTPUT_STATE_KEY_TILE / FC;+ // Per-row terms are hoisted into row_* arrays: a lane owns rows lane_row + slot * ROW_STRIDE.+ constexpr ushort ROW_STRIDE = Ops::THREAD_ELEMENT_ROW_STRIDE;+ constexpr ushort ROW_SLOTS = Ops::THREAD_ELEMENT_ROWS;+ constexpr ushort TOKEN_ROW_SLOTS = TOKEN_FRAGMENTS * ROW_SLOTS;using TokenValueFragment = Fragment<float, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, Ops>;using TokenKeyFragment = OperandFragment<float, TOKEN_FRAGMENTS, 1, Ops>;@@ -89,6 +93,7 @@ KERNEL(DeltaNetChunkedOutputAndState)(if (value_base >= head_v_dim) {return;}+ const short lane_row = Ops::get_position(ushort(lane)).y;const uint hk_idx = hv_idx / (num_v_heads / num_k_heads);const uint num_chunks = (suffix_len + OUTPUT_STATE_CHUNK - 1) / OUTPUT_STATE_CHUNK;const uint conv_dim = 2 * key_dim + value_dim;@@ -115,6 +120,8 @@ KERNEL(DeltaNetChunkedOutputAndState)(};TokenValueFragment output_acc;+ float row_beta[TOKEN_ROW_SLOTS];+ float row_exp_g[TOKEN_ROW_SLOTS];// -- R phase: R = beta (.) (V - e^g (.) (K . S^T)) into residual_new_value_tile ----------if (simdgroup_idx < NUM_TOKEN_TILES) {const uint token_tile_base = simdgroup_idx * TOKEN_TILE;@@ -147,17 +154,23 @@ KERNEL(DeltaNetChunkedOutputAndState)(fragment_mma(output_acc, query_frag, state_frag);}+ METAL_PRAGMA_UNROLL+ for (ushort slot = 0; slot < TOKEN_ROW_SLOTS; ++slot) {+ const uint row = uint(lane_row + slot * ROW_STRIDE);+ const uint token = token_base + token_tile_base + row;+ const bool valid = row < valid_token_rows;+ row_beta[slot] = valid ? beta[token * num_v_heads + hv_idx] : 0.0f;+ row_exp_g[slot] = valid ? fast::exp(g_head[token]) : 0.0f;+ }+const device T* value_input_tile =in_proj + (token_base + token_tile_base) * total_proj_dim + 2 * key_dim + hv_idx * head_v_dim + value_base;residual_acc.map_coords(lane, [&](short row, short col, float state_projection) {if (uint(row) >= valid_token_rows) {return 0.0f;}- const uint token = token_base + token_tile_base + uint(row);- const float beta_t = beta[token * num_v_heads + hv_idx];- const float g_t = g_head[token];const float value_input = float(value_input_tile[uint(row) * total_proj_dim + uint(col)]);- return beta_t * (value_input - fast::exp(g_t) * state_projection);+ return row_beta[row / ROW_STRIDE] * (value_input - row_exp_g[row / ROW_STRIDE] * state_projection);});residual_acc.store(lane, residual_new_value_tile + token_tile_base * VT, int(VT));}@@ -209,13 +222,7 @@ KERNEL(DeltaNetChunkedOutputAndState)(const uint valid_token_rows =token_tile_base < valid_tokens ? min(uint(TOKEN_TILE), valid_tokens - token_tile_base) : 0u;- output_acc.map_coords(lane, [&](short row, short, float value) {- if (uint(row) >= valid_token_rows) {- return 0.0f;- }- const uint token = token_base + token_tile_base + uint(row);- return value * fast::exp(g_head[token]);- });+ output_acc.map_coords(lane, [&](short row, short, float value) { return value * row_exp_g[row / ROW_STRIDE]; });const uint qk_tile_base =chunk_head_base * OUTPUT_STATE_CHUNK * OUTPUT_STATE_CHUNK + token_tile_base * OUTPUT_STATE_CHUNK;@@ -286,14 +293,16 @@ KERNEL(DeltaNetChunkedOutputAndState)(short(OUTPUT_STATE_KEY_TILE)));- key_frag.map_coords(lane, [&](short row, short, float value) {- if (uint(row) >= valid_source_tokens) {- return 0.0f;- }- const uint token = token_base + source_token_block_start + uint(row);- // beta is already folded into Vnew through R.- return value * fast::exp(g_last - g_head[token]);- });+ float row_decay[ROW_SLOTS];+ METAL_PRAGMA_UNROLL+ for (ushort slot = 0; slot < ROW_SLOTS; ++slot) {+ const uint row = uint(lane_row + slot * ROW_STRIDE);+ row_decay[slot] = row < valid_source_tokens+ ? fast::exp(g_last - g_head[token_base + source_token_block_start + row])+ : 0.0f;+ }+ // beta is already folded into Vnew through R.+ key_frag.map_coords(lane, [&](short row, short, float value) { return value * row_decay[row / ROW_STRIDE]; });fragment_mma(state_update_acc, new_value_frag, key_frag);}diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/precompute.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/precompute.metalnew file mode 100644index 000000000..df8e83249--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/precompute.metal@@ -0,0 +1,261 @@+#include <metal_stdlib>+#include "../../common/defines.h"+#include "../../common/dsl.h"+#include "../../common/thread_context.h"+#include "../common/gram.h"+#include "../common/tri_inv.h"++using namespace metal;+using namespace uzu::matmul;++#define BLOCK_SIZE 16u+#define COL_TILE 32u+#define MAX_BLOCKS 4u+#define A_TILE_COUNT 6u+#define MAX_GROUPS 4u+#define PRECOMPUTE_THREADS 128++// q_norm: [suffix_len, key_dim].+// k_norm: [suffix_len, key_dim].+// log_decay: [suffix_len, num_v_heads].+// beta: [suffix_len, num_v_heads].+// g_out: [num_v_heads, suffix_len] chunk-local cumulative log decay.+// qk_scaled_out: [chunks, num_v_heads, CHUNK, CHUNK] decay-scaled causal Q K^T.+// t_out: [chunks, num_v_heads, CHUNK, CHUNK] dense T = (I + A)^-1, bf16.+//+// One threadgroup owns one (chunk, k-head); simdgroup b owns block row b. Per grouped v-head:+// 1. g = prefix sum of log_decay.+// 2. qk_scaled = exp(g_r - g_c) * Q K^T for c <= r.+// 3. A = beta_r * exp(g_r - g_c) * K K^T for c < r.+// 4. T = (I + A)^-1 by block forward substitution.+template <uint HEAD_K_DIM, uint CHUNK_SIZE>+VARIANTS(HEAD_K_DIM, 128)+VARIANTS(CHUNK_SIZE, 64)+KERNEL(DeltaNetChunkedPrecompute)(+ device const float* q_norm,+ device const float* k_norm,+ device const float* log_decay,+ device const float* beta,+ device float* g_out,+ device float* qk_scaled_out,+ device bfloat* t_out,+ constant const uint& num_v_heads,+ constant const uint& num_k_heads,+ constant const uint& key_dim,+ constant const uint& suffix_len,+ threadgroup float a_tiles[A_TILE_COUNT * BLOCK_SIZE * COL_TILE],+ threadgroup float diag_inverses[MAX_BLOCKS * BLOCK_SIZE * BLOCK_SIZE],+ threadgroup float g_tiles[MAX_GROUPS * CHUNK_SIZE],+ threadgroup float beta_tiles[MAX_GROUPS * CHUNK_SIZE],+ const ThreadContext thread_context,+ const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),+ const uint hk_idx GROUPS(num_k_heads),+ const uint thread_idx THREADS(PRECOMPUTE_THREADS)+) {+ using Ops = SimdgroupFragmentOps;+ constexpr ushort ROW_FRAGMENTS = BLOCK_SIZE / Ops::FRAGMENT_ROWS;+ constexpr ushort GRAM_COL_FRAGMENTS = COL_TILE / Ops::FRAGMENT_COLS;+ using GramAccFragment = Fragment<float, ROW_FRAGMENTS, GRAM_COL_FRAGMENTS, Ops>;+ using GramLeftFragment = OperandFragment<float, ROW_FRAGMENTS, 1, Ops>;+ using GramRightFragment = OperandFragment<float, 1, GRAM_COL_FRAGMENTS, Ops, ReadTranspose>;++ using TileFragment = Fragment<float, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;+ using MatrixFragment = OperandFragment<float, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;+ using ValueFragment = OperandFragment<bfloat, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;++ static_assert(HEAD_K_DIM % Ops::FRAGMENT_ROWS == 0, "HEAD_K_DIM must align to fragment K");+ constexpr uint num_blocks = CHUNK_SIZE / BLOCK_SIZE;+ static_assert(num_blocks == MAX_BLOCKS, "one simdgroup per block row");++ const uint lane = thread_context.simd_lane_id;+ const uint block_idx = thread_context.simdgroup_index;+ // First A tile of block row b in threadgroup memory: rows hold b / 2 + 1 tiles.+ auto tile_base = [](uint b) UZU_ALWAYS_INLINE { return b < 2 ? b : 2 * b - 2; };++ const uint chunk_token_base = chunk_idx * CHUNK_SIZE;+ const uint valid_tokens = min(uint(CHUNK_SIZE), suffix_len - chunk_token_base);+ const uint groups_per_head = num_v_heads / num_k_heads;+ const uint row_base = block_idx * BLOCK_SIZE;+ const uint row_token_base = chunk_token_base + row_base;+ const uint valid_rows = valid_tokens > row_base ? min(BLOCK_SIZE, valid_tokens - row_base) : 0u;+ const uint num_col_tiles = block_idx / 2 + 1;++ // Simdgroup b loads log_decay and beta of grouped v-head b ahead of the MMA loop.+ constexpr uint decay_passes = CHUNK_SIZE / METAL_SIMD_SIZE;+ const bool has_group = block_idx < groups_per_head;+ const uint group_hv_idx = hk_idx * groups_per_head + block_idx;+ float decay_values[decay_passes];+ float beta_values[decay_passes];+ METAL_PRAGMA_UNROLL+ for (uint pass = 0; pass < decay_passes; ++pass) {+ const uint local = pass * METAL_SIMD_SIZE + lane;+ const bool valid = has_group && local < valid_tokens;+ const uint offset = (chunk_token_base + local) * num_v_heads + group_hv_idx;+ decay_values[pass] = valid ? log_decay[offset] : 0.0f;+ beta_values[pass] = valid ? beta[offset] : 0.0f;+ }++ // K K^T and Q K^T of this block row up to the diagonal; tail tiles stay zero.+ GramAccFragment kk_acc[2];+ GramAccFragment qk_acc[2];+ METAL_PRAGMA_UNROLL+ for (uint c = 0; c < 2; ++c) {+ kk_acc[c].clear();+ qk_acc[c].clear();+ }+ if (valid_rows != 0) {+ for (uint k_block_start = 0; k_block_start < HEAD_K_DIM; k_block_start += Ops::FRAGMENT_ROWS) {+ const device float* k_rows = k_norm + row_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;+ const device float* q_rows = q_norm + row_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;+ METAL_PRAGMA_UNROLL+ for (uint c = 0; c < 2; ++c) {+ const uint col_base = c * COL_TILE;+ const uint col_token_base = chunk_token_base + col_base;+ const uint tile_cols = min(uint(COL_TILE), uint(CHUNK_SIZE - col_base));+ const uint valid_cols = valid_tokens > col_base ? min(tile_cols, valid_tokens - col_base) : 0u;+ if (c < num_col_tiles && valid_cols != 0) {+ const device float* k_cols = k_norm + col_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;+ accumulate_dual_gram_tile<GramAccFragment, GramLeftFragment, GramRightFragment>(+ kk_acc[c],+ qk_acc[c],+ k_rows,+ q_rows,+ k_cols,+ int(key_dim),+ ushort(valid_rows),+ ushort(valid_cols),+ Ops::FRAGMENT_ROWS,+ valid_rows == BLOCK_SIZE,+ valid_cols == tile_cols,+ ushort(lane)+ );+ }+ }+ }+ }++ if (has_group) {+ float carry = 0.0f;+ METAL_PRAGMA_UNROLL+ for (uint pass = 0; pass < decay_passes; ++pass) {+ const uint local = pass * METAL_SIMD_SIZE + lane;+ const float prefix = carry + simd_prefix_inclusive_sum(decay_values[pass]);+ g_tiles[block_idx * CHUNK_SIZE + local] = prefix;+ beta_tiles[block_idx * CHUNK_SIZE + local] = beta_values[pass];+ if (local < valid_tokens) {+ g_out[group_hv_idx * suffix_len + chunk_token_base + local] = prefix;+ }+ carry += simd_sum(decay_values[pass]);+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ threadgroup float* own_tiles = a_tiles + tile_base(block_idx) * (BLOCK_SIZE * COL_TILE);++ for (uint group = 0; group < groups_per_head; ++group) {+ const threadgroup float* g_tile = g_tiles + group * CHUNK_SIZE;+ const uint head_base = chunk_idx * num_v_heads + hk_idx * groups_per_head + group;+ METAL_PRAGMA_UNROLL+ for (uint c = 0; c < 2; ++c) {+ if (c < num_col_tiles) {+ const uint col_base = c * COL_TILE;+ const short2 tile_dims = short2(min(uint(COL_TILE), uint(CHUNK_SIZE - col_base)), BLOCK_SIZE);+ GramAccFragment scaled = qk_acc[c];+ scaled.map_coords(lane, [&](short r, short col_in_tile, float value) UZU_ALWAYS_INLINE {+ const uint row = row_base + uint(r);+ const uint col = col_base + uint(col_in_tile);+ if (row >= valid_tokens || col >= valid_tokens || col > row) {+ return 0.0f;+ }+ return value * fast::exp(g_tile[row] - g_tile[col]);+ });+ scaled.store_safe(+ lane,+ qk_scaled_out + head_base * CHUNK_SIZE * CHUNK_SIZE + row_base * CHUNK_SIZE + col_base,+ CHUNK_SIZE,+ tile_dims+ );+ }+ }+ }++ for (uint group = 0; group < groups_per_head; ++group) {+ const threadgroup float* g_tile = g_tiles + group * CHUNK_SIZE;+ const threadgroup float* beta_tile = beta_tiles + group * CHUNK_SIZE;+ const uint head_base = chunk_idx * num_v_heads + hk_idx * groups_per_head + group;++ METAL_PRAGMA_UNROLL+ for (uint c = 0; c < 2; ++c) {+ if (c < num_col_tiles) {+ const uint col_base = c * COL_TILE;+ GramAccFragment a = kk_acc[c];+ a.map_coords(lane, [&](short r, short col_in_tile, float value) UZU_ALWAYS_INLINE {+ const uint row = row_base + uint(r);+ const uint col = col_base + uint(col_in_tile);+ if (row >= valid_tokens || col >= valid_tokens || col >= row) {+ return 0.0f;+ }+ return beta_tile[row] * fast::exp(g_tile[row] - g_tile[col]) * value;+ });+ a.store(lane, own_tiles + c * (BLOCK_SIZE * COL_TILE), COL_TILE);+ }+ }+ simdgroup_barrier(mem_flags::mem_threadgroup);+ invert_lower_triangular_block<BLOCK_SIZE, COL_TILE>(+ diag_inverses + block_idx * (BLOCK_SIZE * BLOCK_SIZE),+ own_tiles + (block_idx / 2) * (BLOCK_SIZE * COL_TILE) + (block_idx % 2) * BLOCK_SIZE,+ valid_rows,+ lane+ );++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // Solve this simdgroup's 16-column slice of T; blocks above the slice are zero, solved blocks stay in registers.+ const uint slice_base = block_idx * BLOCK_SIZE;+ device bfloat* t_head = t_out + head_base * CHUNK_SIZE * CHUNK_SIZE + slice_base;+ ValueFragment t_blocks[num_blocks];+ METAL_PRAGMA_UNROLL+ for (uint b = 0; b < num_blocks; ++b) {+ TileFragment t_block;+ t_block.clear();+ if (b >= block_idx) {+ const threadgroup float* b_tiles = a_tiles + tile_base(b) * (BLOCK_SIZE * COL_TILE);+ TileFragment acc_t;+ acc_t.map_coords(lane, [&](short local_row, short local_col, float) UZU_ALWAYS_INLINE {+ return b * BLOCK_SIZE + uint(local_row) == slice_base + uint(local_col) ? 1.0f : 0.0f;+ });+ METAL_PRAGMA_UNROLL+ for (uint prev_block_idx = 0; prev_block_idx < b; ++prev_block_idx) {+ if (prev_block_idx >= block_idx) {+ MatrixFragment a_frag;+ a_frag.load_from(+ lane,+ fragment_source(+ b_tiles + (prev_block_idx / 2) * (BLOCK_SIZE * COL_TILE) + (prev_block_idx % 2) * BLOCK_SIZE,+ COL_TILE+ )+ );+ a_frag.map([](float value) UZU_ALWAYS_INLINE { return -value; });+ fragment_mma(acc_t, a_frag, t_blocks[prev_block_idx]);+ }+ }+ MatrixFragment inv_frag;+ inv_frag.load_from(+ lane,+ fragment_source(+ static_cast<const threadgroup float*>(diag_inverses + b * (BLOCK_SIZE * BLOCK_SIZE)),+ BLOCK_SIZE+ )+ );+ fragment_mma(t_block, inv_frag, acc_t);+ }+ t_block.store(lane, t_head + b * BLOCK_SIZE * CHUNK_SIZE, CHUNK_SIZE);+ METAL_PRAGMA_UNROLL+ for (ushort e = 0; e < ValueFragment::ELEMENTS_PER_FRAGMENT; ++e) {+ t_blocks[b].elements()[e] = bfloat(t_block.elements()[e]);+ }+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/common/tri_inv.h b/crates/uzu-engine/src/backends/metal/kernel/gdn/common/tri_inv.hindex 796fcbe24..947fc5922 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/common/tri_inv.h+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/common/tri_inv.h@@ -5,9 +5,9 @@using namespace metal;-template <uint BLOCK>+template <uint BLOCK, uint A_STRIDE = BLOCK, typename OutPtr>METAL_FUNC void invert_lower_triangular_block(- device float* a_inv_block,+ OutPtr a_inv_block,threadgroup const float* a_tile,const uint block_size,const uint lane@@ -27,7 +27,7 @@ METAL_FUNC void invert_lower_triangular_block(METAL_PRAGMA_UNROLLfor (uint prev_row = 0; prev_row < BLOCK; prev_row++) {if (prev_row < row) {- acc += a_tile[row * BLOCK + prev_row] * inverse_col[prev_row];+ acc += a_tile[row * A_STRIDE + prev_row] * inverse_col[prev_row];}}inverse_col[row] = -acc;diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_pack.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_pack.metalindex 9e076f61f..4834321a7 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_pack.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_pack.metal@@ -1,27 +1,37 @@#include <metal_stdlib>#include "../common/dsl.h"-template <typename StateT, typename InputT>-VARIANTS(StateT, float, bfloat)-VARIANTS(InputT, float, bfloat)-PUBLIC KERNEL(Conv1dPack)(- device const StateT* state_in,- device const InputT* x,- device StateT* padded,- constant const uint& state_stride,- constant const uint& row_stride,+using namespace metal;++// Saves the KERNEL_SIZE - 1 rows before each token tile, which the in-place DeltaNetConvScan overwrites:+// packed[tile, tap] = row (tile * tile_tokens - TAPS + tap) of [conv state ++ x].+//+// state_in: [conv_dim, KERNEL_SIZE - 1]+// x: [suffix_len, row_stride]+// packed: [num_tiles, KERNEL_SIZE - 1, conv_dim]+template <typename T, uint KERNEL_SIZE>+VARIANTS(T, float, bfloat)+VARIANTS(KERNEL_SIZE, 4)+PUBLIC KERNEL(DeltaNetConvPack)(+ device const float* state_in,+ device const T* x,+ device float* packed,constant const uint& suffix_len,- constant const uint& num_channels,- const uint channel_idx AXIS(num_channels, 32),- const uint row_idx AXIS(state_stride + suffix_len, 1)+ constant const uint& tile_tokens,+ constant const uint& row_stride,+ constant const uint& conv_dim,+ const uint channel_idx AXIS(conv_dim, 128),+ const uint tile_idx AXIS(suffix_len.div_ceil(tile_tokens), 1)) {- const uint padded_index = row_idx * row_stride + channel_idx;- if (row_idx < state_stride) {- const uint state_index = channel_idx * state_stride + row_idx;- padded[padded_index] = state_in[state_index];- } else {- const uint token = row_idx - state_stride;- const uint x_index = token * row_stride + channel_idx;- padded[padded_index] = static_cast<StateT>(x[x_index]);+ constexpr uint TAPS = KERNEL_SIZE - 1;+ const size_t first_row = size_t(tile_idx) * tile_tokens;+ for (uint tap = 0; tap < TAPS; ++tap) {+ float value;+ if (tile_idx == 0) {+ value = state_in[channel_idx * TAPS + tap];+ } else {+ value = float(x[(first_row - TAPS + tap) * row_stride + channel_idx]);+ }+ packed[(tile_idx * TAPS + tap) * size_t(conv_dim) + channel_idx] = value;}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_scan.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_scan.metalindex 44aebdbe0..55b84af7d 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_scan.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_scan.metal@@ -4,51 +4,124 @@using namespace metal;-// Multi-token causal conv1d with SiLU for DeltaNet.-// Reads from conv_padded (Conv1dPack output), writes conv'd+SiLU'd-// values back to in_proj with stride out_stride, updates conv state.+// Causal conv1d + SiLU, in place on in_proj. A thread owns CHANNELS channels of one token tile; its first window+// comes from DeltaNetConvPack, so tiles are independent. The last tile writes state_out.//-// conv_padded: [state_stride + suffix_len, row_stride]-// conv_weight: [conv_dim, kernel_size]-// bias: [conv_dim] (optional)-// in_proj: [suffix_len, out_stride] — first conv_dim channels overwritten-// state_out: [conv_dim, state_stride]+// packed: [num_tiles, KERNEL_SIZE - 1, conv_dim]+// conv_weight: [conv_dim, KERNEL_SIZE]+// bias: [conv_dim] (optional)+// in_proj: [suffix_len, out_stride], first conv_dim channels overwritten+// state_out: [conv_dim, KERNEL_SIZE - 1]+// conv_dim and out_stride must be multiples of CHANNELS.++constant constexpr uint CHANNELS = 8;+constant constexpr uint VECTOR_WIDTH = 4;+constant constexpr uint ROWS_IN_FLIGHT = 4;template <typename T>+METAL_FUNC void load_channels(const device T* source, thread float (&values)[CHANNELS]) {+ const device vec<T, VECTOR_WIDTH>* vectors = reinterpret_cast<const device vec<T, VECTOR_WIDTH>*>(source);+ for (uint vector_idx = 0; vector_idx < CHANNELS / VECTOR_WIDTH; ++vector_idx) {+ const vec<float, VECTOR_WIDTH> chunk = vec<float, VECTOR_WIDTH>(vectors[vector_idx]);+ for (uint i = 0; i < VECTOR_WIDTH; ++i) {+ values[vector_idx * VECTOR_WIDTH + i] = chunk[i];+ }+ }+}++template <typename T>+METAL_FUNC void store_channels(device T* destination, const thread float (&values)[CHANNELS]) {+ device vec<T, VECTOR_WIDTH>* vectors = reinterpret_cast<device vec<T, VECTOR_WIDTH>*>(destination);+ for (uint vector_idx = 0; vector_idx < CHANNELS / VECTOR_WIDTH; ++vector_idx) {+ vec<float, VECTOR_WIDTH> chunk;+ for (uint i = 0; i < VECTOR_WIDTH; ++i) {+ chunk[i] = values[vector_idx * VECTOR_WIDTH + i];+ }+ vectors[vector_idx] = vec<T, VECTOR_WIDTH>(chunk);+ }+}++template <typename T, uint KERNEL_SIZE>VARIANTS(T, float, bfloat)+VARIANTS(KERNEL_SIZE, 4)PUBLIC KERNEL(DeltaNetConvScan)(- device const float* conv_padded,+ device const float* packed,device const float* conv_weight,device const float* bias OPTIONAL(has_bias),device T* in_proj,device float* state_out,constant const uint& suffix_len,- constant const uint& kernel_size,- constant const uint& row_stride,- constant const uint& state_stride,+ constant const uint& tile_tokens,constant const uint& conv_dim,constant const uint& out_stride,const bool has_bias SPECIALIZE,- const uint token_idx AXIS(suffix_len + kernel_size - 1, 1),- const uint channel_idx AXIS(conv_dim, 32)+ const uint channel_group AXIS(conv_dim / 8, 32),+ const uint tile_idx AXIS(suffix_len.div_ceil(tile_tokens), 4)) {- if (token_idx < suffix_len) {- const device float* weight_row = conv_weight + channel_idx * kernel_size;+ constexpr uint TAPS = KERNEL_SIZE - 1;+ const uint first_channel = channel_group * CHANNELS;- float acc = has_bias ? float(bias[channel_idx]) : 0.0f;+ float weights[CHANNELS * KERNEL_SIZE];+ const device float4* weight_vectors =+ reinterpret_cast<const device float4*>(conv_weight + first_channel * KERNEL_SIZE);+ for (uint i = 0; i < CHANNELS * KERNEL_SIZE / 4; ++i) {+ const float4 chunk = weight_vectors[i];+ for (uint lane = 0; lane < 4; ++lane) {+ weights[i * 4 + lane] = chunk[lane];+ }+ }+ float biases[CHANNELS];+ for (uint j = 0; j < CHANNELS; ++j) {+ biases[j] = has_bias ? bias[first_channel + j] : 0.0f;+ }- for (uint tap = 0; tap < kernel_size; ++tap) {- const uint padded_row = token_idx + tap;- const uint padded_index = padded_row * row_stride + channel_idx;- acc += float(weight_row[tap]) * conv_padded[padded_index];+ float window[TAPS][CHANNELS];+ for (uint tap = 0; tap < TAPS; ++tap) {+ load_channels(packed + (tile_idx * TAPS + tap) * conv_dim + first_channel, window[tap]);+ }++ const uint tile_begin = tile_idx * tile_tokens;+ const uint tile_end = min(tile_begin + tile_tokens, suffix_len);+ for (uint token = tile_begin; token < tile_end; token += ROWS_IN_FLIGHT) {+ float rows[ROWS_IN_FLIGHT][CHANNELS];+ for (uint r = 0; r < ROWS_IN_FLIGHT; ++r) {+ if (token + r < tile_end) {+ load_channels(in_proj + (token + r) * size_t(out_stride) + first_channel, rows[r]);+ }+ }+ for (uint r = 0; r < ROWS_IN_FLIGHT; ++r) {+ if (token + r < tile_end) {+ float outputs[CHANNELS];+ for (uint j = 0; j < CHANNELS; ++j) {+ float acc = biases[j];+ for (uint tap = 0; tap < TAPS; ++tap) {+ acc = fma(weights[j * KERNEL_SIZE + tap], window[tap][j], acc);+ }+ acc = fma(weights[j * KERNEL_SIZE + TAPS], rows[r][j], acc);+ outputs[j] = activate_silu(acc);+ }+ store_channels(in_proj + (token + r) * size_t(out_stride) + first_channel, outputs);+ for (uint tap = 0; tap + 1 < TAPS; ++tap) {+ for (uint j = 0; j < CHANNELS; ++j) {+ window[tap][j] = window[tap + 1][j];+ }+ }+ for (uint j = 0; j < CHANNELS; ++j) {+ window[TAPS - 1][j] = rows[r][j];+ }+ }}+ }- const size_t dst = size_t(token_idx) * out_stride + channel_idx;- in_proj[dst] = static_cast<T>(activate_silu(acc));- } else {- const uint tap = token_idx - suffix_len;- const size_t padded_index = size_t(token_idx) * row_stride + channel_idx;- const uint state_offset = channel_idx * state_stride;- state_out[state_offset + tap] = conv_padded[padded_index];+ if (tile_end == suffix_len) {+ device float4* state_vectors = reinterpret_cast<device float4*>(state_out + first_channel * TAPS);+ for (uint i = 0; i < CHANNELS * TAPS / 4; ++i) {+ float4 chunk;+ for (uint lane = 0; lane < 4; ++lane) {+ const uint flat = i * 4 + lane;+ chunk[lane] = window[flat % TAPS][flat / TAPS];+ }+ state_vectors[i] = chunk;+ }}}diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rsindex f7fc4f721..66efc68e6 100644--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs@@ -5,7 +5,7 @@ use crate::{backends::common::{Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding, Context, Kernels,kernel::{- Conv1dPackKernel, ConvTreeScanKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel,+ ConvTreeScanKernel, DeltaNetConvPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel,DeltaNetNormGateKernel, DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,StateAdvanceKernel,delta_net_chunked_prefill::{DeltaNetChunkedPrefill, DeltaNetChunkedPrefillArgs},@@ -30,6 +30,9 @@ use crate::{pub(crate) mod tree_verify;const INNER_DATA_TYPE: DataType = DataType::F32;+// Prefill conv: up to CONV_TILES token tiles of at least CONV_MIN_TILE_TOKENS (correctness needs >= KERNEL_SIZE - 1).+const CONV_TILES: u32 = 16;+const CONV_MIN_TILE_TOKENS: u32 = 16;enum DeltaNetSuffixStatus<B: Backend> {Flat {@@ -127,7 +130,7 @@ pub struct DeltaNet<B: Backend> {conv_weight: B::GlobalBuffer,conv_bias: Option<B::GlobalBuffer>,conv_update: <B::Kernels as Kernels>::DeltaNetConvUpdateKernel,- conv_pack: <B::Kernels as Kernels>::Conv1dPackKernel,+ conv_pack: <B::Kernels as Kernels>::DeltaNetConvPackKernel,conv_scan: <B::Kernels as Kernels>::DeltaNetConvScanKernel,conv_tree_scan: <B::Kernels as Kernels>::ConvTreeScanKernel,a_log: B::GlobalBuffer,@@ -164,9 +167,9 @@ impl<B: Backend> DeltaNet<B> {parameter_tree: &ParameterTree<B>,context: &B::Context,) -> Result<(Self, Option<B::GlobalBuffer>), DeltaNetNewError<B>> {- if config.kernel_size < 2 {+ if config.kernel_size != 4 {return Err(DeltaNetNewError::UnsupportedConfiguration(format!(- "kernel_size must be >= 2, got {}",+ "kernel_size must be 4, got {}",config.kernel_size)));}@@ -187,6 +190,12 @@ impl<B: Backend> DeltaNet<B> {let value_dim = config.num_heads * config.value_head_dim;let conv_dim = key_dim * 2 + value_dim;let total_proj_dim = conv_dim + value_dim + config.num_heads + config.num_heads;+ // The prefill conv scan moves 8 channels per thread as vec4 pairs: the in_proj row stride must be a multiple of 8.+ if !total_proj_dim.is_multiple_of(8) {+ return Err(DeltaNetNewError::UnsupportedConfiguration(format!(+ "in_proj width must be a multiple of 8, got {total_proj_dim}"+ )));+ }let (in_projection, in_projection_input_hadamard_factors) =<dyn Linear<B>>::new_with_input_rht_mixed_precision(@@ -212,11 +221,16 @@ impl<B: Backend> DeltaNet<B> {let conv_update =<B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, outer_data_type, conv_config.has_biases).map_err(DeltaNetNewError::Backend)?;- let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, INNER_DATA_TYPE, outer_data_type)- .map_err(DeltaNetNewError::Backend)?;- let conv_scan =- <B::Kernels as Kernels>::DeltaNetConvScanKernel::new(context, outer_data_type, conv_config.has_biases)+ let conv_pack =+ <B::Kernels as Kernels>::DeltaNetConvPackKernel::new(context, outer_data_type, config.kernel_size).map_err(DeltaNetNewError::Backend)?;+ let conv_scan = <B::Kernels as Kernels>::DeltaNetConvScanKernel::new(+ context,+ outer_data_type,+ config.kernel_size,+ conv_config.has_biases,+ )+ .map_err(DeltaNetNewError::Backend)?;let conv_tree_scan = <B::Kernels as Kernels>::ConvTreeScanKernel::new(context,outer_data_type,@@ -274,8 +288,13 @@ impl<B: Backend> DeltaNet<B> {None};- let chunked = <B::Kernels as Kernels>::DeltaNetChunkedPrefill::new(context, outer_data_type, config.head_dim)- .map_err(DeltaNetNewError::Backend)?;+ let chunked = <B::Kernels as Kernels>::DeltaNetChunkedPrefill::new(+ context,+ outer_data_type,+ config.head_dim,+ config.num_heads / config.num_groups,+ )+ .map_err(DeltaNetNewError::Backend)?;let out_projection = <dyn Linear<B>>::new_mixed_precision(value_dim,@@ -534,32 +553,29 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {command_buffer,);} else {- let mut padded = command_buffer- .allocate_scratch_for_shape(- &[batch_dim.size() + self.kernel_size - 1, self.total_proj_dim],- INNER_DATA_TYPE,- )+ let conv_tile_tokens = batch_dim.size().div_ceil(CONV_TILES).max(CONV_MIN_TILE_TOKENS);+ debug_assert!(conv_tile_tokens >= self.kernel_size - 1);+ let mut packed = command_buffer+ .allocate_scratch_for_shape(&[CONV_TILES, self.kernel_size - 1, self.conv_dim], INNER_DATA_TYPE).map_err(MixerEncodeError::Backend)?;self.conv_pack.encode(&state.conv_state,&in_projected,- &mut padded,- self.kernel_size - 1,- self.total_proj_dim,+ &mut packed,batch_dim.size(),+ conv_tile_tokens,+ self.total_proj_dim,self.conv_dim,command_buffer,);self.conv_scan.encode(- &padded,+ &packed,&self.conv_weight,self.conv_bias.as_ref(),&mut in_projected,&mut state.conv_state,batch_dim.size(),- self.kernel_size,- self.total_proj_dim,- self.kernel_size - 1,+ conv_tile_tokens,self.conv_dim,self.total_proj_dim,command_buffer,diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/conv_pack_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/conv_pack_test.rsindex 2b0d2bc27..19e17b92b 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/conv_pack_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/conv_pack_test.rs@@ -9,133 +9,110 @@ use crate::{backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- kernel::Conv1dPackKernel,+ kernel::DeltaNetConvPackKernel,},cpu::Cpu,},- data_type::{DataType, DataTypeRepr},+ data_type::DataTypeRepr,tests::{assert::assert_eq_float,helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},},};+const KERNEL_SIZE: u32 = 4;+const TAPS: u32 = KERNEL_SIZE - 1;+struct Input<T: DataTypeRepr + Float> {- state_in: Box<[T]>,+ state_in: Box<[f32]>,x: Box<[T]>,- state_stride: u32,- row_stride: u32,suffix_len: u32,- num_channels: u32,+ tile_tokens: u32,+ row_stride: u32,+ conv_dim: u32,}fn get_input<T: DataTypeRepr + Float>(- state_stride: u32,- row_stride: u32,suffix_len: u32,- num_channels: u32,+ tile_tokens: u32,+ row_stride: u32,+ conv_dim: u32,) -> Input<T> {- let state_size = num_channels as usize * state_stride as usize;- let x_size = suffix_len as usize * row_stride as usize;-- let state_in: Vec<T> = (0..state_size).map(|i| T::from(0.1 * (i as f64 + 1.0)).unwrap()).collect();- let x: Vec<T> = (0..x_size).map(|i| T::from(-0.05 * (i as f64 + 1.0)).unwrap()).collect();+ let state_in: Vec<f32> = (0..conv_dim * TAPS).map(|i| 0.1 * (i as f32 + 1.0)).collect();+ let x: Vec<T> = (0..suffix_len * row_stride).map(|i| T::from(-0.05 * (i as f64 + 1.0)).unwrap()).collect();Input {state_in: state_in.into_boxed_slice(),x: x.into_boxed_slice(),- state_stride,- row_stride,suffix_len,- num_channels,+ tile_tokens,+ row_stride,+ conv_dim,}}-fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Vec<T> {+fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Vec<f32> {let context = block_on(B::Context::new()).expect("Failed to create Context");- let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, T::data_type(), T::data_type())- .expect("Failed to create Conv1dPackKernel");-- let total_rows = input.state_stride as usize + input.suffix_len as usize;- let padded_size = total_rows * input.row_stride as usize;+ let kernel =+ <<B as Backend>::Kernels as Kernels>::DeltaNetConvPackKernel::new(&context, T::data_type(), KERNEL_SIZE)+ .expect("Failed to create DeltaNetConvPackKernel");- let state = create_buffer_with_data::<B, T>(&context, &input.state_in);+ let num_tiles = input.suffix_len.div_ceil(input.tile_tokens);+ let state = create_buffer_with_data::<B, f32>(&context, &input.state_in);let x = create_buffer_with_data::<B, T>(&context, &input.x);- let mut padded = create_buffer::<B, T>(&context, padded_size);+ let mut packed = create_buffer::<B, f32>(&context, (num_tiles * TAPS * input.conv_dim) as usize);let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");kernel.encode(&state,&x,- &mut padded,- input.state_stride,- input.row_stride,+ &mut packed,input.suffix_len,- input.num_channels,+ input.tile_tokens,+ input.row_stride,+ input.conv_dim,&mut command_buffer,);block_on(command_buffer.end_encoding().submit().wait_until_completed()).expect("Failed to wait command buffer");- buffer_to_vec(&padded)-}--fn get_test_data<T: DataTypeRepr + Float>(- state_stride: u32,- row_stride: u32,- suffix_len: u32,- num_channels: u32,-) -> (Input<T>, Vec<T>) {- let input = get_input::<T>(state_stride, row_stride, suffix_len, num_channels);- let expected = get_output::<Cpu, T>(&input);- (input, expected)+ buffer_to_vec(&packed)}fn test_internal<T: DataTypeRepr + Float + Debug + Display>(- input: &Input<T>,- expected: &[T],+ suffix_len: u32,+ tile_tokens: u32,+ row_stride: u32,+ conv_dim: u32,label: &str,) {- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {- 1e-3f32- } else {- 1e-6- };+ let input = get_input::<T>(suffix_len, tile_tokens, row_stride, conv_dim);+ let expected = get_output::<Cpu, T>(&input);for_each_non_cpu_backend!(|B| {- let output = get_output::<B, T>(input);- let msg = format!("Conv1dPack {} {} (type={})", std::any::type_name::<B>(), label, std::any::type_name::<T>(),);- assert_eq_float::<T>(expected, &output, eps, &msg);+ let output = get_output::<B, T>(&input);+ let msg =+ format!("DeltaNetConvPack {} {} (type={})", std::any::type_name::<B>(), label, std::any::type_name::<T>());+ assert_eq_float::<f32>(&expected, &output, 1e-6, &msg);});}fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {- // 4 channels, state_stride=3 (kernel_size=4), 2 tokens- let (input, expected) = get_test_data::<T>(3, 4, 2, 4);- test_internal(&input, &expected, "basic");-}--fn test_single_token<T: DataTypeRepr + Float + Debug + Display>() {- // 8 channels, state_stride=1, 1 token- let (input, expected) = get_test_data::<T>(1, 8, 1, 8);- test_internal(&input, &expected, "single_token");+ // several tiles, partial last tile+ test_internal::<T>(20, 8, 16, 8, "basic");}-fn test_many_tokens<T: DataTypeRepr + Float + Debug + Display>() {- // Minimal state (kernel_size=2), many tokens- let (input, expected) = get_test_data::<T>(1, 4, 8, 4);- test_internal(&input, &expected, "many_tokens");+fn test_single_tile<T: DataTypeRepr + Float + Debug + Display>() {+ // a single tile takes only the carried state+ test_internal::<T>(5, 16, 8, 8, "single_tile");}-fn test_large<T: DataTypeRepr + Float + Debug + Display>() {- // Larger dimensions closer to real usage- let (input, expected) = get_test_data::<T>(3, 128, 16, 128);- test_internal(&input, &expected, "large");+fn test_exact_tiles<T: DataTypeRepr + Float + Debug + Display>() {+ test_internal::<T>(64, 16, 24, 8, "exact_tiles");}fn test_channels_lt_row_stride<T: DataTypeRepr + Float + Debug + Display>() {- // num_channels < row_stride (padded channels that kernel doesn't write)- let (input, expected) = get_test_data::<T>(2, 8, 3, 6);- test_internal(&input, &expected, "channels_lt_row_stride");+ // conv_dim < row_stride (trailing projection channels are not packed)+ test_internal::<T>(30, 4, 40, 32, "channels_lt_row_stride");}// f32@@ -145,18 +122,13 @@ fn test_basic_f32() {}#[uzu_test]-fn test_single_token_f32() {- test_single_token::<f32>();-}--#[uzu_test]-fn test_many_tokens_f32() {- test_many_tokens::<f32>();+fn test_single_tile_f32() {+ test_single_tile::<f32>();}#[uzu_test]-fn test_large_f32() {- test_large::<f32>();+fn test_exact_tiles_f32() {+ test_exact_tiles::<f32>();}#[uzu_test]@@ -171,16 +143,16 @@ fn test_basic_bf16() {}#[uzu_test]-fn test_single_token_bf16() {- test_single_token::<bf16>();+fn test_single_tile_bf16() {+ test_single_tile::<bf16>();}#[uzu_test]-fn test_many_tokens_bf16() {- test_many_tokens::<bf16>();+fn test_exact_tiles_bf16() {+ test_exact_tiles::<bf16>();}#[uzu_test]-fn test_large_bf16() {- test_large::<bf16>();+fn test_channels_lt_row_stride_bf16() {+ test_channels_lt_row_stride::<bf16>();}diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/delta_net_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/delta_net_test.rsindex 40c0c40a0..20607f3a6 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/delta_net_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/delta_net_test.rs@@ -5,7 +5,7 @@ use uzu_engine_macros::uzu_test;#[cfg(backend = "metal")]use crate::{backends::{- common::kernel::{Conv1dPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel},+ common::kernel::{DeltaNetConvPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel},metal::Metal,},tests::helpers::buffer_prefix_to_vec,@@ -149,16 +149,17 @@ fn test_delta_net_conv_update_small() {assert_close(&cpu_state, &gpu_state, 1e-5, 1e-4, "ConvUpdate state");}-// DeltaNetConvScan+// DeltaNetConvPack + DeltaNetConvScan#[cfg(backend = "metal")]-#[uzu_test]-fn test_delta_net_conv_scan() {+fn test_delta_net_conv_scan_impl<B: Backend>(+ suffix_len: usize,+ tile_tokens: usize,+) {let conv_dim = 32;let kernel_size: usize = 4;let tap_count = kernel_size - 1;- let suffix_len: usize = 16;- let extra_dim = 10;+ let extra_dim = 16;let total_proj_dim = conv_dim + extra_dim;let w: Vec<f32> = (0..conv_dim * kernel_size).map(|i| ((i % 11) as f32) * 0.05 - 0.2).collect();@@ -187,45 +188,46 @@ fn test_delta_net_conv_scan() {}let ref_outputs: Vec<f32> = ref_outputs.iter().copied().map(|value| f32::from(bf16::from_f32(value))).collect();- // Test: Conv1dPack + DeltaNetConvScan on Metal- let context = block_on(<Metal as Backend>::Context::new()).expect("context");- let state_array = create_buffer_with_data::<Metal, f32>(&context, &init_state);- let mut in_proj_array = create_buffer_with_data::<Metal, bf16>(&context, &in_proj);- let w_array = create_buffer_with_data::<Metal, f32>(&context, &w);- let b_array = create_buffer_with_data::<Metal, f32>(&context, &b);+ // Test: DeltaNetConvPack + DeltaNetConvScan on backend B+ let context = block_on(B::Context::new()).expect("context");+ let mut state_array = create_buffer_with_data::<B, f32>(&context, &init_state);+ let mut in_proj_array = create_buffer_with_data::<B, bf16>(&context, &in_proj);+ let w_array = create_buffer_with_data::<B, f32>(&context, &w);+ let b_array = create_buffer_with_data::<B, f32>(&context, &b);- let padded_len = (tap_count + suffix_len) * total_proj_dim;- let mut padded_array = create_buffer::<Metal, f32>(&context, padded_len);- let mut state_out_array = create_buffer::<Metal, f32>(&context, conv_dim * tap_count);+ let num_tiles = suffix_len.div_ceil(tile_tokens);+ let mut packed_array = create_buffer::<B, f32>(&context, num_tiles * tap_count * conv_dim);let pack_kernel =- <<Metal as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, DataType::F32, DataType::BF16)+ <<B as Backend>::Kernels as Kernels>::DeltaNetConvPackKernel::new(&context, DataType::BF16, kernel_size as u32).expect("pack");- let scan_kernel =- <<Metal as Backend>::Kernels as Kernels>::DeltaNetConvScanKernel::new(&context, DataType::BF16, true)- .expect("scan");+ let scan_kernel = <<B as Backend>::Kernels as Kernels>::DeltaNetConvScanKernel::new(+ &context,+ DataType::BF16,+ kernel_size as u32,+ true,+ )+ .expect("scan");let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");pack_kernel.encode(&state_array,&in_proj_array,- &mut padded_array,- tap_count as u32,- total_proj_dim as u32,+ &mut packed_array,suffix_len as u32,+ tile_tokens as u32,+ total_proj_dim as u32,conv_dim as u32,&mut command_buffer,);scan_kernel.encode(- &padded_array,+ &packed_array,&w_array,Some(&b_array),&mut in_proj_array,- &mut state_out_array,+ &mut state_array,suffix_len as u32,- kernel_size as u32,- total_proj_dim as u32,- tap_count as u32,+ tile_tokens as u32,conv_dim as u32,total_proj_dim as u32,&mut command_buffer,@@ -238,13 +240,27 @@ fn test_delta_net_conv_scan() {for t in 0..suffix_len {scan_outputs[t * conv_dim..(t + 1) * conv_dim].copy_from_slice(&in_proj_result[t * total_proj_dim..t * total_proj_dim + conv_dim]);+ // channels past conv_dim are untouched+ assert_eq!(+ in_proj_result[t * total_proj_dim + conv_dim..(t + 1) * total_proj_dim],+ in_proj_f32[t * total_proj_dim + conv_dim..(t + 1) * total_proj_dim],+ );}- let scan_state: Vec<f32> = buffer_to_vec(&state_out_array);+ let scan_state: Vec<f32> = buffer_to_vec(&state_array);assert_close(&ref_outputs, &scan_outputs, 1e-4, 1e-3, "ConvScan output");assert_close(&ref_state, &scan_state, 1e-5, 1e-4, "ConvScan state");}+#[cfg(backend = "metal")]+#[uzu_test]+fn test_delta_net_conv_scan() {+ // single partial tile, shorter than the conv window, exact tiles, several tiles with a partial last one+ for (suffix_len, tile_tokens) in [(2, 32), (96, 32), (100, 32), (70, 8)] {+ for_each_backend!(|B| test_delta_net_conv_scan_impl::<B>(suffix_len, tile_tokens));+ }+}+// DeltaNetUpdate (decode)// The reference is the CPU f32 kernel on the same rounded inputs; a_log, dt_bias, norm weight and state stay f32diff --git a/crates/uzu-engine/unit/backends/metal/kernel/gdn/chunked_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/gdn/chunked_test.rsindex 31f75df7d..c73cedbb4 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/gdn/chunked_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/gdn/chunked_test.rs@@ -1,6 +1,7 @@use futures::executor::block_on;use half::bf16;-use num_traits::cast;+use rand::{RngExt, SeedableRng, rngs::SmallRng};+use rand_distr::Normal;use uzu_engine_macros::uzu_test;use super::*;@@ -8,193 +9,188 @@ use crate::{backends::{common::{Backend, BufferMut, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- kernel::{DeltaNetNormGateKernel, DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel},+ kernel::{DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel},},metal::Metal,},- data_type::DataTypeRepr,tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data},};type BackendKernels<B> = <B as Backend>::Kernels;-const NUM_V_HEADS: usize = 4;-const NUM_K_HEADS: usize = 1;+const NUM_V_HEADS: usize = 6;+const NUM_K_HEADS: usize = 2;const HEAD_K_DIM: usize = 128;const HEAD_V_DIM: usize = 128;+const KEY_DIM: usize = NUM_K_HEADS * HEAD_K_DIM;+const VALUE_DIM: usize = NUM_V_HEADS * HEAD_V_DIM;+const CONV_DIM: usize = 2 * KEY_DIM + VALUE_DIM;+const TOTAL_PROJ_DIM: usize = CONV_DIM + VALUE_DIM + 2 * NUM_V_HEADS;+const WARMUP_TOKENS: usize = 1024;-#[derive(Clone, Copy)]-enum PrefillMode {- Recurrent,- Chunked,+fn random_in_proj(tokens: usize) -> Vec<bf16> {+ let mut rng = SmallRng::seed_from_u64(7);+ let normal = Normal::<f32>::new(0.0, 1.0).unwrap();+ (0..tokens * TOTAL_PROJ_DIM)+ .map(|i| {+ let x: f32 = rng.sample(normal);+ let column = i % TOTAL_PROJ_DIM;+ bf16::from_f32(if column < CONV_DIM {+ x / (1.0 + (-x).exp())+ } else if column >= TOTAL_PROJ_DIM - NUM_V_HEADS {+ 0.5 * x+ } else {+ x+ })+ })+ .collect()}-struct TestCase {- suffix_len: usize,- in_proj: Vec<f32>,- a_log: Vec<f32>,- dt_bias: Vec<f32>,- norm_weight: Vec<f32>,- key_dim: usize,- value_dim: usize,- total_proj_dim: usize,-}--fn test_case(suffix_len: usize) -> TestCase {- let key_dim = NUM_K_HEADS * HEAD_K_DIM;- let value_dim = NUM_V_HEADS * HEAD_V_DIM;- let conv_dim = 2 * key_dim + value_dim;- let total_proj_dim = conv_dim + value_dim + NUM_V_HEADS + NUM_V_HEADS;-- TestCase {- suffix_len,- in_proj: (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect(),- a_log: (0..NUM_V_HEADS).map(|i| -1.5 + (i as f32) * 0.05).collect(),- dt_bias: (0..NUM_V_HEADS).map(|i| 0.3 + (i as f32) * 0.02).collect(),- norm_weight: (0..HEAD_V_DIM).map(|i| 0.9 + (i as f32) * 0.001).collect(),- key_dim,- value_dim,- total_proj_dim,- }-}--fn run_prefill<T: DataTypeRepr>(+/// `None` runs the recurrent kernels. The output is NaN-filled first so unwritten elements fail the comparison.+fn run_prefill(context: &MetalContext,- case: &TestCase,- mode: PrefillMode,-) -> (Vec<f32>, Vec<f32>) {- let in_proj_data: Vec<T> = case.in_proj.iter().copied().map(|value| cast(value).unwrap()).collect();+ in_proj_data: &[bf16],+ chunked: Option<&MetalDeltaNetChunkedPrefill>,+ state: &mut <Metal as Backend>::GlobalBuffer,+) -> Vec<f32> {+ let suffix_len = in_proj_data.len() / TOTAL_PROJ_DIM;+ // a_log = ln U(1, 16) and dt_bias = softplus^-1(logU(1e-3, 1e-1)) as in Qwen3-Next+ let spread = |i: usize| (i as f32 + 0.5) / NUM_V_HEADS as f32;+ let a_log: Vec<f32> = (0..NUM_V_HEADS).map(|i| (1.0 + 15.0 * spread(i)).ln()).collect();+ let dt_bias: Vec<f32> =+ (0..NUM_V_HEADS).map(|i| (1e-3f32.ln() + 100f32.ln() * spread(i)).exp().exp_m1().ln()).collect();let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");- let mut in_proj = command_buffer.allocate_scratch(size_of_val(in_proj_data.as_slice())).unwrap();- in_proj.copyin(&in_proj_data);- let a_log = create_buffer_with_data::<Metal, f32>(context, &case.a_log);- let dt_bias = create_buffer_with_data::<Metal, f32>(context, &case.dt_bias);- let norm_weight = create_buffer_with_data::<Metal, f32>(context, &case.norm_weight);- let state_len = NUM_V_HEADS * HEAD_V_DIM * HEAD_K_DIM;- let mut state = create_buffer::<Metal, f32>(context, state_len);- let mut out = command_buffer.allocate_scratch(case.suffix_len * case.value_dim * size_of::<T>()).unwrap();- let mut q = create_buffer::<Metal, f32>(context, case.suffix_len * case.key_dim);- let mut k = create_buffer::<Metal, f32>(context, case.suffix_len * case.key_dim);- let mut beta = create_buffer::<Metal, f32>(context, case.suffix_len * NUM_V_HEADS);- let mut decay = create_buffer::<Metal, f32>(context, case.suffix_len * NUM_V_HEADS);- command_buffer.encode_fill(&mut state, 0);+ let mut in_proj = command_buffer.allocate_scratch(size_of_val(in_proj_data)).unwrap();+ in_proj.copyin(in_proj_data);+ let a_log = create_buffer_with_data::<Metal, f32>(context, &a_log);+ let dt_bias = create_buffer_with_data::<Metal, f32>(context, &dt_bias);+ let mut out = command_buffer.allocate_scratch(suffix_len * VALUE_DIM * size_of::<bf16>()).unwrap();+ command_buffer.encode_fill(&mut out, 0xFF);- match mode {- PrefillMode::Recurrent => {- <BackendKernels<Metal> as Kernels>::DeltaNetPrefillPrepKernel::new(- context,- T::data_type(),- DataType::F32,- HEAD_K_DIM as u32,- false,- false,+ if let Some(chunked) = chunked {+ chunked+ .encode(+ DeltaNetChunkedPrefillArgs {+ in_projected: &in_proj,+ a_log: &a_log,+ dt_bias: &dt_bias,+ ssm_state: state,+ delta_output: &mut out,+ num_heads: NUM_V_HEADS as u32,+ num_groups: NUM_K_HEADS as u32,+ value_head_dim: HEAD_V_DIM as u32,+ key_dim: KEY_DIM as u32,+ value_dim: VALUE_DIM as u32,+ suffix_len: suffix_len as u32,+ },+ &mut command_buffer,)- .expect("prep")+ .expect("chunked encode");+ } else {+ let mut q = create_buffer::<Metal, f32>(context, suffix_len * KEY_DIM);+ let mut k = create_buffer::<Metal, f32>(context, suffix_len * KEY_DIM);+ let mut beta = create_buffer::<Metal, f32>(context, suffix_len * NUM_V_HEADS);+ let mut decay = create_buffer::<Metal, f32>(context, suffix_len * NUM_V_HEADS);+ <BackendKernels<Metal> as Kernels>::DeltaNetPrefillPrepKernel::new(+ context,+ DataType::BF16,+ DataType::F32,+ HEAD_K_DIM as u32,+ false,+ false,+ )+ .expect("prep")+ .encode(+ &in_proj,+ &a_log,+ &dt_bias,+ &mut q,+ &mut k,+ None::<&mut <Metal as Backend>::GlobalBuffer>,+ &mut beta,+ &mut decay,+ NUM_V_HEADS as u32,+ NUM_K_HEADS as u32,+ KEY_DIM as u32,+ VALUE_DIM as u32,+ suffix_len as u32,+ &mut command_buffer,+ );+ <BackendKernels<Metal> as Kernels>::DeltaNetPrefillKernel::new(context, DataType::BF16, HEAD_K_DIM as u32)+ .expect("prefill").encode(+ &q,+ &k,+ &beta,+ &decay,&in_proj,- &a_log,- &dt_bias,- &mut q,- &mut k,- None::<&mut <Metal as Backend>::GlobalBuffer>,- &mut beta,- &mut decay,+ state,+ &mut out,NUM_V_HEADS as u32,NUM_K_HEADS as u32,- case.key_dim as u32,- case.value_dim as u32,- case.suffix_len as u32,+ HEAD_V_DIM as u32,+ KEY_DIM as u32,+ VALUE_DIM as u32,+ suffix_len as u32,+ HEAD_V_DIM.div_ceil(16) as u32,&mut command_buffer,);- <BackendKernels<Metal> as Kernels>::DeltaNetPrefillKernel::new(context, T::data_type(), HEAD_K_DIM as u32)- .expect("prefill")- .encode(- &q,- &k,- &beta,- &decay,- &in_proj,- &mut state,- &mut out,- NUM_V_HEADS as u32,- NUM_K_HEADS as u32,- HEAD_V_DIM as u32,- case.key_dim as u32,- case.value_dim as u32,- case.suffix_len as u32,- HEAD_V_DIM.div_ceil(16) as u32,- &mut command_buffer,- );- },- PrefillMode::Chunked => {- <BackendKernels<Metal> as Kernels>::DeltaNetChunkedPrefill::new(context, T::data_type(), HEAD_K_DIM as u32)- .expect("chunked")- .expect("chunked unsupported")- .encode(- DeltaNetChunkedPrefillArgs {- in_projected: &in_proj,- a_log: &a_log,- dt_bias: &dt_bias,- ssm_state: &mut state,- delta_output: &mut out,- num_heads: NUM_V_HEADS as u32,- num_groups: NUM_K_HEADS as u32,- value_head_dim: HEAD_V_DIM as u32,- key_dim: case.key_dim as u32,- value_dim: case.value_dim as u32,- suffix_len: case.suffix_len as u32,- },- &mut command_buffer,- )- .expect("chunked encode");- },}-- <BackendKernels<Metal> as Kernels>::DeltaNetNormGateKernel::new(context, T::data_type()).expect("norm").encode(- &mut out,- &in_proj,- &norm_weight,- NUM_V_HEADS as u32,- HEAD_V_DIM as u32,- case.value_dim as u32,- (2 * case.key_dim + case.value_dim) as u32,- case.total_proj_dim as u32,- 1e-6,- case.suffix_len as u32,- &mut command_buffer,- );block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();+ buffer_to_vec::<Metal, bf16>(&out).into_iter().map(f32::from).collect()+}- (buffer_to_vec::<Metal, T>(&out).into_iter().map(|value| cast(value).unwrap()).collect(), buffer_to_vec(&state))+fn check(+ context: &MetalContext,+ chunked: &MetalDeltaNetChunkedPrefill,+ state0: &[f32],+ in_proj: &[bf16],+ calls: usize,+ label: &str,+) {+ let mut chunked_state = create_buffer_with_data::<Metal, f32>(context, state0);+ let mut recurrent_state = create_buffer_with_data::<Metal, f32>(context, state0);+ let chunked_out: Vec<f32> = in_proj+ .chunks(in_proj.len() / calls)+ .flat_map(|tokens| run_prefill(context, tokens, Some(chunked), &mut chunked_state))+ .collect();+ let recurrent_out = run_prefill(context, in_proj, None, &mut recurrent_state);+ let (chunked_state, recurrent_state) =+ (buffer_to_vec::<Metal, f32>(&chunked_state), buffer_to_vec::<Metal, f32>(&recurrent_state));++ assert!(+ is_close::default().abs_tol(5e-3).rel_tol(1e-3).all_close(chunked_out, recurrent_out),+ "chunked output differs from recurrent ({label})"+ );+ assert!(+ is_close::default().abs_tol(1e-2).rel_tol(1e-3).all_close(chunked_state, recurrent_state),+ "chunked state differs from recurrent ({label})"+ );}#[uzu_test]fn chunked_prefill_matches_recurrent_prefill() {let context = block_on(<Metal as Backend>::Context::new()).expect("metal context");- if <BackendKernels<Metal> as Kernels>::DeltaNetChunkedPrefill::new(+ let Some(chunked) = MetalDeltaNetChunkedPrefill::new(&context,- <bf16 as DataTypeRepr>::data_type(),+ DataType::BF16,HEAD_K_DIM as u32,+ (NUM_V_HEADS / NUM_K_HEADS) as u32,)- .expect("chunked")- .is_none()- {+ .unwrap() else {eprintln!("chunked prefill unsupported on this Metal device");return;- }-- for suffix_len in [CHUNK_SIZE - 1, CHUNK_SIZE, CHUNK_SIZE + 1, CHUNK_SIZE * 2 + 1] {- let case = test_case(suffix_len as usize);- let (recurrent_out, recurrent_state) = run_prefill::<bf16>(&context, &case, PrefillMode::Recurrent);- let (chunked_out, chunked_state) = run_prefill::<bf16>(&context, &case, PrefillMode::Chunked);+ };+ let in_proj = random_in_proj(WARMUP_TOKENS + 4096);+ let (warmup, tokens) = in_proj.split_at(WARMUP_TOKENS * TOTAL_PROJ_DIM);+ let mut state = create_buffer_with_data::<Metal, f32>(&context, &vec![0.0; NUM_V_HEADS * HEAD_V_DIM * HEAD_K_DIM]);+ run_prefill(&context, warmup, None, &mut state);+ let state0 = buffer_to_vec::<Metal, f32>(&state);+ assert!(state0.iter().any(|value| *value != 0.0), "initial state is zero");- assert!(- is_close::default().abs_tol(5e-3).rel_tol(1e-3).all_close(chunked_out.iter().copied(), recurrent_out),- "chunked output differs at T={suffix_len}"- );- assert!(- is_close::default().abs_tol(2e-4).rel_tol(1e-3).all_close(chunked_state.iter().copied(), recurrent_state),- "chunked state differs at T={suffix_len}"- );+ for suffix_len in [2 * CHUNK_SIZE as usize + 1, 512, 1024] {+ let label = format!("T={suffix_len}");+ check(&context, &chunked, &state0, &tokens[..suffix_len * TOTAL_PROJ_DIM], 1, &label);}+ check(&context, &chunked, &state0, tokens, 4, "carried 4x1024");}