A high-performance inference engine for AI models
Chunked GDN (#589)
CCYeh6597e3beb7f7
diff --git a/crates/backend-uzu/src/backends/common/kernel/delta_net_chunked_prefill.rs b/crates/backend-uzu/src/backends/common/kernel/delta_net_chunked_prefill.rsnew file mode 100644index 000000000..d6de4bd5f--- /dev/null+++ b/crates/backend-uzu/src/backends/common/kernel/delta_net_chunked_prefill.rs@@ -0,0 +1,64 @@+use std::convert::Infallible;++use crate::{+ backends::common::{Allocation, Backend, Encoder},+ data_type::DataType,+};++pub struct DeltaNetChunkedPrefillArgs<'a, B: Backend> {+ pub in_projected: &'a Allocation<B>,+ pub a_log: &'a Allocation<B>,+ pub dt_bias: &'a Allocation<B>,+ pub ssm_state: &'a mut Allocation<B>,+ pub delta_output: &'a mut Allocation<B>,+ pub num_heads: u32,+ pub num_groups: u32,+ pub value_head_dim: u32,+ pub key_dim: u32,+ pub value_dim: u32,+ pub suffix_len: usize,+}++pub trait DeltaNetChunkedPrefill<B: Backend>: Sized {+ fn new(+ context: &B::Context,+ outer_data_type: DataType,+ head_dim: u32,+ ) -> Result<Option<Self>, B::Error>;++ fn should_use(+ &self,+ suffix_len: usize,+ ) -> bool;++ fn encode(+ &self,+ args: DeltaNetChunkedPrefillArgs<'_, B>,+ encoder: &mut Encoder<B>,+ ) -> Result<(), B::Error>;+}++impl<B: Backend> DeltaNetChunkedPrefill<B> for Infallible {+ fn new(+ _context: &B::Context,+ _outer_data_type: DataType,+ _head_dim: u32,+ ) -> Result<Option<Self>, B::Error> {+ Ok(None)+ }++ fn should_use(+ &self,+ _suffix_len: usize,+ ) -> bool {+ match *self {}+ }++ fn encode(+ &self,+ _args: DeltaNetChunkedPrefillArgs<'_, B>,+ _encoder: &mut Encoder<B>,+ ) -> Result<(), B::Error> {+ match *self {}+ }+}diff --git a/crates/backend-uzu/src/backends/common/kernel/mod.rs b/crates/backend-uzu/src/backends/common/kernel/mod.rsindex c022189a3..59cfe359f 100644--- a/crates/backend-uzu/src/backends/common/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/common/kernel/mod.rs@@ -1,6 +1,7 @@use crate::backends::common::Backend;pub mod attention_gemm;+pub mod delta_net_chunked_prefill;pub mod matmul;include!(concat!(env!("OUT_DIR"), "/traits.rs"));@@ -10,6 +11,7 @@ pub trait Kernels: Sized {autogen_kernels!();type AttentionGemmCore: attention_gemm::AttentionGemmCore<Self::Backend>;+ type DeltaNetChunkedPrefill: delta_net_chunked_prefill::DeltaNetChunkedPrefill<Self::Backend>;type MatmulKernel: matmul::MatmulKernel<Backend = Self::Backend>;}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_scan.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_scan.rssimilarity index 100%rename from crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_scan.rsrename to crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_scan.rsdiff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_update.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_update.rssimilarity index 100%rename from crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_update.rsrename to crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_update.rsdiff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/mod.rssimilarity index 84%rename from crates/backend-uzu/src/backends/cpu/kernel/delta_net/mod.rsrename to crates/backend-uzu/src/backends/cpu/kernel/gdn/mod.rsindex 8f29d2c4f..e0c4e7865 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/mod.rs@@ -3,4 +3,5 @@ pub mod conv_update;pub mod norm_gate;pub mod prefill;pub mod prefill_prep;+pub mod tree_verify;pub mod update;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/norm_gate.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/norm_gate.rssimilarity index 100%rename from crates/backend-uzu/src/backends/cpu/kernel/delta_net/norm_gate.rsrename to crates/backend-uzu/src/backends/cpu/kernel/gdn/norm_gate.rsdiff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/prefill.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill.rssimilarity index 100%rename from crates/backend-uzu/src/backends/cpu/kernel/delta_net/prefill.rsrename to crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill.rsdiff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/prefill_prep.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill_prep.rssimilarity index 91%rename from crates/backend-uzu/src/backends/cpu/kernel/delta_net/prefill_prep.rsrename to crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill_prep.rsindex 64706b59a..00a2f601b 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/prefill_prep.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill_prep.rs@@ -21,6 +21,7 @@ pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(key_dim: u32,value_dim: u32,suffix_len: u32,+ #[specialize] write_log_decay: bool,) {let num_v_heads = num_v_heads as usize;let num_k_heads = num_k_heads as usize;@@ -80,11 +81,15 @@ pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(} else {(1.0 + sp_in.exp()).ln()};- let decay = (-a_log_val.exp() * sp).exp();+ let log_decay = -a_log_val.exp() * sp;unsafe {*beta_out.add(token * num_v_heads + hv) = beta;- *decay_out.add(token * num_v_heads + hv) = decay;+ *decay_out.add(token * num_v_heads + hv) = if write_log_decay {+ log_decay+ } else {+ log_decay.exp()+ };};}}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rssimilarity index 100%rename from crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/mod.rsrename to crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rsdiff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/out.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/out.rssimilarity index 100%rename from crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/out.rsrename to crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/out.rsdiff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/prefix_beta.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/prefix_beta.rssimilarity index 100%rename from crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/prefix_beta.rsrename to crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/prefix_beta.rsdiff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/tree_gram.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rssimilarity index 100%rename from crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/tree_gram.rsrename to crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rsdiff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/tree_update_solve.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rssimilarity index 100%rename from crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/tree_update_solve.rsrename to crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rsdiff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/update.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/update.rssimilarity index 100%rename from crates/backend-uzu/src/backends/cpu/kernel/delta_net/update.rsrename to crates/backend-uzu/src/backends/cpu/kernel/gdn/update.rsdiff --git a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/mod.rsindex f0095bf5a..3d903e94d 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs@@ -4,10 +4,9 @@ use crate::backends::{common::Kernels, cpu::Cpu};mod activation;mod attention;-mod delta_net;mod embedding;mod gated_act_mul;-mod gdn_tree_verify;+mod gdn;mod hadamard_transform;mod logit_soft_cap;mod matmul;@@ -33,5 +32,6 @@ impl Kernels for CpuKernels {autogen_kernels!();type AttentionGemmCore = Infallible;+ type DeltaNetChunkedPrefill = Infallible;type MatmulKernel = matmul::MatmulCpuKernel;}diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rsindex d25889b77..b9871e838 100644--- a/crates/backend-uzu/src/backends/metal/context.rs+++ b/crates/backend-uzu/src/backends/metal/context.rs@@ -132,7 +132,6 @@ impl Context for MetalContext {let gpu_core_count = device.gpu_core_count();let device_tier = device_tier_for_device(gpu_core_count, device.as_ref());-let page_size = MTLSparsePageSize::KB256;let heap_capacity = Metal::ALLOCATION_GRANULARITY;let sparse_pool = MetalSparseHeapPool::new(page_size, heap_capacity);diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/a_diag_inv.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/a_diag_inv.metalnew file mode 100644index 000000000..266139d08--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/a_diag_inv.metal@@ -0,0 +1,92 @@+#include <metal_stdlib>+#include "../../common/defines.h"+#include "../../common/dsl.h"+#include "../common/tri_inv.h"++using namespace metal;++#define DIAG_BLOCK_SIZE 16u++// kk: [chunks, num_k_heads, CHUNK_SIZE, CHUNK_SIZE] K K^T from Gram.+// beta: [suffix_len, num_v_heads].+// g: [num_v_heads, suffix_len] cumulative log decay.+// a_packed: [chunks, num_v_heads, num_blocks, ceil(num_blocks/2),+// DIAG_BLOCK_SIZE, 2*DIAG_BLOCK_SIZE].+// a_inv: [chunks, num_v_heads, num_blocks, DIAG_BLOCK_SIZE, DIAG_BLOCK_SIZE].+//+// One threadgroup owns one (chunk, v-head, column-pair, row-block). It materializes+// A[row,col] = beta[row] * exp(g[row] - g[col]) * kk[row,col] for col < row,+// writes packed column-pair blocks through the diagonal pair, copies the+// diagonal block into threadgroup memory, then writes (I + A_diag)^-1 to a_inv.+template <uint CHUNK_SIZE>+VARIANTS(CHUNK_SIZE, 32, 64)+KERNEL(DeltaNetChunkedADiagInv)(+ device const float* kk,+ device const float* beta,+ device const float* g,+ device float* a_packed,+ device float* a_inv,+ constant const uint& num_v_heads,+ constant const uint& num_k_heads,+ constant const uint& suffix_len,+ threadgroup float diag_a_tile[DIAG_BLOCK_SIZE * DIAG_BLOCK_SIZE],+ const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),+ const uint hv_idx GROUPS(num_v_heads),+ const uint block_col_pair_idx GROUPS(+ CHUNK_SIZE.div_ceil(DIAG_BLOCK_SIZE) * CHUNK_SIZE.div_ceil(DIAG_BLOCK_SIZE).div_ceil(2)+ ),+ const uint lane THREADS(METAL_SIMD_SIZE)+) {+ constexpr uint num_blocks = (CHUNK_SIZE + DIAG_BLOCK_SIZE - 1) / DIAG_BLOCK_SIZE;+ constexpr uint num_col_pairs = (num_blocks + 1) / 2;+ const uint block_idx = block_col_pair_idx / num_col_pairs;+ const uint col_pair_idx = block_col_pair_idx - block_idx * num_col_pairs;+ const uint hk_idx = hv_idx / (num_v_heads / num_k_heads);+ const uint token_base = chunk_idx * CHUNK_SIZE;+ const uint row_base = block_idx * DIAG_BLOCK_SIZE;+ const uint kk_base = (chunk_idx * num_k_heads + hk_idx) * CHUNK_SIZE * CHUNK_SIZE;+ const uint diag_col_pair = block_idx / 2;+ const uint diag_col_offset = (block_idx % 2) * DIAG_BLOCK_SIZE;+ const device float* g_head = g + hv_idx * suffix_len;+ if (col_pair_idx > diag_col_pair) {+ return;+ }++ METAL_PRAGMA_UNROLL+ for (uint idx = lane; idx < DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE; idx += METAL_SIMD_SIZE) {+ const uint local_col = idx % (2 * DIAG_BLOCK_SIZE);+ const uint local_row = (idx / (2 * DIAG_BLOCK_SIZE)) % DIAG_BLOCK_SIZE;+ const uint row = row_base + local_row;+ const uint col = col_pair_idx * 2 * DIAG_BLOCK_SIZE + local_col;+ const uint row_token = token_base + row;+ const uint col_token = token_base + col;++ float value = 0.0f;+ if (row < CHUNK_SIZE && col < CHUNK_SIZE && row_token < suffix_len && col_token < suffix_len && col < row) {+ const float beta_row = beta[row_token * num_v_heads + hv_idx];+ const float g_row = g_head[row_token];+ const float g_col = g_head[col_token];+ value = beta_row * fast::exp(g_row - g_col) * kk[kk_base + row * CHUNK_SIZE + col];+ }++ const uint out_idx =+ (((chunk_idx * num_v_heads + hv_idx) * num_blocks + block_idx) * num_col_pairs + col_pair_idx) *+ (DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE) ++ local_row * (2 * DIAG_BLOCK_SIZE) + local_col;+ a_packed[out_idx] = value;+ if (col_pair_idx == diag_col_pair && local_col >= diag_col_offset &&+ local_col < diag_col_offset + DIAG_BLOCK_SIZE) {+ diag_a_tile[local_row * DIAG_BLOCK_SIZE + local_col - diag_col_offset] = value;+ }+ }++ if (col_pair_idx != diag_col_pair) {+ return;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ device float* a_inv_block =+ a_inv + ((chunk_idx * num_v_heads + hv_idx) * num_blocks + block_idx) * (DIAG_BLOCK_SIZE * DIAG_BLOCK_SIZE);+ const uint block_tokens =+ token_base + row_base < suffix_len ? min(uint(DIAG_BLOCK_SIZE), suffix_len - token_base - row_base) : 0u;+ invert_lower_triangular_block<DIAG_BLOCK_SIZE>(a_inv_block, diag_a_tile, block_tokens, lane);+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/causal_inv.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/causal_inv.metalnew file mode 100644index 000000000..efccb1b1c--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/causal_inv.metal@@ -0,0 +1,102 @@+#include <metal_stdlib>+#include "../../common/defines.h"+#include "../../common/dsl.h"+#include "../../matmul/common/fragment.h"+#include "../../matmul/common/simdgroup_fragment_ops.h"++using namespace metal;+using namespace uzu::matmul;++#define DIAG_BLOCK_SIZE 16u++// Materializes dense T = (I + A)^-1 as bf16 for output and state. This is block+// forward substitution over a_packed plus the per-block inverses from ADiagInv.+template <uint CHUNK_SIZE, uint BV>+VARIANTS(CHUNK_SIZE, 32, 64)+VARIANTS(BV, 16, 32)+KERNEL(DeltaNetChunkedCausalInv)(+ device const float* a_packed,+ device const float* a_inv,+ device bfloat* t_out,+ constant const uint& num_v_heads,+ constant const uint& suffix_len,+ const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),+ const uint hv_idx GROUPS(num_v_heads),+ const uint tile_idx GROUPS(CHUNK_SIZE.div_ceil(BV)),+ const uint lane THREADS(METAL_SIMD_SIZE)+) {+ using Ops = SimdgroupFragmentOps;+ constexpr ushort TOKEN_FRAGMENTS = DIAG_BLOCK_SIZE / Ops::FRAGMENT_ROWS;+ constexpr ushort VALUE_FRAGMENTS = BV / Ops::FRAGMENT_COLS;+ using TileFragment = Fragment<float, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, Ops>;+ using MatrixFragment = OperandFragment<float, TOKEN_FRAGMENTS, TOKEN_FRAGMENTS, Ops>;+ using ValueFragment = OperandFragment<bfloat, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, Ops>;+ using PairMatrixFragment = OperandFragment<float, TOKEN_FRAGMENTS, 2 * TOKEN_FRAGMENTS, Ops>;+ using PairValueFragment = OperandFragment<bfloat, 2 * TOKEN_FRAGMENTS, VALUE_FRAGMENTS, Ops>;++ constexpr uint num_blocks = (CHUNK_SIZE + DIAG_BLOCK_SIZE - 1) / DIAG_BLOCK_SIZE;+ constexpr uint num_col_pairs = (num_blocks + 1) / 2;+ const uint slice_base = tile_idx * BV;++ const device float* a_blocks = a_packed + (chunk_idx * num_v_heads + hv_idx) * num_blocks * num_col_pairs *+ (DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE);+ const device float* inv_blocks =+ a_inv + (chunk_idx * num_v_heads + hv_idx) * num_blocks * (DIAG_BLOCK_SIZE * DIAG_BLOCK_SIZE);+ device bfloat* t_head = t_out + (chunk_idx * num_v_heads + hv_idx) * CHUNK_SIZE * CHUNK_SIZE + slice_base;++ for (uint block_idx = 0; block_idx < num_blocks; ++block_idx) {+ const uint row_base = block_idx * DIAG_BLOCK_SIZE;++ // -- RHS = identity slice: acc[row, col] = (row_base+row == slice_base+col) -+ TileFragment acc_t;+ acc_t.map_coords(lane, [&](short local_row, short local_col, float) {+ const uint global_row = row_base + uint(local_row);+ const uint global_col = slice_base + uint(local_col);+ return global_row == global_col ? 1.0f : 0.0f;+ });++ // -- Forward substitution: subtract A . T_prev (T_prev in bf16, own slice) --+ // a_packed stays positive for block inverse; dense solve subtracts it.+ const uint num_full_pairs = block_idx / 2;+ for (uint pair_idx = 0; pair_idx < num_full_pairs; ++pair_idx) {+ PairMatrixFragment a_frag;+ const device float* a_pair =+ a_blocks + (block_idx * num_col_pairs + pair_idx) * (DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE);+ a_frag.load_from(lane, fragment_source(a_pair, 2 * DIAG_BLOCK_SIZE));+ a_frag.map([](float value) { return -value; });++ PairValueFragment t_prev_frag;+ const device bfloat* t_prev = t_head + pair_idx * 2 * DIAG_BLOCK_SIZE * CHUNK_SIZE;+ t_prev_frag.load_from(lane, fragment_source(t_prev, CHUNK_SIZE));+ fragment_mma(acc_t, a_frag, t_prev_frag);+ }++ for (uint prev_block_idx = num_full_pairs * 2; prev_block_idx < block_idx; ++prev_block_idx) {+ MatrixFragment a_frag;+ const device float* a_block =+ a_blocks + (block_idx * num_col_pairs + prev_block_idx / 2) * (DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE) ++ (prev_block_idx % 2) * DIAG_BLOCK_SIZE;+ a_frag.load_from(lane, fragment_source(a_block, 2 * DIAG_BLOCK_SIZE));+ a_frag.map([](float value) { return -value; });++ ValueFragment t_prev_frag;+ const device bfloat* t_prev = t_head + prev_block_idx * DIAG_BLOCK_SIZE * CHUNK_SIZE;+ t_prev_frag.load_from(lane, fragment_source(t_prev, CHUNK_SIZE));+ fragment_mma(acc_t, a_frag, t_prev_frag);+ }++ // -- Apply the diagonal-block inverse and store T ------------------------+ MatrixFragment inv_frag;+ inv_frag.load_from(+ lane,+ fragment_source(inv_blocks + block_idx * DIAG_BLOCK_SIZE * DIAG_BLOCK_SIZE, DIAG_BLOCK_SIZE)+ );++ TileFragment t_block;+ t_block.clear();+ fragment_mma(t_block, inv_frag, acc_t);+ t_block.store(lane, t_head + row_base * CHUNK_SIZE, CHUNK_SIZE);++ simdgroup_barrier(mem_flags::mem_device);+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/cumsum.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/cumsum.metalnew file mode 100644index 000000000..37e5277e8--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/cumsum.metal@@ -0,0 +1,37 @@+#include <metal_stdlib>+#include "../../common/defines.h"+#include "../../common/dsl.h"++using namespace metal;++template <uint CHUNK_SIZE>+VARIANTS(CHUNK_SIZE, 32, 64)+KERNEL(DeltaNetChunkedCumsum)(+ device const float* log_decay,+ device float* g_out,+ constant const uint& num_v_heads,+ constant const uint& suffix_len,+ const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),+ const uint hv_idx GROUPS(num_v_heads),+ const uint lane THREADS(METAL_SIMD_SIZE)+) {+ const uint token_begin = chunk_idx * CHUNK_SIZE;+ const uint token_end = min(token_begin + CHUNK_SIZE, suffix_len);++ float carry = 0.0f;+ METAL_PRAGMA_UNROLL+ for (uint token_offset = 0; token_offset < CHUNK_SIZE; token_offset += METAL_SIMD_SIZE) {+ const uint token = token_begin + token_offset + lane;+ const bool valid = token < token_end;+ const uint input_offset = token * num_v_heads + hv_idx;+ const uint output_offset = hv_idx * suffix_len + token;+ const float value = valid ? log_decay[input_offset] : 0.0f;+ const float prefix = carry + simd_prefix_inclusive_sum(value);++ if (valid) {+ g_out[output_offset] = prefix;+ }++ carry += simd_sum(value);+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/gram.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/gram.metalnew file mode 100644index 000000000..2566e9499--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/gram.metal@@ -0,0 +1,114 @@+#include <metal_stdlib>+#include "../../common/defines.h"+#include "../../common/dsl.h"+#include "../common/gram.h"++using namespace metal;+using namespace uzu::matmul;++#define ROW_TILE 16u+#define COL_TILE 32u++// q_norm/k_norm: [suffix_len, key_dim], g: [num_v_heads, suffix_len].+// kk_out: [chunks, num_k_heads, CHUNK_SIZE, CHUNK_SIZE].+// qk_scaled_out: [chunks, num_v_heads, CHUNK_SIZE, CHUNK_SIZE].+// Computes K K^T per k-head and decay-scaled Q K^T per grouped v-head.+// MXU did not improve e2e vs simdgroup.+template <uint HEAD_K_DIM, uint CHUNK_SIZE>+VARIANTS(HEAD_K_DIM, 128)+VARIANTS(CHUNK_SIZE, 32, 64)+KERNEL(DeltaNetChunkedGram)(+ device const float* q_norm,+ device const float* k_norm,+ device const float* g,+ device float* kk_out,+ device float* qk_scaled_out,+ constant const uint& num_v_heads,+ constant const uint& num_k_heads,+ constant const uint& key_dim,+ constant const uint& suffix_len,+ const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),+ const uint hk_idx GROUPS(num_k_heads),+ const uint tile_idx GROUPS(CHUNK_SIZE.div_ceil(ROW_TILE) * CHUNK_SIZE.div_ceil(COL_TILE)),+ const uint lane THREADS(METAL_SIMD_SIZE)+) {+ using Ops = SimdgroupFragmentOps;+ using InputType = float;+ constexpr ushort ROW_FRAGMENTS = ROW_TILE / Ops::FRAGMENT_ROWS;+ constexpr ushort COL_FRAGMENTS = COL_TILE / Ops::FRAGMENT_COLS;+ using AccFragment = Fragment<float, ROW_FRAGMENTS, COL_FRAGMENTS, Ops>;+ using LeftFragment = OperandFragment<InputType, ROW_FRAGMENTS, 1, Ops>;+ using RightFragment = OperandFragment<InputType, 1, COL_FRAGMENTS, Ops, ReadTranspose>;++ static_assert(HEAD_K_DIM % Ops::FRAGMENT_ROWS == 0, "HEAD_K_DIM must align to fragment K");++ constexpr uint col_tiles = (CHUNK_SIZE + COL_TILE - 1) / COL_TILE;+ const uint row_tile_idx = tile_idx / col_tiles;+ const uint col_tile_idx = tile_idx - row_tile_idx * col_tiles;+ const uint row_base = row_tile_idx * ROW_TILE;+ const uint col_base = col_tile_idx * COL_TILE;+ const uint chunk_token_base = chunk_idx * CHUNK_SIZE;++ const uint tile_rows = min(uint(ROW_TILE), uint(CHUNK_SIZE - row_base));+ const uint tile_cols = min(uint(COL_TILE), uint(CHUNK_SIZE - col_base));+ const uint row_token_base = chunk_token_base + row_base;+ const uint col_token_base = chunk_token_base + col_base;+ const uint valid_rows = row_token_base < suffix_len ? min(tile_rows, suffix_len - row_token_base) : 0u;+ const uint valid_cols = col_token_base < suffix_len ? min(tile_cols, suffix_len - col_token_base) : 0u;+ // Upper-causal tiles are zero.+ if (valid_rows == 0 || valid_cols == 0 || col_base >= row_base + valid_rows) {+ return;+ }++ AccFragment kk_acc;+ AccFragment qk_acc;+ kk_acc.clear();+ qk_acc.clear();++ for (uint k_block_start = 0; k_block_start < HEAD_K_DIM; k_block_start += Ops::FRAGMENT_ROWS) {+ const device float* k_rows = k_norm + row_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;+ const device float* q_rows = q_norm + row_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;+ const device float* k_cols = k_norm + col_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;++ accumulate_dual_gram_tile<AccFragment, LeftFragment, RightFragment>(+ kk_acc,+ qk_acc,+ k_rows,+ q_rows,+ k_cols,+ int(key_dim),+ ushort(valid_rows),+ ushort(valid_cols),+ Ops::FRAGMENT_ROWS,+ valid_rows == tile_rows,+ valid_cols == tile_cols,+ ushort(lane)+ );+ }++ const short2 tile_dims = short2(tile_cols, tile_rows);+ const uint kk_base = (chunk_idx * num_k_heads + hk_idx) * CHUNK_SIZE * CHUNK_SIZE + row_base * CHUNK_SIZE + col_base;+ kk_acc.store_safe(lane, kk_out + kk_base, CHUNK_SIZE, tile_dims);++ // Store qk for each grouped v-head with decay scale and causal mask.+ const uint valid_tokens = chunk_token_base < suffix_len ? min(uint(CHUNK_SIZE), suffix_len - chunk_token_base) : 0u;+ const uint groups_per_head = num_v_heads / num_k_heads;+ for (uint group = 0; group < groups_per_head; ++group) {+ const uint hv_idx = hk_idx * groups_per_head + group;+ const device float* g_head = g + hv_idx * suffix_len;+ AccFragment scaled = qk_acc;+ scaled.map_coords(lane, [&](short r, short c, float value) {+ const uint row = row_base + uint(r);+ const uint col = col_base + uint(c);+ if (row >= valid_tokens || col >= valid_tokens || col > row) {+ return 0.0f;+ }+ const float g_row = g_head[chunk_token_base + row];+ const float g_col = g_head[chunk_token_base + col];+ return value * fast::exp(g_row - g_col);+ });+ const uint dst_base =+ (chunk_idx * num_v_heads + hv_idx) * CHUNK_SIZE * CHUNK_SIZE + row_base * CHUNK_SIZE + col_base;+ scaled.store_safe(lane, qk_scaled_out + dst_base, CHUNK_SIZE, tile_dims);+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rsnew file mode 100644index 000000000..0ec83e804--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs@@ -0,0 +1,185 @@+use super::super::{+ DeltaNetChunkedADiagInvMetalKernel, DeltaNetChunkedCausalInvMetalKernel, DeltaNetChunkedCumsumMetalKernel,+ DeltaNetChunkedGramMetalKernel, DeltaNetChunkedOutputAndStateMetalKernel, DeltaNetPrefillPrepMetalKernel,+};+use crate::{+ array::size_for_shape,+ backends::{+ common::{+ Backend, Encoder,+ kernel::{+ DeltaNetPrefillPrepKernel,+ delta_net_chunked_prefill::{DeltaNetChunkedPrefill, DeltaNetChunkedPrefillArgs},+ },+ },+ metal::{Metal, MetalContext},+ },+ data_type::DataType,+};++const MXU_MIN_T: usize = 256;+const CHUNK_SIZE: usize = 64;+const BLOCK_SIZE: usize = 16;+const VT: usize = 32;++pub struct MetalDeltaNetChunkedPrefill {+ min_t: usize,+ prep: DeltaNetPrefillPrepMetalKernel,+ cumsum: DeltaNetChunkedCumsumMetalKernel,+ gram: DeltaNetChunkedGramMetalKernel,+ a_diag_inv: DeltaNetChunkedADiagInvMetalKernel,+ causal_inv: DeltaNetChunkedCausalInvMetalKernel,+ output_and_state: DeltaNetChunkedOutputAndStateMetalKernel,+}++impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {+ fn new(+ context: &MetalContext,+ outer_data_type: DataType,+ head_dim: u32,+ ) -> Result<Option<Self>, <Metal as Backend>::Error> {+ if outer_data_type == DataType::F16 {+ return Ok(None);+ }++ if !context.supports_mxu() {+ return Ok(None);+ }++ let scratch_data_type = if outer_data_type == DataType::BF16 {+ DataType::BF16+ } else {+ DataType::F32+ };++ Ok(Some(Self {+ min_t: MXU_MIN_T,+ prep: DeltaNetPrefillPrepMetalKernel::new(context, outer_data_type, head_dim, true)?,+ cumsum: DeltaNetChunkedCumsumMetalKernel::new(context, CHUNK_SIZE as u32)?,+ gram: DeltaNetChunkedGramMetalKernel::new(context, head_dim, CHUNK_SIZE as u32)?,+ a_diag_inv: DeltaNetChunkedADiagInvMetalKernel::new(context, CHUNK_SIZE as u32)?,+ causal_inv: DeltaNetChunkedCausalInvMetalKernel::new(context, CHUNK_SIZE as u32, VT as u32)?,+ output_and_state: DeltaNetChunkedOutputAndStateMetalKernel::new(+ context,+ outer_data_type,+ outer_data_type,+ scratch_data_type,+ head_dim,+ VT as u32,+ true,+ )?,+ }))+ }++ fn should_use(+ &self,+ suffix_len: usize,+ ) -> bool {+ suffix_len >= self.min_t+ }++ fn encode(+ &self,+ args: DeltaNetChunkedPrefillArgs<'_, Metal>,+ encoder: &mut Encoder<Metal>,+ ) -> Result<(), <Metal as Backend>::Error> {+ let suffix_len = args.suffix_len;+ let num_chunks = suffix_len.div_ceil(CHUNK_SIZE);+ let num_blocks = CHUNK_SIZE.div_ceil(BLOCK_SIZE);+ let num_col_pairs = num_blocks.div_ceil(2);++ let mut q_norm =+ encoder.allocate_scratch(size_for_shape(&[suffix_len * args.key_dim as usize], DataType::F32))?;+ let mut k_norm =+ encoder.allocate_scratch(size_for_shape(&[suffix_len * args.key_dim as usize], DataType::F32))?;+ let mut beta =+ encoder.allocate_scratch(size_for_shape(&[suffix_len * args.num_heads as usize], DataType::F32))?;+ let mut log_decay =+ encoder.allocate_scratch(size_for_shape(&[suffix_len * args.num_heads as usize], DataType::F32))?;+ let mut g = encoder.allocate_scratch(size_for_shape(&[suffix_len * args.num_heads as usize], DataType::F32))?;+ let mut kk = encoder.allocate_scratch(size_for_shape(+ &[num_chunks * args.num_groups as usize * CHUNK_SIZE * CHUNK_SIZE],+ DataType::F32,+ ))?;+ let mut qk = encoder.allocate_scratch(size_for_shape(+ &[num_chunks * args.num_heads as usize * CHUNK_SIZE * CHUNK_SIZE],+ DataType::F32,+ ))?;+ let mut a_packed = encoder.allocate_scratch(size_for_shape(+ &[num_chunks * args.num_heads as usize * num_blocks * num_col_pairs * BLOCK_SIZE * 2 * BLOCK_SIZE],+ DataType::F32,+ ))?;+ let mut a_inv = encoder.allocate_scratch(size_for_shape(+ &[num_chunks * args.num_heads as usize * num_blocks * BLOCK_SIZE * BLOCK_SIZE],+ DataType::F32,+ ))?;+ let mut t_mat = encoder.allocate_scratch(size_for_shape(+ &[num_chunks * args.num_heads as usize * CHUNK_SIZE * CHUNK_SIZE],+ DataType::BF16,+ ))?;++ self.prep.encode(+ args.in_projected,+ args.a_log,+ args.dt_bias,+ &mut q_norm,+ &mut k_norm,+ &mut beta,+ &mut log_decay,+ args.num_heads,+ args.num_groups,+ args.key_dim,+ args.value_dim,+ suffix_len as u32,+ encoder,+ );+ self.cumsum.encode(&log_decay, &mut g, args.num_heads, suffix_len as u32, encoder);+ self.gram.encode(+ &q_norm,+ &k_norm,+ &g,+ &mut kk,+ &mut qk,+ args.num_heads,+ args.num_groups,+ args.key_dim,+ suffix_len as u32,+ encoder,+ );+ self.a_diag_inv.encode(+ &kk,+ &beta,+ &g,+ &mut a_packed,+ &mut a_inv,+ args.num_heads,+ args.num_groups,+ suffix_len as u32,+ encoder,+ );+ self.causal_inv.encode(&a_packed, &a_inv, &mut t_mat, args.num_heads, suffix_len as u32, encoder);+ self.output_and_state.encode(+ &q_norm,+ &k_norm,+ args.in_projected,+ &qk,+ &t_mat,+ &g,+ &beta,+ args.ssm_state,+ args.delta_output,+ args.num_heads,+ args.num_groups,+ args.value_head_dim,+ args.key_dim,+ args.value_dim,+ suffix_len as u32,+ encoder,+ );+ Ok(())+ }+}++#[cfg(test)]+#[path = "../../../../../../unit/backends/metal/kernel/gdn/chunked_test.rs"]+mod tests;diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/output_and_state.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/output_and_state.metalnew file mode 100644index 000000000..652a9c8e9--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/output_and_state.metal@@ -0,0 +1,285 @@+#include <metal_stdlib>+#include "../../common/defines.h"+#include "../../common/dsl.h"+#include "../../common/thread_context.h"+#include "../../matmul/common/fragment.h"+#include "../../matmul/common/mxu_fragment_ops.h"+#include "../../matmul/common/simdgroup_fragment_ops.h"++using namespace metal;+using namespace uzu::matmul;++#define OUTPUT_STATE_THREADS 128+#define OUTPUT_STATE_NUM_SIMDGROUPS (OUTPUT_STATE_THREADS / METAL_SIMD_SIZE)+#define OUTPUT_STATE_CHUNK 64+#define OUTPUT_STATE_KEY_TILE (HEAD_K_DIM / OUTPUT_STATE_NUM_SIMDGROUPS)++// q_norm: [suffix_len, key_dim].+// k_norm: [suffix_len, key_dim].+// in_proj: [suffix_len, total_proj_dim]; V starts at offset 2*key_dim.+// qk_scaled: [chunks, num_v_heads, CHUNK, CHUNK] from Gram.+// t_mat: [chunks, num_v_heads, CHUNK, CHUNK] dense causal inverse, bf16.+// g: [num_v_heads, suffix_len] cumulative log decay.+// beta: [suffix_len, num_v_heads].+// state: [num_v_heads, head_v_dim, HEAD_K_DIM], f32 input/output.+// out: [suffix_len, value_dim].+// ScratchT: R/Vnew threadgroup storage precision.+//+// One threadgroup owns one (v-head, value-slice). It loads the state slice as+// S^T in threadgroup memory, then scans chunks:+// 1. R = beta * (V - exp(g) * K * S^T).+// 2. Vnew = T * R.+// 3. out = exp(g) * Q * S^T + qk_scaled * Vnew.+// 4. S^T = exp(g_last) * S^T + (exp(g_last - g) * K)^T * Vnew.+template <typename T, typename O, typename ScratchT, uint HEAD_K_DIM, uint VT, bool USE_MXU>+VARIANTS(T, float, bfloat)+VARIANTS(O, float, bfloat)+VARIANTS(ScratchT, float, bfloat)+VARIANTS(HEAD_K_DIM, 128)+VARIANTS(VT, 32)+VARIANTS(USE_MXU, false, true)+KERNEL(DeltaNetChunkedOutputAndState)(+ device const float* q_norm,+ device const float* k_norm,+ device const T* in_proj,+ device const float* qk_scaled,+ device const bfloat* t_mat,+ device const float* g,+ device const float* beta,+ device float* state,+ device O* out,+ constant const uint& num_v_heads,+ constant const uint& num_k_heads,+ constant const uint& head_v_dim,+ constant const uint& key_dim,+ constant const uint& value_dim,+ constant const uint& suffix_len,+ threadgroup float transposed_state_tile[HEAD_K_DIM * VT],+ threadgroup ScratchT residual_new_value_tile[OUTPUT_STATE_CHUNK * VT],+ const ThreadContext thread_context,+ const uint hv_idx GROUPS(num_v_heads),+ const uint value_tile_idx GROUPS(head_v_dim.div_ceil(VT)),+ const uint thread_idx THREADS(OUTPUT_STATE_THREADS)+) {+ using Ops = metal::conditional_t<USE_MXU, MxuFragmentOps<>, SimdgroupFragmentOps>;+ constexpr ushort FR = Ops::FRAGMENT_ROWS;+ constexpr ushort FC = Ops::FRAGMENT_COLS;+ static_assert(HEAD_K_DIM % FR == 0, "HEAD_K_DIM must be a multiple of fragment rows");+ static_assert(OUTPUT_STATE_CHUNK % FR == 0, "chunk size must be a multiple of fragment rows");+ static_assert(VT % FC == 0, "VT must be a multiple of fragment columns");+ static_assert(OUTPUT_STATE_KEY_TILE % FC == 0, "key tile must be a multiple of fragment columns");++ constexpr uint TOKEN_TILE = (VT >= 32) ? 16u : 32u;+ constexpr uint NUM_TOKEN_TILES = OUTPUT_STATE_CHUNK / TOKEN_TILE;+ constexpr ushort TOKEN_FRAGMENTS = TOKEN_TILE / FR;+ constexpr ushort VALUE_FRAGMENTS = VT / FC;+ constexpr ushort KEY_FRAGMENTS = OUTPUT_STATE_KEY_TILE / FC;++ using TokenValueFragment = Fragment<float, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, Ops>;+ using TokenKeyFragment = OperandFragment<float, TOKEN_FRAGMENTS, 1, Ops>;+ using KeyValueFragment = OperandFragment<float, 1, VALUE_FRAGMENTS, Ops>;+ using ScratchKeyValueFragment = OperandFragment<ScratchT, 1, VALUE_FRAGMENTS, Ops>;+ using ValueKeyFragment = Fragment<float, VALUE_FRAGMENTS, KEY_FRAGMENTS, Ops>;+ using ScratchValueTokenFragment = OperandFragment<ScratchT, VALUE_FRAGMENTS, 1, Ops>;+ using TokenKeyTileFragment = OperandFragment<float, 1, KEY_FRAGMENTS, Ops>;++ const uint lane = thread_context.simd_lane_id;+ const uint simdgroup_idx = thread_context.simdgroup_index;+ const uint value_base = value_tile_idx * VT;+ if (value_base >= head_v_dim) {+ return;+ }+ const uint hk_idx = hv_idx / (num_v_heads / num_k_heads);+ const uint num_chunks = (suffix_len + OUTPUT_STATE_CHUNK - 1) / OUTPUT_STATE_CHUNK;+ const uint conv_dim = 2 * key_dim + value_dim;+ const uint total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;+ const device float* g_head = g + hv_idx * suffix_len;++ // Load the initial state slice transposed into threadgroup memory:+ // transposed_state_tile[key * VT + value] = state[(hv, value_base + value, key)]+ for (uint idx = thread_idx; idx < VT * HEAD_K_DIM; idx += OUTPUT_STATE_THREADS) {+ const uint local_value_idx = idx / HEAD_K_DIM;+ const uint local_key_idx = idx - local_value_idx * HEAD_K_DIM;+ transposed_state_tile[local_key_idx * VT + local_value_idx] =+ state[(hv_idx * head_v_dim + value_base + local_value_idx) * HEAD_K_DIM + local_key_idx];+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ for (uint chunk_idx = 0; chunk_idx < num_chunks; ++chunk_idx) {+ const uint token_base = chunk_idx * OUTPUT_STATE_CHUNK;+ const uint valid_tokens = token_base < suffix_len ? min(uint(OUTPUT_STATE_CHUNK), suffix_len - token_base) : 0u;+ const uint chunk_head_base = (chunk_idx * num_v_heads + hv_idx);++ TokenValueFragment output_acc;+ // -- R phase: R = beta (.) (V - e^g (.) (K . S^T)) into residual_new_value_tile ----------+ if (simdgroup_idx < NUM_TOKEN_TILES) {+ const uint token_tile_base = simdgroup_idx * TOKEN_TILE;+ const uint valid_token_rows =+ token_tile_base < valid_tokens ? min(uint(TOKEN_TILE), valid_tokens - token_tile_base) : 0u;++ TokenValueFragment residual_acc;+ residual_acc.clear();+ const device float* k_tile = k_norm + (token_base + token_tile_base) * key_dim + hk_idx * HEAD_K_DIM;+ output_acc.clear();+ const uint query_tile_base = (token_base + token_tile_base) * key_dim + hk_idx * HEAD_K_DIM;+ for (uint key_block_start = 0; key_block_start < HEAD_K_DIM; key_block_start += FR) {+ TokenKeyFragment key_frag;+ TokenKeyFragment query_frag;+ KeyValueFragment state_frag;+ key_frag.load_from(lane, fragment_source(k_tile + key_block_start, int(key_dim)).bounded(valid_token_rows, FR));+ query_frag.load_from(+ lane,+ fragment_source(q_norm + query_tile_base + key_block_start, int(key_dim)).bounded(valid_token_rows, FR)+ );+ state_frag.load_from(lane, fragment_source(transposed_state_tile + key_block_start * VT, int(VT), 1));+ fragment_mma(residual_acc, key_frag, state_frag);+ fragment_mma(output_acc, query_frag, state_frag);+ }++ const device T* value_input_tile =+ in_proj + (token_base + token_tile_base) * total_proj_dim + 2 * key_dim + hv_idx * head_v_dim + value_base;+ residual_acc.map_coords(lane, [&](short row, short col, float state_projection) {+ if (uint(row) >= valid_token_rows) {+ return 0.0f;+ }+ const uint token = token_base + token_tile_base + uint(row);+ const float beta_t = beta[token * num_v_heads + hv_idx];+ const float g_t = g_head[token];+ const float value_input = float(value_input_tile[uint(row) * total_proj_dim + uint(col)]);+ return beta_t * (value_input - fast::exp(g_t) * state_projection);+ });+ residual_acc.store(lane, residual_new_value_tile + token_tile_base * VT, int(VT));+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // -- Vnew phase: Vnew = T . R (dense bf16 T streamed from device) --------+ TokenValueFragment new_value_acc;+ if (simdgroup_idx < NUM_TOKEN_TILES) {+ const uint token_tile_base = simdgroup_idx * TOKEN_TILE;+ const uint valid_token_rows =+ token_tile_base < valid_tokens ? min(uint(TOKEN_TILE), valid_tokens - token_tile_base) : 0u;++ new_value_acc.clear();+ const device bfloat* causal_inv_rows =+ t_mat + chunk_head_base * OUTPUT_STATE_CHUNK * OUTPUT_STATE_CHUNK + token_tile_base * OUTPUT_STATE_CHUNK;+ // Skip fully upper-causal T tiles.+ const uint causal_col_end = min(uint(OUTPUT_STATE_CHUNK), token_tile_base + TOKEN_TILE);+ for (uint source_token_block_start = 0; source_token_block_start < causal_col_end;+ source_token_block_start += FR) {+ TokenKeyFragment causal_inv_frag;+ ScratchKeyValueFragment residual_frag;+ causal_inv_frag.load_from(+ lane,+ fragment_source(causal_inv_rows + source_token_block_start, int(OUTPUT_STATE_CHUNK))+ .bounded(valid_token_rows, FR)+ );+ residual_frag.load_from(+ lane,+ fragment_source(residual_new_value_tile + source_token_block_start * VT, int(VT), 1)+ );+ fragment_mma(new_value_acc, causal_inv_frag, residual_frag);+ }+ }+ // Barrier before overwriting residual_new_value_tile (R) with Vnew: all simdgroups must be+ // done reading R above.+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (simdgroup_idx < NUM_TOKEN_TILES) {+ const uint token_tile_base = simdgroup_idx * TOKEN_TILE;+ new_value_acc.store(lane, residual_new_value_tile + token_tile_base * VT, int(VT));+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // -- Y phase: Y = e^g (.) (Q . S^T) + A . Vnew ; cast+store to out -------+ if (simdgroup_idx < NUM_TOKEN_TILES) {+ const uint token_tile_base = simdgroup_idx * TOKEN_TILE;+ const uint valid_token_rows =+ token_tile_base < valid_tokens ? min(uint(TOKEN_TILE), valid_tokens - token_tile_base) : 0u;++ output_acc.map_coords(lane, [&](short row, short, float value) {+ if (uint(row) >= valid_token_rows) {+ return 0.0f;+ }+ const uint token = token_base + token_tile_base + uint(row);+ return value * fast::exp(g_head[token]);+ });++ const uint qk_tile_base =+ chunk_head_base * OUTPUT_STATE_CHUNK * OUTPUT_STATE_CHUNK + token_tile_base * OUTPUT_STATE_CHUNK;+ // Skip fully upper-causal qk tiles.+ const uint causal_col_end = min(uint(OUTPUT_STATE_CHUNK), token_tile_base + TOKEN_TILE);+ for (uint source_token_block_start = 0; source_token_block_start < causal_col_end;+ source_token_block_start += FR) {+ const uint valid_source_tokens =+ source_token_block_start < valid_tokens ? min(uint(FR), valid_tokens - source_token_block_start) : 0u;+ TokenKeyFragment qk_frag;+ ScratchKeyValueFragment new_value_frag;+ qk_frag.load_from(+ lane,+ fragment_source(qk_scaled + qk_tile_base + source_token_block_start, int(OUTPUT_STATE_CHUNK))+ .bounded(valid_token_rows, valid_source_tokens)+ );+ new_value_frag.load_from(+ lane,+ fragment_source(residual_new_value_tile + source_token_block_start * VT, int(VT), 1)+ );+ fragment_mma(output_acc, qk_frag, new_value_frag);+ }++ const uint output_tile_base = (token_base + token_tile_base) * value_dim + hv_idx * head_v_dim + value_base;+ output_acc.store_safe(lane, out + output_tile_base, int(value_dim), short2(short(VT), short(valid_token_rows)));+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // -- Update phase: S^T <- exp(g_last) . S^T + (decay_scale (.) K)^T . Vnew -----+ {+ const uint key_tile_base = simdgroup_idx * OUTPUT_STATE_KEY_TILE;+ const uint g_last_token = token_base + (valid_tokens > 0 ? valid_tokens - 1 : 0u);+ const float g_last = g_head[g_last_token];+ const float state_decay = fast::exp(g_last);++ ValueKeyFragment state_update_acc;+ state_update_acc.clear();+ for (uint source_token_block_start = 0; source_token_block_start < OUTPUT_STATE_CHUNK;+ source_token_block_start += FR) {+ const uint valid_source_tokens =+ source_token_block_start < valid_tokens ? min(uint(FR), valid_tokens - source_token_block_start) : 0u;+ ScratchValueTokenFragment new_value_frag;+ TokenKeyTileFragment key_frag;+ new_value_frag.load_from(+ lane,+ fragment_source(residual_new_value_tile + source_token_block_start * VT, 1, int(VT))+ .bounded(short(VT), short(valid_source_tokens))+ );+ const device float* k_tile =+ k_norm + (token_base + source_token_block_start) * key_dim + hk_idx * HEAD_K_DIM + key_tile_base;+ key_frag.load_from(+ lane,+ fragment_source(k_tile, int(key_dim)).bounded(short(valid_source_tokens), short(OUTPUT_STATE_KEY_TILE))+ );+ key_frag.map_coords(lane, [&](short row, short, float value) {+ if (uint(row) >= valid_source_tokens) {+ return 0.0f;+ }+ const uint token = token_base + source_token_block_start + uint(row);+ // beta is already folded into Vnew through R.+ return value * fast::exp(g_last - g_head[token]);+ });+ fragment_mma(state_update_acc, new_value_frag, key_frag);+ }++ state_update_acc.map_coords(lane, [&](short row, short col, float value) {+ return state_decay * transposed_state_tile[(key_tile_base + uint(col)) * VT + uint(row)] + value;+ });+ state_update_acc.store(lane, transposed_state_tile + key_tile_base * VT, 1, int(VT));+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }++ // Write the final state slice back to device, transposing [K, VT] -> [V, K].+ for (uint idx = thread_idx; idx < VT * HEAD_K_DIM; idx += OUTPUT_STATE_THREADS) {+ const uint local_value_idx = idx / HEAD_K_DIM;+ const uint local_key_idx = idx - local_value_idx * HEAD_K_DIM;+ state[(hv_idx * head_v_dim + value_base + local_value_idx) * HEAD_K_DIM + local_key_idx] =+ transposed_state_tile[local_key_idx * VT + local_value_idx];+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/common/gram.h b/crates/backend-uzu/src/backends/metal/kernel/gdn/common/gram.hnew file mode 100644index 000000000..e374a97bb--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/common/gram.h@@ -0,0 +1,47 @@+#pragma once++#include "../../common/defines.h"+#include "../../matmul/common/fragment.h"+#include "../../matmul/common/mxu_fragment_ops.h"+#include "../../matmul/common/simdgroup_fragment_ops.h"+#include <metal_stdlib>++using namespace metal;+using namespace uzu::matmul;++template <typename AccFragment, typename LeftFragment, typename RightFragment, typename T>+METAL_FUNC void accumulate_dual_gram_tile(+ thread AccFragment& kk_acc,+ thread AccFragment& qk_acc,+ const device T* k_rows,+ const device T* q_rows,+ const device T* k_cols,+ const int stride,+ const ushort valid_rows,+ const ushort valid_cols,+ const ushort valid_k,+ const bool full_left,+ const bool full_right,+ const ushort lane+) {+ LeftFragment k_left;+ LeftFragment q_left;+ RightFragment k_right;++ if (full_left) {+ k_left.load_from(lane, fragment_source(k_rows, stride));+ q_left.load_from(lane, fragment_source(q_rows, stride));+ } else {+ k_left.load_from(lane, fragment_source(k_rows, stride).bounded(valid_rows, valid_k));+ q_left.load_from(lane, fragment_source(q_rows, stride).bounded(valid_rows, valid_k));+ }++ if (full_right) {+ k_right.load_from(lane, fragment_source(k_cols, stride));+ } else {+ k_right.load_from(lane, fragment_source(k_cols, stride).bounded(valid_cols, valid_k));+ }++ fragment_mma(kk_acc, k_left, k_right);+ fragment_mma(qk_acc, q_left, k_right);+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/common/tri_inv.h b/crates/backend-uzu/src/backends/metal/kernel/gdn/common/tri_inv.hnew file mode 100644index 000000000..796fcbe24--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/common/tri_inv.h@@ -0,0 +1,41 @@+#pragma once++#include "../../common/defines.h"+#include <metal_stdlib>++using namespace metal;++template <uint BLOCK>+METAL_FUNC void invert_lower_triangular_block(+ device float* a_inv_block,+ threadgroup const float* a_tile,+ const uint block_size,+ const uint lane+) {+ if (lane >= BLOCK) {+ return;+ }++ const uint col = lane;+ float inverse_col[BLOCK] = {};+ inverse_col[col] = 1.0f;++ METAL_PRAGMA_UNROLL+ for (uint row = 0; row < BLOCK; row++) {+ if (row > col && row < block_size) {+ float acc = 0.0f;+ METAL_PRAGMA_UNROLL+ for (uint prev_row = 0; prev_row < BLOCK; prev_row++) {+ if (prev_row < row) {+ acc += a_tile[row * BLOCK + prev_row] * inverse_col[prev_row];+ }+ }+ inverse_col[row] = -acc;+ }+ }++ METAL_PRAGMA_UNROLL+ for (uint row = 0; row < BLOCK; row++) {+ a_inv_block[row * BLOCK + col] = inverse_col[row];+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_scan.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_scan.metalsimilarity index 100%rename from crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_scan.metalrename to crates/backend-uzu/src/backends/metal/kernel/gdn/conv_scan.metaldiff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_update.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_update.metalsimilarity index 100%rename from crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_update.metalrename to crates/backend-uzu/src/backends/metal/kernel/gdn/conv_update.metaldiff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/mod.rsnew file mode 100644index 000000000..60c62629e--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/mod.rs@@ -0,0 +1 @@+pub mod chunked;diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/norm_gate.metalsimilarity index 100%rename from crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metalrename to crates/backend-uzu/src/backends/metal/kernel/gdn/norm_gate.metaldiff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill.metalsimilarity index 97%rename from crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metalrename to crates/backend-uzu/src/backends/metal/kernel/gdn/prefill.metalindex 822cf8918..8f24c8c02 100644--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill.metal@@ -44,8 +44,7 @@ PUBLIC KERNEL(DeltaNetPrefill)(const uint dv_local = tid / METAL_SIMD_SIZE;const uint dv_idx = (dv_group * NUM_SG + dv_local) * DV_PER_SIMDGROUP;- const uint groups_per_head = num_v_heads / num_k_heads;- const uint hk = hv_idx / groups_per_head;+ const uint hk = hv_idx / (num_v_heads / num_k_heads);const uint conv_dim = 2 * key_dim + value_dim;const uint total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;const uint dk_base = lane * ELEMS;diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill_prep.metalsimilarity index 53%rename from crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metalrename to crates/backend-uzu/src/backends/metal/kernel/gdn/prefill_prep.metalindex f78cf62e8..ee6f9cd92 100644--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill_prep.metal@@ -5,10 +5,6 @@using namespace metal;-// Pre-compute normalized q, k, beta, decay for DeltaNet prefill.-// Grid: suffix_len × num_k_heads threadgroups, METAL_SIMD_SIZE threads each (1-// SIMD group).-template <typename T, uint HEAD_K_DIM>VARIANTS(T, float, half, bfloat)VARIANTS(HEAD_K_DIM, 128)@@ -27,6 +23,7 @@ PUBLIC KERNEL(DeltaNetPrefillPrep)(constant const uint& suffix_len,const uint token_idx GROUPS(suffix_len),const uint hk_idx GROUPS(num_k_heads),+ const bool write_log_decay SPECIALIZE,const uint lane THREADS(METAL_SIMD_SIZE)) {static_assert(HEAD_K_DIM % METAL_SIMD_SIZE == 0, "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE");@@ -40,54 +37,41 @@ PUBLIC KERNEL(DeltaNetPrefillPrep)(const uint q_base = tok_offset + hk_idx * HEAD_K_DIM;const uint k_base = tok_offset + key_dim + hk_idx * HEAD_K_DIM;- // Load q and k elements (strided by SIMD width)float q_vals[elems_per_thread];float k_vals[elems_per_thread];float q_sq = 0.0f;float k_sq = 0.0f;for (uint i = 0; i < elems_per_thread; ++i) {- uint idx = lane + i * METAL_SIMD_SIZE;+ const uint idx = lane + i * METAL_SIMD_SIZE;q_vals[i] = float(in_proj[q_base + idx]);k_vals[i] = float(in_proj[k_base + idx]);q_sq += q_vals[i] * q_vals[i];k_sq += k_vals[i] * k_vals[i];}- // L2 normalize via simd_sum (no threadgroup barrier needed)- float q_norm_sq = simd_sum(q_sq);- float k_norm_sq = simd_sum(k_sq);- float q_inv_norm = rsqrt(q_norm_sq + 1e-6f);- float k_inv_norm = rsqrt(k_norm_sq + 1e-6f);- float q_scale = rsqrt(float(HEAD_K_DIM));-- // Normalize and scale q, write q and k- const uint out_q_base = token_idx * key_dim + hk_idx * HEAD_K_DIM;- const uint out_k_base = token_idx * key_dim + hk_idx * HEAD_K_DIM;+ const float q_inv_norm = rsqrt(simd_sum(q_sq) + 1e-6f);+ const float k_inv_norm = rsqrt(simd_sum(k_sq) + 1e-6f);+ const float q_scale = rsqrt(float(HEAD_K_DIM));+ const uint out_base = token_idx * key_dim + hk_idx * HEAD_K_DIM;for (uint i = 0; i < elems_per_thread; ++i) {- uint idx = lane + i * METAL_SIMD_SIZE;- float q_n = q_vals[i] * q_inv_norm * q_scale;- float k_n = k_vals[i] * k_inv_norm;- if (idx < HEAD_K_DIM) {- q_norm_out[out_q_base + idx] = q_n;- k_norm_out[out_k_base + idx] = k_n;- }+ const uint idx = lane + i * METAL_SIMD_SIZE;+ q_norm_out[out_base + idx] = q_vals[i] * q_inv_norm * q_scale;+ k_norm_out[out_base + idx] = k_vals[i] * k_inv_norm;}- // Compute beta and decay for v-heads belonging to this k-head- // groups_per_head v-heads share this k-head; distribute across lanesfor (uint group = lane; group < groups_per_head; group += METAL_SIMD_SIZE) {- uint hv = hk_idx * groups_per_head + group;+ const uint hv = hk_idx * groups_per_head + group;+ const float beta_raw = float(in_proj[tok_offset + conv_dim + value_dim + hv]);+ const float a_raw = float(in_proj[tok_offset + conv_dim + value_dim + num_v_heads + hv]);+ const float log_decay = -fast::exp(a_log[hv]) * activate_softplus(a_raw + dt_bias[hv]);- float beta_raw = float(in_proj[tok_offset + conv_dim + value_dim + hv]);- float beta = 1.0f / (1.0f + fast::exp(-beta_raw));-- float a_raw = float(in_proj[tok_offset + conv_dim + value_dim + num_v_heads + hv]);- float sp = activate_softplus(a_raw + float(dt_bias[hv]));- float decay = fast::exp(-fast::exp(float(a_log[hv])) * sp);-- beta_out[token_idx * num_v_heads + hv] = beta;- decay_out[token_idx * num_v_heads + hv] = decay;+ beta_out[token_idx * num_v_heads + hv] = 1.0f / (1.0f + fast::exp(-beta_raw));+ if (write_log_decay) {+ decay_out[token_idx * num_v_heads + hv] = log_decay;+ } else {+ decay_out[token_idx * num_v_heads + hv] = fast::exp(log_decay);+ }}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/build_tree_out_dispatch_helper.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/build_tree_out_dispatch_helper.rssimilarity index 100%rename from crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/build_tree_out_dispatch_helper.rsrename to crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/build_tree_out_dispatch_helper.rsdiff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/out.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metalsimilarity index 94%rename from crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/out.metalrename to crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metalindex 93564eeff..1a6446c7e 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/out.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metal@@ -1,10 +1,10 @@#include <metal_stdlib>-#include "../common/defines.h"-#include "../common/dsl.h"-#include "../common/thread_context.h"-#include "../matmul/common/fragment.h"-#include "../matmul/common/mxu_fragment_ops.h"-#include "../matmul/common/simdgroup_fragment_ops.h"+#include "../../common/defines.h"+#include "../../common/dsl.h"+#include "../../common/thread_context.h"+#include "../../matmul/common/fragment.h"+#include "../../matmul/common/mxu_fragment_ops.h"+#include "../../matmul/common/simdgroup_fragment_ops.h"using namespace metal;using namespace uzu::matmul;@@ -75,8 +75,7 @@ PUBLIC KERNEL(BuildTreeOut)(const uint batch_idx = batch_value_head_idx / value_heads;const uint value_head_idx = batch_value_head_idx - batch_idx * value_heads;- const uint value_heads_per_qk_head = value_heads / qk_heads;- const uint qk_head_idx = value_head_idx / value_heads_per_qk_head;+ const uint qk_head_idx = value_head_idx / (value_heads / qk_heads);const uint row_base = (row_tile_group_idx * SIMDGROUPS_PER_TG + thread_context.simdgroup_index) * ROWS;const uint value_base = v_tile_idx * MATMUL_COLS;diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/prefix_beta.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix_beta.metalsimilarity index 92%rename from crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/prefix_beta.metalrename to crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix_beta.metalindex 50736b4df..e498d5aeb 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/prefix_beta.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix_beta.metal@@ -1,9 +1,9 @@#include <metal_stdlib>-#include "../activation/activations.h"-#include "../common/defines.h"-#include "../common/dsl.h"-#include "../common/thread_context.h"-#include "../generated/trie.h"+#include "../../activation/activations.h"+#include "../../common/defines.h"+#include "../../common/dsl.h"+#include "../../common/thread_context.h"+#include "../../generated/trie.h"#define ROWS_PER_THREADGROUP 4udiff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_gram.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metalsimilarity index 79%rename from crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_gram.metalrename to crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metalindex 8ee34866c..dc2f1821c 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_gram.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal@@ -1,11 +1,13 @@#include <metal_stdlib>-#include "../common/defines.h"-#include "../common/dsl.h"-#include "../common/thread_context.h"-#include "../generated/trie.h"-#include "../matmul/common/fragment.h"-#include "../matmul/common/mxu_fragment_ops.h"-#include "../matmul/common/simdgroup_fragment_ops.h"+#include "../../common/defines.h"+#include "../../common/dsl.h"+#include "../../common/thread_context.h"+#include "../../generated/trie.h"+#include "../../matmul/common/fragment.h"+#include "../../matmul/common/mxu_fragment_ops.h"+#include "../../matmul/common/simdgroup_fragment_ops.h"+#include "../common/gram.h"+#include "../common/tri_inv.h"using namespace metal;using namespace uzu::matmul;@@ -16,73 +18,6 @@ using namespace uzu::trie;#define NUM_SIMDGROUPS 2u#define INVALID_ROW 0xffffffffu-// Ragged blocks are padded with identity columns so the 16x16 output is fully-// initialized.-METAL_FUNC void invert_tree_gram_diagonal_block(- device float* a_inv_block,- threadgroup const float* diag_a_tile,- const uint block_size,- const uint lane-) {- if (lane >= ROW_TILE) {- return;- }-- const uint col = lane;- float inverse_col[ROW_TILE] = {};- inverse_col[col] = 1.0f;-- METAL_PRAGMA_UNROLL- for (uint row = 0; row < ROW_TILE; row++) {- if (row > col && row < block_size) {- float acc = 0.0f;- METAL_PRAGMA_UNROLL- for (uint prev_row = 0; prev_row < ROW_TILE; prev_row++) {- if (prev_row < row) {- acc += diag_a_tile[row * ROW_TILE + prev_row] * inverse_col[prev_row];- }- }- inverse_col[row] = -acc;- }- }-- METAL_PRAGMA_UNROLL- for (uint row = 0; row < ROW_TILE; row++) {- a_inv_block[row * ROW_TILE + col] = inverse_col[row];- }-}--// Unscaled gram dots for one tile: kk = k[rows].k[cols], qk = q[rows].k[cols].-template <typename T, typename Ops, typename AccFragment, typename LeftFragment, typename RightFragment>-METAL_FUNC void tree_gram_dots(- const device T* k_rows,- const device T* q_rows,- const device T* k_cols,- const uint head_k_dim,- const uint qk_stride,- const bool full_rows,- const uint tile_rows,- const bool full_cols,- const uint tile_cols,- const ushort lane,- thread AccFragment& kk_acc,- thread AccFragment& qk_acc-) {- kk_acc.clear();- qk_acc.clear();- // head_k_dim is a multiple of the fragment K depth, so the contraction has no tail.- for (uint kb = 0; kb < head_k_dim; kb += Ops::FRAGMENT_ROWS) {- LeftFragment k_left;- LeftFragment q_left;- RightFragment k_right;- k_left.load_maybe_bounded(lane, k_rows + kb, qk_stride, full_rows, tile_rows, Ops::FRAGMENT_ROWS);- q_left.load_maybe_bounded(lane, q_rows + kb, qk_stride, full_rows, tile_rows, Ops::FRAGMENT_ROWS);- k_right.load_maybe_bounded(lane, k_cols + kb, qk_stride, full_cols, tile_cols, Ops::FRAGMENT_ROWS);- fragment_mma(kk_acc, k_left, k_right);- fragment_mma(qk_acc, q_left, k_right);- }-}-// kh0 = k @ h0[h0_idx[batch]]^T for one row-tile, walking head_v_dim in COL_TILE// chunks from dv_start by dv_stride. No-op for a zero initial state.template <typename T, typename Ops, typename AccFragment, typename LeftFragment, typename RightFragment>@@ -199,8 +134,7 @@ PUBLIC KERNEL(BuildTreeGram)(using RightFragment = OperandFragment<InputType, 1, COL_FRAGMENTS, Ops, ReadTranspose>;static_assert(COL_TILE == METAL_SIMD_SIZE, "COL_TILE must match the SIMD width");- const uint value_heads_per_key_head = value_heads / k_heads;- const uint key_head_idx = value_head_idx / value_heads_per_key_head;+ const uint key_head_idx = value_head_idx / (value_heads / k_heads);const uint qk_stride = k_heads * head_k_dim;const uint qk_base = (batch_idx * tree_size * k_heads + key_head_idx) * head_k_dim;const uint prefix_base = batch_idx * tree_size * value_heads + value_head_idx;@@ -267,20 +201,24 @@ PUBLIC KERNEL(BuildTreeGram)(AccFragment kk_acc;AccFragment qk_acc;- tree_gram_dots<T, Ops, AccFragment, LeftFragment, RightFragment>(- k_rows,- q_rows,- k_cols,- head_k_dim,- qk_stride,- full_rows,- tile_rows,- full_cols,- tile_cols,- lane,- kk_acc,- qk_acc- );+ kk_acc.clear();+ qk_acc.clear();+ for (uint kb = 0; kb < head_k_dim; kb += Ops::FRAGMENT_ROWS) {+ accumulate_dual_gram_tile<AccFragment, LeftFragment, RightFragment>(+ kk_acc,+ qk_acc,+ k_rows + kb,+ q_rows + kb,+ k_cols + kb,+ int(qk_stride),+ ushort(tile_rows),+ ushort(tile_cols),+ Ops::FRAGMENT_ROWS,+ full_rows,+ full_cols,+ lane+ );+ }const bool has_diag = col_tile_idx == diag_col_tile;simdgroup_barrier(mem_flags::mem_threadgroup);@@ -320,7 +258,7 @@ PUBLIC KERNEL(BuildTreeGram)(simdgroup_barrier(mem_flags::mem_threadgroup);device float* a_inv_block =a_inv + ((batch_idx * value_heads + value_head_idx) * num_blocks + row_tile_idx) * (ROW_TILE * ROW_TILE);- invert_tree_gram_diagonal_block(a_inv_block, diag_a_tile, tile_rows, lane);+ invert_lower_triangular_block<ROW_TILE>(a_inv_block, diag_a_tile, tile_rows, lane);}} else if (col_tile_idx < col_tiles) {// Above-diagonal zero fill; COL_TILE == METAL_SIMD_SIZE, so eachdiff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_update_solve.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metalsimilarity index 97%rename from crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_update_solve.metalrename to crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metalindex 3d4731ef8..8e8627c2e 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_update_solve.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal@@ -1,9 +1,9 @@#include <metal_stdlib>-#include "../common/defines.h"-#include "../common/dsl.h"-#include "../matmul/common/fragment.h"-#include "../matmul/common/mxu_fragment_ops.h"-#include "../matmul/common/simdgroup_fragment_ops.h"+#include "../../common/defines.h"+#include "../../common/dsl.h"+#include "../../matmul/common/fragment.h"+#include "../../matmul/common/mxu_fragment_ops.h"+#include "../../matmul/common/simdgroup_fragment_ops.h"using namespace metal;using namespace uzu::matmul;diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_update_solve_dispatch_helper.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve_dispatch_helper.rssimilarity index 94%rename from crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_update_solve_dispatch_helper.rsrename to crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve_dispatch_helper.rsindex 3a860531d..d3e5c0905 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_update_solve_dispatch_helper.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve_dispatch_helper.rs@@ -51,5 +51,5 @@ pub(crate) fn tree_update_solve_variant(}#[cfg(test)]-#[path = "../../../../../tests/unit/backends/metal/tree_update_solve_dispatch_test.rs"]+#[path = "../../../../../../tests/unit/backends/metal/tree_update_solve_dispatch_test.rs"]mod tests;diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/update.metalsimilarity index 97%rename from crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metalrename to crates/backend-uzu/src/backends/metal/kernel/gdn/update.metalindex 9790551f8..c99b2add8 100644--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/update.metal@@ -43,8 +43,7 @@ PUBLIC KERNEL(DeltaNetUpdate)(const uint sg = tid / METAL_SIMD_SIZE;const uint conv_dim = 2 * key_dim + value_dim;- const uint groups_per_head = num_v_heads / num_k_heads;- const uint hk = hv_idx / groups_per_head;+ const uint hk = hv_idx / (num_v_heads / num_k_heads);// Load + normalize q/k; each simd group recomputes the norm to skip a// barrier.diff --git a/crates/backend-uzu/src/backends/metal/kernel/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/mod.rsindex 02c955e65..f4a414d40 100644--- a/crates/backend-uzu/src/backends/metal/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/mod.rs@@ -7,10 +7,11 @@ use crate::backends::{};pub mod attention;-#[path = "gdn_tree_verify/build_tree_out_dispatch_helper.rs"]+#[path = "gdn/tree_verify/build_tree_out_dispatch_helper.rs"]mod build_tree_out_dispatch_helper;+pub mod gdn;pub mod matmul;-#[path = "gdn_tree_verify/tree_update_solve_dispatch_helper.rs"]+#[path = "gdn/tree_verify/tree_update_solve_dispatch_helper.rs"]mod tree_update_solve_dispatch_helper;pub const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));@@ -24,5 +25,6 @@ impl Kernels for MetalKernels {autogen_kernels!();type AttentionGemmCore = attention::AttentionGemmMetalCore;+ type DeltaNetChunkedPrefill = gdn::chunked::MetalDeltaNetChunkedPrefill;type MatmulKernel = matmul::MatmulMetalKernel;}diff --git a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rsindex 753544f10..f511818a5 100644--- a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs@@ -7,6 +7,7 @@ use crate::{kernel::{Conv1dPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel, DeltaNetNormGateKernel,DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,+ delta_net_chunked_prefill::{DeltaNetChunkedPrefill, DeltaNetChunkedPrefillArgs},},},config::token_mixer::delta_net::DeltaNetConfig,@@ -71,6 +72,7 @@ pub struct DeltaNet<B: Backend> {delta_net_prefill_prep: <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel,delta_net_prefill: <B::Kernels as Kernels>::DeltaNetPrefillKernel,delta_net_norm_gate: <B::Kernels as Kernels>::DeltaNetNormGateKernel,+ chunked: Option<<B::Kernels as Kernels>::DeltaNetChunkedPrefill>,out_projection: Box<dyn Linear<B>>,}@@ -162,15 +164,23 @@ impl<B: Backend> DeltaNet<B> {let delta_net_update =<B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, outer_data_type, config.head_dim as u32).map_err(DeltaNetNewError::Backend)?;- let delta_net_prefill_prep =- <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(context, outer_data_type, config.head_dim as u32)- .map_err(DeltaNetNewError::Backend)?;+ let delta_net_prefill_prep = <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(+ context,+ outer_data_type,+ config.head_dim as u32,+ false,+ )+ .map_err(DeltaNetNewError::Backend)?;let delta_net_prefill =<B::Kernels as Kernels>::DeltaNetPrefillKernel::new(context, outer_data_type, config.head_dim as u32).map_err(DeltaNetNewError::Backend)?;let delta_net_norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, outer_data_type).map_err(DeltaNetNewError::Backend)?;+ let chunked =+ <B::Kernels as Kernels>::DeltaNetChunkedPrefill::new(context, outer_data_type, config.head_dim as u32)+ .map_err(DeltaNetNewError::Backend)?;+let out_projection = <dyn Linear<B>>::new_mixed_precision(value_dim,[hidden_dim],@@ -208,6 +218,7 @@ impl<B: Backend> DeltaNet<B> {delta_net_prefill_prep,delta_net_prefill,delta_net_norm_gate,+ chunked,out_projection,},in_projection_input_hadamard_factors,@@ -333,47 +344,65 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {self.total_proj_dim as u32,encoder,);-- let mut prep_q_norm =- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.key_dim], DataType::F32))?;- let mut prep_k_norm =- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.key_dim], DataType::F32))?;- let mut prep_beta =- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.num_heads], DataType::F32))?;- let mut prep_decay =- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.num_heads], DataType::F32))?;- self.delta_net_prefill_prep.encode(- &in_projected,- &self.a_log,- &self.dt_bias,- &mut prep_q_norm,- &mut prep_k_norm,- &mut prep_beta,- &mut prep_decay,- self.num_heads as u32,- self.num_groups as u32,- self.key_dim as u32,- self.value_dim as u32,- batch_dim.size() as u32,- encoder,- );- self.delta_net_prefill.encode(- &prep_q_norm,- &prep_k_norm,- &prep_beta,- &prep_decay,- &in_projected,- &mut state.ssm_state,- &mut delta_output,- self.num_heads as u32,- self.num_groups as u32,- self.value_head_dim as u32,- self.key_dim as u32,- self.value_dim as u32,- batch_dim.size() as u32,- self.value_head_dim.div_ceil(16) as u32,- encoder,- );+ if let Some(chunked) = self.chunked.as_ref().filter(|chunked| chunked.should_use(batch_dim.size())) {+ chunked.encode(+ DeltaNetChunkedPrefillArgs {+ in_projected: &in_projected,+ a_log: &self.a_log,+ dt_bias: &self.dt_bias,+ ssm_state: &mut state.ssm_state,+ delta_output: &mut delta_output,+ num_heads: self.num_heads as u32,+ num_groups: self.num_groups as u32,+ value_head_dim: self.value_head_dim as u32,+ key_dim: self.key_dim as u32,+ value_dim: self.value_dim as u32,+ suffix_len: batch_dim.size(),+ },+ encoder,+ )?;+ } else {+ let mut prep_q_norm =+ encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.key_dim], DataType::F32))?;+ let mut prep_k_norm =+ encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.key_dim], DataType::F32))?;+ let mut prep_beta =+ encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.num_heads], DataType::F32))?;+ let mut prep_decay =+ encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.num_heads], DataType::F32))?;+ self.delta_net_prefill_prep.encode(+ &in_projected,+ &self.a_log,+ &self.dt_bias,+ &mut prep_q_norm,+ &mut prep_k_norm,+ &mut prep_beta,+ &mut prep_decay,+ self.num_heads as u32,+ self.num_groups as u32,+ self.key_dim as u32,+ self.value_dim as u32,+ batch_dim.size() as u32,+ encoder,+ );+ self.delta_net_prefill.encode(+ &prep_q_norm,+ &prep_k_norm,+ &prep_beta,+ &prep_decay,+ &in_projected,+ &mut state.ssm_state,+ &mut delta_output,+ self.num_heads as u32,+ self.num_groups as u32,+ self.value_head_dim as u32,+ self.key_dim as u32,+ self.value_dim as u32,+ batch_dim.size() as u32,+ self.value_head_dim.div_ceil(16) as u32,+ encoder,+ );+ }self.delta_net_norm_gate.encode(&mut delta_output,&in_projected,diff --git a/crates/backend-uzu/unit/backends/common/kernel/delta_net_test.rs b/crates/backend-uzu/unit/backends/common/kernel/delta_net_test.rsindex 791293de5..afd333ab2 100644--- a/crates/backend-uzu/unit/backends/common/kernel/delta_net_test.rs+++ b/crates/backend-uzu/unit/backends/common/kernel/delta_net_test.rs@@ -330,6 +330,7 @@ fn run_prefill_with_norm_gate_typed<T: ArrayElement>(&context,T::data_type(),head_k_dim as u32,+ false,).unwrap();let prefill_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillKernel::new(@@ -508,6 +509,7 @@ fn test_delta_net_prefill_prep() {&cpu_ctx,DataType::F32,head_k_dim as u32,+ false,).unwrap();let mut cpu_enc = Encoder::new(cpu_ctx.as_ref()).expect("encoder");@@ -548,6 +550,7 @@ fn test_delta_net_prefill_prep() {&context,DataType::F32,head_k_dim as u32,+ false,).unwrap();@@ -620,6 +623,7 @@ fn bench_delta_net_prefill() {&context,DataType::F32,head_k_dim as u32,+ false,).unwrap();let prefill_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillKernel::new(diff --git a/crates/backend-uzu/unit/backends/metal/kernel/gdn/chunked_test.rs b/crates/backend-uzu/unit/backends/metal/kernel/gdn/chunked_test.rsnew file mode 100644index 000000000..0948a6fb5--- /dev/null+++ b/crates/backend-uzu/unit/backends/metal/kernel/gdn/chunked_test.rs@@ -0,0 +1,198 @@+use half::bf16;+use num_traits::cast;+use proc_macros::uzu_test;++use super::*;+use crate::{+ array::ArrayElement,+ backends::{+ common::{+ Context, Kernels,+ kernel::{DeltaNetNormGateKernel, DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel},+ },+ metal::Metal,+ },+ tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+};++type BackendKernels<B> = <B as Backend>::Kernels;++const NUM_V_HEADS: usize = 4;+const NUM_K_HEADS: usize = 1;+const HEAD_K_DIM: usize = 128;+const HEAD_V_DIM: usize = 128;++#[derive(Clone, Copy)]+enum PrefillMode {+ Recurrent,+ Chunked,+}++struct TestCase {+ suffix_len: usize,+ in_proj: Vec<f32>,+ a_log: Vec<f32>,+ dt_bias: Vec<f32>,+ norm_weight: Vec<f32>,+ key_dim: usize,+ value_dim: usize,+ total_proj_dim: usize,+}++fn test_case(suffix_len: usize) -> TestCase {+ let key_dim = NUM_K_HEADS * HEAD_K_DIM;+ let value_dim = NUM_V_HEADS * HEAD_V_DIM;+ let conv_dim = 2 * key_dim + value_dim;+ let total_proj_dim = conv_dim + value_dim + NUM_V_HEADS + NUM_V_HEADS;++ TestCase {+ suffix_len,+ in_proj: (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect(),+ a_log: (0..NUM_V_HEADS).map(|i| -1.5 + (i as f32) * 0.05).collect(),+ dt_bias: (0..NUM_V_HEADS).map(|i| 0.3 + (i as f32) * 0.02).collect(),+ norm_weight: (0..HEAD_V_DIM).map(|i| 0.9 + (i as f32) * 0.001).collect(),+ key_dim,+ value_dim,+ total_proj_dim,+ }+}++fn run_prefill<T: ArrayElement>(+ context: &MetalContext,+ case: &TestCase,+ mode: PrefillMode,+) -> (Vec<f32>, Vec<f32>) {+ let in_proj_data: Vec<T> = case.in_proj.iter().copied().map(|value| cast(value).unwrap()).collect();+ let in_proj = alloc_allocation_with_data::<Metal, T>(context, &in_proj_data);+ let a_log = alloc_allocation_with_data::<Metal, f32>(context, &case.a_log);+ let dt_bias = alloc_allocation_with_data::<Metal, f32>(context, &case.dt_bias);+ let norm_weight = alloc_allocation_with_data::<Metal, f32>(context, &case.norm_weight);+ let state_len = NUM_V_HEADS * HEAD_V_DIM * HEAD_K_DIM;+ let mut state = alloc_allocation::<Metal, f32>(context, state_len);+ let mut out = alloc_allocation::<Metal, T>(context, case.suffix_len * case.value_dim);+ let mut q = alloc_allocation::<Metal, f32>(context, case.suffix_len * case.key_dim);+ let mut k = alloc_allocation::<Metal, f32>(context, case.suffix_len * case.key_dim);+ let mut beta = alloc_allocation::<Metal, f32>(context, case.suffix_len * NUM_V_HEADS);+ let mut decay = alloc_allocation::<Metal, f32>(context, case.suffix_len * NUM_V_HEADS);+ let mut encoder = Encoder::new(context).expect("encoder");+ encoder.encode_fill(&mut state, 0);++ match mode {+ PrefillMode::Recurrent => {+ <BackendKernels<Metal> as Kernels>::DeltaNetPrefillPrepKernel::new(+ context,+ T::data_type(),+ HEAD_K_DIM as u32,+ false,+ )+ .expect("prep")+ .encode(+ &in_proj,+ &a_log,+ &dt_bias,+ &mut q,+ &mut k,+ &mut beta,+ &mut decay,+ NUM_V_HEADS as u32,+ NUM_K_HEADS as u32,+ case.key_dim as u32,+ case.value_dim as u32,+ case.suffix_len as u32,+ &mut encoder,+ );+ <BackendKernels<Metal> as Kernels>::DeltaNetPrefillKernel::new(context, T::data_type(), HEAD_K_DIM as u32)+ .expect("prefill")+ .encode(+ &q,+ &k,+ &beta,+ &decay,+ &in_proj,+ &mut state,+ &mut out,+ NUM_V_HEADS as u32,+ NUM_K_HEADS as u32,+ HEAD_V_DIM as u32,+ case.key_dim as u32,+ case.value_dim as u32,+ case.suffix_len as u32,+ HEAD_V_DIM.div_ceil(16) as u32,+ &mut encoder,+ );+ },+ PrefillMode::Chunked => {+ <BackendKernels<Metal> as Kernels>::DeltaNetChunkedPrefill::new(context, T::data_type(), HEAD_K_DIM as u32)+ .expect("chunked")+ .expect("chunked unsupported")+ .encode(+ DeltaNetChunkedPrefillArgs {+ in_projected: &in_proj,+ a_log: &a_log,+ dt_bias: &dt_bias,+ ssm_state: &mut state,+ delta_output: &mut out,+ num_heads: NUM_V_HEADS as u32,+ num_groups: NUM_K_HEADS as u32,+ value_head_dim: HEAD_V_DIM as u32,+ key_dim: case.key_dim as u32,+ value_dim: case.value_dim as u32,+ suffix_len: case.suffix_len,+ },+ &mut encoder,+ )+ .expect("chunked encode");+ },+ }++ <BackendKernels<Metal> as Kernels>::DeltaNetNormGateKernel::new(context, T::data_type()).expect("norm").encode(+ &mut out,+ &in_proj,+ &norm_weight,+ NUM_V_HEADS as u32,+ HEAD_V_DIM as u32,+ case.value_dim as u32,+ (2 * case.key_dim + case.value_dim) as u32,+ case.total_proj_dim as u32,+ 1e-6,+ case.suffix_len as u32,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ (+ allocation_to_vec::<Metal, T>(&out).into_iter().map(|value| cast(value).unwrap()).collect(),+ allocation_to_vec(&state),+ )+}++#[uzu_test]+fn chunked_prefill_matches_recurrent_prefill() {+ let context = <Metal as Backend>::Context::new().expect("metal context");+ if <BackendKernels<Metal> as Kernels>::DeltaNetChunkedPrefill::new(+ &context,+ <bf16 as ArrayElement>::data_type(),+ HEAD_K_DIM as u32,+ )+ .expect("chunked")+ .is_none()+ {+ eprintln!("chunked prefill unsupported on this Metal device");+ return;+ }++ for suffix_len in [CHUNK_SIZE - 1, CHUNK_SIZE, CHUNK_SIZE + 1, CHUNK_SIZE * 2 + 1] {+ let case = test_case(suffix_len);+ let (recurrent_out, recurrent_state) = run_prefill::<bf16>(&context, &case, PrefillMode::Recurrent);+ let (chunked_out, chunked_state) = run_prefill::<bf16>(&context, &case, PrefillMode::Chunked);++ assert!(+ is_close::default().abs_tol(5e-3).rel_tol(1e-3).all_close(chunked_out.iter().copied(), recurrent_out),+ "chunked output differs at T={suffix_len}"+ );+ assert!(+ is_close::default().abs_tol(2e-4).rel_tol(1e-3).all_close(chunked_state.iter().copied(), recurrent_state),+ "chunked state differs at T={suffix_len}"+ );+ }+}