Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Chunked GDN (#589)

CCYeh6597e3beb7f7
diff --git a/crates/backend-uzu/src/backends/common/kernel/delta_net_chunked_prefill.rs b/crates/backend-uzu/src/backends/common/kernel/delta_net_chunked_prefill.rs
new file mode 100644
index 000000000..d6de4bd5f
--- /dev/null
+++ b/crates/backend-uzu/src/backends/common/kernel/delta_net_chunked_prefill.rs
@@ -0,0 +1,64 @@
+use std::convert::Infallible;
+
+use crate::{
+ backends::common::{Allocation, Backend, Encoder},
+ data_type::DataType,
+};
+
+pub struct DeltaNetChunkedPrefillArgs<'a, B: Backend> {
+ pub in_projected: &'a Allocation<B>,
+ pub a_log: &'a Allocation<B>,
+ pub dt_bias: &'a Allocation<B>,
+ pub ssm_state: &'a mut Allocation<B>,
+ pub delta_output: &'a mut Allocation<B>,
+ pub num_heads: u32,
+ pub num_groups: u32,
+ pub value_head_dim: u32,
+ pub key_dim: u32,
+ pub value_dim: u32,
+ pub suffix_len: usize,
+}
+
+pub trait DeltaNetChunkedPrefill<B: Backend>: Sized {
+ fn new(
+ context: &B::Context,
+ outer_data_type: DataType,
+ head_dim: u32,
+ ) -> Result<Option<Self>, B::Error>;
+
+ fn should_use(
+ &self,
+ suffix_len: usize,
+ ) -> bool;
+
+ fn encode(
+ &self,
+ args: DeltaNetChunkedPrefillArgs<'_, B>,
+ encoder: &mut Encoder<B>,
+ ) -> Result<(), B::Error>;
+}
+
+impl<B: Backend> DeltaNetChunkedPrefill<B> for Infallible {
+ fn new(
+ _context: &B::Context,
+ _outer_data_type: DataType,
+ _head_dim: u32,
+ ) -> Result<Option<Self>, B::Error> {
+ Ok(None)
+ }
+
+ fn should_use(
+ &self,
+ _suffix_len: usize,
+ ) -> bool {
+ match *self {}
+ }
+
+ fn encode(
+ &self,
+ _args: DeltaNetChunkedPrefillArgs<'_, B>,
+ _encoder: &mut Encoder<B>,
+ ) -> Result<(), B::Error> {
+ match *self {}
+ }
+}
diff --git a/crates/backend-uzu/src/backends/common/kernel/mod.rs b/crates/backend-uzu/src/backends/common/kernel/mod.rs
index c022189a3..59cfe359f 100644
--- a/crates/backend-uzu/src/backends/common/kernel/mod.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/mod.rs
@@ -1,6 +1,7 @@
use crate::backends::common::Backend;
pub mod attention_gemm;
+pub mod delta_net_chunked_prefill;
pub mod matmul;
include!(concat!(env!("OUT_DIR"), "/traits.rs"));
@@ -10,6 +11,7 @@ pub trait Kernels: Sized {
autogen_kernels!();
type AttentionGemmCore: attention_gemm::AttentionGemmCore<Self::Backend>;
+ type DeltaNetChunkedPrefill: delta_net_chunked_prefill::DeltaNetChunkedPrefill<Self::Backend>;
type MatmulKernel: matmul::MatmulKernel<Backend = Self::Backend>;
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_scan.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_scan.rs
similarity index 100%
rename from crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_scan.rs
rename to crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_scan.rs
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_update.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_update.rs
similarity index 100%
rename from crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_update.rs
rename to crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_update.rs
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/mod.rs
similarity index 84%
rename from crates/backend-uzu/src/backends/cpu/kernel/delta_net/mod.rs
rename to crates/backend-uzu/src/backends/cpu/kernel/gdn/mod.rs
index 8f29d2c4f..e0c4e7865 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/mod.rs
@@ -3,4 +3,5 @@ pub mod conv_update;
pub mod norm_gate;
pub mod prefill;
pub mod prefill_prep;
+pub mod tree_verify;
pub mod update;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/norm_gate.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/norm_gate.rs
similarity index 100%
rename from crates/backend-uzu/src/backends/cpu/kernel/delta_net/norm_gate.rs
rename to crates/backend-uzu/src/backends/cpu/kernel/gdn/norm_gate.rs
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/prefill.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill.rs
similarity index 100%
rename from crates/backend-uzu/src/backends/cpu/kernel/delta_net/prefill.rs
rename to crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill.rs
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/prefill_prep.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill_prep.rs
similarity index 91%
rename from crates/backend-uzu/src/backends/cpu/kernel/delta_net/prefill_prep.rs
rename to crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill_prep.rs
index 64706b59a..00a2f601b 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/prefill_prep.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill_prep.rs
@@ -21,6 +21,7 @@ pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
key_dim: u32,
value_dim: u32,
suffix_len: u32,
+ #[specialize] write_log_decay: bool,
) {
let num_v_heads = num_v_heads as usize;
let num_k_heads = num_k_heads as usize;
@@ -80,11 +81,15 @@ pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
} else {
(1.0 + sp_in.exp()).ln()
};
- let decay = (-a_log_val.exp() * sp).exp();
+ let log_decay = -a_log_val.exp() * sp;
unsafe {
*beta_out.add(token * num_v_heads + hv) = beta;
- *decay_out.add(token * num_v_heads + hv) = decay;
+ *decay_out.add(token * num_v_heads + hv) = if write_log_decay {
+ log_decay
+ } else {
+ log_decay.exp()
+ };
};
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rs
similarity index 100%
rename from crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/mod.rs
rename to crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rs
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/out.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/out.rs
similarity index 100%
rename from crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/out.rs
rename to crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/out.rs
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/prefix_beta.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/prefix_beta.rs
similarity index 100%
rename from crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/prefix_beta.rs
rename to crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/prefix_beta.rs
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/tree_gram.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs
similarity index 100%
rename from crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/tree_gram.rs
rename to crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/tree_update_solve.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs
similarity index 100%
rename from crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/tree_update_solve.rs
rename to crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/update.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/update.rs
similarity index 100%
rename from crates/backend-uzu/src/backends/cpu/kernel/delta_net/update.rs
rename to crates/backend-uzu/src/backends/cpu/kernel/gdn/update.rs
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
index f0095bf5a..3d903e94d 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
@@ -4,10 +4,9 @@ use crate::backends::{common::Kernels, cpu::Cpu};
mod activation;
mod attention;
-mod delta_net;
mod embedding;
mod gated_act_mul;
-mod gdn_tree_verify;
+mod gdn;
mod hadamard_transform;
mod logit_soft_cap;
mod matmul;
@@ -33,5 +32,6 @@ impl Kernels for CpuKernels {
autogen_kernels!();
type AttentionGemmCore = Infallible;
+ type DeltaNetChunkedPrefill = Infallible;
type MatmulKernel = matmul::MatmulCpuKernel;
}
diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rs
index d25889b77..b9871e838 100644
--- a/crates/backend-uzu/src/backends/metal/context.rs
+++ b/crates/backend-uzu/src/backends/metal/context.rs
@@ -132,7 +132,6 @@ impl Context for MetalContext {
let gpu_core_count = device.gpu_core_count();
let device_tier = device_tier_for_device(gpu_core_count, device.as_ref());
-
let page_size = MTLSparsePageSize::KB256;
let heap_capacity = Metal::ALLOCATION_GRANULARITY;
let sparse_pool = MetalSparseHeapPool::new(page_size, heap_capacity);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/a_diag_inv.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/a_diag_inv.metal
new file mode 100644
index 000000000..266139d08
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/a_diag_inv.metal
@@ -0,0 +1,92 @@
+#include <metal_stdlib>
+#include "../../common/defines.h"
+#include "../../common/dsl.h"
+#include "../common/tri_inv.h"
+
+using namespace metal;
+
+#define DIAG_BLOCK_SIZE 16u
+
+// kk: [chunks, num_k_heads, CHUNK_SIZE, CHUNK_SIZE] K K^T from Gram.
+// beta: [suffix_len, num_v_heads].
+// g: [num_v_heads, suffix_len] cumulative log decay.
+// a_packed: [chunks, num_v_heads, num_blocks, ceil(num_blocks/2),
+// DIAG_BLOCK_SIZE, 2*DIAG_BLOCK_SIZE].
+// a_inv: [chunks, num_v_heads, num_blocks, DIAG_BLOCK_SIZE, DIAG_BLOCK_SIZE].
+//
+// One threadgroup owns one (chunk, v-head, column-pair, row-block). It materializes
+// A[row,col] = beta[row] * exp(g[row] - g[col]) * kk[row,col] for col < row,
+// writes packed column-pair blocks through the diagonal pair, copies the
+// diagonal block into threadgroup memory, then writes (I + A_diag)^-1 to a_inv.
+template <uint CHUNK_SIZE>
+VARIANTS(CHUNK_SIZE, 32, 64)
+KERNEL(DeltaNetChunkedADiagInv)(
+ device const float* kk,
+ device const float* beta,
+ device const float* g,
+ device float* a_packed,
+ device float* a_inv,
+ constant const uint& num_v_heads,
+ constant const uint& num_k_heads,
+ constant const uint& suffix_len,
+ threadgroup float diag_a_tile[DIAG_BLOCK_SIZE * DIAG_BLOCK_SIZE],
+ const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),
+ const uint hv_idx GROUPS(num_v_heads),
+ const uint block_col_pair_idx GROUPS(
+ CHUNK_SIZE.div_ceil(DIAG_BLOCK_SIZE) * CHUNK_SIZE.div_ceil(DIAG_BLOCK_SIZE).div_ceil(2)
+ ),
+ const uint lane THREADS(METAL_SIMD_SIZE)
+) {
+ constexpr uint num_blocks = (CHUNK_SIZE + DIAG_BLOCK_SIZE - 1) / DIAG_BLOCK_SIZE;
+ constexpr uint num_col_pairs = (num_blocks + 1) / 2;
+ const uint block_idx = block_col_pair_idx / num_col_pairs;
+ const uint col_pair_idx = block_col_pair_idx - block_idx * num_col_pairs;
+ const uint hk_idx = hv_idx / (num_v_heads / num_k_heads);
+ const uint token_base = chunk_idx * CHUNK_SIZE;
+ const uint row_base = block_idx * DIAG_BLOCK_SIZE;
+ const uint kk_base = (chunk_idx * num_k_heads + hk_idx) * CHUNK_SIZE * CHUNK_SIZE;
+ const uint diag_col_pair = block_idx / 2;
+ const uint diag_col_offset = (block_idx % 2) * DIAG_BLOCK_SIZE;
+ const device float* g_head = g + hv_idx * suffix_len;
+ if (col_pair_idx > diag_col_pair) {
+ return;
+ }
+
+ METAL_PRAGMA_UNROLL
+ for (uint idx = lane; idx < DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE; idx += METAL_SIMD_SIZE) {
+ const uint local_col = idx % (2 * DIAG_BLOCK_SIZE);
+ const uint local_row = (idx / (2 * DIAG_BLOCK_SIZE)) % DIAG_BLOCK_SIZE;
+ const uint row = row_base + local_row;
+ const uint col = col_pair_idx * 2 * DIAG_BLOCK_SIZE + local_col;
+ const uint row_token = token_base + row;
+ const uint col_token = token_base + col;
+
+ float value = 0.0f;
+ if (row < CHUNK_SIZE && col < CHUNK_SIZE && row_token < suffix_len && col_token < suffix_len && col < row) {
+ const float beta_row = beta[row_token * num_v_heads + hv_idx];
+ const float g_row = g_head[row_token];
+ const float g_col = g_head[col_token];
+ value = beta_row * fast::exp(g_row - g_col) * kk[kk_base + row * CHUNK_SIZE + col];
+ }
+
+ const uint out_idx =
+ (((chunk_idx * num_v_heads + hv_idx) * num_blocks + block_idx) * num_col_pairs + col_pair_idx) *
+ (DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE) +
+ local_row * (2 * DIAG_BLOCK_SIZE) + local_col;
+ a_packed[out_idx] = value;
+ if (col_pair_idx == diag_col_pair && local_col >= diag_col_offset &&
+ local_col < diag_col_offset + DIAG_BLOCK_SIZE) {
+ diag_a_tile[local_row * DIAG_BLOCK_SIZE + local_col - diag_col_offset] = value;
+ }
+ }
+
+ if (col_pair_idx != diag_col_pair) {
+ return;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ device float* a_inv_block =
+ a_inv + ((chunk_idx * num_v_heads + hv_idx) * num_blocks + block_idx) * (DIAG_BLOCK_SIZE * DIAG_BLOCK_SIZE);
+ const uint block_tokens =
+ token_base + row_base < suffix_len ? min(uint(DIAG_BLOCK_SIZE), suffix_len - token_base - row_base) : 0u;
+ invert_lower_triangular_block<DIAG_BLOCK_SIZE>(a_inv_block, diag_a_tile, block_tokens, lane);
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/causal_inv.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/causal_inv.metal
new file mode 100644
index 000000000..efccb1b1c
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/causal_inv.metal
@@ -0,0 +1,102 @@
+#include <metal_stdlib>
+#include "../../common/defines.h"
+#include "../../common/dsl.h"
+#include "../../matmul/common/fragment.h"
+#include "../../matmul/common/simdgroup_fragment_ops.h"
+
+using namespace metal;
+using namespace uzu::matmul;
+
+#define DIAG_BLOCK_SIZE 16u
+
+// Materializes dense T = (I + A)^-1 as bf16 for output and state. This is block
+// forward substitution over a_packed plus the per-block inverses from ADiagInv.
+template <uint CHUNK_SIZE, uint BV>
+VARIANTS(CHUNK_SIZE, 32, 64)
+VARIANTS(BV, 16, 32)
+KERNEL(DeltaNetChunkedCausalInv)(
+ device const float* a_packed,
+ device const float* a_inv,
+ device bfloat* t_out,
+ constant const uint& num_v_heads,
+ constant const uint& suffix_len,
+ const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),
+ const uint hv_idx GROUPS(num_v_heads),
+ const uint tile_idx GROUPS(CHUNK_SIZE.div_ceil(BV)),
+ const uint lane THREADS(METAL_SIMD_SIZE)
+) {
+ using Ops = SimdgroupFragmentOps;
+ constexpr ushort TOKEN_FRAGMENTS = DIAG_BLOCK_SIZE / Ops::FRAGMENT_ROWS;
+ constexpr ushort VALUE_FRAGMENTS = BV / Ops::FRAGMENT_COLS;
+ using TileFragment = Fragment<float, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, Ops>;
+ using MatrixFragment = OperandFragment<float, TOKEN_FRAGMENTS, TOKEN_FRAGMENTS, Ops>;
+ using ValueFragment = OperandFragment<bfloat, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, Ops>;
+ using PairMatrixFragment = OperandFragment<float, TOKEN_FRAGMENTS, 2 * TOKEN_FRAGMENTS, Ops>;
+ using PairValueFragment = OperandFragment<bfloat, 2 * TOKEN_FRAGMENTS, VALUE_FRAGMENTS, Ops>;
+
+ constexpr uint num_blocks = (CHUNK_SIZE + DIAG_BLOCK_SIZE - 1) / DIAG_BLOCK_SIZE;
+ constexpr uint num_col_pairs = (num_blocks + 1) / 2;
+ const uint slice_base = tile_idx * BV;
+
+ const device float* a_blocks = a_packed + (chunk_idx * num_v_heads + hv_idx) * num_blocks * num_col_pairs *
+ (DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE);
+ const device float* inv_blocks =
+ a_inv + (chunk_idx * num_v_heads + hv_idx) * num_blocks * (DIAG_BLOCK_SIZE * DIAG_BLOCK_SIZE);
+ device bfloat* t_head = t_out + (chunk_idx * num_v_heads + hv_idx) * CHUNK_SIZE * CHUNK_SIZE + slice_base;
+
+ for (uint block_idx = 0; block_idx < num_blocks; ++block_idx) {
+ const uint row_base = block_idx * DIAG_BLOCK_SIZE;
+
+ // -- RHS = identity slice: acc[row, col] = (row_base+row == slice_base+col) -
+ TileFragment acc_t;
+ acc_t.map_coords(lane, [&](short local_row, short local_col, float) {
+ const uint global_row = row_base + uint(local_row);
+ const uint global_col = slice_base + uint(local_col);
+ return global_row == global_col ? 1.0f : 0.0f;
+ });
+
+ // -- Forward substitution: subtract A . T_prev (T_prev in bf16, own slice) --
+ // a_packed stays positive for block inverse; dense solve subtracts it.
+ const uint num_full_pairs = block_idx / 2;
+ for (uint pair_idx = 0; pair_idx < num_full_pairs; ++pair_idx) {
+ PairMatrixFragment a_frag;
+ const device float* a_pair =
+ a_blocks + (block_idx * num_col_pairs + pair_idx) * (DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE);
+ a_frag.load_from(lane, fragment_source(a_pair, 2 * DIAG_BLOCK_SIZE));
+ a_frag.map([](float value) { return -value; });
+
+ PairValueFragment t_prev_frag;
+ const device bfloat* t_prev = t_head + pair_idx * 2 * DIAG_BLOCK_SIZE * CHUNK_SIZE;
+ t_prev_frag.load_from(lane, fragment_source(t_prev, CHUNK_SIZE));
+ fragment_mma(acc_t, a_frag, t_prev_frag);
+ }
+
+ for (uint prev_block_idx = num_full_pairs * 2; prev_block_idx < block_idx; ++prev_block_idx) {
+ MatrixFragment a_frag;
+ const device float* a_block =
+ a_blocks + (block_idx * num_col_pairs + prev_block_idx / 2) * (DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE) +
+ (prev_block_idx % 2) * DIAG_BLOCK_SIZE;
+ a_frag.load_from(lane, fragment_source(a_block, 2 * DIAG_BLOCK_SIZE));
+ a_frag.map([](float value) { return -value; });
+
+ ValueFragment t_prev_frag;
+ const device bfloat* t_prev = t_head + prev_block_idx * DIAG_BLOCK_SIZE * CHUNK_SIZE;
+ t_prev_frag.load_from(lane, fragment_source(t_prev, CHUNK_SIZE));
+ fragment_mma(acc_t, a_frag, t_prev_frag);
+ }
+
+ // -- Apply the diagonal-block inverse and store T ------------------------
+ MatrixFragment inv_frag;
+ inv_frag.load_from(
+ lane,
+ fragment_source(inv_blocks + block_idx * DIAG_BLOCK_SIZE * DIAG_BLOCK_SIZE, DIAG_BLOCK_SIZE)
+ );
+
+ TileFragment t_block;
+ t_block.clear();
+ fragment_mma(t_block, inv_frag, acc_t);
+ t_block.store(lane, t_head + row_base * CHUNK_SIZE, CHUNK_SIZE);
+
+ simdgroup_barrier(mem_flags::mem_device);
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/cumsum.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/cumsum.metal
new file mode 100644
index 000000000..37e5277e8
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/cumsum.metal
@@ -0,0 +1,37 @@
+#include <metal_stdlib>
+#include "../../common/defines.h"
+#include "../../common/dsl.h"
+
+using namespace metal;
+
+template <uint CHUNK_SIZE>
+VARIANTS(CHUNK_SIZE, 32, 64)
+KERNEL(DeltaNetChunkedCumsum)(
+ device const float* log_decay,
+ device float* g_out,
+ constant const uint& num_v_heads,
+ constant const uint& suffix_len,
+ const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),
+ const uint hv_idx GROUPS(num_v_heads),
+ const uint lane THREADS(METAL_SIMD_SIZE)
+) {
+ const uint token_begin = chunk_idx * CHUNK_SIZE;
+ const uint token_end = min(token_begin + CHUNK_SIZE, suffix_len);
+
+ float carry = 0.0f;
+ METAL_PRAGMA_UNROLL
+ for (uint token_offset = 0; token_offset < CHUNK_SIZE; token_offset += METAL_SIMD_SIZE) {
+ const uint token = token_begin + token_offset + lane;
+ const bool valid = token < token_end;
+ const uint input_offset = token * num_v_heads + hv_idx;
+ const uint output_offset = hv_idx * suffix_len + token;
+ const float value = valid ? log_decay[input_offset] : 0.0f;
+ const float prefix = carry + simd_prefix_inclusive_sum(value);
+
+ if (valid) {
+ g_out[output_offset] = prefix;
+ }
+
+ carry += simd_sum(value);
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/gram.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/gram.metal
new file mode 100644
index 000000000..2566e9499
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/gram.metal
@@ -0,0 +1,114 @@
+#include <metal_stdlib>
+#include "../../common/defines.h"
+#include "../../common/dsl.h"
+#include "../common/gram.h"
+
+using namespace metal;
+using namespace uzu::matmul;
+
+#define ROW_TILE 16u
+#define COL_TILE 32u
+
+// q_norm/k_norm: [suffix_len, key_dim], g: [num_v_heads, suffix_len].
+// kk_out: [chunks, num_k_heads, CHUNK_SIZE, CHUNK_SIZE].
+// qk_scaled_out: [chunks, num_v_heads, CHUNK_SIZE, CHUNK_SIZE].
+// Computes K K^T per k-head and decay-scaled Q K^T per grouped v-head.
+// MXU did not improve e2e vs simdgroup.
+template <uint HEAD_K_DIM, uint CHUNK_SIZE>
+VARIANTS(HEAD_K_DIM, 128)
+VARIANTS(CHUNK_SIZE, 32, 64)
+KERNEL(DeltaNetChunkedGram)(
+ device const float* q_norm,
+ device const float* k_norm,
+ device const float* g,
+ device float* kk_out,
+ device float* qk_scaled_out,
+ constant const uint& num_v_heads,
+ constant const uint& num_k_heads,
+ constant const uint& key_dim,
+ constant const uint& suffix_len,
+ const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),
+ const uint hk_idx GROUPS(num_k_heads),
+ const uint tile_idx GROUPS(CHUNK_SIZE.div_ceil(ROW_TILE) * CHUNK_SIZE.div_ceil(COL_TILE)),
+ const uint lane THREADS(METAL_SIMD_SIZE)
+) {
+ using Ops = SimdgroupFragmentOps;
+ using InputType = float;
+ constexpr ushort ROW_FRAGMENTS = ROW_TILE / Ops::FRAGMENT_ROWS;
+ constexpr ushort COL_FRAGMENTS = COL_TILE / Ops::FRAGMENT_COLS;
+ using AccFragment = Fragment<float, ROW_FRAGMENTS, COL_FRAGMENTS, Ops>;
+ using LeftFragment = OperandFragment<InputType, ROW_FRAGMENTS, 1, Ops>;
+ using RightFragment = OperandFragment<InputType, 1, COL_FRAGMENTS, Ops, ReadTranspose>;
+
+ static_assert(HEAD_K_DIM % Ops::FRAGMENT_ROWS == 0, "HEAD_K_DIM must align to fragment K");
+
+ constexpr uint col_tiles = (CHUNK_SIZE + COL_TILE - 1) / COL_TILE;
+ const uint row_tile_idx = tile_idx / col_tiles;
+ const uint col_tile_idx = tile_idx - row_tile_idx * col_tiles;
+ const uint row_base = row_tile_idx * ROW_TILE;
+ const uint col_base = col_tile_idx * COL_TILE;
+ const uint chunk_token_base = chunk_idx * CHUNK_SIZE;
+
+ const uint tile_rows = min(uint(ROW_TILE), uint(CHUNK_SIZE - row_base));
+ const uint tile_cols = min(uint(COL_TILE), uint(CHUNK_SIZE - col_base));
+ const uint row_token_base = chunk_token_base + row_base;
+ const uint col_token_base = chunk_token_base + col_base;
+ const uint valid_rows = row_token_base < suffix_len ? min(tile_rows, suffix_len - row_token_base) : 0u;
+ const uint valid_cols = col_token_base < suffix_len ? min(tile_cols, suffix_len - col_token_base) : 0u;
+ // Upper-causal tiles are zero.
+ if (valid_rows == 0 || valid_cols == 0 || col_base >= row_base + valid_rows) {
+ return;
+ }
+
+ AccFragment kk_acc;
+ AccFragment qk_acc;
+ kk_acc.clear();
+ qk_acc.clear();
+
+ for (uint k_block_start = 0; k_block_start < HEAD_K_DIM; k_block_start += Ops::FRAGMENT_ROWS) {
+ const device float* k_rows = k_norm + row_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;
+ const device float* q_rows = q_norm + row_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;
+ const device float* k_cols = k_norm + col_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;
+
+ accumulate_dual_gram_tile<AccFragment, LeftFragment, RightFragment>(
+ kk_acc,
+ qk_acc,
+ k_rows,
+ q_rows,
+ k_cols,
+ int(key_dim),
+ ushort(valid_rows),
+ ushort(valid_cols),
+ Ops::FRAGMENT_ROWS,
+ valid_rows == tile_rows,
+ valid_cols == tile_cols,
+ ushort(lane)
+ );
+ }
+
+ const short2 tile_dims = short2(tile_cols, tile_rows);
+ const uint kk_base = (chunk_idx * num_k_heads + hk_idx) * CHUNK_SIZE * CHUNK_SIZE + row_base * CHUNK_SIZE + col_base;
+ kk_acc.store_safe(lane, kk_out + kk_base, CHUNK_SIZE, tile_dims);
+
+ // Store qk for each grouped v-head with decay scale and causal mask.
+ const uint valid_tokens = chunk_token_base < suffix_len ? min(uint(CHUNK_SIZE), suffix_len - chunk_token_base) : 0u;
+ const uint groups_per_head = num_v_heads / num_k_heads;
+ for (uint group = 0; group < groups_per_head; ++group) {
+ const uint hv_idx = hk_idx * groups_per_head + group;
+ const device float* g_head = g + hv_idx * suffix_len;
+ AccFragment scaled = qk_acc;
+ scaled.map_coords(lane, [&](short r, short c, float value) {
+ const uint row = row_base + uint(r);
+ const uint col = col_base + uint(c);
+ if (row >= valid_tokens || col >= valid_tokens || col > row) {
+ return 0.0f;
+ }
+ const float g_row = g_head[chunk_token_base + row];
+ const float g_col = g_head[chunk_token_base + col];
+ return value * fast::exp(g_row - g_col);
+ });
+ const uint dst_base =
+ (chunk_idx * num_v_heads + hv_idx) * CHUNK_SIZE * CHUNK_SIZE + row_base * CHUNK_SIZE + col_base;
+ scaled.store_safe(lane, qk_scaled_out + dst_base, CHUNK_SIZE, tile_dims);
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs
new file mode 100644
index 000000000..0ec83e804
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs
@@ -0,0 +1,185 @@
+use super::super::{
+ DeltaNetChunkedADiagInvMetalKernel, DeltaNetChunkedCausalInvMetalKernel, DeltaNetChunkedCumsumMetalKernel,
+ DeltaNetChunkedGramMetalKernel, DeltaNetChunkedOutputAndStateMetalKernel, DeltaNetPrefillPrepMetalKernel,
+};
+use crate::{
+ array::size_for_shape,
+ backends::{
+ common::{
+ Backend, Encoder,
+ kernel::{
+ DeltaNetPrefillPrepKernel,
+ delta_net_chunked_prefill::{DeltaNetChunkedPrefill, DeltaNetChunkedPrefillArgs},
+ },
+ },
+ metal::{Metal, MetalContext},
+ },
+ data_type::DataType,
+};
+
+const MXU_MIN_T: usize = 256;
+const CHUNK_SIZE: usize = 64;
+const BLOCK_SIZE: usize = 16;
+const VT: usize = 32;
+
+pub struct MetalDeltaNetChunkedPrefill {
+ min_t: usize,
+ prep: DeltaNetPrefillPrepMetalKernel,
+ cumsum: DeltaNetChunkedCumsumMetalKernel,
+ gram: DeltaNetChunkedGramMetalKernel,
+ a_diag_inv: DeltaNetChunkedADiagInvMetalKernel,
+ causal_inv: DeltaNetChunkedCausalInvMetalKernel,
+ output_and_state: DeltaNetChunkedOutputAndStateMetalKernel,
+}
+
+impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {
+ fn new(
+ context: &MetalContext,
+ outer_data_type: DataType,
+ head_dim: u32,
+ ) -> Result<Option<Self>, <Metal as Backend>::Error> {
+ if outer_data_type == DataType::F16 {
+ return Ok(None);
+ }
+
+ if !context.supports_mxu() {
+ return Ok(None);
+ }
+
+ let scratch_data_type = if outer_data_type == DataType::BF16 {
+ DataType::BF16
+ } else {
+ DataType::F32
+ };
+
+ Ok(Some(Self {
+ min_t: MXU_MIN_T,
+ prep: DeltaNetPrefillPrepMetalKernel::new(context, outer_data_type, head_dim, true)?,
+ cumsum: DeltaNetChunkedCumsumMetalKernel::new(context, CHUNK_SIZE as u32)?,
+ gram: DeltaNetChunkedGramMetalKernel::new(context, head_dim, CHUNK_SIZE as u32)?,
+ a_diag_inv: DeltaNetChunkedADiagInvMetalKernel::new(context, CHUNK_SIZE as u32)?,
+ causal_inv: DeltaNetChunkedCausalInvMetalKernel::new(context, CHUNK_SIZE as u32, VT as u32)?,
+ output_and_state: DeltaNetChunkedOutputAndStateMetalKernel::new(
+ context,
+ outer_data_type,
+ outer_data_type,
+ scratch_data_type,
+ head_dim,
+ VT as u32,
+ true,
+ )?,
+ }))
+ }
+
+ fn should_use(
+ &self,
+ suffix_len: usize,
+ ) -> bool {
+ suffix_len >= self.min_t
+ }
+
+ fn encode(
+ &self,
+ args: DeltaNetChunkedPrefillArgs<'_, Metal>,
+ encoder: &mut Encoder<Metal>,
+ ) -> Result<(), <Metal as Backend>::Error> {
+ let suffix_len = args.suffix_len;
+ let num_chunks = suffix_len.div_ceil(CHUNK_SIZE);
+ let num_blocks = CHUNK_SIZE.div_ceil(BLOCK_SIZE);
+ let num_col_pairs = num_blocks.div_ceil(2);
+
+ let mut q_norm =
+ encoder.allocate_scratch(size_for_shape(&[suffix_len * args.key_dim as usize], DataType::F32))?;
+ let mut k_norm =
+ encoder.allocate_scratch(size_for_shape(&[suffix_len * args.key_dim as usize], DataType::F32))?;
+ let mut beta =
+ encoder.allocate_scratch(size_for_shape(&[suffix_len * args.num_heads as usize], DataType::F32))?;
+ let mut log_decay =
+ encoder.allocate_scratch(size_for_shape(&[suffix_len * args.num_heads as usize], DataType::F32))?;
+ let mut g = encoder.allocate_scratch(size_for_shape(&[suffix_len * args.num_heads as usize], DataType::F32))?;
+ let mut kk = encoder.allocate_scratch(size_for_shape(
+ &[num_chunks * args.num_groups as usize * CHUNK_SIZE * CHUNK_SIZE],
+ DataType::F32,
+ ))?;
+ let mut qk = encoder.allocate_scratch(size_for_shape(
+ &[num_chunks * args.num_heads as usize * CHUNK_SIZE * CHUNK_SIZE],
+ DataType::F32,
+ ))?;
+ let mut a_packed = encoder.allocate_scratch(size_for_shape(
+ &[num_chunks * args.num_heads as usize * num_blocks * num_col_pairs * BLOCK_SIZE * 2 * BLOCK_SIZE],
+ DataType::F32,
+ ))?;
+ let mut a_inv = encoder.allocate_scratch(size_for_shape(
+ &[num_chunks * args.num_heads as usize * num_blocks * BLOCK_SIZE * BLOCK_SIZE],
+ DataType::F32,
+ ))?;
+ let mut t_mat = encoder.allocate_scratch(size_for_shape(
+ &[num_chunks * args.num_heads as usize * CHUNK_SIZE * CHUNK_SIZE],
+ DataType::BF16,
+ ))?;
+
+ self.prep.encode(
+ args.in_projected,
+ args.a_log,
+ args.dt_bias,
+ &mut q_norm,
+ &mut k_norm,
+ &mut beta,
+ &mut log_decay,
+ args.num_heads,
+ args.num_groups,
+ args.key_dim,
+ args.value_dim,
+ suffix_len as u32,
+ encoder,
+ );
+ self.cumsum.encode(&log_decay, &mut g, args.num_heads, suffix_len as u32, encoder);
+ self.gram.encode(
+ &q_norm,
+ &k_norm,
+ &g,
+ &mut kk,
+ &mut qk,
+ args.num_heads,
+ args.num_groups,
+ args.key_dim,
+ suffix_len as u32,
+ encoder,
+ );
+ self.a_diag_inv.encode(
+ &kk,
+ &beta,
+ &g,
+ &mut a_packed,
+ &mut a_inv,
+ args.num_heads,
+ args.num_groups,
+ suffix_len as u32,
+ encoder,
+ );
+ self.causal_inv.encode(&a_packed, &a_inv, &mut t_mat, args.num_heads, suffix_len as u32, encoder);
+ self.output_and_state.encode(
+ &q_norm,
+ &k_norm,
+ args.in_projected,
+ &qk,
+ &t_mat,
+ &g,
+ &beta,
+ args.ssm_state,
+ args.delta_output,
+ args.num_heads,
+ args.num_groups,
+ args.value_head_dim,
+ args.key_dim,
+ args.value_dim,
+ suffix_len as u32,
+ encoder,
+ );
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+#[path = "../../../../../../unit/backends/metal/kernel/gdn/chunked_test.rs"]
+mod tests;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/output_and_state.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/output_and_state.metal
new file mode 100644
index 000000000..652a9c8e9
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/output_and_state.metal
@@ -0,0 +1,285 @@
+#include <metal_stdlib>
+#include "../../common/defines.h"
+#include "../../common/dsl.h"
+#include "../../common/thread_context.h"
+#include "../../matmul/common/fragment.h"
+#include "../../matmul/common/mxu_fragment_ops.h"
+#include "../../matmul/common/simdgroup_fragment_ops.h"
+
+using namespace metal;
+using namespace uzu::matmul;
+
+#define OUTPUT_STATE_THREADS 128
+#define OUTPUT_STATE_NUM_SIMDGROUPS (OUTPUT_STATE_THREADS / METAL_SIMD_SIZE)
+#define OUTPUT_STATE_CHUNK 64
+#define OUTPUT_STATE_KEY_TILE (HEAD_K_DIM / OUTPUT_STATE_NUM_SIMDGROUPS)
+
+// q_norm: [suffix_len, key_dim].
+// k_norm: [suffix_len, key_dim].
+// in_proj: [suffix_len, total_proj_dim]; V starts at offset 2*key_dim.
+// qk_scaled: [chunks, num_v_heads, CHUNK, CHUNK] from Gram.
+// t_mat: [chunks, num_v_heads, CHUNK, CHUNK] dense causal inverse, bf16.
+// g: [num_v_heads, suffix_len] cumulative log decay.
+// beta: [suffix_len, num_v_heads].
+// state: [num_v_heads, head_v_dim, HEAD_K_DIM], f32 input/output.
+// out: [suffix_len, value_dim].
+// ScratchT: R/Vnew threadgroup storage precision.
+//
+// One threadgroup owns one (v-head, value-slice). It loads the state slice as
+// S^T in threadgroup memory, then scans chunks:
+// 1. R = beta * (V - exp(g) * K * S^T).
+// 2. Vnew = T * R.
+// 3. out = exp(g) * Q * S^T + qk_scaled * Vnew.
+// 4. S^T = exp(g_last) * S^T + (exp(g_last - g) * K)^T * Vnew.
+template <typename T, typename O, typename ScratchT, uint HEAD_K_DIM, uint VT, bool USE_MXU>
+VARIANTS(T, float, bfloat)
+VARIANTS(O, float, bfloat)
+VARIANTS(ScratchT, float, bfloat)
+VARIANTS(HEAD_K_DIM, 128)
+VARIANTS(VT, 32)
+VARIANTS(USE_MXU, false, true)
+KERNEL(DeltaNetChunkedOutputAndState)(
+ device const float* q_norm,
+ device const float* k_norm,
+ device const T* in_proj,
+ device const float* qk_scaled,
+ device const bfloat* t_mat,
+ device const float* g,
+ device const float* beta,
+ device float* state,
+ device O* out,
+ constant const uint& num_v_heads,
+ constant const uint& num_k_heads,
+ constant const uint& head_v_dim,
+ constant const uint& key_dim,
+ constant const uint& value_dim,
+ constant const uint& suffix_len,
+ threadgroup float transposed_state_tile[HEAD_K_DIM * VT],
+ threadgroup ScratchT residual_new_value_tile[OUTPUT_STATE_CHUNK * VT],
+ const ThreadContext thread_context,
+ const uint hv_idx GROUPS(num_v_heads),
+ const uint value_tile_idx GROUPS(head_v_dim.div_ceil(VT)),
+ const uint thread_idx THREADS(OUTPUT_STATE_THREADS)
+) {
+ using Ops = metal::conditional_t<USE_MXU, MxuFragmentOps<>, SimdgroupFragmentOps>;
+ constexpr ushort FR = Ops::FRAGMENT_ROWS;
+ constexpr ushort FC = Ops::FRAGMENT_COLS;
+ static_assert(HEAD_K_DIM % FR == 0, "HEAD_K_DIM must be a multiple of fragment rows");
+ static_assert(OUTPUT_STATE_CHUNK % FR == 0, "chunk size must be a multiple of fragment rows");
+ static_assert(VT % FC == 0, "VT must be a multiple of fragment columns");
+ static_assert(OUTPUT_STATE_KEY_TILE % FC == 0, "key tile must be a multiple of fragment columns");
+
+ constexpr uint TOKEN_TILE = (VT >= 32) ? 16u : 32u;
+ constexpr uint NUM_TOKEN_TILES = OUTPUT_STATE_CHUNK / TOKEN_TILE;
+ constexpr ushort TOKEN_FRAGMENTS = TOKEN_TILE / FR;
+ constexpr ushort VALUE_FRAGMENTS = VT / FC;
+ constexpr ushort KEY_FRAGMENTS = OUTPUT_STATE_KEY_TILE / FC;
+
+ using TokenValueFragment = Fragment<float, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, Ops>;
+ using TokenKeyFragment = OperandFragment<float, TOKEN_FRAGMENTS, 1, Ops>;
+ using KeyValueFragment = OperandFragment<float, 1, VALUE_FRAGMENTS, Ops>;
+ using ScratchKeyValueFragment = OperandFragment<ScratchT, 1, VALUE_FRAGMENTS, Ops>;
+ using ValueKeyFragment = Fragment<float, VALUE_FRAGMENTS, KEY_FRAGMENTS, Ops>;
+ using ScratchValueTokenFragment = OperandFragment<ScratchT, VALUE_FRAGMENTS, 1, Ops>;
+ using TokenKeyTileFragment = OperandFragment<float, 1, KEY_FRAGMENTS, Ops>;
+
+ const uint lane = thread_context.simd_lane_id;
+ const uint simdgroup_idx = thread_context.simdgroup_index;
+ const uint value_base = value_tile_idx * VT;
+ if (value_base >= head_v_dim) {
+ return;
+ }
+ const uint hk_idx = hv_idx / (num_v_heads / num_k_heads);
+ const uint num_chunks = (suffix_len + OUTPUT_STATE_CHUNK - 1) / OUTPUT_STATE_CHUNK;
+ const uint conv_dim = 2 * key_dim + value_dim;
+ const uint total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;
+ const device float* g_head = g + hv_idx * suffix_len;
+
+ // Load the initial state slice transposed into threadgroup memory:
+ // transposed_state_tile[key * VT + value] = state[(hv, value_base + value, key)]
+ for (uint idx = thread_idx; idx < VT * HEAD_K_DIM; idx += OUTPUT_STATE_THREADS) {
+ const uint local_value_idx = idx / HEAD_K_DIM;
+ const uint local_key_idx = idx - local_value_idx * HEAD_K_DIM;
+ transposed_state_tile[local_key_idx * VT + local_value_idx] =
+ state[(hv_idx * head_v_dim + value_base + local_value_idx) * HEAD_K_DIM + local_key_idx];
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ for (uint chunk_idx = 0; chunk_idx < num_chunks; ++chunk_idx) {
+ const uint token_base = chunk_idx * OUTPUT_STATE_CHUNK;
+ const uint valid_tokens = token_base < suffix_len ? min(uint(OUTPUT_STATE_CHUNK), suffix_len - token_base) : 0u;
+ const uint chunk_head_base = (chunk_idx * num_v_heads + hv_idx);
+
+ TokenValueFragment output_acc;
+ // -- R phase: R = beta (.) (V - e^g (.) (K . S^T)) into residual_new_value_tile ----------
+ if (simdgroup_idx < NUM_TOKEN_TILES) {
+ const uint token_tile_base = simdgroup_idx * TOKEN_TILE;
+ const uint valid_token_rows =
+ token_tile_base < valid_tokens ? min(uint(TOKEN_TILE), valid_tokens - token_tile_base) : 0u;
+
+ TokenValueFragment residual_acc;
+ residual_acc.clear();
+ const device float* k_tile = k_norm + (token_base + token_tile_base) * key_dim + hk_idx * HEAD_K_DIM;
+ output_acc.clear();
+ const uint query_tile_base = (token_base + token_tile_base) * key_dim + hk_idx * HEAD_K_DIM;
+ for (uint key_block_start = 0; key_block_start < HEAD_K_DIM; key_block_start += FR) {
+ TokenKeyFragment key_frag;
+ TokenKeyFragment query_frag;
+ KeyValueFragment state_frag;
+ key_frag.load_from(lane, fragment_source(k_tile + key_block_start, int(key_dim)).bounded(valid_token_rows, FR));
+ query_frag.load_from(
+ lane,
+ fragment_source(q_norm + query_tile_base + key_block_start, int(key_dim)).bounded(valid_token_rows, FR)
+ );
+ state_frag.load_from(lane, fragment_source(transposed_state_tile + key_block_start * VT, int(VT), 1));
+ fragment_mma(residual_acc, key_frag, state_frag);
+ fragment_mma(output_acc, query_frag, state_frag);
+ }
+
+ const device T* value_input_tile =
+ in_proj + (token_base + token_tile_base) * total_proj_dim + 2 * key_dim + hv_idx * head_v_dim + value_base;
+ residual_acc.map_coords(lane, [&](short row, short col, float state_projection) {
+ if (uint(row) >= valid_token_rows) {
+ return 0.0f;
+ }
+ const uint token = token_base + token_tile_base + uint(row);
+ const float beta_t = beta[token * num_v_heads + hv_idx];
+ const float g_t = g_head[token];
+ const float value_input = float(value_input_tile[uint(row) * total_proj_dim + uint(col)]);
+ return beta_t * (value_input - fast::exp(g_t) * state_projection);
+ });
+ residual_acc.store(lane, residual_new_value_tile + token_tile_base * VT, int(VT));
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // -- Vnew phase: Vnew = T . R (dense bf16 T streamed from device) --------
+ TokenValueFragment new_value_acc;
+ if (simdgroup_idx < NUM_TOKEN_TILES) {
+ const uint token_tile_base = simdgroup_idx * TOKEN_TILE;
+ const uint valid_token_rows =
+ token_tile_base < valid_tokens ? min(uint(TOKEN_TILE), valid_tokens - token_tile_base) : 0u;
+
+ new_value_acc.clear();
+ const device bfloat* causal_inv_rows =
+ t_mat + chunk_head_base * OUTPUT_STATE_CHUNK * OUTPUT_STATE_CHUNK + token_tile_base * OUTPUT_STATE_CHUNK;
+ // Skip fully upper-causal T tiles.
+ const uint causal_col_end = min(uint(OUTPUT_STATE_CHUNK), token_tile_base + TOKEN_TILE);
+ for (uint source_token_block_start = 0; source_token_block_start < causal_col_end;
+ source_token_block_start += FR) {
+ TokenKeyFragment causal_inv_frag;
+ ScratchKeyValueFragment residual_frag;
+ causal_inv_frag.load_from(
+ lane,
+ fragment_source(causal_inv_rows + source_token_block_start, int(OUTPUT_STATE_CHUNK))
+ .bounded(valid_token_rows, FR)
+ );
+ residual_frag.load_from(
+ lane,
+ fragment_source(residual_new_value_tile + source_token_block_start * VT, int(VT), 1)
+ );
+ fragment_mma(new_value_acc, causal_inv_frag, residual_frag);
+ }
+ }
+ // Barrier before overwriting residual_new_value_tile (R) with Vnew: all simdgroups must be
+ // done reading R above.
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (simdgroup_idx < NUM_TOKEN_TILES) {
+ const uint token_tile_base = simdgroup_idx * TOKEN_TILE;
+ new_value_acc.store(lane, residual_new_value_tile + token_tile_base * VT, int(VT));
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // -- Y phase: Y = e^g (.) (Q . S^T) + A . Vnew ; cast+store to out -------
+ if (simdgroup_idx < NUM_TOKEN_TILES) {
+ const uint token_tile_base = simdgroup_idx * TOKEN_TILE;
+ const uint valid_token_rows =
+ token_tile_base < valid_tokens ? min(uint(TOKEN_TILE), valid_tokens - token_tile_base) : 0u;
+
+ output_acc.map_coords(lane, [&](short row, short, float value) {
+ if (uint(row) >= valid_token_rows) {
+ return 0.0f;
+ }
+ const uint token = token_base + token_tile_base + uint(row);
+ return value * fast::exp(g_head[token]);
+ });
+
+ const uint qk_tile_base =
+ chunk_head_base * OUTPUT_STATE_CHUNK * OUTPUT_STATE_CHUNK + token_tile_base * OUTPUT_STATE_CHUNK;
+ // Skip fully upper-causal qk tiles.
+ const uint causal_col_end = min(uint(OUTPUT_STATE_CHUNK), token_tile_base + TOKEN_TILE);
+ for (uint source_token_block_start = 0; source_token_block_start < causal_col_end;
+ source_token_block_start += FR) {
+ const uint valid_source_tokens =
+ source_token_block_start < valid_tokens ? min(uint(FR), valid_tokens - source_token_block_start) : 0u;
+ TokenKeyFragment qk_frag;
+ ScratchKeyValueFragment new_value_frag;
+ qk_frag.load_from(
+ lane,
+ fragment_source(qk_scaled + qk_tile_base + source_token_block_start, int(OUTPUT_STATE_CHUNK))
+ .bounded(valid_token_rows, valid_source_tokens)
+ );
+ new_value_frag.load_from(
+ lane,
+ fragment_source(residual_new_value_tile + source_token_block_start * VT, int(VT), 1)
+ );
+ fragment_mma(output_acc, qk_frag, new_value_frag);
+ }
+
+ const uint output_tile_base = (token_base + token_tile_base) * value_dim + hv_idx * head_v_dim + value_base;
+ output_acc.store_safe(lane, out + output_tile_base, int(value_dim), short2(short(VT), short(valid_token_rows)));
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // -- Update phase: S^T <- exp(g_last) . S^T + (decay_scale (.) K)^T . Vnew -----
+ {
+ const uint key_tile_base = simdgroup_idx * OUTPUT_STATE_KEY_TILE;
+ const uint g_last_token = token_base + (valid_tokens > 0 ? valid_tokens - 1 : 0u);
+ const float g_last = g_head[g_last_token];
+ const float state_decay = fast::exp(g_last);
+
+ ValueKeyFragment state_update_acc;
+ state_update_acc.clear();
+ for (uint source_token_block_start = 0; source_token_block_start < OUTPUT_STATE_CHUNK;
+ source_token_block_start += FR) {
+ const uint valid_source_tokens =
+ source_token_block_start < valid_tokens ? min(uint(FR), valid_tokens - source_token_block_start) : 0u;
+ ScratchValueTokenFragment new_value_frag;
+ TokenKeyTileFragment key_frag;
+ new_value_frag.load_from(
+ lane,
+ fragment_source(residual_new_value_tile + source_token_block_start * VT, 1, int(VT))
+ .bounded(short(VT), short(valid_source_tokens))
+ );
+ const device float* k_tile =
+ k_norm + (token_base + source_token_block_start) * key_dim + hk_idx * HEAD_K_DIM + key_tile_base;
+ key_frag.load_from(
+ lane,
+ fragment_source(k_tile, int(key_dim)).bounded(short(valid_source_tokens), short(OUTPUT_STATE_KEY_TILE))
+ );
+ key_frag.map_coords(lane, [&](short row, short, float value) {
+ if (uint(row) >= valid_source_tokens) {
+ return 0.0f;
+ }
+ const uint token = token_base + source_token_block_start + uint(row);
+ // beta is already folded into Vnew through R.
+ return value * fast::exp(g_last - g_head[token]);
+ });
+ fragment_mma(state_update_acc, new_value_frag, key_frag);
+ }
+
+ state_update_acc.map_coords(lane, [&](short row, short col, float value) {
+ return state_decay * transposed_state_tile[(key_tile_base + uint(col)) * VT + uint(row)] + value;
+ });
+ state_update_acc.store(lane, transposed_state_tile + key_tile_base * VT, 1, int(VT));
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ // Write the final state slice back to device, transposing [K, VT] -> [V, K].
+ for (uint idx = thread_idx; idx < VT * HEAD_K_DIM; idx += OUTPUT_STATE_THREADS) {
+ const uint local_value_idx = idx / HEAD_K_DIM;
+ const uint local_key_idx = idx - local_value_idx * HEAD_K_DIM;
+ state[(hv_idx * head_v_dim + value_base + local_value_idx) * HEAD_K_DIM + local_key_idx] =
+ transposed_state_tile[local_key_idx * VT + local_value_idx];
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/common/gram.h b/crates/backend-uzu/src/backends/metal/kernel/gdn/common/gram.h
new file mode 100644
index 000000000..e374a97bb
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/common/gram.h
@@ -0,0 +1,47 @@
+#pragma once
+
+#include "../../common/defines.h"
+#include "../../matmul/common/fragment.h"
+#include "../../matmul/common/mxu_fragment_ops.h"
+#include "../../matmul/common/simdgroup_fragment_ops.h"
+#include <metal_stdlib>
+
+using namespace metal;
+using namespace uzu::matmul;
+
+template <typename AccFragment, typename LeftFragment, typename RightFragment, typename T>
+METAL_FUNC void accumulate_dual_gram_tile(
+ thread AccFragment& kk_acc,
+ thread AccFragment& qk_acc,
+ const device T* k_rows,
+ const device T* q_rows,
+ const device T* k_cols,
+ const int stride,
+ const ushort valid_rows,
+ const ushort valid_cols,
+ const ushort valid_k,
+ const bool full_left,
+ const bool full_right,
+ const ushort lane
+) {
+ LeftFragment k_left;
+ LeftFragment q_left;
+ RightFragment k_right;
+
+ if (full_left) {
+ k_left.load_from(lane, fragment_source(k_rows, stride));
+ q_left.load_from(lane, fragment_source(q_rows, stride));
+ } else {
+ k_left.load_from(lane, fragment_source(k_rows, stride).bounded(valid_rows, valid_k));
+ q_left.load_from(lane, fragment_source(q_rows, stride).bounded(valid_rows, valid_k));
+ }
+
+ if (full_right) {
+ k_right.load_from(lane, fragment_source(k_cols, stride));
+ } else {
+ k_right.load_from(lane, fragment_source(k_cols, stride).bounded(valid_cols, valid_k));
+ }
+
+ fragment_mma(kk_acc, k_left, k_right);
+ fragment_mma(qk_acc, q_left, k_right);
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/common/tri_inv.h b/crates/backend-uzu/src/backends/metal/kernel/gdn/common/tri_inv.h
new file mode 100644
index 000000000..796fcbe24
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/common/tri_inv.h
@@ -0,0 +1,41 @@
+#pragma once
+
+#include "../../common/defines.h"
+#include <metal_stdlib>
+
+using namespace metal;
+
+template <uint BLOCK>
+METAL_FUNC void invert_lower_triangular_block(
+ device float* a_inv_block,
+ threadgroup const float* a_tile,
+ const uint block_size,
+ const uint lane
+) {
+ if (lane >= BLOCK) {
+ return;
+ }
+
+ const uint col = lane;
+ float inverse_col[BLOCK] = {};
+ inverse_col[col] = 1.0f;
+
+ METAL_PRAGMA_UNROLL
+ for (uint row = 0; row < BLOCK; row++) {
+ if (row > col && row < block_size) {
+ float acc = 0.0f;
+ METAL_PRAGMA_UNROLL
+ for (uint prev_row = 0; prev_row < BLOCK; prev_row++) {
+ if (prev_row < row) {
+ acc += a_tile[row * BLOCK + prev_row] * inverse_col[prev_row];
+ }
+ }
+ inverse_col[row] = -acc;
+ }
+ }
+
+ METAL_PRAGMA_UNROLL
+ for (uint row = 0; row < BLOCK; row++) {
+ a_inv_block[row * BLOCK + col] = inverse_col[row];
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_scan.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_scan.metal
similarity index 100%
rename from crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_scan.metal
rename to crates/backend-uzu/src/backends/metal/kernel/gdn/conv_scan.metal
diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_update.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_update.metal
similarity index 100%
rename from crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_update.metal
rename to crates/backend-uzu/src/backends/metal/kernel/gdn/conv_update.metal
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/mod.rs
new file mode 100644
index 000000000..60c62629e
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/mod.rs
@@ -0,0 +1 @@
+pub mod chunked;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/norm_gate.metal
similarity index 100%
rename from crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metal
rename to crates/backend-uzu/src/backends/metal/kernel/gdn/norm_gate.metal
diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill.metal
similarity index 97%
rename from crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal
rename to crates/backend-uzu/src/backends/metal/kernel/gdn/prefill.metal
index 822cf8918..8f24c8c02 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill.metal
@@ -44,8 +44,7 @@ PUBLIC KERNEL(DeltaNetPrefill)(
const uint dv_local = tid / METAL_SIMD_SIZE;
const uint dv_idx = (dv_group * NUM_SG + dv_local) * DV_PER_SIMDGROUP;
- const uint groups_per_head = num_v_heads / num_k_heads;
- const uint hk = hv_idx / groups_per_head;
+ const uint hk = hv_idx / (num_v_heads / num_k_heads);
const uint conv_dim = 2 * key_dim + value_dim;
const uint total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;
const uint dk_base = lane * ELEMS;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill_prep.metal
similarity index 53%
rename from crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal
rename to crates/backend-uzu/src/backends/metal/kernel/gdn/prefill_prep.metal
index f78cf62e8..ee6f9cd92 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill_prep.metal
@@ -5,10 +5,6 @@
using namespace metal;
-// Pre-compute normalized q, k, beta, decay for DeltaNet prefill.
-// Grid: suffix_len × num_k_heads threadgroups, METAL_SIMD_SIZE threads each (1
-// SIMD group).
-
template <typename T, uint HEAD_K_DIM>
VARIANTS(T, float, half, bfloat)
VARIANTS(HEAD_K_DIM, 128)
@@ -27,6 +23,7 @@ PUBLIC KERNEL(DeltaNetPrefillPrep)(
constant const uint& suffix_len,
const uint token_idx GROUPS(suffix_len),
const uint hk_idx GROUPS(num_k_heads),
+ const bool write_log_decay SPECIALIZE,
const uint lane THREADS(METAL_SIMD_SIZE)
) {
static_assert(HEAD_K_DIM % METAL_SIMD_SIZE == 0, "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE");
@@ -40,54 +37,41 @@ PUBLIC KERNEL(DeltaNetPrefillPrep)(
const uint q_base = tok_offset + hk_idx * HEAD_K_DIM;
const uint k_base = tok_offset + key_dim + hk_idx * HEAD_K_DIM;
- // Load q and k elements (strided by SIMD width)
float q_vals[elems_per_thread];
float k_vals[elems_per_thread];
float q_sq = 0.0f;
float k_sq = 0.0f;
for (uint i = 0; i < elems_per_thread; ++i) {
- uint idx = lane + i * METAL_SIMD_SIZE;
+ const uint idx = lane + i * METAL_SIMD_SIZE;
q_vals[i] = float(in_proj[q_base + idx]);
k_vals[i] = float(in_proj[k_base + idx]);
q_sq += q_vals[i] * q_vals[i];
k_sq += k_vals[i] * k_vals[i];
}
- // L2 normalize via simd_sum (no threadgroup barrier needed)
- float q_norm_sq = simd_sum(q_sq);
- float k_norm_sq = simd_sum(k_sq);
- float q_inv_norm = rsqrt(q_norm_sq + 1e-6f);
- float k_inv_norm = rsqrt(k_norm_sq + 1e-6f);
- float q_scale = rsqrt(float(HEAD_K_DIM));
-
- // Normalize and scale q, write q and k
- const uint out_q_base = token_idx * key_dim + hk_idx * HEAD_K_DIM;
- const uint out_k_base = token_idx * key_dim + hk_idx * HEAD_K_DIM;
+ const float q_inv_norm = rsqrt(simd_sum(q_sq) + 1e-6f);
+ const float k_inv_norm = rsqrt(simd_sum(k_sq) + 1e-6f);
+ const float q_scale = rsqrt(float(HEAD_K_DIM));
+ const uint out_base = token_idx * key_dim + hk_idx * HEAD_K_DIM;
for (uint i = 0; i < elems_per_thread; ++i) {
- uint idx = lane + i * METAL_SIMD_SIZE;
- float q_n = q_vals[i] * q_inv_norm * q_scale;
- float k_n = k_vals[i] * k_inv_norm;
- if (idx < HEAD_K_DIM) {
- q_norm_out[out_q_base + idx] = q_n;
- k_norm_out[out_k_base + idx] = k_n;
- }
+ const uint idx = lane + i * METAL_SIMD_SIZE;
+ q_norm_out[out_base + idx] = q_vals[i] * q_inv_norm * q_scale;
+ k_norm_out[out_base + idx] = k_vals[i] * k_inv_norm;
}
- // Compute beta and decay for v-heads belonging to this k-head
- // groups_per_head v-heads share this k-head; distribute across lanes
for (uint group = lane; group < groups_per_head; group += METAL_SIMD_SIZE) {
- uint hv = hk_idx * groups_per_head + group;
+ const uint hv = hk_idx * groups_per_head + group;
+ const float beta_raw = float(in_proj[tok_offset + conv_dim + value_dim + hv]);
+ const float a_raw = float(in_proj[tok_offset + conv_dim + value_dim + num_v_heads + hv]);
+ const float log_decay = -fast::exp(a_log[hv]) * activate_softplus(a_raw + dt_bias[hv]);
- float beta_raw = float(in_proj[tok_offset + conv_dim + value_dim + hv]);
- float beta = 1.0f / (1.0f + fast::exp(-beta_raw));
-
- float a_raw = float(in_proj[tok_offset + conv_dim + value_dim + num_v_heads + hv]);
- float sp = activate_softplus(a_raw + float(dt_bias[hv]));
- float decay = fast::exp(-fast::exp(float(a_log[hv])) * sp);
-
- beta_out[token_idx * num_v_heads + hv] = beta;
- decay_out[token_idx * num_v_heads + hv] = decay;
+ beta_out[token_idx * num_v_heads + hv] = 1.0f / (1.0f + fast::exp(-beta_raw));
+ if (write_log_decay) {
+ decay_out[token_idx * num_v_heads + hv] = log_decay;
+ } else {
+ decay_out[token_idx * num_v_heads + hv] = fast::exp(log_decay);
+ }
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/build_tree_out_dispatch_helper.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/build_tree_out_dispatch_helper.rs
similarity index 100%
rename from crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/build_tree_out_dispatch_helper.rs
rename to crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/build_tree_out_dispatch_helper.rs
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/out.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metal
similarity index 94%
rename from crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/out.metal
rename to crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metal
index 93564eeff..1a6446c7e 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/out.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metal
@@ -1,10 +1,10 @@
#include <metal_stdlib>
-#include "../common/defines.h"
-#include "../common/dsl.h"
-#include "../common/thread_context.h"
-#include "../matmul/common/fragment.h"
-#include "../matmul/common/mxu_fragment_ops.h"
-#include "../matmul/common/simdgroup_fragment_ops.h"
+#include "../../common/defines.h"
+#include "../../common/dsl.h"
+#include "../../common/thread_context.h"
+#include "../../matmul/common/fragment.h"
+#include "../../matmul/common/mxu_fragment_ops.h"
+#include "../../matmul/common/simdgroup_fragment_ops.h"
using namespace metal;
using namespace uzu::matmul;
@@ -75,8 +75,7 @@ PUBLIC KERNEL(BuildTreeOut)(
const uint batch_idx = batch_value_head_idx / value_heads;
const uint value_head_idx = batch_value_head_idx - batch_idx * value_heads;
- const uint value_heads_per_qk_head = value_heads / qk_heads;
- const uint qk_head_idx = value_head_idx / value_heads_per_qk_head;
+ const uint qk_head_idx = value_head_idx / (value_heads / qk_heads);
const uint row_base = (row_tile_group_idx * SIMDGROUPS_PER_TG + thread_context.simdgroup_index) * ROWS;
const uint value_base = v_tile_idx * MATMUL_COLS;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/prefix_beta.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix_beta.metal
similarity index 92%
rename from crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/prefix_beta.metal
rename to crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix_beta.metal
index 50736b4df..e498d5aeb 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/prefix_beta.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix_beta.metal
@@ -1,9 +1,9 @@
#include <metal_stdlib>
-#include "../activation/activations.h"
-#include "../common/defines.h"
-#include "../common/dsl.h"
-#include "../common/thread_context.h"
-#include "../generated/trie.h"
+#include "../../activation/activations.h"
+#include "../../common/defines.h"
+#include "../../common/dsl.h"
+#include "../../common/thread_context.h"
+#include "../../generated/trie.h"
#define ROWS_PER_THREADGROUP 4u
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_gram.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal
similarity index 79%
rename from crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_gram.metal
rename to crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal
index 8ee34866c..dc2f1821c 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_gram.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal
@@ -1,11 +1,13 @@
#include <metal_stdlib>
-#include "../common/defines.h"
-#include "../common/dsl.h"
-#include "../common/thread_context.h"
-#include "../generated/trie.h"
-#include "../matmul/common/fragment.h"
-#include "../matmul/common/mxu_fragment_ops.h"
-#include "../matmul/common/simdgroup_fragment_ops.h"
+#include "../../common/defines.h"
+#include "../../common/dsl.h"
+#include "../../common/thread_context.h"
+#include "../../generated/trie.h"
+#include "../../matmul/common/fragment.h"
+#include "../../matmul/common/mxu_fragment_ops.h"
+#include "../../matmul/common/simdgroup_fragment_ops.h"
+#include "../common/gram.h"
+#include "../common/tri_inv.h"
using namespace metal;
using namespace uzu::matmul;
@@ -16,73 +18,6 @@ using namespace uzu::trie;
#define NUM_SIMDGROUPS 2u
#define INVALID_ROW 0xffffffffu
-// Ragged blocks are padded with identity columns so the 16x16 output is fully
-// initialized.
-METAL_FUNC void invert_tree_gram_diagonal_block(
- device float* a_inv_block,
- threadgroup const float* diag_a_tile,
- const uint block_size,
- const uint lane
-) {
- if (lane >= ROW_TILE) {
- return;
- }
-
- const uint col = lane;
- float inverse_col[ROW_TILE] = {};
- inverse_col[col] = 1.0f;
-
- METAL_PRAGMA_UNROLL
- for (uint row = 0; row < ROW_TILE; row++) {
- if (row > col && row < block_size) {
- float acc = 0.0f;
- METAL_PRAGMA_UNROLL
- for (uint prev_row = 0; prev_row < ROW_TILE; prev_row++) {
- if (prev_row < row) {
- acc += diag_a_tile[row * ROW_TILE + prev_row] * inverse_col[prev_row];
- }
- }
- inverse_col[row] = -acc;
- }
- }
-
- METAL_PRAGMA_UNROLL
- for (uint row = 0; row < ROW_TILE; row++) {
- a_inv_block[row * ROW_TILE + col] = inverse_col[row];
- }
-}
-
-// Unscaled gram dots for one tile: kk = k[rows].k[cols], qk = q[rows].k[cols].
-template <typename T, typename Ops, typename AccFragment, typename LeftFragment, typename RightFragment>
-METAL_FUNC void tree_gram_dots(
- const device T* k_rows,
- const device T* q_rows,
- const device T* k_cols,
- const uint head_k_dim,
- const uint qk_stride,
- const bool full_rows,
- const uint tile_rows,
- const bool full_cols,
- const uint tile_cols,
- const ushort lane,
- thread AccFragment& kk_acc,
- thread AccFragment& qk_acc
-) {
- kk_acc.clear();
- qk_acc.clear();
- // head_k_dim is a multiple of the fragment K depth, so the contraction has no tail.
- for (uint kb = 0; kb < head_k_dim; kb += Ops::FRAGMENT_ROWS) {
- LeftFragment k_left;
- LeftFragment q_left;
- RightFragment k_right;
- k_left.load_maybe_bounded(lane, k_rows + kb, qk_stride, full_rows, tile_rows, Ops::FRAGMENT_ROWS);
- q_left.load_maybe_bounded(lane, q_rows + kb, qk_stride, full_rows, tile_rows, Ops::FRAGMENT_ROWS);
- k_right.load_maybe_bounded(lane, k_cols + kb, qk_stride, full_cols, tile_cols, Ops::FRAGMENT_ROWS);
- fragment_mma(kk_acc, k_left, k_right);
- fragment_mma(qk_acc, q_left, k_right);
- }
-}
-
// kh0 = k @ h0[h0_idx[batch]]^T for one row-tile, walking head_v_dim in COL_TILE
// chunks from dv_start by dv_stride. No-op for a zero initial state.
template <typename T, typename Ops, typename AccFragment, typename LeftFragment, typename RightFragment>
@@ -199,8 +134,7 @@ PUBLIC KERNEL(BuildTreeGram)(
using RightFragment = OperandFragment<InputType, 1, COL_FRAGMENTS, Ops, ReadTranspose>;
static_assert(COL_TILE == METAL_SIMD_SIZE, "COL_TILE must match the SIMD width");
- const uint value_heads_per_key_head = value_heads / k_heads;
- const uint key_head_idx = value_head_idx / value_heads_per_key_head;
+ const uint key_head_idx = value_head_idx / (value_heads / k_heads);
const uint qk_stride = k_heads * head_k_dim;
const uint qk_base = (batch_idx * tree_size * k_heads + key_head_idx) * head_k_dim;
const uint prefix_base = batch_idx * tree_size * value_heads + value_head_idx;
@@ -267,20 +201,24 @@ PUBLIC KERNEL(BuildTreeGram)(
AccFragment kk_acc;
AccFragment qk_acc;
- tree_gram_dots<T, Ops, AccFragment, LeftFragment, RightFragment>(
- k_rows,
- q_rows,
- k_cols,
- head_k_dim,
- qk_stride,
- full_rows,
- tile_rows,
- full_cols,
- tile_cols,
- lane,
- kk_acc,
- qk_acc
- );
+ kk_acc.clear();
+ qk_acc.clear();
+ for (uint kb = 0; kb < head_k_dim; kb += Ops::FRAGMENT_ROWS) {
+ accumulate_dual_gram_tile<AccFragment, LeftFragment, RightFragment>(
+ kk_acc,
+ qk_acc,
+ k_rows + kb,
+ q_rows + kb,
+ k_cols + kb,
+ int(qk_stride),
+ ushort(tile_rows),
+ ushort(tile_cols),
+ Ops::FRAGMENT_ROWS,
+ full_rows,
+ full_cols,
+ lane
+ );
+ }
const bool has_diag = col_tile_idx == diag_col_tile;
simdgroup_barrier(mem_flags::mem_threadgroup);
@@ -320,7 +258,7 @@ PUBLIC KERNEL(BuildTreeGram)(
simdgroup_barrier(mem_flags::mem_threadgroup);
device float* a_inv_block =
a_inv + ((batch_idx * value_heads + value_head_idx) * num_blocks + row_tile_idx) * (ROW_TILE * ROW_TILE);
- invert_tree_gram_diagonal_block(a_inv_block, diag_a_tile, tile_rows, lane);
+ invert_lower_triangular_block<ROW_TILE>(a_inv_block, diag_a_tile, tile_rows, lane);
}
} else if (col_tile_idx < col_tiles) {
// Above-diagonal zero fill; COL_TILE == METAL_SIMD_SIZE, so each
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_update_solve.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal
similarity index 97%
rename from crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_update_solve.metal
rename to crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal
index 3d4731ef8..8e8627c2e 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_update_solve.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal
@@ -1,9 +1,9 @@
#include <metal_stdlib>
-#include "../common/defines.h"
-#include "../common/dsl.h"
-#include "../matmul/common/fragment.h"
-#include "../matmul/common/mxu_fragment_ops.h"
-#include "../matmul/common/simdgroup_fragment_ops.h"
+#include "../../common/defines.h"
+#include "../../common/dsl.h"
+#include "../../matmul/common/fragment.h"
+#include "../../matmul/common/mxu_fragment_ops.h"
+#include "../../matmul/common/simdgroup_fragment_ops.h"
using namespace metal;
using namespace uzu::matmul;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_update_solve_dispatch_helper.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve_dispatch_helper.rs
similarity index 94%
rename from crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_update_solve_dispatch_helper.rs
rename to crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve_dispatch_helper.rs
index 3a860531d..d3e5c0905 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/tree_update_solve_dispatch_helper.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve_dispatch_helper.rs
@@ -51,5 +51,5 @@ pub(crate) fn tree_update_solve_variant(
}
#[cfg(test)]
-#[path = "../../../../../tests/unit/backends/metal/tree_update_solve_dispatch_test.rs"]
+#[path = "../../../../../../tests/unit/backends/metal/tree_update_solve_dispatch_test.rs"]
mod tests;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/update.metal
similarity index 97%
rename from crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal
rename to crates/backend-uzu/src/backends/metal/kernel/gdn/update.metal
index 9790551f8..c99b2add8 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/update.metal
@@ -43,8 +43,7 @@ PUBLIC KERNEL(DeltaNetUpdate)(
const uint sg = tid / METAL_SIMD_SIZE;
const uint conv_dim = 2 * key_dim + value_dim;
- const uint groups_per_head = num_v_heads / num_k_heads;
- const uint hk = hv_idx / groups_per_head;
+ const uint hk = hv_idx / (num_v_heads / num_k_heads);
// Load + normalize q/k; each simd group recomputes the norm to skip a
// barrier.
diff --git a/crates/backend-uzu/src/backends/metal/kernel/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/mod.rs
index 02c955e65..f4a414d40 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/mod.rs
@@ -7,10 +7,11 @@ use crate::backends::{
};
pub mod attention;
-#[path = "gdn_tree_verify/build_tree_out_dispatch_helper.rs"]
+#[path = "gdn/tree_verify/build_tree_out_dispatch_helper.rs"]
mod build_tree_out_dispatch_helper;
+pub mod gdn;
pub mod matmul;
-#[path = "gdn_tree_verify/tree_update_solve_dispatch_helper.rs"]
+#[path = "gdn/tree_verify/tree_update_solve_dispatch_helper.rs"]
mod tree_update_solve_dispatch_helper;
pub const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));
@@ -24,5 +25,6 @@ impl Kernels for MetalKernels {
autogen_kernels!();
type AttentionGemmCore = attention::AttentionGemmMetalCore;
+ type DeltaNetChunkedPrefill = gdn::chunked::MetalDeltaNetChunkedPrefill;
type MatmulKernel = matmul::MatmulMetalKernel;
}
diff --git a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs
index 753544f10..f511818a5 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs
@@ -7,6 +7,7 @@ use crate::{
kernel::{
Conv1dPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel, DeltaNetNormGateKernel,
DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,
+ delta_net_chunked_prefill::{DeltaNetChunkedPrefill, DeltaNetChunkedPrefillArgs},
},
},
config::token_mixer::delta_net::DeltaNetConfig,
@@ -71,6 +72,7 @@ pub struct DeltaNet<B: Backend> {
delta_net_prefill_prep: <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel,
delta_net_prefill: <B::Kernels as Kernels>::DeltaNetPrefillKernel,
delta_net_norm_gate: <B::Kernels as Kernels>::DeltaNetNormGateKernel,
+ chunked: Option<<B::Kernels as Kernels>::DeltaNetChunkedPrefill>,
out_projection: Box<dyn Linear<B>>,
}
@@ -162,15 +164,23 @@ impl<B: Backend> DeltaNet<B> {
let delta_net_update =
<B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, outer_data_type, config.head_dim as u32)
.map_err(DeltaNetNewError::Backend)?;
- let delta_net_prefill_prep =
- <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(context, outer_data_type, config.head_dim as u32)
- .map_err(DeltaNetNewError::Backend)?;
+ let delta_net_prefill_prep = <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(
+ context,
+ outer_data_type,
+ config.head_dim as u32,
+ false,
+ )
+ .map_err(DeltaNetNewError::Backend)?;
let delta_net_prefill =
<B::Kernels as Kernels>::DeltaNetPrefillKernel::new(context, outer_data_type, config.head_dim as u32)
.map_err(DeltaNetNewError::Backend)?;
let delta_net_norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, outer_data_type)
.map_err(DeltaNetNewError::Backend)?;
+ let chunked =
+ <B::Kernels as Kernels>::DeltaNetChunkedPrefill::new(context, outer_data_type, config.head_dim as u32)
+ .map_err(DeltaNetNewError::Backend)?;
+
let out_projection = <dyn Linear<B>>::new_mixed_precision(
value_dim,
[hidden_dim],
@@ -208,6 +218,7 @@ impl<B: Backend> DeltaNet<B> {
delta_net_prefill_prep,
delta_net_prefill,
delta_net_norm_gate,
+ chunked,
out_projection,
},
in_projection_input_hadamard_factors,
@@ -333,47 +344,65 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
self.total_proj_dim as u32,
encoder,
);
-
- let mut prep_q_norm =
- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.key_dim], DataType::F32))?;
- let mut prep_k_norm =
- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.key_dim], DataType::F32))?;
- let mut prep_beta =
- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.num_heads], DataType::F32))?;
- let mut prep_decay =
- encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.num_heads], DataType::F32))?;
- self.delta_net_prefill_prep.encode(
- &in_projected,
- &self.a_log,
- &self.dt_bias,
- &mut prep_q_norm,
- &mut prep_k_norm,
- &mut prep_beta,
- &mut prep_decay,
- self.num_heads as u32,
- self.num_groups as u32,
- self.key_dim as u32,
- self.value_dim as u32,
- batch_dim.size() as u32,
- encoder,
- );
- self.delta_net_prefill.encode(
- &prep_q_norm,
- &prep_k_norm,
- &prep_beta,
- &prep_decay,
- &in_projected,
- &mut state.ssm_state,
- &mut delta_output,
- self.num_heads as u32,
- self.num_groups as u32,
- self.value_head_dim as u32,
- self.key_dim as u32,
- self.value_dim as u32,
- batch_dim.size() as u32,
- self.value_head_dim.div_ceil(16) as u32,
- encoder,
- );
+ if let Some(chunked) = self.chunked.as_ref().filter(|chunked| chunked.should_use(batch_dim.size())) {
+ chunked.encode(
+ DeltaNetChunkedPrefillArgs {
+ in_projected: &in_projected,
+ a_log: &self.a_log,
+ dt_bias: &self.dt_bias,
+ ssm_state: &mut state.ssm_state,
+ delta_output: &mut delta_output,
+ num_heads: self.num_heads as u32,
+ num_groups: self.num_groups as u32,
+ value_head_dim: self.value_head_dim as u32,
+ key_dim: self.key_dim as u32,
+ value_dim: self.value_dim as u32,
+ suffix_len: batch_dim.size(),
+ },
+ encoder,
+ )?;
+ } else {
+ let mut prep_q_norm =
+ encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.key_dim], DataType::F32))?;
+ let mut prep_k_norm =
+ encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.key_dim], DataType::F32))?;
+ let mut prep_beta =
+ encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.num_heads], DataType::F32))?;
+ let mut prep_decay =
+ encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.num_heads], DataType::F32))?;
+ self.delta_net_prefill_prep.encode(
+ &in_projected,
+ &self.a_log,
+ &self.dt_bias,
+ &mut prep_q_norm,
+ &mut prep_k_norm,
+ &mut prep_beta,
+ &mut prep_decay,
+ self.num_heads as u32,
+ self.num_groups as u32,
+ self.key_dim as u32,
+ self.value_dim as u32,
+ batch_dim.size() as u32,
+ encoder,
+ );
+ self.delta_net_prefill.encode(
+ &prep_q_norm,
+ &prep_k_norm,
+ &prep_beta,
+ &prep_decay,
+ &in_projected,
+ &mut state.ssm_state,
+ &mut delta_output,
+ self.num_heads as u32,
+ self.num_groups as u32,
+ self.value_head_dim as u32,
+ self.key_dim as u32,
+ self.value_dim as u32,
+ batch_dim.size() as u32,
+ self.value_head_dim.div_ceil(16) as u32,
+ encoder,
+ );
+ }
self.delta_net_norm_gate.encode(
&mut delta_output,
&in_projected,
diff --git a/crates/backend-uzu/unit/backends/common/kernel/delta_net_test.rs b/crates/backend-uzu/unit/backends/common/kernel/delta_net_test.rs
index 791293de5..afd333ab2 100644
--- a/crates/backend-uzu/unit/backends/common/kernel/delta_net_test.rs
+++ b/crates/backend-uzu/unit/backends/common/kernel/delta_net_test.rs
@@ -330,6 +330,7 @@ fn run_prefill_with_norm_gate_typed<T: ArrayElement>(
&context,
T::data_type(),
head_k_dim as u32,
+ false,
)
.unwrap();
let prefill_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillKernel::new(
@@ -508,6 +509,7 @@ fn test_delta_net_prefill_prep() {
&cpu_ctx,
DataType::F32,
head_k_dim as u32,
+ false,
)
.unwrap();
let mut cpu_enc = Encoder::new(cpu_ctx.as_ref()).expect("encoder");
@@ -548,6 +550,7 @@ fn test_delta_net_prefill_prep() {
&context,
DataType::F32,
head_k_dim as u32,
+ false,
)
.unwrap();
@@ -620,6 +623,7 @@ fn bench_delta_net_prefill() {
&context,
DataType::F32,
head_k_dim as u32,
+ false,
)
.unwrap();
let prefill_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillKernel::new(
diff --git a/crates/backend-uzu/unit/backends/metal/kernel/gdn/chunked_test.rs b/crates/backend-uzu/unit/backends/metal/kernel/gdn/chunked_test.rs
new file mode 100644
index 000000000..0948a6fb5
--- /dev/null
+++ b/crates/backend-uzu/unit/backends/metal/kernel/gdn/chunked_test.rs
@@ -0,0 +1,198 @@
+use half::bf16;
+use num_traits::cast;
+use proc_macros::uzu_test;
+
+use super::*;
+use crate::{
+ array::ArrayElement,
+ backends::{
+ common::{
+ Context, Kernels,
+ kernel::{DeltaNetNormGateKernel, DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel},
+ },
+ metal::Metal,
+ },
+ tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+};
+
+type BackendKernels<B> = <B as Backend>::Kernels;
+
+const NUM_V_HEADS: usize = 4;
+const NUM_K_HEADS: usize = 1;
+const HEAD_K_DIM: usize = 128;
+const HEAD_V_DIM: usize = 128;
+
+#[derive(Clone, Copy)]
+enum PrefillMode {
+ Recurrent,
+ Chunked,
+}
+
+struct TestCase {
+ suffix_len: usize,
+ in_proj: Vec<f32>,
+ a_log: Vec<f32>,
+ dt_bias: Vec<f32>,
+ norm_weight: Vec<f32>,
+ key_dim: usize,
+ value_dim: usize,
+ total_proj_dim: usize,
+}
+
+fn test_case(suffix_len: usize) -> TestCase {
+ let key_dim = NUM_K_HEADS * HEAD_K_DIM;
+ let value_dim = NUM_V_HEADS * HEAD_V_DIM;
+ let conv_dim = 2 * key_dim + value_dim;
+ let total_proj_dim = conv_dim + value_dim + NUM_V_HEADS + NUM_V_HEADS;
+
+ TestCase {
+ suffix_len,
+ in_proj: (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect(),
+ a_log: (0..NUM_V_HEADS).map(|i| -1.5 + (i as f32) * 0.05).collect(),
+ dt_bias: (0..NUM_V_HEADS).map(|i| 0.3 + (i as f32) * 0.02).collect(),
+ norm_weight: (0..HEAD_V_DIM).map(|i| 0.9 + (i as f32) * 0.001).collect(),
+ key_dim,
+ value_dim,
+ total_proj_dim,
+ }
+}
+
+fn run_prefill<T: ArrayElement>(
+ context: &MetalContext,
+ case: &TestCase,
+ mode: PrefillMode,
+) -> (Vec<f32>, Vec<f32>) {
+ let in_proj_data: Vec<T> = case.in_proj.iter().copied().map(|value| cast(value).unwrap()).collect();
+ let in_proj = alloc_allocation_with_data::<Metal, T>(context, &in_proj_data);
+ let a_log = alloc_allocation_with_data::<Metal, f32>(context, &case.a_log);
+ let dt_bias = alloc_allocation_with_data::<Metal, f32>(context, &case.dt_bias);
+ let norm_weight = alloc_allocation_with_data::<Metal, f32>(context, &case.norm_weight);
+ let state_len = NUM_V_HEADS * HEAD_V_DIM * HEAD_K_DIM;
+ let mut state = alloc_allocation::<Metal, f32>(context, state_len);
+ let mut out = alloc_allocation::<Metal, T>(context, case.suffix_len * case.value_dim);
+ let mut q = alloc_allocation::<Metal, f32>(context, case.suffix_len * case.key_dim);
+ let mut k = alloc_allocation::<Metal, f32>(context, case.suffix_len * case.key_dim);
+ let mut beta = alloc_allocation::<Metal, f32>(context, case.suffix_len * NUM_V_HEADS);
+ let mut decay = alloc_allocation::<Metal, f32>(context, case.suffix_len * NUM_V_HEADS);
+ let mut encoder = Encoder::new(context).expect("encoder");
+ encoder.encode_fill(&mut state, 0);
+
+ match mode {
+ PrefillMode::Recurrent => {
+ <BackendKernels<Metal> as Kernels>::DeltaNetPrefillPrepKernel::new(
+ context,
+ T::data_type(),
+ HEAD_K_DIM as u32,
+ false,
+ )
+ .expect("prep")
+ .encode(
+ &in_proj,
+ &a_log,
+ &dt_bias,
+ &mut q,
+ &mut k,
+ &mut beta,
+ &mut decay,
+ NUM_V_HEADS as u32,
+ NUM_K_HEADS as u32,
+ case.key_dim as u32,
+ case.value_dim as u32,
+ case.suffix_len as u32,
+ &mut encoder,
+ );
+ <BackendKernels<Metal> as Kernels>::DeltaNetPrefillKernel::new(context, T::data_type(), HEAD_K_DIM as u32)
+ .expect("prefill")
+ .encode(
+ &q,
+ &k,
+ &beta,
+ &decay,
+ &in_proj,
+ &mut state,
+ &mut out,
+ NUM_V_HEADS as u32,
+ NUM_K_HEADS as u32,
+ HEAD_V_DIM as u32,
+ case.key_dim as u32,
+ case.value_dim as u32,
+ case.suffix_len as u32,
+ HEAD_V_DIM.div_ceil(16) as u32,
+ &mut encoder,
+ );
+ },
+ PrefillMode::Chunked => {
+ <BackendKernels<Metal> as Kernels>::DeltaNetChunkedPrefill::new(context, T::data_type(), HEAD_K_DIM as u32)
+ .expect("chunked")
+ .expect("chunked unsupported")
+ .encode(
+ DeltaNetChunkedPrefillArgs {
+ in_projected: &in_proj,
+ a_log: &a_log,
+ dt_bias: &dt_bias,
+ ssm_state: &mut state,
+ delta_output: &mut out,
+ num_heads: NUM_V_HEADS as u32,
+ num_groups: NUM_K_HEADS as u32,
+ value_head_dim: HEAD_V_DIM as u32,
+ key_dim: case.key_dim as u32,
+ value_dim: case.value_dim as u32,
+ suffix_len: case.suffix_len,
+ },
+ &mut encoder,
+ )
+ .expect("chunked encode");
+ },
+ }
+
+ <BackendKernels<Metal> as Kernels>::DeltaNetNormGateKernel::new(context, T::data_type()).expect("norm").encode(
+ &mut out,
+ &in_proj,
+ &norm_weight,
+ NUM_V_HEADS as u32,
+ HEAD_V_DIM as u32,
+ case.value_dim as u32,
+ (2 * case.key_dim + case.value_dim) as u32,
+ case.total_proj_dim as u32,
+ 1e-6,
+ case.suffix_len as u32,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ (
+ allocation_to_vec::<Metal, T>(&out).into_iter().map(|value| cast(value).unwrap()).collect(),
+ allocation_to_vec(&state),
+ )
+}
+
+#[uzu_test]
+fn chunked_prefill_matches_recurrent_prefill() {
+ let context = <Metal as Backend>::Context::new().expect("metal context");
+ if <BackendKernels<Metal> as Kernels>::DeltaNetChunkedPrefill::new(
+ &context,
+ <bf16 as ArrayElement>::data_type(),
+ HEAD_K_DIM as u32,
+ )
+ .expect("chunked")
+ .is_none()
+ {
+ eprintln!("chunked prefill unsupported on this Metal device");
+ return;
+ }
+
+ for suffix_len in [CHUNK_SIZE - 1, CHUNK_SIZE, CHUNK_SIZE + 1, CHUNK_SIZE * 2 + 1] {
+ let case = test_case(suffix_len);
+ let (recurrent_out, recurrent_state) = run_prefill::<bf16>(&context, &case, PrefillMode::Recurrent);
+ let (chunked_out, chunked_state) = run_prefill::<bf16>(&context, &case, PrefillMode::Chunked);
+
+ assert!(
+ is_close::default().abs_tol(5e-3).rel_tol(1e-3).all_close(chunked_out.iter().copied(), recurrent_out),
+ "chunked output differs at T={suffix_len}"
+ );
+ assert!(
+ is_close::default().abs_tol(2e-4).rel_tol(1e-3).all_close(chunked_state.iter().copied(), recurrent_state),
+ "chunked state differs at T={suffix_len}"
+ );
+ }
+}