Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Chunked GDN improvements (#978)

CCYehe744e7cf4dc5
diff --git a/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs b/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs
index 648c1ad58..e51798d8f 100644
--- a/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs
@@ -26,6 +26,7 @@ pub trait DeltaNetChunkedPrefill: Sized + Send + Sync {
context: &<Self::Backend as Backend>::Context,
outer_data_type: DataType,
head_dim: u32,
+ groups_per_head: u32,
) -> Result<Option<Self>, <Self::Backend as Backend>::Error>;
fn should_use(
@@ -48,6 +49,7 @@ impl<B: Backend<Kernels: Kernels<DeltaNetChunkedPrefill = Unsupported<B>>>> Delt
_context: &B::Context,
_outer_data_type: DataType,
_head_dim: u32,
+ _groups_per_head: u32,
) -> Result<Option<Self>, B::Error> {
Ok(None)
}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_pack.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_pack.rs
index 9f97cdadc..fe58ff141 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_pack.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_pack.rs
@@ -4,35 +4,34 @@ use uzu_engine_macros::kernel;
use crate::data_type::DataTypeRepr;
-#[kernel(Conv1dPack)]
-#[variants(StateT, f32, bf16)]
-#[variants(InputT, f32, bf16)]
-pub fn conv1d_pack<StateT: DataTypeRepr + Float, InputT: DataTypeRepr + Float>(
- state_in: *const StateT,
- x: *const InputT,
- padded: *mut StateT,
- state_stride: u32,
- row_stride: u32,
+#[kernel(DeltaNetConvPack)]
+#[variants(T, f32, bf16)]
+#[variants(KERNEL_SIZE, 4)]
+pub fn delta_net_conv_pack<T: DataTypeRepr + Float, const KERNEL_SIZE: u32>(
+ state_in: *const f32,
+ x: *const T,
+ packed: *mut f32,
suffix_len: u32,
- num_channels: u32,
+ tile_tokens: u32,
+ row_stride: u32,
+ conv_dim: u32,
) {
- let state_stride = state_stride as usize;
+ let taps = KERNEL_SIZE as usize - 1;
+ let tile_tokens = tile_tokens as usize;
let row_stride = row_stride as usize;
- let suffix_len = suffix_len as usize;
- let num_channels = num_channels as usize;
+ let conv_dim = conv_dim as usize;
- unsafe {
- for channel_idx in 0..num_channels {
- for row_idx in 0..state_stride + suffix_len {
- let padded_index = row_idx * row_stride + channel_idx;
- if row_idx < state_stride {
- let state_index = channel_idx * state_stride + row_idx;
- *padded.add(padded_index) = *state_in.add(state_index);
- } else {
- let token = row_idx - state_stride;
- let x_index = token * row_stride + channel_idx;
- *padded.add(padded_index) = StateT::from((*x.add(x_index)).to_f32().unwrap()).unwrap();
- }
+ for tile in 0..(suffix_len as usize).div_ceil(tile_tokens) {
+ for tap in 0..taps {
+ for channel in 0..conv_dim {
+ let value = unsafe {
+ if tile == 0 {
+ *state_in.add(channel * taps + tap)
+ } else {
+ (*x.add((tile * tile_tokens - taps + tap) * row_stride + channel)).to_f32().unwrap()
+ }
+ };
+ unsafe { *packed.add((tile * taps + tap) * conv_dim + channel) = value };
}
}
}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs
index 991154674..af84400d8 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/conv_scan.rs
@@ -4,69 +4,61 @@ use uzu_engine_macros::kernel;
use crate::{backends::common::gpu_types::activation_type::ActivationType, data_type::DataTypeRepr};
-// Multi-token causal conv1d with SiLU for DeltaNet.
-// Reads from conv_padded (Conv1dPack output), writes conv'd+SiLU'd values
-// back to in_proj with stride out_stride, updates conv state.
+// Multi-token causal conv1d with SiLU for DeltaNet, in place on in_proj. Each token tile starts from its packed rows (see
+// DeltaNetConvPack), so tiles are independent; the last tile writes the new conv state.
//
-// conv_padded: [state_stride + suffix_len, row_stride] — packed history + input
-// conv_weight: [conv_dim, kernel_size]
-// bias: [conv_dim] (optional)
-// in_proj: [suffix_len, out_stride] — first conv_dim channels overwritten
-// state_out: [conv_dim, state_stride]
+// packed: [num_tiles, KERNEL_SIZE - 1, conv_dim]
+// conv_weight: [conv_dim, KERNEL_SIZE]
+// bias: [conv_dim] (optional)
+// in_proj: [suffix_len, out_stride] - first conv_dim channels overwritten
+// state_out: [conv_dim, KERNEL_SIZE - 1]
#[kernel(DeltaNetConvScan)]
#[variants(T, f32, bf16)]
-pub fn delta_net_conv_scan<T: DataTypeRepr + Float>(
- conv_padded: *const f32,
+#[variants(KERNEL_SIZE, 4)]
+pub fn delta_net_conv_scan<T: DataTypeRepr + Float, const KERNEL_SIZE: u32>(
+ packed: *const f32,
conv_weight: *const f32,
#[optional(has_bias)] bias: Option<*const f32>,
in_proj: *mut T,
state_out: *mut f32,
suffix_len: u32,
- kernel_size: u32,
- row_stride: u32,
- state_stride: u32,
+ tile_tokens: u32,
conv_dim: u32,
out_stride: u32,
#[specialize] has_bias: bool,
) {
+ let kernel_size = KERNEL_SIZE as usize;
+ let taps = kernel_size - 1;
let suffix_len = suffix_len as usize;
- let kernel_size = kernel_size as usize;
- let row_stride = row_stride as usize;
- let state_stride = state_stride as usize;
+ let tile_tokens = tile_tokens as usize;
let conv_dim = conv_dim as usize;
let out_stride = out_stride as usize;
- for token in 0..suffix_len {
+ for tile in 0..suffix_len.div_ceil(tile_tokens) {
for channel in 0..conv_dim {
- let w_offset = channel * kernel_size;
-
- let mut acc = if has_bias {
- unsafe { *bias.unwrap().add(channel) }
- } else {
- 0.0f32
- };
-
- for tap in 0..kernel_size {
- let padded_row = token + tap;
- let padded_idx = padded_row * row_stride + channel;
- let sample = unsafe { *conv_padded.add(padded_idx) };
- acc += unsafe { *conv_weight.add(w_offset + tap) } * sample;
- }
-
- unsafe {
- *in_proj.add(token * out_stride + channel) = T::from(ActivationType::SILU.activate(acc)).unwrap();
+ let mut window: Vec<f32> =
+ (0..taps).map(|tap| unsafe { *packed.add((tile * taps + tap) * conv_dim + channel) }).collect();
+ for token in tile * tile_tokens..((tile + 1) * tile_tokens).min(suffix_len) {
+ let sample = unsafe { (*in_proj.add(token * out_stride + channel)).to_f32().unwrap() };
+ let mut acc = if has_bias {
+ unsafe { *bias.unwrap().add(channel) }
+ } else {
+ 0.0f32
+ };
+ for (tap, &previous) in window.iter().enumerate() {
+ acc += unsafe { *conv_weight.add(channel * kernel_size + tap) } * previous;
+ }
+ acc += unsafe { *conv_weight.add(channel * kernel_size + taps) } * sample;
+ unsafe {
+ *in_proj.add(token * out_stride + channel) = T::from(ActivationType::SILU.activate(acc)).unwrap();
+ }
+ window.remove(0);
+ window.push(sample);
}
- }
- }
-
- // state_out layout: [conv_dim, state_stride] (channel-major)
- for channel in 0..conv_dim {
- for tap in 0..state_stride {
- let padded_row = suffix_len + tap;
- let padded_idx = padded_row * row_stride + channel;
- let sample = unsafe { *conv_padded.add(padded_idx) };
- unsafe {
- *state_out.add(channel * state_stride + tap) = sample;
+ if (tile + 1) * tile_tokens >= suffix_len {
+ for (tap, &value) in window.iter().enumerate() {
+ unsafe { *state_out.add(channel * taps + tap) = value };
+ }
}
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/a_diag_inv.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/a_diag_inv.metal
deleted file mode 100644
index 266139d08..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/a_diag_inv.metal
+++ /dev/null
@@ -1,92 +0,0 @@
-#include <metal_stdlib>
-#include "../../common/defines.h"
-#include "../../common/dsl.h"
-#include "../common/tri_inv.h"
-
-using namespace metal;
-
-#define DIAG_BLOCK_SIZE 16u
-
-// kk: [chunks, num_k_heads, CHUNK_SIZE, CHUNK_SIZE] K K^T from Gram.
-// beta: [suffix_len, num_v_heads].
-// g: [num_v_heads, suffix_len] cumulative log decay.
-// a_packed: [chunks, num_v_heads, num_blocks, ceil(num_blocks/2),
-// DIAG_BLOCK_SIZE, 2*DIAG_BLOCK_SIZE].
-// a_inv: [chunks, num_v_heads, num_blocks, DIAG_BLOCK_SIZE, DIAG_BLOCK_SIZE].
-//
-// One threadgroup owns one (chunk, v-head, column-pair, row-block). It materializes
-// A[row,col] = beta[row] * exp(g[row] - g[col]) * kk[row,col] for col < row,
-// writes packed column-pair blocks through the diagonal pair, copies the
-// diagonal block into threadgroup memory, then writes (I + A_diag)^-1 to a_inv.
-template <uint CHUNK_SIZE>
-VARIANTS(CHUNK_SIZE, 32, 64)
-KERNEL(DeltaNetChunkedADiagInv)(
- device const float* kk,
- device const float* beta,
- device const float* g,
- device float* a_packed,
- device float* a_inv,
- constant const uint& num_v_heads,
- constant const uint& num_k_heads,
- constant const uint& suffix_len,
- threadgroup float diag_a_tile[DIAG_BLOCK_SIZE * DIAG_BLOCK_SIZE],
- const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),
- const uint hv_idx GROUPS(num_v_heads),
- const uint block_col_pair_idx GROUPS(
- CHUNK_SIZE.div_ceil(DIAG_BLOCK_SIZE) * CHUNK_SIZE.div_ceil(DIAG_BLOCK_SIZE).div_ceil(2)
- ),
- const uint lane THREADS(METAL_SIMD_SIZE)
-) {
- constexpr uint num_blocks = (CHUNK_SIZE + DIAG_BLOCK_SIZE - 1) / DIAG_BLOCK_SIZE;
- constexpr uint num_col_pairs = (num_blocks + 1) / 2;
- const uint block_idx = block_col_pair_idx / num_col_pairs;
- const uint col_pair_idx = block_col_pair_idx - block_idx * num_col_pairs;
- const uint hk_idx = hv_idx / (num_v_heads / num_k_heads);
- const uint token_base = chunk_idx * CHUNK_SIZE;
- const uint row_base = block_idx * DIAG_BLOCK_SIZE;
- const uint kk_base = (chunk_idx * num_k_heads + hk_idx) * CHUNK_SIZE * CHUNK_SIZE;
- const uint diag_col_pair = block_idx / 2;
- const uint diag_col_offset = (block_idx % 2) * DIAG_BLOCK_SIZE;
- const device float* g_head = g + hv_idx * suffix_len;
- if (col_pair_idx > diag_col_pair) {
- return;
- }
-
- METAL_PRAGMA_UNROLL
- for (uint idx = lane; idx < DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE; idx += METAL_SIMD_SIZE) {
- const uint local_col = idx % (2 * DIAG_BLOCK_SIZE);
- const uint local_row = (idx / (2 * DIAG_BLOCK_SIZE)) % DIAG_BLOCK_SIZE;
- const uint row = row_base + local_row;
- const uint col = col_pair_idx * 2 * DIAG_BLOCK_SIZE + local_col;
- const uint row_token = token_base + row;
- const uint col_token = token_base + col;
-
- float value = 0.0f;
- if (row < CHUNK_SIZE && col < CHUNK_SIZE && row_token < suffix_len && col_token < suffix_len && col < row) {
- const float beta_row = beta[row_token * num_v_heads + hv_idx];
- const float g_row = g_head[row_token];
- const float g_col = g_head[col_token];
- value = beta_row * fast::exp(g_row - g_col) * kk[kk_base + row * CHUNK_SIZE + col];
- }
-
- const uint out_idx =
- (((chunk_idx * num_v_heads + hv_idx) * num_blocks + block_idx) * num_col_pairs + col_pair_idx) *
- (DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE) +
- local_row * (2 * DIAG_BLOCK_SIZE) + local_col;
- a_packed[out_idx] = value;
- if (col_pair_idx == diag_col_pair && local_col >= diag_col_offset &&
- local_col < diag_col_offset + DIAG_BLOCK_SIZE) {
- diag_a_tile[local_row * DIAG_BLOCK_SIZE + local_col - diag_col_offset] = value;
- }
- }
-
- if (col_pair_idx != diag_col_pair) {
- return;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- device float* a_inv_block =
- a_inv + ((chunk_idx * num_v_heads + hv_idx) * num_blocks + block_idx) * (DIAG_BLOCK_SIZE * DIAG_BLOCK_SIZE);
- const uint block_tokens =
- token_base + row_base < suffix_len ? min(uint(DIAG_BLOCK_SIZE), suffix_len - token_base - row_base) : 0u;
- invert_lower_triangular_block<DIAG_BLOCK_SIZE>(a_inv_block, diag_a_tile, block_tokens, lane);
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/causal_inv.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/causal_inv.metal
deleted file mode 100644
index efccb1b1c..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/causal_inv.metal
+++ /dev/null
@@ -1,102 +0,0 @@
-#include <metal_stdlib>
-#include "../../common/defines.h"
-#include "../../common/dsl.h"
-#include "../../matmul/common/fragment.h"
-#include "../../matmul/common/simdgroup_fragment_ops.h"
-
-using namespace metal;
-using namespace uzu::matmul;
-
-#define DIAG_BLOCK_SIZE 16u
-
-// Materializes dense T = (I + A)^-1 as bf16 for output and state. This is block
-// forward substitution over a_packed plus the per-block inverses from ADiagInv.
-template <uint CHUNK_SIZE, uint BV>
-VARIANTS(CHUNK_SIZE, 32, 64)
-VARIANTS(BV, 16, 32)
-KERNEL(DeltaNetChunkedCausalInv)(
- device const float* a_packed,
- device const float* a_inv,
- device bfloat* t_out,
- constant const uint& num_v_heads,
- constant const uint& suffix_len,
- const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),
- const uint hv_idx GROUPS(num_v_heads),
- const uint tile_idx GROUPS(CHUNK_SIZE.div_ceil(BV)),
- const uint lane THREADS(METAL_SIMD_SIZE)
-) {
- using Ops = SimdgroupFragmentOps;
- constexpr ushort TOKEN_FRAGMENTS = DIAG_BLOCK_SIZE / Ops::FRAGMENT_ROWS;
- constexpr ushort VALUE_FRAGMENTS = BV / Ops::FRAGMENT_COLS;
- using TileFragment = Fragment<float, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, Ops>;
- using MatrixFragment = OperandFragment<float, TOKEN_FRAGMENTS, TOKEN_FRAGMENTS, Ops>;
- using ValueFragment = OperandFragment<bfloat, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, Ops>;
- using PairMatrixFragment = OperandFragment<float, TOKEN_FRAGMENTS, 2 * TOKEN_FRAGMENTS, Ops>;
- using PairValueFragment = OperandFragment<bfloat, 2 * TOKEN_FRAGMENTS, VALUE_FRAGMENTS, Ops>;
-
- constexpr uint num_blocks = (CHUNK_SIZE + DIAG_BLOCK_SIZE - 1) / DIAG_BLOCK_SIZE;
- constexpr uint num_col_pairs = (num_blocks + 1) / 2;
- const uint slice_base = tile_idx * BV;
-
- const device float* a_blocks = a_packed + (chunk_idx * num_v_heads + hv_idx) * num_blocks * num_col_pairs *
- (DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE);
- const device float* inv_blocks =
- a_inv + (chunk_idx * num_v_heads + hv_idx) * num_blocks * (DIAG_BLOCK_SIZE * DIAG_BLOCK_SIZE);
- device bfloat* t_head = t_out + (chunk_idx * num_v_heads + hv_idx) * CHUNK_SIZE * CHUNK_SIZE + slice_base;
-
- for (uint block_idx = 0; block_idx < num_blocks; ++block_idx) {
- const uint row_base = block_idx * DIAG_BLOCK_SIZE;
-
- // -- RHS = identity slice: acc[row, col] = (row_base+row == slice_base+col) -
- TileFragment acc_t;
- acc_t.map_coords(lane, [&](short local_row, short local_col, float) {
- const uint global_row = row_base + uint(local_row);
- const uint global_col = slice_base + uint(local_col);
- return global_row == global_col ? 1.0f : 0.0f;
- });
-
- // -- Forward substitution: subtract A . T_prev (T_prev in bf16, own slice) --
- // a_packed stays positive for block inverse; dense solve subtracts it.
- const uint num_full_pairs = block_idx / 2;
- for (uint pair_idx = 0; pair_idx < num_full_pairs; ++pair_idx) {
- PairMatrixFragment a_frag;
- const device float* a_pair =
- a_blocks + (block_idx * num_col_pairs + pair_idx) * (DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE);
- a_frag.load_from(lane, fragment_source(a_pair, 2 * DIAG_BLOCK_SIZE));
- a_frag.map([](float value) { return -value; });
-
- PairValueFragment t_prev_frag;
- const device bfloat* t_prev = t_head + pair_idx * 2 * DIAG_BLOCK_SIZE * CHUNK_SIZE;
- t_prev_frag.load_from(lane, fragment_source(t_prev, CHUNK_SIZE));
- fragment_mma(acc_t, a_frag, t_prev_frag);
- }
-
- for (uint prev_block_idx = num_full_pairs * 2; prev_block_idx < block_idx; ++prev_block_idx) {
- MatrixFragment a_frag;
- const device float* a_block =
- a_blocks + (block_idx * num_col_pairs + prev_block_idx / 2) * (DIAG_BLOCK_SIZE * 2 * DIAG_BLOCK_SIZE) +
- (prev_block_idx % 2) * DIAG_BLOCK_SIZE;
- a_frag.load_from(lane, fragment_source(a_block, 2 * DIAG_BLOCK_SIZE));
- a_frag.map([](float value) { return -value; });
-
- ValueFragment t_prev_frag;
- const device bfloat* t_prev = t_head + prev_block_idx * DIAG_BLOCK_SIZE * CHUNK_SIZE;
- t_prev_frag.load_from(lane, fragment_source(t_prev, CHUNK_SIZE));
- fragment_mma(acc_t, a_frag, t_prev_frag);
- }
-
- // -- Apply the diagonal-block inverse and store T ------------------------
- MatrixFragment inv_frag;
- inv_frag.load_from(
- lane,
- fragment_source(inv_blocks + block_idx * DIAG_BLOCK_SIZE * DIAG_BLOCK_SIZE, DIAG_BLOCK_SIZE)
- );
-
- TileFragment t_block;
- t_block.clear();
- fragment_mma(t_block, inv_frag, acc_t);
- t_block.store(lane, t_head + row_base * CHUNK_SIZE, CHUNK_SIZE);
-
- simdgroup_barrier(mem_flags::mem_device);
- }
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/cumsum.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/cumsum.metal
deleted file mode 100644
index 37e5277e8..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/cumsum.metal
+++ /dev/null
@@ -1,37 +0,0 @@
-#include <metal_stdlib>
-#include "../../common/defines.h"
-#include "../../common/dsl.h"
-
-using namespace metal;
-
-template <uint CHUNK_SIZE>
-VARIANTS(CHUNK_SIZE, 32, 64)
-KERNEL(DeltaNetChunkedCumsum)(
- device const float* log_decay,
- device float* g_out,
- constant const uint& num_v_heads,
- constant const uint& suffix_len,
- const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),
- const uint hv_idx GROUPS(num_v_heads),
- const uint lane THREADS(METAL_SIMD_SIZE)
-) {
- const uint token_begin = chunk_idx * CHUNK_SIZE;
- const uint token_end = min(token_begin + CHUNK_SIZE, suffix_len);
-
- float carry = 0.0f;
- METAL_PRAGMA_UNROLL
- for (uint token_offset = 0; token_offset < CHUNK_SIZE; token_offset += METAL_SIMD_SIZE) {
- const uint token = token_begin + token_offset + lane;
- const bool valid = token < token_end;
- const uint input_offset = token * num_v_heads + hv_idx;
- const uint output_offset = hv_idx * suffix_len + token;
- const float value = valid ? log_decay[input_offset] : 0.0f;
- const float prefix = carry + simd_prefix_inclusive_sum(value);
-
- if (valid) {
- g_out[output_offset] = prefix;
- }
-
- carry += simd_sum(value);
- }
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/gram.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/gram.metal
deleted file mode 100644
index 2566e9499..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/gram.metal
+++ /dev/null
@@ -1,114 +0,0 @@
-#include <metal_stdlib>
-#include "../../common/defines.h"
-#include "../../common/dsl.h"
-#include "../common/gram.h"
-
-using namespace metal;
-using namespace uzu::matmul;
-
-#define ROW_TILE 16u
-#define COL_TILE 32u
-
-// q_norm/k_norm: [suffix_len, key_dim], g: [num_v_heads, suffix_len].
-// kk_out: [chunks, num_k_heads, CHUNK_SIZE, CHUNK_SIZE].
-// qk_scaled_out: [chunks, num_v_heads, CHUNK_SIZE, CHUNK_SIZE].
-// Computes K K^T per k-head and decay-scaled Q K^T per grouped v-head.
-// MXU did not improve e2e vs simdgroup.
-template <uint HEAD_K_DIM, uint CHUNK_SIZE>
-VARIANTS(HEAD_K_DIM, 128)
-VARIANTS(CHUNK_SIZE, 32, 64)
-KERNEL(DeltaNetChunkedGram)(
- device const float* q_norm,
- device const float* k_norm,
- device const float* g,
- device float* kk_out,
- device float* qk_scaled_out,
- constant const uint& num_v_heads,
- constant const uint& num_k_heads,
- constant const uint& key_dim,
- constant const uint& suffix_len,
- const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),
- const uint hk_idx GROUPS(num_k_heads),
- const uint tile_idx GROUPS(CHUNK_SIZE.div_ceil(ROW_TILE) * CHUNK_SIZE.div_ceil(COL_TILE)),
- const uint lane THREADS(METAL_SIMD_SIZE)
-) {
- using Ops = SimdgroupFragmentOps;
- using InputType = float;
- constexpr ushort ROW_FRAGMENTS = ROW_TILE / Ops::FRAGMENT_ROWS;
- constexpr ushort COL_FRAGMENTS = COL_TILE / Ops::FRAGMENT_COLS;
- using AccFragment = Fragment<float, ROW_FRAGMENTS, COL_FRAGMENTS, Ops>;
- using LeftFragment = OperandFragment<InputType, ROW_FRAGMENTS, 1, Ops>;
- using RightFragment = OperandFragment<InputType, 1, COL_FRAGMENTS, Ops, ReadTranspose>;
-
- static_assert(HEAD_K_DIM % Ops::FRAGMENT_ROWS == 0, "HEAD_K_DIM must align to fragment K");
-
- constexpr uint col_tiles = (CHUNK_SIZE + COL_TILE - 1) / COL_TILE;
- const uint row_tile_idx = tile_idx / col_tiles;
- const uint col_tile_idx = tile_idx - row_tile_idx * col_tiles;
- const uint row_base = row_tile_idx * ROW_TILE;
- const uint col_base = col_tile_idx * COL_TILE;
- const uint chunk_token_base = chunk_idx * CHUNK_SIZE;
-
- const uint tile_rows = min(uint(ROW_TILE), uint(CHUNK_SIZE - row_base));
- const uint tile_cols = min(uint(COL_TILE), uint(CHUNK_SIZE - col_base));
- const uint row_token_base = chunk_token_base + row_base;
- const uint col_token_base = chunk_token_base + col_base;
- const uint valid_rows = row_token_base < suffix_len ? min(tile_rows, suffix_len - row_token_base) : 0u;
- const uint valid_cols = col_token_base < suffix_len ? min(tile_cols, suffix_len - col_token_base) : 0u;
- // Upper-causal tiles are zero.
- if (valid_rows == 0 || valid_cols == 0 || col_base >= row_base + valid_rows) {
- return;
- }
-
- AccFragment kk_acc;
- AccFragment qk_acc;
- kk_acc.clear();
- qk_acc.clear();
-
- for (uint k_block_start = 0; k_block_start < HEAD_K_DIM; k_block_start += Ops::FRAGMENT_ROWS) {
- const device float* k_rows = k_norm + row_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;
- const device float* q_rows = q_norm + row_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;
- const device float* k_cols = k_norm + col_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;
-
- accumulate_dual_gram_tile<AccFragment, LeftFragment, RightFragment>(
- kk_acc,
- qk_acc,
- k_rows,
- q_rows,
- k_cols,
- int(key_dim),
- ushort(valid_rows),
- ushort(valid_cols),
- Ops::FRAGMENT_ROWS,
- valid_rows == tile_rows,
- valid_cols == tile_cols,
- ushort(lane)
- );
- }
-
- const short2 tile_dims = short2(tile_cols, tile_rows);
- const uint kk_base = (chunk_idx * num_k_heads + hk_idx) * CHUNK_SIZE * CHUNK_SIZE + row_base * CHUNK_SIZE + col_base;
- kk_acc.store_safe(lane, kk_out + kk_base, CHUNK_SIZE, tile_dims);
-
- // Store qk for each grouped v-head with decay scale and causal mask.
- const uint valid_tokens = chunk_token_base < suffix_len ? min(uint(CHUNK_SIZE), suffix_len - chunk_token_base) : 0u;
- const uint groups_per_head = num_v_heads / num_k_heads;
- for (uint group = 0; group < groups_per_head; ++group) {
- const uint hv_idx = hk_idx * groups_per_head + group;
- const device float* g_head = g + hv_idx * suffix_len;
- AccFragment scaled = qk_acc;
- scaled.map_coords(lane, [&](short r, short c, float value) {
- const uint row = row_base + uint(r);
- const uint col = col_base + uint(c);
- if (row >= valid_tokens || col >= valid_tokens || col > row) {
- return 0.0f;
- }
- const float g_row = g_head[chunk_token_base + row];
- const float g_col = g_head[chunk_token_base + col];
- return value * fast::exp(g_row - g_col);
- });
- const uint dst_base =
- (chunk_idx * num_v_heads + hv_idx) * CHUNK_SIZE * CHUNK_SIZE + row_base * CHUNK_SIZE + col_base;
- scaled.store_safe(lane, qk_scaled_out + dst_base, CHUNK_SIZE, tile_dims);
- }
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/mod.rs
index 93b523059..655a59f70 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/mod.rs
@@ -1,6 +1,5 @@
use super::super::{
- DeltaNetChunkedADiagInvMetalKernel, DeltaNetChunkedCausalInvMetalKernel, DeltaNetChunkedCumsumMetalKernel,
- DeltaNetChunkedGramMetalKernel, DeltaNetChunkedOutputAndStateMetalKernel, DeltaNetPrefillPrepMetalKernel,
+ DeltaNetChunkedOutputAndStateMetalKernel, DeltaNetChunkedPrecomputeMetalKernel, DeltaNetPrefillPrepMetalKernel,
};
use crate::{
backends::{
@@ -18,17 +17,13 @@ use crate::{
const MXU_MIN_T: u32 = 256;
const CHUNK_SIZE: u32 = 64;
-const BLOCK_SIZE: u32 = 16;
const VT: u32 = 32;
const INNER_DATA_TYPE: DataType = DataType::F32;
pub struct MetalDeltaNetChunkedPrefill {
min_t: u32,
prep: DeltaNetPrefillPrepMetalKernel,
- cumsum: DeltaNetChunkedCumsumMetalKernel,
- gram: DeltaNetChunkedGramMetalKernel,
- a_diag_inv: DeltaNetChunkedADiagInvMetalKernel,
- causal_inv: DeltaNetChunkedCausalInvMetalKernel,
+ precompute: DeltaNetChunkedPrecomputeMetalKernel,
output_and_state: DeltaNetChunkedOutputAndStateMetalKernel,
}
@@ -39,6 +34,7 @@ impl DeltaNetChunkedPrefill for MetalDeltaNetChunkedPrefill {
context: &MetalContext,
outer_data_type: DataType,
head_dim: u32,
+ groups_per_head: u32,
) -> Result<Option<Self>, <Metal as Backend>::Error> {
if outer_data_type == DataType::F16 {
return Ok(None);
@@ -48,6 +44,11 @@ impl DeltaNetChunkedPrefill for MetalDeltaNetChunkedPrefill {
return Ok(None);
}
+ // The precompute kernel assigns one grouped v-head to each of its four simdgroups.
+ if groups_per_head > 4 {
+ return Ok(None);
+ }
+
let scratch_data_type = if outer_data_type == DataType::BF16 {
DataType::BF16
} else {
@@ -64,10 +65,7 @@ impl DeltaNetChunkedPrefill for MetalDeltaNetChunkedPrefill {
true,
false,
)?,
- cumsum: DeltaNetChunkedCumsumMetalKernel::new(context, CHUNK_SIZE)?,
- gram: DeltaNetChunkedGramMetalKernel::new(context, head_dim, CHUNK_SIZE)?,
- a_diag_inv: DeltaNetChunkedADiagInvMetalKernel::new(context, CHUNK_SIZE)?,
- causal_inv: DeltaNetChunkedCausalInvMetalKernel::new(context, CHUNK_SIZE, VT)?,
+ precompute: DeltaNetChunkedPrecomputeMetalKernel::new(context, head_dim, CHUNK_SIZE)?,
output_and_state: DeltaNetChunkedOutputAndStateMetalKernel::new(
context,
outer_data_type,
@@ -94,8 +92,6 @@ impl DeltaNetChunkedPrefill for MetalDeltaNetChunkedPrefill {
) -> Result<(), <Metal as Backend>::Error> {
let suffix_len = args.suffix_len;
let num_chunks = suffix_len.div_ceil(CHUNK_SIZE);
- let num_blocks = CHUNK_SIZE.div_ceil(BLOCK_SIZE);
- let num_col_pairs = num_blocks.div_ceil(2);
let mut q_norm = command_buffer.allocate_scratch_for_shape(&[suffix_len, args.key_dim], INNER_DATA_TYPE)?;
let mut k_norm = command_buffer.allocate_scratch_for_shape(&[suffix_len, args.key_dim], INNER_DATA_TYPE)?;
@@ -103,18 +99,8 @@ impl DeltaNetChunkedPrefill for MetalDeltaNetChunkedPrefill {
let mut log_decay =
command_buffer.allocate_scratch_for_shape(&[suffix_len, args.num_heads], INNER_DATA_TYPE)?;
let mut g = command_buffer.allocate_scratch_for_shape(&[suffix_len, args.num_heads], INNER_DATA_TYPE)?;
- let mut kk = command_buffer
- .allocate_scratch_for_shape(&[num_chunks, args.num_groups, CHUNK_SIZE, CHUNK_SIZE], INNER_DATA_TYPE)?;
let mut qk = command_buffer
.allocate_scratch_for_shape(&[num_chunks, args.num_heads, CHUNK_SIZE, CHUNK_SIZE], INNER_DATA_TYPE)?;
- let mut a_packed = command_buffer.allocate_scratch_for_shape(
- &[num_chunks, args.num_heads, num_blocks, num_col_pairs, BLOCK_SIZE, 2 * BLOCK_SIZE],
- INNER_DATA_TYPE,
- )?;
- let mut a_inv = command_buffer.allocate_scratch_for_shape(
- &[num_chunks, args.num_heads, num_blocks, BLOCK_SIZE, BLOCK_SIZE],
- INNER_DATA_TYPE,
- )?;
let mut t_mat = command_buffer
.allocate_scratch_for_shape(&[num_chunks, args.num_heads, CHUNK_SIZE, CHUNK_SIZE], DataType::BF16)?;
@@ -134,31 +120,20 @@ impl DeltaNetChunkedPrefill for MetalDeltaNetChunkedPrefill {
suffix_len,
command_buffer,
);
- self.cumsum.encode(&log_decay, &mut g, args.num_heads, suffix_len, command_buffer);
- self.gram.encode(
+ self.precompute.encode(
&q_norm,
&k_norm,
- &g,
- &mut kk,
+ &log_decay,
+ &beta,
+ &mut g,
&mut qk,
+ &mut t_mat,
args.num_heads,
args.num_groups,
args.key_dim,
suffix_len,
command_buffer,
);
- self.a_diag_inv.encode(
- &kk,
- &beta,
- &g,
- &mut a_packed,
- &mut a_inv,
- args.num_heads,
- args.num_groups,
- suffix_len,
- command_buffer,
- );
- self.causal_inv.encode(&a_packed, &a_inv, &mut t_mat, args.num_heads, suffix_len, command_buffer);
self.output_and_state.encode(
&q_norm,
&k_norm,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/output_and_state.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/output_and_state.metal
index dee18b6aa..36158d572 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/output_and_state.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/output_and_state.metal
@@ -74,6 +74,10 @@ KERNEL(DeltaNetChunkedOutputAndState)(
constexpr ushort TOKEN_FRAGMENTS = TOKEN_TILE / FR;
constexpr ushort VALUE_FRAGMENTS = VT / FC;
constexpr ushort KEY_FRAGMENTS = OUTPUT_STATE_KEY_TILE / FC;
+ // Per-row terms are hoisted into row_* arrays: a lane owns rows lane_row + slot * ROW_STRIDE.
+ constexpr ushort ROW_STRIDE = Ops::THREAD_ELEMENT_ROW_STRIDE;
+ constexpr ushort ROW_SLOTS = Ops::THREAD_ELEMENT_ROWS;
+ constexpr ushort TOKEN_ROW_SLOTS = TOKEN_FRAGMENTS * ROW_SLOTS;
using TokenValueFragment = Fragment<float, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, Ops>;
using TokenKeyFragment = OperandFragment<float, TOKEN_FRAGMENTS, 1, Ops>;
@@ -89,6 +93,7 @@ KERNEL(DeltaNetChunkedOutputAndState)(
if (value_base >= head_v_dim) {
return;
}
+ const short lane_row = Ops::get_position(ushort(lane)).y;
const uint hk_idx = hv_idx / (num_v_heads / num_k_heads);
const uint num_chunks = (suffix_len + OUTPUT_STATE_CHUNK - 1) / OUTPUT_STATE_CHUNK;
const uint conv_dim = 2 * key_dim + value_dim;
@@ -115,6 +120,8 @@ KERNEL(DeltaNetChunkedOutputAndState)(
};
TokenValueFragment output_acc;
+ float row_beta[TOKEN_ROW_SLOTS];
+ float row_exp_g[TOKEN_ROW_SLOTS];
// -- R phase: R = beta (.) (V - e^g (.) (K . S^T)) into residual_new_value_tile ----------
if (simdgroup_idx < NUM_TOKEN_TILES) {
const uint token_tile_base = simdgroup_idx * TOKEN_TILE;
@@ -147,17 +154,23 @@ KERNEL(DeltaNetChunkedOutputAndState)(
fragment_mma(output_acc, query_frag, state_frag);
}
+ METAL_PRAGMA_UNROLL
+ for (ushort slot = 0; slot < TOKEN_ROW_SLOTS; ++slot) {
+ const uint row = uint(lane_row + slot * ROW_STRIDE);
+ const uint token = token_base + token_tile_base + row;
+ const bool valid = row < valid_token_rows;
+ row_beta[slot] = valid ? beta[token * num_v_heads + hv_idx] : 0.0f;
+ row_exp_g[slot] = valid ? fast::exp(g_head[token]) : 0.0f;
+ }
+
const device T* value_input_tile =
in_proj + (token_base + token_tile_base) * total_proj_dim + 2 * key_dim + hv_idx * head_v_dim + value_base;
residual_acc.map_coords(lane, [&](short row, short col, float state_projection) {
if (uint(row) >= valid_token_rows) {
return 0.0f;
}
- const uint token = token_base + token_tile_base + uint(row);
- const float beta_t = beta[token * num_v_heads + hv_idx];
- const float g_t = g_head[token];
const float value_input = float(value_input_tile[uint(row) * total_proj_dim + uint(col)]);
- return beta_t * (value_input - fast::exp(g_t) * state_projection);
+ return row_beta[row / ROW_STRIDE] * (value_input - row_exp_g[row / ROW_STRIDE] * state_projection);
});
residual_acc.store(lane, residual_new_value_tile + token_tile_base * VT, int(VT));
}
@@ -209,13 +222,7 @@ KERNEL(DeltaNetChunkedOutputAndState)(
const uint valid_token_rows =
token_tile_base < valid_tokens ? min(uint(TOKEN_TILE), valid_tokens - token_tile_base) : 0u;
- output_acc.map_coords(lane, [&](short row, short, float value) {
- if (uint(row) >= valid_token_rows) {
- return 0.0f;
- }
- const uint token = token_base + token_tile_base + uint(row);
- return value * fast::exp(g_head[token]);
- });
+ output_acc.map_coords(lane, [&](short row, short, float value) { return value * row_exp_g[row / ROW_STRIDE]; });
const uint qk_tile_base =
chunk_head_base * OUTPUT_STATE_CHUNK * OUTPUT_STATE_CHUNK + token_tile_base * OUTPUT_STATE_CHUNK;
@@ -286,14 +293,16 @@ KERNEL(DeltaNetChunkedOutputAndState)(
short(OUTPUT_STATE_KEY_TILE)
)
);
- key_frag.map_coords(lane, [&](short row, short, float value) {
- if (uint(row) >= valid_source_tokens) {
- return 0.0f;
- }
- const uint token = token_base + source_token_block_start + uint(row);
- // beta is already folded into Vnew through R.
- return value * fast::exp(g_last - g_head[token]);
- });
+ float row_decay[ROW_SLOTS];
+ METAL_PRAGMA_UNROLL
+ for (ushort slot = 0; slot < ROW_SLOTS; ++slot) {
+ const uint row = uint(lane_row + slot * ROW_STRIDE);
+ row_decay[slot] = row < valid_source_tokens
+ ? fast::exp(g_last - g_head[token_base + source_token_block_start + row])
+ : 0.0f;
+ }
+ // beta is already folded into Vnew through R.
+ key_frag.map_coords(lane, [&](short row, short, float value) { return value * row_decay[row / ROW_STRIDE]; });
fragment_mma(state_update_acc, new_value_frag, key_frag);
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/precompute.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/precompute.metal
new file mode 100644
index 000000000..df8e83249
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/precompute.metal
@@ -0,0 +1,261 @@
+#include <metal_stdlib>
+#include "../../common/defines.h"
+#include "../../common/dsl.h"
+#include "../../common/thread_context.h"
+#include "../common/gram.h"
+#include "../common/tri_inv.h"
+
+using namespace metal;
+using namespace uzu::matmul;
+
+#define BLOCK_SIZE 16u
+#define COL_TILE 32u
+#define MAX_BLOCKS 4u
+#define A_TILE_COUNT 6u
+#define MAX_GROUPS 4u
+#define PRECOMPUTE_THREADS 128
+
+// q_norm: [suffix_len, key_dim].
+// k_norm: [suffix_len, key_dim].
+// log_decay: [suffix_len, num_v_heads].
+// beta: [suffix_len, num_v_heads].
+// g_out: [num_v_heads, suffix_len] chunk-local cumulative log decay.
+// qk_scaled_out: [chunks, num_v_heads, CHUNK, CHUNK] decay-scaled causal Q K^T.
+// t_out: [chunks, num_v_heads, CHUNK, CHUNK] dense T = (I + A)^-1, bf16.
+//
+// One threadgroup owns one (chunk, k-head); simdgroup b owns block row b. Per grouped v-head:
+// 1. g = prefix sum of log_decay.
+// 2. qk_scaled = exp(g_r - g_c) * Q K^T for c <= r.
+// 3. A = beta_r * exp(g_r - g_c) * K K^T for c < r.
+// 4. T = (I + A)^-1 by block forward substitution.
+template <uint HEAD_K_DIM, uint CHUNK_SIZE>
+VARIANTS(HEAD_K_DIM, 128)
+VARIANTS(CHUNK_SIZE, 64)
+KERNEL(DeltaNetChunkedPrecompute)(
+ device const float* q_norm,
+ device const float* k_norm,
+ device const float* log_decay,
+ device const float* beta,
+ device float* g_out,
+ device float* qk_scaled_out,
+ device bfloat* t_out,
+ constant const uint& num_v_heads,
+ constant const uint& num_k_heads,
+ constant const uint& key_dim,
+ constant const uint& suffix_len,
+ threadgroup float a_tiles[A_TILE_COUNT * BLOCK_SIZE * COL_TILE],
+ threadgroup float diag_inverses[MAX_BLOCKS * BLOCK_SIZE * BLOCK_SIZE],
+ threadgroup float g_tiles[MAX_GROUPS * CHUNK_SIZE],
+ threadgroup float beta_tiles[MAX_GROUPS * CHUNK_SIZE],
+ const ThreadContext thread_context,
+ const uint chunk_idx GROUPS(suffix_len.div_ceil(CHUNK_SIZE)),
+ const uint hk_idx GROUPS(num_k_heads),
+ const uint thread_idx THREADS(PRECOMPUTE_THREADS)
+) {
+ using Ops = SimdgroupFragmentOps;
+ constexpr ushort ROW_FRAGMENTS = BLOCK_SIZE / Ops::FRAGMENT_ROWS;
+ constexpr ushort GRAM_COL_FRAGMENTS = COL_TILE / Ops::FRAGMENT_COLS;
+ using GramAccFragment = Fragment<float, ROW_FRAGMENTS, GRAM_COL_FRAGMENTS, Ops>;
+ using GramLeftFragment = OperandFragment<float, ROW_FRAGMENTS, 1, Ops>;
+ using GramRightFragment = OperandFragment<float, 1, GRAM_COL_FRAGMENTS, Ops, ReadTranspose>;
+
+ using TileFragment = Fragment<float, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;
+ using MatrixFragment = OperandFragment<float, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;
+ using ValueFragment = OperandFragment<bfloat, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;
+
+ static_assert(HEAD_K_DIM % Ops::FRAGMENT_ROWS == 0, "HEAD_K_DIM must align to fragment K");
+ constexpr uint num_blocks = CHUNK_SIZE / BLOCK_SIZE;
+ static_assert(num_blocks == MAX_BLOCKS, "one simdgroup per block row");
+
+ const uint lane = thread_context.simd_lane_id;
+ const uint block_idx = thread_context.simdgroup_index;
+ // First A tile of block row b in threadgroup memory: rows hold b / 2 + 1 tiles.
+ auto tile_base = [](uint b) UZU_ALWAYS_INLINE { return b < 2 ? b : 2 * b - 2; };
+
+ const uint chunk_token_base = chunk_idx * CHUNK_SIZE;
+ const uint valid_tokens = min(uint(CHUNK_SIZE), suffix_len - chunk_token_base);
+ const uint groups_per_head = num_v_heads / num_k_heads;
+ const uint row_base = block_idx * BLOCK_SIZE;
+ const uint row_token_base = chunk_token_base + row_base;
+ const uint valid_rows = valid_tokens > row_base ? min(BLOCK_SIZE, valid_tokens - row_base) : 0u;
+ const uint num_col_tiles = block_idx / 2 + 1;
+
+ // Simdgroup b loads log_decay and beta of grouped v-head b ahead of the MMA loop.
+ constexpr uint decay_passes = CHUNK_SIZE / METAL_SIMD_SIZE;
+ const bool has_group = block_idx < groups_per_head;
+ const uint group_hv_idx = hk_idx * groups_per_head + block_idx;
+ float decay_values[decay_passes];
+ float beta_values[decay_passes];
+ METAL_PRAGMA_UNROLL
+ for (uint pass = 0; pass < decay_passes; ++pass) {
+ const uint local = pass * METAL_SIMD_SIZE + lane;
+ const bool valid = has_group && local < valid_tokens;
+ const uint offset = (chunk_token_base + local) * num_v_heads + group_hv_idx;
+ decay_values[pass] = valid ? log_decay[offset] : 0.0f;
+ beta_values[pass] = valid ? beta[offset] : 0.0f;
+ }
+
+ // K K^T and Q K^T of this block row up to the diagonal; tail tiles stay zero.
+ GramAccFragment kk_acc[2];
+ GramAccFragment qk_acc[2];
+ METAL_PRAGMA_UNROLL
+ for (uint c = 0; c < 2; ++c) {
+ kk_acc[c].clear();
+ qk_acc[c].clear();
+ }
+ if (valid_rows != 0) {
+ for (uint k_block_start = 0; k_block_start < HEAD_K_DIM; k_block_start += Ops::FRAGMENT_ROWS) {
+ const device float* k_rows = k_norm + row_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;
+ const device float* q_rows = q_norm + row_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;
+ METAL_PRAGMA_UNROLL
+ for (uint c = 0; c < 2; ++c) {
+ const uint col_base = c * COL_TILE;
+ const uint col_token_base = chunk_token_base + col_base;
+ const uint tile_cols = min(uint(COL_TILE), uint(CHUNK_SIZE - col_base));
+ const uint valid_cols = valid_tokens > col_base ? min(tile_cols, valid_tokens - col_base) : 0u;
+ if (c < num_col_tiles && valid_cols != 0) {
+ const device float* k_cols = k_norm + col_token_base * key_dim + hk_idx * HEAD_K_DIM + k_block_start;
+ accumulate_dual_gram_tile<GramAccFragment, GramLeftFragment, GramRightFragment>(
+ kk_acc[c],
+ qk_acc[c],
+ k_rows,
+ q_rows,
+ k_cols,
+ int(key_dim),
+ ushort(valid_rows),
+ ushort(valid_cols),
+ Ops::FRAGMENT_ROWS,
+ valid_rows == BLOCK_SIZE,
+ valid_cols == tile_cols,
+ ushort(lane)
+ );
+ }
+ }
+ }
+ }
+
+ if (has_group) {
+ float carry = 0.0f;
+ METAL_PRAGMA_UNROLL
+ for (uint pass = 0; pass < decay_passes; ++pass) {
+ const uint local = pass * METAL_SIMD_SIZE + lane;
+ const float prefix = carry + simd_prefix_inclusive_sum(decay_values[pass]);
+ g_tiles[block_idx * CHUNK_SIZE + local] = prefix;
+ beta_tiles[block_idx * CHUNK_SIZE + local] = beta_values[pass];
+ if (local < valid_tokens) {
+ g_out[group_hv_idx * suffix_len + chunk_token_base + local] = prefix;
+ }
+ carry += simd_sum(decay_values[pass]);
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ threadgroup float* own_tiles = a_tiles + tile_base(block_idx) * (BLOCK_SIZE * COL_TILE);
+
+ for (uint group = 0; group < groups_per_head; ++group) {
+ const threadgroup float* g_tile = g_tiles + group * CHUNK_SIZE;
+ const uint head_base = chunk_idx * num_v_heads + hk_idx * groups_per_head + group;
+ METAL_PRAGMA_UNROLL
+ for (uint c = 0; c < 2; ++c) {
+ if (c < num_col_tiles) {
+ const uint col_base = c * COL_TILE;
+ const short2 tile_dims = short2(min(uint(COL_TILE), uint(CHUNK_SIZE - col_base)), BLOCK_SIZE);
+ GramAccFragment scaled = qk_acc[c];
+ scaled.map_coords(lane, [&](short r, short col_in_tile, float value) UZU_ALWAYS_INLINE {
+ const uint row = row_base + uint(r);
+ const uint col = col_base + uint(col_in_tile);
+ if (row >= valid_tokens || col >= valid_tokens || col > row) {
+ return 0.0f;
+ }
+ return value * fast::exp(g_tile[row] - g_tile[col]);
+ });
+ scaled.store_safe(
+ lane,
+ qk_scaled_out + head_base * CHUNK_SIZE * CHUNK_SIZE + row_base * CHUNK_SIZE + col_base,
+ CHUNK_SIZE,
+ tile_dims
+ );
+ }
+ }
+ }
+
+ for (uint group = 0; group < groups_per_head; ++group) {
+ const threadgroup float* g_tile = g_tiles + group * CHUNK_SIZE;
+ const threadgroup float* beta_tile = beta_tiles + group * CHUNK_SIZE;
+ const uint head_base = chunk_idx * num_v_heads + hk_idx * groups_per_head + group;
+
+ METAL_PRAGMA_UNROLL
+ for (uint c = 0; c < 2; ++c) {
+ if (c < num_col_tiles) {
+ const uint col_base = c * COL_TILE;
+ GramAccFragment a = kk_acc[c];
+ a.map_coords(lane, [&](short r, short col_in_tile, float value) UZU_ALWAYS_INLINE {
+ const uint row = row_base + uint(r);
+ const uint col = col_base + uint(col_in_tile);
+ if (row >= valid_tokens || col >= valid_tokens || col >= row) {
+ return 0.0f;
+ }
+ return beta_tile[row] * fast::exp(g_tile[row] - g_tile[col]) * value;
+ });
+ a.store(lane, own_tiles + c * (BLOCK_SIZE * COL_TILE), COL_TILE);
+ }
+ }
+ simdgroup_barrier(mem_flags::mem_threadgroup);
+ invert_lower_triangular_block<BLOCK_SIZE, COL_TILE>(
+ diag_inverses + block_idx * (BLOCK_SIZE * BLOCK_SIZE),
+ own_tiles + (block_idx / 2) * (BLOCK_SIZE * COL_TILE) + (block_idx % 2) * BLOCK_SIZE,
+ valid_rows,
+ lane
+ );
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // Solve this simdgroup's 16-column slice of T; blocks above the slice are zero, solved blocks stay in registers.
+ const uint slice_base = block_idx * BLOCK_SIZE;
+ device bfloat* t_head = t_out + head_base * CHUNK_SIZE * CHUNK_SIZE + slice_base;
+ ValueFragment t_blocks[num_blocks];
+ METAL_PRAGMA_UNROLL
+ for (uint b = 0; b < num_blocks; ++b) {
+ TileFragment t_block;
+ t_block.clear();
+ if (b >= block_idx) {
+ const threadgroup float* b_tiles = a_tiles + tile_base(b) * (BLOCK_SIZE * COL_TILE);
+ TileFragment acc_t;
+ acc_t.map_coords(lane, [&](short local_row, short local_col, float) UZU_ALWAYS_INLINE {
+ return b * BLOCK_SIZE + uint(local_row) == slice_base + uint(local_col) ? 1.0f : 0.0f;
+ });
+ METAL_PRAGMA_UNROLL
+ for (uint prev_block_idx = 0; prev_block_idx < b; ++prev_block_idx) {
+ if (prev_block_idx >= block_idx) {
+ MatrixFragment a_frag;
+ a_frag.load_from(
+ lane,
+ fragment_source(
+ b_tiles + (prev_block_idx / 2) * (BLOCK_SIZE * COL_TILE) + (prev_block_idx % 2) * BLOCK_SIZE,
+ COL_TILE
+ )
+ );
+ a_frag.map([](float value) UZU_ALWAYS_INLINE { return -value; });
+ fragment_mma(acc_t, a_frag, t_blocks[prev_block_idx]);
+ }
+ }
+ MatrixFragment inv_frag;
+ inv_frag.load_from(
+ lane,
+ fragment_source(
+ static_cast<const threadgroup float*>(diag_inverses + b * (BLOCK_SIZE * BLOCK_SIZE)),
+ BLOCK_SIZE
+ )
+ );
+ fragment_mma(t_block, inv_frag, acc_t);
+ }
+ t_block.store(lane, t_head + b * BLOCK_SIZE * CHUNK_SIZE, CHUNK_SIZE);
+ METAL_PRAGMA_UNROLL
+ for (ushort e = 0; e < ValueFragment::ELEMENTS_PER_FRAGMENT; ++e) {
+ t_blocks[b].elements()[e] = bfloat(t_block.elements()[e]);
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/common/tri_inv.h b/crates/uzu-engine/src/backends/metal/kernel/gdn/common/tri_inv.h
index 796fcbe24..947fc5922 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/common/tri_inv.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/common/tri_inv.h
@@ -5,9 +5,9 @@
using namespace metal;
-template <uint BLOCK>
+template <uint BLOCK, uint A_STRIDE = BLOCK, typename OutPtr>
METAL_FUNC void invert_lower_triangular_block(
- device float* a_inv_block,
+ OutPtr a_inv_block,
threadgroup const float* a_tile,
const uint block_size,
const uint lane
@@ -27,7 +27,7 @@ METAL_FUNC void invert_lower_triangular_block(
METAL_PRAGMA_UNROLL
for (uint prev_row = 0; prev_row < BLOCK; prev_row++) {
if (prev_row < row) {
- acc += a_tile[row * BLOCK + prev_row] * inverse_col[prev_row];
+ acc += a_tile[row * A_STRIDE + prev_row] * inverse_col[prev_row];
}
}
inverse_col[row] = -acc;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_pack.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_pack.metal
index 9e076f61f..4834321a7 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_pack.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_pack.metal
@@ -1,27 +1,37 @@
#include <metal_stdlib>
#include "../common/dsl.h"
-template <typename StateT, typename InputT>
-VARIANTS(StateT, float, bfloat)
-VARIANTS(InputT, float, bfloat)
-PUBLIC KERNEL(Conv1dPack)(
- device const StateT* state_in,
- device const InputT* x,
- device StateT* padded,
- constant const uint& state_stride,
- constant const uint& row_stride,
+using namespace metal;
+
+// Saves the KERNEL_SIZE - 1 rows before each token tile, which the in-place DeltaNetConvScan overwrites:
+// packed[tile, tap] = row (tile * tile_tokens - TAPS + tap) of [conv state ++ x].
+//
+// state_in: [conv_dim, KERNEL_SIZE - 1]
+// x: [suffix_len, row_stride]
+// packed: [num_tiles, KERNEL_SIZE - 1, conv_dim]
+template <typename T, uint KERNEL_SIZE>
+VARIANTS(T, float, bfloat)
+VARIANTS(KERNEL_SIZE, 4)
+PUBLIC KERNEL(DeltaNetConvPack)(
+ device const float* state_in,
+ device const T* x,
+ device float* packed,
constant const uint& suffix_len,
- constant const uint& num_channels,
- const uint channel_idx AXIS(num_channels, 32),
- const uint row_idx AXIS(state_stride + suffix_len, 1)
+ constant const uint& tile_tokens,
+ constant const uint& row_stride,
+ constant const uint& conv_dim,
+ const uint channel_idx AXIS(conv_dim, 128),
+ const uint tile_idx AXIS(suffix_len.div_ceil(tile_tokens), 1)
) {
- const uint padded_index = row_idx * row_stride + channel_idx;
- if (row_idx < state_stride) {
- const uint state_index = channel_idx * state_stride + row_idx;
- padded[padded_index] = state_in[state_index];
- } else {
- const uint token = row_idx - state_stride;
- const uint x_index = token * row_stride + channel_idx;
- padded[padded_index] = static_cast<StateT>(x[x_index]);
+ constexpr uint TAPS = KERNEL_SIZE - 1;
+ const size_t first_row = size_t(tile_idx) * tile_tokens;
+ for (uint tap = 0; tap < TAPS; ++tap) {
+ float value;
+ if (tile_idx == 0) {
+ value = state_in[channel_idx * TAPS + tap];
+ } else {
+ value = float(x[(first_row - TAPS + tap) * row_stride + channel_idx]);
+ }
+ packed[(tile_idx * TAPS + tap) * size_t(conv_dim) + channel_idx] = value;
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_scan.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_scan.metal
index 44aebdbe0..55b84af7d 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_scan.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/conv_scan.metal
@@ -4,51 +4,124 @@
using namespace metal;
-// Multi-token causal conv1d with SiLU for DeltaNet.
-// Reads from conv_padded (Conv1dPack output), writes conv'd+SiLU'd
-// values back to in_proj with stride out_stride, updates conv state.
+// Causal conv1d + SiLU, in place on in_proj. A thread owns CHANNELS channels of one token tile; its first window
+// comes from DeltaNetConvPack, so tiles are independent. The last tile writes state_out.
//
-// conv_padded: [state_stride + suffix_len, row_stride]
-// conv_weight: [conv_dim, kernel_size]
-// bias: [conv_dim] (optional)
-// in_proj: [suffix_len, out_stride] — first conv_dim channels overwritten
-// state_out: [conv_dim, state_stride]
+// packed: [num_tiles, KERNEL_SIZE - 1, conv_dim]
+// conv_weight: [conv_dim, KERNEL_SIZE]
+// bias: [conv_dim] (optional)
+// in_proj: [suffix_len, out_stride], first conv_dim channels overwritten
+// state_out: [conv_dim, KERNEL_SIZE - 1]
+// conv_dim and out_stride must be multiples of CHANNELS.
+
+constant constexpr uint CHANNELS = 8;
+constant constexpr uint VECTOR_WIDTH = 4;
+constant constexpr uint ROWS_IN_FLIGHT = 4;
template <typename T>
+METAL_FUNC void load_channels(const device T* source, thread float (&values)[CHANNELS]) {
+ const device vec<T, VECTOR_WIDTH>* vectors = reinterpret_cast<const device vec<T, VECTOR_WIDTH>*>(source);
+ for (uint vector_idx = 0; vector_idx < CHANNELS / VECTOR_WIDTH; ++vector_idx) {
+ const vec<float, VECTOR_WIDTH> chunk = vec<float, VECTOR_WIDTH>(vectors[vector_idx]);
+ for (uint i = 0; i < VECTOR_WIDTH; ++i) {
+ values[vector_idx * VECTOR_WIDTH + i] = chunk[i];
+ }
+ }
+}
+
+template <typename T>
+METAL_FUNC void store_channels(device T* destination, const thread float (&values)[CHANNELS]) {
+ device vec<T, VECTOR_WIDTH>* vectors = reinterpret_cast<device vec<T, VECTOR_WIDTH>*>(destination);
+ for (uint vector_idx = 0; vector_idx < CHANNELS / VECTOR_WIDTH; ++vector_idx) {
+ vec<float, VECTOR_WIDTH> chunk;
+ for (uint i = 0; i < VECTOR_WIDTH; ++i) {
+ chunk[i] = values[vector_idx * VECTOR_WIDTH + i];
+ }
+ vectors[vector_idx] = vec<T, VECTOR_WIDTH>(chunk);
+ }
+}
+
+template <typename T, uint KERNEL_SIZE>
VARIANTS(T, float, bfloat)
+VARIANTS(KERNEL_SIZE, 4)
PUBLIC KERNEL(DeltaNetConvScan)(
- device const float* conv_padded,
+ device const float* packed,
device const float* conv_weight,
device const float* bias OPTIONAL(has_bias),
device T* in_proj,
device float* state_out,
constant const uint& suffix_len,
- constant const uint& kernel_size,
- constant const uint& row_stride,
- constant const uint& state_stride,
+ constant const uint& tile_tokens,
constant const uint& conv_dim,
constant const uint& out_stride,
const bool has_bias SPECIALIZE,
- const uint token_idx AXIS(suffix_len + kernel_size - 1, 1),
- const uint channel_idx AXIS(conv_dim, 32)
+ const uint channel_group AXIS(conv_dim / 8, 32),
+ const uint tile_idx AXIS(suffix_len.div_ceil(tile_tokens), 4)
) {
- if (token_idx < suffix_len) {
- const device float* weight_row = conv_weight + channel_idx * kernel_size;
+ constexpr uint TAPS = KERNEL_SIZE - 1;
+ const uint first_channel = channel_group * CHANNELS;
- float acc = has_bias ? float(bias[channel_idx]) : 0.0f;
+ float weights[CHANNELS * KERNEL_SIZE];
+ const device float4* weight_vectors =
+ reinterpret_cast<const device float4*>(conv_weight + first_channel * KERNEL_SIZE);
+ for (uint i = 0; i < CHANNELS * KERNEL_SIZE / 4; ++i) {
+ const float4 chunk = weight_vectors[i];
+ for (uint lane = 0; lane < 4; ++lane) {
+ weights[i * 4 + lane] = chunk[lane];
+ }
+ }
+ float biases[CHANNELS];
+ for (uint j = 0; j < CHANNELS; ++j) {
+ biases[j] = has_bias ? bias[first_channel + j] : 0.0f;
+ }
- for (uint tap = 0; tap < kernel_size; ++tap) {
- const uint padded_row = token_idx + tap;
- const uint padded_index = padded_row * row_stride + channel_idx;
- acc += float(weight_row[tap]) * conv_padded[padded_index];
+ float window[TAPS][CHANNELS];
+ for (uint tap = 0; tap < TAPS; ++tap) {
+ load_channels(packed + (tile_idx * TAPS + tap) * conv_dim + first_channel, window[tap]);
+ }
+
+ const uint tile_begin = tile_idx * tile_tokens;
+ const uint tile_end = min(tile_begin + tile_tokens, suffix_len);
+ for (uint token = tile_begin; token < tile_end; token += ROWS_IN_FLIGHT) {
+ float rows[ROWS_IN_FLIGHT][CHANNELS];
+ for (uint r = 0; r < ROWS_IN_FLIGHT; ++r) {
+ if (token + r < tile_end) {
+ load_channels(in_proj + (token + r) * size_t(out_stride) + first_channel, rows[r]);
+ }
+ }
+ for (uint r = 0; r < ROWS_IN_FLIGHT; ++r) {
+ if (token + r < tile_end) {
+ float outputs[CHANNELS];
+ for (uint j = 0; j < CHANNELS; ++j) {
+ float acc = biases[j];
+ for (uint tap = 0; tap < TAPS; ++tap) {
+ acc = fma(weights[j * KERNEL_SIZE + tap], window[tap][j], acc);
+ }
+ acc = fma(weights[j * KERNEL_SIZE + TAPS], rows[r][j], acc);
+ outputs[j] = activate_silu(acc);
+ }
+ store_channels(in_proj + (token + r) * size_t(out_stride) + first_channel, outputs);
+ for (uint tap = 0; tap + 1 < TAPS; ++tap) {
+ for (uint j = 0; j < CHANNELS; ++j) {
+ window[tap][j] = window[tap + 1][j];
+ }
+ }
+ for (uint j = 0; j < CHANNELS; ++j) {
+ window[TAPS - 1][j] = rows[r][j];
+ }
+ }
}
+ }
- const size_t dst = size_t(token_idx) * out_stride + channel_idx;
- in_proj[dst] = static_cast<T>(activate_silu(acc));
- } else {
- const uint tap = token_idx - suffix_len;
- const size_t padded_index = size_t(token_idx) * row_stride + channel_idx;
- const uint state_offset = channel_idx * state_stride;
- state_out[state_offset + tap] = conv_padded[padded_index];
+ if (tile_end == suffix_len) {
+ device float4* state_vectors = reinterpret_cast<device float4*>(state_out + first_channel * TAPS);
+ for (uint i = 0; i < CHANNELS * TAPS / 4; ++i) {
+ float4 chunk;
+ for (uint lane = 0; lane < 4; ++lane) {
+ const uint flat = i * 4 + lane;
+ chunk[lane] = window[flat % TAPS][flat / TAPS];
+ }
+ state_vectors[i] = chunk;
+ }
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
index f7fc4f721..66efc68e6 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
@@ -5,7 +5,7 @@ use crate::{
backends::common::{
Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding, Context, Kernels,
kernel::{
- Conv1dPackKernel, ConvTreeScanKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel,
+ ConvTreeScanKernel, DeltaNetConvPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel,
DeltaNetNormGateKernel, DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,
StateAdvanceKernel,
delta_net_chunked_prefill::{DeltaNetChunkedPrefill, DeltaNetChunkedPrefillArgs},
@@ -30,6 +30,9 @@ use crate::{
pub(crate) mod tree_verify;
const INNER_DATA_TYPE: DataType = DataType::F32;
+// Prefill conv: up to CONV_TILES token tiles of at least CONV_MIN_TILE_TOKENS (correctness needs >= KERNEL_SIZE - 1).
+const CONV_TILES: u32 = 16;
+const CONV_MIN_TILE_TOKENS: u32 = 16;
enum DeltaNetSuffixStatus<B: Backend> {
Flat {
@@ -127,7 +130,7 @@ pub struct DeltaNet<B: Backend> {
conv_weight: B::GlobalBuffer,
conv_bias: Option<B::GlobalBuffer>,
conv_update: <B::Kernels as Kernels>::DeltaNetConvUpdateKernel,
- conv_pack: <B::Kernels as Kernels>::Conv1dPackKernel,
+ conv_pack: <B::Kernels as Kernels>::DeltaNetConvPackKernel,
conv_scan: <B::Kernels as Kernels>::DeltaNetConvScanKernel,
conv_tree_scan: <B::Kernels as Kernels>::ConvTreeScanKernel,
a_log: B::GlobalBuffer,
@@ -164,9 +167,9 @@ impl<B: Backend> DeltaNet<B> {
parameter_tree: &ParameterTree<B>,
context: &B::Context,
) -> Result<(Self, Option<B::GlobalBuffer>), DeltaNetNewError<B>> {
- if config.kernel_size < 2 {
+ if config.kernel_size != 4 {
return Err(DeltaNetNewError::UnsupportedConfiguration(format!(
- "kernel_size must be >= 2, got {}",
+ "kernel_size must be 4, got {}",
config.kernel_size
)));
}
@@ -187,6 +190,12 @@ impl<B: Backend> DeltaNet<B> {
let value_dim = config.num_heads * config.value_head_dim;
let conv_dim = key_dim * 2 + value_dim;
let total_proj_dim = conv_dim + value_dim + config.num_heads + config.num_heads;
+ // The prefill conv scan moves 8 channels per thread as vec4 pairs: the in_proj row stride must be a multiple of 8.
+ if !total_proj_dim.is_multiple_of(8) {
+ return Err(DeltaNetNewError::UnsupportedConfiguration(format!(
+ "in_proj width must be a multiple of 8, got {total_proj_dim}"
+ )));
+ }
let (in_projection, in_projection_input_hadamard_factors) =
<dyn Linear<B>>::new_with_input_rht_mixed_precision(
@@ -212,11 +221,16 @@ impl<B: Backend> DeltaNet<B> {
let conv_update =
<B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, outer_data_type, conv_config.has_biases)
.map_err(DeltaNetNewError::Backend)?;
- let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, INNER_DATA_TYPE, outer_data_type)
- .map_err(DeltaNetNewError::Backend)?;
- let conv_scan =
- <B::Kernels as Kernels>::DeltaNetConvScanKernel::new(context, outer_data_type, conv_config.has_biases)
+ let conv_pack =
+ <B::Kernels as Kernels>::DeltaNetConvPackKernel::new(context, outer_data_type, config.kernel_size)
.map_err(DeltaNetNewError::Backend)?;
+ let conv_scan = <B::Kernels as Kernels>::DeltaNetConvScanKernel::new(
+ context,
+ outer_data_type,
+ config.kernel_size,
+ conv_config.has_biases,
+ )
+ .map_err(DeltaNetNewError::Backend)?;
let conv_tree_scan = <B::Kernels as Kernels>::ConvTreeScanKernel::new(
context,
outer_data_type,
@@ -274,8 +288,13 @@ impl<B: Backend> DeltaNet<B> {
None
};
- let chunked = <B::Kernels as Kernels>::DeltaNetChunkedPrefill::new(context, outer_data_type, config.head_dim)
- .map_err(DeltaNetNewError::Backend)?;
+ let chunked = <B::Kernels as Kernels>::DeltaNetChunkedPrefill::new(
+ context,
+ outer_data_type,
+ config.head_dim,
+ config.num_heads / config.num_groups,
+ )
+ .map_err(DeltaNetNewError::Backend)?;
let out_projection = <dyn Linear<B>>::new_mixed_precision(
value_dim,
@@ -534,32 +553,29 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
command_buffer,
);
} else {
- let mut padded = command_buffer
- .allocate_scratch_for_shape(
- &[batch_dim.size() + self.kernel_size - 1, self.total_proj_dim],
- INNER_DATA_TYPE,
- )
+ let conv_tile_tokens = batch_dim.size().div_ceil(CONV_TILES).max(CONV_MIN_TILE_TOKENS);
+ debug_assert!(conv_tile_tokens >= self.kernel_size - 1);
+ let mut packed = command_buffer
+ .allocate_scratch_for_shape(&[CONV_TILES, self.kernel_size - 1, self.conv_dim], INNER_DATA_TYPE)
.map_err(MixerEncodeError::Backend)?;
self.conv_pack.encode(
&state.conv_state,
&in_projected,
- &mut padded,
- self.kernel_size - 1,
- self.total_proj_dim,
+ &mut packed,
batch_dim.size(),
+ conv_tile_tokens,
+ self.total_proj_dim,
self.conv_dim,
command_buffer,
);
self.conv_scan.encode(
- &padded,
+ &packed,
&self.conv_weight,
self.conv_bias.as_ref(),
&mut in_projected,
&mut state.conv_state,
batch_dim.size(),
- self.kernel_size,
- self.total_proj_dim,
- self.kernel_size - 1,
+ conv_tile_tokens,
self.conv_dim,
self.total_proj_dim,
command_buffer,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/conv_pack_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/conv_pack_test.rs
index 2b0d2bc27..19e17b92b 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/conv_pack_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/conv_pack_test.rs
@@ -9,133 +9,110 @@ use crate::{
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- kernel::Conv1dPackKernel,
+ kernel::DeltaNetConvPackKernel,
},
cpu::Cpu,
},
- data_type::{DataType, DataTypeRepr},
+ data_type::DataTypeRepr,
tests::{
assert::assert_eq_float,
helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
},
};
+const KERNEL_SIZE: u32 = 4;
+const TAPS: u32 = KERNEL_SIZE - 1;
+
struct Input<T: DataTypeRepr + Float> {
- state_in: Box<[T]>,
+ state_in: Box<[f32]>,
x: Box<[T]>,
- state_stride: u32,
- row_stride: u32,
suffix_len: u32,
- num_channels: u32,
+ tile_tokens: u32,
+ row_stride: u32,
+ conv_dim: u32,
}
fn get_input<T: DataTypeRepr + Float>(
- state_stride: u32,
- row_stride: u32,
suffix_len: u32,
- num_channels: u32,
+ tile_tokens: u32,
+ row_stride: u32,
+ conv_dim: u32,
) -> Input<T> {
- let state_size = num_channels as usize * state_stride as usize;
- let x_size = suffix_len as usize * row_stride as usize;
-
- let state_in: Vec<T> = (0..state_size).map(|i| T::from(0.1 * (i as f64 + 1.0)).unwrap()).collect();
- let x: Vec<T> = (0..x_size).map(|i| T::from(-0.05 * (i as f64 + 1.0)).unwrap()).collect();
+ let state_in: Vec<f32> = (0..conv_dim * TAPS).map(|i| 0.1 * (i as f32 + 1.0)).collect();
+ let x: Vec<T> = (0..suffix_len * row_stride).map(|i| T::from(-0.05 * (i as f64 + 1.0)).unwrap()).collect();
Input {
state_in: state_in.into_boxed_slice(),
x: x.into_boxed_slice(),
- state_stride,
- row_stride,
suffix_len,
- num_channels,
+ tile_tokens,
+ row_stride,
+ conv_dim,
}
}
-fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Vec<T> {
+fn get_output<B: Backend, T: DataTypeRepr + Float>(input: &Input<T>) -> Vec<f32> {
let context = block_on(B::Context::new()).expect("Failed to create Context");
- let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, T::data_type(), T::data_type())
- .expect("Failed to create Conv1dPackKernel");
-
- let total_rows = input.state_stride as usize + input.suffix_len as usize;
- let padded_size = total_rows * input.row_stride as usize;
+ let kernel =
+ <<B as Backend>::Kernels as Kernels>::DeltaNetConvPackKernel::new(&context, T::data_type(), KERNEL_SIZE)
+ .expect("Failed to create DeltaNetConvPackKernel");
- let state = create_buffer_with_data::<B, T>(&context, &input.state_in);
+ let num_tiles = input.suffix_len.div_ceil(input.tile_tokens);
+ let state = create_buffer_with_data::<B, f32>(&context, &input.state_in);
let x = create_buffer_with_data::<B, T>(&context, &input.x);
- let mut padded = create_buffer::<B, T>(&context, padded_size);
+ let mut packed = create_buffer::<B, f32>(&context, (num_tiles * TAPS * input.conv_dim) as usize);
let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
kernel.encode(
&state,
&x,
- &mut padded,
- input.state_stride,
- input.row_stride,
+ &mut packed,
input.suffix_len,
- input.num_channels,
+ input.tile_tokens,
+ input.row_stride,
+ input.conv_dim,
&mut command_buffer,
);
block_on(command_buffer.end_encoding().submit().wait_until_completed()).expect("Failed to wait command buffer");
- buffer_to_vec(&padded)
-}
-
-fn get_test_data<T: DataTypeRepr + Float>(
- state_stride: u32,
- row_stride: u32,
- suffix_len: u32,
- num_channels: u32,
-) -> (Input<T>, Vec<T>) {
- let input = get_input::<T>(state_stride, row_stride, suffix_len, num_channels);
- let expected = get_output::<Cpu, T>(&input);
- (input, expected)
+ buffer_to_vec(&packed)
}
fn test_internal<T: DataTypeRepr + Float + Debug + Display>(
- input: &Input<T>,
- expected: &[T],
+ suffix_len: u32,
+ tile_tokens: u32,
+ row_stride: u32,
+ conv_dim: u32,
label: &str,
) {
- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
- 1e-3f32
- } else {
- 1e-6
- };
+ let input = get_input::<T>(suffix_len, tile_tokens, row_stride, conv_dim);
+ let expected = get_output::<Cpu, T>(&input);
for_each_non_cpu_backend!(|B| {
- let output = get_output::<B, T>(input);
- let msg = format!("Conv1dPack {} {} (type={})", std::any::type_name::<B>(), label, std::any::type_name::<T>(),);
- assert_eq_float::<T>(expected, &output, eps, &msg);
+ let output = get_output::<B, T>(&input);
+ let msg =
+ format!("DeltaNetConvPack {} {} (type={})", std::any::type_name::<B>(), label, std::any::type_name::<T>());
+ assert_eq_float::<f32>(&expected, &output, 1e-6, &msg);
});
}
fn test_basic<T: DataTypeRepr + Float + Debug + Display>() {
- // 4 channels, state_stride=3 (kernel_size=4), 2 tokens
- let (input, expected) = get_test_data::<T>(3, 4, 2, 4);
- test_internal(&input, &expected, "basic");
-}
-
-fn test_single_token<T: DataTypeRepr + Float + Debug + Display>() {
- // 8 channels, state_stride=1, 1 token
- let (input, expected) = get_test_data::<T>(1, 8, 1, 8);
- test_internal(&input, &expected, "single_token");
+ // several tiles, partial last tile
+ test_internal::<T>(20, 8, 16, 8, "basic");
}
-fn test_many_tokens<T: DataTypeRepr + Float + Debug + Display>() {
- // Minimal state (kernel_size=2), many tokens
- let (input, expected) = get_test_data::<T>(1, 4, 8, 4);
- test_internal(&input, &expected, "many_tokens");
+fn test_single_tile<T: DataTypeRepr + Float + Debug + Display>() {
+ // a single tile takes only the carried state
+ test_internal::<T>(5, 16, 8, 8, "single_tile");
}
-fn test_large<T: DataTypeRepr + Float + Debug + Display>() {
- // Larger dimensions closer to real usage
- let (input, expected) = get_test_data::<T>(3, 128, 16, 128);
- test_internal(&input, &expected, "large");
+fn test_exact_tiles<T: DataTypeRepr + Float + Debug + Display>() {
+ test_internal::<T>(64, 16, 24, 8, "exact_tiles");
}
fn test_channels_lt_row_stride<T: DataTypeRepr + Float + Debug + Display>() {
- // num_channels < row_stride (padded channels that kernel doesn't write)
- let (input, expected) = get_test_data::<T>(2, 8, 3, 6);
- test_internal(&input, &expected, "channels_lt_row_stride");
+ // conv_dim < row_stride (trailing projection channels are not packed)
+ test_internal::<T>(30, 4, 40, 32, "channels_lt_row_stride");
}
// f32
@@ -145,18 +122,13 @@ fn test_basic_f32() {
}
#[uzu_test]
-fn test_single_token_f32() {
- test_single_token::<f32>();
-}
-
-#[uzu_test]
-fn test_many_tokens_f32() {
- test_many_tokens::<f32>();
+fn test_single_tile_f32() {
+ test_single_tile::<f32>();
}
#[uzu_test]
-fn test_large_f32() {
- test_large::<f32>();
+fn test_exact_tiles_f32() {
+ test_exact_tiles::<f32>();
}
#[uzu_test]
@@ -171,16 +143,16 @@ fn test_basic_bf16() {
}
#[uzu_test]
-fn test_single_token_bf16() {
- test_single_token::<bf16>();
+fn test_single_tile_bf16() {
+ test_single_tile::<bf16>();
}
#[uzu_test]
-fn test_many_tokens_bf16() {
- test_many_tokens::<bf16>();
+fn test_exact_tiles_bf16() {
+ test_exact_tiles::<bf16>();
}
#[uzu_test]
-fn test_large_bf16() {
- test_large::<bf16>();
+fn test_channels_lt_row_stride_bf16() {
+ test_channels_lt_row_stride::<bf16>();
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/delta_net_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/delta_net_test.rs
index 40c0c40a0..20607f3a6 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/delta_net_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/delta_net_test.rs
@@ -5,7 +5,7 @@ use uzu_engine_macros::uzu_test;
#[cfg(backend = "metal")]
use crate::{
backends::{
- common::kernel::{Conv1dPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel},
+ common::kernel::{DeltaNetConvPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel},
metal::Metal,
},
tests::helpers::buffer_prefix_to_vec,
@@ -149,16 +149,17 @@ fn test_delta_net_conv_update_small() {
assert_close(&cpu_state, &gpu_state, 1e-5, 1e-4, "ConvUpdate state");
}
-// DeltaNetConvScan
+// DeltaNetConvPack + DeltaNetConvScan
#[cfg(backend = "metal")]
-#[uzu_test]
-fn test_delta_net_conv_scan() {
+fn test_delta_net_conv_scan_impl<B: Backend>(
+ suffix_len: usize,
+ tile_tokens: usize,
+) {
let conv_dim = 32;
let kernel_size: usize = 4;
let tap_count = kernel_size - 1;
- let suffix_len: usize = 16;
- let extra_dim = 10;
+ let extra_dim = 16;
let total_proj_dim = conv_dim + extra_dim;
let w: Vec<f32> = (0..conv_dim * kernel_size).map(|i| ((i % 11) as f32) * 0.05 - 0.2).collect();
@@ -187,45 +188,46 @@ fn test_delta_net_conv_scan() {
}
let ref_outputs: Vec<f32> = ref_outputs.iter().copied().map(|value| f32::from(bf16::from_f32(value))).collect();
- // Test: Conv1dPack + DeltaNetConvScan on Metal
- let context = block_on(<Metal as Backend>::Context::new()).expect("context");
- let state_array = create_buffer_with_data::<Metal, f32>(&context, &init_state);
- let mut in_proj_array = create_buffer_with_data::<Metal, bf16>(&context, &in_proj);
- let w_array = create_buffer_with_data::<Metal, f32>(&context, &w);
- let b_array = create_buffer_with_data::<Metal, f32>(&context, &b);
+ // Test: DeltaNetConvPack + DeltaNetConvScan on backend B
+ let context = block_on(B::Context::new()).expect("context");
+ let mut state_array = create_buffer_with_data::<B, f32>(&context, &init_state);
+ let mut in_proj_array = create_buffer_with_data::<B, bf16>(&context, &in_proj);
+ let w_array = create_buffer_with_data::<B, f32>(&context, &w);
+ let b_array = create_buffer_with_data::<B, f32>(&context, &b);
- let padded_len = (tap_count + suffix_len) * total_proj_dim;
- let mut padded_array = create_buffer::<Metal, f32>(&context, padded_len);
- let mut state_out_array = create_buffer::<Metal, f32>(&context, conv_dim * tap_count);
+ let num_tiles = suffix_len.div_ceil(tile_tokens);
+ let mut packed_array = create_buffer::<B, f32>(&context, num_tiles * tap_count * conv_dim);
let pack_kernel =
- <<Metal as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, DataType::F32, DataType::BF16)
+ <<B as Backend>::Kernels as Kernels>::DeltaNetConvPackKernel::new(&context, DataType::BF16, kernel_size as u32)
.expect("pack");
- let scan_kernel =
- <<Metal as Backend>::Kernels as Kernels>::DeltaNetConvScanKernel::new(&context, DataType::BF16, true)
- .expect("scan");
+ let scan_kernel = <<B as Backend>::Kernels as Kernels>::DeltaNetConvScanKernel::new(
+ &context,
+ DataType::BF16,
+ kernel_size as u32,
+ true,
+ )
+ .expect("scan");
let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
pack_kernel.encode(
&state_array,
&in_proj_array,
- &mut padded_array,
- tap_count as u32,
- total_proj_dim as u32,
+ &mut packed_array,
suffix_len as u32,
+ tile_tokens as u32,
+ total_proj_dim as u32,
conv_dim as u32,
&mut command_buffer,
);
scan_kernel.encode(
- &padded_array,
+ &packed_array,
&w_array,
Some(&b_array),
&mut in_proj_array,
- &mut state_out_array,
+ &mut state_array,
suffix_len as u32,
- kernel_size as u32,
- total_proj_dim as u32,
- tap_count as u32,
+ tile_tokens as u32,
conv_dim as u32,
total_proj_dim as u32,
&mut command_buffer,
@@ -238,13 +240,27 @@ fn test_delta_net_conv_scan() {
for t in 0..suffix_len {
scan_outputs[t * conv_dim..(t + 1) * conv_dim]
.copy_from_slice(&in_proj_result[t * total_proj_dim..t * total_proj_dim + conv_dim]);
+ // channels past conv_dim are untouched
+ assert_eq!(
+ in_proj_result[t * total_proj_dim + conv_dim..(t + 1) * total_proj_dim],
+ in_proj_f32[t * total_proj_dim + conv_dim..(t + 1) * total_proj_dim],
+ );
}
- let scan_state: Vec<f32> = buffer_to_vec(&state_out_array);
+ let scan_state: Vec<f32> = buffer_to_vec(&state_array);
assert_close(&ref_outputs, &scan_outputs, 1e-4, 1e-3, "ConvScan output");
assert_close(&ref_state, &scan_state, 1e-5, 1e-4, "ConvScan state");
}
+#[cfg(backend = "metal")]
+#[uzu_test]
+fn test_delta_net_conv_scan() {
+ // single partial tile, shorter than the conv window, exact tiles, several tiles with a partial last one
+ for (suffix_len, tile_tokens) in [(2, 32), (96, 32), (100, 32), (70, 8)] {
+ for_each_backend!(|B| test_delta_net_conv_scan_impl::<B>(suffix_len, tile_tokens));
+ }
+}
+
// DeltaNetUpdate (decode)
// The reference is the CPU f32 kernel on the same rounded inputs; a_log, dt_bias, norm weight and state stay f32
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/gdn/chunked_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/gdn/chunked_test.rs
index 31f75df7d..c73cedbb4 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/gdn/chunked_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/gdn/chunked_test.rs
@@ -1,6 +1,7 @@
use futures::executor::block_on;
use half::bf16;
-use num_traits::cast;
+use rand::{RngExt, SeedableRng, rngs::SmallRng};
+use rand_distr::Normal;
use uzu_engine_macros::uzu_test;
use super::*;
@@ -8,193 +9,188 @@ use crate::{
backends::{
common::{
Backend, BufferMut, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- kernel::{DeltaNetNormGateKernel, DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel},
+ kernel::{DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel},
},
metal::Metal,
},
- data_type::DataTypeRepr,
tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data},
};
type BackendKernels<B> = <B as Backend>::Kernels;
-const NUM_V_HEADS: usize = 4;
-const NUM_K_HEADS: usize = 1;
+const NUM_V_HEADS: usize = 6;
+const NUM_K_HEADS: usize = 2;
const HEAD_K_DIM: usize = 128;
const HEAD_V_DIM: usize = 128;
+const KEY_DIM: usize = NUM_K_HEADS * HEAD_K_DIM;
+const VALUE_DIM: usize = NUM_V_HEADS * HEAD_V_DIM;
+const CONV_DIM: usize = 2 * KEY_DIM + VALUE_DIM;
+const TOTAL_PROJ_DIM: usize = CONV_DIM + VALUE_DIM + 2 * NUM_V_HEADS;
+const WARMUP_TOKENS: usize = 1024;
-#[derive(Clone, Copy)]
-enum PrefillMode {
- Recurrent,
- Chunked,
+fn random_in_proj(tokens: usize) -> Vec<bf16> {
+ let mut rng = SmallRng::seed_from_u64(7);
+ let normal = Normal::<f32>::new(0.0, 1.0).unwrap();
+ (0..tokens * TOTAL_PROJ_DIM)
+ .map(|i| {
+ let x: f32 = rng.sample(normal);
+ let column = i % TOTAL_PROJ_DIM;
+ bf16::from_f32(if column < CONV_DIM {
+ x / (1.0 + (-x).exp())
+ } else if column >= TOTAL_PROJ_DIM - NUM_V_HEADS {
+ 0.5 * x
+ } else {
+ x
+ })
+ })
+ .collect()
}
-struct TestCase {
- suffix_len: usize,
- in_proj: Vec<f32>,
- a_log: Vec<f32>,
- dt_bias: Vec<f32>,
- norm_weight: Vec<f32>,
- key_dim: usize,
- value_dim: usize,
- total_proj_dim: usize,
-}
-
-fn test_case(suffix_len: usize) -> TestCase {
- let key_dim = NUM_K_HEADS * HEAD_K_DIM;
- let value_dim = NUM_V_HEADS * HEAD_V_DIM;
- let conv_dim = 2 * key_dim + value_dim;
- let total_proj_dim = conv_dim + value_dim + NUM_V_HEADS + NUM_V_HEADS;
-
- TestCase {
- suffix_len,
- in_proj: (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect(),
- a_log: (0..NUM_V_HEADS).map(|i| -1.5 + (i as f32) * 0.05).collect(),
- dt_bias: (0..NUM_V_HEADS).map(|i| 0.3 + (i as f32) * 0.02).collect(),
- norm_weight: (0..HEAD_V_DIM).map(|i| 0.9 + (i as f32) * 0.001).collect(),
- key_dim,
- value_dim,
- total_proj_dim,
- }
-}
-
-fn run_prefill<T: DataTypeRepr>(
+/// `None` runs the recurrent kernels. The output is NaN-filled first so unwritten elements fail the comparison.
+fn run_prefill(
context: &MetalContext,
- case: &TestCase,
- mode: PrefillMode,
-) -> (Vec<f32>, Vec<f32>) {
- let in_proj_data: Vec<T> = case.in_proj.iter().copied().map(|value| cast(value).unwrap()).collect();
+ in_proj_data: &[bf16],
+ chunked: Option<&MetalDeltaNetChunkedPrefill>,
+ state: &mut <Metal as Backend>::GlobalBuffer,
+) -> Vec<f32> {
+ let suffix_len = in_proj_data.len() / TOTAL_PROJ_DIM;
+ // a_log = ln U(1, 16) and dt_bias = softplus^-1(logU(1e-3, 1e-1)) as in Qwen3-Next
+ let spread = |i: usize| (i as f32 + 0.5) / NUM_V_HEADS as f32;
+ let a_log: Vec<f32> = (0..NUM_V_HEADS).map(|i| (1.0 + 15.0 * spread(i)).ln()).collect();
+ let dt_bias: Vec<f32> =
+ (0..NUM_V_HEADS).map(|i| (1e-3f32.ln() + 100f32.ln() * spread(i)).exp().exp_m1().ln()).collect();
let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");
- let mut in_proj = command_buffer.allocate_scratch(size_of_val(in_proj_data.as_slice())).unwrap();
- in_proj.copyin(&in_proj_data);
- let a_log = create_buffer_with_data::<Metal, f32>(context, &case.a_log);
- let dt_bias = create_buffer_with_data::<Metal, f32>(context, &case.dt_bias);
- let norm_weight = create_buffer_with_data::<Metal, f32>(context, &case.norm_weight);
- let state_len = NUM_V_HEADS * HEAD_V_DIM * HEAD_K_DIM;
- let mut state = create_buffer::<Metal, f32>(context, state_len);
- let mut out = command_buffer.allocate_scratch(case.suffix_len * case.value_dim * size_of::<T>()).unwrap();
- let mut q = create_buffer::<Metal, f32>(context, case.suffix_len * case.key_dim);
- let mut k = create_buffer::<Metal, f32>(context, case.suffix_len * case.key_dim);
- let mut beta = create_buffer::<Metal, f32>(context, case.suffix_len * NUM_V_HEADS);
- let mut decay = create_buffer::<Metal, f32>(context, case.suffix_len * NUM_V_HEADS);
- command_buffer.encode_fill(&mut state, 0);
+ let mut in_proj = command_buffer.allocate_scratch(size_of_val(in_proj_data)).unwrap();
+ in_proj.copyin(in_proj_data);
+ let a_log = create_buffer_with_data::<Metal, f32>(context, &a_log);
+ let dt_bias = create_buffer_with_data::<Metal, f32>(context, &dt_bias);
+ let mut out = command_buffer.allocate_scratch(suffix_len * VALUE_DIM * size_of::<bf16>()).unwrap();
+ command_buffer.encode_fill(&mut out, 0xFF);
- match mode {
- PrefillMode::Recurrent => {
- <BackendKernels<Metal> as Kernels>::DeltaNetPrefillPrepKernel::new(
- context,
- T::data_type(),
- DataType::F32,
- HEAD_K_DIM as u32,
- false,
- false,
+ if let Some(chunked) = chunked {
+ chunked
+ .encode(
+ DeltaNetChunkedPrefillArgs {
+ in_projected: &in_proj,
+ a_log: &a_log,
+ dt_bias: &dt_bias,
+ ssm_state: state,
+ delta_output: &mut out,
+ num_heads: NUM_V_HEADS as u32,
+ num_groups: NUM_K_HEADS as u32,
+ value_head_dim: HEAD_V_DIM as u32,
+ key_dim: KEY_DIM as u32,
+ value_dim: VALUE_DIM as u32,
+ suffix_len: suffix_len as u32,
+ },
+ &mut command_buffer,
)
- .expect("prep")
+ .expect("chunked encode");
+ } else {
+ let mut q = create_buffer::<Metal, f32>(context, suffix_len * KEY_DIM);
+ let mut k = create_buffer::<Metal, f32>(context, suffix_len * KEY_DIM);
+ let mut beta = create_buffer::<Metal, f32>(context, suffix_len * NUM_V_HEADS);
+ let mut decay = create_buffer::<Metal, f32>(context, suffix_len * NUM_V_HEADS);
+ <BackendKernels<Metal> as Kernels>::DeltaNetPrefillPrepKernel::new(
+ context,
+ DataType::BF16,
+ DataType::F32,
+ HEAD_K_DIM as u32,
+ false,
+ false,
+ )
+ .expect("prep")
+ .encode(
+ &in_proj,
+ &a_log,
+ &dt_bias,
+ &mut q,
+ &mut k,
+ None::<&mut <Metal as Backend>::GlobalBuffer>,
+ &mut beta,
+ &mut decay,
+ NUM_V_HEADS as u32,
+ NUM_K_HEADS as u32,
+ KEY_DIM as u32,
+ VALUE_DIM as u32,
+ suffix_len as u32,
+ &mut command_buffer,
+ );
+ <BackendKernels<Metal> as Kernels>::DeltaNetPrefillKernel::new(context, DataType::BF16, HEAD_K_DIM as u32)
+ .expect("prefill")
.encode(
+ &q,
+ &k,
+ &beta,
+ &decay,
&in_proj,
- &a_log,
- &dt_bias,
- &mut q,
- &mut k,
- None::<&mut <Metal as Backend>::GlobalBuffer>,
- &mut beta,
- &mut decay,
+ state,
+ &mut out,
NUM_V_HEADS as u32,
NUM_K_HEADS as u32,
- case.key_dim as u32,
- case.value_dim as u32,
- case.suffix_len as u32,
+ HEAD_V_DIM as u32,
+ KEY_DIM as u32,
+ VALUE_DIM as u32,
+ suffix_len as u32,
+ HEAD_V_DIM.div_ceil(16) as u32,
&mut command_buffer,
);
- <BackendKernels<Metal> as Kernels>::DeltaNetPrefillKernel::new(context, T::data_type(), HEAD_K_DIM as u32)
- .expect("prefill")
- .encode(
- &q,
- &k,
- &beta,
- &decay,
- &in_proj,
- &mut state,
- &mut out,
- NUM_V_HEADS as u32,
- NUM_K_HEADS as u32,
- HEAD_V_DIM as u32,
- case.key_dim as u32,
- case.value_dim as u32,
- case.suffix_len as u32,
- HEAD_V_DIM.div_ceil(16) as u32,
- &mut command_buffer,
- );
- },
- PrefillMode::Chunked => {
- <BackendKernels<Metal> as Kernels>::DeltaNetChunkedPrefill::new(context, T::data_type(), HEAD_K_DIM as u32)
- .expect("chunked")
- .expect("chunked unsupported")
- .encode(
- DeltaNetChunkedPrefillArgs {
- in_projected: &in_proj,
- a_log: &a_log,
- dt_bias: &dt_bias,
- ssm_state: &mut state,
- delta_output: &mut out,
- num_heads: NUM_V_HEADS as u32,
- num_groups: NUM_K_HEADS as u32,
- value_head_dim: HEAD_V_DIM as u32,
- key_dim: case.key_dim as u32,
- value_dim: case.value_dim as u32,
- suffix_len: case.suffix_len as u32,
- },
- &mut command_buffer,
- )
- .expect("chunked encode");
- },
}
-
- <BackendKernels<Metal> as Kernels>::DeltaNetNormGateKernel::new(context, T::data_type()).expect("norm").encode(
- &mut out,
- &in_proj,
- &norm_weight,
- NUM_V_HEADS as u32,
- HEAD_V_DIM as u32,
- case.value_dim as u32,
- (2 * case.key_dim + case.value_dim) as u32,
- case.total_proj_dim as u32,
- 1e-6,
- case.suffix_len as u32,
- &mut command_buffer,
- );
block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
+ buffer_to_vec::<Metal, bf16>(&out).into_iter().map(f32::from).collect()
+}
- (buffer_to_vec::<Metal, T>(&out).into_iter().map(|value| cast(value).unwrap()).collect(), buffer_to_vec(&state))
+fn check(
+ context: &MetalContext,
+ chunked: &MetalDeltaNetChunkedPrefill,
+ state0: &[f32],
+ in_proj: &[bf16],
+ calls: usize,
+ label: &str,
+) {
+ let mut chunked_state = create_buffer_with_data::<Metal, f32>(context, state0);
+ let mut recurrent_state = create_buffer_with_data::<Metal, f32>(context, state0);
+ let chunked_out: Vec<f32> = in_proj
+ .chunks(in_proj.len() / calls)
+ .flat_map(|tokens| run_prefill(context, tokens, Some(chunked), &mut chunked_state))
+ .collect();
+ let recurrent_out = run_prefill(context, in_proj, None, &mut recurrent_state);
+ let (chunked_state, recurrent_state) =
+ (buffer_to_vec::<Metal, f32>(&chunked_state), buffer_to_vec::<Metal, f32>(&recurrent_state));
+
+ assert!(
+ is_close::default().abs_tol(5e-3).rel_tol(1e-3).all_close(chunked_out, recurrent_out),
+ "chunked output differs from recurrent ({label})"
+ );
+ assert!(
+ is_close::default().abs_tol(1e-2).rel_tol(1e-3).all_close(chunked_state, recurrent_state),
+ "chunked state differs from recurrent ({label})"
+ );
}
#[uzu_test]
fn chunked_prefill_matches_recurrent_prefill() {
let context = block_on(<Metal as Backend>::Context::new()).expect("metal context");
- if <BackendKernels<Metal> as Kernels>::DeltaNetChunkedPrefill::new(
+ let Some(chunked) = MetalDeltaNetChunkedPrefill::new(
&context,
- <bf16 as DataTypeRepr>::data_type(),
+ DataType::BF16,
HEAD_K_DIM as u32,
+ (NUM_V_HEADS / NUM_K_HEADS) as u32,
)
- .expect("chunked")
- .is_none()
- {
+ .unwrap() else {
eprintln!("chunked prefill unsupported on this Metal device");
return;
- }
-
- for suffix_len in [CHUNK_SIZE - 1, CHUNK_SIZE, CHUNK_SIZE + 1, CHUNK_SIZE * 2 + 1] {
- let case = test_case(suffix_len as usize);
- let (recurrent_out, recurrent_state) = run_prefill::<bf16>(&context, &case, PrefillMode::Recurrent);
- let (chunked_out, chunked_state) = run_prefill::<bf16>(&context, &case, PrefillMode::Chunked);
+ };
+ let in_proj = random_in_proj(WARMUP_TOKENS + 4096);
+ let (warmup, tokens) = in_proj.split_at(WARMUP_TOKENS * TOTAL_PROJ_DIM);
+ let mut state = create_buffer_with_data::<Metal, f32>(&context, &vec![0.0; NUM_V_HEADS * HEAD_V_DIM * HEAD_K_DIM]);
+ run_prefill(&context, warmup, None, &mut state);
+ let state0 = buffer_to_vec::<Metal, f32>(&state);
+ assert!(state0.iter().any(|value| *value != 0.0), "initial state is zero");
- assert!(
- is_close::default().abs_tol(5e-3).rel_tol(1e-3).all_close(chunked_out.iter().copied(), recurrent_out),
- "chunked output differs at T={suffix_len}"
- );
- assert!(
- is_close::default().abs_tol(2e-4).rel_tol(1e-3).all_close(chunked_state.iter().copied(), recurrent_state),
- "chunked state differs at T={suffix_len}"
- );
+ for suffix_len in [2 * CHUNK_SIZE as usize + 1, 512, 1024] {
+ let label = format!("T={suffix_len}");
+ check(&context, &chunked, &state0, &tokens[..suffix_len * TOTAL_PROJ_DIM], 1, &label);
}
+ check(&context, &chunked, &state0, tokens, 4, "carried 4x1024");
}