Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Qwen3.5 (#299)

CCYeh98304ccf0a10
diff --git a/CODEOWNERS b/CODEOWNERS
index 6ed9c9df9..3a21cab36 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -4,7 +4,7 @@
/crates/uzu/build @uuuvn
/crates/uzu/examples @alexxale
/crates/uzu/src/backends/cpu @uuuvn
-/crates/uzu/src/backends/metal @LuckyIYI @eugenebokhan @uuuvn
+/crates/uzu/src/backends/metal @LuckyIYI @eugenebokhan @uuuvn @CC-Yeh
/crates/uzu/src/classifier @eugenebokhan
/crates/uzu/src/config @uuuvn
/crates/uzu/src/device @uuuvn
diff --git a/crates/benchmarks/src/runner/runner.rs b/crates/benchmarks/src/runner/runner.rs
index a6b0f382d..81371ca74 100644
--- a/crates/benchmarks/src/runner/runner.rs
+++ b/crates/benchmarks/src/runner/runner.rs
@@ -78,6 +78,9 @@ impl Runner {
let input = Input::Messages(self.task.messages.clone());
+ let warmup_config = RunConfig::default().tokens_limit(1);
+ session.run(input.clone(), warmup_config, Some(|_: Output| true))?;
+
let mut results: Vec<TaskResult> = Vec::new();
for run_index in 0..self.task.number_of_runs {
let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
diff --git a/crates/uzu/src/backends/cpu/kernel/delta_net/conv_scan.rs b/crates/uzu/src/backends/cpu/kernel/delta_net/conv_scan.rs
new file mode 100644
index 000000000..6dfde8ba1
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/delta_net/conv_scan.rs
@@ -0,0 +1,73 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::{ArrayElement, backends::common::gpu_types::ActivationType};
+
+// Multi-token causal conv1d with SiLU for DeltaNet.
+// Reads from conv_padded (Conv1dPack output), writes conv'd+SiLU'd values
+// back to in_proj with stride out_stride, updates conv state.
+//
+// conv_padded: [state_stride + suffix_len, row_stride] — packed history + input
+// conv_weight: [conv_dim, kernel_size]
+// bias: [conv_dim] (optional)
+// in_proj: [suffix_len, out_stride] — first conv_dim channels overwritten
+// state_out: [conv_dim, state_stride]
+#[kernel(DeltaNetConvScan)]
+#[variants(T, f32, f16, bf16)]
+pub fn delta_net_conv_scan<T: ArrayElement + Float>(
+ conv_padded: *const T,
+ conv_weight: *const T,
+ #[optional(has_bias)] bias: Option<*const T>,
+ in_proj: *mut T,
+ state_out: *mut T,
+ suffix_len: u32,
+ kernel_size: u32,
+ row_stride: u32,
+ state_stride: u32,
+ conv_dim: u32,
+ out_stride: u32,
+ #[specialize] has_bias: bool,
+) {
+ let suffix_len = suffix_len as usize;
+ let kernel_size = kernel_size as usize;
+ let row_stride = row_stride as usize;
+ let state_stride = state_stride as usize;
+ let conv_dim = conv_dim as usize;
+ let out_stride = out_stride as usize;
+
+ for token in 0..suffix_len {
+ for channel in 0..conv_dim {
+ let w_offset = channel * kernel_size;
+
+ let mut acc = if has_bias {
+ unsafe { (*bias.unwrap().add(channel)).to_f32().unwrap() }
+ } else {
+ 0.0f32
+ };
+
+ for tap in 0..kernel_size {
+ let padded_row = token + tap;
+ let padded_idx = padded_row * row_stride + channel;
+ let sample = unsafe { (*conv_padded.add(padded_idx)).to_f32().unwrap() };
+ acc += unsafe { (*conv_weight.add(w_offset + tap)).to_f32().unwrap() } * sample;
+ }
+
+ unsafe {
+ *in_proj.add(token * out_stride + channel) = T::from(ActivationType::SILU.activate(acc)).unwrap();
+ }
+ }
+ }
+
+ // state_out layout: [conv_dim, state_stride] (channel-major)
+ for channel in 0..conv_dim {
+ for tap in 0..state_stride {
+ let padded_row = suffix_len + tap;
+ let padded_idx = padded_row * row_stride + channel;
+ let sample = unsafe { (*conv_padded.add(padded_idx)).to_f32().unwrap() };
+ unsafe {
+ *state_out.add(channel * state_stride + tap) = T::from(sample).unwrap();
+ }
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/delta_net/conv_update.rs b/crates/uzu/src/backends/cpu/kernel/delta_net/conv_update.rs
new file mode 100644
index 000000000..a4cb8313d
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/delta_net/conv_update.rs
@@ -0,0 +1,56 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::{ArrayElement, backends::common::gpu_types::ActivationType};
+
+// Single-token causal conv1d with SiLU, in-place.
+#[kernel(DeltaNetConvUpdate)]
+#[variants(T, f32, f16, bf16)]
+pub fn delta_net_conv_update<T: ArrayElement + Float>(
+ conv_weight: *const T,
+ #[optional(has_bias)] bias: Option<*const T>,
+ in_out: *mut T,
+ state: *mut T,
+ kernel_size: u32,
+ conv_dim: u32,
+ state_stride: u32,
+ #[specialize] has_bias: bool,
+) {
+ let kernel_size = kernel_size as usize;
+ let conv_dim = conv_dim as usize;
+ let state_stride = state_stride as usize;
+ let tap_count = kernel_size - 1;
+
+ for channel in 0..conv_dim {
+ let state_offset = channel * state_stride;
+ let weight_offset = channel * kernel_size;
+
+ let x = unsafe { (*in_out.add(channel)).to_f32().unwrap() };
+
+ let mut acc = if has_bias {
+ unsafe { (*bias.unwrap().add(channel)).to_f32().unwrap() }
+ } else {
+ 0.0f32
+ };
+
+ for tap in 0..tap_count {
+ acc += unsafe { (*state.add(state_offset + tap)).to_f32().unwrap() }
+ * unsafe { (*conv_weight.add(weight_offset + tap)).to_f32().unwrap() };
+ }
+ acc += x * unsafe { (*conv_weight.add(weight_offset + tap_count)).to_f32().unwrap() };
+
+ unsafe {
+ *in_out.add(channel) = T::from(ActivationType::SILU.activate(acc)).unwrap();
+ }
+
+ for tap in 1..tap_count {
+ unsafe {
+ *state.add(state_offset + tap - 1) = *state.add(state_offset + tap);
+ }
+ }
+ unsafe {
+ *state.add(state_offset + tap_count - 1) = T::from(x).unwrap();
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/delta_net/mod.rs b/crates/uzu/src/backends/cpu/kernel/delta_net/mod.rs
new file mode 100644
index 000000000..8f29d2c4f
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/delta_net/mod.rs
@@ -0,0 +1,6 @@
+pub mod conv_scan;
+pub mod conv_update;
+pub mod norm_gate;
+pub mod prefill;
+pub mod prefill_prep;
+pub mod update;
diff --git a/crates/uzu/src/backends/cpu/kernel/delta_net/norm_gate.rs b/crates/uzu/src/backends/cpu/kernel/delta_net/norm_gate.rs
new file mode 100644
index 000000000..6edf80099
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/delta_net/norm_gate.rs
@@ -0,0 +1,61 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::{ArrayElement, backends::common::gpu_types::ActivationType};
+
+// In-place RMSNorm + SiLU gate for DeltaNet prefill output.
+//
+// in_out: [suffix_len, value_dim] — raw input, overwritten with final output
+// in_proj: [suffix_len, total_proj_dim] — for reading z values
+// norm_weight: [head_v_dim]
+//
+// Per (token, head): in_out[i] = in_out[i] * inv_rms * norm_weight[i] * silu(z[i])
+// where inv_rms = rsqrt(mean(in_out^2) + epsilon)
+// and z is at in_proj[token * total_proj_dim + conv_dim + hv * head_v_dim + i]
+#[kernel(DeltaNetNormGate)]
+#[variants(T, f32, f16, bf16)]
+pub fn delta_net_norm_gate<T: ArrayElement + Float>(
+ in_out: *mut T,
+ in_proj: *const T,
+ norm_weight: *const T,
+ num_v_heads: u32,
+ head_v_dim: u32,
+ value_dim: u32,
+ conv_dim: u32,
+ total_proj_dim: u32,
+ norm_epsilon: f32,
+ suffix_len: u32,
+) {
+ let in_out_ptr = in_out as *const T;
+ let num_v_heads = num_v_heads as usize;
+ let head_v_dim = head_v_dim as usize;
+ let value_dim = value_dim as usize;
+ let conv_dim = conv_dim as usize;
+ let total_proj_dim = total_proj_dim as usize;
+ let suffix_len = suffix_len as usize;
+
+ for token in 0..suffix_len {
+ for hv in 0..num_v_heads {
+ let base = token * value_dim + hv * head_v_dim;
+
+ let mut sumsq = 0.0f32;
+ for i in 0..head_v_dim {
+ let val = unsafe { (*in_out_ptr.add(base + i)).to_f32().unwrap() };
+ sumsq += val * val;
+ }
+ let inv_rms = 1.0 / (sumsq / head_v_dim as f32 + norm_epsilon).sqrt();
+
+ for i in 0..head_v_dim {
+ let o_i = unsafe { (*in_out_ptr.add(base + i)).to_f32().unwrap() };
+ let norm_w = unsafe { (*norm_weight.add(i)).to_f32().unwrap() };
+ let z_idx = token * total_proj_dim + conv_dim + hv * head_v_dim + i;
+ let z_i = unsafe { (*in_proj.add(z_idx)).to_f32().unwrap() };
+ let final_val = o_i * inv_rms * norm_w * ActivationType::SILU.activate(z_i);
+ unsafe {
+ *in_out.add(base + i) = T::from(final_val).unwrap();
+ }
+ }
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/delta_net/prefill.rs b/crates/uzu/src/backends/cpu/kernel/delta_net/prefill.rs
new file mode 100644
index 000000000..a3fed8274
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/delta_net/prefill.rs
@@ -0,0 +1,74 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(DeltaNetPrefill)]
+#[variants(T, f32, f16, bf16)]
+#[variants(HEAD_K_DIM, 128)]
+pub fn delta_net_prefill<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
+ q_norm: *const f32,
+ k_norm: *const f32,
+ beta_buf: *const f32,
+ decay_buf: *const f32,
+ in_proj: *const T,
+ state: *mut T,
+ out: *mut T,
+ num_v_heads: u32,
+ num_k_heads: u32,
+ head_v_dim: u32,
+ key_dim: u32,
+ value_dim: u32,
+ suffix_len: u32,
+ #[allow(unused)] num_dv_groups: u32,
+) {
+ let state_ptr = state as *const T;
+ let num_v_heads = num_v_heads as usize;
+ let num_k_heads = num_k_heads as usize;
+ let head_k_dim = HEAD_K_DIM as usize;
+ let head_v_dim = head_v_dim as usize;
+ let key_dim = key_dim as usize;
+ let value_dim = value_dim as usize;
+ let conv_dim = 2 * key_dim + value_dim;
+ let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;
+ let suffix_len = suffix_len as usize;
+ let groups_per_head = num_v_heads / num_k_heads;
+
+ for hv in 0..num_v_heads {
+ let hk = hv / groups_per_head;
+
+ for token in 0..suffix_len {
+ let qk_off = token * key_dim + hk * head_k_dim;
+ let decay = unsafe { *decay_buf.add(token * num_v_heads + hv) };
+ let beta = unsafe { *beta_buf.add(token * num_v_heads + hv) };
+
+ for i in 0..head_v_dim {
+ // State layout: [Hv, Dv, Dk]
+ let state_off = (hv * head_v_dim + i) * head_k_dim;
+
+ let mut kv_mem = 0.0f32;
+ for j in 0..head_k_dim {
+ let s = unsafe { (*state_ptr.add(state_off + j)).to_f32().unwrap() };
+ kv_mem += (decay * s) * unsafe { *k_norm.add(qk_off + j) };
+ }
+
+ let v_val = unsafe {
+ (*in_proj.add(token * total_proj_dim + 2 * key_dim + hv * head_v_dim + i)).to_f32().unwrap()
+ };
+ let delta = beta * (v_val - kv_mem);
+
+ let mut o_val = 0.0f32;
+ for j in 0..head_k_dim {
+ let s = unsafe { (*state_ptr.add(state_off + j)).to_f32().unwrap() };
+ let k_j = unsafe { *k_norm.add(qk_off + j) };
+ let new_s = decay * s + k_j * delta;
+ unsafe { *state.add(state_off + j) = T::from(new_s).unwrap() };
+ o_val += new_s * unsafe { *q_norm.add(qk_off + j) };
+ }
+
+ unsafe { *out.add(token * value_dim + hv * head_v_dim + i) = T::from(o_val).unwrap() };
+ }
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/delta_net/prefill_prep.rs b/crates/uzu/src/backends/cpu/kernel/delta_net/prefill_prep.rs
new file mode 100644
index 000000000..a05fcf911
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/delta_net/prefill_prep.rs
@@ -0,0 +1,92 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+// Pre-compute L2-normalized q/k, beta, and decay for all tokens.
+#[kernel(DeltaNetPrefillPrep)]
+#[variants(T, f32, f16, bf16)]
+#[variants(HEAD_K_DIM, 128)]
+pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
+ in_proj: *const T,
+ a_log: *const T,
+ dt_bias: *const T,
+ q_norm_out: *mut f32,
+ k_norm_out: *mut f32,
+ beta_out: *mut f32,
+ decay_out: *mut f32,
+ num_v_heads: u32,
+ num_k_heads: u32,
+ key_dim: u32,
+ value_dim: u32,
+ suffix_len: u32,
+) {
+ let num_v_heads = num_v_heads as usize;
+ let num_k_heads = num_k_heads as usize;
+ let head_k_dim = HEAD_K_DIM as usize;
+ let key_dim = key_dim as usize;
+ let value_dim = value_dim as usize;
+ let conv_dim = 2 * key_dim + value_dim;
+ let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;
+ let suffix_len = suffix_len as usize;
+ let groups_per_head = num_v_heads / num_k_heads;
+
+ for token in 0..suffix_len {
+ let tok_offset = token * total_proj_dim;
+
+ for hk in 0..num_k_heads {
+ // Load and L2-normalize q
+ let q_off = tok_offset + hk * head_k_dim;
+ let mut q_sq = 0.0f32;
+ for j in 0..head_k_dim {
+ let v = unsafe { (*in_proj.add(q_off + j)).to_f32().unwrap() };
+ q_sq += v * v;
+ }
+ let q_inv = 1.0 / (q_sq + 1e-6).sqrt();
+ let q_scale = 1.0 / (head_k_dim as f32).sqrt();
+ for j in 0..head_k_dim {
+ let v = unsafe { (*in_proj.add(q_off + j)).to_f32().unwrap() };
+ unsafe { *q_norm_out.add(token * key_dim + hk * head_k_dim + j) = v * q_inv * q_scale };
+ }
+
+ // Load and L2-normalize k
+ let k_off = tok_offset + key_dim + hk * head_k_dim;
+ let mut k_sq = 0.0f32;
+ for j in 0..head_k_dim {
+ let v = unsafe { (*in_proj.add(k_off + j)).to_f32().unwrap() };
+ k_sq += v * v;
+ }
+ let k_inv = 1.0 / (k_sq + 1e-6).sqrt();
+ for j in 0..head_k_dim {
+ let v = unsafe { (*in_proj.add(k_off + j)).to_f32().unwrap() };
+ unsafe { *k_norm_out.add(token * key_dim + hk * head_k_dim + j) = v * k_inv };
+ }
+
+ // Beta and decay for each v-head of this k-head
+ for group in 0..groups_per_head {
+ let hv = hk * groups_per_head + group;
+
+ let beta_raw = unsafe { (*in_proj.add(tok_offset + conv_dim + value_dim + hv)).to_f32().unwrap() };
+ let beta = 1.0 / (1.0 + (-beta_raw).exp());
+
+ let a_log_val = unsafe { (*a_log.add(hv)).to_f32().unwrap() };
+ let dt_bias_val = unsafe { (*dt_bias.add(hv)).to_f32().unwrap() };
+ let a_raw =
+ unsafe { (*in_proj.add(tok_offset + conv_dim + value_dim + num_v_heads + hv)).to_f32().unwrap() };
+ let sp_in = a_raw + dt_bias_val;
+ let sp = if sp_in > 20.0 {
+ sp_in
+ } else {
+ (1.0 + sp_in.exp()).ln()
+ };
+ let decay = (-a_log_val.exp() * sp).exp();
+
+ unsafe {
+ *beta_out.add(token * num_v_heads + hv) = beta;
+ *decay_out.add(token * num_v_heads + hv) = decay;
+ };
+ }
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/delta_net/update.rs b/crates/uzu/src/backends/cpu/kernel/delta_net/update.rs
new file mode 100644
index 000000000..87b070023
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/delta_net/update.rs
@@ -0,0 +1,144 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::{ArrayElement, backends::common::gpu_types::ActivationType};
+
+// Single-token delta net update: decay state, apply delta rule, RMSNorm + SiLU gate.
+// Steps per v-head:
+// 1. L2-normalize and scale q, k
+// 2. Compute decay from a_log and softplus(a_raw + dt_bias)
+// 3. For each v-dim: read state, compute output, update state with delta rule
+// 4. RMSNorm output, gate with SiLU(z), write result
+#[kernel(DeltaNetUpdate)]
+#[variants(T, f32, f16, bf16)]
+#[variants(HEAD_K_DIM, 128)]
+pub fn delta_net_update<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
+ in_proj: *const T,
+ a_log: *const T,
+ dt_bias: *const T,
+ norm_weight: *const T,
+ state: *mut T,
+ out: *mut T,
+ num_v_heads: u32,
+ num_k_heads: u32,
+ head_v_dim: u32,
+ key_dim: u32,
+ value_dim: u32,
+ norm_epsilon: f32,
+) {
+ let state_ptr = state as *const T;
+
+ let num_v_heads = num_v_heads as usize;
+ let num_k_heads = num_k_heads as usize;
+ debug_assert!(num_v_heads % num_k_heads == 0, "num_v_heads must be a multiple of num_k_heads");
+ let head_k_dim = HEAD_K_DIM as usize;
+ let head_v_dim = head_v_dim as usize;
+ let key_dim = key_dim as usize;
+ let value_dim = value_dim as usize;
+ let conv_dim = 2 * key_dim + value_dim;
+
+ // in_proj layout (after conv):
+ // [0..key_dim): q (conv'd + SiLU)
+ // [key_dim..2*key_dim): k (conv'd + SiLU)
+ // [2*key_dim..2*key_dim+value_dim): v (conv'd + SiLU)
+ // [conv_dim..conv_dim+value_dim): z (raw)
+ // [conv_dim+value_dim..conv_dim+value_dim+num_v_heads): beta_raw (raw)
+ // [conv_dim+value_dim+num_v_heads..conv_dim+value_dim+2*num_v_heads): a_raw (raw)
+
+ for hv in 0..num_v_heads {
+ let hk = hv / (num_v_heads / num_k_heads);
+
+ // Load q and k for this head
+ let q_offset = hk * head_k_dim;
+ let k_offset = key_dim + hk * head_k_dim;
+
+ let mut q = vec![0.0f32; head_k_dim];
+ let mut k = vec![0.0f32; head_k_dim];
+ for j in 0..head_k_dim {
+ q[j] = unsafe { (*in_proj.add(q_offset + j)).to_f32().unwrap() };
+ k[j] = unsafe { (*in_proj.add(k_offset + j)).to_f32().unwrap() };
+ }
+
+ // L2 normalize q and k
+ let q_norm_sq: f32 = q.iter().map(|x| x * x).sum();
+ let k_norm_sq: f32 = k.iter().map(|x| x * x).sum();
+ let q_inv_norm = 1.0 / (q_norm_sq + 1e-6).sqrt();
+ let k_inv_norm = 1.0 / (k_norm_sq + 1e-6).sqrt();
+ for j in 0..head_k_dim {
+ q[j] *= q_inv_norm;
+ k[j] *= k_inv_norm;
+ }
+
+ // Scale q by head_k_dim^-0.5
+ let q_scale = 1.0 / (head_k_dim as f32).sqrt();
+ for j in 0..head_k_dim {
+ q[j] *= q_scale;
+ }
+
+ // Compute gating
+ let beta_raw = unsafe { (*in_proj.add(conv_dim + value_dim + hv)).to_f32().unwrap() };
+ let beta = 1.0 / (1.0 + (-beta_raw).exp()); // sigmoid
+
+ let a_raw = unsafe { (*in_proj.add(conv_dim + value_dim + num_v_heads + hv)).to_f32().unwrap() };
+ let a_log_val = unsafe { (*a_log.add(hv)).to_f32().unwrap() };
+ let dt_bias_val = unsafe { (*dt_bias.add(hv)).to_f32().unwrap() };
+
+ // softplus(a_raw + dt_bias)
+ let sp_input = a_raw + dt_bias_val;
+ let sp = if sp_input > 20.0 {
+ sp_input
+ } else {
+ (1.0 + sp_input.exp()).ln()
+ };
+ let g = -a_log_val.exp() * sp;
+ let decay = g.exp();
+
+ // dot(k, q)
+ let kq_dot: f32 = k.iter().zip(q.iter()).map(|(ki, qi)| ki * qi).sum();
+
+ // State layout: [num_v_heads, head_v_dim, head_k_dim]
+ let mut o = vec![0.0f32; head_v_dim];
+
+ for i in 0..head_v_dim {
+ let v_i = unsafe { (*in_proj.add(2 * key_dim + hv * head_v_dim + i)).to_f32().unwrap() };
+ let state_row = hv * head_v_dim * head_k_dim + i * head_k_dim;
+
+ // Pass 1: read state row S[i,:], compute dot(S[i,:], q) and dot(S[i,:], k)
+ let mut sq_acc = 0.0f32;
+ let mut sk_acc = 0.0f32;
+ for j in 0..head_k_dim {
+ let s = unsafe { (*state_ptr.add(state_row + j)).to_f32().unwrap() };
+ sq_acc += s * q[j];
+ sk_acc += s * k[j];
+ }
+
+ let retrieved_i = decay * sk_acc;
+ let delta_i = beta * (v_i - retrieved_i);
+ o[i] = decay * sq_acc + delta_i * kq_dot;
+
+ // Pass 2: update state row S[i,:]
+ for j in 0..head_k_dim {
+ let s = unsafe { (*state_ptr.add(state_row + j)).to_f32().unwrap() };
+ unsafe {
+ *state.add(state_row + j) = T::from(decay * s + k[j] * delta_i).unwrap();
+ }
+ }
+ }
+
+ // RMSNorm over o
+ let sumsq: f32 = o.iter().map(|x| x * x).sum();
+ let inv_rms = 1.0 / (sumsq / head_v_dim as f32 + norm_epsilon).sqrt();
+
+ // Apply RMSNorm + SiLU gate and write output
+ for i in 0..head_v_dim {
+ let norm_w = unsafe { (*norm_weight.add(i)).to_f32().unwrap() };
+ let z_i = unsafe { (*in_proj.add(conv_dim + hv * head_v_dim + i)).to_f32().unwrap() };
+ let z_silu = ActivationType::SILU.activate(z_i);
+ let final_val = o[i] * inv_rms * norm_w * z_silu;
+ unsafe {
+ *out.add(hv * head_v_dim + i) = T::from(final_val).unwrap();
+ }
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/mod.rs b/crates/uzu/src/backends/cpu/kernel/mod.rs
index f6b5610f5..fd116b887 100644
--- a/crates/uzu/src/backends/cpu/kernel/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/mod.rs
@@ -2,6 +2,7 @@
mod activation;
mod attention;
mod audio;
+mod delta_net;
mod embedding;
mod kv_cache_update;
mod layer_norm;
diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/conv_scan.metal b/crates/uzu/src/backends/metal/kernel/delta_net/conv_scan.metal
new file mode 100644
index 000000000..e0474b957
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/delta_net/conv_scan.metal
@@ -0,0 +1,57 @@
+#include <metal_stdlib>
+#include "../activation/activations.h"
+#include "../common/dsl.h"
+#include "../ssm/ssm_common.h"
+
+using namespace metal;
+
+// Multi-token causal conv1d with SiLU for DeltaNet.
+// Reads from conv_padded (Conv1dPack output), writes conv'd+SiLU'd
+// values back to in_proj with stride out_stride, updates conv state.
+//
+// conv_padded: [state_stride + suffix_len, row_stride]
+// conv_weight: [conv_dim, kernel_size]
+// bias: [conv_dim] (optional)
+// in_proj: [suffix_len, out_stride] — first conv_dim channels overwritten
+// state_out: [conv_dim, state_stride]
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(DeltaNetConvScan)(
+ device const T* conv_padded,
+ device const T* conv_weight,
+ device const T* bias OPTIONAL(has_bias),
+ device T* in_proj,
+ device T* state_out,
+ constant const uint& suffix_len,
+ constant const uint& kernel_size,
+ constant const uint& row_stride,
+ constant const uint& state_stride,
+ constant const uint& conv_dim,
+ constant const uint& out_stride,
+ const bool has_bias SPECIALIZE,
+ const uint token_idx AXIS(suffix_len + kernel_size - 1, 1),
+ const uint channel_idx AXIS(conv_dim, 32)
+) {
+ const uint tap_count = kernel_size - 1;
+
+ if (token_idx < suffix_len) {
+ const device T* weight_row = conv_weight + channel_idx * kernel_size;
+
+ float acc = has_bias ? float(bias[channel_idx]) : 0.0f;
+
+ for (uint tap = 0; tap < kernel_size; ++tap) {
+ const uint padded_row = token_idx + tap;
+ const uint padded_index = padded_row * row_stride + channel_idx;
+ acc += float(weight_row[tap]) * float(conv_padded[padded_index]);
+ }
+
+ const size_t dst = size_t(token_idx) * out_stride + channel_idx;
+ in_proj[dst] = static_cast<T>(activate_silu(acc));
+ } else {
+ const uint tap = token_idx - suffix_len;
+ const size_t padded_index = size_t(token_idx) * row_stride + channel_idx;
+ const uint state_offset = channel_idx * state_stride;
+ state_out[state_offset + tap] = conv_padded[padded_index];
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/conv_update.metal b/crates/uzu/src/backends/metal/kernel/delta_net/conv_update.metal
new file mode 100644
index 000000000..7132561aa
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/delta_net/conv_update.metal
@@ -0,0 +1,41 @@
+#include <metal_stdlib>
+#include "../activation/activations.h"
+#include "../common/dsl.h"
+#include "../ssm/ssm_common.h"
+
+using namespace metal;
+
+// Single-token causal conv1d with SiLU, in-place.
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(DeltaNetConvUpdate)(
+ device const T* conv_weight,
+ device const T* bias OPTIONAL(has_bias),
+ device T* in_out,
+ device T* state,
+ constant const uint& kernel_size,
+ constant const uint& conv_dim,
+ constant const uint& state_stride,
+ const bool has_bias SPECIALIZE,
+ const uint channel_idx AXIS(conv_dim, 256)
+) {
+ const uint tap_count = kernel_size - 1;
+ const uint state_offset = channel_idx * state_stride;
+ const device T* weight_row = conv_weight + channel_idx * kernel_size;
+
+ float x = float(in_out[channel_idx]);
+
+ float acc = has_bias ? float(bias[channel_idx]) : 0.0f;
+
+ for (uint tap = 0; tap < tap_count; ++tap) {
+ acc += float(weight_row[tap]) * float(state[state_offset + tap]);
+ }
+ acc += float(weight_row[tap_count]) * x;
+
+ in_out[channel_idx] = static_cast<T>(activate_silu(acc));
+
+ for (uint tap = 0; tap + 1 < tap_count; ++tap) {
+ state[state_offset + tap] = state[state_offset + tap + 1];
+ }
+ state[state_offset + tap_count - 1] = static_cast<T>(x);
+}
diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal b/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal
new file mode 100644
index 000000000..ba45c4018
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal
@@ -0,0 +1,61 @@
+#include <metal_stdlib>
+#include "../activation/activations.h"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
+#include "../common/threadgroup_reduce.h"
+#include "../ssm/ssm_common.h"
+
+using namespace metal;
+
+// In-place RMSNorm + SiLU gate for DeltaNet prefill output.
+// One threadgroup per (token, head).
+//
+// in_out[i] = in_out[i] * inv_rms * norm_weight[i] * silu(z[i])
+
+// TODO: support different head_v_dim via VARIANTS template when needed
+#define HEAD_V_DIM 128
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(DeltaNetNormGate)(
+ device T* in_out,
+ device const T* in_proj,
+ device const T* norm_weight,
+ constant const uint& num_v_heads,
+ constant const uint& head_v_dim,
+ constant const uint& value_dim,
+ constant const uint& conv_dim,
+ constant const uint& total_proj_dim,
+ constant const float& norm_epsilon,
+ constant const uint& suffix_len,
+ threadgroup float shared_scratch[32],
+ const ThreadContext thread_context,
+ const uint token_idx GROUPS(suffix_len),
+ const uint hv_idx GROUPS(num_v_heads),
+ const uint tid THREADS(HEAD_V_DIM)
+) {
+ const bool active = (tid < head_v_dim);
+ const uint base = token_idx * value_dim + hv_idx * head_v_dim;
+
+ // Load value
+ float o_i = active ? float(in_out[base + tid]) : 0.0f;
+
+ // RMSNorm: cooperative sum of squares
+ float o_sq = active ? o_i * o_i : 0.0f;
+ float o_sumsq = threadgroup_cooperative_reduce_sum<HEAD_V_DIM>(
+ o_sq,
+ shared_scratch,
+ tid,
+ thread_context
+ );
+ float inv_rms = rsqrt(o_sumsq / float(head_v_dim) + norm_epsilon);
+
+ // Apply norm + SiLU gate (in-place)
+ if (active) {
+ float nw = float(norm_weight[tid]);
+ uint z_idx =
+ token_idx * total_proj_dim + conv_dim + hv_idx * head_v_dim + tid;
+ float z_silu = activate_silu(float(in_proj[z_idx]));
+ in_out[base + tid] = static_cast<T>(o_i * inv_rms * nw * z_silu);
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/prefill.metal b/crates/uzu/src/backends/metal/kernel/delta_net/prefill.metal
new file mode 100644
index 000000000..3e5f78703
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/delta_net/prefill.metal
@@ -0,0 +1,126 @@
+#include <metal_stdlib>
+#include "../common/defines.h"
+#include "../common/dsl.h"
+#include "../ssm/ssm_common.h"
+
+using namespace metal;
+
+// DeltaNet prefill. One SIMD group per (v-head, dv_idx), Dk split across lanes.
+// Grid: num_v_heads × num_dv_groups, PREFILL_THREADS threads each.
+
+#define PREFILL_THREADS 256
+
+static_assert(
+ PREFILL_THREADS % METAL_SIMD_SIZE == 0,
+ "PREFILL_THREADS must be a multiple of METAL_SIMD_SIZE"
+);
+
+template <typename T, uint HEAD_K_DIM>
+VARIANTS(T, float, half, bfloat)
+VARIANTS(HEAD_K_DIM, 128)
+PUBLIC KERNEL(DeltaNetPrefill)(
+ device const float* q_norm,
+ device const float* k_norm,
+ device const float* beta_buf,
+ device const float* decay_buf,
+ device const T* in_proj,
+ device T* state,
+ device T* out,
+ constant const uint& num_v_heads,
+ constant const uint& num_k_heads,
+ constant const uint& head_v_dim,
+ constant const uint& key_dim,
+ constant const uint& value_dim,
+ constant const uint& suffix_len,
+ constant const uint& num_dv_groups,
+ const uint hv_idx GROUPS(num_v_heads),
+ const uint dv_group GROUPS(num_dv_groups),
+ const uint tid THREADS(PREFILL_THREADS)
+) {
+ static_assert(
+ HEAD_K_DIM % METAL_SIMD_SIZE == 0,
+ "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE"
+ );
+ constexpr uint elems_per_thread = HEAD_K_DIM / METAL_SIMD_SIZE;
+
+ const uint dk_lane = tid % METAL_SIMD_SIZE;
+ const uint dv_local = tid / METAL_SIMD_SIZE;
+ const uint dv_idx = dv_group * (PREFILL_THREADS / METAL_SIMD_SIZE) + dv_local;
+ const bool active = (dv_idx < head_v_dim);
+
+ const uint groups_per_head = num_v_heads / num_k_heads;
+ const uint hk = hv_idx / groups_per_head;
+ const uint conv_dim = 2 * key_dim + value_dim;
+ const uint total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;
+ const uint dk_base = dk_lane * elems_per_thread;
+
+ // Pointer bases — increment per token instead of re-computing index
+ device const float* q_ptr = q_norm + hk * HEAD_K_DIM + dk_base;
+ device const float* k_ptr = k_norm + hk * HEAD_K_DIM + dk_base;
+ device const float* beta_ptr = beta_buf + hv_idx;
+ device const float* decay_ptr = decay_buf + hv_idx;
+ device const T* v_ptr = in_proj + 2 * key_dim + hv_idx * head_v_dim + dv_idx;
+
+ // State layout: [Hv, Dv, Dk] — contiguous along Dk
+ device T* state_ptr =
+ state + (hv_idx * head_v_dim + dv_idx) * HEAD_K_DIM + dk_base;
+ device T* out_ptr = out + hv_idx * head_v_dim + dv_idx;
+
+ // Load state into registers
+ float s[elems_per_thread];
+ METAL_PRAGMA_UNROLL
+ for (uint i = 0; i < elems_per_thread; ++i)
+ s[i] = active ? float(state_ptr[i]) : 0.0f;
+
+ for (uint token = 0; token < suffix_len; ++token) {
+ float decay = *decay_ptr;
+ float beta = *beta_ptr;
+
+ // Load and cache k_norm (used in both passes)
+ float k[elems_per_thread];
+ METAL_PRAGMA_UNROLL
+ for (uint i = 0; i < elems_per_thread; ++i)
+ k[i] = k_ptr[i];
+
+ // Pass 1: decay state + kv_mem = (decayed S) @ k
+ float kv_partial = 0.0f;
+ METAL_PRAGMA_UNROLL
+ for (uint i = 0; i < elems_per_thread; ++i) {
+ s[i] *= decay;
+ kv_partial += s[i] * k[i];
+ }
+ float kv_mem = simd_sum(kv_partial);
+
+ // Delta
+ float v_val = active ? float(*v_ptr) : 0.0f;
+ float delta = beta * (v_val - kv_mem);
+
+ // Pass 2: update state + output = new_S @ q
+ float out_partial = 0.0f;
+ METAL_PRAGMA_UNROLL
+ for (uint i = 0; i < elems_per_thread; ++i) {
+ s[i] += k[i] * delta;
+ out_partial += s[i] * q_ptr[i];
+ }
+ float o_val = simd_sum(out_partial);
+
+ if (active && dk_lane == 0) {
+ *out_ptr = static_cast<T>(o_val);
+ }
+
+ // Advance pointers to next token
+ q_ptr += key_dim;
+ k_ptr += key_dim;
+ beta_ptr += num_v_heads;
+ decay_ptr += num_v_heads;
+ v_ptr += total_proj_dim;
+ out_ptr += value_dim;
+ }
+
+ // Write final state
+ if (active) {
+ METAL_PRAGMA_UNROLL
+ for (uint i = 0; i < elems_per_thread; ++i)
+ state_ptr[i] = static_cast<T>(s[i]);
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal b/crates/uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal
new file mode 100644
index 000000000..65256e6b1
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal
@@ -0,0 +1,97 @@
+#include <metal_stdlib>
+#include "../common/defines.h"
+#include "../common/dsl.h"
+#include "../ssm/ssm_common.h"
+
+using namespace metal;
+
+// Pre-compute normalized q, k, beta, decay for DeltaNet prefill.
+// Grid: suffix_len × num_k_heads threadgroups, METAL_SIMD_SIZE threads each (1
+// SIMD group).
+
+template <typename T, uint HEAD_K_DIM>
+VARIANTS(T, float, half, bfloat)
+VARIANTS(HEAD_K_DIM, 128)
+PUBLIC KERNEL(DeltaNetPrefillPrep)(
+ device const T* in_proj,
+ device const T* a_log,
+ device const T* dt_bias,
+ device float* q_norm_out,
+ device float* k_norm_out,
+ device float* beta_out,
+ device float* decay_out,
+ constant const uint& num_v_heads,
+ constant const uint& num_k_heads,
+ constant const uint& key_dim,
+ constant const uint& value_dim,
+ constant const uint& suffix_len,
+ const uint token_idx GROUPS(suffix_len),
+ const uint hk_idx GROUPS(num_k_heads),
+ const uint lane THREADS(METAL_SIMD_SIZE)
+) {
+ static_assert(
+ HEAD_K_DIM % METAL_SIMD_SIZE == 0,
+ "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE"
+ );
+ constexpr uint elems_per_thread = HEAD_K_DIM / METAL_SIMD_SIZE;
+
+ const uint conv_dim = 2 * key_dim + value_dim;
+ const uint total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;
+ const uint groups_per_head = num_v_heads / num_k_heads;
+
+ const uint tok_offset = token_idx * total_proj_dim;
+ const uint q_base = tok_offset + hk_idx * HEAD_K_DIM;
+ const uint k_base = tok_offset + key_dim + hk_idx * HEAD_K_DIM;
+
+ // Load q and k elements (strided by SIMD width)
+ float q_vals[elems_per_thread];
+ float k_vals[elems_per_thread];
+ float q_sq = 0.0f;
+ float k_sq = 0.0f;
+
+ for (uint i = 0; i < elems_per_thread; ++i) {
+ uint idx = lane + i * METAL_SIMD_SIZE;
+ q_vals[i] = float(in_proj[q_base + idx]);
+ k_vals[i] = float(in_proj[k_base + idx]);
+ q_sq += q_vals[i] * q_vals[i];
+ k_sq += k_vals[i] * k_vals[i];
+ }
+
+ // L2 normalize via simd_sum (no threadgroup barrier needed)
+ float q_norm_sq = simd_sum(q_sq);
+ float k_norm_sq = simd_sum(k_sq);
+ float q_inv_norm = rsqrt(q_norm_sq + 1e-6f);
+ float k_inv_norm = rsqrt(k_norm_sq + 1e-6f);
+ float q_scale = rsqrt(float(HEAD_K_DIM));
+
+ // Normalize and scale q, write q and k
+ const uint out_q_base = token_idx * key_dim + hk_idx * HEAD_K_DIM;
+ const uint out_k_base = token_idx * key_dim + hk_idx * HEAD_K_DIM;
+
+ for (uint i = 0; i < elems_per_thread; ++i) {
+ uint idx = lane + i * METAL_SIMD_SIZE;
+ float q_n = q_vals[i] * q_inv_norm * q_scale;
+ float k_n = k_vals[i] * k_inv_norm;
+ if (idx < HEAD_K_DIM) {
+ q_norm_out[out_q_base + idx] = q_n;
+ k_norm_out[out_k_base + idx] = k_n;
+ }
+ }
+
+ // Compute beta and decay for v-heads belonging to this k-head
+ // groups_per_head v-heads share this k-head; distribute across lanes
+ for (uint group = lane; group < groups_per_head; group += METAL_SIMD_SIZE) {
+ uint hv = hk_idx * groups_per_head + group;
+
+ float beta_raw = float(in_proj[tok_offset + conv_dim + value_dim + hv]);
+ float beta = 1.0f / (1.0f + fast::exp(-beta_raw));
+
+ float a_raw =
+ float(in_proj[tok_offset + conv_dim + value_dim + num_v_heads + hv]);
+ float sp = softplus(a_raw + float(dt_bias[hv]));
+ float decay = fast::exp(-fast::exp(float(a_log[hv])) * sp);
+
+ beta_out[token_idx * num_v_heads + hv] = beta;
+ decay_out[token_idx * num_v_heads + hv] = decay;
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/update.metal b/crates/uzu/src/backends/metal/kernel/delta_net/update.metal
new file mode 100644
index 000000000..7fdd882a4
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/delta_net/update.metal
@@ -0,0 +1,155 @@
+#include <metal_stdlib>
+#include "../activation/activations.h"
+#include "../common/defines.h"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
+#include "../common/threadgroup_reduce.h"
+#include "../ssm/ssm_common.h"
+
+using namespace metal;
+
+#define UPDATE_THREADS 512
+constexpr constant uint UPDATE_K_SPLIT = 4;
+
+static_assert(
+ UPDATE_THREADS % UPDATE_K_SPLIT == 0,
+ "UPDATE_THREADS must be a multiple of UPDATE_K_SPLIT"
+);
+
+template <typename T, uint HEAD_K_DIM>
+VARIANTS(T, float, half, bfloat)
+VARIANTS(HEAD_K_DIM, 128)
+PUBLIC KERNEL(DeltaNetUpdate)(
+ device const T* in_proj,
+ device const T* a_log,
+ device const T* dt_bias,
+ device const T* norm_weight,
+ device T* state,
+ device T* out,
+ constant const uint& num_v_heads,
+ constant const uint& num_k_heads,
+ constant const uint& head_v_dim,
+ constant const uint& key_dim,
+ constant const uint& value_dim,
+ constant const float& norm_epsilon,
+ threadgroup float shared_qk[512 /* UPDATE_THREADS */ * 2],
+ threadgroup float shared_scratch[METAL_SIMD_SIZE],
+ const ThreadContext thread_context,
+ const uint hv_idx GROUPS(num_v_heads),
+ const uint tid THREADS(UPDATE_THREADS)
+) {
+ static_assert(
+ HEAD_K_DIM % UPDATE_K_SPLIT == 0,
+ "HEAD_K_DIM must be a multiple of UPDATE_K_SPLIT"
+ );
+ constexpr uint bk = HEAD_K_DIM / UPDATE_K_SPLIT;
+
+ const uint vi = tid / UPDATE_K_SPLIT;
+ const uint ki_part = tid % UPDATE_K_SPLIT;
+ const bool active = (vi < head_v_dim);
+
+ threadgroup float* shared_q = shared_qk;
+ threadgroup float* shared_k = shared_qk + UPDATE_THREADS;
+
+ const uint conv_dim = 2 * key_dim + value_dim;
+ const uint groups_per_head = num_v_heads / num_k_heads;
+ const uint hk = hv_idx / groups_per_head;
+
+ if (tid < HEAD_K_DIM) {
+ shared_q[tid] = float(in_proj[hk * HEAD_K_DIM + tid]);
+ shared_k[tid] = float(in_proj[key_dim + hk * HEAD_K_DIM + tid]);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ float q_partial = (tid < HEAD_K_DIM) ? shared_q[tid] * shared_q[tid] : 0.0f;
+ float q_norm_sq = threadgroup_cooperative_reduce_sum<UPDATE_THREADS>(
+ q_partial,
+ shared_scratch,
+ tid,
+ thread_context
+ );
+ float q_inv_norm = rsqrt(q_norm_sq + 1e-6f);
+
+ float k_partial = (tid < HEAD_K_DIM) ? shared_k[tid] * shared_k[tid] : 0.0f;
+ float k_norm_sq = threadgroup_cooperative_reduce_sum<UPDATE_THREADS>(
+ k_partial,
+ shared_scratch,
+ tid,
+ thread_context
+ );
+ float k_inv_norm = rsqrt(k_norm_sq + 1e-6f);
+
+ float q_scale = rsqrt(float(HEAD_K_DIM));
+ if (tid < HEAD_K_DIM) {
+ shared_q[tid] *= q_inv_norm * q_scale;
+ shared_k[tid] *= k_inv_norm;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ float kq_p = (tid < HEAD_K_DIM) ? shared_k[tid] * shared_q[tid] : 0.0f;
+ float kq_dot = threadgroup_cooperative_reduce_sum<UPDATE_THREADS>(
+ kq_p,
+ shared_scratch,
+ tid,
+ thread_context
+ );
+
+ float beta_raw = float(in_proj[conv_dim + value_dim + hv_idx]);
+ float beta = 1.0f / (1.0f + fast::exp(-beta_raw));
+ float a_raw = float(in_proj[conv_dim + value_dim + num_v_heads + hv_idx]);
+ float sp = softplus(a_raw + float(dt_bias[hv_idx]));
+ float decay = fast::exp(-fast::exp(float(a_log[hv_idx])) * sp);
+
+ float v_i =
+ active ? float(in_proj[2 * key_dim + hv_idx * head_v_dim + vi]) : 0.0f;
+ float z_i =
+ active ? float(in_proj[conv_dim + hv_idx * head_v_dim + vi]) : 0.0f;
+
+ // State read — each thread handles bk rows
+ // State layout: [Hv, Dv, Dk]
+ float sq_partial = 0.0f;
+ float sk_partial = 0.0f;
+ const uint state_row_offset =
+ hv_idx * head_v_dim * HEAD_K_DIM + vi * HEAD_K_DIM;
+ const uint k_base = ki_part * bk;
+ if (active) {
+ for (uint j = 0; j < bk; ++j) {
+ float s = float(state[state_row_offset + k_base + j]);
+ sq_partial += s * shared_q[k_base + j];
+ sk_partial += s * shared_k[k_base + j];
+ }
+ }
+
+ // Reduce across k-parts (4 consecutive lanes = one quad group)
+ float sq_acc = quad_sum(sq_partial);
+ float sk_acc = quad_sum(sk_partial);
+
+ float retrieved_i = decay * sk_acc;
+ float delta_i = beta * (v_i - retrieved_i);
+ float o_i = decay * sq_acc + delta_i * kq_dot;
+
+ // State update — each thread updates its k-part
+ if (active) {
+ for (uint j = 0; j < bk; ++j) {
+ uint idx = state_row_offset + k_base + j;
+ float s = float(state[idx]);
+ state[idx] = static_cast<T>(decay * s + shared_k[k_base + j] * delta_i);
+ }
+ }
+
+ // RMSNorm + SiLU gate — only ki_part==0 contributes
+ float o_sq = (active && ki_part == 0) ? o_i * o_i : 0.0f;
+ float o_sumsq = threadgroup_cooperative_reduce_sum<UPDATE_THREADS>(
+ o_sq,
+ shared_scratch,
+ tid,
+ thread_context
+ );
+ float inv_rms = rsqrt(o_sumsq / float(head_v_dim) + norm_epsilon);
+
+ if (active && ki_part == 0) {
+ float nw = float(norm_weight[vi]);
+ float z_silu = activate_silu(z_i);
+ out[hv_idx * head_v_dim + vi] = static_cast<T>(o_i * inv_rms * nw * z_silu);
+ }
+}
diff --git a/crates/uzu/src/config/attention.rs b/crates/uzu/src/config/attention.rs
index 8215b7c37..e213a67da 100644
--- a/crates/uzu/src/config/attention.rs
+++ b/crates/uzu/src/config/attention.rs
@@ -25,6 +25,8 @@ pub struct AttentionConfig {
#[serde(default)]
pub has_gate: bool,
#[serde(default)]
+ pub gate_projection_config: Option<LinearConfig>,
+ #[serde(default)]
pub partial_rope_dim: Option<usize>,
}
diff --git a/crates/uzu/src/config/delta_net.rs b/crates/uzu/src/config/delta_net.rs
index 4e650f702..61a87b2fd 100644
--- a/crates/uzu/src/config/delta_net.rs
+++ b/crates/uzu/src/config/delta_net.rs
@@ -27,4 +27,8 @@ impl DeltaNetAttentionConfig {
pub fn conv_dim(&self) -> usize {
self.key_dim() * 2 + self.value_dim()
}
+
+ pub fn total_proj_dim(&self) -> usize {
+ self.conv_dim() + self.value_dim() + self.num_heads + self.num_heads
+ }
}
diff --git a/crates/uzu/src/encodable_block/delta_net_mixer.rs b/crates/uzu/src/encodable_block/delta_net_mixer.rs
new file mode 100644
index 000000000..a2a60de4f
--- /dev/null
+++ b/crates/uzu/src/encodable_block/delta_net_mixer.rs
@@ -0,0 +1,345 @@
+use std::ops::{Deref, DerefMut};
+
+use thiserror::Error;
+
+use crate::{
+ DataType,
+ backends::common::{
+ Backend, Encoder, Kernels,
+ kernel::{
+ Conv1dPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel, DeltaNetNormGateKernel,
+ DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,
+ },
+ },
+ config::DeltaNetAttentionConfig,
+ encodable_block::linear::{Linear, LinearBlockError},
+ forward_pass::state::{ArrayId, ForwardPassState},
+ parameters::{ParameterLoaderError, ParameterTree, resolve_subtree},
+};
+
+#[derive(Debug, Error)]
+pub enum DeltaNetMixerError<B: Backend> {
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Unsupported configuration: {0}")]
+ UnsupportedConfiguration(String),
+ #[error("Linear error: {0}")]
+ InnerLinearError(#[from] Box<LinearBlockError<B>>),
+ #[error("Parameter loader error: {0}")]
+ ParameterLoaderError(#[from] ParameterLoaderError<B>),
+}
+
+pub(crate) struct DeltaNetMixer<B: Backend> {
+ layer_index: usize,
+ config: DeltaNetAttentionConfig,
+ in_projection: Box<dyn Linear<B>>,
+ out_projection: Box<dyn Linear<B>>,
+ // Decode kernels
+ conv_update: <B::Kernels as Kernels>::DeltaNetConvUpdateKernel,
+ delta_net_update: <B::Kernels as Kernels>::DeltaNetUpdateKernel,
+ // Prefill kernels
+ conv_pack: <B::Kernels as Kernels>::Conv1dPackKernel,
+ conv_scan: <B::Kernels as Kernels>::DeltaNetConvScanKernel,
+ prefill_prep: <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel,
+ delta_net_prefill: <B::Kernels as Kernels>::DeltaNetPrefillKernel,
+ norm_gate: <B::Kernels as Kernels>::DeltaNetNormGateKernel,
+ // Parameters
+ conv_weight: B::Buffer,
+ conv_bias: Option<B::Buffer>,
+ a_log: B::Buffer,
+ dt_bias: B::Buffer,
+ norm_weight: B::Buffer,
+}
+
+impl<B: Backend> DeltaNetMixer<B> {
+ pub(crate) fn new(
+ context: &B::Context,
+ config: DeltaNetAttentionConfig,
+ layer_index: usize,
+ model_dim: usize,
+ decoder_layer_loader: &ParameterTree<B::Context>,
+ ) -> Result<Self, DeltaNetMixerError<B>> {
+ if config.kernel_size < 2 {
+ return Err(DeltaNetMixerError::UnsupportedConfiguration(format!(
+ "kernel_size must be >= 2, got {}",
+ config.kernel_size
+ )));
+ }
+ if config.head_dim != 128 {
+ return Err(DeltaNetMixerError::UnsupportedConfiguration(format!(
+ "head_dim must be 128, got {}",
+ config.head_dim
+ )));
+ }
+ if config.value_head_dim != 128 {
+ return Err(DeltaNetMixerError::UnsupportedConfiguration(format!(
+ "value_head_dim must be 128, got {}",
+ config.value_head_dim
+ )));
+ }
+
+ let data_type: DataType = config.in_proj_config.activation_precision().into();
+ let has_bias = config.conv_config.has_biases;
+
+ // Load weights
+ let mixer_tree = resolve_subtree(decoder_layer_loader, &["mixer"]);
+ let conv_tree = resolve_subtree(&mixer_tree, &["conv", "conv1d"]);
+
+ let in_projection = <dyn Linear<B>>::new(
+ &config.in_proj_config,
+ false,
+ model_dim,
+ [config.total_proj_dim()],
+ context,
+ &resolve_subtree(decoder_layer_loader, &["mixer.in_projection", "mixer.in_proj"]),
+ ArrayId::Main,
+ ArrayId::SsmInProj,
+ )
+ .map_err(|e| DeltaNetMixerError::InnerLinearError(Box::new(e)))?;
+
+ let out_projection = <dyn Linear<B>>::new(
+ &config.out_proj_config,
+ false,
+ config.value_dim(),
+ [model_dim],
+ context,
+ &resolve_subtree(decoder_layer_loader, &["mixer.out_projection", "mixer.out_proj"]),
+ ArrayId::AttentionOutput,
+ ArrayId::Main,
+ )
+ .map_err(|e| DeltaNetMixerError::InnerLinearError(Box::new(e)))?;
+
+ let conv_weight = conv_tree.leaf("weights")?.read_buffer()?;
+ let conv_bias = if has_bias {
+ Some(conv_tree.leaf("biases")?.read_buffer()?)
+ } else {
+ None
+ };
+
+ let a_log = mixer_tree.leaf("a_log")?.read_buffer()?;
+ let dt_bias = mixer_tree.leaf("dt_bias")?.read_buffer()?;
+ let norm_tree = resolve_subtree(&mixer_tree, &["norm", "inner_norm"]);
+ let norm_weight = norm_tree.leaf("scales")?.read_buffer()?;
+
+ // Create kernels
+ let conv_update = <B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, data_type, has_bias)
+ .map_err(DeltaNetMixerError::BackendError)?;
+ let delta_net_update =
+ <B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, data_type, config.head_dim as u32)
+ .map_err(DeltaNetMixerError::BackendError)?;
+ let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, data_type)
+ .map_err(DeltaNetMixerError::BackendError)?;
+ let conv_scan = <B::Kernels as Kernels>::DeltaNetConvScanKernel::new(context, data_type, has_bias)
+ .map_err(DeltaNetMixerError::BackendError)?;
+ let prefill_prep =
+ <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(context, data_type, config.head_dim as u32)
+ .map_err(DeltaNetMixerError::BackendError)?;
+ let delta_net_prefill =
+ <B::Kernels as Kernels>::DeltaNetPrefillKernel::new(context, data_type, config.head_dim as u32)
+ .map_err(DeltaNetMixerError::BackendError)?;
+ let norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, data_type)
+ .map_err(DeltaNetMixerError::BackendError)?;
+
+ Ok(Self {
+ layer_index,
+ config,
+ in_projection,
+ out_projection,
+ conv_update,
+ delta_net_update,
+ conv_pack,
+ conv_scan,
+ prefill_prep,
+ delta_net_prefill,
+ norm_gate,
+ conv_weight,
+ conv_bias,
+ a_log,
+ dt_bias,
+ norm_weight,
+ })
+ }
+
+ fn run_conv_update(
+ &self,
+ state: &mut ForwardPassState<B>,
+ encoder: &mut Encoder<B>,
+ ) {
+ let in_proj = state.array(ArrayId::SsmInProj);
+ let conv_state = state.array(ArrayId::DeltaNetConvState(self.layer_index));
+
+ let kernel_size = self.config.kernel_size;
+
+ self.conv_update.encode(
+ &self.conv_weight,
+ self.conv_bias.as_ref(),
+ in_proj.buffer().borrow_mut().deref_mut(),
+ conv_state.buffer().borrow_mut().deref_mut(),
+ kernel_size as u32,
+ self.config.conv_dim() as u32,
+ (kernel_size - 1) as u32,
+ encoder,
+ );
+ }
+
+ fn run_conv_scan(
+ &self,
+ state: &mut ForwardPassState<B>,
+ encoder: &mut Encoder<B>,
+ suffix_length: usize,
+ ) {
+ let kernel_size = self.config.kernel_size;
+ let state_stride = kernel_size - 1;
+ let conv_dim = self.config.conv_dim();
+ let total_proj_dim = self.config.total_proj_dim();
+
+ let in_proj = state.array(ArrayId::SsmInProj);
+ let conv_state = state.array(ArrayId::DeltaNetConvState(self.layer_index));
+
+ let padded = state.conv_padded_buffer().expect("Missing conv padded buffer");
+
+ self.conv_pack.encode(
+ conv_state.buffer().borrow().deref(),
+ in_proj.buffer().borrow().deref(),
+ padded.buffer().borrow_mut().deref_mut(),
+ state_stride as u32,
+ total_proj_dim as u32,
+ suffix_length as u32,
+ conv_dim as u32,
+ encoder,
+ );
+
+ self.conv_scan.encode(
+ padded.buffer().borrow().deref(),
+ &self.conv_weight,
+ self.conv_bias.as_ref(),
+ in_proj.buffer().borrow_mut().deref_mut(),
+ conv_state.buffer().borrow_mut().deref_mut(),
+ suffix_length as u32,
+ kernel_size as u32,
+ total_proj_dim as u32,
+ state_stride as u32,
+ conv_dim as u32,
+ total_proj_dim as u32,
+ encoder,
+ );
+ }
+
+ fn run_delta_rule(
+ &self,
+ state: &mut ForwardPassState<B>,
+ encoder: &mut Encoder<B>,
+ ) {
+ let in_proj = state.array(ArrayId::SsmInProj);
+ let ssm_state = state.array(ArrayId::DeltaNetSsmState(self.layer_index));
+ let out = state.array(ArrayId::AttentionOutput);
+
+ self.delta_net_update.encode(
+ in_proj.buffer().borrow().deref(),
+ &self.a_log,
+ &self.dt_bias,
+ &self.norm_weight,
+ ssm_state.buffer().borrow_mut().deref_mut(),
+ out.buffer().borrow_mut().deref_mut(),
+ self.config.num_heads as u32,
+ self.config.num_groups as u32,
+ self.config.value_head_dim as u32,
+ self.config.key_dim() as u32,
+ self.config.value_dim() as u32,
+ self.config.norm_config.epsilon,
+ encoder,
+ );
+ }
+
+ fn run_delta_rule_prefill(
+ &self,
+ state: &mut ForwardPassState<B>,
+ encoder: &mut Encoder<B>,
+ suffix_length: usize,
+ ) {
+ let num_dv_groups = ((self.config.value_head_dim + 7) / 8) as u32;
+
+ let in_proj = state.array(ArrayId::SsmInProj);
+ let ssm_state = state.array(ArrayId::DeltaNetSsmState(self.layer_index));
+ let out = state.array(ArrayId::AttentionOutput);
+
+ let aux = state.llm_aux.as_ref().expect("DeltaNet prep buffers not initialized");
+ let prep_q_norm = aux.delta_net_prep_q_norm.as_ref().expect("DeltaNet prep_q_norm not initialized");
+ let prep_k_norm = aux.delta_net_prep_k_norm.as_ref().expect("DeltaNet prep_k_norm not initialized");
+ let prep_beta = aux.delta_net_prep_beta.as_ref().expect("DeltaNet prep_beta not initialized");
+ let prep_decay = aux.delta_net_prep_decay.as_ref().expect("DeltaNet prep_decay not initialized");
+
+ self.prefill_prep.encode(
+ in_proj.buffer().borrow().deref(),
+ &self.a_log,
+ &self.dt_bias,
+ prep_q_norm.buffer().borrow_mut().deref_mut(),
+ prep_k_norm.buffer().borrow_mut().deref_mut(),
+ prep_beta.buffer().borrow_mut().deref_mut(),
+ prep_decay.buffer().borrow_mut().deref_mut(),
+ self.config.num_heads as u32,
+ self.config.num_groups as u32,
+ self.config.key_dim() as u32,
+ self.config.value_dim() as u32,
+ suffix_length as u32,
+ encoder,
+ );
+
+ self.delta_net_prefill.encode(
+ prep_q_norm.buffer().borrow().deref(),
+ prep_k_norm.buffer().borrow().deref(),
+ prep_beta.buffer().borrow().deref(),
+ prep_decay.buffer().borrow().deref(),
+ in_proj.buffer().borrow().deref(),
+ ssm_state.buffer().borrow_mut().deref_mut(),
+ out.buffer().borrow_mut().deref_mut(),
+ self.config.num_heads as u32,
+ self.config.num_groups as u32,
+ self.config.value_head_dim as u32,
+ self.config.key_dim() as u32,
+ self.config.value_dim() as u32,
+ suffix_length as u32,
+ num_dv_groups,
+ encoder,
+ );
+
+ // Norm gate
+ self.norm_gate.encode(
+ out.buffer().borrow_mut().deref_mut(),
+ in_proj.buffer().borrow().deref(),
+ &self.norm_weight,
+ self.config.num_heads as u32,
+ self.config.value_head_dim as u32,
+ self.config.value_dim() as u32,
+ self.config.conv_dim() as u32,
+ self.config.total_proj_dim() as u32,
+ self.config.norm_config.epsilon,
+ suffix_length as u32,
+ encoder,
+ );
+ }
+
+ pub(crate) fn encode(
+ &self,
+ state: &mut ForwardPassState<B>,
+ encoder: &mut Encoder<B>,
+ ) -> Result<(), B::Error> {
+ let active_suffix_length = state.active_suffix_length();
+ if active_suffix_length == 0 {
+ return Ok(());
+ }
+
+ self.in_projection.encode(state, encoder)?;
+
+ if active_suffix_length == 1 {
+ self.run_conv_update(state, encoder);
+ self.run_delta_rule(state, encoder);
+ } else {
+ self.run_conv_scan(state, encoder, active_suffix_length);
+ self.run_delta_rule_prefill(state, encoder, active_suffix_length);
+ }
+
+ self.out_projection.encode(state, encoder)?;
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rs
index 641375307..e9ca29161 100644
--- a/crates/uzu/src/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/encodable_block/layer/executables.rs
@@ -8,8 +8,8 @@ use crate::{
backends::common::{Backend, Encoder},
config::{DecoderLayerConfig, DecoderLayerType, MixerConfig},
encodable_block::{
- Attention, EncodingParameters, Linear, MambaMixer, Mlp, QKNorm, RMSNorm, Rope, ShortConvMixer, TensorAddSwap,
- TensorCopy,
+ Attention, DeltaNetMixer, EncodingParameters, Linear, MambaMixer, Mlp, QKNorm, RMSNorm, Rope, ShortConvMixer,
+ TensorAddSwap, TensorCopy,
},
forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
@@ -68,11 +68,18 @@ impl<B: Backend> LayerExecutables<B> {
MixerConfig::Attention(attention_config) => {
let rope_block = rope.expect("RoPE encoder missing for attention layer");
+ let layer_num_heads = attention_config.num_heads.unwrap_or(num_heads);
+ let layer_num_groups = attention_config.num_groups.unwrap_or(num_groups);
+ let layer_head_dim = attention_config.head_dim.unwrap_or(head_dim);
+
+ let q_dim = layer_num_heads * layer_head_dim;
+ let kv_dim = layer_num_groups * layer_head_dim;
+
let qkv_projection = <dyn Linear<B>>::new(
&attention_config.qkv_projection_config,
attention_config.has_qkv_biases,
model_dim,
- [num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],
+ [q_dim, kv_dim, kv_dim],
context,
&decoder_layer_loader.subtree("mixer.qkv_projection").unwrap(),
ArrayId::Main,
@@ -80,13 +87,18 @@ impl<B: Backend> LayerExecutables<B> {
)
.expect("Failed to create qkv projection");
- let gate_projection = if attention_config.has_gate {
+ let has_gate = attention_config.has_gate || attention_config.gate_projection_config.is_some();
+ let gate_projection = if has_gate {
+ let gate_config = attention_config
+ .gate_projection_config
+ .as_ref()
+ .unwrap_or(&attention_config.qkv_projection_config);
Some(
<dyn Linear<B>>::new(
- &attention_config.qkv_projection_config,
+ gate_config,
false,
model_dim,
- [num_heads * head_dim],
+ [q_dim],
context,
&decoder_layer_loader.subtree("mixer.gate_projection").unwrap(),
ArrayId::Main,
@@ -107,9 +119,9 @@ impl<B: Backend> LayerExecutables<B> {
attention_config.key_norm_config.clone(),
ArrayId::QKV,
&decoder_layer_loader.subtree("mixer").unwrap(),
- num_heads,
- num_groups,
- head_dim,
+ layer_num_heads,
+ layer_num_groups,
+ layer_head_dim,
) {
Ok(qk_norm) => Some(qk_norm),
Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),
@@ -121,7 +133,7 @@ impl<B: Backend> LayerExecutables<B> {
let out_projection = <dyn Linear<B>>::new(
&attention_config.out_projection_config,
attention_config.has_out_biases,
- num_heads * head_dim,
+ q_dim,
[model_dim],
context,
&decoder_layer_loader.subtree("mixer.out_projection").unwrap(),
@@ -138,7 +150,7 @@ impl<B: Backend> LayerExecutables<B> {
attention_config.has_sinks,
attention_config.is_causal.unwrap_or(true),
attention_config.sliding_window_size,
- attention_config.has_gate,
+ has_gate,
)
.expect("Failed to create AttentionWrapper kernel");
@@ -180,7 +192,15 @@ impl<B: Backend> LayerExecutables<B> {
mixer,
}
},
- MixerConfig::DeltaNet(_) => unimplemented!("DeltaNet mixer"),
+ MixerConfig::DeltaNet(delta_net_config) => {
+ let mixer =
+ DeltaNetMixer::new(context, delta_net_config.clone(), layer_index, model_dim, decoder_layer_loader)
+ .expect("Failed to create DeltaNet mixer");
+
+ MixerExecutables::DeltaNet {
+ mixer,
+ }
+ },
};
let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {
@@ -316,6 +336,15 @@ impl<B: Backend> LayerExecutables<B> {
state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());
}
},
+ MixerExecutables::DeltaNet {
+ mixer,
+ } => {
+ mixer.encode(state, encoder)?;
+ #[cfg(feature = "tracing")]
+ if let Some(ref layer_traces) = layer_traces {
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());
+ }
+ },
}
if let Some(post_attention_norm) = &self.post_attention_norm {
diff --git a/crates/uzu/src/encodable_block/layer/mixer.rs b/crates/uzu/src/encodable_block/layer/mixer.rs
index 749ebbc0b..08894a765 100644
--- a/crates/uzu/src/encodable_block/layer/mixer.rs
+++ b/crates/uzu/src/encodable_block/layer/mixer.rs
@@ -4,10 +4,10 @@ use std::rc::Rc;
use crate::{
backends::common::Backend,
- encodable_block::{Attention, MambaMixer, QKNorm, Rope, ShortConvMixer, linear::Linear},
+ encodable_block::{Attention, DeltaNetMixer, MambaMixer, QKNorm, Rope, ShortConvMixer, linear::Linear},
};
-/// Mixer component - either attention, state space, or short conv.
+/// Mixer component - either attention, state space, short conv, or delta net.
pub(crate) enum MixerExecutables<B: Backend> {
Attention {
qkv_projection: Box<dyn Linear<B>>,
@@ -23,4 +23,7 @@ pub(crate) enum MixerExecutables<B: Backend> {
ShortConv {
mixer: ShortConvMixer<B>,
},
+ DeltaNet {
+ mixer: DeltaNetMixer<B>,
+ },
}
diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rs
index ada33e7ab..84f5b1b2d 100644
--- a/crates/uzu/src/encodable_block/mod.rs
+++ b/crates/uzu/src/encodable_block/mod.rs
@@ -2,6 +2,7 @@ mod activation;
mod attention;
mod classifier_layer;
mod decoder;
+mod delta_net_mixer;
mod embedding;
mod layer;
mod layer_norm;
@@ -23,6 +24,7 @@ pub use activation::Activation;
pub use attention::Attention;
pub use classifier_layer::ClassifierLayer;
pub use decoder::Decoder;
+pub(crate) use delta_net_mixer::DeltaNetMixer;
pub use embedding::Embedding;
pub use layer::LayerExecutables;
pub use layer_norm::{LayerNorm, LayerNormError};
diff --git a/crates/uzu/src/forward_pass/cache_layers.rs b/crates/uzu/src/forward_pass/cache_layers.rs
index 38abcabac..3c25ee397 100644
--- a/crates/uzu/src/forward_pass/cache_layers.rs
+++ b/crates/uzu/src/forward_pass/cache_layers.rs
@@ -5,6 +5,7 @@ use crate::{
backends::common::{Backend, Encoder, kernel::kv_cache_update::KVCacheUpdate},
config::DecoderLayerType,
forward_pass::{
+ delta_net_layer::DeltaNetLayer,
kv_cache_layer::{KVCacheLayer, KVCacheLayerState, KVSlice},
model_shape::ModelShape,
short_conv_layer::ShortConvLayer,
@@ -17,6 +18,7 @@ pub enum CacheLayer<B: Backend> {
Transformer(KVCacheLayer<B>),
StateSpace(SSMLayer<B>),
ShortConv(ShortConvLayer<B>),
+ DeltaNet(DeltaNetLayer<B>),
}
#[derive(Clone)]
@@ -24,6 +26,7 @@ pub enum CacheLayerSlice<B: Backend> {
Transformer(KVSlice<B>),
StateSpace,
ShortConv,
+ DeltaNet,
}
const ARRAY_TRANSFORMER_KEYS_LABEL: &str = "cache_layers_transformer_keys";
@@ -32,6 +35,8 @@ const ARRAY_STATE_SPACE_CONV_STATE_LABEL: &str = "cache_layers_state_space_conv_
const ARRAY_STATE_SPACE_SSM_STATE_LABEL: &str = "cache_layers_state_space_ssm_state";
const ARRAY_SHORT_CONV_CONV_STATE_LABEL: &str = "cache_layers_short_conv_conv_state";
const ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL: &str = "cache_layers_short_conv_suffix_state";
+const ARRAY_DELTA_NET_CONV_STATE_LABEL: &str = "cache_layers_delta_net_conv_state";
+const ARRAY_DELTA_NET_SSM_STATE_LABEL: &str = "cache_layers_delta_net_ssm_state";
impl<B: Backend> CacheLayer<B> {
pub fn as_transformer(&self) -> Option<&KVCacheLayer<B>> {
@@ -75,6 +80,20 @@ impl<B: Backend> CacheLayer<B> {
_ => None,
}
}
+
+ pub fn as_delta_net(&self) -> Option<&DeltaNetLayer<B>> {
+ match self {
+ CacheLayer::DeltaNet(layer) => Some(layer),
+ _ => None,
+ }
+ }
+
+ pub fn as_delta_net_mut(&mut self) -> Option<&mut DeltaNetLayer<B>> {
+ match self {
+ CacheLayer::DeltaNet(layer) => Some(layer),
+ _ => None,
+ }
+ }
}
pub struct CacheLayers<B: Backend> {
@@ -184,8 +203,30 @@ impl<B: Backend> CacheLayers<B> {
})
},
DecoderLayerType::DeltaNet {
+ conv_dim,
+ kernel_size,
+ num_heads,
+ head_dim,
+ value_head_dim,
..
- } => unimplemented!("DeltaNet cache layer"),
+ } => {
+ let conv_shape = [*conv_dim, kernel_size.saturating_sub(1)];
+ let ssm_shape = [*num_heads, *value_head_dim, *head_dim];
+ let dtype = model_shape.activation_data_type();
+
+ CacheLayer::DeltaNet(DeltaNetLayer {
+ conv_state: context.create_array_zeros(
+ &conv_shape,
+ dtype,
+ &format!("{ARRAY_DELTA_NET_CONV_STATE_LABEL}_{layer_index}"),
+ ),
+ ssm_state: context.create_array_zeros(
+ &ssm_shape,
+ dtype,
+ &format!("{ARRAY_DELTA_NET_SSM_STATE_LABEL}_{layer_index}"),
+ ),
+ })
+ },
})
.collect();
@@ -216,6 +257,7 @@ impl<B: Backend> CacheLayers<B> {
},
CacheLayer::StateSpace(layer) => layer.zero(),
CacheLayer::ShortConv(layer) => layer.zero(),
+ CacheLayer::DeltaNet(layer) => layer.zero(),
}
}
}
@@ -272,6 +314,7 @@ impl<B: Backend> CacheLayers<B> {
},
CacheLayer::StateSpace(_) => layers.push(CacheLayerSlice::StateSpace),
CacheLayer::ShortConv(_) => layers.push(CacheLayerSlice::ShortConv),
+ CacheLayer::DeltaNet(_) => layers.push(CacheLayerSlice::DeltaNet),
}
}
@@ -391,6 +434,30 @@ impl<B: Backend> CacheLayers<B> {
suffix_state_valid_len: Cell::new(0),
})
},
+ CacheLayer::DeltaNet(layer) => {
+ let conv_shape = layer.conv_state.shape().to_vec();
+ let conv_dtype = layer.conv_state.data_type();
+ let mut new_conv = context.create_array_uninitialized(
+ &conv_shape,
+ conv_dtype,
+ &format!("{ARRAY_DELTA_NET_CONV_STATE_LABEL}_{layer_index}"),
+ );
+ new_conv.copy_from_array(&layer.conv_state);
+
+ let ssm_shape = layer.ssm_state.shape().to_vec();
+ let ssm_dtype = layer.ssm_state.data_type();
+ let mut new_ssm = context.create_array_uninitialized(
+ &ssm_shape,
+ ssm_dtype,
+ &format!("{ARRAY_DELTA_NET_SSM_STATE_LABEL}_{layer_index}"),
+ );
+ new_ssm.copy_from_array(&layer.ssm_state);
+
+ CacheLayer::DeltaNet(DeltaNetLayer {
+ conv_state: new_conv,
+ ssm_state: new_ssm,
+ })
+ },
})
.collect();
diff --git a/crates/uzu/src/forward_pass/delta_net_layer.rs b/crates/uzu/src/forward_pass/delta_net_layer.rs
new file mode 100644
index 000000000..4559e35cd
--- /dev/null
+++ b/crates/uzu/src/forward_pass/delta_net_layer.rs
@@ -0,0 +1,16 @@
+use bytemuck::fill_zeroes;
+
+use crate::{array::Array, backends::common::Backend};
+
+#[derive(Debug)]
+pub struct DeltaNetLayer<B: Backend> {
+ pub conv_state: Array<B>,
+ pub ssm_state: Array<B>,
+}
+
+impl<B: Backend> DeltaNetLayer<B> {
+ pub fn zero(&mut self) {
+ fill_zeroes(self.conv_state.as_bytes_mut());
+ fill_zeroes(self.ssm_state.as_bytes_mut());
+ }
+}
diff --git a/crates/uzu/src/forward_pass/mod.rs b/crates/uzu/src/forward_pass/mod.rs
index cfede3e9a..474f782c6 100644
--- a/crates/uzu/src/forward_pass/mod.rs
+++ b/crates/uzu/src/forward_pass/mod.rs
@@ -1,6 +1,7 @@
pub mod model_shape;
pub mod cache_layers;
+pub mod delta_net_layer;
pub mod kv_cache_layer;
pub mod short_conv_layer;
pub mod ssm_layer;
diff --git a/crates/uzu/src/forward_pass/model_shape.rs b/crates/uzu/src/forward_pass/model_shape.rs
index 7a46bbda5..fae070856 100644
--- a/crates/uzu/src/forward_pass/model_shape.rs
+++ b/crates/uzu/src/forward_pass/model_shape.rs
@@ -28,6 +28,10 @@ pub struct ModelShape {
max_rope_dim: usize,
max_lora_intermediate: Option<usize>,
has_gate: bool,
+ max_delta_net_kernel_size: usize,
+ max_delta_net_key_dim: usize,
+ max_delta_net_num_heads: usize,
+ max_qkv_dim: usize,
}
impl ModelShape {
@@ -76,6 +80,24 @@ impl ModelShape {
max_mamba_kernel_size = max_mamba_kernel_size.max(*kernel_size as usize);
}
}
+ let mut max_delta_net_kernel_size = 0usize;
+ let mut max_delta_net_key_dim = 0usize;
+ let mut max_delta_net_num_heads = 0usize;
+ for layer in layer_types.iter() {
+ if let DecoderLayerType::DeltaNet {
+ kernel_size,
+ num_heads,
+ num_groups,
+ head_dim,
+ ..
+ } = layer
+ {
+ max_delta_net_kernel_size = max_delta_net_kernel_size.max(*kernel_size);
+ max_delta_net_key_dim = max_delta_net_key_dim.max(num_groups * head_dim);
+ max_delta_net_num_heads = max_delta_net_num_heads.max(*num_heads);
+ }
+ }
+
let mut max_rope_dim = 0usize;
let mut max_lora_intermediate = None;
let mut has_gate = false;
@@ -86,6 +108,7 @@ impl ModelShape {
.map(|configs| configs.iter().collect::<Vec<_>>())
.unwrap_or_else(|| vec![&decoder_config.layer_config; num_layers]);
+ let mut max_qkv_dim = 0usize;
for layer_config in &all_layer_configs {
let mut linear_configs = Vec::new();
@@ -93,7 +116,7 @@ impl ModelShape {
MixerConfig::Attention(attn) => {
let hd = attn.head_dim.unwrap_or(decoder_config.head_dim);
max_rope_dim = max_rope_dim.max(attn.partial_rope_dim.unwrap_or(hd));
- has_gate = has_gate || attn.has_gate;
+ has_gate = has_gate || attn.has_gate || attn.gate_projection_config.is_some();
linear_configs.extend([&attn.qkv_projection_config, &attn.out_projection_config]);
},
MixerConfig::Mamba(mamba) => {
@@ -128,6 +151,9 @@ impl ModelShape {
};
}
}
+ if max_qkv_dim == 0 {
+ max_qkv_dim = (decoder_config.num_heads + 2 * decoder_config.num_groups) * decoder_config.head_dim;
+ }
if max_rope_dim == 0 {
max_rope_dim = decoder_config.head_dim;
@@ -158,6 +184,10 @@ impl ModelShape {
max_rope_dim,
max_lora_intermediate,
has_gate,
+ max_delta_net_kernel_size,
+ max_delta_net_key_dim,
+ max_delta_net_num_heads,
+ max_qkv_dim,
}
}
@@ -240,7 +270,7 @@ impl ModelShape {
&self,
suffix_length: usize,
) -> [usize; 2] {
- [suffix_length, (2 * self.num_groups + self.num_heads) * self.head_dim]
+ [suffix_length, self.max_qkv_dim]
}
pub fn logits_shape(
@@ -254,7 +284,21 @@ impl ModelShape {
&self,
suffix_length: usize,
) -> [usize; 2] {
- [suffix_length, self.num_heads * self.head_dim]
+ let attention_dim = self.num_heads * self.head_dim;
+ let delta_net_dim = self
+ .layer_types
+ .iter()
+ .filter_map(|lt| match lt {
+ DecoderLayerType::DeltaNet {
+ num_heads,
+ value_head_dim,
+ ..
+ } => Some(num_heads * value_head_dim),
+ _ => None,
+ })
+ .max()
+ .unwrap_or(0);
+ [suffix_length, attention_dim.max(delta_net_dim)]
}
pub fn gate_shape(
@@ -446,6 +490,51 @@ impl ModelShape {
self.layer_types.iter().any(|layer_type| matches!(layer_type, DecoderLayerType::ShortConv { .. }))
}
+ pub fn has_delta_net_layers(&self) -> bool {
+ self.max_delta_net_kernel_size > 0
+ }
+
+ pub fn delta_net_prep_qk_shape(
+ &self,
+ suffix_length: usize,
+ ) -> Option<[usize; 1]> {
+ if self.has_delta_net_layers() {
+ Some([suffix_length * self.max_delta_net_key_dim])
+ } else {
+ None
+ }
+ }
+
+ pub fn delta_net_prep_beta_decay_shape(
+ &self,
+ suffix_length: usize,
+ ) -> Option<[usize; 1]> {
+ if self.has_delta_net_layers() {
+ Some([suffix_length * self.max_delta_net_num_heads])
+ } else {
+ None
+ }
+ }
+
+ fn max_delta_net_padded_stride(&self) -> usize {
+ self.layer_types
+ .iter()
+ .filter_map(|lt| match lt {
+ DecoderLayerType::DeltaNet {
+ conv_dim,
+ num_heads,
+ value_head_dim,
+ ..
+ } => {
+ let value_dim = num_heads * value_head_dim;
+ Some(conv_dim + value_dim + 2 * num_heads)
+ },
+ _ => None,
+ })
+ .max()
+ .unwrap_or(0)
+ }
+
pub fn max_mamba_conv_dim(&self) -> Option<usize> {
if self.max_mamba_conv_dim == 0 {
None
@@ -481,9 +570,21 @@ impl ModelShape {
&self,
suffix_length: usize,
) -> Option<[usize; 2]> {
- match (self.max_mamba_conv_dim(), self.max_mamba_kernel_size()) {
- (Some(conv_dim), Some(kernel_size)) if kernel_size > 0 => Some([suffix_length + kernel_size - 1, conv_dim]),
+ let mamba = match (self.max_mamba_conv_dim(), self.max_mamba_kernel_size()) {
+ (Some(conv_dim), Some(kernel_size)) if kernel_size > 0 => Some((suffix_length + kernel_size - 1, conv_dim)),
_ => None,
+ };
+ let delta_net = if self.has_delta_net_layers() {
+ let state_stride = self.max_delta_net_kernel_size.saturating_sub(1);
+ let max_padded_stride = self.max_delta_net_padded_stride();
+ Some((suffix_length + state_stride, max_padded_stride))
+ } else {
+ None
+ };
+ match (mamba, delta_net) {
+ (Some((rows_m, cols_m)), Some((rows_d, cols_d))) => Some([rows_m.max(rows_d), cols_m.max(cols_d)]),
+ (Some((rows, cols)), None) | (None, Some((rows, cols))) => Some([rows, cols]),
+ (None, None) => None,
}
}
@@ -574,6 +675,15 @@ impl ModelShape {
DecoderLayerType::ShortConv {
..
} => Some(self.model_dim * 3),
+ DecoderLayerType::DeltaNet {
+ conv_dim,
+ num_heads,
+ value_head_dim,
+ ..
+ } => {
+ let value_dim = num_heads * value_head_dim;
+ Some(conv_dim + value_dim + 2 * num_heads)
+ },
_ => None,
})
.max()
diff --git a/crates/uzu/src/forward_pass/scratch_buffers.rs b/crates/uzu/src/forward_pass/scratch_buffers.rs
index f920895b8..2cefb1d1c 100644
--- a/crates/uzu/src/forward_pass/scratch_buffers.rs
+++ b/crates/uzu/src/forward_pass/scratch_buffers.rs
@@ -34,6 +34,10 @@ pub struct ScratchBuffers<B: Backend> {
pub ssm_c: Option<Array<B>>,
pub ssm_dt: Option<Array<B>>,
pub ssm_z: Option<Array<B>>,
+ pub delta_net_prep_q_norm: Option<Array<B>>,
+ pub delta_net_prep_k_norm: Option<Array<B>>,
+ pub delta_net_prep_beta: Option<Array<B>>,
+ pub delta_net_prep_decay: Option<Array<B>>,
// 3-D
pub rotated_queries: Array<B>,
@@ -128,6 +132,18 @@ impl<B: Backend> ScratchBuffers<B> {
ssm_c: model_shape.ssm_bc_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_c")),
ssm_dt: model_shape.ssm_dt_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_dt")),
ssm_z: model_shape.ssm_z_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_z")),
+ delta_net_prep_q_norm: model_shape
+ .delta_net_prep_qk_shape(max_suffix_len)
+ .map(|shape| alloc(&shape, DataType::F32, "delta_net_prep_q_norm")),
+ delta_net_prep_k_norm: model_shape
+ .delta_net_prep_qk_shape(max_suffix_len)
+ .map(|shape| alloc(&shape, DataType::F32, "delta_net_prep_k_norm")),
+ delta_net_prep_beta: model_shape
+ .delta_net_prep_beta_decay_shape(max_suffix_len)
+ .map(|shape| alloc(&shape, DataType::F32, "delta_net_prep_beta")),
+ delta_net_prep_decay: model_shape
+ .delta_net_prep_beta_decay_shape(max_suffix_len)
+ .map(|shape| alloc(&shape, DataType::F32, "delta_net_prep_decay")),
// 3-D
rotated_queries: alloc(&model_shape.rotated_queries_shape(max_suffix_len), act_ty, "rotated_queries"),
rotated_keys: alloc(&model_shape.rotated_keys_shape(max_suffix_len), act_ty, "rotated_keys"),
diff --git a/crates/uzu/src/forward_pass/state/array_id.rs b/crates/uzu/src/forward_pass/state/array_id.rs
index 09afec89b..281b1470b 100644
--- a/crates/uzu/src/forward_pass/state/array_id.rs
+++ b/crates/uzu/src/forward_pass/state/array_id.rs
@@ -32,6 +32,8 @@ pub enum ArrayId {
SsmZ(usize),
ShortConvState(usize),
ShortConvSuffixState(usize),
+ DeltaNetConvState(usize),
+ DeltaNetSsmState(usize),
RotatedQueries,
RotatedKeys,
diff --git a/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs b/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs
index b204b189d..09a974544 100644
--- a/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs
+++ b/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs
@@ -14,6 +14,11 @@ pub struct LanguageModelGeneratorAuxBuffers<B: Backend> {
pub ssm_c: Option<Array<B>>,
pub ssm_dt: Option<Array<B>>,
pub ssm_z: Option<Array<B>>,
+ // DeltaNet prep buffers
+ pub delta_net_prep_q_norm: Option<Array<B>>,
+ pub delta_net_prep_k_norm: Option<Array<B>>,
+ pub delta_net_prep_beta: Option<Array<B>>,
+ pub delta_net_prep_decay: Option<Array<B>>,
// MoE buffers
pub moe_topk_ids: Option<Array<B>>,
pub moe_topk_probs: Option<Array<B>>,
@@ -93,6 +98,26 @@ impl<B: Backend> LanguageModelGeneratorAuxBuffers<B> {
.as_ref()
.zip(model_shape.ssm_z_shape(suffix_length))
.map(|(buf, shape)| buf.view(&shape)),
+ delta_net_prep_q_norm: scratch
+ .delta_net_prep_q_norm
+ .as_ref()
+ .zip(model_shape.delta_net_prep_qk_shape(suffix_length))
+ .map(|(buf, shape)| buf.view(&shape)),
+ delta_net_prep_k_norm: scratch
+ .delta_net_prep_k_norm
+ .as_ref()
+ .zip(model_shape.delta_net_prep_qk_shape(suffix_length))
+ .map(|(buf, shape)| buf.view(&shape)),
+ delta_net_prep_beta: scratch
+ .delta_net_prep_beta
+ .as_ref()
+ .zip(model_shape.delta_net_prep_beta_decay_shape(suffix_length))
+ .map(|(buf, shape)| buf.view(&shape)),
+ delta_net_prep_decay: scratch
+ .delta_net_prep_decay
+ .as_ref()
+ .zip(model_shape.delta_net_prep_beta_decay_shape(suffix_length))
+ .map(|(buf, shape)| buf.view(&shape)),
moe_topk_ids: moe.zip(scratch.moe_topk_ids.as_ref()).map(|(moe, buf)| {
buf.view(&model_shape.moe_topk_ids_shape(suffix_length, moe.num_active_routed_experts))
}),
diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rs
index d130f7865..8cd427da3 100644
--- a/crates/uzu/src/forward_pass/state/state.rs
+++ b/crates/uzu/src/forward_pass/state/state.rs
@@ -59,7 +59,7 @@ pub struct ForwardPassState<B: Backend> {
token_bitmask: Option<Array<B>>,
pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
pub common_aux: CommonAuxBuffers<B>,
- llm_aux: Option<LanguageModelGeneratorAuxBuffers<B>>,
+ pub llm_aux: Option<LanguageModelGeneratorAuxBuffers<B>>,
mode: ForwardPassMode<B>,
}
@@ -470,6 +470,14 @@ impl<B: Backend> ForwardPassState<B> {
let cache = self.llm_state().cache_layers.borrow();
cache.data[layer_index].as_short_conv().expect("Expected ShortConv layer").suffix_state.clone()
},
+ ArrayId::DeltaNetConvState(layer_index) => {
+ let cache = self.llm_state().cache_layers.borrow();
+ cache.data[layer_index].as_delta_net().expect("Expected DeltaNet layer").conv_state.clone()
+ },
+ ArrayId::DeltaNetSsmState(layer_index) => {
+ let cache = self.llm_state().cache_layers.borrow();
+ cache.data[layer_index].as_delta_net().expect("Expected DeltaNet layer").ssm_state.clone()
+ },
// MoE arrays (LLM only)
ArrayId::MoeTopkIds => {
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rs
index ce7d4c719..58d334a5c 100644
--- a/crates/uzu/src/tracer/trace_validator.rs
+++ b/crates/uzu/src/tracer/trace_validator.rs
@@ -349,6 +349,46 @@ impl<B: Backend> TraceValidator<B> {
}
}
+ // LLM-specific: DeltaNet state validation
+ let delta_net_layers: Vec<usize> = {
+ let cache = state.cache_layers().unwrap().borrow();
+ cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_delta_net().map(|_| index)).collect()
+ };
+
+ for index in delta_net_layers {
+ let conv_state = state.array(ArrayId::DeltaNetConvState(index));
+ let ssm_state = state.array(ArrayId::DeltaNetSsmState(index));
+
+ for path in [
+ format!("updated_state.{}.conv_state", index),
+ format!("activation_trace.layer_results.{}.updated_state.conv_state", index),
+ ] {
+ if let Ok(expected) = traces_view.leaf_array(&path) {
+ results.push(TracerValidationResult {
+ name: path,
+ metrics: Self::validate_array(
+ data_type,
+ &expected,
+ &conv_state,
+ Some(ArrayTransform::SsmConvState),
+ ),
+ });
+ }
+ }
+
+ for path in [
+ format!("updated_state.{}.ssm_state", index),
+ format!("activation_trace.layer_results.{}.updated_state.ssm_state", index),
+ ] {
+ if let Ok(expected) = traces_view.leaf_array(&path) {
+ results.push(TracerValidationResult {
+ name: path,
+ metrics: Self::validate_array(data_type, &expected, &ssm_state, None),
+ });
+ }
+ }
+ }
+
// LLM-specific: Token comparison
let tokens_violation_indices = if let Ok(expected_logits) = traces_view.leaf_array("logits") {
let expected_tokens = Self::get_tokens_from_logits(&expected_logits);
diff --git a/crates/uzu/tests/common/path.rs b/crates/uzu/tests/common/path.rs
index d425f529f..7b6bc38aa 100644
--- a/crates/uzu/tests/common/path.rs
+++ b/crates/uzu/tests/common/path.rs
@@ -10,14 +10,15 @@ pub fn get_version() -> &'static str {
}
pub fn get_test_model_path() -> PathBuf {
+ let model_dir = std::env::var("TEST_MODEL").unwrap_or_else(|_| MODEL_DIR_NAME.to_string());
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("..")
.join("..")
.join("models")
.join(get_version())
- .join(MODEL_DIR_NAME);
+ .join(model_dir);
if !path.exists() {
- panic!("Test model not found at {:?}. Run `./scripts/download_test_model.sh` to fetch it.", path);
+ panic!("Test model not found at {:?}. Set TEST_MODEL env var or run `./scripts/download_test_model.sh`.", path);
}
path
}
diff --git a/crates/uzu/tests/unit/config/attention_test.rs b/crates/uzu/tests/unit/config/attention_test.rs
index 6a7f86d3f..3514eb124 100644
--- a/crates/uzu/tests/unit/config/attention_test.rs
+++ b/crates/uzu/tests/unit/config/attention_test.rs
@@ -77,6 +77,7 @@ fn test_attention_config() {
has_qkv_biases: false,
has_out_biases: false,
has_gate: false,
+ gate_projection_config: None,
partial_rope_dim: None,
};
diff --git a/crates/uzu/tests/unit/config/decoder_layer_test.rs b/crates/uzu/tests/unit/config/decoder_layer_test.rs
index 0c1d08619..a1980219c 100644
--- a/crates/uzu/tests/unit/config/decoder_layer_test.rs
+++ b/crates/uzu/tests/unit/config/decoder_layer_test.rs
@@ -135,6 +135,7 @@ fn test_decoder_layer_config() {
has_qkv_biases: false,
has_out_biases: false,
has_gate: false,
+ gate_projection_config: None,
partial_rope_dim: None,
}),
mlp_config: MLPConfig::Dense(mlp::DenseMLPConfig {
diff --git a/crates/uzu/tests/unit/config/decoder_test.rs b/crates/uzu/tests/unit/config/decoder_test.rs
index 7c1904ee0..30bc4ad8b 100644
--- a/crates/uzu/tests/unit/config/decoder_test.rs
+++ b/crates/uzu/tests/unit/config/decoder_test.rs
@@ -198,6 +198,7 @@ fn test_decoder_config() {
has_qkv_biases: false,
has_out_biases: false,
has_gate: false,
+ gate_projection_config: None,
partial_rope_dim: None,
}),
mlp_config: MLPConfig::Dense(DenseMLPConfig {
diff --git a/crates/uzu/tests/unit/config/transformer_layer_test.rs b/crates/uzu/tests/unit/config/transformer_layer_test.rs
index bb68298a0..fa81a2cb9 100644
--- a/crates/uzu/tests/unit/config/transformer_layer_test.rs
+++ b/crates/uzu/tests/unit/config/transformer_layer_test.rs
@@ -143,6 +143,7 @@ fn test_transformer_layer_config() {
has_qkv_biases: false,
has_out_biases: false,
has_gate: false,
+ gate_projection_config: None,
partial_rope_dim: None,
}),
mlp_config: MLPConfig::Dense(mlp::DenseMLPConfig {
diff --git a/crates/uzu/tests/unit/kernel/delta_net_test.rs b/crates/uzu/tests/unit/kernel/delta_net_test.rs
new file mode 100644
index 000000000..599e10d59
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/delta_net_test.rs
@@ -0,0 +1,687 @@
+#![cfg(metal_backend)]
+
+use std::ops::{Deref, DerefMut};
+
+use uzu::{
+ ArrayContextExt, DataType,
+ backends::{
+ common::{
+ Backend, Context, Encoder, Kernels,
+ kernel::{
+ Conv1dPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel, DeltaNetNormGateKernel,
+ DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,
+ },
+ },
+ cpu::Cpu,
+ metal::Metal,
+ },
+};
+
+fn run_conv_update<B: Backend>(
+ in_proj: &[f32],
+ w: &[f32],
+ b: &[f32],
+ state: &[f32],
+ kernel_size: u32,
+ conv_dim: u32,
+ state_stride: u32,
+) -> (Vec<f32>, Vec<f32>) {
+ let context = B::Context::new().expect("Failed to create context");
+
+ let w_array = context.create_array_from(&[w.len()], w, "w");
+ let b_array = context.create_array_from(&[b.len()], b, "b");
+ let in_out_array = context.create_array_from(&[in_proj.len()], in_proj, "in_out");
+ let state_array = context.create_array_from(&[state.len()], state, "state");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(&context, DataType::F32, true)
+ .expect("Failed to create kernel");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ w_array.buffer().borrow().deref(),
+ Some(b_array.buffer().borrow().deref()),
+ in_out_array.buffer().borrow_mut().deref_mut(),
+ state_array.buffer().borrow_mut().deref_mut(),
+ kernel_size,
+ conv_dim,
+ state_stride,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ let out = in_out_array.as_slice::<f32>()[..conv_dim as usize].to_vec();
+ let new_state = state_array.as_slice::<f32>().to_vec();
+ (out, new_state)
+}
+
+fn run_delta_net_update<B: Backend>(
+ in_proj: &[f32],
+ a_log: &[f32],
+ dt_bias: &[f32],
+ norm_weight: &[f32],
+ state: &[f32],
+ num_v_heads: u32,
+ num_k_heads: u32,
+ head_k_dim: u32,
+ head_v_dim: u32,
+ key_dim: u32,
+ value_dim: u32,
+) -> (Vec<f32>, Vec<f32>) {
+ let context = B::Context::new().expect("Failed to create context");
+
+ let in_proj_array = context.create_array_from(&[in_proj.len()], in_proj, "in_proj");
+ let a_log_array = context.create_array_from(&[a_log.len()], a_log, "a_log");
+ let dt_bias_array = context.create_array_from(&[dt_bias.len()], dt_bias, "dt_bias");
+ let norm_weight_array = context.create_array_from(&[norm_weight.len()], norm_weight, "norm_weight");
+ let state_array = context.create_array_from(&[state.len()], state, "state");
+ let out_array = context.create_array_zeros(&[value_dim as usize], DataType::F32, "out");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::DeltaNetUpdateKernel::new(&context, DataType::F32, head_k_dim)
+ .expect("Failed to create kernel");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ in_proj_array.buffer().borrow().deref(),
+ a_log_array.buffer().borrow().deref(),
+ dt_bias_array.buffer().borrow().deref(),
+ norm_weight_array.buffer().borrow().deref(),
+ state_array.buffer().borrow_mut().deref_mut(),
+ out_array.buffer().borrow_mut().deref_mut(),
+ num_v_heads,
+ num_k_heads,
+ head_v_dim,
+ key_dim,
+ value_dim,
+ 1e-6f32,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ let out = out_array.as_slice::<f32>().to_vec();
+ let new_state = state_array.as_slice::<f32>().to_vec();
+ (out, new_state)
+}
+
+fn assert_close(
+ a: &[f32],
+ b: &[f32],
+ atol: f32,
+ rtol: f32,
+ label: &str,
+) {
+ assert_eq!(a.len(), b.len(), "{label}: length mismatch: {} vs {}", a.len(), b.len());
+ assert!(
+ is_close::default().abs_tol(atol).rel_tol(rtol).all_close(a.iter().copied(), b.iter().copied()),
+ "{label}: values not close enough (atol={atol:.6e}, rtol={rtol:.6e})"
+ );
+}
+
+// DeltaNetConvUpdate
+
+#[test]
+fn test_delta_net_conv_update_small() {
+ let conv_dim = 32;
+ let kernel_size = 4;
+ let tap_count = kernel_size - 1;
+
+ let in_proj: Vec<f32> = (0..conv_dim).map(|i| ((i % 7) as f32) * 0.1 - 0.3).collect();
+ let w: Vec<f32> = (0..conv_dim * kernel_size).map(|i| ((i % 11) as f32) * 0.05 - 0.2).collect();
+ let b: Vec<f32> = (0..conv_dim).map(|i| ((i % 5) as f32) * 0.01).collect();
+ let state: Vec<f32> = (0..conv_dim * tap_count).map(|i| ((i % 13) as f32) * 0.02 - 0.1).collect();
+
+ let (cpu_out, cpu_state) =
+ run_conv_update::<Cpu>(&in_proj, &w, &b, &state, kernel_size as u32, conv_dim as u32, tap_count as u32);
+ let (gpu_out, gpu_state) =
+ run_conv_update::<Metal>(&in_proj, &w, &b, &state, kernel_size as u32, conv_dim as u32, tap_count as u32);
+
+ assert_close(&cpu_out, &gpu_out, 1e-4, 1e-3, "ConvUpdate output");
+ assert_close(&cpu_state, &gpu_state, 1e-5, 1e-4, "ConvUpdate state");
+}
+
+// DeltaNetConvScan
+
+#[test]
+fn test_delta_net_conv_scan() {
+ let conv_dim = 32;
+ let kernel_size: usize = 4;
+ let tap_count = kernel_size - 1;
+ let suffix_len: usize = 16;
+ let extra_dim = 10;
+ let total_proj_dim = conv_dim + extra_dim;
+
+ let w: Vec<f32> = (0..conv_dim * kernel_size).map(|i| ((i % 11) as f32) * 0.05 - 0.2).collect();
+ let b: Vec<f32> = (0..conv_dim).map(|i| ((i % 5) as f32) * 0.01).collect();
+ let init_state: Vec<f32> = (0..conv_dim * tap_count).map(|i| ((i % 13) as f32) * 0.02 - 0.1).collect();
+ let in_proj: Vec<f32> = (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect();
+
+ // Reference: decode conv token-by-token
+ let mut ref_state = init_state.clone();
+ let mut ref_outputs = vec![0.0f32; suffix_len * conv_dim];
+ for t in 0..suffix_len {
+ let token_in: Vec<f32> = in_proj[t * total_proj_dim..t * total_proj_dim + conv_dim].to_vec();
+ let (out, new_state) = run_conv_update::<Cpu>(
+ &token_in,
+ &w,
+ &b,
+ &ref_state,
+ kernel_size as u32,
+ conv_dim as u32,
+ tap_count as u32,
+ );
+ ref_state = new_state;
+ ref_outputs[t * conv_dim..(t + 1) * conv_dim].copy_from_slice(&out);
+ }
+
+ // Test: Conv1dPack + DeltaNetConvScan on Metal
+ let context = <Metal as Backend>::Context::new().expect("context");
+ let state_array = context.create_array_from(&[init_state.len()], &init_state, "state");
+ let in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj, "in_proj");
+ let w_array = context.create_array_from(&[w.len()], &w, "w");
+ let b_array = context.create_array_from(&[b.len()], &b, "b");
+
+ let padded_len = (tap_count + suffix_len) * total_proj_dim;
+ let padded_array = context.create_array_zeros(&[padded_len], DataType::F32, "padded");
+ let state_out_array = context.create_array_zeros(&[conv_dim * tap_count], DataType::F32, "state_out");
+
+ let pack_kernel =
+ <<Metal as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, DataType::F32).expect("pack");
+ let scan_kernel =
+ <<Metal as Backend>::Kernels as Kernels>::DeltaNetConvScanKernel::new(&context, DataType::F32, true)
+ .expect("scan");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ pack_kernel.encode(
+ state_array.buffer().borrow().deref(),
+ in_proj_array.buffer().borrow().deref(),
+ padded_array.buffer().borrow_mut().deref_mut(),
+ tap_count as u32,
+ total_proj_dim as u32,
+ suffix_len as u32,
+ conv_dim as u32,
+ &mut encoder,
+ );
+ scan_kernel.encode(
+ padded_array.buffer().borrow().deref(),
+ w_array.buffer().borrow().deref(),
+ Some(b_array.buffer().borrow().deref()),
+ in_proj_array.buffer().borrow_mut().deref_mut(),
+ state_out_array.buffer().borrow_mut().deref_mut(),
+ suffix_len as u32,
+ kernel_size as u32,
+ total_proj_dim as u32,
+ tap_count as u32,
+ conv_dim as u32,
+ total_proj_dim as u32,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ let in_proj_result = in_proj_array.as_slice::<f32>();
+ let mut scan_outputs = vec![0.0f32; suffix_len * conv_dim];
+ for t in 0..suffix_len {
+ scan_outputs[t * conv_dim..(t + 1) * conv_dim]
+ .copy_from_slice(&in_proj_result[t * total_proj_dim..t * total_proj_dim + conv_dim]);
+ }
+ let scan_state = state_out_array.as_slice::<f32>().to_vec();
+
+ assert_close(&ref_outputs, &scan_outputs, 1e-4, 1e-3, "ConvScan output");
+ assert_close(&ref_state, &scan_state, 1e-5, 1e-4, "ConvScan state");
+}
+
+// DeltaNetUpdate (decode)
+
+fn test_delta_net_update_impl(
+ num_v_heads: usize,
+ num_k_heads: usize,
+ head_k_dim: usize,
+ head_v_dim: usize,
+ label: &str,
+) {
+ let key_dim = num_k_heads * head_k_dim;
+ let value_dim = num_v_heads * head_v_dim;
+ let conv_dim = 2 * key_dim + value_dim;
+ let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;
+ let state_size = num_v_heads * head_k_dim * head_v_dim;
+
+ let in_proj: Vec<f32> = (0..total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect();
+ let a_log: Vec<f32> = (0..num_v_heads).map(|i| -1.5 + (i as f32) * 0.05).collect();
+ let dt_bias: Vec<f32> = (0..num_v_heads).map(|i| 0.3 + (i as f32) * 0.02).collect();
+ let norm_weight: Vec<f32> = (0..head_v_dim).map(|i| 0.9 + (i as f32) * 0.001).collect();
+ let state: Vec<f32> = (0..state_size).map(|i| ((i % 29) as f32) * 0.005 - 0.05).collect();
+
+ let (cpu_out, cpu_state) = run_delta_net_update::<Cpu>(
+ &in_proj,
+ &a_log,
+ &dt_bias,
+ &norm_weight,
+ &state,
+ num_v_heads as u32,
+ num_k_heads as u32,
+ head_k_dim as u32,
+ head_v_dim as u32,
+ key_dim as u32,
+ value_dim as u32,
+ );
+ let (gpu_out, gpu_state) = run_delta_net_update::<Metal>(
+ &in_proj,
+ &a_log,
+ &dt_bias,
+ &norm_weight,
+ &state,
+ num_v_heads as u32,
+ num_k_heads as u32,
+ head_k_dim as u32,
+ head_v_dim as u32,
+ key_dim as u32,
+ value_dim as u32,
+ );
+
+ assert_close(&cpu_out, &gpu_out, 1e-3, 1e-2, &format!("{label} output"));
+ assert_close(&cpu_state, &gpu_state, 1e-4, 1e-3, &format!("{label} state"));
+}
+
+#[test]
+fn test_delta_net_update_qwen35_shapes() {
+ test_delta_net_update_impl(48, 16, 128, 128, "DeltaNetUpdate Qwen3.5");
+}
+
+// DeltaNetPrefill + NormGate
+
+fn run_prefill_with_norm_gate(
+ in_proj: &[f32],
+ a_log: &[f32],
+ dt_bias: &[f32],
+ norm_weight: &[f32],
+ state: &[f32],
+ num_v_heads: usize,
+ num_k_heads: usize,
+ head_k_dim: usize,
+ head_v_dim: usize,
+ suffix_len: usize,
+) -> (Vec<f32>, Vec<f32>) {
+ let key_dim = num_k_heads * head_k_dim;
+ let value_dim = num_v_heads * head_v_dim;
+ let conv_dim = 2 * key_dim + value_dim;
+ let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;
+ let num_dv_groups = (head_v_dim as u32 + 7) / 8;
+
+ let context = <Metal as Backend>::Context::new().expect("context");
+ let in_proj_array = context.create_array_from(&[in_proj.len()], in_proj, "in_proj");
+ let a_log_array = context.create_array_from(&[a_log.len()], a_log, "a_log");
+ let dt_bias_array = context.create_array_from(&[dt_bias.len()], dt_bias, "dt_bias");
+ let norm_weight_array = context.create_array_from(&[norm_weight.len()], norm_weight, "norm_weight");
+ let state_array = context.create_array_from(&[state.len()], state, "state");
+ let out_array = context.create_array_zeros(&[suffix_len * value_dim], DataType::F32, "out");
+ let q_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "q_norm");
+ let k_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "k_norm");
+
+ let beta_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "beta");
+ let decay_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "decay");
+
+ let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(
+ &context,
+ DataType::F32,
+ head_k_dim as u32,
+ )
+ .unwrap();
+ let prefill_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillKernel::new(
+ &context,
+ DataType::F32,
+ head_k_dim as u32,
+ )
+ .unwrap();
+ let norm_k =
+ <<Metal as Backend>::Kernels as Kernels>::DeltaNetNormGateKernel::new(&context, DataType::F32).unwrap();
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ prep_k.encode(
+ in_proj_array.buffer().borrow().deref(),
+ a_log_array.buffer().borrow().deref(),
+ dt_bias_array.buffer().borrow().deref(),
+ q_norm_array.buffer().borrow_mut().deref_mut(),
+ k_norm_array.buffer().borrow_mut().deref_mut(),
+ beta_array.buffer().borrow_mut().deref_mut(),
+ decay_array.buffer().borrow_mut().deref_mut(),
+ num_v_heads as u32,
+ num_k_heads as u32,
+ key_dim as u32,
+ value_dim as u32,
+ suffix_len as u32,
+ &mut encoder,
+ );
+ prefill_k.encode(
+ q_norm_array.buffer().borrow().deref(),
+ k_norm_array.buffer().borrow().deref(),
+ beta_array.buffer().borrow().deref(),
+ decay_array.buffer().borrow().deref(),
+ in_proj_array.buffer().borrow().deref(),
+ state_array.buffer().borrow_mut().deref_mut(),
+ out_array.buffer().borrow_mut().deref_mut(),
+ num_v_heads as u32,
+ num_k_heads as u32,
+ head_v_dim as u32,
+ key_dim as u32,
+ value_dim as u32,
+ suffix_len as u32,
+ num_dv_groups,
+ &mut encoder,
+ );
+ norm_k.encode(
+ out_array.buffer().borrow_mut().deref_mut(),
+ in_proj_array.buffer().borrow().deref(),
+ norm_weight_array.buffer().borrow().deref(),
+ num_v_heads as u32,
+ head_v_dim as u32,
+ value_dim as u32,
+ conv_dim as u32,
+ total_proj_dim as u32,
+ 1e-6f32,
+ suffix_len as u32,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ (out_array.as_slice::<f32>().to_vec(), state_array.as_slice::<f32>().to_vec())
+}
+
+fn test_prefill_norm_gate_impl(
+ num_v_heads: usize,
+ num_k_heads: usize,
+ head_k_dim: usize,
+ head_v_dim: usize,
+ suffix_len: usize,
+ label: &str,
+) {
+ let key_dim = num_k_heads * head_k_dim;
+ let value_dim = num_v_heads * head_v_dim;
+ let conv_dim = 2 * key_dim + value_dim;
+ let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;
+ let state_size = num_v_heads * head_k_dim * head_v_dim;
+
+ let in_proj: Vec<f32> = (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect();
+ let a_log: Vec<f32> = (0..num_v_heads).map(|i| -1.5 + (i as f32) * 0.05).collect();
+ let dt_bias: Vec<f32> = (0..num_v_heads).map(|i| 0.3 + (i as f32) * 0.02).collect();
+ let norm_weight: Vec<f32> = (0..head_v_dim).map(|i| 0.9 + (i as f32) * 0.001).collect();
+ let state: Vec<f32> = (0..state_size).map(|i| ((i % 29) as f32) * 0.005 - 0.05).collect();
+
+ // Reference: fused decode kernel token-by-token
+ let mut ref_state = state.clone();
+ let mut ref_outputs = vec![0.0f32; suffix_len * value_dim];
+ for t in 0..suffix_len {
+ let token_in: Vec<f32> = in_proj[t * total_proj_dim..(t + 1) * total_proj_dim].to_vec();
+ let (out, new_state) = run_delta_net_update::<Cpu>(
+ &token_in,
+ &a_log,
+ &dt_bias,
+ &norm_weight,
+ &ref_state,
+ num_v_heads as u32,
+ num_k_heads as u32,
+ head_k_dim as u32,
+ head_v_dim as u32,
+ key_dim as u32,
+ value_dim as u32,
+ );
+ ref_state = new_state;
+ ref_outputs[t * value_dim..(t + 1) * value_dim].copy_from_slice(&out);
+ }
+
+ let (gpu_out, gpu_state) = run_prefill_with_norm_gate(
+ &in_proj,
+ &a_log,
+ &dt_bias,
+ &norm_weight,
+ &state,
+ num_v_heads,
+ num_k_heads,
+ head_k_dim,
+ head_v_dim,
+ suffix_len,
+ );
+
+ assert_close(&ref_outputs, &gpu_out, 1e-3, 1e-2, &format!("{label} output"));
+ assert_close(&ref_state, &gpu_state, 1e-3, 1e-2, &format!("{label} state"));
+}
+
+#[test]
+fn test_delta_net_prefill_qwen35_shapes() {
+ test_prefill_norm_gate_impl(48, 16, 128, 128, 32, "Prefill+NormGate Qwen3.5");
+}
+
+#[test]
+fn test_delta_net_prefill_prep() {
+ let num_v_heads = 48usize;
+ let num_k_heads = 16usize;
+ let head_k_dim = 128usize;
+ let head_v_dim = 128usize;
+ let suffix_len = 8usize;
+
+ let key_dim = num_k_heads * head_k_dim;
+ let value_dim = num_v_heads * head_v_dim;
+ let conv_dim = 2 * key_dim + value_dim;
+ let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;
+
+ let in_proj: Vec<f32> = (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect();
+ let a_log: Vec<f32> = (0..num_v_heads).map(|i| -1.5 + (i as f32) * 0.05).collect();
+ let dt_bias: Vec<f32> = (0..num_v_heads).map(|i| 0.3 + (i as f32) * 0.02).collect();
+
+ // CPU reference via Kernels trait
+ let cpu_ctx = <Cpu as Backend>::Context::new().expect("cpu context");
+ let cpu_in_proj = cpu_ctx.create_array_from(&[in_proj.len()], &in_proj, "in_proj");
+ let cpu_a_log = cpu_ctx.create_array_from(&[a_log.len()], &a_log, "a_log");
+ let cpu_dt_bias = cpu_ctx.create_array_from(&[dt_bias.len()], &dt_bias, "dt_bias");
+ let cpu_q = cpu_ctx.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "q");
+ let cpu_k = cpu_ctx.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "k");
+ let cpu_beta = cpu_ctx.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "beta");
+ let cpu_decay = cpu_ctx.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "decay");
+
+ let cpu_prep = <<Cpu as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(
+ &cpu_ctx,
+ DataType::F32,
+ head_k_dim as u32,
+ )
+ .unwrap();
+ let mut cpu_enc = Encoder::new(cpu_ctx.as_ref()).expect("encoder");
+ cpu_prep.encode(
+ cpu_in_proj.buffer().borrow().deref(),
+ cpu_a_log.buffer().borrow().deref(),
+ cpu_dt_bias.buffer().borrow().deref(),
+ cpu_q.buffer().borrow_mut().deref_mut(),
+ cpu_k.buffer().borrow_mut().deref_mut(),
+ cpu_beta.buffer().borrow_mut().deref_mut(),
+ cpu_decay.buffer().borrow_mut().deref_mut(),
+ num_v_heads as u32,
+ num_k_heads as u32,
+ key_dim as u32,
+ value_dim as u32,
+ suffix_len as u32,
+ &mut cpu_enc,
+ );
+ cpu_enc.end_encoding().submit().wait_until_completed().unwrap();
+
+ let ref_q = cpu_q.as_slice::<f32>().to_vec();
+ let ref_k = cpu_k.as_slice::<f32>().to_vec();
+ let ref_beta = cpu_beta.as_slice::<f32>().to_vec();
+ let ref_decay = cpu_decay.as_slice::<f32>().to_vec();
+
+ // Metal
+ let context = <Metal as Backend>::Context::new().expect("context");
+ let in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj, "in_proj");
+ let a_log_array = context.create_array_from(&[a_log.len()], &a_log, "a_log");
+ let dt_bias_array = context.create_array_from(&[dt_bias.len()], &dt_bias, "dt_bias");
+ let q_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "q_norm");
+ let k_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "k_norm");
+
+ let beta_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "beta");
+ let decay_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "decay");
+
+ let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(
+ &context,
+ DataType::F32,
+ head_k_dim as u32,
+ )
+ .unwrap();
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("encoder");
+ prep_k.encode(
+ in_proj_array.buffer().borrow().deref(),
+ a_log_array.buffer().borrow().deref(),
+ dt_bias_array.buffer().borrow().deref(),
+ q_norm_array.buffer().borrow_mut().deref_mut(),
+ k_norm_array.buffer().borrow_mut().deref_mut(),
+ beta_array.buffer().borrow_mut().deref_mut(),
+ decay_array.buffer().borrow_mut().deref_mut(),
+ num_v_heads as u32,
+ num_k_heads as u32,
+ key_dim as u32,
+ value_dim as u32,
+ suffix_len as u32,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ let gpu_q = q_norm_array.as_slice::<f32>();
+ let gpu_k = k_norm_array.as_slice::<f32>();
+ let gpu_beta = beta_array.as_slice::<f32>();
+ let gpu_decay = decay_array.as_slice::<f32>();
+
+ assert_close(gpu_q, &ref_q, 1e-4, 1e-3, "prep q_norm");
+ assert_close(gpu_k, &ref_k, 1e-4, 1e-3, "prep k_norm");
+ assert_close(gpu_beta, &ref_beta, 1e-4, 1e-3, "prep beta");
+ assert_close(gpu_decay, &ref_decay, 1e-4, 1e-3, "prep decay");
+}
+
+#[test]
+#[ignore]
+fn bench_delta_net_prefill() {
+ use crate::common::perf::run_perf_with_warmup;
+
+ let num_v_heads = 48usize;
+ let num_k_heads = 16usize;
+ let head_k_dim = 128usize;
+ let head_v_dim = 128usize;
+ let suffix_len = 32usize;
+
+ let key_dim = num_k_heads * head_k_dim;
+ let value_dim = num_v_heads * head_v_dim;
+ let conv_dim = 2 * key_dim + value_dim;
+ let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;
+ let state_size = num_v_heads * head_k_dim * head_v_dim;
+
+ let in_proj: Vec<f32> = (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect();
+ let a_log: Vec<f32> = (0..num_v_heads).map(|i| -1.5 + (i as f32) * 0.05).collect();
+ let dt_bias: Vec<f32> = (0..num_v_heads).map(|i| 0.3 + (i as f32) * 0.02).collect();
+ let norm_weight: Vec<f32> = (0..head_v_dim).map(|i| 0.9 + (i as f32) * 0.001).collect();
+
+ let context = <Metal as Backend>::Context::new().expect("context");
+ let in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj, "in_proj");
+ let a_log_array = context.create_array_from(&[a_log.len()], &a_log, "a_log");
+ let dt_bias_array = context.create_array_from(&[dt_bias.len()], &dt_bias, "dt_bias");
+ let norm_weight_array = context.create_array_from(&[norm_weight.len()], &norm_weight, "norm_weight");
+ let out_array = context.create_array_zeros(&[suffix_len * value_dim], DataType::F32, "out");
+ let q_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "q_norm");
+ let k_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "k_norm");
+
+ let beta_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "beta");
+ let decay_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "decay");
+
+ let num_dv_groups = (head_v_dim as u32 + 7) / 8;
+
+ let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(
+ &context,
+ DataType::F32,
+ head_k_dim as u32,
+ )
+ .unwrap();
+ let prefill_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillKernel::new(
+ &context,
+ DataType::F32,
+ head_k_dim as u32,
+ )
+ .unwrap();
+ let norm_k =
+ <<Metal as Backend>::Kernels as Kernels>::DeltaNetNormGateKernel::new(&context, DataType::F32).unwrap();
+
+ eprintln!("\n=== DeltaNet Prefill Benchmark (Qwen3.5 shapes) ===");
+ eprintln!(
+ " heads={num_v_heads} k_heads={num_k_heads} head_k={head_k_dim} head_v={head_v_dim} tokens={suffix_len}"
+ );
+ eprintln!(" state_size={state_size} ({:.2} MB)", state_size as f64 * 4.0 / 1024.0 / 1024.0);
+
+ let prep_result = run_perf_with_warmup("prep_only", 5, 50, || {
+ let mut encoder = Encoder::new(context.as_ref()).expect("encoder");
+ prep_k.encode(
+ in_proj_array.buffer().borrow().deref(),
+ a_log_array.buffer().borrow().deref(),
+ dt_bias_array.buffer().borrow().deref(),
+ q_norm_array.buffer().borrow_mut().deref_mut(),
+ k_norm_array.buffer().borrow_mut().deref_mut(),
+ beta_array.buffer().borrow_mut().deref_mut(),
+ decay_array.buffer().borrow_mut().deref_mut(),
+ num_v_heads as u32,
+ num_k_heads as u32,
+ key_dim as u32,
+ value_dim as u32,
+ suffix_len as u32,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ });
+ prep_result.print();
+
+ // Benchmark prep + prefill + norm_gate (production path)
+ let mut state_array = context.create_array_zeros(&[state_size], DataType::F32, "state");
+
+ let prefill_result = run_perf_with_warmup("prep+prefill+norm_gate", 5, 50, || {
+ state_array.as_slice_mut::<f32>().fill(0.0);
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("encoder");
+ prep_k.encode(
+ in_proj_array.buffer().borrow().deref(),
+ a_log_array.buffer().borrow().deref(),
+ dt_bias_array.buffer().borrow().deref(),
+ q_norm_array.buffer().borrow_mut().deref_mut(),
+ k_norm_array.buffer().borrow_mut().deref_mut(),
+ beta_array.buffer().borrow_mut().deref_mut(),
+ decay_array.buffer().borrow_mut().deref_mut(),
+ num_v_heads as u32,
+ num_k_heads as u32,
+ key_dim as u32,
+ value_dim as u32,
+ suffix_len as u32,
+ &mut encoder,
+ );
+ prefill_k.encode(
+ q_norm_array.buffer().borrow().deref(),
+ k_norm_array.buffer().borrow().deref(),
+ beta_array.buffer().borrow().deref(),
+ decay_array.buffer().borrow().deref(),
+ in_proj_array.buffer().borrow().deref(),
+ state_array.buffer().borrow_mut().deref_mut(),
+ out_array.buffer().borrow_mut().deref_mut(),
+ num_v_heads as u32,
+ num_k_heads as u32,
+ head_v_dim as u32,
+ key_dim as u32,
+ value_dim as u32,
+ suffix_len as u32,
+ num_dv_groups,
+ &mut encoder,
+ );
+ norm_k.encode(
+ out_array.buffer().borrow_mut().deref_mut(),
+ in_proj_array.buffer().borrow().deref(),
+ norm_weight_array.buffer().borrow().deref(),
+ num_v_heads as u32,
+ head_v_dim as u32,
+ value_dim as u32,
+ conv_dim as u32,
+ total_proj_dim as u32,
+ 1e-6f32,
+ suffix_len as u32,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ });
+ prefill_result.print();
+}
diff --git a/crates/uzu/tests/unit/kernel/mod.rs b/crates/uzu/tests/unit/kernel/mod.rs
index 3e965aafb..92faaafff 100644
--- a/crates/uzu/tests/unit/kernel/mod.rs
+++ b/crates/uzu/tests/unit/kernel/mod.rs
@@ -6,6 +6,7 @@ mod activation_mul_test;
mod activation_test;
mod attention;
mod audio;
+mod delta_net_test;
mod embedding;
mod kv_cache_update_test;
mod layer_norm_test;