A high-performance inference engine for AI models
Qwen3.5 (#299)
CCYeh98304ccf0a10
diff --git a/CODEOWNERS b/CODEOWNERSindex 6ed9c9df9..3a21cab36 100644--- a/CODEOWNERS+++ b/CODEOWNERS@@ -4,7 +4,7 @@/crates/uzu/build @uuuvn/crates/uzu/examples @alexxale/crates/uzu/src/backends/cpu @uuuvn-/crates/uzu/src/backends/metal @LuckyIYI @eugenebokhan @uuuvn+/crates/uzu/src/backends/metal @LuckyIYI @eugenebokhan @uuuvn @CC-Yeh/crates/uzu/src/classifier @eugenebokhan/crates/uzu/src/config @uuuvn/crates/uzu/src/device @uuuvndiff --git a/crates/benchmarks/src/runner/runner.rs b/crates/benchmarks/src/runner/runner.rsindex a6b0f382d..81371ca74 100644--- a/crates/benchmarks/src/runner/runner.rs+++ b/crates/benchmarks/src/runner/runner.rs@@ -78,6 +78,9 @@ impl Runner {let input = Input::Messages(self.task.messages.clone());+ let warmup_config = RunConfig::default().tokens_limit(1);+ session.run(input.clone(), warmup_config, Some(|_: Output| true))?;+let mut results: Vec<TaskResult> = Vec::new();for run_index in 0..self.task.number_of_runs {let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();diff --git a/crates/uzu/src/backends/cpu/kernel/delta_net/conv_scan.rs b/crates/uzu/src/backends/cpu/kernel/delta_net/conv_scan.rsnew file mode 100644index 000000000..6dfde8ba1--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/delta_net/conv_scan.rs@@ -0,0 +1,73 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::{ArrayElement, backends::common::gpu_types::ActivationType};++// Multi-token causal conv1d with SiLU for DeltaNet.+// Reads from conv_padded (Conv1dPack output), writes conv'd+SiLU'd values+// back to in_proj with stride out_stride, updates conv state.+//+// conv_padded: [state_stride + suffix_len, row_stride] — packed history + input+// conv_weight: [conv_dim, kernel_size]+// bias: [conv_dim] (optional)+// in_proj: [suffix_len, out_stride] — first conv_dim channels overwritten+// state_out: [conv_dim, state_stride]+#[kernel(DeltaNetConvScan)]+#[variants(T, f32, f16, bf16)]+pub fn delta_net_conv_scan<T: ArrayElement + Float>(+ conv_padded: *const T,+ conv_weight: *const T,+ #[optional(has_bias)] bias: Option<*const T>,+ in_proj: *mut T,+ state_out: *mut T,+ suffix_len: u32,+ kernel_size: u32,+ row_stride: u32,+ state_stride: u32,+ conv_dim: u32,+ out_stride: u32,+ #[specialize] has_bias: bool,+) {+ let suffix_len = suffix_len as usize;+ let kernel_size = kernel_size as usize;+ let row_stride = row_stride as usize;+ let state_stride = state_stride as usize;+ let conv_dim = conv_dim as usize;+ let out_stride = out_stride as usize;++ for token in 0..suffix_len {+ for channel in 0..conv_dim {+ let w_offset = channel * kernel_size;++ let mut acc = if has_bias {+ unsafe { (*bias.unwrap().add(channel)).to_f32().unwrap() }+ } else {+ 0.0f32+ };++ for tap in 0..kernel_size {+ let padded_row = token + tap;+ let padded_idx = padded_row * row_stride + channel;+ let sample = unsafe { (*conv_padded.add(padded_idx)).to_f32().unwrap() };+ acc += unsafe { (*conv_weight.add(w_offset + tap)).to_f32().unwrap() } * sample;+ }++ unsafe {+ *in_proj.add(token * out_stride + channel) = T::from(ActivationType::SILU.activate(acc)).unwrap();+ }+ }+ }++ // state_out layout: [conv_dim, state_stride] (channel-major)+ for channel in 0..conv_dim {+ for tap in 0..state_stride {+ let padded_row = suffix_len + tap;+ let padded_idx = padded_row * row_stride + channel;+ let sample = unsafe { (*conv_padded.add(padded_idx)).to_f32().unwrap() };+ unsafe {+ *state_out.add(channel * state_stride + tap) = T::from(sample).unwrap();+ }+ }+ }+}diff --git a/crates/uzu/src/backends/cpu/kernel/delta_net/conv_update.rs b/crates/uzu/src/backends/cpu/kernel/delta_net/conv_update.rsnew file mode 100644index 000000000..a4cb8313d--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/delta_net/conv_update.rs@@ -0,0 +1,56 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::{ArrayElement, backends::common::gpu_types::ActivationType};++// Single-token causal conv1d with SiLU, in-place.+#[kernel(DeltaNetConvUpdate)]+#[variants(T, f32, f16, bf16)]+pub fn delta_net_conv_update<T: ArrayElement + Float>(+ conv_weight: *const T,+ #[optional(has_bias)] bias: Option<*const T>,+ in_out: *mut T,+ state: *mut T,+ kernel_size: u32,+ conv_dim: u32,+ state_stride: u32,+ #[specialize] has_bias: bool,+) {+ let kernel_size = kernel_size as usize;+ let conv_dim = conv_dim as usize;+ let state_stride = state_stride as usize;+ let tap_count = kernel_size - 1;++ for channel in 0..conv_dim {+ let state_offset = channel * state_stride;+ let weight_offset = channel * kernel_size;++ let x = unsafe { (*in_out.add(channel)).to_f32().unwrap() };++ let mut acc = if has_bias {+ unsafe { (*bias.unwrap().add(channel)).to_f32().unwrap() }+ } else {+ 0.0f32+ };++ for tap in 0..tap_count {+ acc += unsafe { (*state.add(state_offset + tap)).to_f32().unwrap() }+ * unsafe { (*conv_weight.add(weight_offset + tap)).to_f32().unwrap() };+ }+ acc += x * unsafe { (*conv_weight.add(weight_offset + tap_count)).to_f32().unwrap() };++ unsafe {+ *in_out.add(channel) = T::from(ActivationType::SILU.activate(acc)).unwrap();+ }++ for tap in 1..tap_count {+ unsafe {+ *state.add(state_offset + tap - 1) = *state.add(state_offset + tap);+ }+ }+ unsafe {+ *state.add(state_offset + tap_count - 1) = T::from(x).unwrap();+ }+ }+}diff --git a/crates/uzu/src/backends/cpu/kernel/delta_net/mod.rs b/crates/uzu/src/backends/cpu/kernel/delta_net/mod.rsnew file mode 100644index 000000000..8f29d2c4f--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/delta_net/mod.rs@@ -0,0 +1,6 @@+pub mod conv_scan;+pub mod conv_update;+pub mod norm_gate;+pub mod prefill;+pub mod prefill_prep;+pub mod update;diff --git a/crates/uzu/src/backends/cpu/kernel/delta_net/norm_gate.rs b/crates/uzu/src/backends/cpu/kernel/delta_net/norm_gate.rsnew file mode 100644index 000000000..6edf80099--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/delta_net/norm_gate.rs@@ -0,0 +1,61 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::{ArrayElement, backends::common::gpu_types::ActivationType};++// In-place RMSNorm + SiLU gate for DeltaNet prefill output.+//+// in_out: [suffix_len, value_dim] — raw input, overwritten with final output+// in_proj: [suffix_len, total_proj_dim] — for reading z values+// norm_weight: [head_v_dim]+//+// Per (token, head): in_out[i] = in_out[i] * inv_rms * norm_weight[i] * silu(z[i])+// where inv_rms = rsqrt(mean(in_out^2) + epsilon)+// and z is at in_proj[token * total_proj_dim + conv_dim + hv * head_v_dim + i]+#[kernel(DeltaNetNormGate)]+#[variants(T, f32, f16, bf16)]+pub fn delta_net_norm_gate<T: ArrayElement + Float>(+ in_out: *mut T,+ in_proj: *const T,+ norm_weight: *const T,+ num_v_heads: u32,+ head_v_dim: u32,+ value_dim: u32,+ conv_dim: u32,+ total_proj_dim: u32,+ norm_epsilon: f32,+ suffix_len: u32,+) {+ let in_out_ptr = in_out as *const T;+ let num_v_heads = num_v_heads as usize;+ let head_v_dim = head_v_dim as usize;+ let value_dim = value_dim as usize;+ let conv_dim = conv_dim as usize;+ let total_proj_dim = total_proj_dim as usize;+ let suffix_len = suffix_len as usize;++ for token in 0..suffix_len {+ for hv in 0..num_v_heads {+ let base = token * value_dim + hv * head_v_dim;++ let mut sumsq = 0.0f32;+ for i in 0..head_v_dim {+ let val = unsafe { (*in_out_ptr.add(base + i)).to_f32().unwrap() };+ sumsq += val * val;+ }+ let inv_rms = 1.0 / (sumsq / head_v_dim as f32 + norm_epsilon).sqrt();++ for i in 0..head_v_dim {+ let o_i = unsafe { (*in_out_ptr.add(base + i)).to_f32().unwrap() };+ let norm_w = unsafe { (*norm_weight.add(i)).to_f32().unwrap() };+ let z_idx = token * total_proj_dim + conv_dim + hv * head_v_dim + i;+ let z_i = unsafe { (*in_proj.add(z_idx)).to_f32().unwrap() };+ let final_val = o_i * inv_rms * norm_w * ActivationType::SILU.activate(z_i);+ unsafe {+ *in_out.add(base + i) = T::from(final_val).unwrap();+ }+ }+ }+ }+}diff --git a/crates/uzu/src/backends/cpu/kernel/delta_net/prefill.rs b/crates/uzu/src/backends/cpu/kernel/delta_net/prefill.rsnew file mode 100644index 000000000..a3fed8274--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/delta_net/prefill.rs@@ -0,0 +1,74 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(DeltaNetPrefill)]+#[variants(T, f32, f16, bf16)]+#[variants(HEAD_K_DIM, 128)]+pub fn delta_net_prefill<T: ArrayElement + Float, const HEAD_K_DIM: u32>(+ q_norm: *const f32,+ k_norm: *const f32,+ beta_buf: *const f32,+ decay_buf: *const f32,+ in_proj: *const T,+ state: *mut T,+ out: *mut T,+ num_v_heads: u32,+ num_k_heads: u32,+ head_v_dim: u32,+ key_dim: u32,+ value_dim: u32,+ suffix_len: u32,+ #[allow(unused)] num_dv_groups: u32,+) {+ let state_ptr = state as *const T;+ let num_v_heads = num_v_heads as usize;+ let num_k_heads = num_k_heads as usize;+ let head_k_dim = HEAD_K_DIM as usize;+ let head_v_dim = head_v_dim as usize;+ let key_dim = key_dim as usize;+ let value_dim = value_dim as usize;+ let conv_dim = 2 * key_dim + value_dim;+ let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;+ let suffix_len = suffix_len as usize;+ let groups_per_head = num_v_heads / num_k_heads;++ for hv in 0..num_v_heads {+ let hk = hv / groups_per_head;++ for token in 0..suffix_len {+ let qk_off = token * key_dim + hk * head_k_dim;+ let decay = unsafe { *decay_buf.add(token * num_v_heads + hv) };+ let beta = unsafe { *beta_buf.add(token * num_v_heads + hv) };++ for i in 0..head_v_dim {+ // State layout: [Hv, Dv, Dk]+ let state_off = (hv * head_v_dim + i) * head_k_dim;++ let mut kv_mem = 0.0f32;+ for j in 0..head_k_dim {+ let s = unsafe { (*state_ptr.add(state_off + j)).to_f32().unwrap() };+ kv_mem += (decay * s) * unsafe { *k_norm.add(qk_off + j) };+ }++ let v_val = unsafe {+ (*in_proj.add(token * total_proj_dim + 2 * key_dim + hv * head_v_dim + i)).to_f32().unwrap()+ };+ let delta = beta * (v_val - kv_mem);++ let mut o_val = 0.0f32;+ for j in 0..head_k_dim {+ let s = unsafe { (*state_ptr.add(state_off + j)).to_f32().unwrap() };+ let k_j = unsafe { *k_norm.add(qk_off + j) };+ let new_s = decay * s + k_j * delta;+ unsafe { *state.add(state_off + j) = T::from(new_s).unwrap() };+ o_val += new_s * unsafe { *q_norm.add(qk_off + j) };+ }++ unsafe { *out.add(token * value_dim + hv * head_v_dim + i) = T::from(o_val).unwrap() };+ }+ }+ }+}diff --git a/crates/uzu/src/backends/cpu/kernel/delta_net/prefill_prep.rs b/crates/uzu/src/backends/cpu/kernel/delta_net/prefill_prep.rsnew file mode 100644index 000000000..a05fcf911--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/delta_net/prefill_prep.rs@@ -0,0 +1,92 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++// Pre-compute L2-normalized q/k, beta, and decay for all tokens.+#[kernel(DeltaNetPrefillPrep)]+#[variants(T, f32, f16, bf16)]+#[variants(HEAD_K_DIM, 128)]+pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(+ in_proj: *const T,+ a_log: *const T,+ dt_bias: *const T,+ q_norm_out: *mut f32,+ k_norm_out: *mut f32,+ beta_out: *mut f32,+ decay_out: *mut f32,+ num_v_heads: u32,+ num_k_heads: u32,+ key_dim: u32,+ value_dim: u32,+ suffix_len: u32,+) {+ let num_v_heads = num_v_heads as usize;+ let num_k_heads = num_k_heads as usize;+ let head_k_dim = HEAD_K_DIM as usize;+ let key_dim = key_dim as usize;+ let value_dim = value_dim as usize;+ let conv_dim = 2 * key_dim + value_dim;+ let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;+ let suffix_len = suffix_len as usize;+ let groups_per_head = num_v_heads / num_k_heads;++ for token in 0..suffix_len {+ let tok_offset = token * total_proj_dim;++ for hk in 0..num_k_heads {+ // Load and L2-normalize q+ let q_off = tok_offset + hk * head_k_dim;+ let mut q_sq = 0.0f32;+ for j in 0..head_k_dim {+ let v = unsafe { (*in_proj.add(q_off + j)).to_f32().unwrap() };+ q_sq += v * v;+ }+ let q_inv = 1.0 / (q_sq + 1e-6).sqrt();+ let q_scale = 1.0 / (head_k_dim as f32).sqrt();+ for j in 0..head_k_dim {+ let v = unsafe { (*in_proj.add(q_off + j)).to_f32().unwrap() };+ unsafe { *q_norm_out.add(token * key_dim + hk * head_k_dim + j) = v * q_inv * q_scale };+ }++ // Load and L2-normalize k+ let k_off = tok_offset + key_dim + hk * head_k_dim;+ let mut k_sq = 0.0f32;+ for j in 0..head_k_dim {+ let v = unsafe { (*in_proj.add(k_off + j)).to_f32().unwrap() };+ k_sq += v * v;+ }+ let k_inv = 1.0 / (k_sq + 1e-6).sqrt();+ for j in 0..head_k_dim {+ let v = unsafe { (*in_proj.add(k_off + j)).to_f32().unwrap() };+ unsafe { *k_norm_out.add(token * key_dim + hk * head_k_dim + j) = v * k_inv };+ }++ // Beta and decay for each v-head of this k-head+ for group in 0..groups_per_head {+ let hv = hk * groups_per_head + group;++ let beta_raw = unsafe { (*in_proj.add(tok_offset + conv_dim + value_dim + hv)).to_f32().unwrap() };+ let beta = 1.0 / (1.0 + (-beta_raw).exp());++ let a_log_val = unsafe { (*a_log.add(hv)).to_f32().unwrap() };+ let dt_bias_val = unsafe { (*dt_bias.add(hv)).to_f32().unwrap() };+ let a_raw =+ unsafe { (*in_proj.add(tok_offset + conv_dim + value_dim + num_v_heads + hv)).to_f32().unwrap() };+ let sp_in = a_raw + dt_bias_val;+ let sp = if sp_in > 20.0 {+ sp_in+ } else {+ (1.0 + sp_in.exp()).ln()+ };+ let decay = (-a_log_val.exp() * sp).exp();++ unsafe {+ *beta_out.add(token * num_v_heads + hv) = beta;+ *decay_out.add(token * num_v_heads + hv) = decay;+ };+ }+ }+ }+}diff --git a/crates/uzu/src/backends/cpu/kernel/delta_net/update.rs b/crates/uzu/src/backends/cpu/kernel/delta_net/update.rsnew file mode 100644index 000000000..87b070023--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/delta_net/update.rs@@ -0,0 +1,144 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::{ArrayElement, backends::common::gpu_types::ActivationType};++// Single-token delta net update: decay state, apply delta rule, RMSNorm + SiLU gate.+// Steps per v-head:+// 1. L2-normalize and scale q, k+// 2. Compute decay from a_log and softplus(a_raw + dt_bias)+// 3. For each v-dim: read state, compute output, update state with delta rule+// 4. RMSNorm output, gate with SiLU(z), write result+#[kernel(DeltaNetUpdate)]+#[variants(T, f32, f16, bf16)]+#[variants(HEAD_K_DIM, 128)]+pub fn delta_net_update<T: ArrayElement + Float, const HEAD_K_DIM: u32>(+ in_proj: *const T,+ a_log: *const T,+ dt_bias: *const T,+ norm_weight: *const T,+ state: *mut T,+ out: *mut T,+ num_v_heads: u32,+ num_k_heads: u32,+ head_v_dim: u32,+ key_dim: u32,+ value_dim: u32,+ norm_epsilon: f32,+) {+ let state_ptr = state as *const T;++ let num_v_heads = num_v_heads as usize;+ let num_k_heads = num_k_heads as usize;+ debug_assert!(num_v_heads % num_k_heads == 0, "num_v_heads must be a multiple of num_k_heads");+ let head_k_dim = HEAD_K_DIM as usize;+ let head_v_dim = head_v_dim as usize;+ let key_dim = key_dim as usize;+ let value_dim = value_dim as usize;+ let conv_dim = 2 * key_dim + value_dim;++ // in_proj layout (after conv):+ // [0..key_dim): q (conv'd + SiLU)+ // [key_dim..2*key_dim): k (conv'd + SiLU)+ // [2*key_dim..2*key_dim+value_dim): v (conv'd + SiLU)+ // [conv_dim..conv_dim+value_dim): z (raw)+ // [conv_dim+value_dim..conv_dim+value_dim+num_v_heads): beta_raw (raw)+ // [conv_dim+value_dim+num_v_heads..conv_dim+value_dim+2*num_v_heads): a_raw (raw)++ for hv in 0..num_v_heads {+ let hk = hv / (num_v_heads / num_k_heads);++ // Load q and k for this head+ let q_offset = hk * head_k_dim;+ let k_offset = key_dim + hk * head_k_dim;++ let mut q = vec![0.0f32; head_k_dim];+ let mut k = vec![0.0f32; head_k_dim];+ for j in 0..head_k_dim {+ q[j] = unsafe { (*in_proj.add(q_offset + j)).to_f32().unwrap() };+ k[j] = unsafe { (*in_proj.add(k_offset + j)).to_f32().unwrap() };+ }++ // L2 normalize q and k+ let q_norm_sq: f32 = q.iter().map(|x| x * x).sum();+ let k_norm_sq: f32 = k.iter().map(|x| x * x).sum();+ let q_inv_norm = 1.0 / (q_norm_sq + 1e-6).sqrt();+ let k_inv_norm = 1.0 / (k_norm_sq + 1e-6).sqrt();+ for j in 0..head_k_dim {+ q[j] *= q_inv_norm;+ k[j] *= k_inv_norm;+ }++ // Scale q by head_k_dim^-0.5+ let q_scale = 1.0 / (head_k_dim as f32).sqrt();+ for j in 0..head_k_dim {+ q[j] *= q_scale;+ }++ // Compute gating+ let beta_raw = unsafe { (*in_proj.add(conv_dim + value_dim + hv)).to_f32().unwrap() };+ let beta = 1.0 / (1.0 + (-beta_raw).exp()); // sigmoid++ let a_raw = unsafe { (*in_proj.add(conv_dim + value_dim + num_v_heads + hv)).to_f32().unwrap() };+ let a_log_val = unsafe { (*a_log.add(hv)).to_f32().unwrap() };+ let dt_bias_val = unsafe { (*dt_bias.add(hv)).to_f32().unwrap() };++ // softplus(a_raw + dt_bias)+ let sp_input = a_raw + dt_bias_val;+ let sp = if sp_input > 20.0 {+ sp_input+ } else {+ (1.0 + sp_input.exp()).ln()+ };+ let g = -a_log_val.exp() * sp;+ let decay = g.exp();++ // dot(k, q)+ let kq_dot: f32 = k.iter().zip(q.iter()).map(|(ki, qi)| ki * qi).sum();++ // State layout: [num_v_heads, head_v_dim, head_k_dim]+ let mut o = vec![0.0f32; head_v_dim];++ for i in 0..head_v_dim {+ let v_i = unsafe { (*in_proj.add(2 * key_dim + hv * head_v_dim + i)).to_f32().unwrap() };+ let state_row = hv * head_v_dim * head_k_dim + i * head_k_dim;++ // Pass 1: read state row S[i,:], compute dot(S[i,:], q) and dot(S[i,:], k)+ let mut sq_acc = 0.0f32;+ let mut sk_acc = 0.0f32;+ for j in 0..head_k_dim {+ let s = unsafe { (*state_ptr.add(state_row + j)).to_f32().unwrap() };+ sq_acc += s * q[j];+ sk_acc += s * k[j];+ }++ let retrieved_i = decay * sk_acc;+ let delta_i = beta * (v_i - retrieved_i);+ o[i] = decay * sq_acc + delta_i * kq_dot;++ // Pass 2: update state row S[i,:]+ for j in 0..head_k_dim {+ let s = unsafe { (*state_ptr.add(state_row + j)).to_f32().unwrap() };+ unsafe {+ *state.add(state_row + j) = T::from(decay * s + k[j] * delta_i).unwrap();+ }+ }+ }++ // RMSNorm over o+ let sumsq: f32 = o.iter().map(|x| x * x).sum();+ let inv_rms = 1.0 / (sumsq / head_v_dim as f32 + norm_epsilon).sqrt();++ // Apply RMSNorm + SiLU gate and write output+ for i in 0..head_v_dim {+ let norm_w = unsafe { (*norm_weight.add(i)).to_f32().unwrap() };+ let z_i = unsafe { (*in_proj.add(conv_dim + hv * head_v_dim + i)).to_f32().unwrap() };+ let z_silu = ActivationType::SILU.activate(z_i);+ let final_val = o[i] * inv_rms * norm_w * z_silu;+ unsafe {+ *out.add(hv * head_v_dim + i) = T::from(final_val).unwrap();+ }+ }+ }+}diff --git a/crates/uzu/src/backends/cpu/kernel/mod.rs b/crates/uzu/src/backends/cpu/kernel/mod.rsindex f6b5610f5..fd116b887 100644--- a/crates/uzu/src/backends/cpu/kernel/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/mod.rs@@ -2,6 +2,7 @@mod activation;mod attention;mod audio;+mod delta_net;mod embedding;mod kv_cache_update;mod layer_norm;diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/conv_scan.metal b/crates/uzu/src/backends/metal/kernel/delta_net/conv_scan.metalnew file mode 100644index 000000000..e0474b957--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/delta_net/conv_scan.metal@@ -0,0 +1,57 @@+#include <metal_stdlib>+#include "../activation/activations.h"+#include "../common/dsl.h"+#include "../ssm/ssm_common.h"++using namespace metal;++// Multi-token causal conv1d with SiLU for DeltaNet.+// Reads from conv_padded (Conv1dPack output), writes conv'd+SiLU'd+// values back to in_proj with stride out_stride, updates conv state.+//+// conv_padded: [state_stride + suffix_len, row_stride]+// conv_weight: [conv_dim, kernel_size]+// bias: [conv_dim] (optional)+// in_proj: [suffix_len, out_stride] — first conv_dim channels overwritten+// state_out: [conv_dim, state_stride]++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(DeltaNetConvScan)(+ device const T* conv_padded,+ device const T* conv_weight,+ device const T* bias OPTIONAL(has_bias),+ device T* in_proj,+ device T* state_out,+ constant const uint& suffix_len,+ constant const uint& kernel_size,+ constant const uint& row_stride,+ constant const uint& state_stride,+ constant const uint& conv_dim,+ constant const uint& out_stride,+ const bool has_bias SPECIALIZE,+ const uint token_idx AXIS(suffix_len + kernel_size - 1, 1),+ const uint channel_idx AXIS(conv_dim, 32)+) {+ const uint tap_count = kernel_size - 1;++ if (token_idx < suffix_len) {+ const device T* weight_row = conv_weight + channel_idx * kernel_size;++ float acc = has_bias ? float(bias[channel_idx]) : 0.0f;++ for (uint tap = 0; tap < kernel_size; ++tap) {+ const uint padded_row = token_idx + tap;+ const uint padded_index = padded_row * row_stride + channel_idx;+ acc += float(weight_row[tap]) * float(conv_padded[padded_index]);+ }++ const size_t dst = size_t(token_idx) * out_stride + channel_idx;+ in_proj[dst] = static_cast<T>(activate_silu(acc));+ } else {+ const uint tap = token_idx - suffix_len;+ const size_t padded_index = size_t(token_idx) * row_stride + channel_idx;+ const uint state_offset = channel_idx * state_stride;+ state_out[state_offset + tap] = conv_padded[padded_index];+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/conv_update.metal b/crates/uzu/src/backends/metal/kernel/delta_net/conv_update.metalnew file mode 100644index 000000000..7132561aa--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/delta_net/conv_update.metal@@ -0,0 +1,41 @@+#include <metal_stdlib>+#include "../activation/activations.h"+#include "../common/dsl.h"+#include "../ssm/ssm_common.h"++using namespace metal;++// Single-token causal conv1d with SiLU, in-place.+template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(DeltaNetConvUpdate)(+ device const T* conv_weight,+ device const T* bias OPTIONAL(has_bias),+ device T* in_out,+ device T* state,+ constant const uint& kernel_size,+ constant const uint& conv_dim,+ constant const uint& state_stride,+ const bool has_bias SPECIALIZE,+ const uint channel_idx AXIS(conv_dim, 256)+) {+ const uint tap_count = kernel_size - 1;+ const uint state_offset = channel_idx * state_stride;+ const device T* weight_row = conv_weight + channel_idx * kernel_size;++ float x = float(in_out[channel_idx]);++ float acc = has_bias ? float(bias[channel_idx]) : 0.0f;++ for (uint tap = 0; tap < tap_count; ++tap) {+ acc += float(weight_row[tap]) * float(state[state_offset + tap]);+ }+ acc += float(weight_row[tap_count]) * x;++ in_out[channel_idx] = static_cast<T>(activate_silu(acc));++ for (uint tap = 0; tap + 1 < tap_count; ++tap) {+ state[state_offset + tap] = state[state_offset + tap + 1];+ }+ state[state_offset + tap_count - 1] = static_cast<T>(x);+}diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal b/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metalnew file mode 100644index 000000000..ba45c4018--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal@@ -0,0 +1,61 @@+#include <metal_stdlib>+#include "../activation/activations.h"+#include "../common/dsl.h"+#include "../common/thread_context.h"+#include "../common/threadgroup_reduce.h"+#include "../ssm/ssm_common.h"++using namespace metal;++// In-place RMSNorm + SiLU gate for DeltaNet prefill output.+// One threadgroup per (token, head).+//+// in_out[i] = in_out[i] * inv_rms * norm_weight[i] * silu(z[i])++// TODO: support different head_v_dim via VARIANTS template when needed+#define HEAD_V_DIM 128++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(DeltaNetNormGate)(+ device T* in_out,+ device const T* in_proj,+ device const T* norm_weight,+ constant const uint& num_v_heads,+ constant const uint& head_v_dim,+ constant const uint& value_dim,+ constant const uint& conv_dim,+ constant const uint& total_proj_dim,+ constant const float& norm_epsilon,+ constant const uint& suffix_len,+ threadgroup float shared_scratch[32],+ const ThreadContext thread_context,+ const uint token_idx GROUPS(suffix_len),+ const uint hv_idx GROUPS(num_v_heads),+ const uint tid THREADS(HEAD_V_DIM)+) {+ const bool active = (tid < head_v_dim);+ const uint base = token_idx * value_dim + hv_idx * head_v_dim;++ // Load value+ float o_i = active ? float(in_out[base + tid]) : 0.0f;++ // RMSNorm: cooperative sum of squares+ float o_sq = active ? o_i * o_i : 0.0f;+ float o_sumsq = threadgroup_cooperative_reduce_sum<HEAD_V_DIM>(+ o_sq,+ shared_scratch,+ tid,+ thread_context+ );+ float inv_rms = rsqrt(o_sumsq / float(head_v_dim) + norm_epsilon);++ // Apply norm + SiLU gate (in-place)+ if (active) {+ float nw = float(norm_weight[tid]);+ uint z_idx =+ token_idx * total_proj_dim + conv_dim + hv_idx * head_v_dim + tid;+ float z_silu = activate_silu(float(in_proj[z_idx]));+ in_out[base + tid] = static_cast<T>(o_i * inv_rms * nw * z_silu);+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/prefill.metal b/crates/uzu/src/backends/metal/kernel/delta_net/prefill.metalnew file mode 100644index 000000000..3e5f78703--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/delta_net/prefill.metal@@ -0,0 +1,126 @@+#include <metal_stdlib>+#include "../common/defines.h"+#include "../common/dsl.h"+#include "../ssm/ssm_common.h"++using namespace metal;++// DeltaNet prefill. One SIMD group per (v-head, dv_idx), Dk split across lanes.+// Grid: num_v_heads × num_dv_groups, PREFILL_THREADS threads each.++#define PREFILL_THREADS 256++static_assert(+ PREFILL_THREADS % METAL_SIMD_SIZE == 0,+ "PREFILL_THREADS must be a multiple of METAL_SIMD_SIZE"+);++template <typename T, uint HEAD_K_DIM>+VARIANTS(T, float, half, bfloat)+VARIANTS(HEAD_K_DIM, 128)+PUBLIC KERNEL(DeltaNetPrefill)(+ device const float* q_norm,+ device const float* k_norm,+ device const float* beta_buf,+ device const float* decay_buf,+ device const T* in_proj,+ device T* state,+ device T* out,+ constant const uint& num_v_heads,+ constant const uint& num_k_heads,+ constant const uint& head_v_dim,+ constant const uint& key_dim,+ constant const uint& value_dim,+ constant const uint& suffix_len,+ constant const uint& num_dv_groups,+ const uint hv_idx GROUPS(num_v_heads),+ const uint dv_group GROUPS(num_dv_groups),+ const uint tid THREADS(PREFILL_THREADS)+) {+ static_assert(+ HEAD_K_DIM % METAL_SIMD_SIZE == 0,+ "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE"+ );+ constexpr uint elems_per_thread = HEAD_K_DIM / METAL_SIMD_SIZE;++ const uint dk_lane = tid % METAL_SIMD_SIZE;+ const uint dv_local = tid / METAL_SIMD_SIZE;+ const uint dv_idx = dv_group * (PREFILL_THREADS / METAL_SIMD_SIZE) + dv_local;+ const bool active = (dv_idx < head_v_dim);++ const uint groups_per_head = num_v_heads / num_k_heads;+ const uint hk = hv_idx / groups_per_head;+ const uint conv_dim = 2 * key_dim + value_dim;+ const uint total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;+ const uint dk_base = dk_lane * elems_per_thread;++ // Pointer bases — increment per token instead of re-computing index+ device const float* q_ptr = q_norm + hk * HEAD_K_DIM + dk_base;+ device const float* k_ptr = k_norm + hk * HEAD_K_DIM + dk_base;+ device const float* beta_ptr = beta_buf + hv_idx;+ device const float* decay_ptr = decay_buf + hv_idx;+ device const T* v_ptr = in_proj + 2 * key_dim + hv_idx * head_v_dim + dv_idx;++ // State layout: [Hv, Dv, Dk] — contiguous along Dk+ device T* state_ptr =+ state + (hv_idx * head_v_dim + dv_idx) * HEAD_K_DIM + dk_base;+ device T* out_ptr = out + hv_idx * head_v_dim + dv_idx;++ // Load state into registers+ float s[elems_per_thread];+ METAL_PRAGMA_UNROLL+ for (uint i = 0; i < elems_per_thread; ++i)+ s[i] = active ? float(state_ptr[i]) : 0.0f;++ for (uint token = 0; token < suffix_len; ++token) {+ float decay = *decay_ptr;+ float beta = *beta_ptr;++ // Load and cache k_norm (used in both passes)+ float k[elems_per_thread];+ METAL_PRAGMA_UNROLL+ for (uint i = 0; i < elems_per_thread; ++i)+ k[i] = k_ptr[i];++ // Pass 1: decay state + kv_mem = (decayed S) @ k+ float kv_partial = 0.0f;+ METAL_PRAGMA_UNROLL+ for (uint i = 0; i < elems_per_thread; ++i) {+ s[i] *= decay;+ kv_partial += s[i] * k[i];+ }+ float kv_mem = simd_sum(kv_partial);++ // Delta+ float v_val = active ? float(*v_ptr) : 0.0f;+ float delta = beta * (v_val - kv_mem);++ // Pass 2: update state + output = new_S @ q+ float out_partial = 0.0f;+ METAL_PRAGMA_UNROLL+ for (uint i = 0; i < elems_per_thread; ++i) {+ s[i] += k[i] * delta;+ out_partial += s[i] * q_ptr[i];+ }+ float o_val = simd_sum(out_partial);++ if (active && dk_lane == 0) {+ *out_ptr = static_cast<T>(o_val);+ }++ // Advance pointers to next token+ q_ptr += key_dim;+ k_ptr += key_dim;+ beta_ptr += num_v_heads;+ decay_ptr += num_v_heads;+ v_ptr += total_proj_dim;+ out_ptr += value_dim;+ }++ // Write final state+ if (active) {+ METAL_PRAGMA_UNROLL+ for (uint i = 0; i < elems_per_thread; ++i)+ state_ptr[i] = static_cast<T>(s[i]);+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal b/crates/uzu/src/backends/metal/kernel/delta_net/prefill_prep.metalnew file mode 100644index 000000000..65256e6b1--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal@@ -0,0 +1,97 @@+#include <metal_stdlib>+#include "../common/defines.h"+#include "../common/dsl.h"+#include "../ssm/ssm_common.h"++using namespace metal;++// Pre-compute normalized q, k, beta, decay for DeltaNet prefill.+// Grid: suffix_len × num_k_heads threadgroups, METAL_SIMD_SIZE threads each (1+// SIMD group).++template <typename T, uint HEAD_K_DIM>+VARIANTS(T, float, half, bfloat)+VARIANTS(HEAD_K_DIM, 128)+PUBLIC KERNEL(DeltaNetPrefillPrep)(+ device const T* in_proj,+ device const T* a_log,+ device const T* dt_bias,+ device float* q_norm_out,+ device float* k_norm_out,+ device float* beta_out,+ device float* decay_out,+ constant const uint& num_v_heads,+ constant const uint& num_k_heads,+ constant const uint& key_dim,+ constant const uint& value_dim,+ constant const uint& suffix_len,+ const uint token_idx GROUPS(suffix_len),+ const uint hk_idx GROUPS(num_k_heads),+ const uint lane THREADS(METAL_SIMD_SIZE)+) {+ static_assert(+ HEAD_K_DIM % METAL_SIMD_SIZE == 0,+ "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE"+ );+ constexpr uint elems_per_thread = HEAD_K_DIM / METAL_SIMD_SIZE;++ const uint conv_dim = 2 * key_dim + value_dim;+ const uint total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;+ const uint groups_per_head = num_v_heads / num_k_heads;++ const uint tok_offset = token_idx * total_proj_dim;+ const uint q_base = tok_offset + hk_idx * HEAD_K_DIM;+ const uint k_base = tok_offset + key_dim + hk_idx * HEAD_K_DIM;++ // Load q and k elements (strided by SIMD width)+ float q_vals[elems_per_thread];+ float k_vals[elems_per_thread];+ float q_sq = 0.0f;+ float k_sq = 0.0f;++ for (uint i = 0; i < elems_per_thread; ++i) {+ uint idx = lane + i * METAL_SIMD_SIZE;+ q_vals[i] = float(in_proj[q_base + idx]);+ k_vals[i] = float(in_proj[k_base + idx]);+ q_sq += q_vals[i] * q_vals[i];+ k_sq += k_vals[i] * k_vals[i];+ }++ // L2 normalize via simd_sum (no threadgroup barrier needed)+ float q_norm_sq = simd_sum(q_sq);+ float k_norm_sq = simd_sum(k_sq);+ float q_inv_norm = rsqrt(q_norm_sq + 1e-6f);+ float k_inv_norm = rsqrt(k_norm_sq + 1e-6f);+ float q_scale = rsqrt(float(HEAD_K_DIM));++ // Normalize and scale q, write q and k+ const uint out_q_base = token_idx * key_dim + hk_idx * HEAD_K_DIM;+ const uint out_k_base = token_idx * key_dim + hk_idx * HEAD_K_DIM;++ for (uint i = 0; i < elems_per_thread; ++i) {+ uint idx = lane + i * METAL_SIMD_SIZE;+ float q_n = q_vals[i] * q_inv_norm * q_scale;+ float k_n = k_vals[i] * k_inv_norm;+ if (idx < HEAD_K_DIM) {+ q_norm_out[out_q_base + idx] = q_n;+ k_norm_out[out_k_base + idx] = k_n;+ }+ }++ // Compute beta and decay for v-heads belonging to this k-head+ // groups_per_head v-heads share this k-head; distribute across lanes+ for (uint group = lane; group < groups_per_head; group += METAL_SIMD_SIZE) {+ uint hv = hk_idx * groups_per_head + group;++ float beta_raw = float(in_proj[tok_offset + conv_dim + value_dim + hv]);+ float beta = 1.0f / (1.0f + fast::exp(-beta_raw));++ float a_raw =+ float(in_proj[tok_offset + conv_dim + value_dim + num_v_heads + hv]);+ float sp = softplus(a_raw + float(dt_bias[hv]));+ float decay = fast::exp(-fast::exp(float(a_log[hv])) * sp);++ beta_out[token_idx * num_v_heads + hv] = beta;+ decay_out[token_idx * num_v_heads + hv] = decay;+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/update.metal b/crates/uzu/src/backends/metal/kernel/delta_net/update.metalnew file mode 100644index 000000000..7fdd882a4--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/delta_net/update.metal@@ -0,0 +1,155 @@+#include <metal_stdlib>+#include "../activation/activations.h"+#include "../common/defines.h"+#include "../common/dsl.h"+#include "../common/thread_context.h"+#include "../common/threadgroup_reduce.h"+#include "../ssm/ssm_common.h"++using namespace metal;++#define UPDATE_THREADS 512+constexpr constant uint UPDATE_K_SPLIT = 4;++static_assert(+ UPDATE_THREADS % UPDATE_K_SPLIT == 0,+ "UPDATE_THREADS must be a multiple of UPDATE_K_SPLIT"+);++template <typename T, uint HEAD_K_DIM>+VARIANTS(T, float, half, bfloat)+VARIANTS(HEAD_K_DIM, 128)+PUBLIC KERNEL(DeltaNetUpdate)(+ device const T* in_proj,+ device const T* a_log,+ device const T* dt_bias,+ device const T* norm_weight,+ device T* state,+ device T* out,+ constant const uint& num_v_heads,+ constant const uint& num_k_heads,+ constant const uint& head_v_dim,+ constant const uint& key_dim,+ constant const uint& value_dim,+ constant const float& norm_epsilon,+ threadgroup float shared_qk[512 /* UPDATE_THREADS */ * 2],+ threadgroup float shared_scratch[METAL_SIMD_SIZE],+ const ThreadContext thread_context,+ const uint hv_idx GROUPS(num_v_heads),+ const uint tid THREADS(UPDATE_THREADS)+) {+ static_assert(+ HEAD_K_DIM % UPDATE_K_SPLIT == 0,+ "HEAD_K_DIM must be a multiple of UPDATE_K_SPLIT"+ );+ constexpr uint bk = HEAD_K_DIM / UPDATE_K_SPLIT;++ const uint vi = tid / UPDATE_K_SPLIT;+ const uint ki_part = tid % UPDATE_K_SPLIT;+ const bool active = (vi < head_v_dim);++ threadgroup float* shared_q = shared_qk;+ threadgroup float* shared_k = shared_qk + UPDATE_THREADS;++ const uint conv_dim = 2 * key_dim + value_dim;+ const uint groups_per_head = num_v_heads / num_k_heads;+ const uint hk = hv_idx / groups_per_head;++ if (tid < HEAD_K_DIM) {+ shared_q[tid] = float(in_proj[hk * HEAD_K_DIM + tid]);+ shared_k[tid] = float(in_proj[key_dim + hk * HEAD_K_DIM + tid]);+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ float q_partial = (tid < HEAD_K_DIM) ? shared_q[tid] * shared_q[tid] : 0.0f;+ float q_norm_sq = threadgroup_cooperative_reduce_sum<UPDATE_THREADS>(+ q_partial,+ shared_scratch,+ tid,+ thread_context+ );+ float q_inv_norm = rsqrt(q_norm_sq + 1e-6f);++ float k_partial = (tid < HEAD_K_DIM) ? shared_k[tid] * shared_k[tid] : 0.0f;+ float k_norm_sq = threadgroup_cooperative_reduce_sum<UPDATE_THREADS>(+ k_partial,+ shared_scratch,+ tid,+ thread_context+ );+ float k_inv_norm = rsqrt(k_norm_sq + 1e-6f);++ float q_scale = rsqrt(float(HEAD_K_DIM));+ if (tid < HEAD_K_DIM) {+ shared_q[tid] *= q_inv_norm * q_scale;+ shared_k[tid] *= k_inv_norm;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ float kq_p = (tid < HEAD_K_DIM) ? shared_k[tid] * shared_q[tid] : 0.0f;+ float kq_dot = threadgroup_cooperative_reduce_sum<UPDATE_THREADS>(+ kq_p,+ shared_scratch,+ tid,+ thread_context+ );++ float beta_raw = float(in_proj[conv_dim + value_dim + hv_idx]);+ float beta = 1.0f / (1.0f + fast::exp(-beta_raw));+ float a_raw = float(in_proj[conv_dim + value_dim + num_v_heads + hv_idx]);+ float sp = softplus(a_raw + float(dt_bias[hv_idx]));+ float decay = fast::exp(-fast::exp(float(a_log[hv_idx])) * sp);++ float v_i =+ active ? float(in_proj[2 * key_dim + hv_idx * head_v_dim + vi]) : 0.0f;+ float z_i =+ active ? float(in_proj[conv_dim + hv_idx * head_v_dim + vi]) : 0.0f;++ // State read — each thread handles bk rows+ // State layout: [Hv, Dv, Dk]+ float sq_partial = 0.0f;+ float sk_partial = 0.0f;+ const uint state_row_offset =+ hv_idx * head_v_dim * HEAD_K_DIM + vi * HEAD_K_DIM;+ const uint k_base = ki_part * bk;+ if (active) {+ for (uint j = 0; j < bk; ++j) {+ float s = float(state[state_row_offset + k_base + j]);+ sq_partial += s * shared_q[k_base + j];+ sk_partial += s * shared_k[k_base + j];+ }+ }++ // Reduce across k-parts (4 consecutive lanes = one quad group)+ float sq_acc = quad_sum(sq_partial);+ float sk_acc = quad_sum(sk_partial);++ float retrieved_i = decay * sk_acc;+ float delta_i = beta * (v_i - retrieved_i);+ float o_i = decay * sq_acc + delta_i * kq_dot;++ // State update — each thread updates its k-part+ if (active) {+ for (uint j = 0; j < bk; ++j) {+ uint idx = state_row_offset + k_base + j;+ float s = float(state[idx]);+ state[idx] = static_cast<T>(decay * s + shared_k[k_base + j] * delta_i);+ }+ }++ // RMSNorm + SiLU gate — only ki_part==0 contributes+ float o_sq = (active && ki_part == 0) ? o_i * o_i : 0.0f;+ float o_sumsq = threadgroup_cooperative_reduce_sum<UPDATE_THREADS>(+ o_sq,+ shared_scratch,+ tid,+ thread_context+ );+ float inv_rms = rsqrt(o_sumsq / float(head_v_dim) + norm_epsilon);++ if (active && ki_part == 0) {+ float nw = float(norm_weight[vi]);+ float z_silu = activate_silu(z_i);+ out[hv_idx * head_v_dim + vi] = static_cast<T>(o_i * inv_rms * nw * z_silu);+ }+}diff --git a/crates/uzu/src/config/attention.rs b/crates/uzu/src/config/attention.rsindex 8215b7c37..e213a67da 100644--- a/crates/uzu/src/config/attention.rs+++ b/crates/uzu/src/config/attention.rs@@ -25,6 +25,8 @@ pub struct AttentionConfig {#[serde(default)]pub has_gate: bool,#[serde(default)]+ pub gate_projection_config: Option<LinearConfig>,+ #[serde(default)]pub partial_rope_dim: Option<usize>,}diff --git a/crates/uzu/src/config/delta_net.rs b/crates/uzu/src/config/delta_net.rsindex 4e650f702..61a87b2fd 100644--- a/crates/uzu/src/config/delta_net.rs+++ b/crates/uzu/src/config/delta_net.rs@@ -27,4 +27,8 @@ impl DeltaNetAttentionConfig {pub fn conv_dim(&self) -> usize {self.key_dim() * 2 + self.value_dim()}++ pub fn total_proj_dim(&self) -> usize {+ self.conv_dim() + self.value_dim() + self.num_heads + self.num_heads+ }}diff --git a/crates/uzu/src/encodable_block/delta_net_mixer.rs b/crates/uzu/src/encodable_block/delta_net_mixer.rsnew file mode 100644index 000000000..a2a60de4f--- /dev/null+++ b/crates/uzu/src/encodable_block/delta_net_mixer.rs@@ -0,0 +1,345 @@+use std::ops::{Deref, DerefMut};++use thiserror::Error;++use crate::{+ DataType,+ backends::common::{+ Backend, Encoder, Kernels,+ kernel::{+ Conv1dPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel, DeltaNetNormGateKernel,+ DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,+ },+ },+ config::DeltaNetAttentionConfig,+ encodable_block::linear::{Linear, LinearBlockError},+ forward_pass::state::{ArrayId, ForwardPassState},+ parameters::{ParameterLoaderError, ParameterTree, resolve_subtree},+};++#[derive(Debug, Error)]+pub enum DeltaNetMixerError<B: Backend> {+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Unsupported configuration: {0}")]+ UnsupportedConfiguration(String),+ #[error("Linear error: {0}")]+ InnerLinearError(#[from] Box<LinearBlockError<B>>),+ #[error("Parameter loader error: {0}")]+ ParameterLoaderError(#[from] ParameterLoaderError<B>),+}++pub(crate) struct DeltaNetMixer<B: Backend> {+ layer_index: usize,+ config: DeltaNetAttentionConfig,+ in_projection: Box<dyn Linear<B>>,+ out_projection: Box<dyn Linear<B>>,+ // Decode kernels+ conv_update: <B::Kernels as Kernels>::DeltaNetConvUpdateKernel,+ delta_net_update: <B::Kernels as Kernels>::DeltaNetUpdateKernel,+ // Prefill kernels+ conv_pack: <B::Kernels as Kernels>::Conv1dPackKernel,+ conv_scan: <B::Kernels as Kernels>::DeltaNetConvScanKernel,+ prefill_prep: <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel,+ delta_net_prefill: <B::Kernels as Kernels>::DeltaNetPrefillKernel,+ norm_gate: <B::Kernels as Kernels>::DeltaNetNormGateKernel,+ // Parameters+ conv_weight: B::Buffer,+ conv_bias: Option<B::Buffer>,+ a_log: B::Buffer,+ dt_bias: B::Buffer,+ norm_weight: B::Buffer,+}++impl<B: Backend> DeltaNetMixer<B> {+ pub(crate) fn new(+ context: &B::Context,+ config: DeltaNetAttentionConfig,+ layer_index: usize,+ model_dim: usize,+ decoder_layer_loader: &ParameterTree<B::Context>,+ ) -> Result<Self, DeltaNetMixerError<B>> {+ if config.kernel_size < 2 {+ return Err(DeltaNetMixerError::UnsupportedConfiguration(format!(+ "kernel_size must be >= 2, got {}",+ config.kernel_size+ )));+ }+ if config.head_dim != 128 {+ return Err(DeltaNetMixerError::UnsupportedConfiguration(format!(+ "head_dim must be 128, got {}",+ config.head_dim+ )));+ }+ if config.value_head_dim != 128 {+ return Err(DeltaNetMixerError::UnsupportedConfiguration(format!(+ "value_head_dim must be 128, got {}",+ config.value_head_dim+ )));+ }++ let data_type: DataType = config.in_proj_config.activation_precision().into();+ let has_bias = config.conv_config.has_biases;++ // Load weights+ let mixer_tree = resolve_subtree(decoder_layer_loader, &["mixer"]);+ let conv_tree = resolve_subtree(&mixer_tree, &["conv", "conv1d"]);++ let in_projection = <dyn Linear<B>>::new(+ &config.in_proj_config,+ false,+ model_dim,+ [config.total_proj_dim()],+ context,+ &resolve_subtree(decoder_layer_loader, &["mixer.in_projection", "mixer.in_proj"]),+ ArrayId::Main,+ ArrayId::SsmInProj,+ )+ .map_err(|e| DeltaNetMixerError::InnerLinearError(Box::new(e)))?;++ let out_projection = <dyn Linear<B>>::new(+ &config.out_proj_config,+ false,+ config.value_dim(),+ [model_dim],+ context,+ &resolve_subtree(decoder_layer_loader, &["mixer.out_projection", "mixer.out_proj"]),+ ArrayId::AttentionOutput,+ ArrayId::Main,+ )+ .map_err(|e| DeltaNetMixerError::InnerLinearError(Box::new(e)))?;++ let conv_weight = conv_tree.leaf("weights")?.read_buffer()?;+ let conv_bias = if has_bias {+ Some(conv_tree.leaf("biases")?.read_buffer()?)+ } else {+ None+ };++ let a_log = mixer_tree.leaf("a_log")?.read_buffer()?;+ let dt_bias = mixer_tree.leaf("dt_bias")?.read_buffer()?;+ let norm_tree = resolve_subtree(&mixer_tree, &["norm", "inner_norm"]);+ let norm_weight = norm_tree.leaf("scales")?.read_buffer()?;++ // Create kernels+ let conv_update = <B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, data_type, has_bias)+ .map_err(DeltaNetMixerError::BackendError)?;+ let delta_net_update =+ <B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, data_type, config.head_dim as u32)+ .map_err(DeltaNetMixerError::BackendError)?;+ let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, data_type)+ .map_err(DeltaNetMixerError::BackendError)?;+ let conv_scan = <B::Kernels as Kernels>::DeltaNetConvScanKernel::new(context, data_type, has_bias)+ .map_err(DeltaNetMixerError::BackendError)?;+ let prefill_prep =+ <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(context, data_type, config.head_dim as u32)+ .map_err(DeltaNetMixerError::BackendError)?;+ let delta_net_prefill =+ <B::Kernels as Kernels>::DeltaNetPrefillKernel::new(context, data_type, config.head_dim as u32)+ .map_err(DeltaNetMixerError::BackendError)?;+ let norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, data_type)+ .map_err(DeltaNetMixerError::BackendError)?;++ Ok(Self {+ layer_index,+ config,+ in_projection,+ out_projection,+ conv_update,+ delta_net_update,+ conv_pack,+ conv_scan,+ prefill_prep,+ delta_net_prefill,+ norm_gate,+ conv_weight,+ conv_bias,+ a_log,+ dt_bias,+ norm_weight,+ })+ }++ fn run_conv_update(+ &self,+ state: &mut ForwardPassState<B>,+ encoder: &mut Encoder<B>,+ ) {+ let in_proj = state.array(ArrayId::SsmInProj);+ let conv_state = state.array(ArrayId::DeltaNetConvState(self.layer_index));++ let kernel_size = self.config.kernel_size;++ self.conv_update.encode(+ &self.conv_weight,+ self.conv_bias.as_ref(),+ in_proj.buffer().borrow_mut().deref_mut(),+ conv_state.buffer().borrow_mut().deref_mut(),+ kernel_size as u32,+ self.config.conv_dim() as u32,+ (kernel_size - 1) as u32,+ encoder,+ );+ }++ fn run_conv_scan(+ &self,+ state: &mut ForwardPassState<B>,+ encoder: &mut Encoder<B>,+ suffix_length: usize,+ ) {+ let kernel_size = self.config.kernel_size;+ let state_stride = kernel_size - 1;+ let conv_dim = self.config.conv_dim();+ let total_proj_dim = self.config.total_proj_dim();++ let in_proj = state.array(ArrayId::SsmInProj);+ let conv_state = state.array(ArrayId::DeltaNetConvState(self.layer_index));++ let padded = state.conv_padded_buffer().expect("Missing conv padded buffer");++ self.conv_pack.encode(+ conv_state.buffer().borrow().deref(),+ in_proj.buffer().borrow().deref(),+ padded.buffer().borrow_mut().deref_mut(),+ state_stride as u32,+ total_proj_dim as u32,+ suffix_length as u32,+ conv_dim as u32,+ encoder,+ );++ self.conv_scan.encode(+ padded.buffer().borrow().deref(),+ &self.conv_weight,+ self.conv_bias.as_ref(),+ in_proj.buffer().borrow_mut().deref_mut(),+ conv_state.buffer().borrow_mut().deref_mut(),+ suffix_length as u32,+ kernel_size as u32,+ total_proj_dim as u32,+ state_stride as u32,+ conv_dim as u32,+ total_proj_dim as u32,+ encoder,+ );+ }++ fn run_delta_rule(+ &self,+ state: &mut ForwardPassState<B>,+ encoder: &mut Encoder<B>,+ ) {+ let in_proj = state.array(ArrayId::SsmInProj);+ let ssm_state = state.array(ArrayId::DeltaNetSsmState(self.layer_index));+ let out = state.array(ArrayId::AttentionOutput);++ self.delta_net_update.encode(+ in_proj.buffer().borrow().deref(),+ &self.a_log,+ &self.dt_bias,+ &self.norm_weight,+ ssm_state.buffer().borrow_mut().deref_mut(),+ out.buffer().borrow_mut().deref_mut(),+ self.config.num_heads as u32,+ self.config.num_groups as u32,+ self.config.value_head_dim as u32,+ self.config.key_dim() as u32,+ self.config.value_dim() as u32,+ self.config.norm_config.epsilon,+ encoder,+ );+ }++ fn run_delta_rule_prefill(+ &self,+ state: &mut ForwardPassState<B>,+ encoder: &mut Encoder<B>,+ suffix_length: usize,+ ) {+ let num_dv_groups = ((self.config.value_head_dim + 7) / 8) as u32;++ let in_proj = state.array(ArrayId::SsmInProj);+ let ssm_state = state.array(ArrayId::DeltaNetSsmState(self.layer_index));+ let out = state.array(ArrayId::AttentionOutput);++ let aux = state.llm_aux.as_ref().expect("DeltaNet prep buffers not initialized");+ let prep_q_norm = aux.delta_net_prep_q_norm.as_ref().expect("DeltaNet prep_q_norm not initialized");+ let prep_k_norm = aux.delta_net_prep_k_norm.as_ref().expect("DeltaNet prep_k_norm not initialized");+ let prep_beta = aux.delta_net_prep_beta.as_ref().expect("DeltaNet prep_beta not initialized");+ let prep_decay = aux.delta_net_prep_decay.as_ref().expect("DeltaNet prep_decay not initialized");++ self.prefill_prep.encode(+ in_proj.buffer().borrow().deref(),+ &self.a_log,+ &self.dt_bias,+ prep_q_norm.buffer().borrow_mut().deref_mut(),+ prep_k_norm.buffer().borrow_mut().deref_mut(),+ prep_beta.buffer().borrow_mut().deref_mut(),+ prep_decay.buffer().borrow_mut().deref_mut(),+ self.config.num_heads as u32,+ self.config.num_groups as u32,+ self.config.key_dim() as u32,+ self.config.value_dim() as u32,+ suffix_length as u32,+ encoder,+ );++ self.delta_net_prefill.encode(+ prep_q_norm.buffer().borrow().deref(),+ prep_k_norm.buffer().borrow().deref(),+ prep_beta.buffer().borrow().deref(),+ prep_decay.buffer().borrow().deref(),+ in_proj.buffer().borrow().deref(),+ ssm_state.buffer().borrow_mut().deref_mut(),+ out.buffer().borrow_mut().deref_mut(),+ self.config.num_heads as u32,+ self.config.num_groups as u32,+ self.config.value_head_dim as u32,+ self.config.key_dim() as u32,+ self.config.value_dim() as u32,+ suffix_length as u32,+ num_dv_groups,+ encoder,+ );++ // Norm gate+ self.norm_gate.encode(+ out.buffer().borrow_mut().deref_mut(),+ in_proj.buffer().borrow().deref(),+ &self.norm_weight,+ self.config.num_heads as u32,+ self.config.value_head_dim as u32,+ self.config.value_dim() as u32,+ self.config.conv_dim() as u32,+ self.config.total_proj_dim() as u32,+ self.config.norm_config.epsilon,+ suffix_length as u32,+ encoder,+ );+ }++ pub(crate) fn encode(+ &self,+ state: &mut ForwardPassState<B>,+ encoder: &mut Encoder<B>,+ ) -> Result<(), B::Error> {+ let active_suffix_length = state.active_suffix_length();+ if active_suffix_length == 0 {+ return Ok(());+ }++ self.in_projection.encode(state, encoder)?;++ if active_suffix_length == 1 {+ self.run_conv_update(state, encoder);+ self.run_delta_rule(state, encoder);+ } else {+ self.run_conv_scan(state, encoder, active_suffix_length);+ self.run_delta_rule_prefill(state, encoder, active_suffix_length);+ }++ self.out_projection.encode(state, encoder)?;+ Ok(())+ }+}diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rsindex 641375307..e9ca29161 100644--- a/crates/uzu/src/encodable_block/layer/executables.rs+++ b/crates/uzu/src/encodable_block/layer/executables.rs@@ -8,8 +8,8 @@ use crate::{backends::common::{Backend, Encoder},config::{DecoderLayerConfig, DecoderLayerType, MixerConfig},encodable_block::{- Attention, EncodingParameters, Linear, MambaMixer, Mlp, QKNorm, RMSNorm, Rope, ShortConvMixer, TensorAddSwap,- TensorCopy,+ Attention, DeltaNetMixer, EncodingParameters, Linear, MambaMixer, Mlp, QKNorm, RMSNorm, Rope, ShortConvMixer,+ TensorAddSwap, TensorCopy,},forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,@@ -68,11 +68,18 @@ impl<B: Backend> LayerExecutables<B> {MixerConfig::Attention(attention_config) => {let rope_block = rope.expect("RoPE encoder missing for attention layer");+ let layer_num_heads = attention_config.num_heads.unwrap_or(num_heads);+ let layer_num_groups = attention_config.num_groups.unwrap_or(num_groups);+ let layer_head_dim = attention_config.head_dim.unwrap_or(head_dim);++ let q_dim = layer_num_heads * layer_head_dim;+ let kv_dim = layer_num_groups * layer_head_dim;+let qkv_projection = <dyn Linear<B>>::new(&attention_config.qkv_projection_config,attention_config.has_qkv_biases,model_dim,- [num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],+ [q_dim, kv_dim, kv_dim],context,&decoder_layer_loader.subtree("mixer.qkv_projection").unwrap(),ArrayId::Main,@@ -80,13 +87,18 @@ impl<B: Backend> LayerExecutables<B> {).expect("Failed to create qkv projection");- let gate_projection = if attention_config.has_gate {+ let has_gate = attention_config.has_gate || attention_config.gate_projection_config.is_some();+ let gate_projection = if has_gate {+ let gate_config = attention_config+ .gate_projection_config+ .as_ref()+ .unwrap_or(&attention_config.qkv_projection_config);Some(<dyn Linear<B>>::new(- &attention_config.qkv_projection_config,+ gate_config,false,model_dim,- [num_heads * head_dim],+ [q_dim],context,&decoder_layer_loader.subtree("mixer.gate_projection").unwrap(),ArrayId::Main,@@ -107,9 +119,9 @@ impl<B: Backend> LayerExecutables<B> {attention_config.key_norm_config.clone(),ArrayId::QKV,&decoder_layer_loader.subtree("mixer").unwrap(),- num_heads,- num_groups,- head_dim,+ layer_num_heads,+ layer_num_groups,+ layer_head_dim,) {Ok(qk_norm) => Some(qk_norm),Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),@@ -121,7 +133,7 @@ impl<B: Backend> LayerExecutables<B> {let out_projection = <dyn Linear<B>>::new(&attention_config.out_projection_config,attention_config.has_out_biases,- num_heads * head_dim,+ q_dim,[model_dim],context,&decoder_layer_loader.subtree("mixer.out_projection").unwrap(),@@ -138,7 +150,7 @@ impl<B: Backend> LayerExecutables<B> {attention_config.has_sinks,attention_config.is_causal.unwrap_or(true),attention_config.sliding_window_size,- attention_config.has_gate,+ has_gate,).expect("Failed to create AttentionWrapper kernel");@@ -180,7 +192,15 @@ impl<B: Backend> LayerExecutables<B> {mixer,}},- MixerConfig::DeltaNet(_) => unimplemented!("DeltaNet mixer"),+ MixerConfig::DeltaNet(delta_net_config) => {+ let mixer =+ DeltaNetMixer::new(context, delta_net_config.clone(), layer_index, model_dim, decoder_layer_loader)+ .expect("Failed to create DeltaNet mixer");++ MixerExecutables::DeltaNet {+ mixer,+ }+ },};let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {@@ -316,6 +336,15 @@ impl<B: Backend> LayerExecutables<B> {state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());}},+ MixerExecutables::DeltaNet {+ mixer,+ } => {+ mixer.encode(state, encoder)?;+ #[cfg(feature = "tracing")]+ if let Some(ref layer_traces) = layer_traces {+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());+ }+ },}if let Some(post_attention_norm) = &self.post_attention_norm {diff --git a/crates/uzu/src/encodable_block/layer/mixer.rs b/crates/uzu/src/encodable_block/layer/mixer.rsindex 749ebbc0b..08894a765 100644--- a/crates/uzu/src/encodable_block/layer/mixer.rs+++ b/crates/uzu/src/encodable_block/layer/mixer.rs@@ -4,10 +4,10 @@ use std::rc::Rc;use crate::{backends::common::Backend,- encodable_block::{Attention, MambaMixer, QKNorm, Rope, ShortConvMixer, linear::Linear},+ encodable_block::{Attention, DeltaNetMixer, MambaMixer, QKNorm, Rope, ShortConvMixer, linear::Linear},};-/// Mixer component - either attention, state space, or short conv.+/// Mixer component - either attention, state space, short conv, or delta net.pub(crate) enum MixerExecutables<B: Backend> {Attention {qkv_projection: Box<dyn Linear<B>>,@@ -23,4 +23,7 @@ pub(crate) enum MixerExecutables<B: Backend> {ShortConv {mixer: ShortConvMixer<B>,},+ DeltaNet {+ mixer: DeltaNetMixer<B>,+ },}diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rsindex ada33e7ab..84f5b1b2d 100644--- a/crates/uzu/src/encodable_block/mod.rs+++ b/crates/uzu/src/encodable_block/mod.rs@@ -2,6 +2,7 @@ mod activation;mod attention;mod classifier_layer;mod decoder;+mod delta_net_mixer;mod embedding;mod layer;mod layer_norm;@@ -23,6 +24,7 @@ pub use activation::Activation;pub use attention::Attention;pub use classifier_layer::ClassifierLayer;pub use decoder::Decoder;+pub(crate) use delta_net_mixer::DeltaNetMixer;pub use embedding::Embedding;pub use layer::LayerExecutables;pub use layer_norm::{LayerNorm, LayerNormError};diff --git a/crates/uzu/src/forward_pass/cache_layers.rs b/crates/uzu/src/forward_pass/cache_layers.rsindex 38abcabac..3c25ee397 100644--- a/crates/uzu/src/forward_pass/cache_layers.rs+++ b/crates/uzu/src/forward_pass/cache_layers.rs@@ -5,6 +5,7 @@ use crate::{backends::common::{Backend, Encoder, kernel::kv_cache_update::KVCacheUpdate},config::DecoderLayerType,forward_pass::{+ delta_net_layer::DeltaNetLayer,kv_cache_layer::{KVCacheLayer, KVCacheLayerState, KVSlice},model_shape::ModelShape,short_conv_layer::ShortConvLayer,@@ -17,6 +18,7 @@ pub enum CacheLayer<B: Backend> {Transformer(KVCacheLayer<B>),StateSpace(SSMLayer<B>),ShortConv(ShortConvLayer<B>),+ DeltaNet(DeltaNetLayer<B>),}#[derive(Clone)]@@ -24,6 +26,7 @@ pub enum CacheLayerSlice<B: Backend> {Transformer(KVSlice<B>),StateSpace,ShortConv,+ DeltaNet,}const ARRAY_TRANSFORMER_KEYS_LABEL: &str = "cache_layers_transformer_keys";@@ -32,6 +35,8 @@ const ARRAY_STATE_SPACE_CONV_STATE_LABEL: &str = "cache_layers_state_space_conv_const ARRAY_STATE_SPACE_SSM_STATE_LABEL: &str = "cache_layers_state_space_ssm_state";const ARRAY_SHORT_CONV_CONV_STATE_LABEL: &str = "cache_layers_short_conv_conv_state";const ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL: &str = "cache_layers_short_conv_suffix_state";+const ARRAY_DELTA_NET_CONV_STATE_LABEL: &str = "cache_layers_delta_net_conv_state";+const ARRAY_DELTA_NET_SSM_STATE_LABEL: &str = "cache_layers_delta_net_ssm_state";impl<B: Backend> CacheLayer<B> {pub fn as_transformer(&self) -> Option<&KVCacheLayer<B>> {@@ -75,6 +80,20 @@ impl<B: Backend> CacheLayer<B> {_ => None,}}++ pub fn as_delta_net(&self) -> Option<&DeltaNetLayer<B>> {+ match self {+ CacheLayer::DeltaNet(layer) => Some(layer),+ _ => None,+ }+ }++ pub fn as_delta_net_mut(&mut self) -> Option<&mut DeltaNetLayer<B>> {+ match self {+ CacheLayer::DeltaNet(layer) => Some(layer),+ _ => None,+ }+ }}pub struct CacheLayers<B: Backend> {@@ -184,8 +203,30 @@ impl<B: Backend> CacheLayers<B> {})},DecoderLayerType::DeltaNet {+ conv_dim,+ kernel_size,+ num_heads,+ head_dim,+ value_head_dim,..- } => unimplemented!("DeltaNet cache layer"),+ } => {+ let conv_shape = [*conv_dim, kernel_size.saturating_sub(1)];+ let ssm_shape = [*num_heads, *value_head_dim, *head_dim];+ let dtype = model_shape.activation_data_type();++ CacheLayer::DeltaNet(DeltaNetLayer {+ conv_state: context.create_array_zeros(+ &conv_shape,+ dtype,+ &format!("{ARRAY_DELTA_NET_CONV_STATE_LABEL}_{layer_index}"),+ ),+ ssm_state: context.create_array_zeros(+ &ssm_shape,+ dtype,+ &format!("{ARRAY_DELTA_NET_SSM_STATE_LABEL}_{layer_index}"),+ ),+ })+ },}).collect();@@ -216,6 +257,7 @@ impl<B: Backend> CacheLayers<B> {},CacheLayer::StateSpace(layer) => layer.zero(),CacheLayer::ShortConv(layer) => layer.zero(),+ CacheLayer::DeltaNet(layer) => layer.zero(),}}}@@ -272,6 +314,7 @@ impl<B: Backend> CacheLayers<B> {},CacheLayer::StateSpace(_) => layers.push(CacheLayerSlice::StateSpace),CacheLayer::ShortConv(_) => layers.push(CacheLayerSlice::ShortConv),+ CacheLayer::DeltaNet(_) => layers.push(CacheLayerSlice::DeltaNet),}}@@ -391,6 +434,30 @@ impl<B: Backend> CacheLayers<B> {suffix_state_valid_len: Cell::new(0),})},+ CacheLayer::DeltaNet(layer) => {+ let conv_shape = layer.conv_state.shape().to_vec();+ let conv_dtype = layer.conv_state.data_type();+ let mut new_conv = context.create_array_uninitialized(+ &conv_shape,+ conv_dtype,+ &format!("{ARRAY_DELTA_NET_CONV_STATE_LABEL}_{layer_index}"),+ );+ new_conv.copy_from_array(&layer.conv_state);++ let ssm_shape = layer.ssm_state.shape().to_vec();+ let ssm_dtype = layer.ssm_state.data_type();+ let mut new_ssm = context.create_array_uninitialized(+ &ssm_shape,+ ssm_dtype,+ &format!("{ARRAY_DELTA_NET_SSM_STATE_LABEL}_{layer_index}"),+ );+ new_ssm.copy_from_array(&layer.ssm_state);++ CacheLayer::DeltaNet(DeltaNetLayer {+ conv_state: new_conv,+ ssm_state: new_ssm,+ })+ },}).collect();diff --git a/crates/uzu/src/forward_pass/delta_net_layer.rs b/crates/uzu/src/forward_pass/delta_net_layer.rsnew file mode 100644index 000000000..4559e35cd--- /dev/null+++ b/crates/uzu/src/forward_pass/delta_net_layer.rs@@ -0,0 +1,16 @@+use bytemuck::fill_zeroes;++use crate::{array::Array, backends::common::Backend};++#[derive(Debug)]+pub struct DeltaNetLayer<B: Backend> {+ pub conv_state: Array<B>,+ pub ssm_state: Array<B>,+}++impl<B: Backend> DeltaNetLayer<B> {+ pub fn zero(&mut self) {+ fill_zeroes(self.conv_state.as_bytes_mut());+ fill_zeroes(self.ssm_state.as_bytes_mut());+ }+}diff --git a/crates/uzu/src/forward_pass/mod.rs b/crates/uzu/src/forward_pass/mod.rsindex cfede3e9a..474f782c6 100644--- a/crates/uzu/src/forward_pass/mod.rs+++ b/crates/uzu/src/forward_pass/mod.rs@@ -1,6 +1,7 @@pub mod model_shape;pub mod cache_layers;+pub mod delta_net_layer;pub mod kv_cache_layer;pub mod short_conv_layer;pub mod ssm_layer;diff --git a/crates/uzu/src/forward_pass/model_shape.rs b/crates/uzu/src/forward_pass/model_shape.rsindex 7a46bbda5..fae070856 100644--- a/crates/uzu/src/forward_pass/model_shape.rs+++ b/crates/uzu/src/forward_pass/model_shape.rs@@ -28,6 +28,10 @@ pub struct ModelShape {max_rope_dim: usize,max_lora_intermediate: Option<usize>,has_gate: bool,+ max_delta_net_kernel_size: usize,+ max_delta_net_key_dim: usize,+ max_delta_net_num_heads: usize,+ max_qkv_dim: usize,}impl ModelShape {@@ -76,6 +80,24 @@ impl ModelShape {max_mamba_kernel_size = max_mamba_kernel_size.max(*kernel_size as usize);}}+ let mut max_delta_net_kernel_size = 0usize;+ let mut max_delta_net_key_dim = 0usize;+ let mut max_delta_net_num_heads = 0usize;+ for layer in layer_types.iter() {+ if let DecoderLayerType::DeltaNet {+ kernel_size,+ num_heads,+ num_groups,+ head_dim,+ ..+ } = layer+ {+ max_delta_net_kernel_size = max_delta_net_kernel_size.max(*kernel_size);+ max_delta_net_key_dim = max_delta_net_key_dim.max(num_groups * head_dim);+ max_delta_net_num_heads = max_delta_net_num_heads.max(*num_heads);+ }+ }+let mut max_rope_dim = 0usize;let mut max_lora_intermediate = None;let mut has_gate = false;@@ -86,6 +108,7 @@ impl ModelShape {.map(|configs| configs.iter().collect::<Vec<_>>()).unwrap_or_else(|| vec![&decoder_config.layer_config; num_layers]);+ let mut max_qkv_dim = 0usize;for layer_config in &all_layer_configs {let mut linear_configs = Vec::new();@@ -93,7 +116,7 @@ impl ModelShape {MixerConfig::Attention(attn) => {let hd = attn.head_dim.unwrap_or(decoder_config.head_dim);max_rope_dim = max_rope_dim.max(attn.partial_rope_dim.unwrap_or(hd));- has_gate = has_gate || attn.has_gate;+ has_gate = has_gate || attn.has_gate || attn.gate_projection_config.is_some();linear_configs.extend([&attn.qkv_projection_config, &attn.out_projection_config]);},MixerConfig::Mamba(mamba) => {@@ -128,6 +151,9 @@ impl ModelShape {};}}+ if max_qkv_dim == 0 {+ max_qkv_dim = (decoder_config.num_heads + 2 * decoder_config.num_groups) * decoder_config.head_dim;+ }if max_rope_dim == 0 {max_rope_dim = decoder_config.head_dim;@@ -158,6 +184,10 @@ impl ModelShape {max_rope_dim,max_lora_intermediate,has_gate,+ max_delta_net_kernel_size,+ max_delta_net_key_dim,+ max_delta_net_num_heads,+ max_qkv_dim,}}@@ -240,7 +270,7 @@ impl ModelShape {&self,suffix_length: usize,) -> [usize; 2] {- [suffix_length, (2 * self.num_groups + self.num_heads) * self.head_dim]+ [suffix_length, self.max_qkv_dim]}pub fn logits_shape(@@ -254,7 +284,21 @@ impl ModelShape {&self,suffix_length: usize,) -> [usize; 2] {- [suffix_length, self.num_heads * self.head_dim]+ let attention_dim = self.num_heads * self.head_dim;+ let delta_net_dim = self+ .layer_types+ .iter()+ .filter_map(|lt| match lt {+ DecoderLayerType::DeltaNet {+ num_heads,+ value_head_dim,+ ..+ } => Some(num_heads * value_head_dim),+ _ => None,+ })+ .max()+ .unwrap_or(0);+ [suffix_length, attention_dim.max(delta_net_dim)]}pub fn gate_shape(@@ -446,6 +490,51 @@ impl ModelShape {self.layer_types.iter().any(|layer_type| matches!(layer_type, DecoderLayerType::ShortConv { .. }))}+ pub fn has_delta_net_layers(&self) -> bool {+ self.max_delta_net_kernel_size > 0+ }++ pub fn delta_net_prep_qk_shape(+ &self,+ suffix_length: usize,+ ) -> Option<[usize; 1]> {+ if self.has_delta_net_layers() {+ Some([suffix_length * self.max_delta_net_key_dim])+ } else {+ None+ }+ }++ pub fn delta_net_prep_beta_decay_shape(+ &self,+ suffix_length: usize,+ ) -> Option<[usize; 1]> {+ if self.has_delta_net_layers() {+ Some([suffix_length * self.max_delta_net_num_heads])+ } else {+ None+ }+ }++ fn max_delta_net_padded_stride(&self) -> usize {+ self.layer_types+ .iter()+ .filter_map(|lt| match lt {+ DecoderLayerType::DeltaNet {+ conv_dim,+ num_heads,+ value_head_dim,+ ..+ } => {+ let value_dim = num_heads * value_head_dim;+ Some(conv_dim + value_dim + 2 * num_heads)+ },+ _ => None,+ })+ .max()+ .unwrap_or(0)+ }+pub fn max_mamba_conv_dim(&self) -> Option<usize> {if self.max_mamba_conv_dim == 0 {None@@ -481,9 +570,21 @@ impl ModelShape {&self,suffix_length: usize,) -> Option<[usize; 2]> {- match (self.max_mamba_conv_dim(), self.max_mamba_kernel_size()) {- (Some(conv_dim), Some(kernel_size)) if kernel_size > 0 => Some([suffix_length + kernel_size - 1, conv_dim]),+ let mamba = match (self.max_mamba_conv_dim(), self.max_mamba_kernel_size()) {+ (Some(conv_dim), Some(kernel_size)) if kernel_size > 0 => Some((suffix_length + kernel_size - 1, conv_dim)),_ => None,+ };+ let delta_net = if self.has_delta_net_layers() {+ let state_stride = self.max_delta_net_kernel_size.saturating_sub(1);+ let max_padded_stride = self.max_delta_net_padded_stride();+ Some((suffix_length + state_stride, max_padded_stride))+ } else {+ None+ };+ match (mamba, delta_net) {+ (Some((rows_m, cols_m)), Some((rows_d, cols_d))) => Some([rows_m.max(rows_d), cols_m.max(cols_d)]),+ (Some((rows, cols)), None) | (None, Some((rows, cols))) => Some([rows, cols]),+ (None, None) => None,}}@@ -574,6 +675,15 @@ impl ModelShape {DecoderLayerType::ShortConv {..} => Some(self.model_dim * 3),+ DecoderLayerType::DeltaNet {+ conv_dim,+ num_heads,+ value_head_dim,+ ..+ } => {+ let value_dim = num_heads * value_head_dim;+ Some(conv_dim + value_dim + 2 * num_heads)+ },_ => None,}).max()diff --git a/crates/uzu/src/forward_pass/scratch_buffers.rs b/crates/uzu/src/forward_pass/scratch_buffers.rsindex f920895b8..2cefb1d1c 100644--- a/crates/uzu/src/forward_pass/scratch_buffers.rs+++ b/crates/uzu/src/forward_pass/scratch_buffers.rs@@ -34,6 +34,10 @@ pub struct ScratchBuffers<B: Backend> {pub ssm_c: Option<Array<B>>,pub ssm_dt: Option<Array<B>>,pub ssm_z: Option<Array<B>>,+ pub delta_net_prep_q_norm: Option<Array<B>>,+ pub delta_net_prep_k_norm: Option<Array<B>>,+ pub delta_net_prep_beta: Option<Array<B>>,+ pub delta_net_prep_decay: Option<Array<B>>,// 3-Dpub rotated_queries: Array<B>,@@ -128,6 +132,18 @@ impl<B: Backend> ScratchBuffers<B> {ssm_c: model_shape.ssm_bc_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_c")),ssm_dt: model_shape.ssm_dt_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_dt")),ssm_z: model_shape.ssm_z_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_z")),+ delta_net_prep_q_norm: model_shape+ .delta_net_prep_qk_shape(max_suffix_len)+ .map(|shape| alloc(&shape, DataType::F32, "delta_net_prep_q_norm")),+ delta_net_prep_k_norm: model_shape+ .delta_net_prep_qk_shape(max_suffix_len)+ .map(|shape| alloc(&shape, DataType::F32, "delta_net_prep_k_norm")),+ delta_net_prep_beta: model_shape+ .delta_net_prep_beta_decay_shape(max_suffix_len)+ .map(|shape| alloc(&shape, DataType::F32, "delta_net_prep_beta")),+ delta_net_prep_decay: model_shape+ .delta_net_prep_beta_decay_shape(max_suffix_len)+ .map(|shape| alloc(&shape, DataType::F32, "delta_net_prep_decay")),// 3-Drotated_queries: alloc(&model_shape.rotated_queries_shape(max_suffix_len), act_ty, "rotated_queries"),rotated_keys: alloc(&model_shape.rotated_keys_shape(max_suffix_len), act_ty, "rotated_keys"),diff --git a/crates/uzu/src/forward_pass/state/array_id.rs b/crates/uzu/src/forward_pass/state/array_id.rsindex 09afec89b..281b1470b 100644--- a/crates/uzu/src/forward_pass/state/array_id.rs+++ b/crates/uzu/src/forward_pass/state/array_id.rs@@ -32,6 +32,8 @@ pub enum ArrayId {SsmZ(usize),ShortConvState(usize),ShortConvSuffixState(usize),+ DeltaNetConvState(usize),+ DeltaNetSsmState(usize),RotatedQueries,RotatedKeys,diff --git a/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs b/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rsindex b204b189d..09a974544 100644--- a/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs+++ b/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs@@ -14,6 +14,11 @@ pub struct LanguageModelGeneratorAuxBuffers<B: Backend> {pub ssm_c: Option<Array<B>>,pub ssm_dt: Option<Array<B>>,pub ssm_z: Option<Array<B>>,+ // DeltaNet prep buffers+ pub delta_net_prep_q_norm: Option<Array<B>>,+ pub delta_net_prep_k_norm: Option<Array<B>>,+ pub delta_net_prep_beta: Option<Array<B>>,+ pub delta_net_prep_decay: Option<Array<B>>,// MoE bufferspub moe_topk_ids: Option<Array<B>>,pub moe_topk_probs: Option<Array<B>>,@@ -93,6 +98,26 @@ impl<B: Backend> LanguageModelGeneratorAuxBuffers<B> {.as_ref().zip(model_shape.ssm_z_shape(suffix_length)).map(|(buf, shape)| buf.view(&shape)),+ delta_net_prep_q_norm: scratch+ .delta_net_prep_q_norm+ .as_ref()+ .zip(model_shape.delta_net_prep_qk_shape(suffix_length))+ .map(|(buf, shape)| buf.view(&shape)),+ delta_net_prep_k_norm: scratch+ .delta_net_prep_k_norm+ .as_ref()+ .zip(model_shape.delta_net_prep_qk_shape(suffix_length))+ .map(|(buf, shape)| buf.view(&shape)),+ delta_net_prep_beta: scratch+ .delta_net_prep_beta+ .as_ref()+ .zip(model_shape.delta_net_prep_beta_decay_shape(suffix_length))+ .map(|(buf, shape)| buf.view(&shape)),+ delta_net_prep_decay: scratch+ .delta_net_prep_decay+ .as_ref()+ .zip(model_shape.delta_net_prep_beta_decay_shape(suffix_length))+ .map(|(buf, shape)| buf.view(&shape)),moe_topk_ids: moe.zip(scratch.moe_topk_ids.as_ref()).map(|(moe, buf)| {buf.view(&model_shape.moe_topk_ids_shape(suffix_length, moe.num_active_routed_experts))}),diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rsindex d130f7865..8cd427da3 100644--- a/crates/uzu/src/forward_pass/state/state.rs+++ b/crates/uzu/src/forward_pass/state/state.rs@@ -59,7 +59,7 @@ pub struct ForwardPassState<B: Backend> {token_bitmask: Option<Array<B>>,pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,pub common_aux: CommonAuxBuffers<B>,- llm_aux: Option<LanguageModelGeneratorAuxBuffers<B>>,+ pub llm_aux: Option<LanguageModelGeneratorAuxBuffers<B>>,mode: ForwardPassMode<B>,}@@ -470,6 +470,14 @@ impl<B: Backend> ForwardPassState<B> {let cache = self.llm_state().cache_layers.borrow();cache.data[layer_index].as_short_conv().expect("Expected ShortConv layer").suffix_state.clone()},+ ArrayId::DeltaNetConvState(layer_index) => {+ let cache = self.llm_state().cache_layers.borrow();+ cache.data[layer_index].as_delta_net().expect("Expected DeltaNet layer").conv_state.clone()+ },+ ArrayId::DeltaNetSsmState(layer_index) => {+ let cache = self.llm_state().cache_layers.borrow();+ cache.data[layer_index].as_delta_net().expect("Expected DeltaNet layer").ssm_state.clone()+ },// MoE arrays (LLM only)ArrayId::MoeTopkIds => {diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rsindex ce7d4c719..58d334a5c 100644--- a/crates/uzu/src/tracer/trace_validator.rs+++ b/crates/uzu/src/tracer/trace_validator.rs@@ -349,6 +349,46 @@ impl<B: Backend> TraceValidator<B> {}}+ // LLM-specific: DeltaNet state validation+ let delta_net_layers: Vec<usize> = {+ let cache = state.cache_layers().unwrap().borrow();+ cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_delta_net().map(|_| index)).collect()+ };++ for index in delta_net_layers {+ let conv_state = state.array(ArrayId::DeltaNetConvState(index));+ let ssm_state = state.array(ArrayId::DeltaNetSsmState(index));++ for path in [+ format!("updated_state.{}.conv_state", index),+ format!("activation_trace.layer_results.{}.updated_state.conv_state", index),+ ] {+ if let Ok(expected) = traces_view.leaf_array(&path) {+ results.push(TracerValidationResult {+ name: path,+ metrics: Self::validate_array(+ data_type,+ &expected,+ &conv_state,+ Some(ArrayTransform::SsmConvState),+ ),+ });+ }+ }++ for path in [+ format!("updated_state.{}.ssm_state", index),+ format!("activation_trace.layer_results.{}.updated_state.ssm_state", index),+ ] {+ if let Ok(expected) = traces_view.leaf_array(&path) {+ results.push(TracerValidationResult {+ name: path,+ metrics: Self::validate_array(data_type, &expected, &ssm_state, None),+ });+ }+ }+ }+// LLM-specific: Token comparisonlet tokens_violation_indices = if let Ok(expected_logits) = traces_view.leaf_array("logits") {let expected_tokens = Self::get_tokens_from_logits(&expected_logits);diff --git a/crates/uzu/tests/common/path.rs b/crates/uzu/tests/common/path.rsindex d425f529f..7b6bc38aa 100644--- a/crates/uzu/tests/common/path.rs+++ b/crates/uzu/tests/common/path.rs@@ -10,14 +10,15 @@ pub fn get_version() -> &'static str {}pub fn get_test_model_path() -> PathBuf {+ let model_dir = std::env::var("TEST_MODEL").unwrap_or_else(|_| MODEL_DIR_NAME.to_string());let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..").join("..").join("models").join(get_version())- .join(MODEL_DIR_NAME);+ .join(model_dir);if !path.exists() {- panic!("Test model not found at {:?}. Run `./scripts/download_test_model.sh` to fetch it.", path);+ panic!("Test model not found at {:?}. Set TEST_MODEL env var or run `./scripts/download_test_model.sh`.", path);}path}diff --git a/crates/uzu/tests/unit/config/attention_test.rs b/crates/uzu/tests/unit/config/attention_test.rsindex 6a7f86d3f..3514eb124 100644--- a/crates/uzu/tests/unit/config/attention_test.rs+++ b/crates/uzu/tests/unit/config/attention_test.rs@@ -77,6 +77,7 @@ fn test_attention_config() {has_qkv_biases: false,has_out_biases: false,has_gate: false,+ gate_projection_config: None,partial_rope_dim: None,};diff --git a/crates/uzu/tests/unit/config/decoder_layer_test.rs b/crates/uzu/tests/unit/config/decoder_layer_test.rsindex 0c1d08619..a1980219c 100644--- a/crates/uzu/tests/unit/config/decoder_layer_test.rs+++ b/crates/uzu/tests/unit/config/decoder_layer_test.rs@@ -135,6 +135,7 @@ fn test_decoder_layer_config() {has_qkv_biases: false,has_out_biases: false,has_gate: false,+ gate_projection_config: None,partial_rope_dim: None,}),mlp_config: MLPConfig::Dense(mlp::DenseMLPConfig {diff --git a/crates/uzu/tests/unit/config/decoder_test.rs b/crates/uzu/tests/unit/config/decoder_test.rsindex 7c1904ee0..30bc4ad8b 100644--- a/crates/uzu/tests/unit/config/decoder_test.rs+++ b/crates/uzu/tests/unit/config/decoder_test.rs@@ -198,6 +198,7 @@ fn test_decoder_config() {has_qkv_biases: false,has_out_biases: false,has_gate: false,+ gate_projection_config: None,partial_rope_dim: None,}),mlp_config: MLPConfig::Dense(DenseMLPConfig {diff --git a/crates/uzu/tests/unit/config/transformer_layer_test.rs b/crates/uzu/tests/unit/config/transformer_layer_test.rsindex bb68298a0..fa81a2cb9 100644--- a/crates/uzu/tests/unit/config/transformer_layer_test.rs+++ b/crates/uzu/tests/unit/config/transformer_layer_test.rs@@ -143,6 +143,7 @@ fn test_transformer_layer_config() {has_qkv_biases: false,has_out_biases: false,has_gate: false,+ gate_projection_config: None,partial_rope_dim: None,}),mlp_config: MLPConfig::Dense(mlp::DenseMLPConfig {diff --git a/crates/uzu/tests/unit/kernel/delta_net_test.rs b/crates/uzu/tests/unit/kernel/delta_net_test.rsnew file mode 100644index 000000000..599e10d59--- /dev/null+++ b/crates/uzu/tests/unit/kernel/delta_net_test.rs@@ -0,0 +1,687 @@+#![cfg(metal_backend)]++use std::ops::{Deref, DerefMut};++use uzu::{+ ArrayContextExt, DataType,+ backends::{+ common::{+ Backend, Context, Encoder, Kernels,+ kernel::{+ Conv1dPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel, DeltaNetNormGateKernel,+ DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,+ },+ },+ cpu::Cpu,+ metal::Metal,+ },+};++fn run_conv_update<B: Backend>(+ in_proj: &[f32],+ w: &[f32],+ b: &[f32],+ state: &[f32],+ kernel_size: u32,+ conv_dim: u32,+ state_stride: u32,+) -> (Vec<f32>, Vec<f32>) {+ let context = B::Context::new().expect("Failed to create context");++ let w_array = context.create_array_from(&[w.len()], w, "w");+ let b_array = context.create_array_from(&[b.len()], b, "b");+ let in_out_array = context.create_array_from(&[in_proj.len()], in_proj, "in_out");+ let state_array = context.create_array_from(&[state.len()], state, "state");++ let kernel = <<B as Backend>::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(&context, DataType::F32, true)+ .expect("Failed to create kernel");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ w_array.buffer().borrow().deref(),+ Some(b_array.buffer().borrow().deref()),+ in_out_array.buffer().borrow_mut().deref_mut(),+ state_array.buffer().borrow_mut().deref_mut(),+ kernel_size,+ conv_dim,+ state_stride,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ let out = in_out_array.as_slice::<f32>()[..conv_dim as usize].to_vec();+ let new_state = state_array.as_slice::<f32>().to_vec();+ (out, new_state)+}++fn run_delta_net_update<B: Backend>(+ in_proj: &[f32],+ a_log: &[f32],+ dt_bias: &[f32],+ norm_weight: &[f32],+ state: &[f32],+ num_v_heads: u32,+ num_k_heads: u32,+ head_k_dim: u32,+ head_v_dim: u32,+ key_dim: u32,+ value_dim: u32,+) -> (Vec<f32>, Vec<f32>) {+ let context = B::Context::new().expect("Failed to create context");++ let in_proj_array = context.create_array_from(&[in_proj.len()], in_proj, "in_proj");+ let a_log_array = context.create_array_from(&[a_log.len()], a_log, "a_log");+ let dt_bias_array = context.create_array_from(&[dt_bias.len()], dt_bias, "dt_bias");+ let norm_weight_array = context.create_array_from(&[norm_weight.len()], norm_weight, "norm_weight");+ let state_array = context.create_array_from(&[state.len()], state, "state");+ let out_array = context.create_array_zeros(&[value_dim as usize], DataType::F32, "out");++ let kernel = <<B as Backend>::Kernels as Kernels>::DeltaNetUpdateKernel::new(&context, DataType::F32, head_k_dim)+ .expect("Failed to create kernel");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ in_proj_array.buffer().borrow().deref(),+ a_log_array.buffer().borrow().deref(),+ dt_bias_array.buffer().borrow().deref(),+ norm_weight_array.buffer().borrow().deref(),+ state_array.buffer().borrow_mut().deref_mut(),+ out_array.buffer().borrow_mut().deref_mut(),+ num_v_heads,+ num_k_heads,+ head_v_dim,+ key_dim,+ value_dim,+ 1e-6f32,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ let out = out_array.as_slice::<f32>().to_vec();+ let new_state = state_array.as_slice::<f32>().to_vec();+ (out, new_state)+}++fn assert_close(+ a: &[f32],+ b: &[f32],+ atol: f32,+ rtol: f32,+ label: &str,+) {+ assert_eq!(a.len(), b.len(), "{label}: length mismatch: {} vs {}", a.len(), b.len());+ assert!(+ is_close::default().abs_tol(atol).rel_tol(rtol).all_close(a.iter().copied(), b.iter().copied()),+ "{label}: values not close enough (atol={atol:.6e}, rtol={rtol:.6e})"+ );+}++// DeltaNetConvUpdate++#[test]+fn test_delta_net_conv_update_small() {+ let conv_dim = 32;+ let kernel_size = 4;+ let tap_count = kernel_size - 1;++ let in_proj: Vec<f32> = (0..conv_dim).map(|i| ((i % 7) as f32) * 0.1 - 0.3).collect();+ let w: Vec<f32> = (0..conv_dim * kernel_size).map(|i| ((i % 11) as f32) * 0.05 - 0.2).collect();+ let b: Vec<f32> = (0..conv_dim).map(|i| ((i % 5) as f32) * 0.01).collect();+ let state: Vec<f32> = (0..conv_dim * tap_count).map(|i| ((i % 13) as f32) * 0.02 - 0.1).collect();++ let (cpu_out, cpu_state) =+ run_conv_update::<Cpu>(&in_proj, &w, &b, &state, kernel_size as u32, conv_dim as u32, tap_count as u32);+ let (gpu_out, gpu_state) =+ run_conv_update::<Metal>(&in_proj, &w, &b, &state, kernel_size as u32, conv_dim as u32, tap_count as u32);++ assert_close(&cpu_out, &gpu_out, 1e-4, 1e-3, "ConvUpdate output");+ assert_close(&cpu_state, &gpu_state, 1e-5, 1e-4, "ConvUpdate state");+}++// DeltaNetConvScan++#[test]+fn test_delta_net_conv_scan() {+ let conv_dim = 32;+ let kernel_size: usize = 4;+ let tap_count = kernel_size - 1;+ let suffix_len: usize = 16;+ let extra_dim = 10;+ let total_proj_dim = conv_dim + extra_dim;++ let w: Vec<f32> = (0..conv_dim * kernel_size).map(|i| ((i % 11) as f32) * 0.05 - 0.2).collect();+ let b: Vec<f32> = (0..conv_dim).map(|i| ((i % 5) as f32) * 0.01).collect();+ let init_state: Vec<f32> = (0..conv_dim * tap_count).map(|i| ((i % 13) as f32) * 0.02 - 0.1).collect();+ let in_proj: Vec<f32> = (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect();++ // Reference: decode conv token-by-token+ let mut ref_state = init_state.clone();+ let mut ref_outputs = vec![0.0f32; suffix_len * conv_dim];+ for t in 0..suffix_len {+ let token_in: Vec<f32> = in_proj[t * total_proj_dim..t * total_proj_dim + conv_dim].to_vec();+ let (out, new_state) = run_conv_update::<Cpu>(+ &token_in,+ &w,+ &b,+ &ref_state,+ kernel_size as u32,+ conv_dim as u32,+ tap_count as u32,+ );+ ref_state = new_state;+ ref_outputs[t * conv_dim..(t + 1) * conv_dim].copy_from_slice(&out);+ }++ // Test: Conv1dPack + DeltaNetConvScan on Metal+ let context = <Metal as Backend>::Context::new().expect("context");+ let state_array = context.create_array_from(&[init_state.len()], &init_state, "state");+ let in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj, "in_proj");+ let w_array = context.create_array_from(&[w.len()], &w, "w");+ let b_array = context.create_array_from(&[b.len()], &b, "b");++ let padded_len = (tap_count + suffix_len) * total_proj_dim;+ let padded_array = context.create_array_zeros(&[padded_len], DataType::F32, "padded");+ let state_out_array = context.create_array_zeros(&[conv_dim * tap_count], DataType::F32, "state_out");++ let pack_kernel =+ <<Metal as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, DataType::F32).expect("pack");+ let scan_kernel =+ <<Metal as Backend>::Kernels as Kernels>::DeltaNetConvScanKernel::new(&context, DataType::F32, true)+ .expect("scan");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ pack_kernel.encode(+ state_array.buffer().borrow().deref(),+ in_proj_array.buffer().borrow().deref(),+ padded_array.buffer().borrow_mut().deref_mut(),+ tap_count as u32,+ total_proj_dim as u32,+ suffix_len as u32,+ conv_dim as u32,+ &mut encoder,+ );+ scan_kernel.encode(+ padded_array.buffer().borrow().deref(),+ w_array.buffer().borrow().deref(),+ Some(b_array.buffer().borrow().deref()),+ in_proj_array.buffer().borrow_mut().deref_mut(),+ state_out_array.buffer().borrow_mut().deref_mut(),+ suffix_len as u32,+ kernel_size as u32,+ total_proj_dim as u32,+ tap_count as u32,+ conv_dim as u32,+ total_proj_dim as u32,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ let in_proj_result = in_proj_array.as_slice::<f32>();+ let mut scan_outputs = vec![0.0f32; suffix_len * conv_dim];+ for t in 0..suffix_len {+ scan_outputs[t * conv_dim..(t + 1) * conv_dim]+ .copy_from_slice(&in_proj_result[t * total_proj_dim..t * total_proj_dim + conv_dim]);+ }+ let scan_state = state_out_array.as_slice::<f32>().to_vec();++ assert_close(&ref_outputs, &scan_outputs, 1e-4, 1e-3, "ConvScan output");+ assert_close(&ref_state, &scan_state, 1e-5, 1e-4, "ConvScan state");+}++// DeltaNetUpdate (decode)++fn test_delta_net_update_impl(+ num_v_heads: usize,+ num_k_heads: usize,+ head_k_dim: usize,+ head_v_dim: usize,+ label: &str,+) {+ let key_dim = num_k_heads * head_k_dim;+ let value_dim = num_v_heads * head_v_dim;+ let conv_dim = 2 * key_dim + value_dim;+ let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;+ let state_size = num_v_heads * head_k_dim * head_v_dim;++ let in_proj: Vec<f32> = (0..total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect();+ let a_log: Vec<f32> = (0..num_v_heads).map(|i| -1.5 + (i as f32) * 0.05).collect();+ let dt_bias: Vec<f32> = (0..num_v_heads).map(|i| 0.3 + (i as f32) * 0.02).collect();+ let norm_weight: Vec<f32> = (0..head_v_dim).map(|i| 0.9 + (i as f32) * 0.001).collect();+ let state: Vec<f32> = (0..state_size).map(|i| ((i % 29) as f32) * 0.005 - 0.05).collect();++ let (cpu_out, cpu_state) = run_delta_net_update::<Cpu>(+ &in_proj,+ &a_log,+ &dt_bias,+ &norm_weight,+ &state,+ num_v_heads as u32,+ num_k_heads as u32,+ head_k_dim as u32,+ head_v_dim as u32,+ key_dim as u32,+ value_dim as u32,+ );+ let (gpu_out, gpu_state) = run_delta_net_update::<Metal>(+ &in_proj,+ &a_log,+ &dt_bias,+ &norm_weight,+ &state,+ num_v_heads as u32,+ num_k_heads as u32,+ head_k_dim as u32,+ head_v_dim as u32,+ key_dim as u32,+ value_dim as u32,+ );++ assert_close(&cpu_out, &gpu_out, 1e-3, 1e-2, &format!("{label} output"));+ assert_close(&cpu_state, &gpu_state, 1e-4, 1e-3, &format!("{label} state"));+}++#[test]+fn test_delta_net_update_qwen35_shapes() {+ test_delta_net_update_impl(48, 16, 128, 128, "DeltaNetUpdate Qwen3.5");+}++// DeltaNetPrefill + NormGate++fn run_prefill_with_norm_gate(+ in_proj: &[f32],+ a_log: &[f32],+ dt_bias: &[f32],+ norm_weight: &[f32],+ state: &[f32],+ num_v_heads: usize,+ num_k_heads: usize,+ head_k_dim: usize,+ head_v_dim: usize,+ suffix_len: usize,+) -> (Vec<f32>, Vec<f32>) {+ let key_dim = num_k_heads * head_k_dim;+ let value_dim = num_v_heads * head_v_dim;+ let conv_dim = 2 * key_dim + value_dim;+ let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;+ let num_dv_groups = (head_v_dim as u32 + 7) / 8;++ let context = <Metal as Backend>::Context::new().expect("context");+ let in_proj_array = context.create_array_from(&[in_proj.len()], in_proj, "in_proj");+ let a_log_array = context.create_array_from(&[a_log.len()], a_log, "a_log");+ let dt_bias_array = context.create_array_from(&[dt_bias.len()], dt_bias, "dt_bias");+ let norm_weight_array = context.create_array_from(&[norm_weight.len()], norm_weight, "norm_weight");+ let state_array = context.create_array_from(&[state.len()], state, "state");+ let out_array = context.create_array_zeros(&[suffix_len * value_dim], DataType::F32, "out");+ let q_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "q_norm");+ let k_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "k_norm");++ let beta_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "beta");+ let decay_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "decay");++ let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(+ &context,+ DataType::F32,+ head_k_dim as u32,+ )+ .unwrap();+ let prefill_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillKernel::new(+ &context,+ DataType::F32,+ head_k_dim as u32,+ )+ .unwrap();+ let norm_k =+ <<Metal as Backend>::Kernels as Kernels>::DeltaNetNormGateKernel::new(&context, DataType::F32).unwrap();++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ prep_k.encode(+ in_proj_array.buffer().borrow().deref(),+ a_log_array.buffer().borrow().deref(),+ dt_bias_array.buffer().borrow().deref(),+ q_norm_array.buffer().borrow_mut().deref_mut(),+ k_norm_array.buffer().borrow_mut().deref_mut(),+ beta_array.buffer().borrow_mut().deref_mut(),+ decay_array.buffer().borrow_mut().deref_mut(),+ num_v_heads as u32,+ num_k_heads as u32,+ key_dim as u32,+ value_dim as u32,+ suffix_len as u32,+ &mut encoder,+ );+ prefill_k.encode(+ q_norm_array.buffer().borrow().deref(),+ k_norm_array.buffer().borrow().deref(),+ beta_array.buffer().borrow().deref(),+ decay_array.buffer().borrow().deref(),+ in_proj_array.buffer().borrow().deref(),+ state_array.buffer().borrow_mut().deref_mut(),+ out_array.buffer().borrow_mut().deref_mut(),+ num_v_heads as u32,+ num_k_heads as u32,+ head_v_dim as u32,+ key_dim as u32,+ value_dim as u32,+ suffix_len as u32,+ num_dv_groups,+ &mut encoder,+ );+ norm_k.encode(+ out_array.buffer().borrow_mut().deref_mut(),+ in_proj_array.buffer().borrow().deref(),+ norm_weight_array.buffer().borrow().deref(),+ num_v_heads as u32,+ head_v_dim as u32,+ value_dim as u32,+ conv_dim as u32,+ total_proj_dim as u32,+ 1e-6f32,+ suffix_len as u32,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ (out_array.as_slice::<f32>().to_vec(), state_array.as_slice::<f32>().to_vec())+}++fn test_prefill_norm_gate_impl(+ num_v_heads: usize,+ num_k_heads: usize,+ head_k_dim: usize,+ head_v_dim: usize,+ suffix_len: usize,+ label: &str,+) {+ let key_dim = num_k_heads * head_k_dim;+ let value_dim = num_v_heads * head_v_dim;+ let conv_dim = 2 * key_dim + value_dim;+ let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;+ let state_size = num_v_heads * head_k_dim * head_v_dim;++ let in_proj: Vec<f32> = (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect();+ let a_log: Vec<f32> = (0..num_v_heads).map(|i| -1.5 + (i as f32) * 0.05).collect();+ let dt_bias: Vec<f32> = (0..num_v_heads).map(|i| 0.3 + (i as f32) * 0.02).collect();+ let norm_weight: Vec<f32> = (0..head_v_dim).map(|i| 0.9 + (i as f32) * 0.001).collect();+ let state: Vec<f32> = (0..state_size).map(|i| ((i % 29) as f32) * 0.005 - 0.05).collect();++ // Reference: fused decode kernel token-by-token+ let mut ref_state = state.clone();+ let mut ref_outputs = vec![0.0f32; suffix_len * value_dim];+ for t in 0..suffix_len {+ let token_in: Vec<f32> = in_proj[t * total_proj_dim..(t + 1) * total_proj_dim].to_vec();+ let (out, new_state) = run_delta_net_update::<Cpu>(+ &token_in,+ &a_log,+ &dt_bias,+ &norm_weight,+ &ref_state,+ num_v_heads as u32,+ num_k_heads as u32,+ head_k_dim as u32,+ head_v_dim as u32,+ key_dim as u32,+ value_dim as u32,+ );+ ref_state = new_state;+ ref_outputs[t * value_dim..(t + 1) * value_dim].copy_from_slice(&out);+ }++ let (gpu_out, gpu_state) = run_prefill_with_norm_gate(+ &in_proj,+ &a_log,+ &dt_bias,+ &norm_weight,+ &state,+ num_v_heads,+ num_k_heads,+ head_k_dim,+ head_v_dim,+ suffix_len,+ );++ assert_close(&ref_outputs, &gpu_out, 1e-3, 1e-2, &format!("{label} output"));+ assert_close(&ref_state, &gpu_state, 1e-3, 1e-2, &format!("{label} state"));+}++#[test]+fn test_delta_net_prefill_qwen35_shapes() {+ test_prefill_norm_gate_impl(48, 16, 128, 128, 32, "Prefill+NormGate Qwen3.5");+}++#[test]+fn test_delta_net_prefill_prep() {+ let num_v_heads = 48usize;+ let num_k_heads = 16usize;+ let head_k_dim = 128usize;+ let head_v_dim = 128usize;+ let suffix_len = 8usize;++ let key_dim = num_k_heads * head_k_dim;+ let value_dim = num_v_heads * head_v_dim;+ let conv_dim = 2 * key_dim + value_dim;+ let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;++ let in_proj: Vec<f32> = (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect();+ let a_log: Vec<f32> = (0..num_v_heads).map(|i| -1.5 + (i as f32) * 0.05).collect();+ let dt_bias: Vec<f32> = (0..num_v_heads).map(|i| 0.3 + (i as f32) * 0.02).collect();++ // CPU reference via Kernels trait+ let cpu_ctx = <Cpu as Backend>::Context::new().expect("cpu context");+ let cpu_in_proj = cpu_ctx.create_array_from(&[in_proj.len()], &in_proj, "in_proj");+ let cpu_a_log = cpu_ctx.create_array_from(&[a_log.len()], &a_log, "a_log");+ let cpu_dt_bias = cpu_ctx.create_array_from(&[dt_bias.len()], &dt_bias, "dt_bias");+ let cpu_q = cpu_ctx.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "q");+ let cpu_k = cpu_ctx.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "k");+ let cpu_beta = cpu_ctx.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "beta");+ let cpu_decay = cpu_ctx.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "decay");++ let cpu_prep = <<Cpu as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(+ &cpu_ctx,+ DataType::F32,+ head_k_dim as u32,+ )+ .unwrap();+ let mut cpu_enc = Encoder::new(cpu_ctx.as_ref()).expect("encoder");+ cpu_prep.encode(+ cpu_in_proj.buffer().borrow().deref(),+ cpu_a_log.buffer().borrow().deref(),+ cpu_dt_bias.buffer().borrow().deref(),+ cpu_q.buffer().borrow_mut().deref_mut(),+ cpu_k.buffer().borrow_mut().deref_mut(),+ cpu_beta.buffer().borrow_mut().deref_mut(),+ cpu_decay.buffer().borrow_mut().deref_mut(),+ num_v_heads as u32,+ num_k_heads as u32,+ key_dim as u32,+ value_dim as u32,+ suffix_len as u32,+ &mut cpu_enc,+ );+ cpu_enc.end_encoding().submit().wait_until_completed().unwrap();++ let ref_q = cpu_q.as_slice::<f32>().to_vec();+ let ref_k = cpu_k.as_slice::<f32>().to_vec();+ let ref_beta = cpu_beta.as_slice::<f32>().to_vec();+ let ref_decay = cpu_decay.as_slice::<f32>().to_vec();++ // Metal+ let context = <Metal as Backend>::Context::new().expect("context");+ let in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj, "in_proj");+ let a_log_array = context.create_array_from(&[a_log.len()], &a_log, "a_log");+ let dt_bias_array = context.create_array_from(&[dt_bias.len()], &dt_bias, "dt_bias");+ let q_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "q_norm");+ let k_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "k_norm");++ let beta_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "beta");+ let decay_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "decay");++ let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(+ &context,+ DataType::F32,+ head_k_dim as u32,+ )+ .unwrap();++ let mut encoder = Encoder::new(context.as_ref()).expect("encoder");+ prep_k.encode(+ in_proj_array.buffer().borrow().deref(),+ a_log_array.buffer().borrow().deref(),+ dt_bias_array.buffer().borrow().deref(),+ q_norm_array.buffer().borrow_mut().deref_mut(),+ k_norm_array.buffer().borrow_mut().deref_mut(),+ beta_array.buffer().borrow_mut().deref_mut(),+ decay_array.buffer().borrow_mut().deref_mut(),+ num_v_heads as u32,+ num_k_heads as u32,+ key_dim as u32,+ value_dim as u32,+ suffix_len as u32,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ let gpu_q = q_norm_array.as_slice::<f32>();+ let gpu_k = k_norm_array.as_slice::<f32>();+ let gpu_beta = beta_array.as_slice::<f32>();+ let gpu_decay = decay_array.as_slice::<f32>();++ assert_close(gpu_q, &ref_q, 1e-4, 1e-3, "prep q_norm");+ assert_close(gpu_k, &ref_k, 1e-4, 1e-3, "prep k_norm");+ assert_close(gpu_beta, &ref_beta, 1e-4, 1e-3, "prep beta");+ assert_close(gpu_decay, &ref_decay, 1e-4, 1e-3, "prep decay");+}++#[test]+#[ignore]+fn bench_delta_net_prefill() {+ use crate::common::perf::run_perf_with_warmup;++ let num_v_heads = 48usize;+ let num_k_heads = 16usize;+ let head_k_dim = 128usize;+ let head_v_dim = 128usize;+ let suffix_len = 32usize;++ let key_dim = num_k_heads * head_k_dim;+ let value_dim = num_v_heads * head_v_dim;+ let conv_dim = 2 * key_dim + value_dim;+ let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;+ let state_size = num_v_heads * head_k_dim * head_v_dim;++ let in_proj: Vec<f32> = (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect();+ let a_log: Vec<f32> = (0..num_v_heads).map(|i| -1.5 + (i as f32) * 0.05).collect();+ let dt_bias: Vec<f32> = (0..num_v_heads).map(|i| 0.3 + (i as f32) * 0.02).collect();+ let norm_weight: Vec<f32> = (0..head_v_dim).map(|i| 0.9 + (i as f32) * 0.001).collect();++ let context = <Metal as Backend>::Context::new().expect("context");+ let in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj, "in_proj");+ let a_log_array = context.create_array_from(&[a_log.len()], &a_log, "a_log");+ let dt_bias_array = context.create_array_from(&[dt_bias.len()], &dt_bias, "dt_bias");+ let norm_weight_array = context.create_array_from(&[norm_weight.len()], &norm_weight, "norm_weight");+ let out_array = context.create_array_zeros(&[suffix_len * value_dim], DataType::F32, "out");+ let q_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "q_norm");+ let k_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "k_norm");++ let beta_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "beta");+ let decay_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "decay");++ let num_dv_groups = (head_v_dim as u32 + 7) / 8;++ let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(+ &context,+ DataType::F32,+ head_k_dim as u32,+ )+ .unwrap();+ let prefill_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillKernel::new(+ &context,+ DataType::F32,+ head_k_dim as u32,+ )+ .unwrap();+ let norm_k =+ <<Metal as Backend>::Kernels as Kernels>::DeltaNetNormGateKernel::new(&context, DataType::F32).unwrap();++ eprintln!("\n=== DeltaNet Prefill Benchmark (Qwen3.5 shapes) ===");+ eprintln!(+ " heads={num_v_heads} k_heads={num_k_heads} head_k={head_k_dim} head_v={head_v_dim} tokens={suffix_len}"+ );+ eprintln!(" state_size={state_size} ({:.2} MB)", state_size as f64 * 4.0 / 1024.0 / 1024.0);++ let prep_result = run_perf_with_warmup("prep_only", 5, 50, || {+ let mut encoder = Encoder::new(context.as_ref()).expect("encoder");+ prep_k.encode(+ in_proj_array.buffer().borrow().deref(),+ a_log_array.buffer().borrow().deref(),+ dt_bias_array.buffer().borrow().deref(),+ q_norm_array.buffer().borrow_mut().deref_mut(),+ k_norm_array.buffer().borrow_mut().deref_mut(),+ beta_array.buffer().borrow_mut().deref_mut(),+ decay_array.buffer().borrow_mut().deref_mut(),+ num_v_heads as u32,+ num_k_heads as u32,+ key_dim as u32,+ value_dim as u32,+ suffix_len as u32,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ });+ prep_result.print();++ // Benchmark prep + prefill + norm_gate (production path)+ let mut state_array = context.create_array_zeros(&[state_size], DataType::F32, "state");++ let prefill_result = run_perf_with_warmup("prep+prefill+norm_gate", 5, 50, || {+ state_array.as_slice_mut::<f32>().fill(0.0);++ let mut encoder = Encoder::new(context.as_ref()).expect("encoder");+ prep_k.encode(+ in_proj_array.buffer().borrow().deref(),+ a_log_array.buffer().borrow().deref(),+ dt_bias_array.buffer().borrow().deref(),+ q_norm_array.buffer().borrow_mut().deref_mut(),+ k_norm_array.buffer().borrow_mut().deref_mut(),+ beta_array.buffer().borrow_mut().deref_mut(),+ decay_array.buffer().borrow_mut().deref_mut(),+ num_v_heads as u32,+ num_k_heads as u32,+ key_dim as u32,+ value_dim as u32,+ suffix_len as u32,+ &mut encoder,+ );+ prefill_k.encode(+ q_norm_array.buffer().borrow().deref(),+ k_norm_array.buffer().borrow().deref(),+ beta_array.buffer().borrow().deref(),+ decay_array.buffer().borrow().deref(),+ in_proj_array.buffer().borrow().deref(),+ state_array.buffer().borrow_mut().deref_mut(),+ out_array.buffer().borrow_mut().deref_mut(),+ num_v_heads as u32,+ num_k_heads as u32,+ head_v_dim as u32,+ key_dim as u32,+ value_dim as u32,+ suffix_len as u32,+ num_dv_groups,+ &mut encoder,+ );+ norm_k.encode(+ out_array.buffer().borrow_mut().deref_mut(),+ in_proj_array.buffer().borrow().deref(),+ norm_weight_array.buffer().borrow().deref(),+ num_v_heads as u32,+ head_v_dim as u32,+ value_dim as u32,+ conv_dim as u32,+ total_proj_dim as u32,+ 1e-6f32,+ suffix_len as u32,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ });+ prefill_result.print();+}diff --git a/crates/uzu/tests/unit/kernel/mod.rs b/crates/uzu/tests/unit/kernel/mod.rsindex 3e965aafb..92faaafff 100644--- a/crates/uzu/tests/unit/kernel/mod.rs+++ b/crates/uzu/tests/unit/kernel/mod.rs@@ -6,6 +6,7 @@ mod activation_mul_test;mod activation_test;mod attention;mod audio;+mod delta_net_test;mod embedding;mod kv_cache_update_test;mod layer_norm_test;