A high-performance inference engine for AI models
Faster Qwen3.5 (#474)
CCYeh85f64ef14334
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_scan.rs b/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_scan.rsindex 5b6d9f562..36bc40ea5 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_scan.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_scan.rs@@ -16,11 +16,11 @@ use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};#[kernel(DeltaNetConvScan)]#[variants(T, f32, f16, bf16)]pub fn delta_net_conv_scan<T: ArrayElement + Float>(- conv_padded: *const T,- conv_weight: *const T,- #[optional(has_bias)] bias: Option<*const T>,+ conv_padded: *const f32,+ conv_weight: *const f32,+ #[optional(has_bias)] bias: Option<*const f32>,in_proj: *mut T,- state_out: *mut T,+ state_out: *mut f32,suffix_len: u32,kernel_size: u32,row_stride: u32,@@ -41,7 +41,7 @@ pub fn delta_net_conv_scan<T: ArrayElement + Float>(let w_offset = channel * kernel_size;let mut acc = if has_bias {- unsafe { (*bias.unwrap().add(channel)).to_f32().unwrap() }+ unsafe { *bias.unwrap().add(channel) }} else {0.0f32};@@ -49,8 +49,8 @@ pub fn delta_net_conv_scan<T: ArrayElement + Float>(for tap in 0..kernel_size {let padded_row = token + tap;let padded_idx = padded_row * row_stride + channel;- let sample = unsafe { (*conv_padded.add(padded_idx)).to_f32().unwrap() };- acc += unsafe { (*conv_weight.add(w_offset + tap)).to_f32().unwrap() } * sample;+ let sample = unsafe { *conv_padded.add(padded_idx) };+ acc += unsafe { *conv_weight.add(w_offset + tap) } * sample;}unsafe {@@ -64,9 +64,9 @@ pub fn delta_net_conv_scan<T: ArrayElement + Float>(for tap in 0..state_stride {let padded_row = suffix_len + tap;let padded_idx = padded_row * row_stride + channel;- let sample = unsafe { (*conv_padded.add(padded_idx)).to_f32().unwrap() };+ let sample = unsafe { *conv_padded.add(padded_idx) };unsafe {- *state_out.add(channel * state_stride + tap) = T::from(sample).unwrap();+ *state_out.add(channel * state_stride + tap) = sample;}}}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_update.rs b/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_update.rsindex a72a8c931..3da868b84 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_update.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_update.rs@@ -8,10 +8,10 @@ use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};#[kernel(DeltaNetConvUpdate)]#[variants(T, f32, f16, bf16)]pub fn delta_net_conv_update<T: ArrayElement + Float>(- conv_weight: *const T,- #[optional(has_bias)] bias: Option<*const T>,+ conv_weight: *const f32,+ #[optional(has_bias)] bias: Option<*const f32>,in_out: *mut T,- state: *mut T,+ state: *mut f32,kernel_size: u32,conv_dim: u32,state_stride: u32,@@ -29,16 +29,15 @@ pub fn delta_net_conv_update<T: ArrayElement + Float>(let x = unsafe { (*in_out.add(channel)).to_f32().unwrap() };let mut acc = if has_bias {- unsafe { (*bias.unwrap().add(channel)).to_f32().unwrap() }+ unsafe { *bias.unwrap().add(channel) }} else {0.0f32};for tap in 0..tap_count {- acc += unsafe { (*state.add(state_offset + tap)).to_f32().unwrap() }- * unsafe { (*conv_weight.add(weight_offset + tap)).to_f32().unwrap() };+ acc += unsafe { *state.add(state_offset + tap) } * unsafe { *conv_weight.add(weight_offset + tap) };}- acc += x * unsafe { (*conv_weight.add(weight_offset + tap_count)).to_f32().unwrap() };+ acc += x * unsafe { *conv_weight.add(weight_offset + tap_count) };unsafe {*in_out.add(channel) = T::from(ActivationType::SILU.activate(acc)).unwrap();@@ -50,7 +49,7 @@ pub fn delta_net_conv_update<T: ArrayElement + Float>(}}unsafe {- *state.add(state_offset + tap_count - 1) = T::from(x).unwrap();+ *state.add(state_offset + tap_count - 1) = x;}}}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/ssm/conv1d.rs b/crates/backend-uzu/src/backends/cpu/kernel/ssm/conv1d.rsindex f96abd837..5e0961382 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/ssm/conv1d.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/ssm/conv1d.rs@@ -5,11 +5,12 @@ use proc_macros::kernel;use crate::array::ArrayElement;#[kernel(Conv1dPack)]-#[variants(T, f32, f16, bf16)]-pub fn conv1d_pack<T: ArrayElement + Float>(- state_in: *const T,- x: *const T,- padded: *mut T,+#[variants(StateT, f32, bf16)]+#[variants(InputT, f32, bf16)]+pub fn conv1d_pack<StateT: ArrayElement + Float, InputT: ArrayElement + Float>(+ state_in: *const StateT,+ x: *const InputT,+ padded: *mut StateT,state_stride: u32,row_stride: u32,suffix_len: u32,@@ -30,7 +31,7 @@ pub fn conv1d_pack<T: ArrayElement + Float>(} else {let token = row_idx - state_stride;let x_index = token * row_stride + channel_idx;- *padded.add(padded_index) = *x.add(x_index);+ *padded.add(padded_index) = StateT::from((*x.add(x_index)).to_f32().unwrap()).unwrap();}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_scan.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_scan.metalindex efaf791ac..8aa1f1d0b 100644--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_scan.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_scan.metal@@ -17,11 +17,11 @@ using namespace metal;template <typename T>VARIANTS(T, float, half, bfloat)PUBLIC KERNEL(DeltaNetConvScan)(- device const T* conv_padded,- device const T* conv_weight,- device const T* bias OPTIONAL(has_bias),+ device const float* conv_padded,+ device const float* conv_weight,+ device const float* bias OPTIONAL(has_bias),device T* in_proj,- device T* state_out,+ device float* state_out,constant const uint& suffix_len,constant const uint& kernel_size,constant const uint& row_stride,@@ -33,14 +33,14 @@ PUBLIC KERNEL(DeltaNetConvScan)(const uint channel_idx AXIS(conv_dim, 32)) {if (token_idx < suffix_len) {- const device T* weight_row = conv_weight + channel_idx * kernel_size;+ const device float* weight_row = conv_weight + channel_idx * kernel_size;float acc = has_bias ? float(bias[channel_idx]) : 0.0f;for (uint tap = 0; tap < kernel_size; ++tap) {const uint padded_row = token_idx + tap;const uint padded_index = padded_row * row_stride + channel_idx;- acc += float(weight_row[tap]) * float(conv_padded[padded_index]);+ acc += float(weight_row[tap]) * conv_padded[padded_index];}const size_t dst = size_t(token_idx) * out_stride + channel_idx;diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_update.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_update.metalindex 8b9119f41..d9450a0f6 100644--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_update.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_update.metal@@ -8,10 +8,10 @@ using namespace metal;template <typename T>VARIANTS(T, float, half, bfloat)PUBLIC KERNEL(DeltaNetConvUpdate)(- device const T* conv_weight,- device const T* bias OPTIONAL(has_bias),+ device const float* conv_weight,+ device const float* bias OPTIONAL(has_bias),device T* in_out,- device T* state,+ device float* state,constant const uint& kernel_size,constant const uint& conv_dim,constant const uint& state_stride,@@ -20,7 +20,7 @@ PUBLIC KERNEL(DeltaNetConvUpdate)() {const uint tap_count = kernel_size - 1;const uint state_offset = channel_idx * state_stride;- const device T* weight_row = conv_weight + channel_idx * kernel_size;+ const device float* weight_row = conv_weight + channel_idx * kernel_size;float x = float(in_out[channel_idx]);@@ -36,5 +36,5 @@ PUBLIC KERNEL(DeltaNetConvUpdate)(for (uint tap = 0; tap + 1 < tap_count; ++tap) {state[state_offset + tap] = state[state_offset + tap + 1];}- state[state_offset + tap_count - 1] = static_cast<T>(x);+ state[state_offset + tap_count - 1] = x;}diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metalindex 75e19ddf6..f8811bd0b 100644--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metal@@ -19,7 +19,7 @@ VARIANTS(T, float, half, bfloat)PUBLIC KERNEL(DeltaNetNormGate)(device T* in_out,device const T* in_proj,- device const T* norm_weight,+ device const float* norm_weight,constant const uint& num_v_heads,constant const uint& head_v_dim,constant const uint& value_dim,diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metalindex b26376bb2..51d988d62 100644--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal@@ -4,10 +4,11 @@using namespace metal;-// DeltaNet prefill. One SIMD group per (v-head, dv_idx), Dk split across lanes.-// Grid: num_v_heads × num_dv_groups, PREFILL_THREADS threads each.+// DeltaNet prefill: one simdgroup owns 4 value rows while lanes cover Dk via+// float4. Grid is num_v_heads x num_dv_groups.-#define PREFILL_THREADS 256+#define PREFILL_THREADS 128+#define DV_PER_SIMDGROUP 4static_assert(PREFILL_THREADS % METAL_SIMD_SIZE == 0,@@ -23,7 +24,7 @@ PUBLIC KERNEL(DeltaNetPrefill)(device const float* beta_buf,device const float* decay_buf,device const T* in_proj,- device T* state,+ device float* state,device T* out,constant const uint& num_v_heads,constant const uint& num_k_heads,@@ -40,86 +41,80 @@ PUBLIC KERNEL(DeltaNetPrefill)(HEAD_K_DIM % METAL_SIMD_SIZE == 0,"HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE");- constexpr uint elems_per_thread = HEAD_K_DIM / METAL_SIMD_SIZE;+ constexpr uint ELEMS = HEAD_K_DIM / METAL_SIMD_SIZE;+ constexpr uint NUM_SG = PREFILL_THREADS / METAL_SIMD_SIZE;+ static_assert(ELEMS == 4, "float4 prefill requires ELEMS == 4");+ static_assert(DV_PER_SIMDGROUP == 4, "packs dv rows into a float4");- const uint dk_lane = tid % METAL_SIMD_SIZE;+ const uint lane = tid % METAL_SIMD_SIZE;const uint dv_local = tid / METAL_SIMD_SIZE;- const uint dv_idx = dv_group * (PREFILL_THREADS / METAL_SIMD_SIZE) + dv_local;- const bool active = (dv_idx < head_v_dim);+ const uint dv_idx = (dv_group * NUM_SG + dv_local) * DV_PER_SIMDGROUP;const uint groups_per_head = num_v_heads / num_k_heads;const uint hk = hv_idx / groups_per_head;const uint conv_dim = 2 * key_dim + value_dim;const uint total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;- const uint dk_base = dk_lane * elems_per_thread;+ const uint dk_base = lane * ELEMS;- // Pointer bases — increment per token instead of re-computing indexdevice const float* q_ptr = q_norm + hk * HEAD_K_DIM + dk_base;device const float* k_ptr = k_norm + hk * HEAD_K_DIM + dk_base;device const float* beta_ptr = beta_buf + hv_idx;device const float* decay_ptr = decay_buf + hv_idx;- device const T* v_ptr = in_proj + 2 * key_dim + hv_idx * head_v_dim + dv_idx;+ device const T* v_ptr = in_proj + 2 * key_dim + hv_idx * head_v_dim;+ device T* out_ptr = out + hv_idx * head_v_dim;- // State layout: [Hv, Dv, Dk] — contiguous along Dk- device T* state_ptr =- state + (hv_idx * head_v_dim + dv_idx) * HEAD_K_DIM + dk_base;- device T* out_ptr = out + hv_idx * head_v_dim + dv_idx;-- // Load state into registers- float s[elems_per_thread];+ float4 s[DV_PER_SIMDGROUP];METAL_PRAGMA_UNROLL- for (uint i = 0; i < elems_per_thread; ++i)- s[i] = active ? float(state_ptr[i]) : 0.0f;+ for (uint i = 0; i < DV_PER_SIMDGROUP; ++i) {+ uint dv = dv_idx + i;+ s[i] = *reinterpret_cast<device const float4*>(+ state + (hv_idx * head_v_dim + dv) * HEAD_K_DIM + dk_base+ );+ }for (uint token = 0; token < suffix_len; ++token) {float decay = *decay_ptr;float beta = *beta_ptr;+ float4 k = *reinterpret_cast<device const float4*>(k_ptr);+ float4 q = *reinterpret_cast<device const float4*>(q_ptr);- // Load and cache k_norm (used in both passes)- float k[elems_per_thread];METAL_PRAGMA_UNROLL- for (uint i = 0; i < elems_per_thread; ++i)- k[i] = k_ptr[i];-- // Pass 1: decay state + kv_mem = (decayed S) @ k- float kv_partial = 0.0f;- METAL_PRAGMA_UNROLL- for (uint i = 0; i < elems_per_thread; ++i) {+ for (uint i = 0; i < DV_PER_SIMDGROUP; ++i)s[i] *= decay;- kv_partial += s[i] * k[i];- }- float kv_mem = simd_sum(kv_partial);+ float4 kv_mem =+ float4(dot(s[0], k), dot(s[1], k), dot(s[2], k), dot(s[3], k));+ kv_mem = simd_sum(kv_mem);- // Delta- float v_val = active ? float(*v_ptr) : 0.0f;- float delta = beta * (v_val - kv_mem);-- // Pass 2: update state + output = new_S @ q- float out_partial = 0.0f;+ float4 o;METAL_PRAGMA_UNROLL- for (uint i = 0; i < elems_per_thread; ++i) {- s[i] += k[i] * delta;- out_partial += s[i] * q_ptr[i];+ for (uint i = 0; i < DV_PER_SIMDGROUP; ++i) {+ uint dv = dv_idx + i;+ float v_val = float(v_ptr[token * total_proj_dim + dv]);+ float delta = beta * (v_val - kv_mem[i]);+ s[i] += k * delta;+ o[i] = dot(s[i], q);}- float o_val = simd_sum(out_partial);-- if (active && dk_lane == 0) {- *out_ptr = static_cast<T>(o_val);+ o = simd_sum(o);++ if (lane == 0) {+ METAL_PRAGMA_UNROLL+ for (uint i = 0; i < DV_PER_SIMDGROUP; ++i) {+ uint dv = dv_idx + i;+ out_ptr[token * value_dim + dv] = static_cast<T>(o[i]);+ }}- // Advance pointers to next tokenq_ptr += key_dim;k_ptr += key_dim;beta_ptr += num_v_heads;decay_ptr += num_v_heads;- v_ptr += total_proj_dim;- out_ptr += value_dim;}- // Write final state- if (active) {- METAL_PRAGMA_UNROLL- for (uint i = 0; i < elems_per_thread; ++i)- state_ptr[i] = static_cast<T>(s[i]);+ METAL_PRAGMA_UNROLL+ for (uint i = 0; i < DV_PER_SIMDGROUP; ++i) {+ uint dv = dv_idx + i;+ *reinterpret_cast<device float4*>(+ state + (hv_idx * head_v_dim + dv) * HEAD_K_DIM + dk_base+ ) = s[i];}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metalindex 6365c2092..937d73719 100644--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal@@ -14,8 +14,8 @@ VARIANTS(T, float, half, bfloat)VARIANTS(HEAD_K_DIM, 128)PUBLIC KERNEL(DeltaNetPrefillPrep)(device const T* in_proj,- device const T* a_log,- device const T* dt_bias,+ device const float* a_log,+ device const float* dt_bias,device float* q_norm_out,device float* k_norm_out,device float* beta_out,diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metalindex 094cfe0c3..9a6dc76bf 100644--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal@@ -8,22 +8,20 @@using namespace metal;#define UPDATE_THREADS 512-constexpr constant uint UPDATE_K_SPLIT = 4;--static_assert(- UPDATE_THREADS % UPDATE_K_SPLIT == 0,- "UPDATE_THREADS must be a multiple of UPDATE_K_SPLIT"-);+// Bounds shared_o, which holds one o_i per dv for the cross-dv RMSNorm.+#define UPDATE_MAX_HEAD_V_DIM 128+// One simd group per output dim dv; 32 lanes cover Dk (coalesced state IO,+// reduction via simd_sum). Activations are model dtype T; state stays float.template <typename T, uint HEAD_K_DIM>VARIANTS(T, float, half, bfloat)VARIANTS(HEAD_K_DIM, 128)PUBLIC KERNEL(DeltaNetUpdate)(device const T* in_proj,- device const T* a_log,- device const T* dt_bias,- device const T* norm_weight,- device T* state,+ device const float* a_log,+ device const float* dt_bias,+ device const float* norm_weight,+ device float* state,device T* out,constant const uint& num_v_heads,constant const uint& num_k_heads,@@ -31,124 +29,113 @@ PUBLIC KERNEL(DeltaNetUpdate)(constant const uint& key_dim,constant const uint& value_dim,constant const float& norm_epsilon,- threadgroup float shared_qk[512 /* UPDATE_THREADS */ * 2],+ threadgroup float shared_o[UPDATE_MAX_HEAD_V_DIM],threadgroup float shared_scratch[METAL_SIMD_SIZE],const ThreadContext thread_context,const uint hv_idx GROUPS(num_v_heads),const uint tid THREADS(UPDATE_THREADS)) {static_assert(- HEAD_K_DIM % UPDATE_K_SPLIT == 0,- "HEAD_K_DIM must be a multiple of UPDATE_K_SPLIT"+ HEAD_K_DIM % METAL_SIMD_SIZE == 0,+ "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE");- constexpr uint bk = HEAD_K_DIM / UPDATE_K_SPLIT;-- const uint vi = tid / UPDATE_K_SPLIT;- const uint ki_part = tid % UPDATE_K_SPLIT;- const bool active = (vi < head_v_dim);+ constexpr uint ELEMS =+ HEAD_K_DIM / METAL_SIMD_SIZE; // Dk per lane (128/32 = 4)+ constexpr uint NUM_SG =+ UPDATE_THREADS / METAL_SIMD_SIZE; // simd groups / tg (16)- threadgroup float* shared_q = shared_qk;- threadgroup float* shared_k = shared_qk + UPDATE_THREADS;+ const uint lane = tid % METAL_SIMD_SIZE;+ const uint sg = tid / METAL_SIMD_SIZE;const uint conv_dim = 2 * key_dim + value_dim;const uint groups_per_head = num_v_heads / num_k_heads;const uint hk = hv_idx / groups_per_head;- if (tid < HEAD_K_DIM) {- shared_q[tid] = float(in_proj[hk * HEAD_K_DIM + tid]);- shared_k[tid] = float(in_proj[key_dim + hk * HEAD_K_DIM + tid]);+ // Load + normalize q/k; each simd group recomputes the norm to skip a+ // barrier.+ float q[ELEMS];+ float k[ELEMS];+ float q_sq = 0.0f;+ float k_sq = 0.0f;+ for (uint i = 0; i < ELEMS; ++i) {+ const uint dk = lane + METAL_SIMD_SIZE * i;+ q[i] = float(in_proj[hk * HEAD_K_DIM + dk]);+ k[i] = float(in_proj[key_dim + hk * HEAD_K_DIM + dk]);+ q_sq += q[i] * q[i];+ k_sq += k[i] * k[i];}- threadgroup_barrier(mem_flags::mem_threadgroup);-- float q_partial = (tid < HEAD_K_DIM) ? shared_q[tid] * shared_q[tid] : 0.0f;- float q_norm_sq =- threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(- q_partial,- shared_scratch,- thread_context- );- float q_inv_norm = rsqrt(q_norm_sq + 1e-6f);-- float k_partial = (tid < HEAD_K_DIM) ? shared_k[tid] * shared_k[tid] : 0.0f;- float k_norm_sq =- threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(- k_partial,- shared_scratch,- thread_context- );- float k_inv_norm = rsqrt(k_norm_sq + 1e-6f);-- float q_scale = rsqrt(float(HEAD_K_DIM));- if (tid < HEAD_K_DIM) {- shared_q[tid] *= q_inv_norm * q_scale;- shared_k[tid] *= k_inv_norm;+ const float q_inv_norm = rsqrt(simd_sum(q_sq) + 1e-6f);+ const float k_inv_norm = rsqrt(simd_sum(k_sq) + 1e-6f);+ const float q_scale = rsqrt(float(HEAD_K_DIM));+ float kq_partial = 0.0f;+ for (uint i = 0; i < ELEMS; ++i) {+ q[i] *= q_inv_norm * q_scale;+ k[i] *= k_inv_norm;+ kq_partial += q[i] * k[i];}- threadgroup_barrier(mem_flags::mem_threadgroup);-- float kq_p = (tid < HEAD_K_DIM) ? shared_k[tid] * shared_q[tid] : 0.0f;- float kq_dot =- threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(- kq_p,- shared_scratch,- thread_context- );-- float beta_raw = float(in_proj[conv_dim + value_dim + hv_idx]);- float beta = 1.0f / (1.0f + fast::exp(-beta_raw));- float a_raw = float(in_proj[conv_dim + value_dim + num_v_heads + hv_idx]);- float sp = activate_softplus(a_raw + float(dt_bias[hv_idx]));- float decay = fast::exp(-fast::exp(float(a_log[hv_idx])) * sp);-- float v_i =- active ? float(in_proj[2 * key_dim + hv_idx * head_v_dim + vi]) : 0.0f;- float z_i =- active ? float(in_proj[conv_dim + hv_idx * head_v_dim + vi]) : 0.0f;-- // State read — each thread handles bk rows- // State layout: [Hv, Dv, Dk]- float sq_partial = 0.0f;- float sk_partial = 0.0f;- const uint state_row_offset =- hv_idx * head_v_dim * HEAD_K_DIM + vi * HEAD_K_DIM;- const uint k_base = ki_part * bk;- if (active) {- for (uint j = 0; j < bk; ++j) {- float s = float(state[state_row_offset + k_base + j]);- sq_partial += s * shared_q[k_base + j];- sk_partial += s * shared_k[k_base + j];+ const float kq_dot = simd_sum(kq_partial);++ // beta / decay (scalar per head)+ const float beta_raw = float(in_proj[conv_dim + value_dim + hv_idx]);+ const float beta = 1.0f / (1.0f + fast::exp(-beta_raw));+ const float a_raw =+ float(in_proj[conv_dim + value_dim + num_v_heads + hv_idx]);+ const float sp = activate_softplus(a_raw + float(dt_bias[hv_idx]));+ const float decay = fast::exp(-fast::exp(float(a_log[hv_idx])) * sp);++ // Delta rule over the dv owned by this simd group. State is [Hv, Dv, Dk].+ for (uint dv = sg; dv < head_v_dim; dv += NUM_SG) {+ const uint state_row = (hv_idx * head_v_dim + dv) * HEAD_K_DIM;+ const float v_i = float(in_proj[2 * key_dim + hv_idx * head_v_dim + dv]);++ float s[ELEMS];+ float sq_partial = 0.0f;+ float sk_partial = 0.0f;+ for (uint i = 0; i < ELEMS; ++i) {+ const uint dk = lane + METAL_SIMD_SIZE * i;+ s[i] = state[state_row + dk];+ sq_partial += s[i] * q[i];+ sk_partial += s[i] * k[i];}- }+ const float sq_acc = simd_sum(sq_partial);+ const float sk_acc = simd_sum(sk_partial);- // Reduce across k-parts (4 consecutive lanes = one quad group)- float sq_acc = quad_sum(sq_partial);- float sk_acc = quad_sum(sk_partial);+ const float retrieved_i = decay * sk_acc;+ const float delta_i = beta * (v_i - retrieved_i);+ const float o_i = decay * sq_acc + delta_i * kq_dot;- float retrieved_i = decay * sk_acc;- float delta_i = beta * (v_i - retrieved_i);- float o_i = decay * sq_acc + delta_i * kq_dot;+ for (uint i = 0; i < ELEMS; ++i) {+ const uint dk = lane + METAL_SIMD_SIZE * i;+ state[state_row + dk] = decay * s[i] + k[i] * delta_i;+ }- // State update — each thread updates its k-part- if (active) {- for (uint j = 0; j < bk; ++j) {- uint idx = state_row_offset + k_base + j;- float s = float(state[idx]);- state[idx] = static_cast<T>(decay * s + shared_k[k_base + j] * delta_i);+ if (lane == 0) {+ shared_o[dv] = o_i;}}+ threadgroup_barrier(mem_flags::mem_threadgroup);- // RMSNorm + SiLU gate — only ki_part==0 contributes- float o_sq = (active && ki_part == 0) ? o_i * o_i : 0.0f;- float o_sumsq =+ // RMSNorm over all dv, then SiLU gate + write out.+ float o_sq = 0.0f;+ for (uint dv = tid; dv < head_v_dim; dv += UPDATE_THREADS) {+ const float o = shared_o[dv];+ o_sq += o * o;+ }+ const float o_sumsq =threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(o_sq,shared_scratch,thread_context);- float inv_rms = rsqrt(o_sumsq / float(head_v_dim) + norm_epsilon);-- if (active && ki_part == 0) {- float nw = float(norm_weight[vi]);- float z_silu = activate_silu(z_i);- out[hv_idx * head_v_dim + vi] = static_cast<T>(o_i * inv_rms * nw * z_silu);+ const float inv_rms = rsqrt(o_sumsq / float(head_v_dim) + norm_epsilon);++ for (uint dv = sg; dv < head_v_dim; dv += NUM_SG) {+ if (lane == 0) {+ const float nw = float(norm_weight[dv]);+ const float z_i = float(in_proj[conv_dim + hv_idx * head_v_dim + dv]);+ const float z_silu = activate_silu(z_i);+ out[hv_idx * head_v_dim + dv] =+ static_cast<T>(shared_o[dv] * inv_rms * nw * z_silu);+ }}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/ssm/conv1d.metal b/crates/backend-uzu/src/backends/metal/kernel/ssm/conv1d.metalindex e6100b52e..e84becbdd 100644--- a/crates/backend-uzu/src/backends/metal/kernel/ssm/conv1d.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/ssm/conv1d.metal@@ -4,12 +4,13 @@using namespace uzu::activation_type;-template <typename T>-VARIANTS(T, float, half, bfloat)+template <typename StateT, typename InputT>+VARIANTS(StateT, float, bfloat)+VARIANTS(InputT, float, bfloat)PUBLIC KERNEL(Conv1dPack)(- device const T* state_in,- device const T* x,- device T* padded,+ device const StateT* state_in,+ device const InputT* x,+ device StateT* padded,constant const uint& state_stride,constant const uint& row_stride,constant const uint& suffix_len,@@ -24,7 +25,7 @@ PUBLIC KERNEL(Conv1dPack)(} else {const uint token = row_idx - state_stride;const uint x_index = token * row_stride + channel_idx;- padded[padded_index] = x[x_index];+ padded[padded_index] = static_cast<StateT>(x[x_index]);}}@@ -165,4 +166,4 @@ PUBLIC KERNEL(Conv1dScan)(const float sample = float(padded[padded_index]);state_out[state_offset + tap] = static_cast<T>(sample);}-}\ No newline at end of file+}diff --git a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rsindex 89e60f0c2..f2826776c 100644--- a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs+++ b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs@@ -55,7 +55,9 @@ pub struct DeltaNetMixer<B: Backend> {a_log: Allocation<B>,dt_bias: Allocation<B>,norm_weight: Allocation<B>,- inner_data_type: DataType,+ // Activation/scratch dtype (model dtype, e.g. bf16). The recurrent ssm_state+ // and small weights (a_log/dt_bias/norm/conv) stay f32 inside the kernels.+ outer_data_type: DataType,}pub struct DeltaNetArguments<'a, B: Backend> {@@ -109,7 +111,7 @@ impl<B: Backend> DeltaNetMixer<B> {context,outer_data_type,outer_data_type,- inner_data_type,+ outer_data_type,¶meter_tree.subtree("in_proj")?,)?;@@ -119,7 +121,7 @@ impl<B: Backend> DeltaNetMixer<B> {false,context,outer_data_type,- inner_data_type,+ outer_data_type,outer_data_type,¶meter_tree.subtree("out_proj")?,)?;@@ -141,22 +143,22 @@ impl<B: Backend> DeltaNetMixer<B> {.read_allocation()?;// Create kernels- let conv_update = <B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, inner_data_type, has_bias)+ let conv_update = <B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, outer_data_type, has_bias).map_err(DeltaNetMixerError::BackendError)?;let delta_net_update =- <B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, inner_data_type, config.head_dim as u32)+ <B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, outer_data_type, config.head_dim as u32).map_err(DeltaNetMixerError::BackendError)?;- let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, inner_data_type)+ let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, inner_data_type, outer_data_type).map_err(DeltaNetMixerError::BackendError)?;- let conv_scan = <B::Kernels as Kernels>::DeltaNetConvScanKernel::new(context, inner_data_type, has_bias)+ let conv_scan = <B::Kernels as Kernels>::DeltaNetConvScanKernel::new(context, outer_data_type, has_bias).map_err(DeltaNetMixerError::BackendError)?;let prefill_prep =- <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(context, inner_data_type, config.head_dim as u32)+ <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(context, outer_data_type, config.head_dim as u32).map_err(DeltaNetMixerError::BackendError)?;let delta_net_prefill =- <B::Kernels as Kernels>::DeltaNetPrefillKernel::new(context, inner_data_type, config.head_dim as u32)+ <B::Kernels as Kernels>::DeltaNetPrefillKernel::new(context, outer_data_type, config.head_dim as u32).map_err(DeltaNetMixerError::BackendError)?;- let norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, inner_data_type)+ let norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, outer_data_type).map_err(DeltaNetMixerError::BackendError)?;Ok((@@ -184,7 +186,7 @@ impl<B: Backend> DeltaNetMixer<B> {a_log,dt_bias,norm_weight,- inner_data_type,+ outer_data_type,},in_projection_input_hadamard_factors,))@@ -220,8 +222,8 @@ impl<B: Backend> DeltaNetMixer<B> {let state_stride = kernel_size - 1;let conv_dim = self.conv_dim;let total_proj_dim = self.total_proj_dim;- let mut padded = encoder- .allocate_scratch(size_for_shape(&[suffix_length + state_stride, total_proj_dim], self.inner_data_type))?;+ let mut padded =+ encoder.allocate_scratch(size_for_shape(&[suffix_length + state_stride, total_proj_dim], DataType::F32))?;self.conv_pack.encode(&layer.conv_state,@@ -259,7 +261,7 @@ impl<B: Backend> DeltaNetMixer<B> {encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {let mut out =- encoder.allocate_scratch(size_for_shape(&[active_row_count, self.value_dim], self.inner_data_type))?;+ encoder.allocate_scratch(size_for_shape(&[active_row_count, self.value_dim], self.outer_data_type))?;self.delta_net_update.encode(in_proj,&self.a_log,@@ -285,7 +287,7 @@ impl<B: Backend> DeltaNetMixer<B> {suffix_length: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let num_dv_groups = self.value_head_dim.div_ceil(8) as u32;+ let num_dv_groups = self.value_head_dim.div_ceil(16) as u32;let mut prep_q_norm =encoder.allocate_scratch(size_for_shape(&[suffix_length * self.key_dim], DataType::F32))?;let mut prep_k_norm =@@ -312,7 +314,7 @@ impl<B: Backend> DeltaNetMixer<B> {);let mut out =- encoder.allocate_scratch(size_for_shape(&[suffix_length, self.value_dim], self.inner_data_type))?;+ encoder.allocate_scratch(size_for_shape(&[suffix_length, self.value_dim], self.outer_data_type))?;self.delta_net_prefill.encode(&prep_q_norm,&prep_k_norm,diff --git a/crates/backend-uzu/src/encodable_block/mamba_mixer/mod.rs b/crates/backend-uzu/src/encodable_block/mamba_mixer/mod.rsindex 417995a52..5497d7b4d 100644--- a/crates/backend-uzu/src/encodable_block/mamba_mixer/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mamba_mixer/mod.rs@@ -151,7 +151,7 @@ impl<B: Backend> MambaMixer<B> {true,).map_err(MambaMixerError::BackendError)?;- let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, inner_data_type)+ let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, inner_data_type, inner_data_type).map_err(MambaMixerError::BackendError)?;let ssd_prefill = SSDPrefillBlock::new(context, inner_data_type).map_err(MambaMixerError::BackendError)?;let ssd_update = <B::Kernels as Kernels>::SSDUpdateKernel::new(context, inner_data_type, true)diff --git a/crates/backend-uzu/src/forward_pass/cache_layers.rs b/crates/backend-uzu/src/forward_pass/cache_layers.rsindex 044f12a03..f7770c98f 100644--- a/crates/backend-uzu/src/forward_pass/cache_layers.rs+++ b/crates/backend-uzu/src/forward_pass/cache_layers.rs@@ -222,11 +222,12 @@ impl<B: Backend> CacheLayers<B> {})},AnyTokenMixerConfig::DeltaNetConfig(c) => {- let data_type = DataType::F32;+ let conv_data_type = DataType::F32;+ let ssm_data_type = DataType::F32;let conv_shape = [c.conv_dim(), c.kernel_size.saturating_sub(1)];let ssm_shape = [c.num_heads, c.value_head_dim, c.head_dim];- let conv_bytes = size_for_shape(&conv_shape, data_type);- let ssm_bytes = size_for_shape(&ssm_shape, data_type);+ let conv_bytes = size_for_shape(&conv_shape, conv_data_type);+ let ssm_bytes = size_for_shape(&ssm_shape, ssm_data_type);CacheLayer::DeltaNet(DeltaNetLayer {conv_state: context@@ -237,7 +238,7 @@ impl<B: Backend> CacheLayers<B> {.create_allocation(ssm_bytes, AllocationType::Global).expect("Failed to create delta net ssm allocation"),ssm_shape,- data_type,+ data_type: ssm_data_type,})},};diff --git a/crates/backend-uzu/tests/unit/kernel/delta_net_test.rs b/crates/backend-uzu/tests/unit/kernel/delta_net_test.rsindex 24275dc30..8338bcda0 100644--- a/crates/backend-uzu/tests/unit/kernel/delta_net_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/delta_net_test.rs@@ -1,7 +1,7 @@#![cfg(metal_backend)]use backend_uzu::{- array::ArrayContextExt,+ array::{ArrayContextExt, ArrayElement},backends::{common::{Backend, Context, Encoder, Kernels,@@ -15,6 +15,7 @@ use backend_uzu::{},data_type::DataType,};+use half::bf16;use crate::uzu_test;@@ -153,13 +154,15 @@ fn test_delta_net_conv_scan() {let w: Vec<f32> = (0..conv_dim * kernel_size).map(|i| ((i % 11) as f32) * 0.05 - 0.2).collect();let b: Vec<f32> = (0..conv_dim).map(|i| ((i % 5) as f32) * 0.01).collect();let init_state: Vec<f32> = (0..conv_dim * tap_count).map(|i| ((i % 13) as f32) * 0.02 - 0.1).collect();- let in_proj: Vec<f32> = (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect();+ let in_proj: Vec<bf16> =+ (0..suffix_len * total_proj_dim).map(|i| bf16::from_f32(((i % 37) as f32) * 0.02 - 0.3)).collect();+ let in_proj_f32: Vec<f32> = in_proj.iter().copied().map(f32::from).collect();// Reference: decode conv token-by-tokenlet mut ref_state = init_state.clone();let mut ref_outputs = vec![0.0f32; suffix_len * conv_dim];for t in 0..suffix_len {- let token_in: Vec<f32> = in_proj[t * total_proj_dim..t * total_proj_dim + conv_dim].to_vec();+ let token_in: Vec<f32> = in_proj_f32[t * total_proj_dim..t * total_proj_dim + conv_dim].to_vec();let (out, new_state) = run_conv_update::<Cpu>(&token_in,&w,@@ -172,6 +175,7 @@ fn test_delta_net_conv_scan() {ref_state = new_state;ref_outputs[t * conv_dim..(t + 1) * conv_dim].copy_from_slice(&out);}+ let ref_outputs: Vec<f32> = ref_outputs.iter().copied().map(|value| f32::from(bf16::from_f32(value))).collect();// Test: Conv1dPack + DeltaNetConvScan on Metallet context = <Metal as Backend>::Context::new().expect("context");@@ -185,9 +189,10 @@ fn test_delta_net_conv_scan() {let mut state_out_array = context.create_array_zeros(&[conv_dim * tap_count], DataType::F32).into_allocation();let pack_kernel =- <<Metal as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, DataType::F32).expect("pack");+ <<Metal as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, DataType::F32, DataType::BF16)+ .expect("pack");let scan_kernel =- <<Metal as Backend>::Kernels as Kernels>::DeltaNetConvScanKernel::new(&context, DataType::F32, true)+ <<Metal as Backend>::Kernels as Kernels>::DeltaNetConvScanKernel::new(&context, DataType::BF16, true).expect("scan");let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");@@ -217,7 +222,8 @@ fn test_delta_net_conv_scan() {);encoder.end_encoding().submit().wait_until_completed().unwrap();- let in_proj_result: Vec<f32> = crate::common::helpers::allocation_to_vec(&in_proj_array);+ let in_proj_result: Vec<bf16> = crate::common::helpers::allocation_to_vec(&in_proj_array);+ let in_proj_result: Vec<f32> = in_proj_result.into_iter().map(f32::from).collect();let mut scan_outputs = vec![0.0f32; suffix_len * conv_dim];for t in 0..suffix_len {scan_outputs[t * conv_dim..(t + 1) * conv_dim]@@ -288,8 +294,8 @@ fn test_delta_net_update_qwen35_shapes() {// DeltaNetPrefill + NormGate-fn run_prefill_with_norm_gate(- in_proj: &[f32],+fn run_prefill_with_norm_gate_typed<T: ArrayElement>(+ in_proj: &[T],a_log: &[f32],dt_bias: &[f32],norm_weight: &[f32],@@ -304,7 +310,7 @@ fn run_prefill_with_norm_gate(let value_dim = num_v_heads * head_v_dim;let conv_dim = 2 * key_dim + value_dim;let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;- let num_dv_groups = (head_v_dim as u32).div_ceil(8);+ let num_dv_groups = head_v_dim.div_ceil(16) as u32;let context = <Metal as Backend>::Context::new().expect("context");let in_proj_array = context.create_array_from(&[in_proj.len()], in_proj);@@ -312,7 +318,7 @@ fn run_prefill_with_norm_gate(let dt_bias_array = context.create_array_from(&[dt_bias.len()], dt_bias);let norm_weight_array = context.create_array_from(&[norm_weight.len()], norm_weight);let mut state_array = context.create_array_from(&[state.len()], state).into_allocation();- let mut out_array = context.create_array_zeros(&[suffix_len * value_dim], DataType::F32).into_allocation();+ let mut out_array = context.create_array_zeros(&[suffix_len * value_dim], T::data_type()).into_allocation();let mut q_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();let mut k_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();@@ -321,18 +327,18 @@ fn run_prefill_with_norm_gate(let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(&context,- DataType::F32,+ T::data_type(),head_k_dim as u32,).unwrap();let prefill_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillKernel::new(&context,- DataType::F32,+ T::data_type(),head_k_dim as u32,).unwrap();let norm_k =- <<Metal as Backend>::Kernels as Kernels>::DeltaNetNormGateKernel::new(&context, DataType::F32).unwrap();+ <<Metal as Backend>::Kernels as Kernels>::DeltaNetNormGateKernel::new(&context, T::data_type()).unwrap();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");prep_k.encode(@@ -382,15 +388,20 @@ fn run_prefill_with_norm_gate();encoder.end_encoding().submit().wait_until_completed().unwrap();- (crate::common::helpers::allocation_to_vec(&out_array), crate::common::helpers::allocation_to_vec(&state_array))+ let out: Vec<T> = crate::common::helpers::allocation_to_vec(&out_array);+ let out = out.into_iter().map(|value| value.to_f32().expect("output to f32")).collect();+ let state = crate::common::helpers::allocation_to_vec(&state_array);+ (out, state)}-fn test_prefill_norm_gate_impl(+fn test_prefill_norm_gate_impl<T: ArrayElement>(num_v_heads: usize,num_k_heads: usize,head_k_dim: usize,head_v_dim: usize,suffix_len: usize,+ output_atol: f32,+ output_rtol: f32,label: &str,) {let key_dim = num_k_heads * head_k_dim;@@ -400,6 +411,16 @@ fn test_prefill_norm_gate_impl(let state_size = num_v_heads * head_k_dim * head_v_dim;let in_proj: Vec<f32> = (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect();+ let in_proj_typed: Vec<T> = in_proj+ .iter()+ .copied()+ .map(|value| <T as num_traits::NumCast>::from(value).expect("input to activation dtype"))+ .collect();+ let reference_in_proj: Vec<f32> = in_proj_typed+ .iter()+ .copied()+ .map(|value| <f32 as num_traits::NumCast>::from(value).expect("activation dtype to f32"))+ .collect();let a_log: Vec<f32> = (0..num_v_heads).map(|i| -1.5 + (i as f32) * 0.05).collect();let dt_bias: Vec<f32> = (0..num_v_heads).map(|i| 0.3 + (i as f32) * 0.02).collect();let norm_weight: Vec<f32> = (0..head_v_dim).map(|i| 0.9 + (i as f32) * 0.001).collect();@@ -408,8 +429,9 @@ fn test_prefill_norm_gate_impl(// Reference: fused decode kernel token-by-tokenlet mut ref_state = state.clone();let mut ref_outputs = vec![0.0f32; suffix_len * value_dim];- for t in 0..suffix_len {- let token_in: Vec<f32> = in_proj[t * total_proj_dim..(t + 1) * total_proj_dim].to_vec();+ for token_index in 0..suffix_len {+ let token_in: Vec<f32> =+ reference_in_proj[token_index * total_proj_dim..(token_index + 1) * total_proj_dim].to_vec();let (out, new_state) = run_delta_net_update::<Cpu>(&token_in,&a_log,@@ -424,11 +446,11 @@ fn test_prefill_norm_gate_impl(value_dim as u32,);ref_state = new_state;- ref_outputs[t * value_dim..(t + 1) * value_dim].copy_from_slice(&out);+ ref_outputs[token_index * value_dim..(token_index + 1) * value_dim].copy_from_slice(&out);}- let (gpu_out, gpu_state) = run_prefill_with_norm_gate(- &in_proj,+ let (gpu_out, gpu_state) = run_prefill_with_norm_gate_typed(+ &in_proj_typed,&a_log,&dt_bias,&norm_weight,@@ -440,13 +462,18 @@ fn test_prefill_norm_gate_impl(suffix_len,);- assert_close(&ref_outputs, &gpu_out, 1e-3, 1e-2, &format!("{label} output"));+ assert_close(&ref_outputs, &gpu_out, output_atol, output_rtol, &format!("{label} output"));assert_close(&ref_state, &gpu_state, 1e-3, 1e-2, &format!("{label} state"));}#[uzu_test]fn test_delta_net_prefill_qwen35_shapes() {- test_prefill_norm_gate_impl(48, 16, 128, 128, 32, "Prefill+NormGate Qwen3.5");+ test_prefill_norm_gate_impl::<f32>(48, 16, 128, 128, 32, 1e-3, 1e-2, "Prefill+NormGate Qwen3.5");+}++#[uzu_test]+fn test_delta_net_prefill_qwen35_shapes_bf16() {+ test_prefill_norm_gate_impl::<bf16>(48, 16, 128, 128, 32, 2e-2, 5e-2, "Prefill+NormGate Qwen3.5 BF16");}#[uzu_test]@@ -586,7 +613,7 @@ fn bench_delta_net_prefill() {let mut beta_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();let mut decay_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();- let num_dv_groups = (head_v_dim as u32).div_ceil(8);+ let num_dv_groups = head_v_dim.div_ceil(16) as u32;let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(&context,diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rsindex 70b82cb26..183732e73 100644--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs@@ -8,7 +8,7 @@ use backend_uzu::{},data_type::DataType,};-use half::{bf16, f16};+use half::bf16;use num_traits::Float;use crate::{common::assert::assert_eq_float, uzu_test};@@ -46,7 +46,7 @@ fn get_input<T: ArrayElement + Float>(fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");- let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, T::data_type())+ let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, T::data_type(), T::data_type()).expect("Failed to create Conv1dPackKernel");let state_size = input.state_in.len();@@ -159,27 +159,6 @@ fn test_channels_lt_row_stride_f32() {test_channels_lt_row_stride::<f32>();}-// f16-#[uzu_test]-fn test_basic_f16() {- test_basic::<f16>();-}--#[uzu_test]-fn test_single_token_f16() {- test_single_token::<f16>();-}--#[uzu_test]-fn test_many_tokens_f16() {- test_many_tokens::<f16>();-}--#[uzu_test]-fn test_large_f16() {- test_large::<f16>();-}-// bf16#[uzu_test]fn test_basic_bf16() {