Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Faster Qwen3.5 (#474)

CCYeh85f64ef14334
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_scan.rs b/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_scan.rs
index 5b6d9f562..36bc40ea5 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_scan.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_scan.rs
@@ -16,11 +16,11 @@ use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};
#[kernel(DeltaNetConvScan)]
#[variants(T, f32, f16, bf16)]
pub fn delta_net_conv_scan<T: ArrayElement + Float>(
- conv_padded: *const T,
- conv_weight: *const T,
- #[optional(has_bias)] bias: Option<*const T>,
+ conv_padded: *const f32,
+ conv_weight: *const f32,
+ #[optional(has_bias)] bias: Option<*const f32>,
in_proj: *mut T,
- state_out: *mut T,
+ state_out: *mut f32,
suffix_len: u32,
kernel_size: u32,
row_stride: u32,
@@ -41,7 +41,7 @@ pub fn delta_net_conv_scan<T: ArrayElement + Float>(
let w_offset = channel * kernel_size;
let mut acc = if has_bias {
- unsafe { (*bias.unwrap().add(channel)).to_f32().unwrap() }
+ unsafe { *bias.unwrap().add(channel) }
} else {
0.0f32
};
@@ -49,8 +49,8 @@ pub fn delta_net_conv_scan<T: ArrayElement + Float>(
for tap in 0..kernel_size {
let padded_row = token + tap;
let padded_idx = padded_row * row_stride + channel;
- let sample = unsafe { (*conv_padded.add(padded_idx)).to_f32().unwrap() };
- acc += unsafe { (*conv_weight.add(w_offset + tap)).to_f32().unwrap() } * sample;
+ let sample = unsafe { *conv_padded.add(padded_idx) };
+ acc += unsafe { *conv_weight.add(w_offset + tap) } * sample;
}
unsafe {
@@ -64,9 +64,9 @@ pub fn delta_net_conv_scan<T: ArrayElement + Float>(
for tap in 0..state_stride {
let padded_row = suffix_len + tap;
let padded_idx = padded_row * row_stride + channel;
- let sample = unsafe { (*conv_padded.add(padded_idx)).to_f32().unwrap() };
+ let sample = unsafe { *conv_padded.add(padded_idx) };
unsafe {
- *state_out.add(channel * state_stride + tap) = T::from(sample).unwrap();
+ *state_out.add(channel * state_stride + tap) = sample;
}
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_update.rs b/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_update.rs
index a72a8c931..3da868b84 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_update.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/delta_net/conv_update.rs
@@ -8,10 +8,10 @@ use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};
#[kernel(DeltaNetConvUpdate)]
#[variants(T, f32, f16, bf16)]
pub fn delta_net_conv_update<T: ArrayElement + Float>(
- conv_weight: *const T,
- #[optional(has_bias)] bias: Option<*const T>,
+ conv_weight: *const f32,
+ #[optional(has_bias)] bias: Option<*const f32>,
in_out: *mut T,
- state: *mut T,
+ state: *mut f32,
kernel_size: u32,
conv_dim: u32,
state_stride: u32,
@@ -29,16 +29,15 @@ pub fn delta_net_conv_update<T: ArrayElement + Float>(
let x = unsafe { (*in_out.add(channel)).to_f32().unwrap() };
let mut acc = if has_bias {
- unsafe { (*bias.unwrap().add(channel)).to_f32().unwrap() }
+ unsafe { *bias.unwrap().add(channel) }
} else {
0.0f32
};
for tap in 0..tap_count {
- acc += unsafe { (*state.add(state_offset + tap)).to_f32().unwrap() }
- * unsafe { (*conv_weight.add(weight_offset + tap)).to_f32().unwrap() };
+ acc += unsafe { *state.add(state_offset + tap) } * unsafe { *conv_weight.add(weight_offset + tap) };
}
- acc += x * unsafe { (*conv_weight.add(weight_offset + tap_count)).to_f32().unwrap() };
+ acc += x * unsafe { *conv_weight.add(weight_offset + tap_count) };
unsafe {
*in_out.add(channel) = T::from(ActivationType::SILU.activate(acc)).unwrap();
@@ -50,7 +49,7 @@ pub fn delta_net_conv_update<T: ArrayElement + Float>(
}
}
unsafe {
- *state.add(state_offset + tap_count - 1) = T::from(x).unwrap();
+ *state.add(state_offset + tap_count - 1) = x;
}
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/ssm/conv1d.rs b/crates/backend-uzu/src/backends/cpu/kernel/ssm/conv1d.rs
index f96abd837..5e0961382 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/ssm/conv1d.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/ssm/conv1d.rs
@@ -5,11 +5,12 @@ use proc_macros::kernel;
use crate::array::ArrayElement;
#[kernel(Conv1dPack)]
-#[variants(T, f32, f16, bf16)]
-pub fn conv1d_pack<T: ArrayElement + Float>(
- state_in: *const T,
- x: *const T,
- padded: *mut T,
+#[variants(StateT, f32, bf16)]
+#[variants(InputT, f32, bf16)]
+pub fn conv1d_pack<StateT: ArrayElement + Float, InputT: ArrayElement + Float>(
+ state_in: *const StateT,
+ x: *const InputT,
+ padded: *mut StateT,
state_stride: u32,
row_stride: u32,
suffix_len: u32,
@@ -30,7 +31,7 @@ pub fn conv1d_pack<T: ArrayElement + Float>(
} else {
let token = row_idx - state_stride;
let x_index = token * row_stride + channel_idx;
- *padded.add(padded_index) = *x.add(x_index);
+ *padded.add(padded_index) = StateT::from((*x.add(x_index)).to_f32().unwrap()).unwrap();
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_scan.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_scan.metal
index efaf791ac..8aa1f1d0b 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_scan.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_scan.metal
@@ -17,11 +17,11 @@ using namespace metal;
template <typename T>
VARIANTS(T, float, half, bfloat)
PUBLIC KERNEL(DeltaNetConvScan)(
- device const T* conv_padded,
- device const T* conv_weight,
- device const T* bias OPTIONAL(has_bias),
+ device const float* conv_padded,
+ device const float* conv_weight,
+ device const float* bias OPTIONAL(has_bias),
device T* in_proj,
- device T* state_out,
+ device float* state_out,
constant const uint& suffix_len,
constant const uint& kernel_size,
constant const uint& row_stride,
@@ -33,14 +33,14 @@ PUBLIC KERNEL(DeltaNetConvScan)(
const uint channel_idx AXIS(conv_dim, 32)
) {
if (token_idx < suffix_len) {
- const device T* weight_row = conv_weight + channel_idx * kernel_size;
+ const device float* weight_row = conv_weight + channel_idx * kernel_size;
float acc = has_bias ? float(bias[channel_idx]) : 0.0f;
for (uint tap = 0; tap < kernel_size; ++tap) {
const uint padded_row = token_idx + tap;
const uint padded_index = padded_row * row_stride + channel_idx;
- acc += float(weight_row[tap]) * float(conv_padded[padded_index]);
+ acc += float(weight_row[tap]) * conv_padded[padded_index];
}
const size_t dst = size_t(token_idx) * out_stride + channel_idx;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_update.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_update.metal
index 8b9119f41..d9450a0f6 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_update.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/conv_update.metal
@@ -8,10 +8,10 @@ using namespace metal;
template <typename T>
VARIANTS(T, float, half, bfloat)
PUBLIC KERNEL(DeltaNetConvUpdate)(
- device const T* conv_weight,
- device const T* bias OPTIONAL(has_bias),
+ device const float* conv_weight,
+ device const float* bias OPTIONAL(has_bias),
device T* in_out,
- device T* state,
+ device float* state,
constant const uint& kernel_size,
constant const uint& conv_dim,
constant const uint& state_stride,
@@ -20,7 +20,7 @@ PUBLIC KERNEL(DeltaNetConvUpdate)(
) {
const uint tap_count = kernel_size - 1;
const uint state_offset = channel_idx * state_stride;
- const device T* weight_row = conv_weight + channel_idx * kernel_size;
+ const device float* weight_row = conv_weight + channel_idx * kernel_size;
float x = float(in_out[channel_idx]);
@@ -36,5 +36,5 @@ PUBLIC KERNEL(DeltaNetConvUpdate)(
for (uint tap = 0; tap + 1 < tap_count; ++tap) {
state[state_offset + tap] = state[state_offset + tap + 1];
}
- state[state_offset + tap_count - 1] = static_cast<T>(x);
+ state[state_offset + tap_count - 1] = x;
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metal
index 75e19ddf6..f8811bd0b 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metal
@@ -19,7 +19,7 @@ VARIANTS(T, float, half, bfloat)
PUBLIC KERNEL(DeltaNetNormGate)(
device T* in_out,
device const T* in_proj,
- device const T* norm_weight,
+ device const float* norm_weight,
constant const uint& num_v_heads,
constant const uint& head_v_dim,
constant const uint& value_dim,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal
index b26376bb2..51d988d62 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal
@@ -4,10 +4,11 @@
using namespace metal;
-// DeltaNet prefill. One SIMD group per (v-head, dv_idx), Dk split across lanes.
-// Grid: num_v_heads × num_dv_groups, PREFILL_THREADS threads each.
+// DeltaNet prefill: one simdgroup owns 4 value rows while lanes cover Dk via
+// float4. Grid is num_v_heads x num_dv_groups.
-#define PREFILL_THREADS 256
+#define PREFILL_THREADS 128
+#define DV_PER_SIMDGROUP 4
static_assert(
PREFILL_THREADS % METAL_SIMD_SIZE == 0,
@@ -23,7 +24,7 @@ PUBLIC KERNEL(DeltaNetPrefill)(
device const float* beta_buf,
device const float* decay_buf,
device const T* in_proj,
- device T* state,
+ device float* state,
device T* out,
constant const uint& num_v_heads,
constant const uint& num_k_heads,
@@ -40,86 +41,80 @@ PUBLIC KERNEL(DeltaNetPrefill)(
HEAD_K_DIM % METAL_SIMD_SIZE == 0,
"HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE"
);
- constexpr uint elems_per_thread = HEAD_K_DIM / METAL_SIMD_SIZE;
+ constexpr uint ELEMS = HEAD_K_DIM / METAL_SIMD_SIZE;
+ constexpr uint NUM_SG = PREFILL_THREADS / METAL_SIMD_SIZE;
+ static_assert(ELEMS == 4, "float4 prefill requires ELEMS == 4");
+ static_assert(DV_PER_SIMDGROUP == 4, "packs dv rows into a float4");
- const uint dk_lane = tid % METAL_SIMD_SIZE;
+ const uint lane = tid % METAL_SIMD_SIZE;
const uint dv_local = tid / METAL_SIMD_SIZE;
- const uint dv_idx = dv_group * (PREFILL_THREADS / METAL_SIMD_SIZE) + dv_local;
- const bool active = (dv_idx < head_v_dim);
+ const uint dv_idx = (dv_group * NUM_SG + dv_local) * DV_PER_SIMDGROUP;
const uint groups_per_head = num_v_heads / num_k_heads;
const uint hk = hv_idx / groups_per_head;
const uint conv_dim = 2 * key_dim + value_dim;
const uint total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;
- const uint dk_base = dk_lane * elems_per_thread;
+ const uint dk_base = lane * ELEMS;
- // Pointer bases — increment per token instead of re-computing index
device const float* q_ptr = q_norm + hk * HEAD_K_DIM + dk_base;
device const float* k_ptr = k_norm + hk * HEAD_K_DIM + dk_base;
device const float* beta_ptr = beta_buf + hv_idx;
device const float* decay_ptr = decay_buf + hv_idx;
- device const T* v_ptr = in_proj + 2 * key_dim + hv_idx * head_v_dim + dv_idx;
+ device const T* v_ptr = in_proj + 2 * key_dim + hv_idx * head_v_dim;
+ device T* out_ptr = out + hv_idx * head_v_dim;
- // State layout: [Hv, Dv, Dk] — contiguous along Dk
- device T* state_ptr =
- state + (hv_idx * head_v_dim + dv_idx) * HEAD_K_DIM + dk_base;
- device T* out_ptr = out + hv_idx * head_v_dim + dv_idx;
-
- // Load state into registers
- float s[elems_per_thread];
+ float4 s[DV_PER_SIMDGROUP];
METAL_PRAGMA_UNROLL
- for (uint i = 0; i < elems_per_thread; ++i)
- s[i] = active ? float(state_ptr[i]) : 0.0f;
+ for (uint i = 0; i < DV_PER_SIMDGROUP; ++i) {
+ uint dv = dv_idx + i;
+ s[i] = *reinterpret_cast<device const float4*>(
+ state + (hv_idx * head_v_dim + dv) * HEAD_K_DIM + dk_base
+ );
+ }
for (uint token = 0; token < suffix_len; ++token) {
float decay = *decay_ptr;
float beta = *beta_ptr;
+ float4 k = *reinterpret_cast<device const float4*>(k_ptr);
+ float4 q = *reinterpret_cast<device const float4*>(q_ptr);
- // Load and cache k_norm (used in both passes)
- float k[elems_per_thread];
METAL_PRAGMA_UNROLL
- for (uint i = 0; i < elems_per_thread; ++i)
- k[i] = k_ptr[i];
-
- // Pass 1: decay state + kv_mem = (decayed S) @ k
- float kv_partial = 0.0f;
- METAL_PRAGMA_UNROLL
- for (uint i = 0; i < elems_per_thread; ++i) {
+ for (uint i = 0; i < DV_PER_SIMDGROUP; ++i)
s[i] *= decay;
- kv_partial += s[i] * k[i];
- }
- float kv_mem = simd_sum(kv_partial);
+ float4 kv_mem =
+ float4(dot(s[0], k), dot(s[1], k), dot(s[2], k), dot(s[3], k));
+ kv_mem = simd_sum(kv_mem);
- // Delta
- float v_val = active ? float(*v_ptr) : 0.0f;
- float delta = beta * (v_val - kv_mem);
-
- // Pass 2: update state + output = new_S @ q
- float out_partial = 0.0f;
+ float4 o;
METAL_PRAGMA_UNROLL
- for (uint i = 0; i < elems_per_thread; ++i) {
- s[i] += k[i] * delta;
- out_partial += s[i] * q_ptr[i];
+ for (uint i = 0; i < DV_PER_SIMDGROUP; ++i) {
+ uint dv = dv_idx + i;
+ float v_val = float(v_ptr[token * total_proj_dim + dv]);
+ float delta = beta * (v_val - kv_mem[i]);
+ s[i] += k * delta;
+ o[i] = dot(s[i], q);
}
- float o_val = simd_sum(out_partial);
-
- if (active && dk_lane == 0) {
- *out_ptr = static_cast<T>(o_val);
+ o = simd_sum(o);
+
+ if (lane == 0) {
+ METAL_PRAGMA_UNROLL
+ for (uint i = 0; i < DV_PER_SIMDGROUP; ++i) {
+ uint dv = dv_idx + i;
+ out_ptr[token * value_dim + dv] = static_cast<T>(o[i]);
+ }
}
- // Advance pointers to next token
q_ptr += key_dim;
k_ptr += key_dim;
beta_ptr += num_v_heads;
decay_ptr += num_v_heads;
- v_ptr += total_proj_dim;
- out_ptr += value_dim;
}
- // Write final state
- if (active) {
- METAL_PRAGMA_UNROLL
- for (uint i = 0; i < elems_per_thread; ++i)
- state_ptr[i] = static_cast<T>(s[i]);
+ METAL_PRAGMA_UNROLL
+ for (uint i = 0; i < DV_PER_SIMDGROUP; ++i) {
+ uint dv = dv_idx + i;
+ *reinterpret_cast<device float4*>(
+ state + (hv_idx * head_v_dim + dv) * HEAD_K_DIM + dk_base
+ ) = s[i];
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal
index 6365c2092..937d73719 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal
@@ -14,8 +14,8 @@ VARIANTS(T, float, half, bfloat)
VARIANTS(HEAD_K_DIM, 128)
PUBLIC KERNEL(DeltaNetPrefillPrep)(
device const T* in_proj,
- device const T* a_log,
- device const T* dt_bias,
+ device const float* a_log,
+ device const float* dt_bias,
device float* q_norm_out,
device float* k_norm_out,
device float* beta_out,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal
index 094cfe0c3..9a6dc76bf 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal
@@ -8,22 +8,20 @@
using namespace metal;
#define UPDATE_THREADS 512
-constexpr constant uint UPDATE_K_SPLIT = 4;
-
-static_assert(
- UPDATE_THREADS % UPDATE_K_SPLIT == 0,
- "UPDATE_THREADS must be a multiple of UPDATE_K_SPLIT"
-);
+// Bounds shared_o, which holds one o_i per dv for the cross-dv RMSNorm.
+#define UPDATE_MAX_HEAD_V_DIM 128
+// One simd group per output dim dv; 32 lanes cover Dk (coalesced state IO,
+// reduction via simd_sum). Activations are model dtype T; state stays float.
template <typename T, uint HEAD_K_DIM>
VARIANTS(T, float, half, bfloat)
VARIANTS(HEAD_K_DIM, 128)
PUBLIC KERNEL(DeltaNetUpdate)(
device const T* in_proj,
- device const T* a_log,
- device const T* dt_bias,
- device const T* norm_weight,
- device T* state,
+ device const float* a_log,
+ device const float* dt_bias,
+ device const float* norm_weight,
+ device float* state,
device T* out,
constant const uint& num_v_heads,
constant const uint& num_k_heads,
@@ -31,124 +29,113 @@ PUBLIC KERNEL(DeltaNetUpdate)(
constant const uint& key_dim,
constant const uint& value_dim,
constant const float& norm_epsilon,
- threadgroup float shared_qk[512 /* UPDATE_THREADS */ * 2],
+ threadgroup float shared_o[UPDATE_MAX_HEAD_V_DIM],
threadgroup float shared_scratch[METAL_SIMD_SIZE],
const ThreadContext thread_context,
const uint hv_idx GROUPS(num_v_heads),
const uint tid THREADS(UPDATE_THREADS)
) {
static_assert(
- HEAD_K_DIM % UPDATE_K_SPLIT == 0,
- "HEAD_K_DIM must be a multiple of UPDATE_K_SPLIT"
+ HEAD_K_DIM % METAL_SIMD_SIZE == 0,
+ "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE"
);
- constexpr uint bk = HEAD_K_DIM / UPDATE_K_SPLIT;
-
- const uint vi = tid / UPDATE_K_SPLIT;
- const uint ki_part = tid % UPDATE_K_SPLIT;
- const bool active = (vi < head_v_dim);
+ constexpr uint ELEMS =
+ HEAD_K_DIM / METAL_SIMD_SIZE; // Dk per lane (128/32 = 4)
+ constexpr uint NUM_SG =
+ UPDATE_THREADS / METAL_SIMD_SIZE; // simd groups / tg (16)
- threadgroup float* shared_q = shared_qk;
- threadgroup float* shared_k = shared_qk + UPDATE_THREADS;
+ const uint lane = tid % METAL_SIMD_SIZE;
+ const uint sg = tid / METAL_SIMD_SIZE;
const uint conv_dim = 2 * key_dim + value_dim;
const uint groups_per_head = num_v_heads / num_k_heads;
const uint hk = hv_idx / groups_per_head;
- if (tid < HEAD_K_DIM) {
- shared_q[tid] = float(in_proj[hk * HEAD_K_DIM + tid]);
- shared_k[tid] = float(in_proj[key_dim + hk * HEAD_K_DIM + tid]);
+ // Load + normalize q/k; each simd group recomputes the norm to skip a
+ // barrier.
+ float q[ELEMS];
+ float k[ELEMS];
+ float q_sq = 0.0f;
+ float k_sq = 0.0f;
+ for (uint i = 0; i < ELEMS; ++i) {
+ const uint dk = lane + METAL_SIMD_SIZE * i;
+ q[i] = float(in_proj[hk * HEAD_K_DIM + dk]);
+ k[i] = float(in_proj[key_dim + hk * HEAD_K_DIM + dk]);
+ q_sq += q[i] * q[i];
+ k_sq += k[i] * k[i];
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- float q_partial = (tid < HEAD_K_DIM) ? shared_q[tid] * shared_q[tid] : 0.0f;
- float q_norm_sq =
- threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(
- q_partial,
- shared_scratch,
- thread_context
- );
- float q_inv_norm = rsqrt(q_norm_sq + 1e-6f);
-
- float k_partial = (tid < HEAD_K_DIM) ? shared_k[tid] * shared_k[tid] : 0.0f;
- float k_norm_sq =
- threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(
- k_partial,
- shared_scratch,
- thread_context
- );
- float k_inv_norm = rsqrt(k_norm_sq + 1e-6f);
-
- float q_scale = rsqrt(float(HEAD_K_DIM));
- if (tid < HEAD_K_DIM) {
- shared_q[tid] *= q_inv_norm * q_scale;
- shared_k[tid] *= k_inv_norm;
+ const float q_inv_norm = rsqrt(simd_sum(q_sq) + 1e-6f);
+ const float k_inv_norm = rsqrt(simd_sum(k_sq) + 1e-6f);
+ const float q_scale = rsqrt(float(HEAD_K_DIM));
+ float kq_partial = 0.0f;
+ for (uint i = 0; i < ELEMS; ++i) {
+ q[i] *= q_inv_norm * q_scale;
+ k[i] *= k_inv_norm;
+ kq_partial += q[i] * k[i];
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- float kq_p = (tid < HEAD_K_DIM) ? shared_k[tid] * shared_q[tid] : 0.0f;
- float kq_dot =
- threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(
- kq_p,
- shared_scratch,
- thread_context
- );
-
- float beta_raw = float(in_proj[conv_dim + value_dim + hv_idx]);
- float beta = 1.0f / (1.0f + fast::exp(-beta_raw));
- float a_raw = float(in_proj[conv_dim + value_dim + num_v_heads + hv_idx]);
- float sp = activate_softplus(a_raw + float(dt_bias[hv_idx]));
- float decay = fast::exp(-fast::exp(float(a_log[hv_idx])) * sp);
-
- float v_i =
- active ? float(in_proj[2 * key_dim + hv_idx * head_v_dim + vi]) : 0.0f;
- float z_i =
- active ? float(in_proj[conv_dim + hv_idx * head_v_dim + vi]) : 0.0f;
-
- // State read — each thread handles bk rows
- // State layout: [Hv, Dv, Dk]
- float sq_partial = 0.0f;
- float sk_partial = 0.0f;
- const uint state_row_offset =
- hv_idx * head_v_dim * HEAD_K_DIM + vi * HEAD_K_DIM;
- const uint k_base = ki_part * bk;
- if (active) {
- for (uint j = 0; j < bk; ++j) {
- float s = float(state[state_row_offset + k_base + j]);
- sq_partial += s * shared_q[k_base + j];
- sk_partial += s * shared_k[k_base + j];
+ const float kq_dot = simd_sum(kq_partial);
+
+ // beta / decay (scalar per head)
+ const float beta_raw = float(in_proj[conv_dim + value_dim + hv_idx]);
+ const float beta = 1.0f / (1.0f + fast::exp(-beta_raw));
+ const float a_raw =
+ float(in_proj[conv_dim + value_dim + num_v_heads + hv_idx]);
+ const float sp = activate_softplus(a_raw + float(dt_bias[hv_idx]));
+ const float decay = fast::exp(-fast::exp(float(a_log[hv_idx])) * sp);
+
+ // Delta rule over the dv owned by this simd group. State is [Hv, Dv, Dk].
+ for (uint dv = sg; dv < head_v_dim; dv += NUM_SG) {
+ const uint state_row = (hv_idx * head_v_dim + dv) * HEAD_K_DIM;
+ const float v_i = float(in_proj[2 * key_dim + hv_idx * head_v_dim + dv]);
+
+ float s[ELEMS];
+ float sq_partial = 0.0f;
+ float sk_partial = 0.0f;
+ for (uint i = 0; i < ELEMS; ++i) {
+ const uint dk = lane + METAL_SIMD_SIZE * i;
+ s[i] = state[state_row + dk];
+ sq_partial += s[i] * q[i];
+ sk_partial += s[i] * k[i];
}
- }
+ const float sq_acc = simd_sum(sq_partial);
+ const float sk_acc = simd_sum(sk_partial);
- // Reduce across k-parts (4 consecutive lanes = one quad group)
- float sq_acc = quad_sum(sq_partial);
- float sk_acc = quad_sum(sk_partial);
+ const float retrieved_i = decay * sk_acc;
+ const float delta_i = beta * (v_i - retrieved_i);
+ const float o_i = decay * sq_acc + delta_i * kq_dot;
- float retrieved_i = decay * sk_acc;
- float delta_i = beta * (v_i - retrieved_i);
- float o_i = decay * sq_acc + delta_i * kq_dot;
+ for (uint i = 0; i < ELEMS; ++i) {
+ const uint dk = lane + METAL_SIMD_SIZE * i;
+ state[state_row + dk] = decay * s[i] + k[i] * delta_i;
+ }
- // State update — each thread updates its k-part
- if (active) {
- for (uint j = 0; j < bk; ++j) {
- uint idx = state_row_offset + k_base + j;
- float s = float(state[idx]);
- state[idx] = static_cast<T>(decay * s + shared_k[k_base + j] * delta_i);
+ if (lane == 0) {
+ shared_o[dv] = o_i;
}
}
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- // RMSNorm + SiLU gate — only ki_part==0 contributes
- float o_sq = (active && ki_part == 0) ? o_i * o_i : 0.0f;
- float o_sumsq =
+ // RMSNorm over all dv, then SiLU gate + write out.
+ float o_sq = 0.0f;
+ for (uint dv = tid; dv < head_v_dim; dv += UPDATE_THREADS) {
+ const float o = shared_o[dv];
+ o_sq += o * o;
+ }
+ const float o_sumsq =
threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(
o_sq,
shared_scratch,
thread_context
);
- float inv_rms = rsqrt(o_sumsq / float(head_v_dim) + norm_epsilon);
-
- if (active && ki_part == 0) {
- float nw = float(norm_weight[vi]);
- float z_silu = activate_silu(z_i);
- out[hv_idx * head_v_dim + vi] = static_cast<T>(o_i * inv_rms * nw * z_silu);
+ const float inv_rms = rsqrt(o_sumsq / float(head_v_dim) + norm_epsilon);
+
+ for (uint dv = sg; dv < head_v_dim; dv += NUM_SG) {
+ if (lane == 0) {
+ const float nw = float(norm_weight[dv]);
+ const float z_i = float(in_proj[conv_dim + hv_idx * head_v_dim + dv]);
+ const float z_silu = activate_silu(z_i);
+ out[hv_idx * head_v_dim + dv] =
+ static_cast<T>(shared_o[dv] * inv_rms * nw * z_silu);
+ }
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/ssm/conv1d.metal b/crates/backend-uzu/src/backends/metal/kernel/ssm/conv1d.metal
index e6100b52e..e84becbdd 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/ssm/conv1d.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/ssm/conv1d.metal
@@ -4,12 +4,13 @@
using namespace uzu::activation_type;
-template <typename T>
-VARIANTS(T, float, half, bfloat)
+template <typename StateT, typename InputT>
+VARIANTS(StateT, float, bfloat)
+VARIANTS(InputT, float, bfloat)
PUBLIC KERNEL(Conv1dPack)(
- device const T* state_in,
- device const T* x,
- device T* padded,
+ device const StateT* state_in,
+ device const InputT* x,
+ device StateT* padded,
constant const uint& state_stride,
constant const uint& row_stride,
constant const uint& suffix_len,
@@ -24,7 +25,7 @@ PUBLIC KERNEL(Conv1dPack)(
} else {
const uint token = row_idx - state_stride;
const uint x_index = token * row_stride + channel_idx;
- padded[padded_index] = x[x_index];
+ padded[padded_index] = static_cast<StateT>(x[x_index]);
}
}
@@ -165,4 +166,4 @@ PUBLIC KERNEL(Conv1dScan)(
const float sample = float(padded[padded_index]);
state_out[state_offset + tap] = static_cast<T>(sample);
}
-}
\ No newline at end of file
+}
diff --git a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs
index 89e60f0c2..f2826776c 100644
--- a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs
+++ b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs
@@ -55,7 +55,9 @@ pub struct DeltaNetMixer<B: Backend> {
a_log: Allocation<B>,
dt_bias: Allocation<B>,
norm_weight: Allocation<B>,
- inner_data_type: DataType,
+ // Activation/scratch dtype (model dtype, e.g. bf16). The recurrent ssm_state
+ // and small weights (a_log/dt_bias/norm/conv) stay f32 inside the kernels.
+ outer_data_type: DataType,
}
pub struct DeltaNetArguments<'a, B: Backend> {
@@ -109,7 +111,7 @@ impl<B: Backend> DeltaNetMixer<B> {
context,
outer_data_type,
outer_data_type,
- inner_data_type,
+ outer_data_type,
&parameter_tree.subtree("in_proj")?,
)?;
@@ -119,7 +121,7 @@ impl<B: Backend> DeltaNetMixer<B> {
false,
context,
outer_data_type,
- inner_data_type,
+ outer_data_type,
outer_data_type,
&parameter_tree.subtree("out_proj")?,
)?;
@@ -141,22 +143,22 @@ impl<B: Backend> DeltaNetMixer<B> {
.read_allocation()?;
// Create kernels
- let conv_update = <B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, inner_data_type, has_bias)
+ let conv_update = <B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, outer_data_type, has_bias)
.map_err(DeltaNetMixerError::BackendError)?;
let delta_net_update =
- <B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, inner_data_type, config.head_dim as u32)
+ <B::Kernels as Kernels>::DeltaNetUpdateKernel::new(context, outer_data_type, config.head_dim as u32)
.map_err(DeltaNetMixerError::BackendError)?;
- let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, inner_data_type)
+ let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, inner_data_type, outer_data_type)
.map_err(DeltaNetMixerError::BackendError)?;
- let conv_scan = <B::Kernels as Kernels>::DeltaNetConvScanKernel::new(context, inner_data_type, has_bias)
+ let conv_scan = <B::Kernels as Kernels>::DeltaNetConvScanKernel::new(context, outer_data_type, has_bias)
.map_err(DeltaNetMixerError::BackendError)?;
let prefill_prep =
- <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(context, inner_data_type, config.head_dim as u32)
+ <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(context, outer_data_type, config.head_dim as u32)
.map_err(DeltaNetMixerError::BackendError)?;
let delta_net_prefill =
- <B::Kernels as Kernels>::DeltaNetPrefillKernel::new(context, inner_data_type, config.head_dim as u32)
+ <B::Kernels as Kernels>::DeltaNetPrefillKernel::new(context, outer_data_type, config.head_dim as u32)
.map_err(DeltaNetMixerError::BackendError)?;
- let norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, inner_data_type)
+ let norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, outer_data_type)
.map_err(DeltaNetMixerError::BackendError)?;
Ok((
@@ -184,7 +186,7 @@ impl<B: Backend> DeltaNetMixer<B> {
a_log,
dt_bias,
norm_weight,
- inner_data_type,
+ outer_data_type,
},
in_projection_input_hadamard_factors,
))
@@ -220,8 +222,8 @@ impl<B: Backend> DeltaNetMixer<B> {
let state_stride = kernel_size - 1;
let conv_dim = self.conv_dim;
let total_proj_dim = self.total_proj_dim;
- let mut padded = encoder
- .allocate_scratch(size_for_shape(&[suffix_length + state_stride, total_proj_dim], self.inner_data_type))?;
+ let mut padded =
+ encoder.allocate_scratch(size_for_shape(&[suffix_length + state_stride, total_proj_dim], DataType::F32))?;
self.conv_pack.encode(
&layer.conv_state,
@@ -259,7 +261,7 @@ impl<B: Backend> DeltaNetMixer<B> {
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
let mut out =
- encoder.allocate_scratch(size_for_shape(&[active_row_count, self.value_dim], self.inner_data_type))?;
+ encoder.allocate_scratch(size_for_shape(&[active_row_count, self.value_dim], self.outer_data_type))?;
self.delta_net_update.encode(
in_proj,
&self.a_log,
@@ -285,7 +287,7 @@ impl<B: Backend> DeltaNetMixer<B> {
suffix_length: usize,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let num_dv_groups = self.value_head_dim.div_ceil(8) as u32;
+ let num_dv_groups = self.value_head_dim.div_ceil(16) as u32;
let mut prep_q_norm =
encoder.allocate_scratch(size_for_shape(&[suffix_length * self.key_dim], DataType::F32))?;
let mut prep_k_norm =
@@ -312,7 +314,7 @@ impl<B: Backend> DeltaNetMixer<B> {
);
let mut out =
- encoder.allocate_scratch(size_for_shape(&[suffix_length, self.value_dim], self.inner_data_type))?;
+ encoder.allocate_scratch(size_for_shape(&[suffix_length, self.value_dim], self.outer_data_type))?;
self.delta_net_prefill.encode(
&prep_q_norm,
&prep_k_norm,
diff --git a/crates/backend-uzu/src/encodable_block/mamba_mixer/mod.rs b/crates/backend-uzu/src/encodable_block/mamba_mixer/mod.rs
index 417995a52..5497d7b4d 100644
--- a/crates/backend-uzu/src/encodable_block/mamba_mixer/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mamba_mixer/mod.rs
@@ -151,7 +151,7 @@ impl<B: Backend> MambaMixer<B> {
true,
)
.map_err(MambaMixerError::BackendError)?;
- let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, inner_data_type)
+ let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, inner_data_type, inner_data_type)
.map_err(MambaMixerError::BackendError)?;
let ssd_prefill = SSDPrefillBlock::new(context, inner_data_type).map_err(MambaMixerError::BackendError)?;
let ssd_update = <B::Kernels as Kernels>::SSDUpdateKernel::new(context, inner_data_type, true)
diff --git a/crates/backend-uzu/src/forward_pass/cache_layers.rs b/crates/backend-uzu/src/forward_pass/cache_layers.rs
index 044f12a03..f7770c98f 100644
--- a/crates/backend-uzu/src/forward_pass/cache_layers.rs
+++ b/crates/backend-uzu/src/forward_pass/cache_layers.rs
@@ -222,11 +222,12 @@ impl<B: Backend> CacheLayers<B> {
})
},
AnyTokenMixerConfig::DeltaNetConfig(c) => {
- let data_type = DataType::F32;
+ let conv_data_type = DataType::F32;
+ let ssm_data_type = DataType::F32;
let conv_shape = [c.conv_dim(), c.kernel_size.saturating_sub(1)];
let ssm_shape = [c.num_heads, c.value_head_dim, c.head_dim];
- let conv_bytes = size_for_shape(&conv_shape, data_type);
- let ssm_bytes = size_for_shape(&ssm_shape, data_type);
+ let conv_bytes = size_for_shape(&conv_shape, conv_data_type);
+ let ssm_bytes = size_for_shape(&ssm_shape, ssm_data_type);
CacheLayer::DeltaNet(DeltaNetLayer {
conv_state: context
@@ -237,7 +238,7 @@ impl<B: Backend> CacheLayers<B> {
.create_allocation(ssm_bytes, AllocationType::Global)
.expect("Failed to create delta net ssm allocation"),
ssm_shape,
- data_type,
+ data_type: ssm_data_type,
})
},
};
diff --git a/crates/backend-uzu/tests/unit/kernel/delta_net_test.rs b/crates/backend-uzu/tests/unit/kernel/delta_net_test.rs
index 24275dc30..8338bcda0 100644
--- a/crates/backend-uzu/tests/unit/kernel/delta_net_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/delta_net_test.rs
@@ -1,7 +1,7 @@
#![cfg(metal_backend)]
use backend_uzu::{
- array::ArrayContextExt,
+ array::{ArrayContextExt, ArrayElement},
backends::{
common::{
Backend, Context, Encoder, Kernels,
@@ -15,6 +15,7 @@ use backend_uzu::{
},
data_type::DataType,
};
+use half::bf16;
use crate::uzu_test;
@@ -153,13 +154,15 @@ fn test_delta_net_conv_scan() {
let w: Vec<f32> = (0..conv_dim * kernel_size).map(|i| ((i % 11) as f32) * 0.05 - 0.2).collect();
let b: Vec<f32> = (0..conv_dim).map(|i| ((i % 5) as f32) * 0.01).collect();
let init_state: Vec<f32> = (0..conv_dim * tap_count).map(|i| ((i % 13) as f32) * 0.02 - 0.1).collect();
- let in_proj: Vec<f32> = (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect();
+ let in_proj: Vec<bf16> =
+ (0..suffix_len * total_proj_dim).map(|i| bf16::from_f32(((i % 37) as f32) * 0.02 - 0.3)).collect();
+ let in_proj_f32: Vec<f32> = in_proj.iter().copied().map(f32::from).collect();
// Reference: decode conv token-by-token
let mut ref_state = init_state.clone();
let mut ref_outputs = vec![0.0f32; suffix_len * conv_dim];
for t in 0..suffix_len {
- let token_in: Vec<f32> = in_proj[t * total_proj_dim..t * total_proj_dim + conv_dim].to_vec();
+ let token_in: Vec<f32> = in_proj_f32[t * total_proj_dim..t * total_proj_dim + conv_dim].to_vec();
let (out, new_state) = run_conv_update::<Cpu>(
&token_in,
&w,
@@ -172,6 +175,7 @@ fn test_delta_net_conv_scan() {
ref_state = new_state;
ref_outputs[t * conv_dim..(t + 1) * conv_dim].copy_from_slice(&out);
}
+ let ref_outputs: Vec<f32> = ref_outputs.iter().copied().map(|value| f32::from(bf16::from_f32(value))).collect();
// Test: Conv1dPack + DeltaNetConvScan on Metal
let context = <Metal as Backend>::Context::new().expect("context");
@@ -185,9 +189,10 @@ fn test_delta_net_conv_scan() {
let mut state_out_array = context.create_array_zeros(&[conv_dim * tap_count], DataType::F32).into_allocation();
let pack_kernel =
- <<Metal as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, DataType::F32).expect("pack");
+ <<Metal as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, DataType::F32, DataType::BF16)
+ .expect("pack");
let scan_kernel =
- <<Metal as Backend>::Kernels as Kernels>::DeltaNetConvScanKernel::new(&context, DataType::F32, true)
+ <<Metal as Backend>::Kernels as Kernels>::DeltaNetConvScanKernel::new(&context, DataType::BF16, true)
.expect("scan");
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
@@ -217,7 +222,8 @@ fn test_delta_net_conv_scan() {
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let in_proj_result: Vec<f32> = crate::common::helpers::allocation_to_vec(&in_proj_array);
+ let in_proj_result: Vec<bf16> = crate::common::helpers::allocation_to_vec(&in_proj_array);
+ let in_proj_result: Vec<f32> = in_proj_result.into_iter().map(f32::from).collect();
let mut scan_outputs = vec![0.0f32; suffix_len * conv_dim];
for t in 0..suffix_len {
scan_outputs[t * conv_dim..(t + 1) * conv_dim]
@@ -288,8 +294,8 @@ fn test_delta_net_update_qwen35_shapes() {
// DeltaNetPrefill + NormGate
-fn run_prefill_with_norm_gate(
- in_proj: &[f32],
+fn run_prefill_with_norm_gate_typed<T: ArrayElement>(
+ in_proj: &[T],
a_log: &[f32],
dt_bias: &[f32],
norm_weight: &[f32],
@@ -304,7 +310,7 @@ fn run_prefill_with_norm_gate(
let value_dim = num_v_heads * head_v_dim;
let conv_dim = 2 * key_dim + value_dim;
let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;
- let num_dv_groups = (head_v_dim as u32).div_ceil(8);
+ let num_dv_groups = head_v_dim.div_ceil(16) as u32;
let context = <Metal as Backend>::Context::new().expect("context");
let in_proj_array = context.create_array_from(&[in_proj.len()], in_proj);
@@ -312,7 +318,7 @@ fn run_prefill_with_norm_gate(
let dt_bias_array = context.create_array_from(&[dt_bias.len()], dt_bias);
let norm_weight_array = context.create_array_from(&[norm_weight.len()], norm_weight);
let mut state_array = context.create_array_from(&[state.len()], state).into_allocation();
- let mut out_array = context.create_array_zeros(&[suffix_len * value_dim], DataType::F32).into_allocation();
+ let mut out_array = context.create_array_zeros(&[suffix_len * value_dim], T::data_type()).into_allocation();
let mut q_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();
let mut k_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();
@@ -321,18 +327,18 @@ fn run_prefill_with_norm_gate(
let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(
&context,
- DataType::F32,
+ T::data_type(),
head_k_dim as u32,
)
.unwrap();
let prefill_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillKernel::new(
&context,
- DataType::F32,
+ T::data_type(),
head_k_dim as u32,
)
.unwrap();
let norm_k =
- <<Metal as Backend>::Kernels as Kernels>::DeltaNetNormGateKernel::new(&context, DataType::F32).unwrap();
+ <<Metal as Backend>::Kernels as Kernels>::DeltaNetNormGateKernel::new(&context, T::data_type()).unwrap();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
prep_k.encode(
@@ -382,15 +388,20 @@ fn run_prefill_with_norm_gate(
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- (crate::common::helpers::allocation_to_vec(&out_array), crate::common::helpers::allocation_to_vec(&state_array))
+ let out: Vec<T> = crate::common::helpers::allocation_to_vec(&out_array);
+ let out = out.into_iter().map(|value| value.to_f32().expect("output to f32")).collect();
+ let state = crate::common::helpers::allocation_to_vec(&state_array);
+ (out, state)
}
-fn test_prefill_norm_gate_impl(
+fn test_prefill_norm_gate_impl<T: ArrayElement>(
num_v_heads: usize,
num_k_heads: usize,
head_k_dim: usize,
head_v_dim: usize,
suffix_len: usize,
+ output_atol: f32,
+ output_rtol: f32,
label: &str,
) {
let key_dim = num_k_heads * head_k_dim;
@@ -400,6 +411,16 @@ fn test_prefill_norm_gate_impl(
let state_size = num_v_heads * head_k_dim * head_v_dim;
let in_proj: Vec<f32> = (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect();
+ let in_proj_typed: Vec<T> = in_proj
+ .iter()
+ .copied()
+ .map(|value| <T as num_traits::NumCast>::from(value).expect("input to activation dtype"))
+ .collect();
+ let reference_in_proj: Vec<f32> = in_proj_typed
+ .iter()
+ .copied()
+ .map(|value| <f32 as num_traits::NumCast>::from(value).expect("activation dtype to f32"))
+ .collect();
let a_log: Vec<f32> = (0..num_v_heads).map(|i| -1.5 + (i as f32) * 0.05).collect();
let dt_bias: Vec<f32> = (0..num_v_heads).map(|i| 0.3 + (i as f32) * 0.02).collect();
let norm_weight: Vec<f32> = (0..head_v_dim).map(|i| 0.9 + (i as f32) * 0.001).collect();
@@ -408,8 +429,9 @@ fn test_prefill_norm_gate_impl(
// Reference: fused decode kernel token-by-token
let mut ref_state = state.clone();
let mut ref_outputs = vec![0.0f32; suffix_len * value_dim];
- for t in 0..suffix_len {
- let token_in: Vec<f32> = in_proj[t * total_proj_dim..(t + 1) * total_proj_dim].to_vec();
+ for token_index in 0..suffix_len {
+ let token_in: Vec<f32> =
+ reference_in_proj[token_index * total_proj_dim..(token_index + 1) * total_proj_dim].to_vec();
let (out, new_state) = run_delta_net_update::<Cpu>(
&token_in,
&a_log,
@@ -424,11 +446,11 @@ fn test_prefill_norm_gate_impl(
value_dim as u32,
);
ref_state = new_state;
- ref_outputs[t * value_dim..(t + 1) * value_dim].copy_from_slice(&out);
+ ref_outputs[token_index * value_dim..(token_index + 1) * value_dim].copy_from_slice(&out);
}
- let (gpu_out, gpu_state) = run_prefill_with_norm_gate(
- &in_proj,
+ let (gpu_out, gpu_state) = run_prefill_with_norm_gate_typed(
+ &in_proj_typed,
&a_log,
&dt_bias,
&norm_weight,
@@ -440,13 +462,18 @@ fn test_prefill_norm_gate_impl(
suffix_len,
);
- assert_close(&ref_outputs, &gpu_out, 1e-3, 1e-2, &format!("{label} output"));
+ assert_close(&ref_outputs, &gpu_out, output_atol, output_rtol, &format!("{label} output"));
assert_close(&ref_state, &gpu_state, 1e-3, 1e-2, &format!("{label} state"));
}
#[uzu_test]
fn test_delta_net_prefill_qwen35_shapes() {
- test_prefill_norm_gate_impl(48, 16, 128, 128, 32, "Prefill+NormGate Qwen3.5");
+ test_prefill_norm_gate_impl::<f32>(48, 16, 128, 128, 32, 1e-3, 1e-2, "Prefill+NormGate Qwen3.5");
+}
+
+#[uzu_test]
+fn test_delta_net_prefill_qwen35_shapes_bf16() {
+ test_prefill_norm_gate_impl::<bf16>(48, 16, 128, 128, 32, 2e-2, 5e-2, "Prefill+NormGate Qwen3.5 BF16");
}
#[uzu_test]
@@ -586,7 +613,7 @@ fn bench_delta_net_prefill() {
let mut beta_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();
let mut decay_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();
- let num_dv_groups = (head_v_dim as u32).div_ceil(8);
+ let num_dv_groups = head_v_dim.div_ceil(16) as u32;
let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(
&context,
diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs
index 70b82cb26..183732e73 100644
--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs
@@ -8,7 +8,7 @@ use backend_uzu::{
},
data_type::DataType,
};
-use half::{bf16, f16};
+use half::bf16;
use num_traits::Float;
use crate::{common::assert::assert_eq_float, uzu_test};
@@ -46,7 +46,7 @@ fn get_input<T: ArrayElement + Float>(
fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
- let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, T::data_type())
+ let kernel = <<B as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, T::data_type(), T::data_type())
.expect("Failed to create Conv1dPackKernel");
let state_size = input.state_in.len();
@@ -159,27 +159,6 @@ fn test_channels_lt_row_stride_f32() {
test_channels_lt_row_stride::<f32>();
}
-// f16
-#[uzu_test]
-fn test_basic_f16() {
- test_basic::<f16>();
-}
-
-#[uzu_test]
-fn test_single_token_f16() {
- test_single_token::<f16>();
-}
-
-#[uzu_test]
-fn test_many_tokens_f16() {
- test_many_tokens::<f16>();
-}
-
-#[uzu_test]
-fn test_large_f16() {
- test_large::<f16>();
-}
-
// bf16
#[uzu_test]
fn test_basic_bf16() {