A high-performance inference engine for AI models
speedup delta net prefil (#489)
eugene3e194a193eff
diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metalindex 51d988d62..aebfcdfe6 100644--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal@@ -60,17 +60,19 @@ PUBLIC KERNEL(DeltaNetPrefill)(device const float* k_ptr = k_norm + hk * HEAD_K_DIM + dk_base;device const float* beta_ptr = beta_buf + hv_idx;device const float* decay_ptr = decay_buf + hv_idx;- device const T* v_ptr = in_proj + 2 * key_dim + hv_idx * head_v_dim;- device T* out_ptr = out + hv_idx * head_v_dim;+ device const T* v_row = in_proj + 2 * key_dim + hv_idx * head_v_dim + dv_idx;+ device T* out_row = out + hv_idx * head_v_dim + dv_idx;+ // State layout: [Hv, Dv, Dk], contiguous along Dk (HEAD_K_DIM floats/row).+ // Unrolled by hand: METAL_PRAGMA_UNROLL is ignored under metal4.0 -O2, so a+ // loop over s[i] keeps the array in an alloca and reloads it every token.+ device const float* s_in =+ state + (hv_idx * head_v_dim + dv_idx) * HEAD_K_DIM + dk_base;float4 s[DV_PER_SIMDGROUP];- METAL_PRAGMA_UNROLL- for (uint i = 0; i < DV_PER_SIMDGROUP; ++i) {- uint dv = dv_idx + i;- s[i] = *reinterpret_cast<device const float4*>(- state + (hv_idx * head_v_dim + dv) * HEAD_K_DIM + dk_base- );- }+ s[0] = *reinterpret_cast<device const float4*>(s_in + 0 * HEAD_K_DIM);+ s[1] = *reinterpret_cast<device const float4*>(s_in + 1 * HEAD_K_DIM);+ s[2] = *reinterpret_cast<device const float4*>(s_in + 2 * HEAD_K_DIM);+ s[3] = *reinterpret_cast<device const float4*>(s_in + 3 * HEAD_K_DIM);for (uint token = 0; token < suffix_len; ++token) {float decay = *decay_ptr;@@ -78,43 +80,47 @@ PUBLIC KERNEL(DeltaNetPrefill)(float4 k = *reinterpret_cast<device const float4*>(k_ptr);float4 q = *reinterpret_cast<device const float4*>(q_ptr);- METAL_PRAGMA_UNROLL- for (uint i = 0; i < DV_PER_SIMDGROUP; ++i)- s[i] *= decay;+ s[0] *= decay;+ s[1] *= decay;+ s[2] *= decay;+ s[3] *= decay;float4 kv_mem =float4(dot(s[0], k), dot(s[1], k), dot(s[2], k), dot(s[3], k));kv_mem = simd_sum(kv_mem);- float4 o;- METAL_PRAGMA_UNROLL- for (uint i = 0; i < DV_PER_SIMDGROUP; ++i) {- uint dv = dv_idx + i;- float v_val = float(v_ptr[token * total_proj_dim + dv]);- float delta = beta * (v_val - kv_mem[i]);- s[i] += k * delta;- o[i] = dot(s[i], q);- }+ float4 v_val = float4(+ float(v_row[0]),+ float(v_row[1]),+ float(v_row[2]),+ float(v_row[3])+ );+ float4 delta = beta * (v_val - kv_mem);+ s[0] += k * delta[0];+ s[1] += k * delta[1];+ s[2] += k * delta[2];+ s[3] += k * delta[3];+ float4 o = float4(dot(s[0], q), dot(s[1], q), dot(s[2], q), dot(s[3], q));o = simd_sum(o);if (lane == 0) {- METAL_PRAGMA_UNROLL- for (uint i = 0; i < DV_PER_SIMDGROUP; ++i) {- uint dv = dv_idx + i;- out_ptr[token * value_dim + dv] = static_cast<T>(o[i]);- }+ out_row[0] = static_cast<T>(o[0]);+ out_row[1] = static_cast<T>(o[1]);+ out_row[2] = static_cast<T>(o[2]);+ out_row[3] = static_cast<T>(o[3]);}q_ptr += key_dim;k_ptr += key_dim;beta_ptr += num_v_heads;decay_ptr += num_v_heads;+ v_row += total_proj_dim;+ out_row += value_dim;}- METAL_PRAGMA_UNROLL- for (uint i = 0; i < DV_PER_SIMDGROUP; ++i) {- uint dv = dv_idx + i;- *reinterpret_cast<device float4*>(- state + (hv_idx * head_v_dim + dv) * HEAD_K_DIM + dk_base- ) = s[i];- }+ device float* s_out =+ state + (hv_idx * head_v_dim + dv_idx) * HEAD_K_DIM + dk_base;+ *reinterpret_cast<device float4*>(s_out + 0 * HEAD_K_DIM) = s[0];+ *reinterpret_cast<device float4*>(s_out + 1 * HEAD_K_DIM) = s[1];+ *reinterpret_cast<device float4*>(s_out + 2 * HEAD_K_DIM) = s[2];+ *reinterpret_cast<device float4*>(s_out + 3 * HEAD_K_DIM) = s[3];}