A high-performance inference engine for AI models
manual unrolling + deleted unused code
LuckyIYI05b66ff64ab9
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex 83ab64e71..70fcca455 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -37,7 +37,7 @@ pub(crate) use ssm::MambaMixerEncodable;pub use ssm::{Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,SSDPrefillArguments, SSDPrefillKernel, SSDPrefillMode, SSDUpdateArguments,- SSDUpdateKernel, SSMKernelError, SSMUpdateArguments, SSMUpdateKernel,+ SSDUpdateKernel, SSMKernelError,SplitInProjArguments, SplitInProjKernel,};pub use tensor_add_bias::TensorAddBias;diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metalindex 46b5150b4..dc90432d4 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal@@ -1189,21 +1189,45 @@ void qmv_fast_impl(for (int k = 0; k < in_vec_size; k += block_size) {U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);- for (int row = 0; row < results_per_simdgroup; row++) {- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);- const device T* sl = scales + row * in_vec_size_g;- U s = sl[0];-+ {+ auto wl0 = (const device uint8_t*)(ws);+ auto wl1 = (const device uint8_t*)(ws + in_vec_size_w);+ auto wl2 = (const device uint8_t*)(ws + 2 * in_vec_size_w);+ auto wl3 = (const device uint8_t*)(ws + 3 * in_vec_size_w);++ U s0 = static_cast<U>(scales[0]);+ U s1 = static_cast<U>(scales[in_vec_size_g]);+ U s2 = static_cast<U>(scales[2 * in_vec_size_g]);+ U s3 = static_cast<U>(scales[3 * in_vec_size_g]);+if (kUseMlxQuant) {- const device T* bl = biases + row * in_vec_size_g;- U b = bl[0];- result[row] += qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);+ U b0 = static_cast<U>(biases[0]);+ U b1 = static_cast<U>(biases[in_vec_size_g]);+ U b2 = static_cast<U>(biases[2 * in_vec_size_g]);+ U b3 = static_cast<U>(biases[3 * in_vec_size_g]);+ result[0] += qdot<U, values_per_thread, bits>(wl0, x_thread, s0, b0, sum);+ result[1] += qdot<U, values_per_thread, bits>(wl1, x_thread, s1, b1, sum);+ result[2] += qdot<U, values_per_thread, bits>(wl2, x_thread, s2, b2, sum);+ result[3] += qdot<U, values_per_thread, bits>(wl3, x_thread, s3, b3, sum);} else {- const device uint8_t* zl = zps + row * zp_stride;- uint8_t zp_byte = *zl;- U zp = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte >> 4) : (zp_byte & 0x0F));- if (bits == 8) zp = static_cast<U>(zp_byte);- result[row] += qdot_zero_point<U, values_per_thread, bits>(wl, x_thread, s, zp);+ uint8_t zp_byte0 = zps[0];+ uint8_t zp_byte1 = zps[zp_stride];+ uint8_t zp_byte2 = zps[2 * zp_stride];+ uint8_t zp_byte3 = zps[3 * zp_stride];+ U zp0 = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte0 >> 4) : (zp_byte0 & 0x0F));+ U zp1 = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte1 >> 4) : (zp_byte1 & 0x0F));+ U zp2 = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte2 >> 4) : (zp_byte2 & 0x0F));+ U zp3 = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte3 >> 4) : (zp_byte3 & 0x0F));+ if (bits == 8) {+ zp0 = static_cast<U>(zp_byte0);+ zp1 = static_cast<U>(zp_byte1);+ zp2 = static_cast<U>(zp_byte2);+ zp3 = static_cast<U>(zp_byte3);+ }+ result[0] += qdot_zero_point<U, values_per_thread, bits>(wl0, x_thread, s0, zp0);+ result[1] += qdot_zero_point<U, values_per_thread, bits>(wl1, x_thread, s1, zp1);+ result[2] += qdot_zero_point<U, values_per_thread, bits>(wl2, x_thread, s2, zp2);+ result[3] += qdot_zero_point<U, values_per_thread, bits>(wl3, x_thread, s3, zp3);}}diff --git a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs b/crates/uzu/src/backends/metal/kernel/ssm/mod.rsindex 8fd1ce3e3..9b23b0f10 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/ssm/mod.rs@@ -17,7 +17,6 @@ pub mod mamba;pub mod split_inproj;pub mod ssd_prefill;pub mod ssd_update;-pub mod ssm_update;pub use conv1d_scan::{Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,@@ -26,4 +25,3 @@ pub(crate) use mamba::MambaMixerEncodable;pub use split_inproj::{SplitInProjArguments, SplitInProjKernel};pub use ssd_prefill::{SSDPrefillArguments, SSDPrefillKernel, SSDPrefillMode};pub use ssd_update::{SSDUpdateArguments, SSDUpdateKernel};-pub use ssm_update::{SSMUpdateArguments, SSMUpdateKernel};diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssm_update.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssm_update.metaldeleted file mode 100644index b88fb4af1..000000000--- a/crates/uzu/src/backends/metal/kernel/ssm/ssm_update.metal+++ /dev/null@@ -1,107 +0,0 @@-#include <metal_stdlib>-#include "../definitions.metal"--using namespace metal;--struct SILU {- template <typename T>- T operator()(T x) {- float xf = float(x);- float y = 1.0f / (1.0f + fast::exp(-fabs(xf)));- float out = (xf < 0.0f) ? (1.0f - y) * xf : y * xf;- return static_cast<T>(out);- }-};--struct Softplus {- template <typename T>- T operator()(T x) {- float xf = float(x);- float y = log(1.0f + fast::exp(xf));- return static_cast<T>((xf > 20.0f) ? xf : y);- }-};--struct Exp {- template <typename T>- T operator()(T x) {- float xf = float(x);- return static_cast<T>(fast::exp(xf));- };-};--// Mamba SSM naming:-// X – SILU-activated hidden stream after the conv block-// Z – gate activations that modulate the residual output-// A – diagonal state transition spectrum (constant per layer)-// B – input projection that injects X into the state-// C – output projection from state to residual-// D – per-head skip/identity weight--template <typename T>-kernel void ssm_update_kernel(- device const T* x [[ buffer(0) ]], // (b, h) hidden stream X- device const T* dt [[ buffer(1) ]], // (b, h) delta / step size- device const T* A [[ buffer(2) ]], // (n) spectrum A- device const T* B [[ buffer(3) ]], // (b, n) input projection B- device const T* C [[ buffer(4) ]], // (b, n) output projection C- device const T* D [[ buffer(5) ]], // (h) skip weights D- device const T* z [[ buffer(6) ]], // (b, h) gate Z- device const T* state [[ buffer(7) ]], // (b, h, n)- device T* y [[ buffer(8) ]], // (b, h)- device T* next_state [[ buffer(9) ]], // (b, h, n)- uint3 grid_idx [[ thread_position_in_grid ]],- uint3 grid_size [[ threads_per_grid ]]-) {- const int state_size = 16; // matches reference- const int channel_size = grid_size.y;-- const int batch_idx = grid_idx.x;- const int channel_idx = grid_idx.y;-- const int cb_start_idx = batch_idx * state_size;- const int x_idx = batch_idx * channel_size + channel_idx;- const int state_start_idx = x_idx * state_size;-- T this_x = x[x_idx];- this_x = SILU{}(this_x);-- T this_z = z[x_idx];- this_z = SILU{}(this_z);-- T delta = Softplus{}(dt[x_idx]);-- T temp = T(0);- #pragma unroll- for (int i = 0; i < state_size; ++i) {- int cb_idx = cb_start_idx + i;- int state_idx = state_start_idx + i;- T this_new_state = state[state_idx] * Exp{}(A[i] * delta) + B[cb_idx] * delta * this_x;- next_state[state_idx] = this_new_state;- temp = temp + this_new_state * C[cb_idx];- }- temp = temp + D[channel_idx] * this_x;- temp = temp * this_z;- y[x_idx] = temp;-}--#define instantiate_ssm_update_kernel(type_name, type) \- template [[host_name("ssm_update_kernel_" #type_name)]] \- kernel void ssm_update_kernel<type>( \- device const type* x [[ buffer(0) ]], \- device const type* dt [[ buffer(1) ]], \- device const type* A [[ buffer(2) ]], \- device const type* B [[ buffer(3) ]], \- device const type* C [[ buffer(4) ]], \- device const type* D [[ buffer(5) ]], \- device const type* z [[ buffer(6) ]], \- device const type* state [[ buffer(7) ]], \- device type* y [[ buffer(8) ]], \- device type* next_state [[ buffer(9) ]], \- uint3 grid_idx [[ thread_position_in_grid ]], \- uint3 grid_size [[ threads_per_grid ]]);--instantiate_ssm_update_kernel(float, float);-instantiate_ssm_update_kernel(bfloat, bfloat);-instantiate_ssm_update_kernel(half, half);-diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssm_update.rs b/crates/uzu/src/backends/metal/kernel/ssm/ssm_update.rsdeleted file mode 100644index c5bd9a337..000000000--- a/crates/uzu/src/backends/metal/kernel/ssm/ssm_update.rs+++ /dev/null@@ -1,84 +0,0 @@-use metal::{- Buffer as MTLBuffer, ComputeCommandEncoderRef,- ComputePipelineState as MTLComputePipelineState, MTLSize,-};--use super::{SSMKernelError, fn_suffix};-use crate::backends::metal::{KernelDataType, MTLContext};--pub struct SSMUpdateKernel {- pipeline: MTLComputePipelineState,-}--pub struct SSMUpdateArguments<'a> {- /// X – Silu-activated hidden stream (per head/channel) coming from the conv.- pub x: &'a MTLBuffer, // buffer(0)- /// Δt – learned step sizes per head.- pub dt: &'a MTLBuffer, // buffer(1)- /// A – diagonal decay coefficients shared across tokens (eigenvalues).- pub a: &'a MTLBuffer, // buffer(2)- /// B – input projection that injects X into the state space.- pub b: &'a MTLBuffer, // buffer(3)- /// C – output projection that maps state to residual updates.- pub c: &'a MTLBuffer, // buffer(4)- /// D – per-head skip/identity weights.- pub d: &'a MTLBuffer, // buffer(5)- /// Z – post-gate activations that modulate the residual output.- pub z: &'a MTLBuffer, // buffer(6)- /// Current state tensor.- pub state: &'a MTLBuffer, // buffer(7)- /// Y – residual output buffer.- pub y: &'a MTLBuffer, // buffer(8)- /// Next state tensor (same buffer as `state`, but written after update).- pub next_state: &'a MTLBuffer, // buffer(9)- pub batch_size: usize,- pub channels: usize,-}--impl SSMUpdateKernel {- pub fn new(- context: &MTLContext,- data_type: KernelDataType,- ) -> Result<Self, SSMKernelError> {- let fn_name = format!("ssm_update_kernel_{}", fn_suffix(data_type));- let pipeline = context- .compute_pipeline_state(&fn_name, None)- .map_err(SSMKernelError::MetalError)?;- Ok(Self {- pipeline,- })- }-- pub fn encode(- &self,- compute_encoder: &ComputeCommandEncoderRef,- args: SSMUpdateArguments,- ) -> Result<(), SSMKernelError> {- compute_encoder.set_compute_pipeline_state(&self.pipeline);-- compute_encoder.set_buffer(0, Some(args.x), 0);- compute_encoder.set_buffer(1, Some(args.dt), 0);- compute_encoder.set_buffer(2, Some(args.a), 0);- compute_encoder.set_buffer(3, Some(args.b), 0);- compute_encoder.set_buffer(4, Some(args.c), 0);- compute_encoder.set_buffer(5, Some(args.d), 0);- compute_encoder.set_buffer(6, Some(args.z), 0);- compute_encoder.set_buffer(7, Some(args.state), 0);- compute_encoder.set_buffer(8, Some(args.y), 0);- compute_encoder.set_buffer(9, Some(args.next_state), 0);-- let threads_per_threadgroup = MTLSize {- width: 32,- height: 32,- depth: 1,- };- let total_threads = MTLSize {- width: args.batch_size as u64,- height: args.channels as u64,- depth: 1,- };- compute_encoder- .dispatch_threads(total_threads, threads_per_threadgroup);- Ok(())- }-}diff --git a/crates/uzu/tests/common/mod.rs b/crates/uzu/tests/common/mod.rsindex 1c1f9b325..1b740a064 100644--- a/crates/uzu/tests/common/mod.rs+++ b/crates/uzu/tests/common/mod.rs@@ -3,7 +3,8 @@ use std::path::PathBuf;use uzu::VERSION;-pub const MODEL_DIR_NAME: &str = "Llama-3.2-1B-Instruct";++pub const MODEL_DIR_NAME: &str = "Llamba-1B-4bit-mlx";pub const MODEL_FILE_NAME: &str = "model.safetensors";pub const TRACES_FILE_NAME: &str = "traces.safetensors";diff --git a/crates/uzu/tests/ssm_update_test.rs b/crates/uzu/tests/ssm_update_test.rsindex fa4b40986..611f73eac 100644--- a/crates/uzu/tests/ssm_update_test.rs+++ b/crates/uzu/tests/ssm_update_test.rs@@ -5,8 +5,7 @@ use metal::MTLResourceOptions;use uzu::backends::metal::{KernelDataType, MTLContext,kernel::ssm::{- SSDUpdateArguments, SSDUpdateKernel, SSMUpdateArguments,- SSMUpdateKernel,+ SSDUpdateArguments, SSDUpdateKernel,},};fn ssm_update_ref_bf16(@@ -177,135 +176,6 @@ fn create_context() -> Option<MTLContext> {MTLContext::new(device, command_queue).ok()}-#[test]-fn ssm_update_bf16() {- let Some(ctx) = create_context() else {- eprintln!("Skipping: no Metal device");- return;- };- let bsz = 2usize;- let h = 4usize;- let n = 16usize;-- let x: Vec<bf16> = (0..bsz * h)- .map(|i| bf16::from_f32(((i % 7) as f32) * 0.1 - 0.2))- .collect();- let dt: Vec<bf16> =- (0..bsz * h).map(|i| bf16::from_f32(((i % 5) as f32) * 0.03)).collect();- let a: Vec<bf16> = (0..n)- .map(|i| bf16::from_f32(((i % 11) as f32) * 0.02 - 0.1))- .collect();- let b: Vec<bf16> = (0..bsz * n)- .map(|i| bf16::from_f32(((i % 13) as f32) * 0.01))- .collect();- let c: Vec<bf16> = (0..bsz * n)- .map(|i| bf16::from_f32(((i % 17) as f32) * 0.015))- .collect();- let d: Vec<bf16> =- (0..h).map(|i| bf16::from_f32(((i % 3) as f32) * 0.05)).collect();- let z: Vec<bf16> = (0..bsz * h)- .map(|i| bf16::from_f32(((i % 7) as f32) * 0.1 - 0.1))- .collect();- let state: Vec<bf16> = (0..bsz * h * n)- .map(|i| bf16::from_f32(((i % 23) as f32) * 0.01 - 0.05))- .collect();-- let (y_exp, ns_exp) =- ssm_update_ref_bf16(&x, &dt, &a, &b, &c, &d, &z, &state, bsz, h);- let y_exp_f32: Vec<f32> = y_exp.iter().map(|&v| v.to_f32()).collect();- let ns_exp_f32: Vec<f32> = ns_exp.iter().map(|&v| v.to_f32()).collect();-- let x_buf = ctx.device.new_buffer_with_data(- x.as_ptr() as *const _,- (x.len() * std::mem::size_of::<bf16>()) as u64,- MTLResourceOptions::StorageModeShared,- );- let dt_buf = ctx.device.new_buffer_with_data(- dt.as_ptr() as *const _,- (dt.len() * std::mem::size_of::<bf16>()) as u64,- MTLResourceOptions::StorageModeShared,- );- let a_buf = ctx.device.new_buffer_with_data(- a.as_ptr() as *const _,- (a.len() * std::mem::size_of::<bf16>()) as u64,- MTLResourceOptions::StorageModeShared,- );- let b_buf = ctx.device.new_buffer_with_data(- b.as_ptr() as *const _,- (b.len() * std::mem::size_of::<bf16>()) as u64,- MTLResourceOptions::StorageModeShared,- );- let c_buf = ctx.device.new_buffer_with_data(- c.as_ptr() as *const _,- (c.len() * std::mem::size_of::<bf16>()) as u64,- MTLResourceOptions::StorageModeShared,- );- let d_buf = ctx.device.new_buffer_with_data(- d.as_ptr() as *const _,- (d.len() * std::mem::size_of::<bf16>()) as u64,- MTLResourceOptions::StorageModeShared,- );- let z_buf = ctx.device.new_buffer_with_data(- z.as_ptr() as *const _,- (z.len() * std::mem::size_of::<bf16>()) as u64,- MTLResourceOptions::StorageModeShared,- );- let s_buf = ctx.device.new_buffer_with_data(- state.as_ptr() as *const _,- (state.len() * std::mem::size_of::<bf16>()) as u64,- MTLResourceOptions::StorageModeShared,- );- let y_buf = ctx.device.new_buffer(- (bsz * h * std::mem::size_of::<bf16>()) as u64,- MTLResourceOptions::StorageModeShared,- );- let ns_buf = ctx.device.new_buffer(- (bsz * h * n * std::mem::size_of::<bf16>()) as u64,- MTLResourceOptions::StorageModeShared,- );-- let kernel = SSMUpdateKernel::new(&ctx, KernelDataType::BFloat16).unwrap();- let cb_ref = ctx.command_queue.new_command_buffer();- let cb = cb_ref.to_owned();- let enc = cb.new_compute_command_encoder();- kernel- .encode(- &enc,- SSMUpdateArguments {- x: &x_buf,- dt: &dt_buf,- a: &a_buf,- b: &b_buf,- c: &c_buf,- d: &d_buf,- z: &z_buf,- state: &s_buf,- y: &y_buf,- next_state: &ns_buf,- batch_size: bsz,- channels: h,- },- )- .unwrap();- enc.end_encoding();- cb_ref.commit();- cb_ref.wait_until_completed();-- let y_ptr = y_buf.contents() as *const bf16;- let y_out = unsafe { std::slice::from_raw_parts(y_ptr, bsz * h) };- let y_out_f32: Vec<f32> = y_out.iter().map(|&v| v.to_f32()).collect();- let ns_ptr = ns_buf.contents() as *const bf16;- let ns_out = unsafe { std::slice::from_raw_parts(ns_ptr, bsz * h * n) };- let ns_out_f32: Vec<f32> = ns_out.iter().map(|&v| v.to_f32()).collect();- let tol = 2e-2;- for (i, (a, b)) in y_out_f32.iter().zip(y_exp_f32.iter()).enumerate() {- assert!((a - b).abs() < tol, "y {} {} {}", i, a, b);- }- for (i, (a, b)) in ns_out_f32.iter().zip(ns_exp_f32.iter()).enumerate() {- assert!((a - b).abs() < tol, "ns {} {} {}", i, a, b);- }-}-#[test]fn ssd_update_with_z_bf16() {let Some(ctx) = create_context() else {