Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

manual unrolling + deleted unused code

LuckyIYI05b66ff64ab9
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 83ab64e71..70fcca455 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -37,7 +37,7 @@ pub(crate) use ssm::MambaMixerEncodable;
pub use ssm::{
Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,
SSDPrefillArguments, SSDPrefillKernel, SSDPrefillMode, SSDUpdateArguments,
- SSDUpdateKernel, SSMKernelError, SSMUpdateArguments, SSMUpdateKernel,
+ SSDUpdateKernel, SSMKernelError,
SplitInProjArguments, SplitInProjKernel,
};
pub use tensor_add_bias::TensorAddBias;
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
index 46b5150b4..dc90432d4 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
@@ -1189,21 +1189,45 @@ void qmv_fast_impl(
for (int k = 0; k < in_vec_size; k += block_size) {
U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);
- for (int row = 0; row < results_per_simdgroup; row++) {
- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
- const device T* sl = scales + row * in_vec_size_g;
- U s = sl[0];
-
+ {
+ auto wl0 = (const device uint8_t*)(ws);
+ auto wl1 = (const device uint8_t*)(ws + in_vec_size_w);
+ auto wl2 = (const device uint8_t*)(ws + 2 * in_vec_size_w);
+ auto wl3 = (const device uint8_t*)(ws + 3 * in_vec_size_w);
+
+ U s0 = static_cast<U>(scales[0]);
+ U s1 = static_cast<U>(scales[in_vec_size_g]);
+ U s2 = static_cast<U>(scales[2 * in_vec_size_g]);
+ U s3 = static_cast<U>(scales[3 * in_vec_size_g]);
+
if (kUseMlxQuant) {
- const device T* bl = biases + row * in_vec_size_g;
- U b = bl[0];
- result[row] += qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);
+ U b0 = static_cast<U>(biases[0]);
+ U b1 = static_cast<U>(biases[in_vec_size_g]);
+ U b2 = static_cast<U>(biases[2 * in_vec_size_g]);
+ U b3 = static_cast<U>(biases[3 * in_vec_size_g]);
+ result[0] += qdot<U, values_per_thread, bits>(wl0, x_thread, s0, b0, sum);
+ result[1] += qdot<U, values_per_thread, bits>(wl1, x_thread, s1, b1, sum);
+ result[2] += qdot<U, values_per_thread, bits>(wl2, x_thread, s2, b2, sum);
+ result[3] += qdot<U, values_per_thread, bits>(wl3, x_thread, s3, b3, sum);
} else {
- const device uint8_t* zl = zps + row * zp_stride;
- uint8_t zp_byte = *zl;
- U zp = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte >> 4) : (zp_byte & 0x0F));
- if (bits == 8) zp = static_cast<U>(zp_byte);
- result[row] += qdot_zero_point<U, values_per_thread, bits>(wl, x_thread, s, zp);
+ uint8_t zp_byte0 = zps[0];
+ uint8_t zp_byte1 = zps[zp_stride];
+ uint8_t zp_byte2 = zps[2 * zp_stride];
+ uint8_t zp_byte3 = zps[3 * zp_stride];
+ U zp0 = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte0 >> 4) : (zp_byte0 & 0x0F));
+ U zp1 = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte1 >> 4) : (zp_byte1 & 0x0F));
+ U zp2 = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte2 >> 4) : (zp_byte2 & 0x0F));
+ U zp3 = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte3 >> 4) : (zp_byte3 & 0x0F));
+ if (bits == 8) {
+ zp0 = static_cast<U>(zp_byte0);
+ zp1 = static_cast<U>(zp_byte1);
+ zp2 = static_cast<U>(zp_byte2);
+ zp3 = static_cast<U>(zp_byte3);
+ }
+ result[0] += qdot_zero_point<U, values_per_thread, bits>(wl0, x_thread, s0, zp0);
+ result[1] += qdot_zero_point<U, values_per_thread, bits>(wl1, x_thread, s1, zp1);
+ result[2] += qdot_zero_point<U, values_per_thread, bits>(wl2, x_thread, s2, zp2);
+ result[3] += qdot_zero_point<U, values_per_thread, bits>(wl3, x_thread, s3, zp3);
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs b/crates/uzu/src/backends/metal/kernel/ssm/mod.rs
index 8fd1ce3e3..9b23b0f10 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/ssm/mod.rs
@@ -17,7 +17,6 @@ pub mod mamba;
pub mod split_inproj;
pub mod ssd_prefill;
pub mod ssd_update;
-pub mod ssm_update;
pub use conv1d_scan::{
Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel,
@@ -26,4 +25,3 @@ pub(crate) use mamba::MambaMixerEncodable;
pub use split_inproj::{SplitInProjArguments, SplitInProjKernel};
pub use ssd_prefill::{SSDPrefillArguments, SSDPrefillKernel, SSDPrefillMode};
pub use ssd_update::{SSDUpdateArguments, SSDUpdateKernel};
-pub use ssm_update::{SSMUpdateArguments, SSMUpdateKernel};
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssm_update.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssm_update.metal
deleted file mode 100644
index b88fb4af1..000000000
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssm_update.metal
+++ /dev/null
@@ -1,107 +0,0 @@
-#include <metal_stdlib>
-#include "../definitions.metal"
-
-using namespace metal;
-
-struct SILU {
- template <typename T>
- T operator()(T x) {
- float xf = float(x);
- float y = 1.0f / (1.0f + fast::exp(-fabs(xf)));
- float out = (xf < 0.0f) ? (1.0f - y) * xf : y * xf;
- return static_cast<T>(out);
- }
-};
-
-struct Softplus {
- template <typename T>
- T operator()(T x) {
- float xf = float(x);
- float y = log(1.0f + fast::exp(xf));
- return static_cast<T>((xf > 20.0f) ? xf : y);
- }
-};
-
-struct Exp {
- template <typename T>
- T operator()(T x) {
- float xf = float(x);
- return static_cast<T>(fast::exp(xf));
- };
-};
-
-// Mamba SSM naming:
-// X – SILU-activated hidden stream after the conv block
-// Z – gate activations that modulate the residual output
-// A – diagonal state transition spectrum (constant per layer)
-// B – input projection that injects X into the state
-// C – output projection from state to residual
-// D – per-head skip/identity weight
-
-template <typename T>
-kernel void ssm_update_kernel(
- device const T* x [[ buffer(0) ]], // (b, h) hidden stream X
- device const T* dt [[ buffer(1) ]], // (b, h) delta / step size
- device const T* A [[ buffer(2) ]], // (n) spectrum A
- device const T* B [[ buffer(3) ]], // (b, n) input projection B
- device const T* C [[ buffer(4) ]], // (b, n) output projection C
- device const T* D [[ buffer(5) ]], // (h) skip weights D
- device const T* z [[ buffer(6) ]], // (b, h) gate Z
- device const T* state [[ buffer(7) ]], // (b, h, n)
- device T* y [[ buffer(8) ]], // (b, h)
- device T* next_state [[ buffer(9) ]], // (b, h, n)
- uint3 grid_idx [[ thread_position_in_grid ]],
- uint3 grid_size [[ threads_per_grid ]]
-) {
- const int state_size = 16; // matches reference
- const int channel_size = grid_size.y;
-
- const int batch_idx = grid_idx.x;
- const int channel_idx = grid_idx.y;
-
- const int cb_start_idx = batch_idx * state_size;
- const int x_idx = batch_idx * channel_size + channel_idx;
- const int state_start_idx = x_idx * state_size;
-
- T this_x = x[x_idx];
- this_x = SILU{}(this_x);
-
- T this_z = z[x_idx];
- this_z = SILU{}(this_z);
-
- T delta = Softplus{}(dt[x_idx]);
-
- T temp = T(0);
- #pragma unroll
- for (int i = 0; i < state_size; ++i) {
- int cb_idx = cb_start_idx + i;
- int state_idx = state_start_idx + i;
- T this_new_state = state[state_idx] * Exp{}(A[i] * delta) + B[cb_idx] * delta * this_x;
- next_state[state_idx] = this_new_state;
- temp = temp + this_new_state * C[cb_idx];
- }
- temp = temp + D[channel_idx] * this_x;
- temp = temp * this_z;
- y[x_idx] = temp;
-}
-
-#define instantiate_ssm_update_kernel(type_name, type) \
- template [[host_name("ssm_update_kernel_" #type_name)]] \
- kernel void ssm_update_kernel<type>( \
- device const type* x [[ buffer(0) ]], \
- device const type* dt [[ buffer(1) ]], \
- device const type* A [[ buffer(2) ]], \
- device const type* B [[ buffer(3) ]], \
- device const type* C [[ buffer(4) ]], \
- device const type* D [[ buffer(5) ]], \
- device const type* z [[ buffer(6) ]], \
- device const type* state [[ buffer(7) ]], \
- device type* y [[ buffer(8) ]], \
- device type* next_state [[ buffer(9) ]], \
- uint3 grid_idx [[ thread_position_in_grid ]], \
- uint3 grid_size [[ threads_per_grid ]]);
-
-instantiate_ssm_update_kernel(float, float);
-instantiate_ssm_update_kernel(bfloat, bfloat);
-instantiate_ssm_update_kernel(half, half);
-
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssm_update.rs b/crates/uzu/src/backends/metal/kernel/ssm/ssm_update.rs
deleted file mode 100644
index c5bd9a337..000000000
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssm_update.rs
+++ /dev/null
@@ -1,84 +0,0 @@
-use metal::{
- Buffer as MTLBuffer, ComputeCommandEncoderRef,
- ComputePipelineState as MTLComputePipelineState, MTLSize,
-};
-
-use super::{SSMKernelError, fn_suffix};
-use crate::backends::metal::{KernelDataType, MTLContext};
-
-pub struct SSMUpdateKernel {
- pipeline: MTLComputePipelineState,
-}
-
-pub struct SSMUpdateArguments<'a> {
- /// X – Silu-activated hidden stream (per head/channel) coming from the conv.
- pub x: &'a MTLBuffer, // buffer(0)
- /// Δt – learned step sizes per head.
- pub dt: &'a MTLBuffer, // buffer(1)
- /// A – diagonal decay coefficients shared across tokens (eigenvalues).
- pub a: &'a MTLBuffer, // buffer(2)
- /// B – input projection that injects X into the state space.
- pub b: &'a MTLBuffer, // buffer(3)
- /// C – output projection that maps state to residual updates.
- pub c: &'a MTLBuffer, // buffer(4)
- /// D – per-head skip/identity weights.
- pub d: &'a MTLBuffer, // buffer(5)
- /// Z – post-gate activations that modulate the residual output.
- pub z: &'a MTLBuffer, // buffer(6)
- /// Current state tensor.
- pub state: &'a MTLBuffer, // buffer(7)
- /// Y – residual output buffer.
- pub y: &'a MTLBuffer, // buffer(8)
- /// Next state tensor (same buffer as `state`, but written after update).
- pub next_state: &'a MTLBuffer, // buffer(9)
- pub batch_size: usize,
- pub channels: usize,
-}
-
-impl SSMUpdateKernel {
- pub fn new(
- context: &MTLContext,
- data_type: KernelDataType,
- ) -> Result<Self, SSMKernelError> {
- let fn_name = format!("ssm_update_kernel_{}", fn_suffix(data_type));
- let pipeline = context
- .compute_pipeline_state(&fn_name, None)
- .map_err(SSMKernelError::MetalError)?;
- Ok(Self {
- pipeline,
- })
- }
-
- pub fn encode(
- &self,
- compute_encoder: &ComputeCommandEncoderRef,
- args: SSMUpdateArguments,
- ) -> Result<(), SSMKernelError> {
- compute_encoder.set_compute_pipeline_state(&self.pipeline);
-
- compute_encoder.set_buffer(0, Some(args.x), 0);
- compute_encoder.set_buffer(1, Some(args.dt), 0);
- compute_encoder.set_buffer(2, Some(args.a), 0);
- compute_encoder.set_buffer(3, Some(args.b), 0);
- compute_encoder.set_buffer(4, Some(args.c), 0);
- compute_encoder.set_buffer(5, Some(args.d), 0);
- compute_encoder.set_buffer(6, Some(args.z), 0);
- compute_encoder.set_buffer(7, Some(args.state), 0);
- compute_encoder.set_buffer(8, Some(args.y), 0);
- compute_encoder.set_buffer(9, Some(args.next_state), 0);
-
- let threads_per_threadgroup = MTLSize {
- width: 32,
- height: 32,
- depth: 1,
- };
- let total_threads = MTLSize {
- width: args.batch_size as u64,
- height: args.channels as u64,
- depth: 1,
- };
- compute_encoder
- .dispatch_threads(total_threads, threads_per_threadgroup);
- Ok(())
- }
-}
diff --git a/crates/uzu/tests/common/mod.rs b/crates/uzu/tests/common/mod.rs
index 1c1f9b325..1b740a064 100644
--- a/crates/uzu/tests/common/mod.rs
+++ b/crates/uzu/tests/common/mod.rs
@@ -3,7 +3,8 @@ use std::path::PathBuf;
use uzu::VERSION;
-pub const MODEL_DIR_NAME: &str = "Llama-3.2-1B-Instruct";
+
+pub const MODEL_DIR_NAME: &str = "Llamba-1B-4bit-mlx";
pub const MODEL_FILE_NAME: &str = "model.safetensors";
pub const TRACES_FILE_NAME: &str = "traces.safetensors";
diff --git a/crates/uzu/tests/ssm_update_test.rs b/crates/uzu/tests/ssm_update_test.rs
index fa4b40986..611f73eac 100644
--- a/crates/uzu/tests/ssm_update_test.rs
+++ b/crates/uzu/tests/ssm_update_test.rs
@@ -5,8 +5,7 @@ use metal::MTLResourceOptions;
use uzu::backends::metal::{
KernelDataType, MTLContext,
kernel::ssm::{
- SSDUpdateArguments, SSDUpdateKernel, SSMUpdateArguments,
- SSMUpdateKernel,
+ SSDUpdateArguments, SSDUpdateKernel,
},
};
fn ssm_update_ref_bf16(
@@ -177,135 +176,6 @@ fn create_context() -> Option<MTLContext> {
MTLContext::new(device, command_queue).ok()
}
-#[test]
-fn ssm_update_bf16() {
- let Some(ctx) = create_context() else {
- eprintln!("Skipping: no Metal device");
- return;
- };
- let bsz = 2usize;
- let h = 4usize;
- let n = 16usize;
-
- let x: Vec<bf16> = (0..bsz * h)
- .map(|i| bf16::from_f32(((i % 7) as f32) * 0.1 - 0.2))
- .collect();
- let dt: Vec<bf16> =
- (0..bsz * h).map(|i| bf16::from_f32(((i % 5) as f32) * 0.03)).collect();
- let a: Vec<bf16> = (0..n)
- .map(|i| bf16::from_f32(((i % 11) as f32) * 0.02 - 0.1))
- .collect();
- let b: Vec<bf16> = (0..bsz * n)
- .map(|i| bf16::from_f32(((i % 13) as f32) * 0.01))
- .collect();
- let c: Vec<bf16> = (0..bsz * n)
- .map(|i| bf16::from_f32(((i % 17) as f32) * 0.015))
- .collect();
- let d: Vec<bf16> =
- (0..h).map(|i| bf16::from_f32(((i % 3) as f32) * 0.05)).collect();
- let z: Vec<bf16> = (0..bsz * h)
- .map(|i| bf16::from_f32(((i % 7) as f32) * 0.1 - 0.1))
- .collect();
- let state: Vec<bf16> = (0..bsz * h * n)
- .map(|i| bf16::from_f32(((i % 23) as f32) * 0.01 - 0.05))
- .collect();
-
- let (y_exp, ns_exp) =
- ssm_update_ref_bf16(&x, &dt, &a, &b, &c, &d, &z, &state, bsz, h);
- let y_exp_f32: Vec<f32> = y_exp.iter().map(|&v| v.to_f32()).collect();
- let ns_exp_f32: Vec<f32> = ns_exp.iter().map(|&v| v.to_f32()).collect();
-
- let x_buf = ctx.device.new_buffer_with_data(
- x.as_ptr() as *const _,
- (x.len() * std::mem::size_of::<bf16>()) as u64,
- MTLResourceOptions::StorageModeShared,
- );
- let dt_buf = ctx.device.new_buffer_with_data(
- dt.as_ptr() as *const _,
- (dt.len() * std::mem::size_of::<bf16>()) as u64,
- MTLResourceOptions::StorageModeShared,
- );
- let a_buf = ctx.device.new_buffer_with_data(
- a.as_ptr() as *const _,
- (a.len() * std::mem::size_of::<bf16>()) as u64,
- MTLResourceOptions::StorageModeShared,
- );
- let b_buf = ctx.device.new_buffer_with_data(
- b.as_ptr() as *const _,
- (b.len() * std::mem::size_of::<bf16>()) as u64,
- MTLResourceOptions::StorageModeShared,
- );
- let c_buf = ctx.device.new_buffer_with_data(
- c.as_ptr() as *const _,
- (c.len() * std::mem::size_of::<bf16>()) as u64,
- MTLResourceOptions::StorageModeShared,
- );
- let d_buf = ctx.device.new_buffer_with_data(
- d.as_ptr() as *const _,
- (d.len() * std::mem::size_of::<bf16>()) as u64,
- MTLResourceOptions::StorageModeShared,
- );
- let z_buf = ctx.device.new_buffer_with_data(
- z.as_ptr() as *const _,
- (z.len() * std::mem::size_of::<bf16>()) as u64,
- MTLResourceOptions::StorageModeShared,
- );
- let s_buf = ctx.device.new_buffer_with_data(
- state.as_ptr() as *const _,
- (state.len() * std::mem::size_of::<bf16>()) as u64,
- MTLResourceOptions::StorageModeShared,
- );
- let y_buf = ctx.device.new_buffer(
- (bsz * h * std::mem::size_of::<bf16>()) as u64,
- MTLResourceOptions::StorageModeShared,
- );
- let ns_buf = ctx.device.new_buffer(
- (bsz * h * n * std::mem::size_of::<bf16>()) as u64,
- MTLResourceOptions::StorageModeShared,
- );
-
- let kernel = SSMUpdateKernel::new(&ctx, KernelDataType::BFloat16).unwrap();
- let cb_ref = ctx.command_queue.new_command_buffer();
- let cb = cb_ref.to_owned();
- let enc = cb.new_compute_command_encoder();
- kernel
- .encode(
- &enc,
- SSMUpdateArguments {
- x: &x_buf,
- dt: &dt_buf,
- a: &a_buf,
- b: &b_buf,
- c: &c_buf,
- d: &d_buf,
- z: &z_buf,
- state: &s_buf,
- y: &y_buf,
- next_state: &ns_buf,
- batch_size: bsz,
- channels: h,
- },
- )
- .unwrap();
- enc.end_encoding();
- cb_ref.commit();
- cb_ref.wait_until_completed();
-
- let y_ptr = y_buf.contents() as *const bf16;
- let y_out = unsafe { std::slice::from_raw_parts(y_ptr, bsz * h) };
- let y_out_f32: Vec<f32> = y_out.iter().map(|&v| v.to_f32()).collect();
- let ns_ptr = ns_buf.contents() as *const bf16;
- let ns_out = unsafe { std::slice::from_raw_parts(ns_ptr, bsz * h * n) };
- let ns_out_f32: Vec<f32> = ns_out.iter().map(|&v| v.to_f32()).collect();
- let tol = 2e-2;
- for (i, (a, b)) in y_out_f32.iter().zip(y_exp_f32.iter()).enumerate() {
- assert!((a - b).abs() < tol, "y {} {} {}", i, a, b);
- }
- for (i, (a, b)) in ns_out_f32.iter().zip(ns_exp_f32.iter()).enumerate() {
- assert!((a - b).abs() < tol, "ns {} {} {}", i, a, b);
- }
-}
-
#[test]
fn ssd_update_with_z_bf16() {
let Some(ctx) = create_context() else {