A high-performance inference engine for AI models
qk norm update
LuckyIYIb0ff5a6c963d
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex a653c6832..d3aa2e285 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -58,6 +58,7 @@ itertools = "0.14.0"build-debug = []build-clean = []tracing = []+metal-nax = [][[test]]name = "moe"diff --git a/crates/uzu/build/metal/mod.rs b/crates/uzu/build/metal/mod.rsindex e4f31303d..b4abe8ed2 100644--- a/crates/uzu/build/metal/mod.rs+++ b/crates/uzu/build/metal/mod.rs@@ -1,4 +1,7 @@-use std::{env, fs, path::PathBuf};+use std::{+ env, fs,+ path::{Path, PathBuf},+};use anyhow::Context;use compiler::{MetalCompiler, ObjectInfo};@@ -7,12 +10,34 @@ use proc_macro2::TokenStream;use quote::quote;use walkdir::WalkDir;+use crate::debug_log;+mod ast;mod bindgen;mod compiler;mod toolchain;+fn is_nax_source(path: &Path) -> bool {+ path.file_name()+ .and_then(|s| s.to_str())+ .map(|s| s.contains("_nax"))+ .unwrap_or(false)+}+pub async fn main() -> anyhow::Result<()> {+ let out_dir =+ PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);++ let nax_enabled = cfg!(feature = "metal-nax");+ debug_log!("metal nax enabled: {nax_enabled}");++ let nax_mode_path = out_dir.join("metal_nax_mode");+ let desired_nax_mode = if nax_enabled { "1" } else { "0" };+ let prev_nax_mode =+ fs::read_to_string(&nax_mode_path).ok().map(|s| s.trim().to_owned());+ let nax_mode_changed =+ prev_nax_mode.as_deref() != Some(desired_nax_mode);+let dsl = MetalCompiler::new().context("cannot create metal dsl")?;let src_dir = PathBuf::from(@@ -29,6 +54,7 @@ pub async fn main() -> anyhow::Result<()> {== Some("metal")}).map(|e| e.into_path())+ .filter(|p| nax_enabled || !is_nax_source(p)).collect();let num_concurrent_compiles =@@ -45,7 +71,7 @@ pub async fn main() -> anyhow::Result<()> {let kernels: Vec<ObjectInfo> =results.into_iter().map(|(o, _)| o).collect();- if !all_cached {+ if !all_cached || nax_mode_changed {dsl.link(&kernels).await.context("cannot link objects")?;let bindings = kernels@@ -68,6 +94,13 @@ pub async fn main() -> anyhow::Result<()> {fs::write(&out_path, prettyplease::unparse(&parsed)).with_context(|| format!("cannot write bindings file {}", out_path.display()),)?;++ fs::write(&nax_mode_path, desired_nax_mode).with_context(|| {+ format!(+ "cannot write nax mode file {}",+ nax_mode_path.display()+ )+ })?;}Ok(())diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rsindex f3b588543..d1eb88618 100644--- a/crates/uzu/src/backends/metal/context.rs+++ b/crates/uzu/src/backends/metal/context.rs@@ -164,7 +164,7 @@ impl MTLContext {/// Returns true if NAX kernels are available on this device.pub fn is_nax_available(&self) -> bool {- self.architecture.is_nax_available()+ cfg!(feature = "metal-nax") && self.architecture.is_nax_available()}/// Returns true if this is a high-performance device (Pro/Max class).diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rsindex 0d77b7dbd..70ad744f0 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs@@ -23,16 +23,6 @@ pub enum QKNormTarget {Both,}-impl QKNormTarget {- fn to_mask(self) -> u32 {- match self {- QKNormTarget::QueryHeads => 0b01,- QKNormTarget::KeyHeads => 0b10,- QKNormTarget::Both => 0b11,- }- }-}-pub struct RMSNormKernel {pipeline: MTLComputePipelineState,kernel_type: RMSNormKernelType,@@ -311,38 +301,49 @@ impl RMSNormKernel {&args.scale_offset as *const f32 as *const std::ffi::c_void,);- let active_type_mask = args.target.to_mask();+ // Determine which contiguous head range to normalize.+ //+ // QKV layout per token: [Q heads][K heads][V heads]+ // We only ever normalize within [Q] and/or [K].+ let (head_offset, head_count): (u32, u32) = match args.target {+ QKNormTarget::QueryHeads => (0, args.num_q_heads as u32),+ QKNormTarget::KeyHeads => (args.num_q_heads as u32, args.num_kv_heads as u32),+ QKNormTarget::Both => (0, (args.num_q_heads + args.num_kv_heads) as u32),+ };++ if args.batch_size <= 0 || args.head_dim <= 0 || head_count == 0 {+ return Ok(());+ }++ // Pass head range to the kernel.compute_encoder.set_bytes(9,size_of::<u32>() as u64,- &active_type_mask as *const u32 as *const std::ffi::c_void,+ &head_offset as *const u32 as *const std::ffi::c_void,+ );+ compute_encoder.set_bytes(+ 10,+ size_of::<u32>() as u64,+ &head_count as *const u32 as *const std::ffi::c_void,);- // Calculate threadgroup dimensions based on target- let total_heads_to_dispatch = match args.target {- QKNormTarget::QueryHeads => {- // Process Q heads only: dispatch for indices 0 to num_q_heads-1- args.num_q_heads- },- QKNormTarget::KeyHeads => {- // Process K heads only: dispatch for indices 0 to num_q_heads+num_kv_heads-1- // (kernel will skip Q heads based on target enum)- args.num_q_heads + args.num_kv_heads- },- QKNormTarget::Both => {- // Process both Q and K heads: dispatch for indices 0 to num_q_heads+num_kv_heads-1- args.num_q_heads + args.num_kv_heads- },- };+ // One SIMD-group per head, multiple heads per threadgroup.+ let simd_width = self.pipeline.thread_execution_width() as u64;+ let max_threads = self.pipeline.max_total_threads_per_threadgroup() as u64;+ let max_heads_per_threadgroup = (max_threads / simd_width).max(1);+ let heads_per_threadgroup =+ (head_count as u64).min(max_heads_per_threadgroup).max(1);++ let num_head_tiles = (head_count as u64).div_ceil(heads_per_threadgroup);let threadgroups_per_grid = MTLSize {width: args.batch_size as u64,- height: total_heads_to_dispatch as u64,+ height: num_head_tiles,depth: 1,};let threads_per_threadgroup = MTLSize {- width: 1024,+ width: heads_per_threadgroup * simd_width,height: 1,depth: 1,};diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metalindex 060fb09d8..6ad763d0b 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal@@ -254,7 +254,85 @@ void rms_norm_core(FOREACH_RMS_COMBO(DEFINE_RMS_KERNEL)#undef DEFINE_RMS_KERNEL-// Generate QK norm kernels+// QK norm: normalize per-head vectors (small head_dim) efficiently.+//+// Strategy:+// - One SIMD-group (32 threads) processes one head.+// - A threadgroup processes multiple heads for a given token (batch).+// - No threadgroup-wide reductions/barriers: we only use simd_sum within a SIMD-group.+template <typename InputT, typename ScaleT, typename OutputT, typename AccumT, bool FULL_LAYER>+static inline void qk_norm_head_core(+ const device InputT* input_data,+ const device ScaleT* scales_data,+ device OutputT* output_data,+ uint element_count,+ constant float& epsilon,+ constant float& scale_offset,+ uint lane_id+) {+ if (element_count == 0u)+ return;++ AccumT partial_sum = static_cast<AccumT>(0.0f);++ // Sum of squares: each lane processes GRAIN_SIZE elements per iteration.+ for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;+ base_i += SIMD_SIZE * GRAIN_SIZE) {+ AccumT vals[GRAIN_SIZE];+ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ uint i = base_i + j;+ vals[j] =+ (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;+ }+ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ partial_sum += vals[j] * vals[j];+ }+ }++ // SIMD-group reduction.+ AccumT total_sum = simd_sum(partial_sum);++ // RMS factor.+ AccumT mean_square =+ static_cast<AccumT>(total_sum) / static_cast<AccumT>(element_count);+ AccumT rms_norm = rsqrt(mean_square + static_cast<AccumT>(epsilon));++ // Normalize + scale.+ for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;+ base_i += SIMD_SIZE * GRAIN_SIZE) {+ AccumT vals[GRAIN_SIZE];+ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ uint i = base_i + j;+ vals[j] =+ (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;+ }++ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ uint i = base_i + j;+ if (i >= element_count)+ continue;++ AccumT normalized_high = vals[j] * rms_norm;++ if (FULL_LAYER) {+ AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) ++ static_cast<AccumT>(scale_offset);+ output_data[i] =+ static_cast<OutputT>(normalized_high * scale_value_high);+ } else {+ OutputT normalized_low = static_cast<OutputT>(normalized_high);+ OutputT scale_value_low = static_cast<OutputT>(+ static_cast<AccumT>(scales_data[i]) ++ static_cast<AccumT>(scale_offset)+ );+ OutputT product_low = normalized_low * scale_value_low;+ output_data[i] = static_cast<OutputT>(product_low);+ }+ }+ }+}++// Generate QK norm kernels (batched, head-tiled).#define DEFINE_QK_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \[[max_total_threads_per_threadgroup(1024)]] kernel void qk_norm_##SUF( \const device IN* qkv_input [[buffer(0)]], \@@ -266,51 +344,48 @@ FOREACH_RMS_COMBO(DEFINE_RMS_KERNEL)constant uint& head_dim [[buffer(6)]], \constant float& epsilon [[buffer(7)]], \constant float& scale_offset [[buffer(8)]], \- constant uint& active_type_mask [[buffer(9)]], \- uint3 threadgroup_position_in_grid [[threadgroup_position_in_grid]], \- uint3 thread_position_in_threadgroup [[thread_position_in_threadgroup]] \+ constant uint& head_offset [[buffer(9)]], \+ constant uint& head_count [[buffer(10)]], \+ uint3 tgid [[threadgroup_position_in_grid]], \+ uint lid [[thread_index_in_threadgroup]], \+ uint3 threads_per_threadgroup [[threads_per_threadgroup]] \) { \- uint batch_idx = threadgroup_position_in_grid.x; \- uint sequential_head_idx = threadgroup_position_in_grid.y; \- uint thread_in_threadgroup = thread_position_in_threadgroup.x; \+ const uint batch_idx = tgid.x; \+ const uint tile_idx = tgid.y; \\if (batch_idx >= batch_size) \return; \- \- uint qk_type, actual_head_idx, logical_head_idx; \- if (sequential_head_idx < num_q_heads) { \- qk_type = 0; \- actual_head_idx = sequential_head_idx; \- logical_head_idx = actual_head_idx; \- } else { \- qk_type = 1; \- actual_head_idx = sequential_head_idx - num_q_heads; \- logical_head_idx = num_q_heads + actual_head_idx; \- } \- \- if ((active_type_mask & (1 << qk_type)) == 0) \+ if (head_count == 0u || head_dim == 0u) \return; \- if (qk_type == 0 && actual_head_idx >= num_q_heads) \+ \+ const uint heads_per_tg = threads_per_threadgroup.x / SIMD_SIZE; \+ if (heads_per_tg == 0u) \return; \- if (qk_type == 1 && actual_head_idx >= num_kv_heads) \+ \+ const uint simd_group_id = lid / SIMD_SIZE; \+ const uint lane_id = lid % SIMD_SIZE; \+ const uint head_idx = tile_idx * heads_per_tg + simd_group_id; \+ \+ if (head_idx >= head_count) \return; \\- const uint total_heads_in_buffer = num_q_heads + 2 * num_kv_heads; \- const uint slice_offset = batch_idx * total_heads_in_buffer * head_dim + \- logical_head_idx * head_dim; \+ const uint total_heads_in_buffer = num_q_heads + 2u * num_kv_heads; \+ const uint logical_head_idx = head_offset + head_idx; \+ if (logical_head_idx >= total_heads_in_buffer) \+ return; \\- threadgroup ACC shared_sum[SIMD_SIZE]; \+ const ulong slice_offset = \+ (ulong)batch_idx * (ulong)total_heads_in_buffer * (ulong)head_dim + \+ (ulong)logical_head_idx * (ulong)head_dim; \\- rms_norm_core<IN, SC, OUT, ACC>( \+ qk_norm_head_core<IN, SC, OUT, ACC, FULL_LAYER>( \qkv_input + slice_offset, \scales, \qkv_output + slice_offset, \head_dim, \epsilon, \scale_offset, \- shared_sum, \- thread_in_threadgroup, \- FULL_LAYER \+ lane_id \); \}