Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

qk norm update

LuckyIYIb0ff5a6c963d
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index a653c6832..d3aa2e285 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -58,6 +58,7 @@ itertools = "0.14.0"
build-debug = []
build-clean = []
tracing = []
+metal-nax = []
[[test]]
name = "moe"
diff --git a/crates/uzu/build/metal/mod.rs b/crates/uzu/build/metal/mod.rs
index e4f31303d..b4abe8ed2 100644
--- a/crates/uzu/build/metal/mod.rs
+++ b/crates/uzu/build/metal/mod.rs
@@ -1,4 +1,7 @@
-use std::{env, fs, path::PathBuf};
+use std::{
+ env, fs,
+ path::{Path, PathBuf},
+};
use anyhow::Context;
use compiler::{MetalCompiler, ObjectInfo};
@@ -7,12 +10,34 @@ use proc_macro2::TokenStream;
use quote::quote;
use walkdir::WalkDir;
+use crate::debug_log;
+
mod ast;
mod bindgen;
mod compiler;
mod toolchain;
+fn is_nax_source(path: &Path) -> bool {
+ path.file_name()
+ .and_then(|s| s.to_str())
+ .map(|s| s.contains("_nax"))
+ .unwrap_or(false)
+}
+
pub async fn main() -> anyhow::Result<()> {
+ let out_dir =
+ PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);
+
+ let nax_enabled = cfg!(feature = "metal-nax");
+ debug_log!("metal nax enabled: {nax_enabled}");
+
+ let nax_mode_path = out_dir.join("metal_nax_mode");
+ let desired_nax_mode = if nax_enabled { "1" } else { "0" };
+ let prev_nax_mode =
+ fs::read_to_string(&nax_mode_path).ok().map(|s| s.trim().to_owned());
+ let nax_mode_changed =
+ prev_nax_mode.as_deref() != Some(desired_nax_mode);
+
let dsl = MetalCompiler::new().context("cannot create metal dsl")?;
let src_dir = PathBuf::from(
@@ -29,6 +54,7 @@ pub async fn main() -> anyhow::Result<()> {
== Some("metal")
})
.map(|e| e.into_path())
+ .filter(|p| nax_enabled || !is_nax_source(p))
.collect();
let num_concurrent_compiles =
@@ -45,7 +71,7 @@ pub async fn main() -> anyhow::Result<()> {
let kernels: Vec<ObjectInfo> =
results.into_iter().map(|(o, _)| o).collect();
- if !all_cached {
+ if !all_cached || nax_mode_changed {
dsl.link(&kernels).await.context("cannot link objects")?;
let bindings = kernels
@@ -68,6 +94,13 @@ pub async fn main() -> anyhow::Result<()> {
fs::write(&out_path, prettyplease::unparse(&parsed)).with_context(
|| format!("cannot write bindings file {}", out_path.display()),
)?;
+
+ fs::write(&nax_mode_path, desired_nax_mode).with_context(|| {
+ format!(
+ "cannot write nax mode file {}",
+ nax_mode_path.display()
+ )
+ })?;
}
Ok(())
diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rs
index f3b588543..d1eb88618 100644
--- a/crates/uzu/src/backends/metal/context.rs
+++ b/crates/uzu/src/backends/metal/context.rs
@@ -164,7 +164,7 @@ impl MTLContext {
/// Returns true if NAX kernels are available on this device.
pub fn is_nax_available(&self) -> bool {
- self.architecture.is_nax_available()
+ cfg!(feature = "metal-nax") && self.architecture.is_nax_available()
}
/// Returns true if this is a high-performance device (Pro/Max class).
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
index 0d77b7dbd..70ad744f0 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/mod.rs
@@ -23,16 +23,6 @@ pub enum QKNormTarget {
Both,
}
-impl QKNormTarget {
- fn to_mask(self) -> u32 {
- match self {
- QKNormTarget::QueryHeads => 0b01,
- QKNormTarget::KeyHeads => 0b10,
- QKNormTarget::Both => 0b11,
- }
- }
-}
-
pub struct RMSNormKernel {
pipeline: MTLComputePipelineState,
kernel_type: RMSNormKernelType,
@@ -311,38 +301,49 @@ impl RMSNormKernel {
&args.scale_offset as *const f32 as *const std::ffi::c_void,
);
- let active_type_mask = args.target.to_mask();
+ // Determine which contiguous head range to normalize.
+ //
+ // QKV layout per token: [Q heads][K heads][V heads]
+ // We only ever normalize within [Q] and/or [K].
+ let (head_offset, head_count): (u32, u32) = match args.target {
+ QKNormTarget::QueryHeads => (0, args.num_q_heads as u32),
+ QKNormTarget::KeyHeads => (args.num_q_heads as u32, args.num_kv_heads as u32),
+ QKNormTarget::Both => (0, (args.num_q_heads + args.num_kv_heads) as u32),
+ };
+
+ if args.batch_size <= 0 || args.head_dim <= 0 || head_count == 0 {
+ return Ok(());
+ }
+
+ // Pass head range to the kernel.
compute_encoder.set_bytes(
9,
size_of::<u32>() as u64,
- &active_type_mask as *const u32 as *const std::ffi::c_void,
+ &head_offset as *const u32 as *const std::ffi::c_void,
+ );
+ compute_encoder.set_bytes(
+ 10,
+ size_of::<u32>() as u64,
+ &head_count as *const u32 as *const std::ffi::c_void,
);
- // Calculate threadgroup dimensions based on target
- let total_heads_to_dispatch = match args.target {
- QKNormTarget::QueryHeads => {
- // Process Q heads only: dispatch for indices 0 to num_q_heads-1
- args.num_q_heads
- },
- QKNormTarget::KeyHeads => {
- // Process K heads only: dispatch for indices 0 to num_q_heads+num_kv_heads-1
- // (kernel will skip Q heads based on target enum)
- args.num_q_heads + args.num_kv_heads
- },
- QKNormTarget::Both => {
- // Process both Q and K heads: dispatch for indices 0 to num_q_heads+num_kv_heads-1
- args.num_q_heads + args.num_kv_heads
- },
- };
+ // One SIMD-group per head, multiple heads per threadgroup.
+ let simd_width = self.pipeline.thread_execution_width() as u64;
+ let max_threads = self.pipeline.max_total_threads_per_threadgroup() as u64;
+ let max_heads_per_threadgroup = (max_threads / simd_width).max(1);
+ let heads_per_threadgroup =
+ (head_count as u64).min(max_heads_per_threadgroup).max(1);
+
+ let num_head_tiles = (head_count as u64).div_ceil(heads_per_threadgroup);
let threadgroups_per_grid = MTLSize {
width: args.batch_size as u64,
- height: total_heads_to_dispatch as u64,
+ height: num_head_tiles,
depth: 1,
};
let threads_per_threadgroup = MTLSize {
- width: 1024,
+ width: heads_per_threadgroup * simd_width,
height: 1,
depth: 1,
};
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
index 060fb09d8..6ad763d0b 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
@@ -254,7 +254,85 @@ void rms_norm_core(
FOREACH_RMS_COMBO(DEFINE_RMS_KERNEL)
#undef DEFINE_RMS_KERNEL
-// Generate QK norm kernels
+// QK norm: normalize per-head vectors (small head_dim) efficiently.
+//
+// Strategy:
+// - One SIMD-group (32 threads) processes one head.
+// - A threadgroup processes multiple heads for a given token (batch).
+// - No threadgroup-wide reductions/barriers: we only use simd_sum within a SIMD-group.
+template <typename InputT, typename ScaleT, typename OutputT, typename AccumT, bool FULL_LAYER>
+static inline void qk_norm_head_core(
+ const device InputT* input_data,
+ const device ScaleT* scales_data,
+ device OutputT* output_data,
+ uint element_count,
+ constant float& epsilon,
+ constant float& scale_offset,
+ uint lane_id
+) {
+ if (element_count == 0u)
+ return;
+
+ AccumT partial_sum = static_cast<AccumT>(0.0f);
+
+ // Sum of squares: each lane processes GRAIN_SIZE elements per iteration.
+ for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;
+ base_i += SIMD_SIZE * GRAIN_SIZE) {
+ AccumT vals[GRAIN_SIZE];
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ uint i = base_i + j;
+ vals[j] =
+ (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
+ }
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ partial_sum += vals[j] * vals[j];
+ }
+ }
+
+ // SIMD-group reduction.
+ AccumT total_sum = simd_sum(partial_sum);
+
+ // RMS factor.
+ AccumT mean_square =
+ static_cast<AccumT>(total_sum) / static_cast<AccumT>(element_count);
+ AccumT rms_norm = rsqrt(mean_square + static_cast<AccumT>(epsilon));
+
+ // Normalize + scale.
+ for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;
+ base_i += SIMD_SIZE * GRAIN_SIZE) {
+ AccumT vals[GRAIN_SIZE];
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ uint i = base_i + j;
+ vals[j] =
+ (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
+ }
+
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ uint i = base_i + j;
+ if (i >= element_count)
+ continue;
+
+ AccumT normalized_high = vals[j] * rms_norm;
+
+ if (FULL_LAYER) {
+ AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) +
+ static_cast<AccumT>(scale_offset);
+ output_data[i] =
+ static_cast<OutputT>(normalized_high * scale_value_high);
+ } else {
+ OutputT normalized_low = static_cast<OutputT>(normalized_high);
+ OutputT scale_value_low = static_cast<OutputT>(
+ static_cast<AccumT>(scales_data[i]) +
+ static_cast<AccumT>(scale_offset)
+ );
+ OutputT product_low = normalized_low * scale_value_low;
+ output_data[i] = static_cast<OutputT>(product_low);
+ }
+ }
+ }
+}
+
+// Generate QK norm kernels (batched, head-tiled).
#define DEFINE_QK_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \
[[max_total_threads_per_threadgroup(1024)]] kernel void qk_norm_##SUF( \
const device IN* qkv_input [[buffer(0)]], \
@@ -266,51 +344,48 @@ FOREACH_RMS_COMBO(DEFINE_RMS_KERNEL)
constant uint& head_dim [[buffer(6)]], \
constant float& epsilon [[buffer(7)]], \
constant float& scale_offset [[buffer(8)]], \
- constant uint& active_type_mask [[buffer(9)]], \
- uint3 threadgroup_position_in_grid [[threadgroup_position_in_grid]], \
- uint3 thread_position_in_threadgroup [[thread_position_in_threadgroup]] \
+ constant uint& head_offset [[buffer(9)]], \
+ constant uint& head_count [[buffer(10)]], \
+ uint3 tgid [[threadgroup_position_in_grid]], \
+ uint lid [[thread_index_in_threadgroup]], \
+ uint3 threads_per_threadgroup [[threads_per_threadgroup]] \
) { \
- uint batch_idx = threadgroup_position_in_grid.x; \
- uint sequential_head_idx = threadgroup_position_in_grid.y; \
- uint thread_in_threadgroup = thread_position_in_threadgroup.x; \
+ const uint batch_idx = tgid.x; \
+ const uint tile_idx = tgid.y; \
\
if (batch_idx >= batch_size) \
return; \
- \
- uint qk_type, actual_head_idx, logical_head_idx; \
- if (sequential_head_idx < num_q_heads) { \
- qk_type = 0; \
- actual_head_idx = sequential_head_idx; \
- logical_head_idx = actual_head_idx; \
- } else { \
- qk_type = 1; \
- actual_head_idx = sequential_head_idx - num_q_heads; \
- logical_head_idx = num_q_heads + actual_head_idx; \
- } \
- \
- if ((active_type_mask & (1 << qk_type)) == 0) \
+ if (head_count == 0u || head_dim == 0u) \
return; \
- if (qk_type == 0 && actual_head_idx >= num_q_heads) \
+ \
+ const uint heads_per_tg = threads_per_threadgroup.x / SIMD_SIZE; \
+ if (heads_per_tg == 0u) \
return; \
- if (qk_type == 1 && actual_head_idx >= num_kv_heads) \
+ \
+ const uint simd_group_id = lid / SIMD_SIZE; \
+ const uint lane_id = lid % SIMD_SIZE; \
+ const uint head_idx = tile_idx * heads_per_tg + simd_group_id; \
+ \
+ if (head_idx >= head_count) \
return; \
\
- const uint total_heads_in_buffer = num_q_heads + 2 * num_kv_heads; \
- const uint slice_offset = batch_idx * total_heads_in_buffer * head_dim + \
- logical_head_idx * head_dim; \
+ const uint total_heads_in_buffer = num_q_heads + 2u * num_kv_heads; \
+ const uint logical_head_idx = head_offset + head_idx; \
+ if (logical_head_idx >= total_heads_in_buffer) \
+ return; \
\
- threadgroup ACC shared_sum[SIMD_SIZE]; \
+ const ulong slice_offset = \
+ (ulong)batch_idx * (ulong)total_heads_in_buffer * (ulong)head_dim + \
+ (ulong)logical_head_idx * (ulong)head_dim; \
\
- rms_norm_core<IN, SC, OUT, ACC>( \
+ qk_norm_head_core<IN, SC, OUT, ACC, FULL_LAYER>( \
qkv_input + slice_offset, \
scales, \
qkv_output + slice_offset, \
head_dim, \
epsilon, \
scale_offset, \
- shared_sum, \
- thread_in_threadgroup, \
- FULL_LAYER \
+ lane_id \
); \
}