Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Misc backends cleanup (#472)

uuuvnd19240f14570
diff --git a/crates/backend-uzu/build/cpu/compiler.rs b/crates/backend-uzu/build/cpu/compiler.rs
index c6d3cc0ae..211fd5d2c 100644
--- a/crates/backend-uzu/build/cpu/compiler.rs
+++ b/crates/backend-uzu/build/cpu/compiler.rs
@@ -411,11 +411,11 @@ impl CpuCompiler {
FunctionArgumentType::Buffer(access) => {
let (buffer_ptr, buffer_ptr_wrapper) = match access {
KernelBufferAccess::Read => (
- quote! { (&*crate::backends::cpu::BufferDowncastExt::downcast(__dsl_buffer).get()).as_ptr() },
+ quote! { (&*crate::backends::cpu::buffer::BufferDowncastExt::downcast(__dsl_buffer).get()).as_ptr() },
quote! { crate::utils::pointers::SendPtr }
),
KernelBufferAccess::ReadWrite => (
- quote! { (&mut *crate::backends::cpu::BufferDowncastExt::downcast(__dsl_buffer).get()).as_mut_ptr() },
+ quote! { (&mut *crate::backends::cpu::buffer::BufferDowncastExt::downcast(__dsl_buffer).get()).as_mut_ptr() },
quote! { crate::utils::pointers::SendPtrMut },
),
};
diff --git a/crates/backend-uzu/build/metal/bindgen/arguments.rs b/crates/backend-uzu/build/metal/bindgen/arguments.rs
index 53faf9a4c..8015425c5 100644
--- a/crates/backend-uzu/build/metal/bindgen/arguments.rs
+++ b/crates/backend-uzu/build/metal/bindgen/arguments.rs
@@ -370,7 +370,7 @@ fn emit_buffer_set(buffer: &BufferArgument) -> TokenStream {
let name = &buffer.name;
let buffer_index = buffer.buffer_index;
let unconditional_set = quote! {
- compute_encoder.set_buffer(Some(crate::backends::metal::BufferDowncastExt::downcast(#name.0)), #name.1, #buffer_index);
+ compute_encoder.set_buffer(Some(crate::backends::metal::buffer::BufferDowncastExt::downcast(#name.0)), #name.1, #buffer_index);
};
match &buffer.condition {
Some(condition) => {
diff --git a/crates/backend-uzu/build/metal/bindgen/dispatch.rs b/crates/backend-uzu/build/metal/bindgen/dispatch.rs
index 35edc980f..20310f384 100644
--- a/crates/backend-uzu/build/metal/bindgen/dispatch.rs
+++ b/crates/backend-uzu/build/metal/bindgen/dispatch.rs
@@ -85,7 +85,7 @@ fn build_indirect_dispatch(
let dispatch_code = quote! {
compute_encoder.dispatch_threadgroups_indirect(
- crate::backends::metal::BufferDowncastExt::downcast(__dsl_indirect_dispatch_buffer.0),
+ crate::backends::metal::buffer::BufferDowncastExt::downcast(__dsl_indirect_dispatch_buffer.0),
__dsl_indirect_dispatch_buffer.1,
MTLSize::new(#((#threads) as usize, )*),
);
diff --git a/crates/backend-uzu/src/backends/common/context.rs b/crates/backend-uzu/src/backends/common/context.rs
index 466fc3414..102897d1b 100644
--- a/crates/backend-uzu/src/backends/common/context.rs
+++ b/crates/backend-uzu/src/backends/common/context.rs
@@ -1,4 +1,4 @@
-use std::{env, path::Path, rc::Rc};
+use std::{path::Path, rc::Rc};
use crate::backends::common::{Allocation, AllocationPool, AllocationType, Backend, CommandBuffer};
@@ -12,10 +12,6 @@ pub trait Context: Sized {
model_path: &Path,
) -> Result<usize, <Self::Backend as Backend>::Error>;
- fn is_high_performance(&self) -> bool;
-
- fn debug_active(&self) -> bool;
-
fn create_command_buffer(
&self
) -> Result<<<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Initial, <Self::Backend as Backend>::Error>;
@@ -53,8 +49,4 @@ pub trait Context: Sized {
fn stop_capture(&self) -> Result<(), <Self::Backend as Backend>::Error>;
fn sparse_buffers_supported(&self) -> bool;
-
- fn tf32_enabled() -> bool {
- env::var("UZU_TF32").map(|v| v == "1" || v.eq_ignore_ascii_case("true")).unwrap_or(false)
- }
}
diff --git a/crates/backend-uzu/src/backends/cpu/argmax.rs b/crates/backend-uzu/src/backends/cpu/argmax.rs
deleted file mode 100644
index afa61d24b..000000000
--- a/crates/backend-uzu/src/backends/cpu/argmax.rs
+++ /dev/null
@@ -1,185 +0,0 @@
-#[cfg(target_arch = "aarch64")]
-use std::arch::aarch64::*;
-
-use num_traits::NumCast;
-
-use crate::ArrayElement;
-
-pub fn optimized_argmax<T: ArrayElement>(input: &[T]) -> usize {
- if input.is_empty() {
- return 0;
- }
-
- unsafe {
- let len = input.len();
- let ptr = input.as_ptr();
-
- let mut max_val = NumCast::from(*ptr).unwrap_or(f32::NEG_INFINITY);
- let mut max_idx = 0;
- let mut i = 1;
-
- // 32x aggressive unrolling for medium arrays with manual optimization
- while i + 32 <= len {
- // Process 4 chunks of 8 elements each for better instruction pipeline utilization
- for chunk in 0..4 {
- let base = i + chunk * 8;
- for j in 0..8 {
- let val: f32 = NumCast::from(*ptr.add(base + j)).unwrap_or(f32::NEG_INFINITY);
- if val > max_val || (val == max_val && base + j < max_idx) {
- max_val = val;
- max_idx = base + j;
- }
- }
- }
- i += 32;
- }
-
- // Handle remaining 16x chunks
- while i + 16 <= len {
- for j in 0..16 {
- let val: f32 = NumCast::from(*ptr.add(i + j)).unwrap_or(f32::NEG_INFINITY);
- if val > max_val || (val == max_val && i + j < max_idx) {
- max_val = val;
- max_idx = i + j;
- }
- }
- i += 16;
- }
-
- // Handle remaining 8x chunks
- while i + 8 <= len {
- for j in 0..8 {
- let val: f32 = NumCast::from(*ptr.add(i + j)).unwrap_or(f32::NEG_INFINITY);
- if val > max_val || (val == max_val && i + j < max_idx) {
- max_val = val;
- max_idx = i + j;
- }
- }
- i += 8;
- }
-
- // Handle remaining elements
- while i < len {
- let val: f32 = NumCast::from(*ptr.add(i)).unwrap_or(f32::NEG_INFINITY);
- if val > max_val || (val == max_val && i < max_idx) {
- max_val = val;
- max_idx = i;
- }
- i += 1;
- }
-
- max_idx
- }
-}
-
-pub fn simple_argmax<T: ArrayElement>(input: &[T]) -> usize {
- input
- .iter()
- .enumerate()
- .fold((0, f32::NEG_INFINITY), |(best_index, best_value), (index, &value)| {
- let value_f32 = NumCast::from(value).unwrap_or(f32::NEG_INFINITY);
- if value_f32 > best_value || (value_f32 == best_value && index < best_index) {
- (index, value_f32)
- } else {
- (best_index, best_value)
- }
- })
- .0
-}
-
-#[cfg(target_arch = "aarch64")]
-#[target_feature(enable = "neon")]
-pub unsafe fn neon_argmax_f32(input: &[f32]) -> usize {
- unsafe {
- if input.is_empty() {
- return 0;
- }
-
- let len = input.len();
- let ptr = input.as_ptr();
-
- if len < 4 {
- return simple_argmax(input);
- }
-
- // Initialize with first 4 elements
- let mut max_vals = vld1q_f32(ptr);
- let mut max_indices = vdupq_n_u32(0);
- let increment = vdupq_n_u32(4);
-
- // Set initial indices [0, 1, 2, 3]
- max_indices = vsetq_lane_u32(0, max_indices, 0);
- max_indices = vsetq_lane_u32(1, max_indices, 1);
- max_indices = vsetq_lane_u32(2, max_indices, 2);
- max_indices = vsetq_lane_u32(3, max_indices, 3);
-
- let mut current_indices = max_indices;
- let mut i = 4;
-
- // Process 4 elements at a time
- while i + 4 <= len {
- let vals = vld1q_f32(ptr.add(i));
- current_indices = vaddq_u32(current_indices, increment);
-
- // Compare values
- let mask = vcgtq_f32(vals, max_vals);
-
- // Update max values and indices where vals > max_vals
- max_vals = vbslq_f32(mask, vals, max_vals);
- max_indices = vbslq_u32(mask, current_indices, max_indices);
-
- i += 4;
- }
-
- // Extract the maximum from the vector
- let max_vals_array: [f32; 4] = std::mem::transmute(max_vals);
- let max_indices_array: [u32; 4] = std::mem::transmute(max_indices);
-
- let mut global_max = max_vals_array[0];
- let mut global_max_idx = max_indices_array[0] as usize;
-
- for j in 1..4 {
- if max_vals_array[j] > global_max
- || (max_vals_array[j] == global_max && (max_indices_array[j] as usize) < global_max_idx)
- {
- global_max = max_vals_array[j];
- global_max_idx = max_indices_array[j] as usize;
- }
- }
-
- // Handle remaining elements
- while i < len {
- let val = *ptr.add(i);
- if val > global_max || (val == global_max && i < global_max_idx) {
- global_max = val;
- global_max_idx = i;
- }
- i += 1;
- }
-
- global_max_idx
- }
-}
-
-#[cfg(target_arch = "aarch64")]
-pub fn neon_optimized_argmax<T: ArrayElement>(input: &[T]) -> usize {
- // Check if NEON is available at runtime
- if !std::arch::is_aarch64_feature_detected!("neon") {
- return simple_argmax(input);
- }
-
- // For f32, use direct NEON implementation
- if std::any::TypeId::of::<T>() == std::any::TypeId::of::<f32>() {
- unsafe {
- let f32_slice = std::slice::from_raw_parts(input.as_ptr() as *const f32, input.len());
- return neon_argmax_f32(f32_slice);
- }
- }
-
- // For other types, fall back to simple
- simple_argmax(input)
-}
-
-#[cfg(test)]
-#[path = "../../../tests/unit/backends/cpu/argmax_test.rs"]
-mod tests;
diff --git a/crates/backend-uzu/src/backends/cpu/command_buffer.rs b/crates/backend-uzu/src/backends/cpu/command_buffer.rs
index 0e81bcd14..de5be1dec 100644
--- a/crates/backend-uzu/src/backends/cpu/command_buffer.rs
+++ b/crates/backend-uzu/src/backends/cpu/command_buffer.rs
@@ -9,7 +9,7 @@ use crate::{
AccessFlags, Buffer, BufferRangeMut, BufferRangeRef, CommandBuffer, CommandBufferCompleted,
CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
},
- cpu::{BufferDowncastExt, Cpu, error::CpuError},
+ cpu::{Cpu, buffer::BufferDowncastExt, error::CpuError},
},
utils::pointers::{SendPtr, SendPtrMut},
};
diff --git a/crates/backend-uzu/src/backends/cpu/context.rs b/crates/backend-uzu/src/backends/cpu/context.rs
index b252bc06c..78d340c86 100644
--- a/crates/backend-uzu/src/backends/cpu/context.rs
+++ b/crates/backend-uzu/src/backends/cpu/context.rs
@@ -35,14 +35,6 @@ impl Context for CpuContext {
Ok(1)
}
- fn is_high_performance(&self) -> bool {
- false
- }
-
- fn debug_active(&self) -> bool {
- false
- }
-
fn create_buffer(
&self,
size: usize,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
index 50e90c825..a688e3a70 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
@@ -10,7 +10,8 @@ use crate::{
},
},
cpu::{
- BufferDowncastExt, Cpu, context::CpuContext, error::CpuError, kernel::matmul::quant::encode_quantized_gemm,
+ Cpu, buffer::BufferDowncastExt, context::CpuContext, error::CpuError,
+ kernel::matmul::quant::encode_quantized_gemm,
},
},
dispatch_dtype,
diff --git a/crates/backend-uzu/src/backends/cpu/mod.rs b/crates/backend-uzu/src/backends/cpu/mod.rs
index 2f3f6086d..b98cb0309 100644
--- a/crates/backend-uzu/src/backends/cpu/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/mod.rs
@@ -7,7 +7,4 @@ mod error;
mod kernel;
mod sparse;
-pub mod argmax;
-
pub use backend::Cpu;
-pub use buffer::BufferDowncastExt;
diff --git a/crates/backend-uzu/src/backends/metal/command_buffer.rs b/crates/backend-uzu/src/backends/metal/command_buffer.rs
index 3b939927c..3a29cc4df 100644
--- a/crates/backend-uzu/src/backends/metal/command_buffer.rs
+++ b/crates/backend-uzu/src/backends/metal/command_buffer.rs
@@ -6,7 +6,7 @@ use metal::{
};
use objc2::{Message, rc::Retained, runtime::ProtocolObject};
-use super::{BufferDowncastExt, Metal};
+use super::{Metal, buffer::BufferDowncastExt};
use crate::{
backends::{
common::{
diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rs
index b42be7720..61b28844d 100644
--- a/crates/backend-uzu/src/backends/metal/context.rs
+++ b/crates/backend-uzu/src/backends/metal/context.rs
@@ -16,11 +16,10 @@ use metal::{
use objc2::{rc::Retained, runtime::ProtocolObject};
use super::{
- DeviceExt, Metal,
- device_capabilities::MetalDeviceCapabilities,
+ Metal,
error::MetalError,
kernel,
- metal_extensions::{DeviceGeneration, LibraryPipelineExtensions},
+ metal_extensions::{DeviceExt, LibraryPipelineExtensions},
};
use crate::{
backends::{
@@ -42,7 +41,7 @@ pub struct MetalContext {
timeline_value: AtomicU64,
allocator: Rc<Allocator<Metal>>,
peak_memory_usage: RefCell<usize>,
- device_capabilities: MetalDeviceCapabilities,
+ gpu_core_count: u32,
library: Retained<ProtocolObject<dyn MTLLibrary>>,
pipeline_cache: RefCell<HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>>,
sparse_heap_pool: RefCell<MetalSparseHeapPool>,
@@ -52,15 +51,15 @@ pub struct MetalContext {
}
impl MetalContext {
+ pub fn supports_mxu(&self) -> bool {
+ self.device.supports_mxu()
+ }
+
pub(super) fn update_peak_memory_usage(&self) {
let mut peak_memory_usage_borrow = self.peak_memory_usage.borrow_mut();
*peak_memory_usage_borrow = peak_memory_usage_borrow.max(self.device.current_allocated_size());
}
- pub fn device_generation(&self) -> DeviceGeneration {
- self.device_capabilities.generation
- }
-
pub fn compute_pipeline_state(
&self,
cache_key: &str,
@@ -131,7 +130,7 @@ impl Context for MetalContext {
.new_library_with_data(kernel::MTLB)
.map_err(|nserror| MetalError::CannotCreateLibrary(nserror.to_string()))?;
- let device_capabilities = MetalDeviceCapabilities::from_device(&device);
+ let gpu_core_count = device.gpu_core_count();
let page_size = MTLSparsePageSize::KB256;
let heap_capacity = 64 * 4 * page_size.in_bytes();
@@ -148,7 +147,7 @@ impl Context for MetalContext {
timeline_value: AtomicU64::new(0),
allocator: Allocator::new(weak_self.clone()),
peak_memory_usage: RefCell::new(0),
- device_capabilities,
+ gpu_core_count,
library,
pipeline_cache: RefCell::new(HashMap::new()),
sparse_heap_pool: RefCell::new(sparse_pool),
@@ -158,15 +157,11 @@ impl Context for MetalContext {
}))
}
- fn is_high_performance(&self) -> bool {
- self.device_capabilities.is_high_performance()
- }
-
fn recommended_async_batch_size(
&self,
model_path: &Path,
) -> Result<usize, MetalError> {
- let cores = self.device_capabilities.gpu_core_count;
+ let cores = self.gpu_core_count;
let model_size = ModelSize::from_path(model_path)?;
Ok(match (model_size, cores) {
(ModelSize::Large, c) if c > 20 => 32,
@@ -178,11 +173,6 @@ impl Context for MetalContext {
})
}
- fn debug_active(&self) -> bool {
- let upper = std::env::var("METAL_DEVICE_WRAPPER_TYPE").unwrap_or_default().to_ascii_uppercase();
- matches!(upper.as_str(), "1" | "YES" | "TRUE")
- }
-
fn create_buffer(
&self,
size: usize,
diff --git a/crates/backend-uzu/src/backends/metal/device_capabilities.rs b/crates/backend-uzu/src/backends/metal/device_capabilities.rs
deleted file mode 100644
index 5a74d6144..000000000
--- a/crates/backend-uzu/src/backends/metal/device_capabilities.rs
+++ /dev/null
@@ -1,71 +0,0 @@
-use bytesize::ByteSize;
-use metal::{MTLDevice, MTLDeviceExt};
-use objc2::runtime::ProtocolObject;
-
-use super::metal_extensions::{DeviceExt, DeviceGeneration};
-
-const HIGH_PERFORMANCE_CORE_THRESHOLD: u32 = 10;
-
-#[derive(Debug, Clone)]
-pub struct MetalDeviceCapabilities {
- pub generation: DeviceGeneration,
- pub family_name: String,
- pub gpu_core_count: u32,
- pub max_threadgroup_memory: ByteSize,
- pub shared_memory_size: ByteSize,
- pub supports_simd_group: bool,
- pub supports_simd_group_matrix: bool,
- pub supports_simd_reduction: bool,
- pub supports_simd_shuffle_and_fill: bool,
- pub supports_simd_shuffles_and_broadcast: bool,
- pub supports_mxu: bool,
- pub supports_tls: bool,
-}
-
-impl MetalDeviceCapabilities {
- pub fn from_device(device: &ProtocolObject<dyn MTLDevice>) -> Self {
- let architecture_name = device.architecture().name();
- let generation = Self::parse_generation(&architecture_name);
- let max_threadgroup_memory = ByteSize(device.max_threadgroup_memory_length() as u64);
-
- Self {
- generation,
- family_name: device.family_name(),
- gpu_core_count: device.gpu_core_count(),
- max_threadgroup_memory,
- shared_memory_size: device.shared_memory_size(),
- supports_simd_group: device.supports_simd_group(),
- supports_simd_group_matrix: device.supports_simd_group_matrix(),
- supports_simd_reduction: device.supports_simd_reduction(),
- supports_simd_shuffle_and_fill: device.supports_simd_shuffle_and_fill(),
- supports_simd_shuffles_and_broadcast: device.supports_simd_shuffles_and_broadcast(),
- supports_mxu: device.supports_mxu(),
- supports_tls: device.supports_tls(),
- }
- }
-
- pub fn is_high_performance(&self) -> bool {
- self.gpu_core_count > HIGH_PERFORMANCE_CORE_THRESHOLD
- }
-
- fn parse_generation(architecture_name: &str) -> DeviceGeneration {
- let Some(suffix_start) = architecture_name.find("applegpu_g") else {
- return DeviceGeneration::Unknown(0);
- };
- let after_prefix = &architecture_name[suffix_start + "applegpu_g".len()..];
-
- let digit_end = after_prefix
- .char_indices()
- .find(|(_, character)| !character.is_ascii_digit())
- .map_or(after_prefix.len(), |(index, _)| index);
-
- if digit_end == 0 {
- return DeviceGeneration::Unknown(0);
- }
-
- match after_prefix[..digit_end].parse::<u8>() {
- Ok(n) => DeviceGeneration::from_generation_number(n),
- Err(_) => DeviceGeneration::Unknown(0),
- }
- }
-}
diff --git a/crates/backend-uzu/src/backends/metal/metal_extensions/device_extensions.rs b/crates/backend-uzu/src/backends/metal/metal_extensions/device_extensions.rs
index aca8e93a7..9066dd628 100644
--- a/crates/backend-uzu/src/backends/metal/metal_extensions/device_extensions.rs
+++ b/crates/backend-uzu/src/backends/metal/metal_extensions/device_extensions.rs
@@ -1,16 +1,12 @@
use core::mem::transmute;
-use bytesize::ByteSize;
use metal::MTLDevice;
use objc2::{
Message,
ffi::objc_msgSend,
- msg_send,
- rc::Retained,
runtime::{NSObjectProtocol, ProtocolObject, Sel},
sel,
};
-use objc2_foundation::NSString;
// Used to bypass objc2's debug-mode class validation for proxy objects like
// `CaptureMTLDevice` which forward messages at the ObjC runtime level (via
@@ -28,54 +24,7 @@ unsafe fn raw_msg_send<T: Message + ?Sized, R>(
unsafe { send(object_pointer, sel) }
}
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum DeviceGeneration {
- Gen13,
- Gen14,
- Gen15,
- Gen16,
- Gen17,
- Gen18,
- Unknown(u8),
-}
-
-impl DeviceGeneration {
- pub fn generation_number(&self) -> u8 {
- match self {
- Self::Gen13 => 13,
- Self::Gen14 => 14,
- Self::Gen15 => 15,
- Self::Gen16 => 16,
- Self::Gen17 => 17,
- Self::Gen18 => 18,
- Self::Unknown(n) => *n,
- }
- }
-
- pub(crate) fn from_generation_number(generation_number: u8) -> Self {
- match generation_number {
- 13 => Self::Gen13,
- 14 => Self::Gen14,
- 15 => Self::Gen15,
- 16 => Self::Gen16,
- 17 => Self::Gen17,
- 18 => Self::Gen18,
- _ => Self::Unknown(generation_number),
- }
- }
-}
-
pub trait DeviceExt: MTLDevice + Message + NSObjectProtocol + Sized {
- /// Human-readable chip name, e.g. "M2 Max", "M4 Pro", "A17 Pro".
- fn family_name(&self) -> String {
- if self.respondsToSelector(sel!(familyName)) {
- let family_name: Retained<NSString> = unsafe { msg_send![self, familyName] };
- family_name.to_string()
- } else {
- "Unknown".into()
- }
- }
-
/// Number of GPU shader cores.
fn gpu_core_count(&self) -> u32 {
if self.respondsToSelector(sel!(gpuCoreCount)) {
@@ -85,60 +34,6 @@ pub trait DeviceExt: MTLDevice + Message + NSObjectProtocol + Sized {
}
}
- /// Total unified (shared) memory.
- fn shared_memory_size(&self) -> ByteSize {
- if self.respondsToSelector(sel!(sharedMemorySize)) {
- ByteSize(unsafe { raw_msg_send(self, sel!(sharedMemorySize)) })
- } else {
- ByteSize::gib(8)
- }
- }
-
- /// Whether the GPU supports SIMD group operations.
- fn supports_simd_group(&self) -> bool {
- if self.respondsToSelector(sel!(supportsSIMDGroup)) {
- unsafe { raw_msg_send(self, sel!(supportsSIMDGroup)) }
- } else {
- false
- }
- }
-
- /// Whether the GPU supports `simdgroup_matrix`.
- fn supports_simd_group_matrix(&self) -> bool {
- if self.respondsToSelector(sel!(supportsSIMDGroupMatrix)) {
- unsafe { raw_msg_send(self, sel!(supportsSIMDGroupMatrix)) }
- } else {
- false
- }
- }
-
- /// Whether the GPU supports SIMD reduction operations.
- fn supports_simd_reduction(&self) -> bool {
- if self.respondsToSelector(sel!(supportsSIMDReduction)) {
- unsafe { raw_msg_send(self, sel!(supportsSIMDReduction)) }
- } else {
- false
- }
- }
-
- /// Whether the GPU supports SIMD shuffle-and-fill operations.
- fn supports_simd_shuffle_and_fill(&self) -> bool {
- if self.respondsToSelector(sel!(supportsSIMDShuffleAndFill)) {
- unsafe { raw_msg_send(self, sel!(supportsSIMDShuffleAndFill)) }
- } else {
- false
- }
- }
-
- /// Whether the GPU supports SIMD shuffles and broadcast.
- fn supports_simd_shuffles_and_broadcast(&self) -> bool {
- if self.respondsToSelector(sel!(supportsSIMDShufflesAndBroadcast)) {
- unsafe { raw_msg_send(self, sel!(supportsSIMDShufflesAndBroadcast)) }
- } else {
- false
- }
- }
-
/// Whether the GPU has a Matrix eXtension Unit (neural accelerator for compute).
/// True on M5+ (Gen18+), false on M1-M4.
fn supports_mxu(&self) -> bool {
@@ -149,15 +44,6 @@ pub trait DeviceExt: MTLDevice + Message + NSObjectProtocol + Sized {
}
}
- /// Whether the GPU supports Thread-Local Storage.
- fn supports_tls(&self) -> bool {
- if self.respondsToSelector(sel!(supportsTLS)) {
- unsafe { raw_msg_send(self, sel!(supportsTLS)) }
- } else {
- false
- }
- }
-
/// Whether the GPU supports placement sparse resources.
fn supports_placement_sparse_resources(&self) -> bool {
if self.respondsToSelector(sel!(supportsPlacementSparse)) {
diff --git a/crates/backend-uzu/src/backends/metal/metal_extensions/mod.rs b/crates/backend-uzu/src/backends/metal/metal_extensions/mod.rs
index 6ddbead8f..3476c3a4f 100644
--- a/crates/backend-uzu/src/backends/metal/metal_extensions/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/metal_extensions/mod.rs
@@ -7,7 +7,7 @@ mod sparse_page_size_extensions;
pub use compute_command_encoder_extensions_set_value::ComputeEncoderSetValue;
pub use data_type::MetalDataTypeExt;
-pub use device_extensions::{DeviceExt, DeviceGeneration};
+pub use device_extensions::DeviceExt;
pub use function_constant_values_extensions_set_value::FunctionConstantValuesSetValue;
pub use library_extensions_pipeline::LibraryPipelineExtensions;
pub use sparse_page_size_extensions::SparsePageSizeExt;
diff --git a/crates/backend-uzu/src/backends/metal/mod.rs b/crates/backend-uzu/src/backends/metal/mod.rs
index 955ae6cc1..e84dd4691 100644
--- a/crates/backend-uzu/src/backends/metal/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/mod.rs
@@ -3,15 +3,11 @@ mod buffer;
mod command_buffer;
mod context;
mod dense_buffer;
-mod device_capabilities;
mod error;
mod kernel;
mod metal_extensions;
mod sparse;
pub use backend::Metal;
-pub use buffer::BufferDowncastExt;
pub use context::MetalContext;
-pub use device_capabilities::MetalDeviceCapabilities;
-pub use kernel::matmul::{GemmDispatchPath, GemmKernel};
-pub use metal_extensions::{DeviceExt, DeviceGeneration};
+pub use kernel::matmul::GemmDispatchPath;
diff --git a/crates/backend-uzu/src/backends/mod.rs b/crates/backend-uzu/src/backends/mod.rs
index dbf208ad2..8cf23bf33 100644
--- a/crates/backend-uzu/src/backends/mod.rs
+++ b/crates/backend-uzu/src/backends/mod.rs
@@ -6,10 +6,11 @@ pub mod metal;
macro_rules! select_backend {
($expr:expr, $unk:expr) => {{
- let mut default = "cpu";
- if cfg!(metal_backend) {
- default = "metal";
- }
+ let default = if cfg!(metal_backend) {
+ "metal"
+ } else {
+ "cpu"
+ };
match std::env::var("UZU_BACKEND").map(|s| s.to_lowercase()).as_deref().unwrap_or(default) {
"cpu" => {
diff --git a/crates/backend-uzu/src/language_model/mod.rs b/crates/backend-uzu/src/language_model/mod.rs
index 881a2d811..187ac3bee 100644
--- a/crates/backend-uzu/src/language_model/mod.rs
+++ b/crates/backend-uzu/src/language_model/mod.rs
@@ -6,8 +6,5 @@ pub mod language_model_generator_context;
pub mod result;
pub mod rng;
-#[cfg(feature = "tracing")]
-pub mod sampler;
-
// Re-export main types
pub use language_model_generator::{LanguageModelGenerator, LanguageModelGeneratorTrait};
diff --git a/crates/backend-uzu/src/language_model/sampler.rs b/crates/backend-uzu/src/language_model/sampler.rs
deleted file mode 100644
index 9c371232a..000000000
--- a/crates/backend-uzu/src/language_model/sampler.rs
+++ /dev/null
@@ -1,47 +0,0 @@
-use ndarray::{ArrayView, IxDyn};
-
-#[cfg(target_arch = "aarch64")]
-use crate::backends::cpu::argmax::neon_optimized_argmax;
-use crate::{ArrayElement, backends::cpu::argmax::simple_argmax};
-
-pub trait LogitsSampler {
- fn sample<T: ArrayElement>(
- &self,
- logits: ArrayView<T, IxDyn>,
- ) -> Vec<u64>;
-}
-
-pub struct ArgmaxSampler {}
-
-impl LogitsSampler for ArgmaxSampler {
- fn sample<T: ArrayElement>(
- &self,
- logits: ArrayView<T, IxDyn>,
- ) -> Vec<u64> {
- let mut result = Vec::<u64>::with_capacity(logits.shape()[0]);
-
- for row in logits.rows() {
- let max_index = if let Some(slice) = row.as_slice_memory_order() {
- #[cfg(target_arch = "aarch64")]
- {
- if slice.len() >= 16 {
- neon_optimized_argmax(slice)
- } else {
- simple_argmax(slice)
- }
- }
- #[cfg(not(target_arch = "aarch64"))]
- {
- simple_argmax(slice)
- }
- } else {
- let data: Vec<T> = row.iter().cloned().collect();
- simple_argmax(&data)
- };
-
- result.push(max_index as u64);
- }
-
- result
- }
-}
diff --git a/crates/backend-uzu/src/lib.rs b/crates/backend-uzu/src/lib.rs
index 6e42bc8d3..b20d02be8 100644
--- a/crates/backend-uzu/src/lib.rs
+++ b/crates/backend-uzu/src/lib.rs
@@ -37,10 +37,7 @@ pub mod _private {
forward_pass::{
cache_layers::CacheLayers, kv_cache_layer::KVCacheLayer, token_inputs::TokenInputs, traces::ActivationTrace,
},
- language_model::{
- language_model_generator_context::LanguageModelGeneratorContext,
- sampler::{ArgmaxSampler, LogitsSampler},
- },
+ language_model::language_model_generator_context::LanguageModelGeneratorContext,
parameters::{ParameterLoaderError, ParameterTree},
};
}
diff --git a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
index 715ad141e..834cebf31 100644
--- a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
+++ b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
@@ -15,8 +15,8 @@ use std::{
use backend_uzu::{
_private::{
- ActivationTrace, AnyModelConfig, ArgmaxSampler, CacheLayers, Classifier, DecoderDecodeInput, KVCacheLayer,
- LanguageModelGeneratorContext, LogitsSampler, ParameterLoaderError, ParameterTree, TokenInputs,
+ ActivationTrace, AnyModelConfig, CacheLayers, Classifier, DecoderDecodeInput, KVCacheLayer,
+ LanguageModelGeneratorContext, ParameterLoaderError, ParameterTree, TokenInputs,
},
Array, ArrayElement, DataType, ParameterLoader,
backends::common::{Allocation, AllocationType, Backend, Context, Encoder},
@@ -28,10 +28,39 @@ use backend_uzu::{
},
};
use half::{bf16, f16};
-use ndarray::{IxDyn, s};
+use ndarray::{ArrayView, IxDyn, s};
use num_traits::NumCast;
use crate::common;
+
+fn argmax<T: ArrayElement>(input: &[T]) -> usize {
+ input
+ .iter()
+ .enumerate()
+ .fold((0, f32::NEG_INFINITY), |(best_index, best_value), (index, &value)| {
+ let value_f32 = NumCast::from(value).unwrap_or(f32::NEG_INFINITY);
+ if value_f32 > best_value || (value_f32 == best_value && index < best_index) {
+ (index, value_f32)
+ } else {
+ (best_index, best_value)
+ }
+ })
+ .0
+}
+
+fn sample_argmax<T: ArrayElement>(logits: ArrayView<T, IxDyn>) -> Vec<u64> {
+ let mut result = Vec::with_capacity(logits.shape()[0]);
+ for row in logits.rows() {
+ let max_index = if let Some(slice) = row.as_slice_memory_order() {
+ argmax(slice)
+ } else {
+ let data: Vec<T> = row.iter().copied().collect();
+ argmax(&data)
+ };
+ result.push(max_index as u64);
+ }
+ result
+}
// ============================================================================
// Validation Types
// ============================================================================
@@ -1006,15 +1035,10 @@ impl<B: Backend> TraceValidator<B> {
fn get_tokens_from_logits(logits: &Array<B>) -> Vec<u64> {
let data_type = logits.data_type();
match data_type {
- DataType::F16 => Self::get_tokens_from_logits_of_type::<f16>(logits),
- DataType::BF16 => Self::get_tokens_from_logits_of_type::<bf16>(logits),
- DataType::F32 => Self::get_tokens_from_logits_of_type::<f32>(logits),
+ DataType::F16 => sample_argmax(logits.as_view::<f16>()),
+ DataType::BF16 => sample_argmax(logits.as_view::<bf16>()),
+ DataType::F32 => sample_argmax(logits.as_view::<f32>()),
_ => panic!("Unsupported data type: {:?}", data_type),
}
}
-
- fn get_tokens_from_logits_of_type<Precision: ArrayElement>(logits: &Array<B>) -> Vec<u64> {
- let sampler = ArgmaxSampler {};
- sampler.sample(logits.as_view::<Precision>())
- }
}
diff --git a/crates/backend-uzu/tests/unit/backends/cpu/argmax_test.rs b/crates/backend-uzu/tests/unit/backends/cpu/argmax_test.rs
deleted file mode 100644
index a3985cd32..000000000
--- a/crates/backend-uzu/tests/unit/backends/cpu/argmax_test.rs
+++ /dev/null
@@ -1,110 +0,0 @@
-#![cfg(target_arch = "aarch64")]
-
-use std::time::Instant;
-
-use super::*;
-
-#[test]
-fn test_neon_argmax_correctness() {
- // Single deterministic 64k array test with random values
- const SIZE: usize = 64 * 1024;
- const SEED: u64 = 12345;
-
- // Simple deterministic random number generator
- let mut rng_state = SEED;
- let mut data = Vec::with_capacity(SIZE);
-
- for _ in 0..SIZE {
- rng_state = rng_state.wrapping_mul(1103515245).wrapping_add(12345);
- let random_f32 = (rng_state as f32) / (u64::MAX as f32);
- data.push(random_f32);
- }
-
- // Set a known max at position 42_000
- data[42_000] = f32::MAX;
-
- // Compute expected result with naive method
- let expected = simple_argmax(&data);
- // Compare NEON result to expected
- let neon_result = neon_optimized_argmax(&data);
-
- assert_eq!(expected, neon_result, "64k array test failed: expected {}, got {}", expected, neon_result);
- assert_eq!(expected, 42_000, "Expected max should be at index 42_000");
-}
-
-#[test]
-fn test_neon_performance_128k() {
- const SEQ_LEN: usize = 128 * 1024;
- let batch_size = 2;
- const SEED: u64 = 67890;
-
- // Generate random data with deterministic seed
- let mut rng_state = SEED;
- let mut data = Vec::with_capacity(batch_size * SEQ_LEN);
-
- for _ in 0..(batch_size * SEQ_LEN) {
- rng_state = rng_state.wrapping_mul(1103515245).wrapping_add(12345);
- let random_f32 = (rng_state as f32) / (u64::MAX as f32);
- data.push(random_f32);
- }
-
- // Set known max positions
- let expected_indices = vec![42_000, 100_000];
- for (batch_idx, &max_pos) in expected_indices.iter().enumerate() {
- let offset = batch_idx * SEQ_LEN + max_pos;
- data[offset] = f32::MAX;
- }
-
- // Test each batch separately
- for (batch_idx, &expected_idx) in expected_indices.iter().enumerate() {
- let start_offset = batch_idx * SEQ_LEN;
- let end_offset = start_offset + SEQ_LEN;
- let batch_data = &data[start_offset..end_offset];
-
- // Test with NEON implementation
- let start = Instant::now();
- let neon_result = neon_optimized_argmax(batch_data);
- let neon_duration = start.elapsed();
-
- // Test with simple implementation
- let start = Instant::now();
- let simple_result = simple_argmax(batch_data);
- let simple_duration = start.elapsed();
-
- // Test with optimized implementation (manual unrolling)
- let start = Instant::now();
- let optimized_result = optimized_argmax(batch_data);
- let optimized_duration = start.elapsed();
-
- if batch_idx == 0 {
- println!("128k argmax performance comparison:");
- println!("NEON: {:.3} ms", neon_duration.as_secs_f64() * 1000.0);
- println!("Simple: {:.3} ms", simple_duration.as_secs_f64() * 1000.0);
- println!("Optimized: {:.3} ms", optimized_duration.as_secs_f64() * 1000.0);
-
- if simple_duration.as_nanos() > 0 && neon_duration.as_nanos() > 0 {
- let speedup_simple = simple_duration.as_secs_f64() / neon_duration.as_secs_f64();
- let speedup_optimized = optimized_duration.as_secs_f64() / neon_duration.as_secs_f64();
- println!("NEON vs Simple speedup: {:.2}x", speedup_simple);
- println!("NEON vs Optimized speedup: {:.2}x", speedup_optimized);
- }
- }
-
- // Verify all methods give the same result
- assert_eq!(
- neon_result, expected_idx,
- "NEON: Batch {} argmax mismatch: expected {}, got {}",
- batch_idx, expected_idx, neon_result
- );
- assert_eq!(
- simple_result, expected_idx,
- "Simple: Batch {} argmax mismatch: expected {}, got {}",
- batch_idx, expected_idx, simple_result
- );
- assert_eq!(
- optimized_result, expected_idx,
- "Optimized: Batch {} argmax mismatch: expected {}, got {}",
- batch_idx, expected_idx, optimized_result
- );
- }
-}
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
index 5428d5f67..744c36d19 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
@@ -9,7 +9,7 @@ use backend_uzu::{
Backend, Context,
kernel::{Kernels, matmul::MatmulKernel},
},
- metal::{DeviceExt, GemmDispatchPath, Metal, MetalContext},
+ metal::{GemmDispatchPath, Metal, MetalContext},
},
};
use half::bf16;
@@ -23,7 +23,7 @@ use crate::common::{
fn gemm_paths_for_hw(context: &MetalContext) -> Vec<GemmDispatchPath> {
let mut paths = vec![GemmDispatchPath::Simdgroup];
- if context.device.supports_mxu() {
+ if context.supports_mxu() {
paths.push(GemmDispatchPath::Mxu);
}
paths