A high-performance inference engine for AI models
Misc backends cleanup (#472)
uuuvnd19240f14570
diff --git a/crates/backend-uzu/build/cpu/compiler.rs b/crates/backend-uzu/build/cpu/compiler.rsindex c6d3cc0ae..211fd5d2c 100644--- a/crates/backend-uzu/build/cpu/compiler.rs+++ b/crates/backend-uzu/build/cpu/compiler.rs@@ -411,11 +411,11 @@ impl CpuCompiler {FunctionArgumentType::Buffer(access) => {let (buffer_ptr, buffer_ptr_wrapper) = match access {KernelBufferAccess::Read => (- quote! { (&*crate::backends::cpu::BufferDowncastExt::downcast(__dsl_buffer).get()).as_ptr() },+ quote! { (&*crate::backends::cpu::buffer::BufferDowncastExt::downcast(__dsl_buffer).get()).as_ptr() },quote! { crate::utils::pointers::SendPtr }),KernelBufferAccess::ReadWrite => (- quote! { (&mut *crate::backends::cpu::BufferDowncastExt::downcast(__dsl_buffer).get()).as_mut_ptr() },+ quote! { (&mut *crate::backends::cpu::buffer::BufferDowncastExt::downcast(__dsl_buffer).get()).as_mut_ptr() },quote! { crate::utils::pointers::SendPtrMut },),};diff --git a/crates/backend-uzu/build/metal/bindgen/arguments.rs b/crates/backend-uzu/build/metal/bindgen/arguments.rsindex 53faf9a4c..8015425c5 100644--- a/crates/backend-uzu/build/metal/bindgen/arguments.rs+++ b/crates/backend-uzu/build/metal/bindgen/arguments.rs@@ -370,7 +370,7 @@ fn emit_buffer_set(buffer: &BufferArgument) -> TokenStream {let name = &buffer.name;let buffer_index = buffer.buffer_index;let unconditional_set = quote! {- compute_encoder.set_buffer(Some(crate::backends::metal::BufferDowncastExt::downcast(#name.0)), #name.1, #buffer_index);+ compute_encoder.set_buffer(Some(crate::backends::metal::buffer::BufferDowncastExt::downcast(#name.0)), #name.1, #buffer_index);};match &buffer.condition {Some(condition) => {diff --git a/crates/backend-uzu/build/metal/bindgen/dispatch.rs b/crates/backend-uzu/build/metal/bindgen/dispatch.rsindex 35edc980f..20310f384 100644--- a/crates/backend-uzu/build/metal/bindgen/dispatch.rs+++ b/crates/backend-uzu/build/metal/bindgen/dispatch.rs@@ -85,7 +85,7 @@ fn build_indirect_dispatch(let dispatch_code = quote! {compute_encoder.dispatch_threadgroups_indirect(- crate::backends::metal::BufferDowncastExt::downcast(__dsl_indirect_dispatch_buffer.0),+ crate::backends::metal::buffer::BufferDowncastExt::downcast(__dsl_indirect_dispatch_buffer.0),__dsl_indirect_dispatch_buffer.1,MTLSize::new(#((#threads) as usize, )*),);diff --git a/crates/backend-uzu/src/backends/common/context.rs b/crates/backend-uzu/src/backends/common/context.rsindex 466fc3414..102897d1b 100644--- a/crates/backend-uzu/src/backends/common/context.rs+++ b/crates/backend-uzu/src/backends/common/context.rs@@ -1,4 +1,4 @@-use std::{env, path::Path, rc::Rc};+use std::{path::Path, rc::Rc};use crate::backends::common::{Allocation, AllocationPool, AllocationType, Backend, CommandBuffer};@@ -12,10 +12,6 @@ pub trait Context: Sized {model_path: &Path,) -> Result<usize, <Self::Backend as Backend>::Error>;- fn is_high_performance(&self) -> bool;-- fn debug_active(&self) -> bool;-fn create_command_buffer(&self) -> Result<<<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Initial, <Self::Backend as Backend>::Error>;@@ -53,8 +49,4 @@ pub trait Context: Sized {fn stop_capture(&self) -> Result<(), <Self::Backend as Backend>::Error>;fn sparse_buffers_supported(&self) -> bool;-- fn tf32_enabled() -> bool {- env::var("UZU_TF32").map(|v| v == "1" || v.eq_ignore_ascii_case("true")).unwrap_or(false)- }}diff --git a/crates/backend-uzu/src/backends/cpu/argmax.rs b/crates/backend-uzu/src/backends/cpu/argmax.rsdeleted file mode 100644index afa61d24b..000000000--- a/crates/backend-uzu/src/backends/cpu/argmax.rs+++ /dev/null@@ -1,185 +0,0 @@-#[cfg(target_arch = "aarch64")]-use std::arch::aarch64::*;--use num_traits::NumCast;--use crate::ArrayElement;--pub fn optimized_argmax<T: ArrayElement>(input: &[T]) -> usize {- if input.is_empty() {- return 0;- }-- unsafe {- let len = input.len();- let ptr = input.as_ptr();-- let mut max_val = NumCast::from(*ptr).unwrap_or(f32::NEG_INFINITY);- let mut max_idx = 0;- let mut i = 1;-- // 32x aggressive unrolling for medium arrays with manual optimization- while i + 32 <= len {- // Process 4 chunks of 8 elements each for better instruction pipeline utilization- for chunk in 0..4 {- let base = i + chunk * 8;- for j in 0..8 {- let val: f32 = NumCast::from(*ptr.add(base + j)).unwrap_or(f32::NEG_INFINITY);- if val > max_val || (val == max_val && base + j < max_idx) {- max_val = val;- max_idx = base + j;- }- }- }- i += 32;- }-- // Handle remaining 16x chunks- while i + 16 <= len {- for j in 0..16 {- let val: f32 = NumCast::from(*ptr.add(i + j)).unwrap_or(f32::NEG_INFINITY);- if val > max_val || (val == max_val && i + j < max_idx) {- max_val = val;- max_idx = i + j;- }- }- i += 16;- }-- // Handle remaining 8x chunks- while i + 8 <= len {- for j in 0..8 {- let val: f32 = NumCast::from(*ptr.add(i + j)).unwrap_or(f32::NEG_INFINITY);- if val > max_val || (val == max_val && i + j < max_idx) {- max_val = val;- max_idx = i + j;- }- }- i += 8;- }-- // Handle remaining elements- while i < len {- let val: f32 = NumCast::from(*ptr.add(i)).unwrap_or(f32::NEG_INFINITY);- if val > max_val || (val == max_val && i < max_idx) {- max_val = val;- max_idx = i;- }- i += 1;- }-- max_idx- }-}--pub fn simple_argmax<T: ArrayElement>(input: &[T]) -> usize {- input- .iter()- .enumerate()- .fold((0, f32::NEG_INFINITY), |(best_index, best_value), (index, &value)| {- let value_f32 = NumCast::from(value).unwrap_or(f32::NEG_INFINITY);- if value_f32 > best_value || (value_f32 == best_value && index < best_index) {- (index, value_f32)- } else {- (best_index, best_value)- }- })- .0-}--#[cfg(target_arch = "aarch64")]-#[target_feature(enable = "neon")]-pub unsafe fn neon_argmax_f32(input: &[f32]) -> usize {- unsafe {- if input.is_empty() {- return 0;- }-- let len = input.len();- let ptr = input.as_ptr();-- if len < 4 {- return simple_argmax(input);- }-- // Initialize with first 4 elements- let mut max_vals = vld1q_f32(ptr);- let mut max_indices = vdupq_n_u32(0);- let increment = vdupq_n_u32(4);-- // Set initial indices [0, 1, 2, 3]- max_indices = vsetq_lane_u32(0, max_indices, 0);- max_indices = vsetq_lane_u32(1, max_indices, 1);- max_indices = vsetq_lane_u32(2, max_indices, 2);- max_indices = vsetq_lane_u32(3, max_indices, 3);-- let mut current_indices = max_indices;- let mut i = 4;-- // Process 4 elements at a time- while i + 4 <= len {- let vals = vld1q_f32(ptr.add(i));- current_indices = vaddq_u32(current_indices, increment);-- // Compare values- let mask = vcgtq_f32(vals, max_vals);-- // Update max values and indices where vals > max_vals- max_vals = vbslq_f32(mask, vals, max_vals);- max_indices = vbslq_u32(mask, current_indices, max_indices);-- i += 4;- }-- // Extract the maximum from the vector- let max_vals_array: [f32; 4] = std::mem::transmute(max_vals);- let max_indices_array: [u32; 4] = std::mem::transmute(max_indices);-- let mut global_max = max_vals_array[0];- let mut global_max_idx = max_indices_array[0] as usize;-- for j in 1..4 {- if max_vals_array[j] > global_max- || (max_vals_array[j] == global_max && (max_indices_array[j] as usize) < global_max_idx)- {- global_max = max_vals_array[j];- global_max_idx = max_indices_array[j] as usize;- }- }-- // Handle remaining elements- while i < len {- let val = *ptr.add(i);- if val > global_max || (val == global_max && i < global_max_idx) {- global_max = val;- global_max_idx = i;- }- i += 1;- }-- global_max_idx- }-}--#[cfg(target_arch = "aarch64")]-pub fn neon_optimized_argmax<T: ArrayElement>(input: &[T]) -> usize {- // Check if NEON is available at runtime- if !std::arch::is_aarch64_feature_detected!("neon") {- return simple_argmax(input);- }-- // For f32, use direct NEON implementation- if std::any::TypeId::of::<T>() == std::any::TypeId::of::<f32>() {- unsafe {- let f32_slice = std::slice::from_raw_parts(input.as_ptr() as *const f32, input.len());- return neon_argmax_f32(f32_slice);- }- }-- // For other types, fall back to simple- simple_argmax(input)-}--#[cfg(test)]-#[path = "../../../tests/unit/backends/cpu/argmax_test.rs"]-mod tests;diff --git a/crates/backend-uzu/src/backends/cpu/command_buffer.rs b/crates/backend-uzu/src/backends/cpu/command_buffer.rsindex 0e81bcd14..de5be1dec 100644--- a/crates/backend-uzu/src/backends/cpu/command_buffer.rs+++ b/crates/backend-uzu/src/backends/cpu/command_buffer.rs@@ -9,7 +9,7 @@ use crate::{AccessFlags, Buffer, BufferRangeMut, BufferRangeRef, CommandBuffer, CommandBufferCompleted,CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,},- cpu::{BufferDowncastExt, Cpu, error::CpuError},+ cpu::{Cpu, buffer::BufferDowncastExt, error::CpuError},},utils::pointers::{SendPtr, SendPtrMut},};diff --git a/crates/backend-uzu/src/backends/cpu/context.rs b/crates/backend-uzu/src/backends/cpu/context.rsindex b252bc06c..78d340c86 100644--- a/crates/backend-uzu/src/backends/cpu/context.rs+++ b/crates/backend-uzu/src/backends/cpu/context.rs@@ -35,14 +35,6 @@ impl Context for CpuContext {Ok(1)}- fn is_high_performance(&self) -> bool {- false- }-- fn debug_active(&self) -> bool {- false- }-fn create_buffer(&self,size: usize,diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rsindex 50e90c825..a688e3a70 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs@@ -10,7 +10,8 @@ use crate::{},},cpu::{- BufferDowncastExt, Cpu, context::CpuContext, error::CpuError, kernel::matmul::quant::encode_quantized_gemm,+ Cpu, buffer::BufferDowncastExt, context::CpuContext, error::CpuError,+ kernel::matmul::quant::encode_quantized_gemm,},},dispatch_dtype,diff --git a/crates/backend-uzu/src/backends/cpu/mod.rs b/crates/backend-uzu/src/backends/cpu/mod.rsindex 2f3f6086d..b98cb0309 100644--- a/crates/backend-uzu/src/backends/cpu/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/mod.rs@@ -7,7 +7,4 @@ mod error;mod kernel;mod sparse;-pub mod argmax;-pub use backend::Cpu;-pub use buffer::BufferDowncastExt;diff --git a/crates/backend-uzu/src/backends/metal/command_buffer.rs b/crates/backend-uzu/src/backends/metal/command_buffer.rsindex 3b939927c..3a29cc4df 100644--- a/crates/backend-uzu/src/backends/metal/command_buffer.rs+++ b/crates/backend-uzu/src/backends/metal/command_buffer.rs@@ -6,7 +6,7 @@ use metal::{};use objc2::{Message, rc::Retained, runtime::ProtocolObject};-use super::{BufferDowncastExt, Metal};+use super::{Metal, buffer::BufferDowncastExt};use crate::{backends::{common::{diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rsindex b42be7720..61b28844d 100644--- a/crates/backend-uzu/src/backends/metal/context.rs+++ b/crates/backend-uzu/src/backends/metal/context.rs@@ -16,11 +16,10 @@ use metal::{use objc2::{rc::Retained, runtime::ProtocolObject};use super::{- DeviceExt, Metal,- device_capabilities::MetalDeviceCapabilities,+ Metal,error::MetalError,kernel,- metal_extensions::{DeviceGeneration, LibraryPipelineExtensions},+ metal_extensions::{DeviceExt, LibraryPipelineExtensions},};use crate::{backends::{@@ -42,7 +41,7 @@ pub struct MetalContext {timeline_value: AtomicU64,allocator: Rc<Allocator<Metal>>,peak_memory_usage: RefCell<usize>,- device_capabilities: MetalDeviceCapabilities,+ gpu_core_count: u32,library: Retained<ProtocolObject<dyn MTLLibrary>>,pipeline_cache: RefCell<HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>>,sparse_heap_pool: RefCell<MetalSparseHeapPool>,@@ -52,15 +51,15 @@ pub struct MetalContext {}impl MetalContext {+ pub fn supports_mxu(&self) -> bool {+ self.device.supports_mxu()+ }+pub(super) fn update_peak_memory_usage(&self) {let mut peak_memory_usage_borrow = self.peak_memory_usage.borrow_mut();*peak_memory_usage_borrow = peak_memory_usage_borrow.max(self.device.current_allocated_size());}- pub fn device_generation(&self) -> DeviceGeneration {- self.device_capabilities.generation- }-pub fn compute_pipeline_state(&self,cache_key: &str,@@ -131,7 +130,7 @@ impl Context for MetalContext {.new_library_with_data(kernel::MTLB).map_err(|nserror| MetalError::CannotCreateLibrary(nserror.to_string()))?;- let device_capabilities = MetalDeviceCapabilities::from_device(&device);+ let gpu_core_count = device.gpu_core_count();let page_size = MTLSparsePageSize::KB256;let heap_capacity = 64 * 4 * page_size.in_bytes();@@ -148,7 +147,7 @@ impl Context for MetalContext {timeline_value: AtomicU64::new(0),allocator: Allocator::new(weak_self.clone()),peak_memory_usage: RefCell::new(0),- device_capabilities,+ gpu_core_count,library,pipeline_cache: RefCell::new(HashMap::new()),sparse_heap_pool: RefCell::new(sparse_pool),@@ -158,15 +157,11 @@ impl Context for MetalContext {}))}- fn is_high_performance(&self) -> bool {- self.device_capabilities.is_high_performance()- }-fn recommended_async_batch_size(&self,model_path: &Path,) -> Result<usize, MetalError> {- let cores = self.device_capabilities.gpu_core_count;+ let cores = self.gpu_core_count;let model_size = ModelSize::from_path(model_path)?;Ok(match (model_size, cores) {(ModelSize::Large, c) if c > 20 => 32,@@ -178,11 +173,6 @@ impl Context for MetalContext {})}- fn debug_active(&self) -> bool {- let upper = std::env::var("METAL_DEVICE_WRAPPER_TYPE").unwrap_or_default().to_ascii_uppercase();- matches!(upper.as_str(), "1" | "YES" | "TRUE")- }-fn create_buffer(&self,size: usize,diff --git a/crates/backend-uzu/src/backends/metal/device_capabilities.rs b/crates/backend-uzu/src/backends/metal/device_capabilities.rsdeleted file mode 100644index 5a74d6144..000000000--- a/crates/backend-uzu/src/backends/metal/device_capabilities.rs+++ /dev/null@@ -1,71 +0,0 @@-use bytesize::ByteSize;-use metal::{MTLDevice, MTLDeviceExt};-use objc2::runtime::ProtocolObject;--use super::metal_extensions::{DeviceExt, DeviceGeneration};--const HIGH_PERFORMANCE_CORE_THRESHOLD: u32 = 10;--#[derive(Debug, Clone)]-pub struct MetalDeviceCapabilities {- pub generation: DeviceGeneration,- pub family_name: String,- pub gpu_core_count: u32,- pub max_threadgroup_memory: ByteSize,- pub shared_memory_size: ByteSize,- pub supports_simd_group: bool,- pub supports_simd_group_matrix: bool,- pub supports_simd_reduction: bool,- pub supports_simd_shuffle_and_fill: bool,- pub supports_simd_shuffles_and_broadcast: bool,- pub supports_mxu: bool,- pub supports_tls: bool,-}--impl MetalDeviceCapabilities {- pub fn from_device(device: &ProtocolObject<dyn MTLDevice>) -> Self {- let architecture_name = device.architecture().name();- let generation = Self::parse_generation(&architecture_name);- let max_threadgroup_memory = ByteSize(device.max_threadgroup_memory_length() as u64);-- Self {- generation,- family_name: device.family_name(),- gpu_core_count: device.gpu_core_count(),- max_threadgroup_memory,- shared_memory_size: device.shared_memory_size(),- supports_simd_group: device.supports_simd_group(),- supports_simd_group_matrix: device.supports_simd_group_matrix(),- supports_simd_reduction: device.supports_simd_reduction(),- supports_simd_shuffle_and_fill: device.supports_simd_shuffle_and_fill(),- supports_simd_shuffles_and_broadcast: device.supports_simd_shuffles_and_broadcast(),- supports_mxu: device.supports_mxu(),- supports_tls: device.supports_tls(),- }- }-- pub fn is_high_performance(&self) -> bool {- self.gpu_core_count > HIGH_PERFORMANCE_CORE_THRESHOLD- }-- fn parse_generation(architecture_name: &str) -> DeviceGeneration {- let Some(suffix_start) = architecture_name.find("applegpu_g") else {- return DeviceGeneration::Unknown(0);- };- let after_prefix = &architecture_name[suffix_start + "applegpu_g".len()..];-- let digit_end = after_prefix- .char_indices()- .find(|(_, character)| !character.is_ascii_digit())- .map_or(after_prefix.len(), |(index, _)| index);-- if digit_end == 0 {- return DeviceGeneration::Unknown(0);- }-- match after_prefix[..digit_end].parse::<u8>() {- Ok(n) => DeviceGeneration::from_generation_number(n),- Err(_) => DeviceGeneration::Unknown(0),- }- }-}diff --git a/crates/backend-uzu/src/backends/metal/metal_extensions/device_extensions.rs b/crates/backend-uzu/src/backends/metal/metal_extensions/device_extensions.rsindex aca8e93a7..9066dd628 100644--- a/crates/backend-uzu/src/backends/metal/metal_extensions/device_extensions.rs+++ b/crates/backend-uzu/src/backends/metal/metal_extensions/device_extensions.rs@@ -1,16 +1,12 @@use core::mem::transmute;-use bytesize::ByteSize;use metal::MTLDevice;use objc2::{Message,ffi::objc_msgSend,- msg_send,- rc::Retained,runtime::{NSObjectProtocol, ProtocolObject, Sel},sel,};-use objc2_foundation::NSString;// Used to bypass objc2's debug-mode class validation for proxy objects like// `CaptureMTLDevice` which forward messages at the ObjC runtime level (via@@ -28,54 +24,7 @@ unsafe fn raw_msg_send<T: Message + ?Sized, R>(unsafe { send(object_pointer, sel) }}-#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum DeviceGeneration {- Gen13,- Gen14,- Gen15,- Gen16,- Gen17,- Gen18,- Unknown(u8),-}--impl DeviceGeneration {- pub fn generation_number(&self) -> u8 {- match self {- Self::Gen13 => 13,- Self::Gen14 => 14,- Self::Gen15 => 15,- Self::Gen16 => 16,- Self::Gen17 => 17,- Self::Gen18 => 18,- Self::Unknown(n) => *n,- }- }-- pub(crate) fn from_generation_number(generation_number: u8) -> Self {- match generation_number {- 13 => Self::Gen13,- 14 => Self::Gen14,- 15 => Self::Gen15,- 16 => Self::Gen16,- 17 => Self::Gen17,- 18 => Self::Gen18,- _ => Self::Unknown(generation_number),- }- }-}-pub trait DeviceExt: MTLDevice + Message + NSObjectProtocol + Sized {- /// Human-readable chip name, e.g. "M2 Max", "M4 Pro", "A17 Pro".- fn family_name(&self) -> String {- if self.respondsToSelector(sel!(familyName)) {- let family_name: Retained<NSString> = unsafe { msg_send![self, familyName] };- family_name.to_string()- } else {- "Unknown".into()- }- }-/// Number of GPU shader cores.fn gpu_core_count(&self) -> u32 {if self.respondsToSelector(sel!(gpuCoreCount)) {@@ -85,60 +34,6 @@ pub trait DeviceExt: MTLDevice + Message + NSObjectProtocol + Sized {}}- /// Total unified (shared) memory.- fn shared_memory_size(&self) -> ByteSize {- if self.respondsToSelector(sel!(sharedMemorySize)) {- ByteSize(unsafe { raw_msg_send(self, sel!(sharedMemorySize)) })- } else {- ByteSize::gib(8)- }- }-- /// Whether the GPU supports SIMD group operations.- fn supports_simd_group(&self) -> bool {- if self.respondsToSelector(sel!(supportsSIMDGroup)) {- unsafe { raw_msg_send(self, sel!(supportsSIMDGroup)) }- } else {- false- }- }-- /// Whether the GPU supports `simdgroup_matrix`.- fn supports_simd_group_matrix(&self) -> bool {- if self.respondsToSelector(sel!(supportsSIMDGroupMatrix)) {- unsafe { raw_msg_send(self, sel!(supportsSIMDGroupMatrix)) }- } else {- false- }- }-- /// Whether the GPU supports SIMD reduction operations.- fn supports_simd_reduction(&self) -> bool {- if self.respondsToSelector(sel!(supportsSIMDReduction)) {- unsafe { raw_msg_send(self, sel!(supportsSIMDReduction)) }- } else {- false- }- }-- /// Whether the GPU supports SIMD shuffle-and-fill operations.- fn supports_simd_shuffle_and_fill(&self) -> bool {- if self.respondsToSelector(sel!(supportsSIMDShuffleAndFill)) {- unsafe { raw_msg_send(self, sel!(supportsSIMDShuffleAndFill)) }- } else {- false- }- }-- /// Whether the GPU supports SIMD shuffles and broadcast.- fn supports_simd_shuffles_and_broadcast(&self) -> bool {- if self.respondsToSelector(sel!(supportsSIMDShufflesAndBroadcast)) {- unsafe { raw_msg_send(self, sel!(supportsSIMDShufflesAndBroadcast)) }- } else {- false- }- }-/// Whether the GPU has a Matrix eXtension Unit (neural accelerator for compute)./// True on M5+ (Gen18+), false on M1-M4.fn supports_mxu(&self) -> bool {@@ -149,15 +44,6 @@ pub trait DeviceExt: MTLDevice + Message + NSObjectProtocol + Sized {}}- /// Whether the GPU supports Thread-Local Storage.- fn supports_tls(&self) -> bool {- if self.respondsToSelector(sel!(supportsTLS)) {- unsafe { raw_msg_send(self, sel!(supportsTLS)) }- } else {- false- }- }-/// Whether the GPU supports placement sparse resources.fn supports_placement_sparse_resources(&self) -> bool {if self.respondsToSelector(sel!(supportsPlacementSparse)) {diff --git a/crates/backend-uzu/src/backends/metal/metal_extensions/mod.rs b/crates/backend-uzu/src/backends/metal/metal_extensions/mod.rsindex 6ddbead8f..3476c3a4f 100644--- a/crates/backend-uzu/src/backends/metal/metal_extensions/mod.rs+++ b/crates/backend-uzu/src/backends/metal/metal_extensions/mod.rs@@ -7,7 +7,7 @@ mod sparse_page_size_extensions;pub use compute_command_encoder_extensions_set_value::ComputeEncoderSetValue;pub use data_type::MetalDataTypeExt;-pub use device_extensions::{DeviceExt, DeviceGeneration};+pub use device_extensions::DeviceExt;pub use function_constant_values_extensions_set_value::FunctionConstantValuesSetValue;pub use library_extensions_pipeline::LibraryPipelineExtensions;pub use sparse_page_size_extensions::SparsePageSizeExt;diff --git a/crates/backend-uzu/src/backends/metal/mod.rs b/crates/backend-uzu/src/backends/metal/mod.rsindex 955ae6cc1..e84dd4691 100644--- a/crates/backend-uzu/src/backends/metal/mod.rs+++ b/crates/backend-uzu/src/backends/metal/mod.rs@@ -3,15 +3,11 @@ mod buffer;mod command_buffer;mod context;mod dense_buffer;-mod device_capabilities;mod error;mod kernel;mod metal_extensions;mod sparse;pub use backend::Metal;-pub use buffer::BufferDowncastExt;pub use context::MetalContext;-pub use device_capabilities::MetalDeviceCapabilities;-pub use kernel::matmul::{GemmDispatchPath, GemmKernel};-pub use metal_extensions::{DeviceExt, DeviceGeneration};+pub use kernel::matmul::GemmDispatchPath;diff --git a/crates/backend-uzu/src/backends/mod.rs b/crates/backend-uzu/src/backends/mod.rsindex dbf208ad2..8cf23bf33 100644--- a/crates/backend-uzu/src/backends/mod.rs+++ b/crates/backend-uzu/src/backends/mod.rs@@ -6,10 +6,11 @@ pub mod metal;macro_rules! select_backend {($expr:expr, $unk:expr) => {{- let mut default = "cpu";- if cfg!(metal_backend) {- default = "metal";- }+ let default = if cfg!(metal_backend) {+ "metal"+ } else {+ "cpu"+ };match std::env::var("UZU_BACKEND").map(|s| s.to_lowercase()).as_deref().unwrap_or(default) {"cpu" => {diff --git a/crates/backend-uzu/src/language_model/mod.rs b/crates/backend-uzu/src/language_model/mod.rsindex 881a2d811..187ac3bee 100644--- a/crates/backend-uzu/src/language_model/mod.rs+++ b/crates/backend-uzu/src/language_model/mod.rs@@ -6,8 +6,5 @@ pub mod language_model_generator_context;pub mod result;pub mod rng;-#[cfg(feature = "tracing")]-pub mod sampler;-// Re-export main typespub use language_model_generator::{LanguageModelGenerator, LanguageModelGeneratorTrait};diff --git a/crates/backend-uzu/src/language_model/sampler.rs b/crates/backend-uzu/src/language_model/sampler.rsdeleted file mode 100644index 9c371232a..000000000--- a/crates/backend-uzu/src/language_model/sampler.rs+++ /dev/null@@ -1,47 +0,0 @@-use ndarray::{ArrayView, IxDyn};--#[cfg(target_arch = "aarch64")]-use crate::backends::cpu::argmax::neon_optimized_argmax;-use crate::{ArrayElement, backends::cpu::argmax::simple_argmax};--pub trait LogitsSampler {- fn sample<T: ArrayElement>(- &self,- logits: ArrayView<T, IxDyn>,- ) -> Vec<u64>;-}--pub struct ArgmaxSampler {}--impl LogitsSampler for ArgmaxSampler {- fn sample<T: ArrayElement>(- &self,- logits: ArrayView<T, IxDyn>,- ) -> Vec<u64> {- let mut result = Vec::<u64>::with_capacity(logits.shape()[0]);-- for row in logits.rows() {- let max_index = if let Some(slice) = row.as_slice_memory_order() {- #[cfg(target_arch = "aarch64")]- {- if slice.len() >= 16 {- neon_optimized_argmax(slice)- } else {- simple_argmax(slice)- }- }- #[cfg(not(target_arch = "aarch64"))]- {- simple_argmax(slice)- }- } else {- let data: Vec<T> = row.iter().cloned().collect();- simple_argmax(&data)- };-- result.push(max_index as u64);- }-- result- }-}diff --git a/crates/backend-uzu/src/lib.rs b/crates/backend-uzu/src/lib.rsindex 6e42bc8d3..b20d02be8 100644--- a/crates/backend-uzu/src/lib.rs+++ b/crates/backend-uzu/src/lib.rs@@ -37,10 +37,7 @@ pub mod _private {forward_pass::{cache_layers::CacheLayers, kv_cache_layer::KVCacheLayer, token_inputs::TokenInputs, traces::ActivationTrace,},- language_model::{- language_model_generator_context::LanguageModelGeneratorContext,- sampler::{ArgmaxSampler, LogitsSampler},- },+ language_model::language_model_generator_context::LanguageModelGeneratorContext,parameters::{ParameterLoaderError, ParameterTree},};}diff --git a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs b/crates/backend-uzu/tests/integration/tracer/trace_validator.rsindex 715ad141e..834cebf31 100644--- a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs+++ b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs@@ -15,8 +15,8 @@ use std::{use backend_uzu::{_private::{- ActivationTrace, AnyModelConfig, ArgmaxSampler, CacheLayers, Classifier, DecoderDecodeInput, KVCacheLayer,- LanguageModelGeneratorContext, LogitsSampler, ParameterLoaderError, ParameterTree, TokenInputs,+ ActivationTrace, AnyModelConfig, CacheLayers, Classifier, DecoderDecodeInput, KVCacheLayer,+ LanguageModelGeneratorContext, ParameterLoaderError, ParameterTree, TokenInputs,},Array, ArrayElement, DataType, ParameterLoader,backends::common::{Allocation, AllocationType, Backend, Context, Encoder},@@ -28,10 +28,39 @@ use backend_uzu::{},};use half::{bf16, f16};-use ndarray::{IxDyn, s};+use ndarray::{ArrayView, IxDyn, s};use num_traits::NumCast;use crate::common;++fn argmax<T: ArrayElement>(input: &[T]) -> usize {+ input+ .iter()+ .enumerate()+ .fold((0, f32::NEG_INFINITY), |(best_index, best_value), (index, &value)| {+ let value_f32 = NumCast::from(value).unwrap_or(f32::NEG_INFINITY);+ if value_f32 > best_value || (value_f32 == best_value && index < best_index) {+ (index, value_f32)+ } else {+ (best_index, best_value)+ }+ })+ .0+}++fn sample_argmax<T: ArrayElement>(logits: ArrayView<T, IxDyn>) -> Vec<u64> {+ let mut result = Vec::with_capacity(logits.shape()[0]);+ for row in logits.rows() {+ let max_index = if let Some(slice) = row.as_slice_memory_order() {+ argmax(slice)+ } else {+ let data: Vec<T> = row.iter().copied().collect();+ argmax(&data)+ };+ result.push(max_index as u64);+ }+ result+}// ============================================================================// Validation Types// ============================================================================@@ -1006,15 +1035,10 @@ impl<B: Backend> TraceValidator<B> {fn get_tokens_from_logits(logits: &Array<B>) -> Vec<u64> {let data_type = logits.data_type();match data_type {- DataType::F16 => Self::get_tokens_from_logits_of_type::<f16>(logits),- DataType::BF16 => Self::get_tokens_from_logits_of_type::<bf16>(logits),- DataType::F32 => Self::get_tokens_from_logits_of_type::<f32>(logits),+ DataType::F16 => sample_argmax(logits.as_view::<f16>()),+ DataType::BF16 => sample_argmax(logits.as_view::<bf16>()),+ DataType::F32 => sample_argmax(logits.as_view::<f32>()),_ => panic!("Unsupported data type: {:?}", data_type),}}-- fn get_tokens_from_logits_of_type<Precision: ArrayElement>(logits: &Array<B>) -> Vec<u64> {- let sampler = ArgmaxSampler {};- sampler.sample(logits.as_view::<Precision>())- }}diff --git a/crates/backend-uzu/tests/unit/backends/cpu/argmax_test.rs b/crates/backend-uzu/tests/unit/backends/cpu/argmax_test.rsdeleted file mode 100644index a3985cd32..000000000--- a/crates/backend-uzu/tests/unit/backends/cpu/argmax_test.rs+++ /dev/null@@ -1,110 +0,0 @@-#![cfg(target_arch = "aarch64")]--use std::time::Instant;--use super::*;--#[test]-fn test_neon_argmax_correctness() {- // Single deterministic 64k array test with random values- const SIZE: usize = 64 * 1024;- const SEED: u64 = 12345;-- // Simple deterministic random number generator- let mut rng_state = SEED;- let mut data = Vec::with_capacity(SIZE);-- for _ in 0..SIZE {- rng_state = rng_state.wrapping_mul(1103515245).wrapping_add(12345);- let random_f32 = (rng_state as f32) / (u64::MAX as f32);- data.push(random_f32);- }-- // Set a known max at position 42_000- data[42_000] = f32::MAX;-- // Compute expected result with naive method- let expected = simple_argmax(&data);- // Compare NEON result to expected- let neon_result = neon_optimized_argmax(&data);-- assert_eq!(expected, neon_result, "64k array test failed: expected {}, got {}", expected, neon_result);- assert_eq!(expected, 42_000, "Expected max should be at index 42_000");-}--#[test]-fn test_neon_performance_128k() {- const SEQ_LEN: usize = 128 * 1024;- let batch_size = 2;- const SEED: u64 = 67890;-- // Generate random data with deterministic seed- let mut rng_state = SEED;- let mut data = Vec::with_capacity(batch_size * SEQ_LEN);-- for _ in 0..(batch_size * SEQ_LEN) {- rng_state = rng_state.wrapping_mul(1103515245).wrapping_add(12345);- let random_f32 = (rng_state as f32) / (u64::MAX as f32);- data.push(random_f32);- }-- // Set known max positions- let expected_indices = vec![42_000, 100_000];- for (batch_idx, &max_pos) in expected_indices.iter().enumerate() {- let offset = batch_idx * SEQ_LEN + max_pos;- data[offset] = f32::MAX;- }-- // Test each batch separately- for (batch_idx, &expected_idx) in expected_indices.iter().enumerate() {- let start_offset = batch_idx * SEQ_LEN;- let end_offset = start_offset + SEQ_LEN;- let batch_data = &data[start_offset..end_offset];-- // Test with NEON implementation- let start = Instant::now();- let neon_result = neon_optimized_argmax(batch_data);- let neon_duration = start.elapsed();-- // Test with simple implementation- let start = Instant::now();- let simple_result = simple_argmax(batch_data);- let simple_duration = start.elapsed();-- // Test with optimized implementation (manual unrolling)- let start = Instant::now();- let optimized_result = optimized_argmax(batch_data);- let optimized_duration = start.elapsed();-- if batch_idx == 0 {- println!("128k argmax performance comparison:");- println!("NEON: {:.3} ms", neon_duration.as_secs_f64() * 1000.0);- println!("Simple: {:.3} ms", simple_duration.as_secs_f64() * 1000.0);- println!("Optimized: {:.3} ms", optimized_duration.as_secs_f64() * 1000.0);-- if simple_duration.as_nanos() > 0 && neon_duration.as_nanos() > 0 {- let speedup_simple = simple_duration.as_secs_f64() / neon_duration.as_secs_f64();- let speedup_optimized = optimized_duration.as_secs_f64() / neon_duration.as_secs_f64();- println!("NEON vs Simple speedup: {:.2}x", speedup_simple);- println!("NEON vs Optimized speedup: {:.2}x", speedup_optimized);- }- }-- // Verify all methods give the same result- assert_eq!(- neon_result, expected_idx,- "NEON: Batch {} argmax mismatch: expected {}, got {}",- batch_idx, expected_idx, neon_result- );- assert_eq!(- simple_result, expected_idx,- "Simple: Batch {} argmax mismatch: expected {}, got {}",- batch_idx, expected_idx, simple_result- );- assert_eq!(- optimized_result, expected_idx,- "Optimized: Batch {} argmax mismatch: expected {}, got {}",- batch_idx, expected_idx, optimized_result- );- }-}diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rsindex 5428d5f67..744c36d19 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs@@ -9,7 +9,7 @@ use backend_uzu::{Backend, Context,kernel::{Kernels, matmul::MatmulKernel},},- metal::{DeviceExt, GemmDispatchPath, Metal, MetalContext},+ metal::{GemmDispatchPath, Metal, MetalContext},},};use half::bf16;@@ -23,7 +23,7 @@ use crate::common::{fn gemm_paths_for_hw(context: &MetalContext) -> Vec<GemmDispatchPath> {let mut paths = vec![GemmDispatchPath::Simdgroup];- if context.device.supports_mxu() {+ if context.supports_mxu() {paths.push(GemmDispatchPath::Mxu);}paths